tp: final fixes in preparation for mass table migration
Add all the remaining classes/functions which are necessary for a mass
table migration: mainly this is stuff like missing Schema functions,
Iterators etc. Also address some minor remaining issues.
Change-Id: I74b62d27bbe0ec3caaca8598df8c5dabe0537249
diff --git a/python/generators/trace_processor_table/public.py b/python/generators/trace_processor_table/public.py
index 8a5493d..abd0aec 100644
--- a/python/generators/trace_processor_table/public.py
+++ b/python/generators/trace_processor_table/public.py
@@ -40,6 +40,8 @@
"""
NONE = 0
SORTED = auto()
+ HIDDEN = auto()
+ DENSE = auto()
SET_ID = auto()
@@ -122,8 +124,7 @@
columns: The columns in this table.
tabledoc: Documentation for this table. Can include documentation overrides
for auto-added columns (i.e. id and type) and aliases added in
- |wrapping_sql_view|. parent: The parent ("super-class") table for this
- table.
+ |wrapping_sql_view|.
parent: The parent table for this table. All columns are inherited from the
specified table.
wrapping_sql_view: See |WrappingSqlView|.
@@ -131,8 +132,8 @@
class_name: str
sql_name: str
columns: List[Column]
- tabledoc: TableDoc
parent: Optional['Table'] = None
+ tabledoc: Optional[TableDoc] = None
wrapping_sql_view: Optional[WrappingSqlView] = None
diff --git a/python/generators/trace_processor_table/serialize.py b/python/generators/trace_processor_table/serialize.py
index 589544a..4aee5b5 100644
--- a/python/generators/trace_processor_table/serialize.py
+++ b/python/generators/trace_processor_table/serialize.py
@@ -19,7 +19,6 @@
from python.generators.trace_processor_table.public import ColumnFlag
from python.generators.trace_processor_table.util import ParsedTable
from python.generators.trace_processor_table.util import ParsedColumn
-from python.generators.trace_processor_table.util import to_cpp_flags
class ColumnSerializer:
@@ -104,8 +103,8 @@
return None
storage = f'ColumnStorage<ColumnType::{self.name}::stored_type>'
- # TODO(lalitm): add support for dense columns.
- return f'''{self.name}_({storage}::Create<false>())'''
+ dense = str(ColumnFlag.DENSE in self.flags).lower()
+ return f'''{self.name}_({storage}::Create<{dense}>())'''
def column_init(self) -> Optional[str]:
if self.is_implicit_id or self.is_implicit_type:
@@ -158,6 +157,41 @@
name = self.name
return f' ColumnStorage<ColumnType::{name}::stored_type> {name}_;'
+ def iterator_getter(self) -> Optional[str]:
+ name = self.name
+ return f'''
+ ColumnType::{self.name}::type {name}() const {{
+ const auto& col = table_->{name}();
+ return col.GetAtIdx(its_[col.overlay_index()].index());
+ }}
+ '''
+
+ def iterator_setter(self) -> Optional[str]:
+ if self.is_implicit_id or self.is_implicit_type:
+ return None
+ return f'''
+ void set_{self.name}(ColumnType::{self.name}::non_optional_type v) {{
+ auto* col = mutable_table_->mutable_{self.name}();
+ col->SetAtIdx(its_[col->overlay_index()].index(), v);
+ }}
+ '''
+
+ def static_schema(self) -> Optional[str]:
+ if self.is_implicit_id or self.is_implicit_type:
+ return None
+ return f'''
+ schema.columns.emplace_back(Table::Schema::Column{{
+ "{self.name}", ColumnType::{self.name}::SqlValueType(), false,
+ {str(ColumnFlag.SORTED in self.flags).lower()},
+ {str(ColumnFlag.HIDDEN in self.flags).lower()},
+ {str(ColumnFlag.SET_ID in self.flags).lower()}}});
+ '''
+
+ def row_eq(self) -> Optional[str]:
+ if self.is_implicit_id or self.is_implicit_type:
+ return None
+ return f'ColumnType::{self.name}::Equals({self.name}, other.{self.name})'
+
class TableSerializer(object):
"""Functions for seralizing a single Table into C++."""
@@ -209,6 +243,7 @@
ColumnSerializer.parent_row_initializer, delimiter=', ')
row_init = self.foreach_col(
ColumnSerializer.row_initializer, delimiter=',\n ')
+ row_eq = self.foreach_col(ColumnSerializer.row_eq, delimiter=' &&\n ')
return f'''
struct Row : public {self.parent_class_name}::Row {{
Row({param},
@@ -218,6 +253,10 @@
type_ = "{self.table.sql_name}";
}}
{self.foreach_col(ColumnSerializer.row_field)}
+
+ bool operator==(const {self.table_name}::Row& other) const {{
+ return type() == other.type() && {row_eq};
+ }}
}};
'''
@@ -295,6 +334,55 @@
type_.Append(string_pool_->InternString(row.type()));
'''
+ def const_iterator(self) -> str:
+ iterator_getters = self.foreach_col(
+ ColumnSerializer.iterator_getter, delimiter='\n')
+ return f'''
+ class ConstIterator;
+ class ConstIterator : public macros_internal::AbstractConstIterator<
+ ConstIterator, {self.table_name}, RowNumber, ConstRowReference> {{
+ public:
+ {iterator_getters}
+
+ protected:
+ explicit ConstIterator(const {self.table_name}* table,
+ std::vector<ColumnStorageOverlay> overlays)
+ : AbstractConstIterator(table, std::move(overlays)) {{}}
+
+ uint32_t CurrentRowNumber() const {{
+ return its_.back().index();
+ }}
+
+ private:
+ friend class {self.table_name};
+ friend class AbstractConstIterator;
+ }};
+ '''
+
+ def iterator(self) -> str:
+ iterator_setters = self.foreach_col(
+ ColumnSerializer.iterator_setter, delimiter='\n')
+ return f'''
+ class Iterator : public ConstIterator {{
+ public:
+ {iterator_setters}
+
+ RowReference row_reference() const {{
+ return RowReference(mutable_table_, CurrentRowNumber());
+ }}
+
+ private:
+ friend class {self.table_name};
+
+ explicit Iterator({self.table_name}* table,
+ std::vector<ColumnStorageOverlay> overlays)
+ : ConstIterator(table, std::move(overlays)),
+ mutable_table_(table) {{}}
+
+ {self.table_name}* mutable_table_ = nullptr;
+ }};
+ '''
+
def serialize(self) -> str:
return f'''
class {self.table_name} : public macros_internal::MacroTable {{
@@ -307,10 +395,6 @@
{self.foreach_col(ColumnSerializer.coltype_enum)}
}};
{self.row_struct().strip()}
- struct IdAndRow {{
- uint32_t row;
- Id id;
- }};
struct ColumnFlag {{
{self.foreach_col(ColumnSerializer.flag)}
}};
@@ -331,11 +415,49 @@
{self.const_row_reference_struct().strip()}
{self.row_reference_struct().strip()}
+ {self.const_iterator().strip()}
+ {self.iterator().strip()}
+
+ struct IdAndRow {{
+ Id id;
+ uint32_t row;
+ RowReference row_reference;
+ RowNumber row_number;
+ }};
+
{self.constructor().strip()}
~{self.table_name}() override;
static const char* Name() {{ return "{self.table.sql_name}"; }}
+ static Table::Schema ComputeStaticSchema() {{
+ Table::Schema schema;
+ schema.columns.emplace_back(Table::Schema::Column{{
+ "id", SqlValue::Type::kLong, true, true, false, false}});
+ schema.columns.emplace_back(Table::Schema::Column{{
+ "type", SqlValue::Type::kString, false, false, false, false}});
+ {self.foreach_col(ColumnSerializer.static_schema)}
+ return schema;
+ }}
+
+ ConstIterator IterateRows() const {{
+ return ConstIterator(this, CopyOverlays());
+ }}
+
+ Iterator IterateRows() {{ return Iterator(this, CopyOverlays()); }}
+
+ ConstIterator FilterToIterator(
+ const std::vector<Constraint>& cs,
+ RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) const {{
+ return ConstIterator(this, FilterAndApplyToOverlays(cs, opt));
+ }}
+
+ Iterator FilterToIterator(
+ const std::vector<Constraint>& cs,
+ RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) {{
+ return Iterator(this, FilterAndApplyToOverlays(cs, opt));
+ }}
+
void ShrinkToFit() {{
{self.foreach_col(ColumnSerializer.shrink_to_fit)}
}}
@@ -356,7 +478,8 @@
{self.insert_common().strip()}
{self.foreach_col(ColumnSerializer.append)}
UpdateSelfOverlayAfterInsert();
- return IdAndRow{{row_number, std::move(id)}};
+ return IdAndRow{{std::move(id), row_number, RowReference(this, row_number),
+ RowNumber(row_number)}};
}}
{self.foreach_col(ColumnSerializer.accessor)}
@@ -395,3 +518,22 @@
#endif // {ifdef_guard}
'''.strip()
+
+
+def to_cpp_flags(raw_flag: ColumnFlag) -> str:
+ """Converts a ColumnFlag to the C++ flags which it represents
+
+ It is not valid to call this function with ColumnFlag.NONE as in this case
+ defaults for that column should be implicitly used."""
+
+ assert raw_flag != ColumnFlag.NONE
+ flags = []
+ if ColumnFlag.SORTED in raw_flag:
+ flags.append('Column::Flag::kSorted')
+ if ColumnFlag.HIDDEN in raw_flag:
+ flags.append('Column::Flag::kHidden')
+ if ColumnFlag.DENSE in raw_flag:
+ flags.append('Column::Flag::kDense')
+ if ColumnFlag.SET_ID in raw_flag:
+ flags.append('Column::Flag::kSetId')
+ return ' | '.join(flags)
diff --git a/python/generators/trace_processor_table/util.py b/python/generators/trace_processor_table/util.py
index 407bfc1..b6c38d8 100644
--- a/python/generators/trace_processor_table/util.py
+++ b/python/generators/trace_processor_table/util.py
@@ -19,12 +19,14 @@
from typing import List
from typing import Set
from typing import Optional
+from typing import Union
from python.generators.trace_processor_table.public import Alias
from python.generators.trace_processor_table.public import Column
from python.generators.trace_processor_table.public import ColumnDoc
from python.generators.trace_processor_table.public import ColumnFlag
from python.generators.trace_processor_table.public import CppColumnType
+from python.generators.trace_processor_table.public import CppDouble
from python.generators.trace_processor_table.public import CppInt32
from python.generators.trace_processor_table.public import CppInt64
from python.generators.trace_processor_table.public import CppOptional
@@ -101,6 +103,8 @@
return ParsedType('int32_t')
if isinstance(col_type, CppUint32):
return ParsedType('uint32_t')
+ if isinstance(col_type, CppDouble):
+ return ParsedType('double')
if isinstance(col_type, CppString):
return ParsedType('StringPool::Id')
@@ -161,36 +165,24 @@
wrapping_view = table.wrapping_sql_view
return wrapping_view.view_name if wrapping_view else table.sql_name
-
-def to_cpp_flags(raw_flag: ColumnFlag) -> str:
- """Converts a ColumnFlag to the C++ flags which it represents
-
- It is not valid to call this function with ColumnFlag.NONE as in this case
- defaults for that column should be implicitly used."""
-
- assert raw_flag != ColumnFlag.NONE
- flags = []
- if ColumnFlag.SORTED in raw_flag:
- flags.append('Column::Flag::kSorted')
- if ColumnFlag.SET_ID in raw_flag:
- flags.append('Column::Flag::kSetId')
- return ' | '.join(flags)
-
-
def _create_implicit_columns_for_root(parsed: ParsedTable
) -> List[ParsedColumn]:
"""Given a root table, returns the implicit id and type columns."""
- assert parsed.table.parent is None
+ table = parsed.table
+ assert table.parent is None
- sql_name = public_sql_name(parsed.table)
+ sql_name = public_sql_name(table)
+ id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None
+ type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None
return [
ParsedColumn(
Column('id', CppSelfTableId(), ColumnFlag.SORTED),
- ColumnDoc(doc=f'Unique idenitifier for this {sql_name}.'),
+ _to_column_doc(id_doc) if id_doc else ColumnDoc(
+ doc=f'Unique idenitifier for this {sql_name}.'),
is_implicit_id=True),
ParsedColumn(
Column('type', CppString(), ColumnFlag.NONE),
- ColumnDoc(doc='''
+ _to_column_doc(type_doc) if type_doc else ColumnDoc(doc='''
The name of the "most-specific" child table containing this
row.
'''),
@@ -226,6 +218,14 @@
return result
+def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]:
+ """Cooerces a user specified ColumnDoc or string into a ColumnDoc."""
+
+ if doc is None or isinstance(doc, ColumnDoc):
+ return doc
+ return ColumnDoc(doc=doc)
+
+
def parse_tables_from_files(input_paths: List[str]) -> List[ParsedTable]:
"""Creates a list of tables with the associated paths."""
@@ -244,9 +244,10 @@
# Create the list of parsed columns
for i, parsed in enumerate(sorted_tables):
parsed_columns: List[ParsedColumn]
+ table = parsed.table
- if parsed.table.parent:
- parsed_parent = parsed_tables[parsed.table.parent.class_name]
+ if table.parent:
+ parsed_parent = parsed_tables[table.parent.class_name]
parsed_columns = [
dataclasses.replace(c, is_ancestor=True)
for c in parsed_parent.columns
@@ -254,19 +255,11 @@
else:
parsed_columns = _create_implicit_columns_for_root(parsed)
- for c in parsed.table.columns:
- doc = parsed.table.tabledoc.columns.get(c.name)
- columndoc: Optional[ColumnDoc]
- if not doc:
- columndoc = None
- elif isinstance(doc, ColumnDoc):
- columndoc = doc
- else:
- columndoc = ColumnDoc(doc=doc)
- parsed_columns.append(ParsedColumn(c, columndoc))
+ for c in table.columns:
+ doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None
+ parsed_columns.append(ParsedColumn(c, _to_column_doc(doc)))
- new_table = dataclasses.replace(parsed, columns=parsed_columns)
- parsed_tables[parsed.table.class_name] = new_table
- sorted_tables[i] = new_table
+ sorted_tables[i] = dataclasses.replace(parsed, columns=parsed_columns)
+ parsed_tables[parsed.table.class_name] = sorted_tables[i]
return sorted_tables
