tp: final fixes in preparation for mass table migration Add all the remaining classes/functions which are necessary for a mass table migration: mainly this is stuff like missing Schema functions, Iterators etc. Also address some minor remaining issues. Change-Id: I74b62d27bbe0ec3caaca8598df8c5dabe0537249
diff --git a/python/generators/trace_processor_table/public.py b/python/generators/trace_processor_table/public.py index 8a5493d..abd0aec 100644 --- a/python/generators/trace_processor_table/public.py +++ b/python/generators/trace_processor_table/public.py
@@ -40,6 +40,8 @@ """ NONE = 0 SORTED = auto() + HIDDEN = auto() + DENSE = auto() SET_ID = auto() @@ -122,8 +124,7 @@ columns: The columns in this table. tabledoc: Documentation for this table. Can include documentation overrides for auto-added columns (i.e. id and type) and aliases added in - |wrapping_sql_view|. parent: The parent ("super-class") table for this - table. + |wrapping_sql_view|. parent: The parent table for this table. All columns are inherited from the specified table. wrapping_sql_view: See |WrappingSqlView|. @@ -131,8 +132,8 @@ class_name: str sql_name: str columns: List[Column] - tabledoc: TableDoc parent: Optional['Table'] = None + tabledoc: Optional[TableDoc] = None wrapping_sql_view: Optional[WrappingSqlView] = None
diff --git a/python/generators/trace_processor_table/serialize.py b/python/generators/trace_processor_table/serialize.py index 589544a..4aee5b5 100644 --- a/python/generators/trace_processor_table/serialize.py +++ b/python/generators/trace_processor_table/serialize.py
@@ -19,7 +19,6 @@ from python.generators.trace_processor_table.public import ColumnFlag from python.generators.trace_processor_table.util import ParsedTable from python.generators.trace_processor_table.util import ParsedColumn -from python.generators.trace_processor_table.util import to_cpp_flags class ColumnSerializer: @@ -104,8 +103,8 @@ return None storage = f'ColumnStorage<ColumnType::{self.name}::stored_type>' - # TODO(lalitm): add support for dense columns. - return f'''{self.name}_({storage}::Create<false>())''' + dense = str(ColumnFlag.DENSE in self.flags).lower() + return f'''{self.name}_({storage}::Create<{dense}>())''' def column_init(self) -> Optional[str]: if self.is_implicit_id or self.is_implicit_type: @@ -158,6 +157,41 @@ name = self.name return f' ColumnStorage<ColumnType::{name}::stored_type> {name}_;' + def iterator_getter(self) -> Optional[str]: + name = self.name + return f''' + ColumnType::{self.name}::type {name}() const {{ + const auto& col = table_->{name}(); + return col.GetAtIdx(its_[col.overlay_index()].index()); + }} + ''' + + def iterator_setter(self) -> Optional[str]: + if self.is_implicit_id or self.is_implicit_type: + return None + return f''' + void set_{self.name}(ColumnType::{self.name}::non_optional_type v) {{ + auto* col = mutable_table_->mutable_{self.name}(); + col->SetAtIdx(its_[col->overlay_index()].index(), v); + }} + ''' + + def static_schema(self) -> Optional[str]: + if self.is_implicit_id or self.is_implicit_type: + return None + return f''' + schema.columns.emplace_back(Table::Schema::Column{{ + "{self.name}", ColumnType::{self.name}::SqlValueType(), false, + {str(ColumnFlag.SORTED in self.flags).lower()}, + {str(ColumnFlag.HIDDEN in self.flags).lower()}, + {str(ColumnFlag.SET_ID in self.flags).lower()}}}); + ''' + + def row_eq(self) -> Optional[str]: + if self.is_implicit_id or self.is_implicit_type: + return None + return f'ColumnType::{self.name}::Equals({self.name}, other.{self.name})' + class TableSerializer(object): """Functions for seralizing a single Table into C++.""" @@ -209,6 +243,7 @@ ColumnSerializer.parent_row_initializer, delimiter=', ') row_init = self.foreach_col( ColumnSerializer.row_initializer, delimiter=',\n ') + row_eq = self.foreach_col(ColumnSerializer.row_eq, delimiter=' &&\n ') return f''' struct Row : public {self.parent_class_name}::Row {{ Row({param}, @@ -218,6 +253,10 @@ type_ = "{self.table.sql_name}"; }} {self.foreach_col(ColumnSerializer.row_field)} + + bool operator==(const {self.table_name}::Row& other) const {{ + return type() == other.type() && {row_eq}; + }} }}; ''' @@ -295,6 +334,55 @@ type_.Append(string_pool_->InternString(row.type())); ''' + def const_iterator(self) -> str: + iterator_getters = self.foreach_col( + ColumnSerializer.iterator_getter, delimiter='\n') + return f''' + class ConstIterator; + class ConstIterator : public macros_internal::AbstractConstIterator< + ConstIterator, {self.table_name}, RowNumber, ConstRowReference> {{ + public: + {iterator_getters} + + protected: + explicit ConstIterator(const {self.table_name}* table, + std::vector<ColumnStorageOverlay> overlays) + : AbstractConstIterator(table, std::move(overlays)) {{}} + + uint32_t CurrentRowNumber() const {{ + return its_.back().index(); + }} + + private: + friend class {self.table_name}; + friend class AbstractConstIterator; + }}; + ''' + + def iterator(self) -> str: + iterator_setters = self.foreach_col( + ColumnSerializer.iterator_setter, delimiter='\n') + return f''' + class Iterator : public ConstIterator {{ + public: + {iterator_setters} + + RowReference row_reference() const {{ + return RowReference(mutable_table_, CurrentRowNumber()); + }} + + private: + friend class {self.table_name}; + + explicit Iterator({self.table_name}* table, + std::vector<ColumnStorageOverlay> overlays) + : ConstIterator(table, std::move(overlays)), + mutable_table_(table) {{}} + + {self.table_name}* mutable_table_ = nullptr; + }}; + ''' + def serialize(self) -> str: return f''' class {self.table_name} : public macros_internal::MacroTable {{ @@ -307,10 +395,6 @@ {self.foreach_col(ColumnSerializer.coltype_enum)} }}; {self.row_struct().strip()} - struct IdAndRow {{ - uint32_t row; - Id id; - }}; struct ColumnFlag {{ {self.foreach_col(ColumnSerializer.flag)} }}; @@ -331,11 +415,49 @@ {self.const_row_reference_struct().strip()} {self.row_reference_struct().strip()} + {self.const_iterator().strip()} + {self.iterator().strip()} + + struct IdAndRow {{ + Id id; + uint32_t row; + RowReference row_reference; + RowNumber row_number; + }}; + {self.constructor().strip()} ~{self.table_name}() override; static const char* Name() {{ return "{self.table.sql_name}"; }} + static Table::Schema ComputeStaticSchema() {{ + Table::Schema schema; + schema.columns.emplace_back(Table::Schema::Column{{ + "id", SqlValue::Type::kLong, true, true, false, false}}); + schema.columns.emplace_back(Table::Schema::Column{{ + "type", SqlValue::Type::kString, false, false, false, false}}); + {self.foreach_col(ColumnSerializer.static_schema)} + return schema; + }} + + ConstIterator IterateRows() const {{ + return ConstIterator(this, CopyOverlays()); + }} + + Iterator IterateRows() {{ return Iterator(this, CopyOverlays()); }} + + ConstIterator FilterToIterator( + const std::vector<Constraint>& cs, + RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) const {{ + return ConstIterator(this, FilterAndApplyToOverlays(cs, opt)); + }} + + Iterator FilterToIterator( + const std::vector<Constraint>& cs, + RowMap::OptimizeFor opt = RowMap::OptimizeFor::kMemory) {{ + return Iterator(this, FilterAndApplyToOverlays(cs, opt)); + }} + void ShrinkToFit() {{ {self.foreach_col(ColumnSerializer.shrink_to_fit)} }} @@ -356,7 +478,8 @@ {self.insert_common().strip()} {self.foreach_col(ColumnSerializer.append)} UpdateSelfOverlayAfterInsert(); - return IdAndRow{{row_number, std::move(id)}}; + return IdAndRow{{std::move(id), row_number, RowReference(this, row_number), + RowNumber(row_number)}}; }} {self.foreach_col(ColumnSerializer.accessor)} @@ -395,3 +518,22 @@ #endif // {ifdef_guard} '''.strip() + + +def to_cpp_flags(raw_flag: ColumnFlag) -> str: + """Converts a ColumnFlag to the C++ flags which it represents + + It is not valid to call this function with ColumnFlag.NONE as in this case + defaults for that column should be implicitly used.""" + + assert raw_flag != ColumnFlag.NONE + flags = [] + if ColumnFlag.SORTED in raw_flag: + flags.append('Column::Flag::kSorted') + if ColumnFlag.HIDDEN in raw_flag: + flags.append('Column::Flag::kHidden') + if ColumnFlag.DENSE in raw_flag: + flags.append('Column::Flag::kDense') + if ColumnFlag.SET_ID in raw_flag: + flags.append('Column::Flag::kSetId') + return ' | '.join(flags)
diff --git a/python/generators/trace_processor_table/util.py b/python/generators/trace_processor_table/util.py index 407bfc1..b6c38d8 100644 --- a/python/generators/trace_processor_table/util.py +++ b/python/generators/trace_processor_table/util.py
@@ -19,12 +19,14 @@ from typing import List from typing import Set from typing import Optional +from typing import Union from python.generators.trace_processor_table.public import Alias from python.generators.trace_processor_table.public import Column from python.generators.trace_processor_table.public import ColumnDoc from python.generators.trace_processor_table.public import ColumnFlag from python.generators.trace_processor_table.public import CppColumnType +from python.generators.trace_processor_table.public import CppDouble from python.generators.trace_processor_table.public import CppInt32 from python.generators.trace_processor_table.public import CppInt64 from python.generators.trace_processor_table.public import CppOptional @@ -101,6 +103,8 @@ return ParsedType('int32_t') if isinstance(col_type, CppUint32): return ParsedType('uint32_t') + if isinstance(col_type, CppDouble): + return ParsedType('double') if isinstance(col_type, CppString): return ParsedType('StringPool::Id') @@ -161,36 +165,24 @@ wrapping_view = table.wrapping_sql_view return wrapping_view.view_name if wrapping_view else table.sql_name - -def to_cpp_flags(raw_flag: ColumnFlag) -> str: - """Converts a ColumnFlag to the C++ flags which it represents - - It is not valid to call this function with ColumnFlag.NONE as in this case - defaults for that column should be implicitly used.""" - - assert raw_flag != ColumnFlag.NONE - flags = [] - if ColumnFlag.SORTED in raw_flag: - flags.append('Column::Flag::kSorted') - if ColumnFlag.SET_ID in raw_flag: - flags.append('Column::Flag::kSetId') - return ' | '.join(flags) - - def _create_implicit_columns_for_root(parsed: ParsedTable ) -> List[ParsedColumn]: """Given a root table, returns the implicit id and type columns.""" - assert parsed.table.parent is None + table = parsed.table + assert table.parent is None - sql_name = public_sql_name(parsed.table) + sql_name = public_sql_name(table) + id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None + type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None return [ ParsedColumn( Column('id', CppSelfTableId(), ColumnFlag.SORTED), - ColumnDoc(doc=f'Unique idenitifier for this {sql_name}.'), + _to_column_doc(id_doc) if id_doc else ColumnDoc( + doc=f'Unique idenitifier for this {sql_name}.'), is_implicit_id=True), ParsedColumn( Column('type', CppString(), ColumnFlag.NONE), - ColumnDoc(doc=''' + _to_column_doc(type_doc) if type_doc else ColumnDoc(doc=''' The name of the "most-specific" child table containing this row. '''), @@ -226,6 +218,14 @@ return result +def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]: + """Cooerces a user specified ColumnDoc or string into a ColumnDoc.""" + + if doc is None or isinstance(doc, ColumnDoc): + return doc + return ColumnDoc(doc=doc) + + def parse_tables_from_files(input_paths: List[str]) -> List[ParsedTable]: """Creates a list of tables with the associated paths.""" @@ -244,9 +244,10 @@ # Create the list of parsed columns for i, parsed in enumerate(sorted_tables): parsed_columns: List[ParsedColumn] + table = parsed.table - if parsed.table.parent: - parsed_parent = parsed_tables[parsed.table.parent.class_name] + if table.parent: + parsed_parent = parsed_tables[table.parent.class_name] parsed_columns = [ dataclasses.replace(c, is_ancestor=True) for c in parsed_parent.columns @@ -254,19 +255,11 @@ else: parsed_columns = _create_implicit_columns_for_root(parsed) - for c in parsed.table.columns: - doc = parsed.table.tabledoc.columns.get(c.name) - columndoc: Optional[ColumnDoc] - if not doc: - columndoc = None - elif isinstance(doc, ColumnDoc): - columndoc = doc - else: - columndoc = ColumnDoc(doc=doc) - parsed_columns.append(ParsedColumn(c, columndoc)) + for c in table.columns: + doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None + parsed_columns.append(ParsedColumn(c, _to_column_doc(doc))) - new_table = dataclasses.replace(parsed, columns=parsed_columns) - parsed_tables[parsed.table.class_name] = new_table - sorted_tables[i] = new_table + sorted_tables[i] = dataclasses.replace(parsed, columns=parsed_columns) + parsed_tables[parsed.table.class_name] = sorted_tables[i] return sorted_tables