blob: 444ec82ebfc2e09b09678d3d903f29af2269e163 [file] [log] [blame]
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import dataclasses
from dataclasses import dataclass
import importlib
import sys
from typing import Dict
from typing import List
from typing import Set
from typing import Optional
from typing import Union
from python.generators.trace_processor_table.public import Alias
from python.generators.trace_processor_table.public import Column
from python.generators.trace_processor_table.public import ColumnDoc
from python.generators.trace_processor_table.public import ColumnFlag
from python.generators.trace_processor_table.public import CppColumnType
from python.generators.trace_processor_table.public import CppDouble
from python.generators.trace_processor_table.public import CppInt32
from python.generators.trace_processor_table.public import CppInt64
from python.generators.trace_processor_table.public import CppOptional
from python.generators.trace_processor_table.public import CppSelfTableId
from python.generators.trace_processor_table.public import CppString
from python.generators.trace_processor_table.public import CppTableId
from python.generators.trace_processor_table.public import CppUint32
from python.generators.trace_processor_table.public import Table
@dataclass
class ParsedType:
"""Result of parsing a CppColumnType into its parts."""
cpp_type: str
is_optional: bool = False
is_alias: bool = False
alias_underlying_name: Optional[str] = None
is_self_id: bool = False
id_table: Optional[Table] = None
def cpp_type_with_optionality(self) -> str:
"""Returns the C++ type wrapping with base::Optional if necessary."""
# ThreadTable and ProcessTable are special for legacy reasons as they were
# around even before the advent of C++ macro tables. Because of this a lot
# of code was written assuming that upid and utid were uint32 (e.g. indexing
# directly into vectors using them) and it was decided this behaviour was
# too expensive in engineering cost to fix given the trivial benefit. For
# this reason, continue to maintain this illusion.
if self.id_table and self.id_table.class_name in ('ThreadTable',
'ProcessTable'):
cpp_type = 'uint32_t'
else:
cpp_type = self.cpp_type
if self.is_optional:
return f'std::optional<{cpp_type}>'
return cpp_type
@dataclass(frozen=True)
class ParsedColumn:
"""Representation of a column parsed from a Python definition."""
column: Column
doc: Optional[ColumnDoc]
# Whether this column is the implicit "id" column which is added by while
# parsing the tables rather than by the user.
is_implicit_id: bool = False
# Whether this column is the implicit "type" column which is added by while
# parsing the tables rather than by the user.
is_implicit_type: bool = False
# Whether this column comes from copying a column from the ancestor. If this
# is set to false, the user explicitly specified it for this table.
is_ancestor: bool = False
@dataclass(frozen=True)
class ParsedTable:
"""Representation of a table parsed from a Python definition."""
table: Table
columns: List[ParsedColumn]
def parse_type_with_cols(table: Table, cols: List[Column],
col_type: CppColumnType) -> ParsedType:
"""Parses a CppColumnType into its constiuent parts."""
if isinstance(col_type, CppInt64):
return ParsedType('int64_t')
if isinstance(col_type, CppInt32):
return ParsedType('int32_t')
if isinstance(col_type, CppUint32):
return ParsedType('uint32_t')
if isinstance(col_type, CppDouble):
return ParsedType('double')
if isinstance(col_type, CppString):
return ParsedType('StringPool::Id')
if isinstance(col_type, Alias):
col = next(c for c in cols if c.name == col_type.underlying_column)
return ParsedType(
parse_type(table, col.type).cpp_type,
is_alias=True,
alias_underlying_name=col.name)
if isinstance(col_type, CppTableId):
return ParsedType(
f'{col_type.table.class_name}::Id', id_table=col_type.table)
if isinstance(col_type, CppSelfTableId):
return ParsedType(
f'{table.class_name}::Id', is_self_id=True, id_table=table)
if isinstance(col_type, CppOptional):
inner = parse_type(table, col_type.inner)
assert not inner.is_optional, 'Nested optional not allowed'
return dataclasses.replace(inner, is_optional=True)
raise Exception(f'Unknown type {col_type}')
def parse_type(table: Table, col_type: CppColumnType) -> ParsedType:
"""Parses a CppColumnType into its constiuent parts."""
return parse_type_with_cols(table, table.columns, col_type)
def typed_column_type(table: Table, col: ParsedColumn) -> str:
"""Returns the TypedColumn/IdColumn C++ type for a given column."""
parsed = parse_type(table, col.column.type)
if col.is_implicit_id:
return f'IdColumn<{parsed.cpp_type}>'
return f'TypedColumn<{parsed.cpp_type_with_optionality()}>'
def data_layer_type(table: Table, col: ParsedColumn) -> str:
"""Returns the DataLayer C++ type for a given column."""
parsed = parse_type(table, col.column.type)
if col.is_implicit_id:
return 'column::IdStorage'
if parsed.cpp_type == 'StringPool::Id':
return 'column::StringStorage'
if ColumnFlag.SET_ID in col.column.flags:
return 'column::SetIdStorage'
return f'column::NumericStorage<ColumnType::{col.column.name}::non_optional_stored_type>'
def find_table_deps(table: Table) -> List[Table]:
"""Finds all the other table class names this table depends on.
By "depends", we mean this table in C++ would need the dependency to be
defined (or included) before this table is defined."""
deps: Dict[str, Table] = {}
if table.parent:
deps[table.parent.class_name] = table.parent
for c in table.columns:
# Aliases cannot have dependencies so simply ignore them: trying to parse
# them before adding implicit columns can cause issues.
if isinstance(c.type, Alias):
continue
id_table = parse_type(table, c.type).id_table
if id_table:
deps[id_table.class_name] = id_table
return list(deps.values())
def public_sql_name(table: Table) -> str:
"""Extracts SQL name for the table which should be publicised."""
wrapping_view = table.wrapping_sql_view
return wrapping_view.view_name if wrapping_view else table.sql_name
def _create_implicit_columns_for_root(table: Table) -> List[ParsedColumn]:
"""Given a root table, returns the implicit id and type columns."""
assert table.parent is None
sql_name = public_sql_name(table)
id_doc = table.tabledoc.columns.get('id') if table.tabledoc else None
type_doc = table.tabledoc.columns.get('type') if table.tabledoc else None
return [
ParsedColumn(
Column('id', CppSelfTableId(), ColumnFlag.SORTED),
_to_column_doc(id_doc) if id_doc else ColumnDoc(
doc=f'Unique identifier for this {sql_name}.'),
is_implicit_id=True),
ParsedColumn(
Column('type', CppString(), ColumnFlag.NONE),
_to_column_doc(type_doc) if type_doc else ColumnDoc(doc='''
The name of the "most-specific" child table containing this
row.
'''),
is_implicit_type=True,
)
]
def _topological_sort_table_and_deps(parsed: List[Table]) -> List[Table]:
"""Topologically sorts a list of tables (i.e. dependenices appear earlier).
See [1] for information on a topological sort. We do this to allow
dependencies to be processed and appear ealier than their dependents.
[1] https://en.wikipedia.org/wiki/Topological_sorting"""
visited: Set[str] = set()
result: List[Table] = []
# Topological sorting is really just a DFS where we put the nodes in the list
# after any dependencies.
def dfs(t: Table):
if t.class_name in visited:
return
visited.add(t.class_name)
for dep in find_table_deps(t):
dfs(dep)
result.append(t)
for p in parsed:
dfs(p)
return result
def _to_column_doc(doc: Union[ColumnDoc, str, None]) -> Optional[ColumnDoc]:
"""Cooerces a user specified ColumnDoc or string into a ColumnDoc."""
if doc is None or isinstance(doc, ColumnDoc):
return doc
return ColumnDoc(doc=doc)
def parse_tables_from_modules(modules: List[str]) -> List[ParsedTable]:
"""Creates a list of tables with the associated paths."""
# Create a mapping from the table to a "parsed" version of the table.
tables: Dict[str, Table] = {}
for module in modules:
imported = importlib.import_module(module)
run_tables: List[Table] = imported.__dict__['ALL_TABLES']
for table in run_tables:
existing_table = tables.get(table.class_name)
assert not existing_table or existing_table == table
tables[table.class_name] = table
# Sort all the tables: note that this list may include tables which are not
# in |tables| dictionary due to dependencies on tables which live in a file
# not covered by |input_paths|.
sorted_tables = _topological_sort_table_and_deps(list(tables.values()))
parsed_tables: Dict[str, ParsedTable] = {}
for table in sorted_tables:
parsed_columns: List[ParsedColumn]
if table.parent:
parsed_parent = parsed_tables[table.parent.class_name]
parsed_columns = [
dataclasses.replace(c, is_ancestor=True)
for c in parsed_parent.columns
]
else:
parsed_columns = _create_implicit_columns_for_root(table)
for c in table.columns:
doc = table.tabledoc.columns.get(c.name) if table.tabledoc else None
parsed_columns.append(ParsedColumn(c, _to_column_doc(doc)))
parsed_tables[table.class_name] = ParsedTable(table, parsed_columns)
# Only return tables which come directly from |input_paths|. This stops us
# generating tables which were not requested.
return [
parsed_tables[p.class_name]
for p in sorted_tables
if p.class_name in tables
]