stdlib: Small cleanup of docs generation library
Change-Id: I882eeff37262bea59f282b9dcc47d7038736104d
diff --git a/infra/perfetto.dev/src/gen_stdlib_docs_md.py b/infra/perfetto.dev/src/gen_stdlib_docs_md.py
index 6fc5283..e1e60bd 100644
--- a/infra/perfetto.dev/src/gen_stdlib_docs_md.py
+++ b/infra/perfetto.dev/src/gen_stdlib_docs_md.py
@@ -22,216 +22,7 @@
import json
from typing import Any, List, Dict
-
-# Escapes special characters in a markdown table.
-def escape_in_table(desc: str):
- return desc.replace('|', '\\|')
-
-
-# Responsible for module level markdown generation.
-class ModuleMd:
-
- def __init__(self, module_name: str, module_files: List[Dict[str,
- Any]]) -> None:
- self.module_name = module_name
- self.files_md = sorted([
- FileMd(module_name, file_dict) for file_dict in module_files
- ], key=lambda x: x.import_key)
- self.summary_objs = '\n'.join(
- file.summary_objs for file in self.files_md if file.summary_objs)
- self.summary_funs = '\n'.join(
- file.summary_funs for file in self.files_md if file.summary_funs)
- self.summary_view_funs = '\n'.join(file.summary_view_funs
- for file in self.files_md
- if file.summary_view_funs)
- self.summary_macros = '\n'.join(
- file.summary_macros for file in self.files_md if file.summary_macros)
-
- def print_description(self):
- if not self.files_md:
- return ''
-
- long_s = []
- long_s.append(f'## Module: {self.module_name}')
-
- if self.module_name == 'prelude':
- # Prelude is a special module which is automatically imported and doesn't
- # have any include keys.
- objs = '\n'.join(obj for file in self.files_md for obj in file.objs)
- if objs:
- long_s.append('#### Views/Tables')
- long_s.append(objs)
- funs = '\n'.join(fun for file in self.files_md for fun in file.funs)
- if funs:
- long_s.append('#### Functions')
- long_s.append(funs)
- table_funs = '\n'.join(
- view_fun for file in self.files_md for view_fun in file.view_funs)
- if table_funs:
- long_s.append('#### Table Functions')
- long_s.append(table_funs)
- macros = '\n'.join(
- macro for file in self.files_md for macro in file.macros)
- if macros:
- long_s.append('#### Macros')
- long_s.append(macros)
- return '\n'.join(long_s)
-
- for file in self.files_md:
- if not any((file.objs, file.funs, file.view_funs, file.macros)):
- continue
-
- long_s.append(f'### {file.import_key}')
- if file.objs:
- long_s.append('#### Views/Tables')
- long_s.append('\n'.join(file.objs))
- if file.funs:
- long_s.append('#### Functions')
- long_s.append('\n'.join(file.funs))
- if file.view_funs:
- long_s.append('#### Table Functions')
- long_s.append('\n'.join(file.view_funs))
- if file.macros:
- long_s.append('#### Macros')
- long_s.append('\n'.join(file.macros))
-
- return '\n'.join(long_s)
-
-
-# Responsible for file level markdown generation.
-class FileMd:
-
- def __init__(self, module_name, file_dict):
- self.import_key = file_dict['import_key']
- import_key_name = self.import_key if module_name != 'prelude' else 'N/A'
- self.objs, self.funs, self.view_funs, self.macros = [], [], [], []
- summary_objs_list, summary_funs_list, summary_view_funs_list, summary_macros_list = [], [], [], []
-
- # Add imports if in file.
- for data in file_dict['imports']:
- # Anchor
- anchor = f'''obj/{module_name}/{data['name']}'''
-
- # Add summary of imported view/table
- summary_objs_list.append(f'''[{data['name']}](#{anchor})|'''
- f'''{import_key_name}|'''
- f'''{escape_in_table(data['summary_desc'])}''')
-
- self.objs.append(f'''\n\n<a name="{anchor}"></a>'''
- f'''**{data['name']}**, {data['type']}\n\n'''
- f'''{escape_in_table(data['desc'])}\n''')
-
- self.objs.append(
- 'Column | Type | Description\n------ | --- | -----------')
- for name, info in data['cols'].items():
- self.objs.append(
- f'{name} | {info["type"]} | {escape_in_table(info["desc"])}')
-
- self.objs.append('\n\n')
-
- # Add functions if in file
- for data in file_dict['functions']:
- # Anchor
- anchor = f'''fun/{module_name}/{data['name']}'''
-
- # Add summary of imported function
- summary_funs_list.append(f'''[{data['name']}](#{anchor})|'''
- f'''{import_key_name}|'''
- f'''{data['return_type']}|'''
- f'''{escape_in_table(data['summary_desc'])}''')
- self.funs.append(
- f'''\n\n<a name="{anchor}"></a>'''
- f'''**{data['name']}**\n\n'''
- f'''{data['desc']}\n\n'''
- f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''')
- if data['args']:
- self.funs.append('Argument | Type | Description\n'
- '-------- | ---- | -----------')
- for name, arg_dict in data['args'].items():
- self.funs.append(
- f'''{name} | {arg_dict['type']} | {escape_in_table(arg_dict['desc'])}'''
- )
-
- self.funs.append('\n\n')
-
- # Add table functions if in file
- for data in file_dict['table_functions']:
- # Anchor
- anchor = rf'''view_fun/{module_name}/{data['name']}'''
- # Add summary of imported view function
- summary_view_funs_list.append(
- f'''[{data['name']}](#{anchor})|'''
- f'''{import_key_name}|'''
- f'''{escape_in_table(data['summary_desc'])}''')
-
- self.view_funs.append(f'''\n\n<a name="{anchor}"></a>'''
- f'''**{data['name']}**\n'''
- f'''{data['desc']}\n\n''')
- if data['args']:
- self.view_funs.append('Argument | Type | Description\n'
- '-------- | ---- | -----------')
- for name, arg_dict in data['args'].items():
- self.view_funs.append(
- f'''{name} | {arg_dict['type']} | {escape_in_table(arg_dict['desc'])}'''
- )
- self.view_funs.append('\n')
- self.view_funs.append('Column | Type | Description\n'
- '------ | -- | -----------')
- for name, column in data['cols'].items():
- self.view_funs.append(f'{name} | {column["type"]} | {column["desc"]}')
-
- self.view_funs.append('\n\n')
-
- # Add macros if in file
- for data in file_dict['macros']:
- # Anchor
- anchor = rf'''macro/{module_name}/{data['name']}'''
- # Add summary of imported view function
- summary_macros_list.append(f'''[{data['name']}](#{anchor})|'''
- f'''{import_key_name}|'''
- f'''{escape_in_table(data['summary_desc'])}''')
-
- self.macros.append(
- f'''\n\n<a name="{anchor}"></a>'''
- f'''**{data['name']}**\n'''
- f'''{data['desc']}\n\n'''
- f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''')
- if data['args']:
- self.macros.append('Argument | Type | Description\n'
- '-------- | ---- | -----------')
- for name, arg_dict in data['args'].items():
- self.macros.append(
- f'''{name} | {arg_dict['type']} | {escape_in_table(arg_dict['desc'])}'''
- )
- self.macros.append('\n')
- self.macros.append('\n\n')
-
- self.summary_objs = '\n'.join(summary_objs_list)
- self.summary_funs = '\n'.join(summary_funs_list)
- self.summary_view_funs = '\n'.join(summary_view_funs_list)
- self.summary_macros = '\n'.join(summary_macros_list)
-
-
-def main():
- parser = argparse.ArgumentParser()
- parser.add_argument('--input', required=True)
- parser.add_argument('--output', required=True)
- args = parser.parse_args()
-
- with open(args.input) as f:
- modules_json_dict = json.load(f)
-
- modules_dict: Dict[str, ModuleMd] = {}
-
- for module_name, module_files in modules_json_dict.items():
- # Remove 'common' when it has been removed from the code.
- if module_name not in ['deprecated', 'common']:
- modules_dict[module_name] = ModuleMd(module_name, module_files)
-
- prelude_module = modules_dict.pop('prelude')
-
- with open(args.output, 'w') as f:
- f.write('''
+INTRODUCTION = '''
# PerfettoSQL standard library
*This page documents the PerfettoSQL standard library.*
@@ -271,71 +62,294 @@
<!-- TODO(b/290185551): talk about experimental module and contributions. -->
## Summary
-''')
+'''
+
+
+def _escape_in_table(desc: str):
+ """Escapes special characters in a markdown table."""
+ return desc.replace('|', '\\|')
+
+
+def _md_table(cols: List[str]):
+ col_str = ' | '.join(cols) + '\n'
+ lines = ['-' * len(col) for col in cols]
+ underlines = ' | '.join(lines)
+ return col_str + underlines
+
+
+def _write_summary(sql_type: str, table_cols: List[str],
+ summary_objs: List[str]) -> str:
+ table_data = '\n'.join(s.strip() for s in summary_objs if s)
+ return f"""
+### {sql_type}
+
+{_md_table(table_cols)}
+{table_data}
+
+"""
+
+
+class FileMd:
+ """Responsible for file level markdown generation."""
+
+ def __init__(self, module_name, file_dict):
+ self.import_key = file_dict['import_key']
+ import_key_name = self.import_key if module_name != 'prelude' else 'N/A'
+ self.objs, self.funs, self.view_funs, self.macros = [], [], [], []
+ summary_objs_list, summary_funs_list, summary_view_funs_list, summary_macros_list = [], [], [], []
+
+ # Add imports if in file.
+ for data in file_dict['imports']:
+ # Anchor
+ anchor = f'''obj/{module_name}/{data['name']}'''
+
+ # Add summary of imported view/table
+ summary_objs_list.append(f'''[{data['name']}](#{anchor})|'''
+ f'''{import_key_name}|'''
+ f'''{_escape_in_table(data['summary_desc'])}''')
+
+ self.objs.append(f'''\n\n<a name="{anchor}"></a>'''
+ f'''**{data['name']}**, {data['type']}\n\n'''
+ f'''{_escape_in_table(data['desc'])}\n''')
+
+ self.objs.append(_md_table(['Column', 'Type', 'Description']))
+ for name, info in data['cols'].items():
+ self.objs.append(
+ f'{name} | {info["type"]} | {_escape_in_table(info["desc"])}')
+
+ self.objs.append('\n\n')
+
+ # Add functions if in file
+ for data in file_dict['functions']:
+ # Anchor
+ anchor = f'''fun/{module_name}/{data['name']}'''
+
+ # Add summary of imported function
+ summary_funs_list.append(f'''[{data['name']}](#{anchor})|'''
+ f'''{import_key_name}|'''
+ f'''{data['return_type']}|'''
+ f'''{_escape_in_table(data['summary_desc'])}''')
+ self.funs.append(
+ f'''\n\n<a name="{anchor}"></a>'''
+ f'''**{data['name']}**\n\n'''
+ f'''{data['desc']}\n\n'''
+ f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''')
+ if data['args']:
+ self.funs.append(_md_table(['Argument', 'Type', 'Description']))
+ for name, arg_dict in data['args'].items():
+ self.funs.append(
+ f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}'''
+ )
+
+ self.funs.append('\n\n')
+
+ # Add table functions if in file
+ for data in file_dict['table_functions']:
+ # Anchor
+ anchor = rf'''view_fun/{module_name}/{data['name']}'''
+ # Add summary of imported view function
+ summary_view_funs_list.append(
+ f'''[{data['name']}](#{anchor})|'''
+ f'''{import_key_name}|'''
+ f'''{_escape_in_table(data['summary_desc'])}''')
+
+ self.view_funs.append(f'''\n\n<a name="{anchor}"></a>'''
+ f'''**{data['name']}**\n'''
+ f'''{data['desc']}\n\n''')
+ if data['args']:
+ self.funs.append(_md_table(['Argument', 'Type', 'Description']))
+ for name, arg_dict in data['args'].items():
+ self.view_funs.append(
+ f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}'''
+ )
+ self.view_funs.append('\n')
+ self.view_funs.append(_md_table(['Column', 'Type', 'Description']))
+ for name, column in data['cols'].items():
+ self.view_funs.append(f'{name} | {column["type"]} | {column["desc"]}')
+
+ self.view_funs.append('\n\n')
+
+ # Add macros if in file
+ for data in file_dict['macros']:
+ # Anchor
+ anchor = rf'''macro/{module_name}/{data['name']}'''
+ # Add summary of imported view function
+ summary_macros_list.append(
+ f'''[{data['name']}](#{anchor})|'''
+ f'''{import_key_name}|'''
+ f'''{_escape_in_table(data['summary_desc'])}''')
+
+ self.macros.append(
+ f'''\n\n<a name="{anchor}"></a>'''
+ f'''**{data['name']}**\n'''
+ f'''{data['desc']}\n\n'''
+ f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''')
+ if data['args']:
+ self.macros.append(_md_table(['Argument', 'Type', 'Description']))
+ for name, arg_dict in data['args'].items():
+ self.macros.append(
+ f'''{name} | {arg_dict['type']} | {_escape_in_table(arg_dict['desc'])}'''
+ )
+ self.macros.append('\n')
+ self.macros.append('\n\n')
+
+ self.summary_objs = '\n'.join(summary_objs_list)
+ self.summary_funs = '\n'.join(summary_funs_list)
+ self.summary_view_funs = '\n'.join(summary_view_funs_list)
+ self.summary_macros = '\n'.join(summary_macros_list)
+
+
+class ModuleMd:
+ """Responsible for module level markdown generation."""
+
+ def __init__(self, module_name: str, module_files: List[Dict[str,
+ Any]]) -> None:
+ self.module_name = module_name
+ self.files_md = sorted(
+ [FileMd(module_name, file_dict) for file_dict in module_files],
+ key=lambda x: x.import_key)
+ self.summary_objs = '\n'.join(
+ file.summary_objs for file in self.files_md if file.summary_objs)
+ self.summary_funs = '\n'.join(
+ file.summary_funs for file in self.files_md if file.summary_funs)
+ self.summary_view_funs = '\n'.join(file.summary_view_funs
+ for file in self.files_md
+ if file.summary_view_funs)
+ self.summary_macros = '\n'.join(
+ file.summary_macros for file in self.files_md if file.summary_macros)
+
+ def get_prelude_description(self) -> str:
+ if not self.module_name == 'prelude':
+ raise ValueError("Only callable on prelude module")
+
+ lines = []
+ lines.append(f'## Module: {self.module_name}')
+
+ # Prelude is a special module which is automatically imported and doesn't
+ # have any include keys.
+ objs = '\n'.join(obj for file in self.files_md for obj in file.objs)
+ if objs:
+ lines.append('#### Views/Tables')
+ lines.append(objs)
+
+ funs = '\n'.join(fun for file in self.files_md for fun in file.funs)
+ if funs:
+ lines.append('#### Functions')
+ lines.append(funs)
+
+ table_funs = '\n'.join(
+ view_fun for file in self.files_md for view_fun in file.view_funs)
+ if table_funs:
+ lines.append('#### Table Functions')
+ lines.append(table_funs)
+
+ macros = '\n'.join(macro for file in self.files_md for macro in file.macros)
+ if macros:
+ lines.append('#### Macros')
+ lines.append(macros)
+
+ return '\n'.join(lines)
+
+ def get_description(self) -> str:
+ if not self.files_md:
+ return ''
+
+ if self.module_name == 'prelude':
+ raise ValueError("Can't be called with prelude module")
+
+ lines = []
+ lines.append(f'## Module: {self.module_name}')
+
+ for file in self.files_md:
+ if not any((file.objs, file.funs, file.view_funs, file.macros)):
+ continue
+
+ lines.append(f'### {file.import_key}')
+ if file.objs:
+ lines.append('#### Views/Tables')
+ lines.append('\n'.join(file.objs))
+ if file.funs:
+ lines.append('#### Functions')
+ lines.append('\n'.join(file.funs))
+ if file.view_funs:
+ lines.append('#### Table Functions')
+ lines.append('\n'.join(file.view_funs))
+ if file.macros:
+ lines.append('#### Macros')
+ lines.append('\n'.join(file.macros))
+
+ return '\n'.join(lines)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--input', required=True)
+ parser.add_argument('--output', required=True)
+ args = parser.parse_args()
+
+ with open(args.input) as f:
+ modules_json_dict = json.load(f)
+
+ # Fetch the modules from json documentation.
+ modules_dict: Dict[str, ModuleMd] = {}
+ for module_name, module_files in modules_json_dict.items():
+ # Remove 'common' when it has been removed from the code.
+ if module_name not in ['deprecated', 'common']:
+ modules_dict[module_name] = ModuleMd(module_name, module_files)
+
+ prelude_module = modules_dict.pop('prelude')
+
+ with open(args.output, 'w') as f:
+ f.write(INTRODUCTION)
summary_objs = [prelude_module.summary_objs
] if prelude_module.summary_objs else []
summary_objs += [
module.summary_objs
- for name, module in modules_dict.items()
- if (module.summary_objs and name != 'experimental')
+ for module in modules_dict.values()
+ if (module.summary_objs)
]
summary_funs = [prelude_module.summary_funs
] if prelude_module.summary_funs else []
- summary_funs += [
- module.summary_funs
- for name, module in modules_dict.items()
- if (module.summary_funs and name != 'experimental')
- ]
+ summary_funs += [module.summary_funs for module in modules_dict.values()]
summary_view_funs = [prelude_module.summary_view_funs
] if prelude_module.summary_view_funs else []
summary_view_funs += [
- module.summary_view_funs
- for name, module in modules_dict.items()
- if (module.summary_view_funs and name != 'experimental')
+ module.summary_view_funs for module in modules_dict.values()
]
summary_macros = [prelude_module.summary_macros
] if prelude_module.summary_macros else []
summary_macros += [
- module.summary_macros
- for name, module in modules_dict.items()
- if (module.summary_macros and name != 'experimental')
+ module.summary_macros for module in modules_dict.values()
]
if summary_objs:
- f.write('### Views/tables\n\n'
- 'Name | Import | Description\n'
- '---- | ------ | -----------\n')
- f.write('\n'.join(summary_objs))
- f.write('\n')
+ f.write(
+ _write_summary('Views/tables', ['Name', 'Import', 'Description'],
+ summary_objs))
if summary_funs:
- f.write('### Functions\n\n'
- 'Name | Import | Return type | Description\n'
- '---- | ------ | ----------- | -----------\n')
- f.write('\n'.join(summary_funs))
- f.write('\n')
+ f.write(
+ _write_summary('Functions',
+ ['Name', 'Import', 'Return type', 'Description'],
+ summary_funs))
if summary_view_funs:
- f.write('### Table Functions\n\n'
- 'Name | Import | Description\n'
- '---- | ------ | -----------\n')
- f.write('\n'.join(summary_view_funs))
- f.write('\n')
+ f.write(
+ _write_summary('Table functions', ['Name', 'Import', 'Description'],
+ summary_view_funs))
if summary_macros:
- f.write('### Macros\n\n'
- 'Name | Import | Description\n'
- '---- | ------ | -----------\n')
- f.write('\n'.join(summary_macros))
- f.write('\n')
+ f.write(
+ _write_summary('Macros', ['Name', 'Import', 'Description'],
+ summary_macros))
f.write('\n\n')
- f.write(prelude_module.print_description())
+ f.write(prelude_module.get_prelude_description())
f.write('\n')
f.write('\n'.join(
- module.print_description() for module in modules_dict.values()))
+ module.get_description() for module in modules_dict.values()))
return 0
diff --git a/python/generators/sql_processing/docs_parse.py b/python/generators/sql_processing/docs_parse.py
index 6e8cd60..9062973 100644
--- a/python/generators/sql_processing/docs_parse.py
+++ b/python/generators/sql_processing/docs_parse.py
@@ -17,7 +17,7 @@
from dataclasses import dataclass
import re
import sys
-from typing import Any, Dict, List, Optional, Set, Tuple, NamedTuple
+from typing import Dict, List, Optional, Set, NamedTuple
from python.generators.sql_processing.docs_extractor import DocsExtractor
from python.generators.sql_processing.utils import ObjKind
@@ -31,30 +31,21 @@
from python.generators.sql_processing.utils import ARG_ANNOTATION_PATTERN
-def is_internal(name: str) -> bool:
+def _is_internal(name: str) -> bool:
return re.match(r'^_.*', name, re.IGNORECASE) is not None
-def is_snake_case(s: str) -> bool:
- """Returns true if the string is snake_case."""
+def _is_snake_case(s: str) -> bool:
return re.fullmatch(r'^[a-z_0-9]*$', s) is not None
-# Parse a SQL comment (i.e. -- Foo\n -- bar.) into a string (i.e. "Foo bar.").
def parse_comment(comment: str) -> str:
+ """Parse a SQL comment (i.e. -- Foo\n -- bar.) into a string (i.e. "Foo bar.")."""
return ' '.join(line.strip().lstrip('--').lstrip()
for line in comment.strip().split('\n'))
-
-class Arg(NamedTuple):
- # TODO(b/307926059): the type is missing on old-style documentation for
- # tables. Make it "str" after stdlib is migrated.
- type: Optional[str]
- description: str
-
-
-# Returns: error message if the name is not correct, None otherwise.
def get_module_prefix_error(name: str, path: str, module: str) -> Optional[str]:
+ """Returns error message if the name is not correct, None otherwise."""
prefix = name.lower().split('_')[0]
if module in ["common", "prelude", "deprecated"]:
if prefix == module:
@@ -77,6 +68,13 @@
f'with one of following names: {", ".join(allowed_prefixes)}')
+class Arg(NamedTuple):
+ # TODO(b/307926059): the type is missing on old-style documentation for
+ # tables. Make it "str" after stdlib is migrated.
+ type: Optional[str]
+ description: str
+
+
class AbstractDocParser(ABC):
@dataclass
@@ -244,7 +242,7 @@
f'{type} "{self.name}": CREATE OR REPLACE is not allowed in stdlib '
f'as standard library modules can only included once. Please just '
f'use CREATE instead.')
- if is_internal(self.name):
+ if _is_internal(self.name):
return None
is_perfetto_table_or_view = (
@@ -294,12 +292,12 @@
f'use CREATE instead.')
# Ignore internal functions.
- if is_internal(self.name):
+ if _is_internal(self.name):
return None
name = self._parse_name()
- if not is_snake_case(name):
+ if not _is_snake_case(name):
self._error(f'Function name "{name}" is not snake_case'
f' (should be {name.casefold()})')
@@ -345,14 +343,14 @@
f'use CREATE instead.')
# Ignore internal functions.
- if is_internal(self.name):
+ if _is_internal(self.name):
return None
self._validate_only_contains_annotations(doc.annotations,
{'@arg', '@column'})
name = self._parse_name()
- if not is_snake_case(name):
+ if not _is_snake_case(name):
self._error(f'Function name "{name}" is not snake_case'
f' (should be "{name.casefold()}")')
@@ -396,13 +394,13 @@
f'use CREATE instead.')
# Ignore internal macros.
- if is_internal(self.name):
+ if _is_internal(self.name):
return None
self._validate_only_contains_annotations(doc.annotations, set())
name = self._parse_name()
- if not is_snake_case(name):
+ if not _is_snake_case(name):
self._error(f'Macro name "{name}" is not snake_case'
f' (should be "{name.casefold()}")')
@@ -416,6 +414,7 @@
class ParsedFile:
+ """Data class containing all of the docmentation of single SQL file"""
errors: List[str] = []
table_views: List[TableOrView] = []
functions: List[Function] = []
@@ -432,9 +431,9 @@
self.macros = macros
-# Reads the provided SQL and, if possible, generates a dictionary with data
-# from documentation together with errors from validation of the schema.
def parse_file(path: str, sql: str) -> Optional[ParsedFile]:
+ """Reads the provided SQL and, if possible, generates a dictionary with data
+ from documentation together with errors from validation of the schema."""
if sys.platform.startswith('win'):
path = path.replace('\\', '/')
