tp: Create stdlib utils python lib
Bug: 255535171
Change-Id: I7c173dac52ee70d59d6495f10bf68dfad89c2567
diff --git a/tools/gen_stdlib_docs_json.py b/tools/gen_stdlib_docs_json.py
index 2b6b8a9..e6953d0 100755
--- a/tools/gen_stdlib_docs_json.py
+++ b/tools/gen_stdlib_docs_json.py
@@ -17,311 +17,13 @@
import os
import sys
import json
-import re
from collections import defaultdict
-from sql_modules_utils import *
-from typing import Union, List
+from typing import Dict
-# Creates a JSON file with documentation for stdlib files.
+ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(os.path.join(ROOT_DIR))
-REPLACEMENT_HEADER = '''/*
- * Copyright (C) 2022 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/*
- *******************************************************************************
- * AUTOGENERATED BY tools/gen_stdlib_docs - DO NOT EDIT
- *******************************************************************************
- */
-
- #include <string.h>
-'''
-
-
-# Stores documentation for CREATE {TABLE|VIEW} with comment split into
-# segments.
-class TableViewDocs:
-
- def __init__(self, name: str, type: str, desc: List[str], columns: List[str]):
- self.name = name
- self.type = type
- self.desc = desc
- self.columns = columns
-
- # Contructs new TableViewDocs from whole comment, by splitting it on typed
- # lines.
- @staticmethod
- def create_from_comment(comment: List[str],
- matches: tuple) -> "TableViewDocs":
- obj_type, name = matches[:2]
-
- # Ignore internal tables and views.
- if re.match(r"^internal_.*", name):
- return None
-
- col_start = None
-
- # Splits code into segments by finding beginning of column segment.
- for i, line in enumerate(comment):
- # Ignore only '--' line.
- if line == "--":
- continue
-
- m = re.match(typed_comment_pattern(), line)
-
- # Ignore untyped lines
- if not m:
- continue
-
- line_type = m.group(1)
- if line_type == "column" and not col_start:
- col_start = i
- continue
-
- return TableViewDocs(name, obj_type, comment[:col_start],
- comment[col_start:])
-
- def parse_comment(self) -> dict:
- return {
- 'name': self.name,
- 'type': self.type,
- 'desc': parse_desc(self),
- 'cols': parse_columns(self)
- }
-
-
-# Stores documentation for CREATE_FUNCTION with comment split into segments.
-class FunctionDocs:
-
- def __init__(
- self,
- name: str,
- desc: str,
- args: List[str],
- ret: List[str],
- ):
- self.name = name
- self.desc = desc
- self.args = args
- self.ret = ret
-
- @staticmethod
- def create_from_comment(comment: List[str], matches: tuple) -> "FunctionDocs":
- name = matches[0]
-
- # Ignore internal functions.
- if re.match(r"^INTERNAL_.*", name):
- return None
-
- start_args, start_ret = None, None
-
- # Splits code into segments by finding beginning of args and ret segments.
- for i, line in enumerate(comment):
- # Ignore only '--' line.
- if line == "--":
- continue
-
- m = re.match(typed_comment_pattern(), line)
-
- # Ignore untyped lines
- if not m:
- continue
-
- line_type = m.group(1)
- if line_type == "arg" and not start_args:
- start_args = i
- continue
-
- if line_type == "ret" and not start_ret:
- start_ret = i
- continue
-
- return FunctionDocs(name, comment[:start_args],
- comment[start_args:start_ret], comment[start_ret:])
-
- def parse_comment(self) -> dict:
- ret_type, ret_desc = parse_ret(self)
- return {
- 'name': self.name,
- 'desc': parse_desc(self),
- 'args': parse_args(self),
- 'return_type': ret_type,
- 'return_desc': ret_desc
- }
-
-
-# Stores documentation for CREATE_VIEW_FUNCTION with comment split into
-# segments.
-class ViewFunctionDocs:
-
- def __init__(
- self,
- name: str,
- desc: List[str],
- args: List[str],
- columns: List[str],
- ):
- self.name = name
- self.desc = desc
- self.args = args
- self.columns = columns
-
- @staticmethod
- def create_from_comment(comment: List[str],
- matches: tuple) -> "ViewFunctionDocs":
- name = matches[0]
-
- # Ignore internal functions.
- if re.match(r"^INTERNAL_.*", name):
- return None
-
- start_args, start_cols = None, None
-
- # Splits code into segments by finding beginning of args and cols segments.
- for i, line in enumerate(comment):
- # Ignore only '--' line.
- if line == "--":
- continue
-
- m = re.match(typed_comment_pattern(), line)
-
- # Ignore untyped lines
- if not m:
- continue
-
- line_type = m.group(1)
- if line_type == "arg" and not start_args:
- start_args = i
- continue
-
- if line_type == "column" and not start_cols:
- start_cols = i
- continue
-
- return ViewFunctionDocs(
- name,
- comment[:start_args],
- comment[start_args:start_cols],
- comment[start_cols:],
- )
-
- def parse_comment(self) -> dict:
- return {
- 'name': self.name,
- 'desc': parse_desc(self),
- 'args': parse_args(self),
- 'cols': parse_columns(self)
- }
-
-
-def get_text(line: str, no_break_line: bool = True) -> str:
- line = line.lstrip('--').strip()
- if not line:
- return '' if no_break_line else '\n'
- return line
-
-
-def parse_desc(docs: Union["TableViewDocs", "FunctionDocs", "ViewFunctionDocs"]
- ) -> str:
- desc_lines = [get_text(line, False) for line in docs.desc]
- return ' '.join(desc_lines).strip('\n').strip()
-
-
-# Whether comment segment about columns contain proper schema. Can be matched
-# against parsed SQL data by setting `use_data_from_sql`.
-def parse_columns(docs: Union["TableViewDocs", "ViewFunctionDocs"]) -> dict:
- cols = {}
- last_col = None
- last_desc = []
- for line in docs.columns:
- # Ignore only '--' line.
- if line == "--" or not line.startswith("-- @column"):
- last_desc.append(get_text(line))
- continue
-
- # Look for '-- @column' line as a column description
- m = re.match(column_pattern(), line)
- if last_col:
- cols[last_col] = ' '.join(last_desc)
- last_col, last_desc = m.group(1), [m.group(2)]
-
- cols[last_col] = ' '.join(last_desc)
- return cols
-
-
-def parse_args(docs: "FunctionDocs") -> dict:
- args = {}
- last_arg, last_desc, last_type = None, [], None
- for line in docs.args:
- # Ignore only '--' line.
- if line == "--" or not line.startswith("-- @arg"):
- last_desc.append(get_text(line))
- continue
-
- m = re.match(args_pattern(), line)
- if last_arg:
- args[last_arg] = {'type': last_type, 'desc': ' '.join(last_desc)}
- last_arg, last_type, last_desc = m.group(1), m.group(2), [m.group(3)]
-
- args[last_arg] = {'type': last_type, 'desc': ' '.join(last_desc)}
- return args
-
-
-# Whether comment segment about return contain proper schema. Matches against
-# parsed SQL data.
-def parse_ret(docs: "FunctionDocs") -> tuple:
- desc = []
- for line in docs.ret:
- # Ignore only '--' line.
- if line == "--" or not line.startswith("-- @ret"):
- desc.append(get_text(line))
-
- m = re.match(function_return_pattern(), line)
- ret_type, desc = m.group(1), [m.group(2)]
- return (ret_type, ' '.join(desc))
-
-
-# After matching file to pattern, fetches and validates related documentation.
-def parse_docs_for_sql_object_type(sql: str, pattern: str,
- docs_object: type) -> List:
- ret = []
- line_id_to_match = match_pattern(pattern, sql)
- lines = sql.split("\n")
- for line_id, matches in line_id_to_match.items():
- # Fetch comment by looking at lines over beginning of match in reverse
- # order.
- comment = fetch_comment(lines[line_id - 1::-1])
- docs = docs_object.create_from_comment(comment, matches)
- if docs:
- ret.append(docs.parse_comment())
-
- return ret
-
-
-def parse_file(sql: str):
- return {
- 'imports':
- parse_docs_for_sql_object_type(sql, create_table_view_pattern(),
- TableViewDocs),
- 'functions':
- parse_docs_for_sql_object_type(sql, create_function_pattern(),
- FunctionDocs),
- 'view_functions':
- parse_docs_for_sql_object_type(sql, create_view_function_pattern(),
- ViewFunctionDocs)
- }
+from python.generators.stdlib_docs.stdlib import *
def main():
@@ -345,20 +47,19 @@
sql_files = args.sql_files
# Extract the SQL output from each file.
- sql_outputs = {}
+ sql_outputs: Dict[str, str] = {}
for file_name in sql_files:
with open(file_name, 'r') as f:
relpath = os.path.relpath(file_name, args.root_dir)
sql_outputs[relpath] = f.read()
modules = defaultdict(list)
-
# Add documentation from each file
for path, sql in sql_outputs.items():
module_name = path.split("/")[0]
import_key = path.split(".sql")[0].replace("/", ".")
- docs = parse_file(sql)
+ docs = parse_file_to_dict(path, sql)[0]
if not any(docs.values()):
continue
file_dict = {'import_key': import_key, **docs}
