[DNS] tp: simplify stdlib docs generation and make it more robust
The docs generator had some minor bugs with writing output to stderr
instead of propogating errors back to the caller. As part of trying to
fix this, I realised that the amount of state which is passed between
functions makes it very difficult to understand what is happening.
Moreover, there seemed to be quite a lot of duplication (i.e. repeated
parisng using regexes, merging annotations across multiple lines) o
subtle logic and unnecessary complexity in how we were finding the
boundaries of annotations.
To address this, rework the docs generator to instead be split into two
stages: a) extraction which is responsible for extracting the comments from
the SQL and "tokenizing" into description and annotations (including
merging sequential lines related to the same column) b) parsinng which
is repsonsble for actually verifying the semantics of the extracted
documentation and parsing into the JSON dictionary format expected by
the markdown generator
Bug: 283524256
Change-Id: Iccad2e64a4cb02411e39914d1b6cdbaa343a3611
diff --git a/tools/check_sql_modules.py b/tools/check_sql_modules.py
index 051f1a9..8037fca 100755
--- a/tools/check_sql_modules.py
+++ b/tools/check_sql_modules.py
@@ -16,32 +16,30 @@
# This tool checks that every SQL object created without prefix
# 'internal_' is documented with proper schema.
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
import os
import sys
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(os.path.join(ROOT_DIR))
-FILE_DIR = ROOT_DIR
-from python.generators.stdlib_docs.stdlib import *
+from python.generators.stdlib_docs.parse import parse_file_to_dict
def main():
-
errors = []
- metrics_sources = os.path.join(FILE_DIR, "src", "trace_processor", "stdlib")
+ metrics_sources = os.path.join(ROOT_DIR, "src", "trace_processor", "stdlib")
for root, _, files in os.walk(metrics_sources, topdown=True):
for f in files:
path = os.path.join(root, f)
- if path.endswith(".sql"):
- with open(path) as f:
- sql = f.read()
- errors += parse_file_to_dict(path, sql)[1]
- sys.stderr.write("\n\n".join(errors))
+ if not path.endswith(".sql"):
+ continue
+ with open(path, 'r') as f:
+ sql = f.read()
+
+ res = parse_file_to_dict(path, sql)
+ errors += res if isinstance(res, list) else []
+ sys.stderr.write("\n".join(errors))
+ sys.stderr.write("\n")
return 0 if not errors else 1
