blob: bc5b08f214379191e9997fc4f9ab1bb30bdd23ff [file] [log] [blame]
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from enum import Enum
import re
import os
from typing import Dict, List
NAME = r'[a-zA-Z_\d\{\}]+'
ANY_WORDS = r'[^\s].*'
ANY_NON_QUOTE = r'[^\']*.*'
TYPE = r'[a-zA-Z]+'
SQL = r'[\s\S]*?'
WS = r'\s*'
COMMENT = r' --[^\n]*\n'
COMMENTS = rf'(?:{COMMENT})*'
ARG = rf'{COMMENTS} {NAME} {TYPE}'
ARG_PATTERN = rf'({COMMENTS}) ({NAME}) ({TYPE})'
ARGS = rf'(?:{ARG})?(?: ,{ARG})*'
# Make the pattern more readable by allowing the use of spaces
# and replace then with a wildcard in a separate step.
# NOTE: two whitespaces next to each other are really bad for performance.
# Take special care to avoid them.
def update_pattern(pattern):
return pattern.replace(' ', WS)
CREATE_TABLE_VIEW_PATTERN = update_pattern(
# Match create table/view and catch type
fr'^CREATE (OR REPLACE)? (VIRTUAL|PERFETTO)?'
fr' (TABLE|VIEW) (?:IF NOT EXISTS)?'
# Catch the name and optional schema.
fr' ({NAME}) (?: \( ({ARGS}) \) )? (?:AS|USING)? .*')
CREATE_TABLE_AS_PATTERN = update_pattern(fr'^CREATE TABLE ({NAME}) AS')
CREATE_VIEW_AS_PATTERN = update_pattern(fr'^CREATE VIEW ({NAME}) AS')
DROP_TABLE_VIEW_PATTERN = update_pattern(fr'^DROP (TABLE|VIEW) IF EXISTS '
fr'({NAME});$')
INCLUDE_ALL_PATTERN = update_pattern(
fr'^INCLUDE PERFETTO MODULE [a-zA-Z0-9_\.]*\*;')
CREATE_FUNCTION_PATTERN = update_pattern(
# Function name.
fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) "
# Args: anything in the brackets.
fr" \( ({ARGS}) \)"
# Type: word after RETURNS.
fr"({COMMENTS})"
fr" RETURNS ({TYPE}) AS ")
CREATE_TABLE_FUNCTION_PATTERN = update_pattern(
fr"CREATE (OR REPLACE)? PERFETTO FUNCTION ({NAME}) "
# Args: anything in the brackets.
fr" \( ({ARGS}) \) "
# Type: table definition after RETURNS.
fr"({COMMENTS})"
fr" RETURNS TABLE \( ({ARGS}) \) AS ")
CREATE_MACRO_PATTERN = update_pattern(
fr"CREATE (OR REPLACE)? PERFETTO MACRO ({NAME}) "
# Args: anything in the brackets.
fr" \( ({ARGS}) \) "
# Type: word after RETURNS.
fr"({COMMENTS})"
fr" RETURNS ({TYPE})")
COLUMN_ANNOTATION_PATTERN = update_pattern(fr'^ ({NAME}) ({ANY_WORDS})')
NAME_AND_TYPE_PATTERN = update_pattern(fr' ({NAME})\s+({TYPE}) ')
ARG_ANNOTATION_PATTERN = fr'\s*{NAME_AND_TYPE_PATTERN}\s+({ANY_WORDS})'
ARG_DEFINITION_PATTERN = update_pattern(ARG_PATTERN)
FUNCTION_RETURN_PATTERN = update_pattern(fr'^ ({TYPE})\s+({ANY_WORDS})')
ANY_PATTERN = r'(?:\s|.)*'
class ObjKind(str, Enum):
table_view = 'table_view'
function = 'function'
table_function = 'table_function'
macro = 'macro'
PATTERN_BY_KIND = {
ObjKind.table_view: CREATE_TABLE_VIEW_PATTERN,
ObjKind.function: CREATE_FUNCTION_PATTERN,
ObjKind.table_function: CREATE_TABLE_FUNCTION_PATTERN,
ObjKind.macro: CREATE_MACRO_PATTERN
}
ALLOWED_PREFIXES = {
'counters': 'counter',
'chrome/util': 'cr',
'intervals': 'interval',
'graphs': 'graph',
'slices': 'slice',
}
# Allows for nonstandard object names.
OBJECT_NAME_ALLOWLIST = {
'graphs/partition.sql': ['tree_structural_partition_by_group'],
'slices/with_context.sql': ['process_slice', 'thread_slice'],
'slices/cpu_time.sql': ['thread_slice_cpu_time']
}
# Given a regex pattern and a string to match against, returns all the
# matching positions. Specifically, it returns a dictionary from the line
# number of the match to the regex match object.
# Note: this resuts a dict[int, re.Match], but re.Match exists only in later
# versions of python3, prior to that it was _sre.SRE_Match.
def match_pattern(pattern: str, file_str: str) -> Dict[int, object]:
line_number_to_matches = {}
for match in re.finditer(pattern, file_str, re.MULTILINE):
line_id = file_str[:match.start()].count('\n')
line_number_to_matches[line_id] = match.groups()
return line_number_to_matches
# Given a list of lines in a text and the line number, scans backwards to find
# all the comments.
def extract_comment(lines: List[str], line_number: int) -> List[str]:
comments = []
for line in lines[line_number - 1::-1]:
# Break on empty line, as that suggests it is no longer a part of
# this comment.
if not line or not line.startswith('--'):
break
comments.append(line)
# Reverse as the above was reversed
comments.reverse()
return comments
# Given SQL string check whether any of the words is used, and create error
# string if needed.
def check_banned_words(sql: str, path: str) -> List[str]:
lines = [l.strip() for l in sql.split('\n')]
errors = []
# Ban the use of LIKE in non-comment lines.
for line in lines:
if line.startswith('--'):
continue
if 'like' in line.casefold():
errors.append(
'LIKE is banned in trace processor metrics. Prefer GLOB instead.\n'
f'Offending file: {path}\n')
continue
if 'create_function' in line.casefold():
errors.append('CREATE_FUNCTION is deprecated in trace processor. '
'Use CREATE PERFETTO FUNCTION instead.\n'
f'Offending file: {path}')
if 'create_view_function' in line.casefold():
errors.append(
'CREATE_VIEW_FUNCTION is deprecated in trace processor. '
'Use CREATE PERFETTO FUNCTION $name RETURNS TABLE instead.\n'
f'Offending file: {path}')
if 'import(' in line.casefold():
errors.append('SELECT IMPORT is deprecated in trace processor. '
'Use INCLUDE PERFETTO MODULE instead.\n'
f'Offending file: {path}')
return errors
# Given SQL string check whether there is (not allowlisted) usage of
# CREATE TABLE {name} AS.
def check_banned_create_table_as(sql: str, filename: str, stdlib_path: str,
allowlist: Dict[str, List[str]]) -> List[str]:
errors = []
for _, matches in match_pattern(CREATE_TABLE_AS_PATTERN, sql).items():
name = matches[0]
# Normalize paths before checking presence in the allowlist so it will
# work on Windows for the Chrome stdlib presubmit.
allowlist_normpath = dict(
(os.path.normpath(path), tables) for path, tables in allowlist.items())
allowlist_key = os.path.normpath(filename[len(stdlib_path):])
if allowlist_key not in allowlist_normpath:
errors.append(f"CREATE TABLE '{name}' is deprecated. "
"Use CREATE PERFETTO TABLE instead.\n"
f"Offending file: {filename}\n")
continue
if name not in allowlist_normpath[allowlist_key]:
errors.append(
f"Table '{name}' uses CREATE TABLE which is deprecated "
"and this table is not allowlisted. Use CREATE PERFETTO TABLE.\n"
f"Offending file: {filename}\n")
return errors
# Given SQL string check whether there is usage of CREATE VIEW {name} AS.
def check_banned_create_view_as(sql: str, filename: str) -> List[str]:
errors = []
for _, matches in match_pattern(CREATE_VIEW_AS_PATTERN, sql).items():
name = matches[0]
errors.append(f"CREATE VIEW '{name}' is deprecated. "
"Use CREATE PERFETTO VIEW instead.\n"
f"Offending file: {filename}\n")
return errors
# Given SQL string check whether there is usage of CREATE VIEW {name} AS.
def check_banned_include_all(sql: str, filename: str) -> List[str]:
errors = []
for _, matches in match_pattern(INCLUDE_ALL_PATTERN, sql).items():
errors.append(
f"INCLUDE PERFETTO MODULE with wildcards is not allowed in stdlib. "
f"Import specific modules instead. Offending file: {filename}")
return errors