Support YAML directly in TP as query args
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc index 74c36fc..5a8a44b 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc
@@ -1117,38 +1117,49 @@ return base::OkStatus(); } - // Detect QueryGraph format by the #!querygraph shebang on the first line. - // This is invalid SQL (# is never valid), so it fails loudly if fed to SQLite. - std::string_view content(file.sql); - // Skip leading whitespace. - size_t pos = 0; - while (pos < content.size() && - (content[pos] == ' ' || content[pos] == '\n' || content[pos] == '\r' || - content[pos] == '\t')) { - ++pos; + // Determine caller format from execution stack. + bool caller_is_yaml = !execution_stack_.empty() && + execution_stack_.back().is_yaml; + + // Pick content based on caller format preference. + // YAML callers prefer YAML, SQL callers prefer SQL, with fallback. + std::string content; + bool is_yaml = false; + if (caller_is_yaml && !file.yaml.empty()) { + content = file.yaml; + is_yaml = true; + } else if (!caller_is_yaml && !file.sql.empty()) { + content = file.sql; + } else if (!file.yaml.empty()) { + content = file.yaml; + is_yaml = true; + } else { + content = file.sql; } - bool is_querygraph = - content.substr(pos).substr(0, 13) == "#!querygraph\n" || - content.substr(pos).substr(0, 13) == "#!querygraph\r" || - content.substr(pos).substr(0, 12) == "#!querygraph"; - if (is_querygraph) { - auto compiled = pfgraph::CompilePfGraphYaml(file.sql); + + // If content is YAML, compile to SQL. + if (is_yaml) { + auto compiled = pfgraph::CompilePfGraphYaml(content); if (!compiled.ok()) { return base::ErrStatus("querygraph compilation of module '%s': %s", key.c_str(), compiled.status().message().c_str()); } - file.sql = *compiled; + content = *compiled; } - // Push include frame onto execution stack. The main loop will process it. + // Store compiled content back for re-use (avoid double compilation). + file.sql = content; + + // Push include frame onto execution stack with format tracking. execution_stack_.push_back({FrameType::kInclude, - SqlSource::FromModuleInclude(file.sql, key), + SqlSource::FromModuleInclude(content, key), /*parser=*/nullptr, /*accumulated_stats=*/{}, /*current_stmt=*/std::nullopt, key, &file, /*traceback_sql=*/parser.statement_sql(), /*wildcard_modules=*/{}, /*wildcard_index=*/0, /*wildcard_traceback_sql=*/ - SqlSource::FromTraceProcessorImplementation("")}); + SqlSource::FromTraceProcessorImplementation(""), + /*is_yaml=*/is_yaml}); return base::OkStatus(); }
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h index 81e4398..0d20b2e 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.h
@@ -305,6 +305,9 @@ wildcard_modules; size_t wildcard_index = 0; SqlSource wildcard_traceback_sql; + + // Whether the current file is QueryGraph YAML format. + bool is_yaml = false; }; void RegisterStaticTable(dataframe::Dataframe*, const std::string&);
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc index cfa062f..5f1d8f9 100644 --- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc +++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine_unittest.cc
@@ -40,7 +40,7 @@ sql_modules::RegisteredPackage result; for (const auto& file : files) { result.modules[file.first] = - sql_modules::RegisteredPackage::ModuleFile{file.second, false}; + sql_modules::RegisteredPackage::ModuleFile{/*sql=*/file.second, /*yaml=*/"", /*included=*/false}; } return result; }
diff --git a/src/trace_processor/perfetto_sql/pfgraph/pfgraph_unittest.cc b/src/trace_processor/perfetto_sql/pfgraph/pfgraph_unittest.cc index 66b5106..e713404 100644 --- a/src/trace_processor/perfetto_sql/pfgraph/pfgraph_unittest.cc +++ b/src/trace_processor/perfetto_sql/pfgraph/pfgraph_unittest.cc
@@ -628,7 +628,7 @@ TEST(PfGraphYamlTest, SimpleYamlPipeline) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test result: @@ -651,7 +651,7 @@ TEST(PfGraphYamlTest, YamlWithImports) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: android.test imports: [android.process_metadata, slices.with_context] @@ -668,7 +668,7 @@ TEST(PfGraphYamlTest, YamlFunction) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test _my_func: @@ -691,7 +691,7 @@ TEST(PfGraphYamlTest, YamlComputedAndWindow) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test result: @@ -714,7 +714,7 @@ TEST(PfGraphYamlTest, YamlGroupByAgg) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test result: @@ -735,7 +735,7 @@ TEST(PfGraphYamlTest, YamlSqlSource) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test result: @@ -753,7 +753,7 @@ TEST(PfGraphYamlTest, YamlPipelineReference) { auto result = CompilePfGraphYaml(R"yaml( -#!querygraph +#!perfetto_querygraph_yaml module: test _base:
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn index b693214..b29dbf3 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/android/BUILD.gn
@@ -30,10 +30,12 @@ "winscope", ] sources = [ + "anrs.pfgraph.yaml", "anrs.sql", "app_process_starts.sql", "battery.sql", "battery_stats.sql", + "binder.pfgraph.yaml", "binder.sql", "binder_breakdown.sql", "bitmaps.sql", @@ -44,12 +46,15 @@ "dvfs.sql", "entity_state_residency.sql", "freezer.sql", + "garbage_collection.pfgraph.yaml", "garbage_collection.sql", + "input.pfgraph.yaml", "input.sql", "io.sql", "job_scheduler.sql", "job_scheduler_states.sql", "kernel_wakelocks.sql", + "monitor_contention.pfgraph.yaml", "monitor_contention.sql", "network_packets.sql", "oom_adjuster.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/anrs.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/android/anrs.pfgraph.yaml index da7281c..fe1b65d 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/anrs.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/android/anrs.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: android.anrs imports: [android.process_metadata]
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/binder.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/android/binder.pfgraph.yaml index f4ff7ba..6ebe267 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/binder.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/android/binder.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: android.binder imports: [android.process_metadata, android.suspend, slices.flow, slices.with_context]
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/garbage_collection.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/android/garbage_collection.pfgraph.yaml index d4f2f29..90749ac 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/garbage_collection.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/android/garbage_collection.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: android.garbage_collection imports: [android.startup.startups, intervals.intersect, slices.with_context]
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/input.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/android/input.pfgraph.yaml index 2cc8c7e..36e40b3 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/input.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/android/input.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: android.input imports: [android.frames.timeline, intervals.intersect, slices.with_context]
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/monitor_contention.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/android/monitor_contention.pfgraph.yaml index 2a23475..df1ae58 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/monitor_contention.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/android/monitor_contention.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: android.monitor_contention imports: [android.suspend]
diff --git a/src/trace_processor/perfetto_sql/stdlib/chrome/perfetto_sql_files.gni b/src/trace_processor/perfetto_sql/stdlib/chrome/perfetto_sql_files.gni index e9b20c7..5d32b16 100644 --- a/src/trace_processor/perfetto_sql/stdlib/chrome/perfetto_sql_files.gni +++ b/src/trace_processor/perfetto_sql/stdlib/chrome/perfetto_sql_files.gni
@@ -28,6 +28,7 @@ "scroll_jank/scroll_jank_cause_map.sql", "scroll_jank/scroll_jank_cause_utils.sql", "scroll_jank/scroll_jank_intervals.sql", + "scroll_jank/scroll_jank_v3.pfgraph.yaml", "scroll_jank/scroll_jank_v3.sql", "scroll_jank/scroll_jank_v3_cause.sql", "scroll_jank/scroll_offsets.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/chrome/scroll_jank/scroll_jank_v3.pfgraph.yaml b/src/trace_processor/perfetto_sql/stdlib/chrome/scroll_jank/scroll_jank_v3.pfgraph.yaml index a24bbbd..a5cc427 100644 --- a/src/trace_processor/perfetto_sql/stdlib/chrome/scroll_jank/scroll_jank_v3.pfgraph.yaml +++ b/src/trace_processor/perfetto_sql/stdlib/chrome/scroll_jank/scroll_jank_v3.pfgraph.yaml
@@ -1,4 +1,4 @@ -#!querygraph +#!perfetto_querygraph_yaml module: chrome.scroll_jank.scroll_jank_v3 imports: [chrome.metadata, chrome.scroll_jank.scroll_jank_v3_cause, chrome.event_latency]
diff --git a/src/trace_processor/shell/BUILD.gn b/src/trace_processor/shell/BUILD.gn index aac1d2d..4f3cfc6 100644 --- a/src/trace_processor/shell/BUILD.gn +++ b/src/trace_processor/shell/BUILD.gn
@@ -101,6 +101,7 @@ deps = [ "../../../gn:default_deps", "../../base", + "../perfetto_sql/pfgraph", ] public_deps = [ "../../../include/perfetto/trace_processor:trace_processor" ] }
diff --git a/src/trace_processor/shell/query.cc b/src/trace_processor/shell/query.cc index 58b0c11..5682636 100644 --- a/src/trace_processor/shell/query.cc +++ b/src/trace_processor/shell/query.cc
@@ -33,6 +33,7 @@ #include "perfetto/ext/base/status_or.h" #include "perfetto/ext/base/string_utils.h" #include "perfetto/trace_processor/basic_types.h" +#include "src/trace_processor/perfetto_sql/pfgraph/pfgraph_compiler.h" #include "perfetto/trace_processor/iterator.h" #include "perfetto/trace_processor/trace_processor.h" @@ -182,10 +183,19 @@ base::Status RunQueries(TraceProcessor* trace_processor, const std::string& queries, bool expect_output) { - if (expect_output) { - return RunQueriesAndPrintResult(trace_processor, queries, stdout); + // Detect QueryGraph YAML format and compile to SQL. + std::string effective_queries = queries; + if (queries.find("#!perfetto_querygraph_yaml") != std::string::npos) { + auto compiled = pfgraph::CompilePfGraphYaml(queries); + if (!compiled.ok()) { + return compiled.status(); + } + effective_queries = *compiled; } - return RunQueriesWithoutOutput(trace_processor, queries); + if (expect_output) { + return RunQueriesAndPrintResult(trace_processor, effective_queries, stdout); + } + return RunQueriesWithoutOutput(trace_processor, effective_queries); } base::Status RunQueriesFromFile(TraceProcessor* trace_processor,
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc index 60bce0b..fff42e3 100644 --- a/src/trace_processor/trace_processor_impl.cc +++ b/src/trace_processor/trace_processor_impl.cc
@@ -270,8 +270,27 @@ "Module name '%s' must start with package name '%s.' as prefix.", module_name.c_str(), name.c_str()); } - new_package.modules.Insert(module_name, - {module_name_and_sql.second, false}); + const std::string& content = module_name_and_sql.second; + bool is_yaml = content.find("#!perfetto_querygraph_yaml") != std::string::npos; + + // Check if this module already exists (dual format: both .sql and .yaml). + auto* existing = new_package.modules.Find(module_name); + if (existing) { + if (is_yaml) { + existing->yaml = content; + } else { + existing->sql = content; + } + } else { + sql_modules::RegisteredPackage::ModuleFile mf; + if (is_yaml) { + mf.yaml = content; + } else { + mf.sql = content; + } + mf.included = false; + new_package.modules.Insert(module_name, std::move(mf)); + } } return base::StatusOr<sql_modules::RegisteredPackage>(std::move(new_package)); }
diff --git a/src/trace_processor/trace_processor_shell.cc b/src/trace_processor/trace_processor_shell.cc index 4803b1e..87c39e7 100644 --- a/src/trace_processor/trace_processor_shell.cc +++ b/src/trace_processor/trace_processor_shell.cc
@@ -1087,8 +1087,8 @@ return base::ErrStatus("Unable to read query graph file %s", options.query_graph_path.c_str()); } - // Detect format by #!querygraph shebang. - bool is_querygraph = pfgraph_source.find("#!querygraph") != std::string::npos; + // Detect format by #!perfetto_querygraph_yaml shebang. + bool is_querygraph = pfgraph_source.find("#!perfetto_querygraph_yaml") != std::string::npos; auto compiled = is_querygraph ? pfgraph::CompilePfGraphYaml(pfgraph_source) : pfgraph::CompilePfGraph(pfgraph_source);
diff --git a/src/trace_processor/util/sql_modules.h b/src/trace_processor/util/sql_modules.h index a93eb3d..5b34679 100644 --- a/src/trace_processor/util/sql_modules.h +++ b/src/trace_processor/util/sql_modules.h
@@ -36,7 +36,8 @@ // function. struct RegisteredPackage { struct ModuleFile { - std::string sql; + std::string sql; // SQL content (may be empty if only YAML exists). + std::string yaml; // YAML content (may be empty if only SQL exists). bool included; }; base::FlatHashMap<std::string, ModuleFile> modules; @@ -51,8 +52,20 @@ return str; } +// Checks if a path is a QueryGraph YAML file (.pfgraph.yaml extension). +inline bool IsQueryGraphYaml(const std::string& path) { + static const char kSuffix[] = ".pfgraph.yaml"; + return path.size() > strlen(kSuffix) && + path.substr(path.size() - strlen(kSuffix)) == kSuffix; +} + inline std::string GetIncludeKey(const std::string& path) { base::StringView path_view(path); + // Handle .pfgraph.yaml compound extension. + if (IsQueryGraphYaml(path)) { + auto stripped = path.substr(0, path.size() - strlen(".pfgraph.yaml")); + return ReplaceSlashWithDot(stripped); + } auto path_no_extension = path_view.substr(0, path_view.rfind('.')); return ReplaceSlashWithDot(path_no_extension.ToStdString()); }
diff --git a/tools/gen_amalgamated_sql.py b/tools/gen_amalgamated_sql.py index 68b91fc..1a3fba4 100755 --- a/tools/gen_amalgamated_sql.py +++ b/tools/gen_amalgamated_sql.py
@@ -65,6 +65,13 @@ ''' +def strip_extension(filename: str): + """Strip file extension, handling compound extensions like .pfgraph.yaml.""" + if filename.endswith('.pfgraph.yaml'): + return filename[:-len('.pfgraph.yaml')] + '_pfgraph' + return os.path.splitext(filename)[0] + + def filename_to_variable(filename: str): return "k" + "".join([ x.capitalize() @@ -117,7 +124,7 @@ # Create the C++ variable for each SQL file. for path, sql in sql_outputs.items(): - variable = filename_to_variable(os.path.splitext(path)[0]) + variable = filename_to_variable(strip_extension(path)) output.write('\nconst char {}[] = '.format(variable)) # MSVC doesn't like string literals that are individually longer than 16k. # However it's still fine "if" "we" "concatenate" "many" "of" "them". @@ -137,7 +144,7 @@ # Create mapping of filename to variable name for each variable. output.write("\nconst FileToSql kFileToSql[] = {") for path in sql_outputs.keys(): - variable = filename_to_variable(os.path.splitext(path)[0]) + variable = filename_to_variable(strip_extension(path)) # This is for Windows which has \ as a path separator. path = path.replace("\\", "/")