trace_processor: add template substitution support for SQL metrics
Also implement the basic process counter metrics to prove the concept.
Context: go/perfetto-metrics
Bug: 129747127
Change-Id: Ia9304328f6368bca085135fac0178243ffc0d9df
diff --git a/Android.bp b/Android.bp
index 62bf1cf..3544957 100644
--- a/Android.bp
+++ b/Android.bp
@@ -19,6 +19,7 @@
srcs: [
"src/trace_processor/metrics/android/android_mem.sql",
"src/trace_processor/metrics/android/android_mem_lmk.sql",
+ "src/trace_processor/metrics/android/android_mem_proc_counters.sql",
],
cmd: "$(location tools/gen_merged_sql_metrics.py) --cpp_out=$(out) $(in)",
out: [
diff --git a/BUILD b/BUILD
index 164767e..489b8a8 100644
--- a/BUILD
+++ b/BUILD
@@ -26,6 +26,7 @@
srcs = [
"src/trace_processor/metrics/android/android_mem.sql",
"src/trace_processor/metrics/android/android_mem_lmk.sql",
+ "src/trace_processor/metrics/android/android_mem_proc_counters.sql",
],
outs = [
"src/trace_processor/metrics/sql_metrics.h",
diff --git a/BUILD.gn b/BUILD.gn
index 084bf9d..830a488 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -138,7 +138,10 @@
deps += [ "src/profiling/memory:unittests" ]
}
if (perfetto_build_standalone && !is_android) {
- deps += [ "src/trace_processor:unittests" ]
+ deps += [
+ "src/trace_processor:unittests",
+ "src/trace_processor/metrics:unittests",
+ ]
}
}
diff --git a/include/perfetto/trace_processor/basic_types.h b/include/perfetto/trace_processor/basic_types.h
index 535969c..32ca2cd 100644
--- a/include/perfetto/trace_processor/basic_types.h
+++ b/include/perfetto/trace_processor/basic_types.h
@@ -19,10 +19,11 @@
#include <stdint.h>
+#include "perfetto/base/logging.h"
+
namespace perfetto {
namespace trace_processor {
-
struct Config {
uint64_t window_size_ns = 180 * 1000 * 1000 * 1000ULL; // 3 minutes.
};
@@ -37,6 +38,11 @@
kDouble,
};
+ double AsDouble() {
+ PERFETTO_CHECK(type == kDouble);
+ return double_value;
+ }
+
// Up to 1 of these fields can be accessed depending on |type|.
union {
// This string will be owned by the iterator that returned it and is valid
diff --git a/src/trace_processor/metrics/BUILD.gn b/src/trace_processor/metrics/BUILD.gn
index 1f0fdbf..2cc26cb 100644
--- a/src/trace_processor/metrics/BUILD.gn
+++ b/src/trace_processor/metrics/BUILD.gn
@@ -17,6 +17,7 @@
sql_files = [
"android/android_mem.sql",
"android/android_mem_lmk.sql",
+ "android/android_mem_proc_counters.sql",
]
config("gen_config") {
@@ -52,3 +53,16 @@
"../../protozero:protozero",
]
}
+
+source_set("unittests") {
+ testonly = true
+ sources = [
+ "metrics_unittest.cc",
+ ]
+ deps = [
+ ":lib",
+ "../../../buildtools:sqlite",
+ "../../../gn:default_deps",
+ "../../../gn:gtest_deps",
+ ]
+}
diff --git a/src/trace_processor/metrics/android/android_mem.sql b/src/trace_processor/metrics/android/android_mem.sql
index 7646381..4d44723 100644
--- a/src/trace_processor/metrics/android/android_mem.sql
+++ b/src/trace_processor/metrics/android/android_mem.sql
@@ -16,3 +16,15 @@
-- Create all the views used to generate the Android Memory metrics proto.
SELECT RUN_METRIC('android_mem_lmk.sql');
+
+-- Generate the process counter metrics.
+SELECT RUN_METRIC('android_mem_proc_counters.sql',
+ 'table_name',
+ 'file_rss',
+ 'counter_names',
+ '("mem.rss.anon")');
+SELECT RUN_METRIC('android_mem_proc_counters.sql',
+ 'table_name',
+ 'anon_rss',
+ 'counter_names',
+ '("mem.rss.anon")');
diff --git a/src/trace_processor/metrics/android/android_mem_proc_counters.sql b/src/trace_processor/metrics/android/android_mem_proc_counters.sql
new file mode 100644
index 0000000..241bbb2
--- /dev/null
+++ b/src/trace_processor/metrics/android/android_mem_proc_counters.sql
@@ -0,0 +1,36 @@
+--
+-- Copyright 2019 The Android Open Source Project
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- https://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE VIEW {{table_name}}_span AS
+SELECT
+ ts,
+ LEAD(ts, 1, ts) OVER(PARTITION BY counter_id ORDER BY ts) - ts AS dur,
+ ref AS upid,
+ value
+FROM counters
+WHERE name IN {{counter_names}} AND ref IS NOT NULL AND ref_type = 'upid';
+
+CREATE VIEW {{table_name}} AS
+SELECT
+ process.name,
+ MIN(span.value),
+ MAX(span.value),
+ SUM(span.value * span.dur) / SUM(span.dur)
+FROM {{table_name}}_span as span JOIN process USING(upid)
+WHERE NOT (process.name IS NULL OR process.name = '')
+GROUP BY 1
+HAVING SUM(span.dur) > 0
+ORDER BY 1;
diff --git a/src/trace_processor/metrics/metrics.cc b/src/trace_processor/metrics/metrics.cc
index 48a6f19..2abd37d 100644
--- a/src/trace_processor/metrics/metrics.cc
+++ b/src/trace_processor/metrics/metrics.cc
@@ -16,6 +16,10 @@
#include "src/trace_processor/metrics/metrics.h"
+#include <regex>
+#include <unordered_map>
+#include <vector>
+
#include "perfetto/base/string_utils.h"
#include "perfetto/metrics/android/mem_metric.pbzero.h"
#include "perfetto/metrics/metrics.pbzero.h"
@@ -26,10 +30,44 @@
namespace trace_processor {
namespace metrics {
+namespace {
+// TODO(lalitm): delete this and use sqlite_utils when that is cleaned up of
+// trace processor dependencies.
+const char* ExtractSqliteValue(sqlite3_value* value) {
+ auto type = sqlite3_value_type(value);
+ PERFETTO_DCHECK(type == SQLITE_TEXT);
+ return reinterpret_cast<const char*>(sqlite3_value_text(value));
+}
+} // namespace
+
+int TemplateReplace(
+ const std::string& raw_text,
+ const std::unordered_map<std::string, std::string>& substitutions,
+ std::string* out) {
+ std::regex re(R"(\{\{\s*(\w*)\s*\}\})", std::regex_constants::ECMAScript);
+
+ auto it = std::sregex_iterator(raw_text.begin(), raw_text.end(), re);
+ auto regex_end = std::sregex_iterator();
+ auto start = raw_text.begin();
+ for (; it != regex_end; ++it) {
+ out->insert(out->end(), start, raw_text.begin() + it->position(0));
+
+ auto value_it = substitutions.find(it->str(1));
+ if (value_it == substitutions.end())
+ return 1;
+
+ const auto& value = value_it->second;
+ std::copy(value.begin(), value.end(), std::back_inserter(*out));
+ start = raw_text.begin() + it->position(0) + it->length(0);
+ }
+ out->insert(out->end(), start, raw_text.end());
+ return 0;
+}
+
void RunMetric(sqlite3_context* ctx, int argc, sqlite3_value** argv) {
auto* tp = static_cast<TraceProcessor*>(sqlite3_user_data(ctx));
if (argc == 0 || sqlite3_value_type(argv[0]) != SQLITE_TEXT) {
- sqlite3_result_error(ctx, "Invalid call to RUN_METRIC", -1);
+ sqlite3_result_error(ctx, "RUN_METRIC: Invalid arguments", -1);
return;
}
@@ -37,20 +75,43 @@
reinterpret_cast<const char*>(sqlite3_value_text(argv[0]));
const char* sql = sql_metrics::GetBundledMetric(filename);
if (!sql) {
- sqlite3_result_error(ctx, "Unknown filename provided to RUN_METRIC", -1);
+ sqlite3_result_error(ctx, "RUN_METRIC: Unknown filename provided", -1);
return;
}
- for (const auto& query : base::SplitString(sql, ";\n\n")) {
- PERFETTO_DLOG("Executing query in RUN_METRIC: %s", query.c_str());
+ std::unordered_map<std::string, std::string> substitutions;
+ for (int i = 1; i < argc; i += 2) {
+ if (sqlite3_value_type(argv[i]) != SQLITE_TEXT) {
+ sqlite3_result_error(ctx, "RUN_METRIC: Invalid args", -1);
+ return;
+ }
- auto it = tp->ExecuteQuery(query);
+ auto* key_str = ExtractSqliteValue(argv[i]);
+ auto* value_str = ExtractSqliteValue(argv[i + 1]);
+ substitutions[key_str] = value_str;
+ }
+
+ for (const auto& query : base::SplitString(sql, ";\n")) {
+ std::string buffer;
+ int ret = TemplateReplace(query, substitutions, &buffer);
+ if (ret) {
+ sqlite3_result_error(
+ ctx, "RUN_METRIC: Error when performing substitution", -1);
+ return;
+ }
+
+ PERFETTO_DLOG("RUN_METRIC: Executing query: %s", buffer.c_str());
+ auto it = tp->ExecuteQuery(buffer);
if (auto opt_error = it.GetLastError()) {
- sqlite3_result_error(ctx, "Error when running RUN_METRIC file", -1);
+ char* error =
+ sqlite3_mprintf("RUN_METRIC: Error when running file %s: %s",
+ filename, opt_error->c_str());
+ sqlite3_result_error(ctx, error, -1);
+ sqlite3_free(error);
return;
} else if (it.Next()) {
sqlite3_result_error(
- ctx, "RUN_METRIC functions should not produce any output", -1);
+ ctx, "RUN_METRIC: functions should not produce any output", -1);
return;
}
}
@@ -66,7 +127,7 @@
return 1;
}
- auto queries = base::SplitString(sql_metrics::kAndroidMem, ";\n\n");
+ auto queries = base::SplitString(sql_metrics::kAndroidMem, ";\n");
for (const auto& query : queries) {
PERFETTO_DLOG("Executing query: %s", query.c_str());
auto prep_it = tp->ExecuteQuery(query);
@@ -96,15 +157,32 @@
PERFETTO_CHECK(has_next);
PERFETTO_CHECK(it.Get(0).type == SqlValue::Type::kLong);
+ has_next = it.Next();
+ PERFETTO_DCHECK(!has_next);
+
auto* memory = metrics.set_android_mem();
memory->set_system_metrics()->set_lmks()->set_total_count(
static_cast<int32_t>(it.Get(0).long_value));
+
+ it = tp->ExecuteQuery("SELECT * from anon_rss;");
+ while (it.Next()) {
+ const char* name = it.Get(0).string_value;
+
+ auto* process = memory->add_process_metrics();
+ process->set_process_name(name);
+
+ auto* anon = process->set_overall_counters()->set_anon_rss();
+ anon->set_min(it.Get(1).AsDouble());
+ anon->set_max(it.Get(2).AsDouble());
+ anon->set_avg(it.Get(3).AsDouble());
+ }
+ if (auto opt_error = it.GetLastError()) {
+ PERFETTO_ELOG("SQLite error: %s", opt_error->c_str());
+ return 1;
+ }
+
metrics.Finalize();
-
*metrics_proto = delegate.StitchSlices();
-
- has_next = it.Next();
- PERFETTO_DCHECK(!has_next);
return 0;
}
diff --git a/src/trace_processor/metrics/metrics.h b/src/trace_processor/metrics/metrics.h
index 324d436..4c85ecb 100644
--- a/src/trace_processor/metrics/metrics.h
+++ b/src/trace_processor/metrics/metrics.h
@@ -18,6 +18,7 @@
#define SRC_TRACE_PROCESSOR_METRICS_METRICS_H_
#include <sqlite3.h>
+#include <unordered_map>
#include <vector>
#include "perfetto/trace_processor/trace_processor.h"
@@ -26,6 +27,16 @@
namespace trace_processor {
namespace metrics {
+// Replaces templated variables inside |raw_text| using the substitution given
+// by |substitutions| writing the result to |out|.
+// The syntax followed is a cut-down variant of Jinja. This means variables that
+// are to be replaced use {{variable-name}} in the raw text with subsitutions
+// containing a mapping from (variable-name -> replacement).
+int TemplateReplace(
+ const std::string& raw_text,
+ const std::unordered_map<std::string, std::string>& substitutions,
+ std::string* out);
+
// This function implements the RUN_METRIC SQL function.
void RunMetric(sqlite3_context* ctx, int argc, sqlite3_value** argv);
diff --git a/src/trace_processor/metrics/metrics_unittest.cc b/src/trace_processor/metrics/metrics_unittest.cc
new file mode 100644
index 0000000..110ae26
--- /dev/null
+++ b/src/trace_processor/metrics/metrics_unittest.cc
@@ -0,0 +1,56 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/metrics/metrics.h"
+
+#include <vector>
+
+#include "gtest/gtest.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace metrics {
+
+namespace {
+
+std::string RunTemplateReplace(
+ const std::string& str,
+ std::unordered_map<std::string, std::string> subs) {
+ std::string out;
+ EXPECT_EQ(TemplateReplace(str, subs, &out), 0);
+ return out;
+}
+
+TEST(MetricsTest, TemplateReplace) {
+ auto res = RunTemplateReplace("no templates here", {});
+ ASSERT_EQ(res, "no templates here");
+
+ res = RunTemplateReplace("{{justtemplate}}", {{"justtemplate", "result"}});
+ ASSERT_EQ(res, "result");
+
+ res = RunTemplateReplace("{{temp1}} {{temp2}}!",
+ {{"temp1", "hello"}, {"temp2", "world"}});
+ ASSERT_EQ(res, "hello world!");
+
+ std::string unused;
+ ASSERT_NE(TemplateReplace("{{missing}}", {{}}, &unused), 0);
+}
+
+} // namespace
+
+} // namespace metrics
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/sqlite_utils.h b/src/trace_processor/sqlite_utils.h
index 7f5d732..5bd1e5f 100644
--- a/src/trace_processor/sqlite_utils.h
+++ b/src/trace_processor/sqlite_utils.h
@@ -136,12 +136,15 @@
// uint64_t at all given that SQLite doesn't support it.
template <>
-inline std::string ExtractSqliteValue(sqlite3_value* value) {
+inline const char* ExtractSqliteValue(sqlite3_value* value) {
auto type = sqlite3_value_type(value);
PERFETTO_DCHECK(type == SQLITE_TEXT);
- const auto* extracted =
- reinterpret_cast<const char*>(sqlite3_value_text(value));
- return std::string(extracted);
+ return reinterpret_cast<const char*>(sqlite3_value_text(value));
+}
+
+template <>
+inline std::string ExtractSqliteValue(sqlite3_value* value) {
+ return ExtractSqliteValue<const char*>(value);
}
template <typename T>
diff --git a/tools/gen_merged_sql_metrics.py b/tools/gen_merged_sql_metrics.py
index 071423c..ba3b0c7 100755
--- a/tools/gen_merged_sql_metrics.py
+++ b/tools/gen_merged_sql_metrics.py
@@ -87,20 +87,19 @@
args = parser.parse_args()
# Extract the SQL output from each file.
- escaped_sql_outputs = {}
+ sql_outputs = {}
for file_name in args.sql_files:
with open(file_name, 'r') as f:
basename = os.path.basename(file_name)
-
- # Escape any quote characters.
- escaped_sql_outputs[basename] = "".join(f.readlines())
+ sql_outputs[basename] = "".join(
+ x for x in f.readlines() if not x.startswith('--'))
with open(args.cpp_out, 'w+') as output:
output.write(REPLACEMENT_HEADER)
output.write(NAMESPACE_BEGIN)
# Create the C++ variable for each SQL file.
- for name, sql in escaped_sql_outputs.items():
+ for name, sql in sql_outputs.items():
variable = filename_to_variable(os.path.splitext(name)[0])
output.write('\nconst char {}[] = R"gendelimiter(\n{})gendelimiter";\n'
.format(variable, sql))
@@ -109,7 +108,7 @@
# Create mapping of filename to variable name for each variable.
output.write("\nconst FileToSql kFileToSql[] = {")
- for name in escaped_sql_outputs.keys():
+ for name in sql_outputs.keys():
variable = filename_to_variable(os.path.splitext(name)[0])
output.write('\n {{"{}", {}}},\n'.format(name, variable))
output.write("};\n")
