metrics: Implement loading metric extension directory
You can now pass in --metric-extension flag to trace_processor_shell.
The flag is in the format disk_path:virtual_path.
Any internal metric proto/sql files matching the virtual_path location
is skipped. All protos in the disk_path are loaded with
"protos/perfetto/metrics/virtual_path/" location, and all SQL files are
loaded in the "virtual_path/" location.
Design doc: go/perfetto-runtime-metrics-import
Bug: 182165266
Change-Id: I6ce3d5881219e5099b6eb9c6f653b1dfded51297
diff --git a/include/perfetto/ext/base/file_utils.h b/include/perfetto/ext/base/file_utils.h
index bf33d70..0acd537 100644
--- a/include/perfetto/ext/base/file_utils.h
+++ b/include/perfetto/ext/base/file_utils.h
@@ -21,9 +21,11 @@
#include <stddef.h>
#include <string>
+#include <vector>
#include "perfetto/base/build_config.h"
#include "perfetto/base/export.h"
+#include "perfetto/base/status.h"
#include "perfetto/ext/base/scoped_file.h"
#include "perfetto/ext/base/utils.h"
@@ -78,6 +80,17 @@
// Wrapper around access(path, F_OK).
bool FileExists(const std::string& path);
+// Gets the extension for a filename. If the file has two extensions, returns
+// only the last one (foo.pb.gz => .gz). Returns empty string if there is no
+// extension.
+std::string GetFileExtension(const std::string& filename);
+
+// Puts the path to all files under |dir_path| in |output|, recursively walking
+// subdirectories. File paths are relative to |dir_path|. Only files are
+// included, not directories.
+base::Status ListFilesRecursive(const std::string& dir_path,
+ std::vector<std::string>& output);
+
} // namespace base
} // namespace perfetto
diff --git a/include/perfetto/ext/base/string_utils.h b/include/perfetto/ext/base/string_utils.h
index 13b8f16..a44ea73 100644
--- a/include/perfetto/ext/base/string_utils.h
+++ b/include/perfetto/ext/base/string_utils.h
@@ -97,6 +97,8 @@
bool StartsWith(const std::string& str, const std::string& prefix);
bool EndsWith(const std::string& str, const std::string& suffix);
+bool StartsWithAny(const std::string& str,
+ const std::vector<std::string>& prefixes);
bool Contains(const std::string& haystack, const std::string& needle);
bool Contains(const std::string& haystack, char needle);
size_t Find(const StringView& needle, const StringView& haystack);
diff --git a/include/perfetto/trace_processor/basic_types.h b/include/perfetto/trace_processor/basic_types.h
index 2b38b6d..05735b9 100644
--- a/include/perfetto/trace_processor/basic_types.h
+++ b/include/perfetto/trace_processor/basic_types.h
@@ -23,6 +23,7 @@
#include <stdint.h>
#include <functional>
#include <string>
+#include <vector>
#include "perfetto/base/export.h"
#include "perfetto/base/logging.h"
@@ -34,6 +35,10 @@
// simpler (e.g. use arrays instead of vectors).
constexpr size_t kMaxCpus = 128;
+// All metrics protos are in this directory. When loading metric extensions, the
+// protos are mounted onto a virtual path inside this directory.
+constexpr char kMetricProtoRoot[] = "protos/perfetto/metrics/";
+
// Enum which encodes how trace processor should try to sort the ingested data.
enum class SortingMode {
// This option allows trace processor to use built-in heuristics about how to
@@ -106,6 +111,10 @@
// the trace before that event. See the ennu documenetation for more details.
DropFtraceDataBefore drop_ftrace_data_before =
DropFtraceDataBefore::kTracingStarted;
+
+ // Any built-in metric proto or sql files matching these paths are skipped
+ // during trace processor metric initialization.
+ std::vector<std::string> skip_builtin_metric_paths;
};
// Represents a dynamically typed value returned by SQL.
diff --git a/include/perfetto/trace_processor/trace_processor.h b/include/perfetto/trace_processor/trace_processor.h
index c949695..77a7889 100644
--- a/include/perfetto/trace_processor/trace_processor.h
+++ b/include/perfetto/trace_processor/trace_processor.h
@@ -57,6 +57,13 @@
// proto builder functions when computing metrics.
virtual util::Status ExtendMetricsProto(const uint8_t* data, size_t size) = 0;
+ // Behaves exactly as ExtendMetricsProto, except any FileDescriptor with
+ // filename matching a prefix in |skip_prefixes| is skipped.
+ virtual util::Status ExtendMetricsProto(
+ const uint8_t* data,
+ size_t size,
+ const std::vector<std::string>& skip_prefixes) = 0;
+
// Computes the given metrics on the loded portion of the trace. If
// successful, the output argument |metrics_proto| will be filled with the
// proto-encoded bytes for the message TraceMetrics in
diff --git a/src/base/file_utils.cc b/src/base/file_utils.cc
index 9b92af2..baa9b2e 100644
--- a/src/base/file_utils.cc
+++ b/src/base/file_utils.cc
@@ -20,10 +20,13 @@
#include <sys/types.h>
#include <algorithm>
+#include <string>
+#include <vector>
#include "perfetto/base/build_config.h"
#include "perfetto/base/logging.h"
#include "perfetto/base/platform_handle.h"
+#include "perfetto/base/status.h"
#include "perfetto/ext/base/scoped_file.h"
#include "perfetto/ext/base/utils.h"
@@ -208,5 +211,39 @@
#endif
}
+base::Status ListFilesRecursive(const std::string& dir_path,
+ std::vector<std::string>& output) {
+#if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
+ // TODO(b/182165266): Write the windows equivalent of this function.
+ return base::ErrStatus("ListFilesRecursive not supported in windows yet");
+#else
+ DIR* dir = opendir(dir_path.c_str());
+ if (dir == nullptr) {
+ return base::ErrStatus("Failed to open directory %s", dir_path.c_str());
+ }
+ for (auto* dirent = readdir(dir); dirent != nullptr; dirent = readdir(dir)) {
+ if (strcmp(dirent->d_name, ".") == 0 || strcmp(dirent->d_name, "..") == 0) {
+ continue;
+ }
+ if (dirent->d_type == DT_DIR) {
+ std::string full_path = dir_path + '/' + dirent->d_name;
+ auto status = ListFilesRecursive(full_path, output);
+ if (!status.ok())
+ return status;
+ } else if (dirent->d_type == DT_REG) {
+ output.push_back(dirent->d_name);
+ }
+ }
+ return base::OkStatus();
+#endif
+}
+
+std::string GetFileExtension(const std::string& filename) {
+ auto ext_idx = filename.rfind('.');
+ if (ext_idx == std::string::npos)
+ return std::string();
+ return filename.substr(ext_idx);
+}
+
} // namespace base
} // namespace perfetto
diff --git a/src/base/string_utils.cc b/src/base/string_utils.cc
index 975e42c..a6ee833 100644
--- a/src/base/string_utils.cc
+++ b/src/base/string_utils.cc
@@ -18,12 +18,12 @@
#include <locale.h>
#include <string.h>
+#include <algorithm>
#if PERFETTO_BUILDFLAG(PERFETTO_OS_APPLE)
#include <xlocale.h>
#endif
-#include <algorithm>
#include <cinttypes>
#include "perfetto/base/logging.h"
@@ -88,6 +88,13 @@
return str.compare(0, prefix.length(), prefix) == 0;
}
+bool StartsWithAny(const std::string& str,
+ const std::vector<std::string>& prefixes) {
+ return std::any_of(
+ prefixes.begin(), prefixes.end(),
+ [&str](const std::string& prefix) { return StartsWith(str, prefix); });
+}
+
bool EndsWith(const std::string& str, const std::string& suffix) {
if (suffix.size() > str.size())
return false;
diff --git a/src/base/string_utils_unittest.cc b/src/base/string_utils_unittest.cc
index 5c41f16..2c7108d 100644
--- a/src/base/string_utils_unittest.cc
+++ b/src/base/string_utils_unittest.cc
@@ -156,6 +156,14 @@
EXPECT_FALSE(StartsWith("", "ab"));
}
+TEST(StringUtilsTest, StartsWithAny) {
+ EXPECT_FALSE(StartsWithAny("", {"a", "b"}));
+ EXPECT_FALSE(StartsWithAny("abcd", {}));
+ EXPECT_FALSE(StartsWithAny("", {}));
+ EXPECT_TRUE(StartsWithAny("abcd", {"ac", "ab"}));
+ EXPECT_FALSE(StartsWithAny("abcd", {"bc", "ac"}));
+}
+
TEST(StringUtilsTest, EndsWith) {
EXPECT_TRUE(EndsWith("", ""));
EXPECT_TRUE(EndsWith("abc", ""));
diff --git a/src/trace_processor/importers/proto/proto_trace_reader.cc b/src/trace_processor/importers/proto/proto_trace_reader.cc
index be5f1ff..4b133a8 100644
--- a/src/trace_processor/importers/proto/proto_trace_reader.cc
+++ b/src/trace_processor/importers/proto/proto_trace_reader.cc
@@ -69,6 +69,7 @@
auto extension = decoder.extension_set();
return context_->descriptor_pool_->AddFromFileDescriptorSet(
extension.data, extension.size,
+ /*skip_prefixes*/ {},
/*merge_existing_messages=*/true);
}
diff --git a/src/trace_processor/trace_database_integrationtest.cc b/src/trace_processor/trace_database_integrationtest.cc
index c6a8c62..e427f93 100644
--- a/src/trace_processor/trace_database_integrationtest.cc
+++ b/src/trace_processor/trace_database_integrationtest.cc
@@ -35,6 +35,44 @@
constexpr size_t kMaxChunkSize = 4 * 1024 * 1024;
+TEST(TraceProcessorCustomConfigTest, SkipInternalMetricsMatchingMountPath) {
+ auto config = Config();
+ config.skip_builtin_metric_paths = {"android/"};
+ auto processor = TraceProcessor::CreateInstance(config);
+ processor->NotifyEndOfFile();
+
+ // Check that andorid metrics have not been loaded.
+ auto it = processor->ExecuteQuery(
+ "select count(*) from trace_metrics "
+ "where name = 'android_cpu';");
+ ASSERT_TRUE(it.Next());
+ ASSERT_EQ(it.Get(0).type, SqlValue::kLong);
+ ASSERT_EQ(it.Get(0).long_value, 0);
+
+ // Check that other metrics have been loaded.
+ it = processor->ExecuteQuery(
+ "select count(*) from trace_metrics "
+ "where name = 'trace_metadata';");
+ ASSERT_TRUE(it.Next());
+ ASSERT_EQ(it.Get(0).type, SqlValue::kLong);
+ ASSERT_EQ(it.Get(0).long_value, 1);
+}
+
+TEST(TraceProcessorCustomConfigTest, HandlesMalformedMountPath) {
+ auto config = Config();
+ config.skip_builtin_metric_paths = {"", "androi"};
+ auto processor = TraceProcessor::CreateInstance(config);
+ processor->NotifyEndOfFile();
+
+ // Check that andorid metrics have been loaded.
+ auto it = processor->ExecuteQuery(
+ "select count(*) from trace_metrics "
+ "where name = 'android_cpu';");
+ ASSERT_TRUE(it.Next());
+ ASSERT_EQ(it.Get(0).type, SqlValue::kLong);
+ ASSERT_EQ(it.Get(0).long_value, 1);
+}
+
class TraceProcessorIntegrationTest : public ::testing::Test {
public:
TraceProcessorIntegrationTest()
@@ -397,6 +435,7 @@
ASSERT_TRUE(it.Next());
EXPECT_STREQ(it.Get(0).string_value, "123e4567-e89b-12d3-a456-426655443322");
}
+
} // namespace
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc
index 97a1ba9..0d7b5ea 100644
--- a/src/trace_processor/trace_processor_impl.cc
+++ b/src/trace_processor/trace_processor_impl.cc
@@ -634,14 +634,37 @@
}
}
+std::vector<std::string> SanitizeMetricMountPaths(
+ const std::vector<std::string>& mount_paths) {
+ std::vector<std::string> sanitized;
+ for (const auto& path : mount_paths) {
+ if (path.length() == 0)
+ continue;
+ sanitized.push_back(path);
+ if (path.back() != '/')
+ sanitized.back().append("/");
+ }
+ return sanitized;
+}
+
void SetupMetrics(TraceProcessor* tp,
sqlite3* db,
- std::vector<metrics::SqlMetricFile>* sql_metrics) {
- tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size());
+ std::vector<metrics::SqlMetricFile>* sql_metrics,
+ const std::vector<std::string>& extension_paths) {
+ const std::vector<std::string> sanitized_extension_paths =
+ SanitizeMetricMountPaths(extension_paths);
+ std::vector<std::string> skip_prefixes;
+ for (const auto& path : sanitized_extension_paths) {
+ skip_prefixes.push_back(kMetricProtoRoot + path);
+ }
+ tp->ExtendMetricsProto(kMetricsDescriptor.data(), kMetricsDescriptor.size(),
+ skip_prefixes);
tp->ExtendMetricsProto(kAllChromeMetricsDescriptor.data(),
- kAllChromeMetricsDescriptor.size());
+ kAllChromeMetricsDescriptor.size(), skip_prefixes);
for (const auto& file_to_sql : metrics::sql_metrics::kFileToSql) {
+ if (base::StartsWithAny(file_to_sql.path, sanitized_extension_paths))
+ continue;
tp->RegisterMetric(file_to_sql.path, file_to_sql.sql);
}
@@ -731,7 +754,7 @@
CreateValueAtMaxTsFunction(db);
CreateUnwrapMetricProtoFunction(db);
- SetupMetrics(this, *db_, &sql_metrics_);
+ SetupMetrics(this, *db_, &sql_metrics_, cfg.skip_builtin_metric_paths);
// Setup the query cache.
query_cache_.reset(new QueryCache());
@@ -1012,7 +1035,15 @@
util::Status TraceProcessorImpl::ExtendMetricsProto(const uint8_t* data,
size_t size) {
- util::Status status = pool_.AddFromFileDescriptorSet(data, size);
+ return ExtendMetricsProto(data, size, /*skip_prefixes*/ {});
+}
+
+util::Status TraceProcessorImpl::ExtendMetricsProto(
+ const uint8_t* data,
+ size_t size,
+ const std::vector<std::string>& skip_prefixes) {
+ util::Status status =
+ pool_.AddFromFileDescriptorSet(data, size, skip_prefixes);
if (!status.ok())
return status;
diff --git a/src/trace_processor/trace_processor_impl.h b/src/trace_processor/trace_processor_impl.h
index cb9ef9f..5db351c 100644
--- a/src/trace_processor/trace_processor_impl.h
+++ b/src/trace_processor/trace_processor_impl.h
@@ -61,6 +61,11 @@
util::Status ExtendMetricsProto(const uint8_t* data, size_t size) override;
+ util::Status ExtendMetricsProto(
+ const uint8_t* data,
+ size_t size,
+ const std::vector<std::string>& skip_prefixes) override;
+
util::Status ComputeMetric(const std::vector<std::string>& metric_names,
std::vector<uint8_t>* metrics) override;
@@ -118,7 +123,6 @@
uint64_t bytes_parsed_ = 0;
};
-
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/trace_processor/trace_processor_shell.cc b/src/trace_processor/trace_processor_shell.cc
index 9b2d671..b50c172 100644
--- a/src/trace_processor/trace_processor_shell.cc
+++ b/src/trace_processor/trace_processor_shell.cc
@@ -21,6 +21,7 @@
#include <cinttypes>
#include <functional>
#include <iostream>
+#include <unordered_set>
#include <vector>
#include <google/protobuf/compiler/parser.h>
@@ -30,6 +31,7 @@
#include "perfetto/base/build_config.h"
#include "perfetto/base/logging.h"
+#include "perfetto/base/status.h"
#include "perfetto/base/time.h"
#include "perfetto/ext/base/file_utils.h"
#include "perfetto/ext/base/getopt.h"
@@ -42,6 +44,7 @@
#include "perfetto/trace_processor/trace_processor.h"
#include "src/trace_processor/metrics/chrome/all_chrome_metrics.descriptor.h"
#include "src/trace_processor/metrics/metrics.descriptor.h"
+#include "src/trace_processor/metrics/metrics.h"
#include "src/trace_processor/util/proto_to_json.h"
#include "src/trace_processor/util/status_macros.h"
@@ -343,23 +346,28 @@
return g_tp->RegisterMetric(path, sql);
}
-util::Status ExtendMetricsProto(const std::string& extend_metrics_proto,
- google::protobuf::DescriptorPool* pool) {
- google::protobuf::FileDescriptorSet desc_set;
-
- base::ScopedFile file(base::OpenFile(extend_metrics_proto, O_RDONLY));
+base::Status ParseToFileDescriptorProto(
+ const std::string& filename,
+ google::protobuf::FileDescriptorProto* file_desc) {
+ base::ScopedFile file(base::OpenFile(filename, O_RDONLY));
if (file.get() == -1) {
- return util::ErrStatus("Failed to open proto file %s",
- extend_metrics_proto.c_str());
+ return base::ErrStatus("Failed to open proto file %s", filename.c_str());
}
google::protobuf::io::FileInputStream stream(file.get());
ErrorPrinter printer;
google::protobuf::io::Tokenizer tokenizer(&stream, &printer);
- auto* file_desc = desc_set.add_file();
google::protobuf::compiler::Parser parser;
parser.Parse(&tokenizer, file_desc);
+ return base::OkStatus();
+}
+
+util::Status ExtendMetricsProto(const std::string& extend_metrics_proto,
+ google::protobuf::DescriptorPool* pool) {
+ google::protobuf::FileDescriptorSet desc_set;
+ auto* file_desc = desc_set.add_file();
+ RETURN_IF_ERROR(ParseToFileDescriptorProto(extend_metrics_proto, file_desc));
file_desc->set_name(BaseName(extend_metrics_proto));
pool->BuildFile(*file_desc);
@@ -656,6 +664,33 @@
return util::OkStatus();
}
+class MetricExtension {
+ public:
+ void SetDiskPath(std::string path) {
+ AddTrailingSlashIfNeeded(path);
+ disk_path_ = std::move(path);
+ }
+ void SetVirtualPath(std::string path) {
+ AddTrailingSlashIfNeeded(path);
+ virtual_path_ = std::move(path);
+ }
+
+ // Disk location. Ends with a trailing slash.
+ const std::string& disk_path() const { return disk_path_; }
+ // Virtual location. Ends with a trailing slash.
+ const std::string& virtual_path() const { return virtual_path_; }
+
+ private:
+ std::string disk_path_;
+ std::string virtual_path_;
+
+ static void AddTrailingSlashIfNeeded(std::string& path) {
+ if (path.length() > 0 && path[path.length() - 1] != '/') {
+ path.push_back('/');
+ }
+ }
+};
+
struct CommandLineOptions {
std::string perf_file_path;
std::string query_file_path;
@@ -665,6 +700,7 @@
std::string metric_output;
std::string trace_file_path;
std::string port_number;
+ std::vector<std::string> raw_metric_extensions;
bool launch_shell = false;
bool enable_httpd = false;
bool wide = false;
@@ -716,7 +752,12 @@
writing the resulting trace into FILE.
--full-sort Forces the trace processor into performing
a full sort ignoring any windowing
- logic.)",
+ logic.
+ --metric-extension DISK_PATH:VIRTUAL_PATH
+ Loads metric proto and sql files from
+ DISK_PATH/protos and DISK_PATH/sql
+ respectively, and mounts them onto
+ VIRTUAL_PATH.)",
argv[0]);
}
@@ -728,6 +769,7 @@
OPT_METRICS_OUTPUT,
OPT_FORCE_FULL_SORT,
OPT_HTTP_PORT,
+ OPT_METRIC_EXTENSION,
};
static const option long_options[] = {
@@ -746,6 +788,7 @@
{"metrics-output", required_argument, nullptr, OPT_METRICS_OUTPUT},
{"full-sort", no_argument, nullptr, OPT_FORCE_FULL_SORT},
{"http-port", required_argument, nullptr, OPT_HTTP_PORT},
+ {"metric-extension", required_argument, nullptr, OPT_METRIC_EXTENSION},
{nullptr, 0, nullptr, 0}};
bool explicit_interactive = false;
@@ -832,6 +875,11 @@
continue;
}
+ if (option == OPT_METRIC_EXTENSION) {
+ command_line_options.raw_metric_extensions.push_back(optarg);
+ continue;
+ }
+
PrintUsage(argv);
exit(option == 'h' ? 0 : 1);
}
@@ -858,16 +906,20 @@
PrintUsage(argv);
exit(1);
}
+
return command_line_options;
}
void ExtendPoolWithBinaryDescriptor(google::protobuf::DescriptorPool& pool,
const void* data,
- int size) {
+ int size,
+ std::vector<std::string>& skip_prefixes) {
google::protobuf::FileDescriptorSet desc_set;
desc_set.ParseFromArray(data, size);
- for (const auto& desc : desc_set.file()) {
- pool.BuildFile(desc);
+ for (const auto& file_desc : desc_set.file()) {
+ if (base::StartsWithAny(file_desc.name(), skip_prefixes))
+ continue;
+ pool.BuildFile(file_desc);
}
}
@@ -941,16 +993,151 @@
return util::OkStatus();
}
-util::Status RunMetrics(const CommandLineOptions& options) {
- // Descriptor pool used for printing output as textproto.
- // Building on top of generated pool so default protos in
- // google.protobuf.descriptor.proto are available.
+base::Status ParseSingleMetricExtensionPath(const std::string& raw_extension,
+ MetricExtension& parsed_extension) {
+ std::vector<std::string> parts = base::SplitString(raw_extension, ":");
+ if (parts.size() != 2 || parts[0].length() == 0 || parts[1].length() == 0) {
+ return base::ErrStatus(
+ "--metric-extension-dir must be of format disk_path:virtual_path");
+ }
+
+ parsed_extension.SetDiskPath(std::move(parts[0]));
+ parsed_extension.SetVirtualPath(std::move(parts[1]));
+
+ if (parsed_extension.virtual_path() == "shell/") {
+ return base::Status(
+ "Cannot have 'shell/' as metric extension virtual path.");
+ }
+ return util::OkStatus();
+}
+
+base::Status CheckForDuplicateMetricExtension(
+ const std::vector<MetricExtension>& metric_extensions) {
+ std::unordered_set<std::string> disk_paths;
+ std::unordered_set<std::string> virtual_paths;
+ for (const auto& extension : metric_extensions) {
+ auto ret = disk_paths.insert(extension.disk_path());
+ if (!ret.second) {
+ return base::ErrStatus(
+ "Another metric extension is already using disk path %s",
+ extension.disk_path().c_str());
+ }
+ ret = virtual_paths.insert(extension.virtual_path());
+ if (!ret.second) {
+ return base::ErrStatus(
+ "Another metric extension is already using virtual path %s",
+ extension.virtual_path().c_str());
+ }
+ }
+ return base::OkStatus();
+}
+
+base::Status ParseMetricExtensionPaths(
+ const std::vector<std::string>& raw_metric_extensions,
+ std::vector<MetricExtension>& metric_extensions) {
+ for (const auto& raw_extension : raw_metric_extensions) {
+ metric_extensions.push_back({});
+ RETURN_IF_ERROR(ParseSingleMetricExtensionPath(raw_extension,
+ metric_extensions.back()));
+ }
+ return CheckForDuplicateMetricExtension(metric_extensions);
+}
+
+base::Status LoadMetricExtensionProtos(const std::string& proto_root,
+ const std::string& mount_path) {
+ if (!base::FileExists(proto_root)) {
+ return base::ErrStatus(
+ "Directory %s does not exist. Metric extension directory must contain "
+ "a 'sql/' and 'protos/' subdirectory.",
+ proto_root.c_str());
+ }
+ std::vector<std::string> proto_files;
+ RETURN_IF_ERROR(base::ListFilesRecursive(proto_root, proto_files));
+
+ google::protobuf::FileDescriptorSet parsed_protos;
+ for (const auto& file_path : proto_files) {
+ if (base::GetFileExtension(file_path) != ".proto")
+ continue;
+ auto* file_desc = parsed_protos.add_file();
+ ParseToFileDescriptorProto(proto_root + file_path, file_desc);
+ file_desc->set_name(mount_path + file_path);
+ }
+
+ std::vector<uint8_t> serialized_filedescset;
+ serialized_filedescset.resize(parsed_protos.ByteSizeLong());
+ parsed_protos.SerializeToArray(
+ serialized_filedescset.data(),
+ static_cast<int>(serialized_filedescset.size()));
+
+ RETURN_IF_ERROR(g_tp->ExtendMetricsProto(serialized_filedescset.data(),
+ serialized_filedescset.size()));
+
+ return base::OkStatus();
+}
+
+base::Status LoadMetricExtensionSql(const std::string& sql_root,
+ const std::string& mount_path) {
+ if (!base::FileExists(sql_root)) {
+ return base::ErrStatus(
+ "Directory %s does not exist. Metric extension directory must contain "
+ "a 'sql/' and 'protos/' subdirectory.",
+ sql_root.c_str());
+ }
+
+ std::vector<std::string> sql_files;
+ RETURN_IF_ERROR(base::ListFilesRecursive(sql_root, sql_files));
+ for (const auto& file_path : sql_files) {
+ if (base::GetFileExtension(file_path) != ".sql")
+ continue;
+ std::string file_contents;
+ if (!base::ReadFile(sql_root + file_path, &file_contents)) {
+ return base::ErrStatus("Cannot read file %s", file_path.c_str());
+ }
+ RETURN_IF_ERROR(
+ g_tp->RegisterMetric(mount_path + file_path, file_contents));
+ }
+
+ return base::OkStatus();
+}
+
+base::Status LoadMetricExtension(const MetricExtension& extension) {
+ const std::string& disk_path = extension.disk_path();
+ const std::string& virtual_path = extension.virtual_path();
+
+ if (!base::FileExists(disk_path)) {
+ return base::ErrStatus("Metric extension directory %s does not exist",
+ disk_path.c_str());
+ }
+
+ // Note: Proto files must be loaded first, because we determine whether an SQL
+ // file is a metric or not by checking if the name matches a field of the root
+ // TraceMetrics proto.
+ RETURN_IF_ERROR(LoadMetricExtensionProtos(disk_path + "protos/",
+ kMetricProtoRoot + virtual_path));
+ RETURN_IF_ERROR(LoadMetricExtensionSql(disk_path + "sql/", virtual_path));
+
+ return base::OkStatus();
+}
+
+util::Status RunMetrics(const CommandLineOptions& options,
+ std::vector<MetricExtension>& metric_extensions) {
+ // Descriptor pool used for printing output as textproto. Building on top of
+ // generated pool so default protos in google.protobuf.descriptor.proto are
+ // available.
google::protobuf::DescriptorPool pool(
google::protobuf::DescriptorPool::generated_pool());
+ // TODO(b/182165266): There is code duplication here with trace_processor_impl
+ // SetupMetrics. This will be removed when we switch the output formatter to
+ // use internal DescriptorPool.
+ std::vector<std::string> skip_prefixes;
+ for (const auto& ext : metric_extensions) {
+ skip_prefixes.push_back(kMetricProtoRoot + ext.virtual_path());
+ }
ExtendPoolWithBinaryDescriptor(pool, kMetricsDescriptor.data(),
- kMetricsDescriptor.size());
+ kMetricsDescriptor.size(), skip_prefixes);
ExtendPoolWithBinaryDescriptor(pool, kAllChromeMetricsDescriptor.data(),
- kAllChromeMetricsDescriptor.size());
+ kAllChromeMetricsDescriptor.size(),
+ skip_prefixes);
std::vector<std::string> metrics;
for (base::StringSplitter ss(options.metric_names, ','); ss.Next();) {
@@ -995,6 +1182,7 @@
} else {
format = OutputFormat::kTextProto;
}
+
return RunMetrics(std::move(metrics), format, pool);
}
@@ -1058,6 +1246,14 @@
? SortingMode::kForceFullSort
: SortingMode::kDefaultHeuristics;
+ std::vector<MetricExtension> metric_extensions;
+ RETURN_IF_ERROR(ParseMetricExtensionPaths(options.raw_metric_extensions,
+ metric_extensions));
+
+ for (const auto& extension : metric_extensions) {
+ config.skip_builtin_metric_paths.push_back(extension.virtual_path());
+ }
+
std::unique_ptr<TraceProcessor> tp = TraceProcessor::CreateInstance(config);
g_tp = tp.get();
@@ -1066,6 +1262,13 @@
tp->EnableMetatrace();
}
+ // We load all the metric extensions even when --run-metrics arg is not there,
+ // because we want the metrics to be available in interactive mode or when
+ // used in UI using httpd.
+ for (const auto& extension : metric_extensions) {
+ RETURN_IF_ERROR(LoadMetricExtension(extension));
+ }
+
base::TimeNanos t_load{};
if (!options.trace_file_path.empty()) {
base::TimeNanos t_load_start = base::GetWallTimeNs();
@@ -1097,7 +1300,7 @@
}
if (!options.metric_names.empty()) {
- RETURN_IF_ERROR(RunMetrics(options));
+ RETURN_IF_ERROR(RunMetrics(options, metric_extensions));
}
if (!options.query_file_path.empty()) {
diff --git a/src/trace_processor/util/descriptors.cc b/src/trace_processor/util/descriptors.cc
index 068da6f..4990b38 100644
--- a/src/trace_processor/util/descriptors.cc
+++ b/src/trace_processor/util/descriptors.cc
@@ -15,6 +15,8 @@
*/
#include "src/trace_processor/util/descriptors.h"
+
+#include "perfetto/ext/base/string_utils.h"
#include "perfetto/ext/base/string_view.h"
#include "perfetto/protozero/field.h"
#include "perfetto/protozero/scattered_heap_buffer.h"
@@ -208,15 +210,16 @@
util::Status DescriptorPool::AddFromFileDescriptorSet(
const uint8_t* file_descriptor_set_proto,
size_t size,
+ const std::vector<std::string>& skip_prefixes,
bool merge_existing_messages) {
- // First pass: extract all the message descriptors from the file and add them
- // to the pool.
protos::pbzero::FileDescriptorSet::Decoder proto(file_descriptor_set_proto,
size);
std::vector<ExtensionInfo> extensions;
for (auto it = proto.file(); it; ++it) {
protos::pbzero::FileDescriptorProto::Decoder file(*it);
- std::string file_name = file.name().ToStdString();
+ const std::string file_name = file.name().ToStdString();
+ if (base::StartsWithAny(file_name, skip_prefixes))
+ continue;
if (processed_files_.find(file_name) != processed_files_.end()) {
// This file has been loaded once already. Skip.
continue;
diff --git a/src/trace_processor/util/descriptors.h b/src/trace_processor/util/descriptors.h
index a5c7cde..e4da7a0 100644
--- a/src/trace_processor/util/descriptors.h
+++ b/src/trace_processor/util/descriptors.h
@@ -158,9 +158,12 @@
class DescriptorPool {
public:
+ // Adds Descriptors from file_descriptor_set_proto. Ignores any FileDescriptor
+ // with name matching a prefix in |skip_prefixes|.
base::Status AddFromFileDescriptorSet(
const uint8_t* file_descriptor_set_proto,
size_t size,
+ const std::vector<std::string>& skip_prefixes = {},
bool merge_existing_messages = false);
base::Optional<uint32_t> FindDescriptorIdx(
