| /* | |
| * Copyright (C) 2022 The Android Open Source Project | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #include "src/trace_processor/trace_summary/summary.h" | |
| #include <algorithm> | |
| #include <cstddef> | |
| #include <cstdint> | |
| #include <memory> | |
| #include <optional> | |
| #include <set> | |
| #include <string> | |
| #include <string_view> | |
| #include <tuple> | |
| #include <unordered_set> | |
| #include <utility> | |
| #include <vector> | |
| #include "perfetto/base/build_config.h" | |
| #include "perfetto/base/logging.h" | |
| #include "perfetto/base/status.h" | |
| #include "perfetto/ext/base/flat_hash_map.h" | |
| #include "perfetto/ext/base/murmur_hash.h" | |
| #include "perfetto/ext/base/status_macros.h" | |
| #include "perfetto/ext/base/status_or.h" | |
| #include "perfetto/ext/base/string_utils.h" | |
| #include "perfetto/protozero/field.h" | |
| #include "perfetto/protozero/scattered_heap_buffer.h" | |
| #include "perfetto/trace_processor/basic_types.h" | |
| #include "perfetto/trace_processor/iterator.h" | |
| #include "perfetto/trace_processor/trace_processor.h" | |
| #include "src/protozero/text_to_proto/text_to_proto.h" | |
| #include "src/trace_processor/perfetto_sql/generator/structured_query_generator.h" | |
| #include "src/trace_processor/trace_summary/trace_summary.descriptor.h" | |
| #include "src/trace_processor/util/descriptors.h" | |
| #include "src/trace_processor/util/protozero_to_text.h" | |
| #include "protos/perfetto/trace_summary/file.pbzero.h" | |
| #include "protos/perfetto/trace_summary/v2_metric.pbzero.h" | |
| #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) | |
| #include <zlib.h> | |
| #endif | |
| namespace perfetto::trace_processor::summary { | |
| namespace { | |
| using perfetto_sql::generator::StructuredQueryGenerator; | |
| using TraceMetricV2Spec = protos::pbzero::TraceMetricV2Spec; | |
| using TraceMetricV2TemplateSpec = protos::pbzero::TraceMetricV2TemplateSpec; | |
| using TraceMetricV2Bundle = protos::pbzero::TraceMetricV2Bundle; | |
| using TraceSummarySpec = protos::pbzero::TraceSummarySpec; | |
| using TraceSummary = protos::pbzero::TraceSummary; | |
| using PerfettoSqlStructuredQuery = protos::pbzero::PerfettoSqlStructuredQuery; | |
| using InternedDimensionSpec = TraceMetricV2Spec::InternedDimensionSpec; | |
| base::StatusOr<uint64_t> HashOf(const SqlValue& val) { | |
| base::MurmurHashCombiner hasher; | |
| hasher.Combine(val.type); | |
| if (val.is_null()) { | |
| } else if (val.type == SqlValue::kLong) { | |
| hasher.Combine(val.long_value); | |
| } else if (val.type == SqlValue::kDouble) { | |
| hasher.Combine(val.double_value); | |
| } else if (val.type == SqlValue::kString) { | |
| hasher.Combine(std::string_view(val.string_value)); | |
| } else { | |
| return base::ErrStatus("Unsupported SqlValue type %d for hashing", | |
| static_cast<int>(val.type)); | |
| } | |
| return hasher.digest(); | |
| } | |
| struct Metric { | |
| std::string id; | |
| std::string query; | |
| protozero::ConstBytes spec; | |
| std::vector<std::string> interned_dimension_queries; | |
| }; | |
| base::Status ExpandMetricTemplates( | |
| const std::vector<TraceSummarySpec::Decoder>& spec_decoders, | |
| std::vector<std::vector<uint8_t>>& synthetic_protos) { | |
| protozero::HeapBuffered<TraceMetricV2Spec> expanded; | |
| for (const auto& spec : spec_decoders) { | |
| for (auto it = spec.metric_template_spec(); it; ++it) { | |
| TraceMetricV2TemplateSpec::Decoder tmpl(*it); | |
| std::string id_prefix = tmpl.id_prefix().ToStdString(); | |
| if (id_prefix.empty()) { | |
| return base::ErrStatus( | |
| "Metric template with empty id_prefix field: this is not allowed"); | |
| } | |
| if (tmpl.has_value_columns() && tmpl.has_value_column_specs()) { | |
| return base::ErrStatus( | |
| "Metric template has both value_columns and value_column_specs " | |
| "defined: this is not allowed"); | |
| } | |
| struct ValueColumnInfo { | |
| std::string name; | |
| std::optional<TraceMetricV2Spec::MetricUnit> unit; | |
| std::string custom_unit; | |
| std::optional<TraceMetricV2Spec::MetricPolarity> polarity; | |
| }; | |
| std::vector<ValueColumnInfo> value_column_infos; | |
| if (tmpl.has_value_columns()) { | |
| for (auto vc_it = tmpl.value_columns(); vc_it; ++vc_it) { | |
| value_column_infos.push_back( | |
| {vc_it->as_std_string(), std::nullopt, "", std::nullopt}); | |
| } | |
| } else { | |
| for (auto vcs_it = tmpl.value_column_specs(); vcs_it; ++vcs_it) { | |
| TraceMetricV2TemplateSpec::ValueColumnSpec::Decoder value_spec( | |
| *vcs_it); | |
| std::optional<TraceMetricV2Spec::MetricUnit> unit; | |
| if (value_spec.has_unit()) { | |
| unit = | |
| static_cast<TraceMetricV2Spec::MetricUnit>(value_spec.unit()); | |
| } | |
| std::optional<TraceMetricV2Spec::MetricPolarity> polarity; | |
| if (value_spec.has_polarity()) { | |
| polarity = static_cast<TraceMetricV2Spec::MetricPolarity>( | |
| value_spec.polarity()); | |
| } | |
| value_column_infos.push_back({value_spec.name().ToStdString(), unit, | |
| value_spec.custom_unit().ToStdString(), | |
| polarity}); | |
| } | |
| } | |
| for (const auto& info : value_column_infos) { | |
| expanded.Reset(); | |
| expanded->set_id(id_prefix + "_" + info.name); | |
| expanded->set_value(info.name); | |
| if (info.unit) { | |
| expanded->set_unit(*info.unit); | |
| } | |
| if (!info.custom_unit.empty()) { | |
| expanded->set_custom_unit(info.custom_unit); | |
| } | |
| if (info.polarity) { | |
| expanded->set_polarity(*info.polarity); | |
| } | |
| for (auto dim = tmpl.dimensions(); dim; ++dim) { | |
| protozero::ConstChars dim_str = *dim; | |
| expanded->add_dimensions(dim_str.data, dim_str.size); | |
| } | |
| for (auto dim = tmpl.dimensions_specs(); dim; ++dim) { | |
| protozero::ConstBytes dim_spec = *dim; | |
| expanded->add_dimensions_specs()->AppendRawProtoBytes(dim_spec.data, | |
| dim_spec.size); | |
| } | |
| if (tmpl.has_query()) { | |
| expanded->set_query()->AppendRawProtoBytes(tmpl.query().data, | |
| tmpl.query().size); | |
| } | |
| if (!tmpl.disable_auto_bundling()) { | |
| expanded->set_bundle_id(id_prefix); | |
| } | |
| if (tmpl.has_dimension_uniqueness()) { | |
| expanded->set_dimension_uniqueness( | |
| static_cast<TraceMetricV2Spec::DimensionUniqueness>( | |
| tmpl.dimension_uniqueness())); | |
| } | |
| for (auto ms_it = tmpl.interned_dimension_specs(); ms_it; ++ms_it) { | |
| expanded->add_interned_dimension_specs()->AppendRawProtoBytes( | |
| ms_it->data(), ms_it->size()); | |
| } | |
| synthetic_protos.push_back(expanded.SerializeAsArray()); | |
| } | |
| } | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status WriteMetadata(TraceProcessor* processor, | |
| const std::string& metadata_sql, | |
| TraceSummary* summary) { | |
| auto it = processor->ExecuteQuery(metadata_sql); | |
| RETURN_IF_ERROR(it.Status()); | |
| uint32_t col_count = it.ColumnCount(); | |
| // This can happen if there is no metadata. Just early return in that case. | |
| if (col_count == 0) { | |
| return base::OkStatus(); | |
| } | |
| // Otherwise we expect a strict schema of (key, value). | |
| if (col_count != 2 || it.GetColumnName(0) != "key" || | |
| it.GetColumnName(1) != "value") { | |
| return base::ErrStatus( | |
| "Metadata query did not match schema of (key, value)"); | |
| } | |
| while (it.Next()) { | |
| auto key = it.Get(0); | |
| if (key.type != SqlValue::kString) { | |
| return base::ErrStatus( | |
| "Key column in metadata query was not of type string"); | |
| } | |
| // Silently ignore any null values. | |
| auto value = it.Get(1); | |
| if (value.is_null()) { | |
| continue; | |
| } | |
| if (value.type != SqlValue::kString) { | |
| return base::ErrStatus( | |
| "Key column in metadata query was not of type string or null"); | |
| } | |
| auto* metadata = summary->add_metadata(); | |
| metadata->set_key(key.AsString()); | |
| metadata->set_value(value.AsString()); | |
| } | |
| return it.Status(); | |
| } | |
| struct Dimension { | |
| std::string name; | |
| TraceMetricV2Spec::DimensionType type; | |
| [[maybe_unused]] bool operator==(const Dimension& other) const { | |
| return std::tie(name, type) == std::tie(other.name, other.type); | |
| } | |
| [[maybe_unused]] bool operator!=(const Dimension& other) const { | |
| return !(*this == other); | |
| } | |
| }; | |
| base::StatusOr<std::vector<Dimension>> GetDimensions( | |
| const TraceMetricV2Spec::Decoder& spec_decoder) { | |
| if (spec_decoder.has_dimensions_specs() && spec_decoder.has_dimensions()) { | |
| return base::ErrStatus( | |
| "Both dimensions and dimension_specs defined for metric '%s'. Only " | |
| "one is allowed", | |
| spec_decoder.id().ToStdString().c_str()); | |
| } | |
| std::vector<Dimension> dimensions; | |
| if (spec_decoder.dimensions_specs()) { | |
| for (auto dim = spec_decoder.dimensions_specs(); dim; ++dim) { | |
| TraceMetricV2Spec::DimensionSpec::Decoder dim_spec(*dim); | |
| std::string_view dim_name = dim_spec.name().ToStdStringView(); | |
| auto dimension_type = | |
| static_cast<TraceMetricV2Spec::DimensionType>(dim_spec.type()); | |
| if (dimension_type == TraceMetricV2Spec::DIMENSION_TYPE_UNSPECIFIED) { | |
| return base::ErrStatus( | |
| "Dimension '%.*s' in metric '%s' has unspecified type", | |
| int(dim_name.size()), dim_name.data(), | |
| spec_decoder.id().ToStdString().c_str()); | |
| } | |
| dimensions.push_back({dim_spec.name().ToStdString(), dimension_type}); | |
| } | |
| } else { | |
| for (auto dim = spec_decoder.dimensions(); dim; ++dim) { | |
| dimensions.push_back({dim->as_std_string(), | |
| TraceMetricV2Spec::DIMENSION_TYPE_UNSPECIFIED}); | |
| } | |
| } | |
| return dimensions; | |
| } | |
| struct DimensionWithIndex : Dimension { | |
| uint32_t index; | |
| }; | |
| base::StatusOr<std::vector<DimensionWithIndex>> GetDimensionsWithIndex( | |
| const TraceMetricV2Spec::Decoder& spec_decoder, | |
| perfetto::trace_processor::Iterator& it) { | |
| ASSIGN_OR_RETURN(auto dimensions, GetDimensions(spec_decoder)); | |
| std::vector<DimensionWithIndex> output; | |
| uint32_t col_count = it.ColumnCount(); | |
| for (const auto& dim : dimensions) { | |
| std::optional<uint32_t> dim_index; | |
| for (uint32_t i = 0; i < col_count; ++i) { | |
| if (it.GetColumnName(i) == dim.name) { | |
| dim_index = i; | |
| break; | |
| } | |
| } | |
| if (!dim_index) { | |
| return base::ErrStatus( | |
| "Dimensions column '%s' not found in the query result for metric " | |
| "'%s'", | |
| dim.name.c_str(), spec_decoder.id().ToStdString().c_str()); | |
| } | |
| output.push_back({dim, *dim_index}); | |
| } | |
| return output; | |
| } | |
| base::Status WriteDimension( | |
| const DimensionWithIndex& dim_with_index, | |
| const std::string& metric_or_bundle_name, | |
| Iterator& query_it, | |
| protos::pbzero::TraceMetricV2Bundle::Row::Dimension* dimension, | |
| base::MurmurHashCombiner* hasher) { | |
| const auto& dimension_value = query_it.Get(dim_with_index.index); | |
| hasher->Combine(dimension_value.type); | |
| if (dimension_value.is_null()) { | |
| // Accept null value for all dimension types. | |
| dimension->set_null_value(); | |
| return base::OkStatus(); | |
| } | |
| switch (dim_with_index.type) { | |
| case TraceMetricV2Spec::STRING: { | |
| if (dimension_value.type != SqlValue::kString) { | |
| return base::ErrStatus( | |
| "Expected string for dimension '%s' in metric or bundle '%s', got " | |
| "%d", | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str(), | |
| dimension_value.type); | |
| } | |
| const char* dimension_str = dimension_value.string_value; | |
| hasher->Combine(base::StringView(dimension_str)); | |
| dimension->set_string_value(dimension_str); | |
| break; | |
| } | |
| case TraceMetricV2Spec::INT64: { | |
| if (dimension_value.type != SqlValue::kLong) { | |
| return base::ErrStatus( | |
| "Expected int64 for dimension '%s' in metric or bundle '%s', got " | |
| "%d", | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str(), | |
| dimension_value.type); | |
| } | |
| int64_t dim_value = dimension_value.long_value; | |
| hasher->Combine(dim_value); | |
| dimension->set_int64_value(dim_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DOUBLE: { | |
| if (dimension_value.type != SqlValue::kDouble) { | |
| return base::ErrStatus( | |
| "Expected double for dimension '%s' in metric or bundle '%s', got " | |
| "%d", | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str(), | |
| dimension_value.type); | |
| } | |
| double dim_value = dimension_value.AsDouble(); | |
| hasher->Combine(dim_value); | |
| dimension->set_double_value(dim_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::BOOLEAN: { | |
| if (dimension_value.type != SqlValue::kLong) { | |
| return base::ErrStatus( | |
| "Expected bool for dimension '%s' in metric or bundle '%s', got " | |
| "%d", | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str(), | |
| dimension_value.type); | |
| } | |
| if (dimension_value.long_value != 0 && dimension_value.long_value != 1) { | |
| return base::ErrStatus( | |
| "Expected bool 0 or 1 for dimension '%s' in metric or bundle '%s', " | |
| "got %" PRId64, | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str(), | |
| dimension_value.long_value); | |
| } | |
| bool dim_value = dimension_value.long_value != 0; | |
| hasher->Combine(dim_value); | |
| dimension->set_bool_value(dim_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DIMENSION_TYPE_UNSPECIFIED: | |
| if (dimension_value.type == SqlValue::kLong) { | |
| int64_t dim_value = dimension_value.long_value; | |
| hasher->Combine(dim_value); | |
| dimension->set_int64_value(dim_value); | |
| } else if (dimension_value.type == SqlValue::kDouble) { | |
| double dim_value = dimension_value.AsDouble(); | |
| hasher->Combine(dim_value); | |
| dimension->set_double_value(dim_value); | |
| } else if (dimension_value.type == SqlValue::kString) { | |
| const char* dimension_str = dimension_value.string_value; | |
| hasher->Combine(base::StringView(dimension_str)); | |
| dimension->set_string_value(dimension_str); | |
| } else if (dimension_value.type == SqlValue::kBytes) { | |
| return base::ErrStatus( | |
| "Received bytes for dimension '%s' in metric or bundle '%s': this " | |
| "is not supported", | |
| dim_with_index.name.c_str(), metric_or_bundle_name.c_str()); | |
| } else { | |
| PERFETTO_FATAL("Null dimension should have been handled above"); | |
| } | |
| break; | |
| } | |
| return base::OkStatus(); | |
| } | |
| struct InternedDimensionKey { | |
| std::string key_column_name; | |
| uint64_t key_hash; | |
| bool operator==(const InternedDimensionKey& o) const { | |
| return key_column_name == o.key_column_name && key_hash == o.key_hash; | |
| } | |
| struct Hasher { | |
| size_t operator()(const InternedDimensionKey& k) const { | |
| return base::MurmurHashCombine(k.key_column_name, k.key_hash); | |
| } | |
| }; | |
| }; | |
| base::Status WriteInternedDimensionValue( | |
| const SqlValue& col_value, | |
| TraceMetricV2Spec::DimensionType type, | |
| TraceMetricV2Bundle::Row::Dimension* value) { | |
| if (col_value.is_null()) { | |
| value->set_null_value(); | |
| return base::OkStatus(); | |
| } | |
| switch (type) { | |
| case TraceMetricV2Spec::DimensionType::STRING: { | |
| if (col_value.type != SqlValue::kString) { | |
| return base::ErrStatus("Expected string for interned dimension column"); | |
| } | |
| value->set_string_value(col_value.string_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DimensionType::INT64: { | |
| if (col_value.type != SqlValue::kLong) { | |
| return base::ErrStatus("Expected int64 for interned dimension column"); | |
| } | |
| value->set_int64_value(col_value.long_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DimensionType::DOUBLE: { | |
| if (col_value.type != SqlValue::kDouble) { | |
| return base::ErrStatus("Expected double for interned dimension column"); | |
| } | |
| value->set_double_value(col_value.double_value); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DimensionType::BOOLEAN: { | |
| if (col_value.type != SqlValue::kLong) { | |
| return base::ErrStatus("Expected bool for interned dimension column"); | |
| } | |
| if (col_value.long_value != 0 && col_value.long_value != 1) { | |
| return base::ErrStatus( | |
| "Expected bool 0 or 1 for interned dimension column, got %" PRId64, | |
| col_value.long_value); | |
| } | |
| value->set_bool_value(col_value.long_value != 0); | |
| break; | |
| } | |
| case TraceMetricV2Spec::DimensionType::DIMENSION_TYPE_UNSPECIFIED: | |
| PERFETTO_FATAL("Unspecified type for interned dimension column"); | |
| break; | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status VerifyBundleHasConsistentSpecs( | |
| const std::string& bundle_id, | |
| const std::vector<const Metric*>& metrics) { | |
| if (metrics.empty()) { | |
| return base::ErrStatus("Empty metric bundle %s: this is not allowed", | |
| bundle_id.c_str()); | |
| } | |
| if (metrics.size() == 1) { | |
| return base::OkStatus(); | |
| } | |
| const Metric* first = metrics.front(); | |
| TraceMetricV2Spec::Decoder first_spec(first->spec); | |
| ASSIGN_OR_RETURN(auto first_dims, GetDimensions(first_spec)); | |
| for (const Metric* metric : metrics) { | |
| TraceMetricV2Spec::Decoder spec(metric->spec); | |
| if (spec.bundle_id().ToStdStringView() != | |
| first_spec.bundle_id().ToStdStringView()) { | |
| return base::ErrStatus( | |
| "Metric '%s' in bundle '%s' has different bundle_id than the " | |
| "first metric '%s': this is not allowed", | |
| metric->id.c_str(), bundle_id.c_str(), first->id.c_str()); | |
| } | |
| if (spec.dimension_uniqueness() != first_spec.dimension_uniqueness()) { | |
| return base::ErrStatus( | |
| "Metric '%s' in bundle '%s' has different dimension_uniqueness than " | |
| "the first metric '%s': this is not allowed", | |
| metric->id.c_str(), bundle_id.c_str(), first->id.c_str()); | |
| } | |
| ASSIGN_OR_RETURN(auto dims, GetDimensions(spec)); | |
| if (dims != first_dims) { | |
| return base::ErrStatus( | |
| "Metric '%s' in bundle '%s' has different dimensions than the first " | |
| "metric '%s': this is not allowed", | |
| metric->id.c_str(), bundle_id.c_str(), first->id.c_str()); | |
| } | |
| if (first->query != metric->query) { | |
| return base::ErrStatus( | |
| "Metric '%s' in bundle '%s' has different query than the first " | |
| "metric '%s': this is not allowed", | |
| metric->id.c_str(), bundle_id.c_str(), first->id.c_str()); | |
| } | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status ValidateInternedDimensionSpecs( | |
| const TraceMetricV2Spec::Decoder& spec) { | |
| ASSIGN_OR_RETURN(auto dimensions, GetDimensions(spec)); | |
| std::set<std::string> interned_dimension_keys; | |
| for (auto it = spec.interned_dimension_specs(); it; ++it) { | |
| InternedDimensionSpec::Decoder ms(*it); | |
| if (!ms.has_key_column_spec()) { | |
| return base::ErrStatus( | |
| "key_column_spec must be specified in interned_dimension_specs"); | |
| } | |
| InternedDimensionSpec::ColumnSpec::Decoder key_col_spec( | |
| ms.key_column_spec()); | |
| base::StringView key_column_name = key_col_spec.name(); | |
| if (!interned_dimension_keys.insert(key_column_name.ToStdString()).second) { | |
| return base::ErrStatus( | |
| "Duplicate key column '%.*s' found in interned_dimension_specs", | |
| static_cast<int>(key_column_name.size()), key_column_name.data()); | |
| } | |
| bool key_in_dimensions = false; | |
| for (const auto& dim : dimensions) { | |
| if (base::StringView(dim.name) == key_column_name) { | |
| key_in_dimensions = true; | |
| break; | |
| } | |
| } | |
| if (!key_in_dimensions) { | |
| return base::ErrStatus( | |
| "Key column '%.*s' in interned dimension bundle not found in metric " | |
| "dimensions", | |
| static_cast<int>(key_column_name.size()), key_column_name.data()); | |
| } | |
| std::set<std::string> all_column_names; | |
| all_column_names.insert(key_column_name.ToStdString()); | |
| for (auto dcs_it = ms.data_column_specs(); dcs_it; ++dcs_it) { | |
| InternedDimensionSpec::ColumnSpec::Decoder data_col_spec(*dcs_it); | |
| base::StringView data_col_name = data_col_spec.name(); | |
| if (!all_column_names.insert(data_col_name.ToStdString()).second) { | |
| return base::ErrStatus( | |
| "Duplicate column name '%.*s' found in interned dimension bundle", | |
| static_cast<int>(data_col_name.size()), data_col_name.data()); | |
| } | |
| } | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status CheckForDuplicateAndInsert( | |
| uint64_t hash, | |
| base::StringView key_column_name, | |
| std::unordered_set<uint64_t>& seen_keys) { | |
| if (!seen_keys.insert(hash).second) { | |
| return base::ErrStatus( | |
| "Duplicate key found in interned dimension bundle with key column " | |
| "'%.*s'", | |
| static_cast<int>(key_column_name.size()), key_column_name.data()); | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status WriteInternedDimensionBundles( | |
| TraceProcessor* processor, | |
| const TraceMetricV2Spec::Decoder& spec, | |
| const std::vector<std::string>& interned_dimension_queries, | |
| const std::unordered_set<InternedDimensionKey, | |
| InternedDimensionKey::Hasher>& | |
| interned_dim_keys_in_metric_bundle, | |
| TraceMetricV2Bundle* bundle) { | |
| RETURN_IF_ERROR(ValidateInternedDimensionSpecs(spec)); | |
| size_t interned_dimension_idx = 0; | |
| for (auto it = spec.interned_dimension_specs(); it; ++it) { | |
| InternedDimensionSpec::Decoder ms(*it); | |
| InternedDimensionSpec::ColumnSpec::Decoder key_col_spec( | |
| ms.key_column_spec()); | |
| base::StringView key_column_name = key_col_spec.name(); | |
| auto* interned_dimension_bundle = bundle->add_interned_dimension_bundles(); | |
| const std::string& sql = | |
| interned_dimension_queries[interned_dimension_idx++]; | |
| auto query_it = processor->ExecuteQuery(sql); | |
| if (!query_it.Status().ok()) { | |
| return base::ErrStatus( | |
| "Error while executing query for interned dimension bundle with key " | |
| "'%.*s': %s", | |
| static_cast<int>(key_column_name.size()), key_column_name.data(), | |
| query_it.Status().c_message()); | |
| } | |
| std::vector<protozero::ConstBytes> col_specs_bytes; | |
| col_specs_bytes.push_back(ms.key_column_spec()); | |
| for (auto dcs_it = ms.data_column_specs(); dcs_it; ++dcs_it) { | |
| col_specs_bytes.push_back(*dcs_it); | |
| } | |
| std::vector<std::pair<uint32_t, TraceMetricV2Spec::DimensionType>> | |
| column_infos; | |
| for (const auto& col_spec_bytes : col_specs_bytes) { | |
| InternedDimensionSpec::ColumnSpec::Decoder col_spec(col_spec_bytes); | |
| std::optional<uint32_t> column_index; | |
| for (uint32_t i = 0; i < query_it.ColumnCount(); ++i) { | |
| if (base::StringView(query_it.GetColumnName(i)) == col_spec.name()) { | |
| column_index = i; | |
| break; | |
| } | |
| } | |
| if (!column_index) { | |
| return base::ErrStatus( | |
| "Column '%.*s' not found in the query result for interned " | |
| "dimension " | |
| "bundle " | |
| "with key '%.*s'", | |
| static_cast<int>(col_spec.name().size), col_spec.name().data, | |
| static_cast<int>(key_column_name.size()), key_column_name.data()); | |
| } | |
| column_infos.emplace_back( | |
| *column_index, | |
| static_cast<TraceMetricV2Spec::DimensionType>(col_spec.type())); | |
| } | |
| std::unordered_set<uint64_t> seen_keys; | |
| while (query_it.Next()) { | |
| const auto& key_val = query_it.Get(column_infos[0].first); | |
| ASSIGN_OR_RETURN(uint64_t key_hash, HashOf(key_val)); | |
| // If the key was not in the metric bundle, we don't need to output | |
| // its interned data. | |
| if (!interned_dim_keys_in_metric_bundle.count( | |
| {key_column_name.ToStdString(), key_hash})) { | |
| continue; | |
| } | |
| RETURN_IF_ERROR( | |
| CheckForDuplicateAndInsert(key_hash, key_column_name, seen_keys)); | |
| auto* row = interned_dimension_bundle->add_interned_dimension_rows(); | |
| RETURN_IF_ERROR(WriteInternedDimensionValue( | |
| query_it.Get(column_infos[0].first), column_infos[0].second, | |
| row->set_key_dimension_value())); | |
| for (size_t i = 1; i < column_infos.size(); ++i) { | |
| const auto& info = column_infos[i]; | |
| const auto& col_value = query_it.Get(info.first); | |
| RETURN_IF_ERROR(WriteInternedDimensionValue( | |
| col_value, info.second, row->add_interned_dimension_values())); | |
| } | |
| } | |
| RETURN_IF_ERROR(query_it.Status()); | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status CreateQueriesAndComputeMetrics(TraceProcessor* processor, | |
| const std::vector<Metric>& metrics, | |
| TraceSummary* summary) { | |
| base::FlatHashMap<std::string, std::vector<const Metric*>> metrics_by_bundle; | |
| for (const Metric& m : metrics) { | |
| TraceMetricV2Spec::Decoder spec_decoder(m.spec); | |
| std::string bundle_id = spec_decoder.bundle_id().ToStdString(); | |
| if (bundle_id.empty()) { | |
| bundle_id = m.id; | |
| } | |
| metrics_by_bundle[bundle_id].push_back(&m); | |
| } | |
| for (auto it = metrics_by_bundle.GetIterator(); it; ++it) { | |
| const auto& value = it.value(); | |
| RETURN_IF_ERROR(VerifyBundleHasConsistentSpecs(it.key(), value)); | |
| const std::string& bundle_id = it.key(); | |
| auto* bundle = summary->add_metric_bundles(); | |
| bundle->set_bundle_id(bundle_id); | |
| for (const Metric* metric : value) { | |
| bundle->add_specs()->AppendRawProtoBytes(metric->spec.data, | |
| metric->spec.size); | |
| } | |
| const Metric* first = value.front(); | |
| TraceMetricV2Spec::Decoder first_spec(first->spec); | |
| std::unordered_set<std::string> interned_dim_key_cols; | |
| for (auto ms_it = first_spec.interned_dimension_specs(); ms_it; ++ms_it) { | |
| InternedDimensionSpec::Decoder ms_decoder(*ms_it); | |
| interned_dim_key_cols.insert(InternedDimensionSpec::ColumnSpec::Decoder( | |
| ms_decoder.key_column_spec()) | |
| .name() | |
| .ToStdString()); | |
| } | |
| std::unordered_set<InternedDimensionKey, InternedDimensionKey::Hasher> | |
| interned_dim_keys_in_metric_bundle; | |
| auto query_it = processor->ExecuteQuery(first->query); | |
| if (!query_it.Status().ok()) { | |
| return base::ErrStatus( | |
| "Error while executing query for metric bundle '%s': %s", | |
| bundle_id.c_str(), query_it.Status().c_message()); | |
| } | |
| PerfettoSqlStructuredQuery::Decoder query(first_spec.query()); | |
| // The sql.column_names field documents what columns the SQL query itself | |
| // returns (before structured query transformations). We can only validate | |
| // this when there are no transformations that modify the output schema: | |
| // - group_by transforms columns into group keys + aggregates | |
| // - select_columns transforms columns via selection/aliasing/expressions | |
| // Other operations (filters, order_by, limit, offset) preserve columns. | |
| if (query.has_sql() && !query.has_group_by() && | |
| !query.has_select_columns()) { | |
| PerfettoSqlStructuredQuery::Sql::Decoder sql_query(query.sql()); | |
| if (sql_query.has_column_names()) { | |
| std::set<std::string> actual_column_names; | |
| for (uint32_t i = 0; i < query_it.ColumnCount(); ++i) { | |
| actual_column_names.insert(query_it.GetColumnName(i)); | |
| } | |
| std::set<std::string> expected_column_names; | |
| for (auto col_it = sql_query.column_names(); col_it; ++col_it) { | |
| expected_column_names.insert(col_it->as_std_string()); | |
| } | |
| if (!std::includes( | |
| actual_column_names.begin(), actual_column_names.end(), | |
| expected_column_names.begin(), expected_column_names.end())) { | |
| std::vector<std::string> expected_vec(expected_column_names.begin(), | |
| expected_column_names.end()); | |
| std::vector<std::string> actual_vec(actual_column_names.begin(), | |
| actual_column_names.end()); | |
| return base::ErrStatus( | |
| "Not all columns expected in metrics bundle '%s' were found. " | |
| "Expected: [%s], Actual: [%s]", | |
| bundle_id.c_str(), base::Join(expected_vec, ", ").c_str(), | |
| base::Join(actual_vec, ", ").c_str()); | |
| } | |
| } | |
| } | |
| ASSIGN_OR_RETURN(std::vector<DimensionWithIndex> dimensions_with_index, | |
| GetDimensionsWithIndex(first_spec, query_it)); | |
| std::vector<uint32_t> value_indices; | |
| for (const auto* metric : value) { | |
| TraceMetricV2Spec::Decoder spec(metric->spec); | |
| std::string value_column_name = spec.value().ToStdString(); | |
| std::optional<uint32_t> value_index; | |
| for (uint32_t i = 0; i < query_it.ColumnCount(); ++i) { | |
| if (query_it.GetColumnName(i) == value_column_name) { | |
| value_index = i; | |
| break; | |
| } | |
| } | |
| if (!value_index) { | |
| return base::ErrStatus( | |
| "Column '%s' not found in the query result for metric '%s'", | |
| value_column_name.c_str(), spec.id().ToStdString().c_str()); | |
| } | |
| value_indices.push_back(*value_index); | |
| } | |
| bool is_unique_dimensions = | |
| first_spec.dimension_uniqueness() == TraceMetricV2Spec::UNIQUE; | |
| std::unordered_set<uint64_t> seen_dimensions; | |
| while (query_it.Next()) { | |
| bool all_null = true; | |
| for (size_t i = 0; i < value.size(); ++i) { | |
| const auto& metric_value_column = query_it.Get(value_indices[i]); | |
| if (!metric_value_column.is_null()) { | |
| all_null = false; | |
| break; | |
| } | |
| } | |
| // If all values are null, we skip writing the row. | |
| if (all_null) { | |
| continue; | |
| } | |
| auto* row = bundle->add_row(); | |
| base::MurmurHashCombiner hasher; | |
| for (const auto& dim : dimensions_with_index) { | |
| RETURN_IF_ERROR(WriteDimension(dim, bundle_id, query_it, | |
| row->add_dimension(), &hasher)); | |
| if (interned_dim_key_cols.count(dim.name)) { | |
| const auto& key_val = query_it.Get(dim.index); | |
| ASSIGN_OR_RETURN(uint64_t value_hash, HashOf(key_val)); | |
| interned_dim_keys_in_metric_bundle.insert({dim.name, value_hash}); | |
| } | |
| } | |
| uint64_t hash = hasher.digest(); | |
| if (is_unique_dimensions && !seen_dimensions.insert(hash).second) { | |
| return base::ErrStatus( | |
| "Duplicate dimensions found for metric bundle '%s': this is not " | |
| "allowed", | |
| bundle_id.c_str()); | |
| } | |
| for (size_t i = 0; i < value.size(); ++i) { | |
| const auto& metric_value_column = query_it.Get(value_indices[i]); | |
| auto* row_value = row->add_values(); | |
| if (metric_value_column.is_null()) { | |
| row_value->set_null_value(); | |
| continue; | |
| } | |
| switch (metric_value_column.type) { | |
| case SqlValue::kLong: | |
| row_value->set_double_value( | |
| static_cast<double>(metric_value_column.long_value)); | |
| break; | |
| case SqlValue::kDouble: | |
| row_value->set_double_value(metric_value_column.double_value); | |
| break; | |
| case SqlValue::kNull: | |
| PERFETTO_FATAL("Null value should have been skipped"); | |
| case SqlValue::kString: | |
| case SqlValue::kBytes: | |
| return base::ErrStatus( | |
| "Received string/bytes for value column in metric '%s': this " | |
| "is not supported", | |
| it.value()[i]->id.c_str()); | |
| } | |
| } | |
| } | |
| RETURN_IF_ERROR(query_it.Status()); | |
| if (!first->interned_dimension_queries.empty()) { | |
| RETURN_IF_ERROR(WriteInternedDimensionBundles( | |
| processor, first_spec, first->interned_dimension_queries, | |
| interned_dim_keys_in_metric_bundle, bundle)); | |
| } | |
| } | |
| return base::OkStatus(); | |
| } | |
| base::Status CreateQueriesAndComputeMetrics( | |
| TraceProcessor* processor, | |
| const DescriptorPool& pool, | |
| const std::vector<StructuredQueryGenerator::Query>& queries, | |
| const std::vector<Metric>& metrics, | |
| const std::optional<std::string>& metadata_sql, | |
| std::vector<uint8_t>* output, | |
| const TraceSummaryOutputSpec& output_spec) { | |
| for (const auto& query : queries) { | |
| auto it = processor->ExecuteQuery("CREATE PERFETTO TABLE " + | |
| query.table_name + " AS " + query.sql); | |
| PERFETTO_CHECK(!it.Next()); | |
| if (!it.Status().ok()) { | |
| return base::ErrStatus("Error while executing shared query %s: %s", | |
| query.id.c_str(), it.Status().c_message()); | |
| } | |
| } | |
| protozero::HeapBuffered<TraceSummary> summary; | |
| RETURN_IF_ERROR( | |
| CreateQueriesAndComputeMetrics(processor, metrics, summary.get())); | |
| if (metadata_sql) { | |
| RETURN_IF_ERROR(WriteMetadata(processor, *metadata_sql, summary.get())); | |
| } | |
| switch (output_spec.format) { | |
| case TraceSummaryOutputSpec::Format::kBinaryProto: | |
| *output = summary.SerializeAsArray(); | |
| break; | |
| case TraceSummaryOutputSpec::Format::kTextProto: | |
| std::vector<uint8_t> proto = summary.SerializeAsArray(); | |
| std::string out = protozero_to_text::ProtozeroToText( | |
| pool, ".perfetto.protos.TraceSummary", | |
| protozero::ConstBytes{proto.data(), proto.size()}); | |
| *output = std::vector<uint8_t>(out.begin(), out.end()); | |
| break; | |
| } | |
| // Compress the output if requested. If Zlib compression is requested but | |
| // not supported, return an error. | |
| if (output_spec.compression == TraceSummaryOutputSpec::Compression::kZlib && | |
| !output->empty()) { | |
| #if PERFETTO_BUILDFLAG(PERFETTO_ZLIB) | |
| auto compressed_size = compressBound(static_cast<uint32_t>(output->size())); | |
| auto compressed_buffer = std::make_unique<uint8_t[]>(compressed_size); | |
| int res = compress(compressed_buffer.get(), &compressed_size, | |
| output->data(), static_cast<uint32_t>(output->size())); | |
| if (res != Z_OK) { | |
| return base::ErrStatus("Failed to compress trace summary output"); | |
| } | |
| output->assign( | |
| compressed_buffer.get(), | |
| compressed_buffer.get() + static_cast<uint32_t>(compressed_size)); | |
| #else | |
| return base::ErrStatus( | |
| "Zlib compression requested but is not supported on this platform."); | |
| #endif | |
| } | |
| return base::OkStatus(); | |
| } | |
| } // namespace | |
| base::Status Summarize(TraceProcessor* processor, | |
| const DescriptorPool& pool, | |
| const TraceSummaryComputationSpec& computation, | |
| const std::vector<TraceSummarySpecBytes>& specs, | |
| std::vector<uint8_t>* output, | |
| const TraceSummaryOutputSpec& output_spec) { | |
| std::vector<std::vector<uint8_t>> synthetic_protos; | |
| std::vector<TraceSummarySpec::Decoder> spec_decoders; | |
| for (uint32_t i = 0; i < specs.size(); ++i) { | |
| switch (specs[i].format) { | |
| case TraceSummarySpecBytes::Format::kBinaryProto: | |
| spec_decoders.emplace_back(specs[i].ptr, specs[i].size); | |
| break; | |
| case TraceSummarySpecBytes::Format::kTextProto: | |
| synthetic_protos.emplace_back(); | |
| ASSIGN_OR_RETURN( | |
| synthetic_protos.back(), | |
| protozero::TextToProto( | |
| kTraceSummaryDescriptor.data(), kTraceSummaryDescriptor.size(), | |
| ".perfetto.protos.TraceSummarySpec", "-", | |
| std::string_view(reinterpret_cast<const char*>(specs[i].ptr), | |
| specs[i].size), | |
| /*allow_unknown_fields=*/true)); | |
| spec_decoders.emplace_back(synthetic_protos.back().data(), | |
| synthetic_protos.back().size()); | |
| break; | |
| } | |
| } | |
| perfetto_sql::generator::StructuredQueryGenerator generator; | |
| for (const auto& spec : spec_decoders) { | |
| for (auto it = spec.query(); it; ++it) { | |
| ASSIGN_OR_RETURN(std::ignore, generator.AddQuery(it->data(), it->size())); | |
| } | |
| } | |
| std::vector<TraceMetricV2Spec::Decoder> metric_decoders; | |
| for (const auto& spec : spec_decoders) { | |
| for (auto it = spec.metric_spec(); it; ++it) { | |
| metric_decoders.emplace_back(*it); | |
| } | |
| } | |
| std::vector<std::vector<uint8_t>> expanded_metrics; | |
| RETURN_IF_ERROR(ExpandMetricTemplates(spec_decoders, expanded_metrics)); | |
| for (const auto& expanded : expanded_metrics) { | |
| metric_decoders.emplace_back(expanded.data(), expanded.size()); | |
| } | |
| // If `v2_metric_ids` is an empty vector, we will not compute any metrics. | |
| std::vector<std::string> metric_ids; | |
| if (computation.v2_metric_ids) { | |
| metric_ids = std::move(*computation.v2_metric_ids); | |
| } else { | |
| // If `v2_metric_ids` is not specified, we will compute all metrics | |
| // specified in the summary specs. | |
| for (const auto& spec : metric_decoders) { | |
| metric_ids.push_back(spec.id().ToStdString()); | |
| } | |
| } | |
| base::FlatHashMap<std::string, size_t> queries_per_metric; | |
| std::vector<Metric> metrics; | |
| for (const auto& id : metric_ids) { | |
| if (base::CaseInsensitiveEqual(id, "all")) { | |
| return base::ErrStatus( | |
| "Metric has id 'all' which is not allowed as this is a reserved " | |
| "name. Please use a different id for your metric"); | |
| } | |
| queries_per_metric.Insert(id, metrics.size()); | |
| metrics.emplace_back(Metric{id, {}, {}, {}}); | |
| } | |
| for (const auto& m : metric_decoders) { | |
| std::string id = m.id().ToStdString(); | |
| if (id.empty()) { | |
| return base::ErrStatus("Metric with empty id field: this is not allowed"); | |
| } | |
| // Only compute metrics which were populated in the map (i.e. the ones | |
| // which were specified in the `computation.v2_metric_ids` field). | |
| size_t* idx = queries_per_metric.Find(id); | |
| if (!idx) { | |
| continue; | |
| } | |
| Metric* metric = &metrics[*idx]; | |
| if (!metric->query.empty()) { | |
| return base::ErrStatus( | |
| "Duplicate definitions for metric '%s' received: this is not " | |
| "allowed", | |
| id.c_str()); | |
| } | |
| base::StatusOr<std::string> query_or = | |
| generator.Generate(m.query().data, m.query().size); | |
| if (!query_or.ok()) { | |
| return base::ErrStatus("Unable to build query for metric '%s': %s", | |
| id.c_str(), query_or.status().c_message()); | |
| } | |
| metric->query = *query_or; | |
| metric->spec = protozero::ConstBytes{ | |
| m.begin(), | |
| static_cast<size_t>(m.end() - m.begin()), | |
| }; | |
| for (auto it = m.interned_dimension_specs(); it; ++it) { | |
| InternedDimensionSpec::Decoder ms(*it); | |
| base::StatusOr<std::string> interned_dimension_query_or = | |
| generator.Generate(ms.query().data, ms.query().size); | |
| if (!interned_dimension_query_or.ok()) { | |
| return base::ErrStatus( | |
| "Unable to build interned dimension query for metric '%s': %s", | |
| id.c_str(), interned_dimension_query_or.status().c_message()); | |
| } | |
| metric->interned_dimension_queries.push_back( | |
| *interned_dimension_query_or); | |
| } | |
| } | |
| std::optional<std::string> metadata_sql; | |
| if (computation.metadata_query_id) { | |
| ASSIGN_OR_RETURN(metadata_sql, | |
| generator.GenerateById(*computation.metadata_query_id)); | |
| } | |
| for (const auto& preamble : generator.ComputePreambles()) { | |
| auto it = processor->ExecuteQuery(preamble); | |
| if (it.Next()) { | |
| return base::ErrStatus( | |
| "Preamble query returned results. Preambles must not return. Only " | |
| "the last statement of the `sql` field can return results."); | |
| } | |
| PERFETTO_CHECK(!it.Next()); | |
| RETURN_IF_ERROR(it.Status()); | |
| } | |
| for (const auto& module : generator.ComputeReferencedModules()) { | |
| auto it = processor->ExecuteQuery("INCLUDE PERFETTO MODULE " + module); | |
| PERFETTO_CHECK(!it.Next()); | |
| RETURN_IF_ERROR(it.Status()); | |
| } | |
| auto queries = generator.referenced_queries(); | |
| base::Status status = CreateQueriesAndComputeMetrics( | |
| processor, pool, queries, metrics, metadata_sql, output, output_spec); | |
| // Make sure to cleanup all the queries. | |
| for (const auto& query : queries) { | |
| auto it = | |
| processor->ExecuteQuery("DROP TABLE IF EXISTS " + query.table_name); | |
| PERFETTO_CHECK(!it.Next()); | |
| PERFETTO_CHECK(it.Status().ok()); | |
| } | |
| return status; | |
| } | |
| } // namespace perfetto::trace_processor::summary |