Merge "Add proposed trendy teams for CTS modules" into main
diff --git a/Android.bp b/Android.bp
index 9aa955d..8b07472 100644
--- a/Android.bp
+++ b/Android.bp
@@ -2391,6 +2391,7 @@
":perfetto_src_shared_lib_test_utils",
":perfetto_src_trace_processor_containers_containers",
":perfetto_src_trace_processor_db_column_column",
+ ":perfetto_src_trace_processor_db_compare",
":perfetto_src_trace_processor_db_db",
":perfetto_src_trace_processor_db_minimal",
":perfetto_src_trace_processor_export_json",
@@ -2424,6 +2425,7 @@
":perfetto_src_trace_processor_importers_systrace_full",
":perfetto_src_trace_processor_importers_systrace_systrace_line",
":perfetto_src_trace_processor_importers_systrace_systrace_parser",
+ ":perfetto_src_trace_processor_importers_zip_full",
":perfetto_src_trace_processor_lib",
":perfetto_src_trace_processor_metatrace",
":perfetto_src_trace_processor_metrics_metrics",
@@ -12561,6 +12563,14 @@
],
}
+// GN: //src/trace_processor/importers/zip:full
+filegroup {
+ name: "perfetto_src_trace_processor_importers_zip_full",
+ srcs: [
+ "src/trace_processor/importers/zip/zip_trace_reader.cc",
+ ],
+}
+
// GN: //src/trace_processor:lib
filegroup {
name: "perfetto_src_trace_processor_lib",
@@ -15089,6 +15099,7 @@
":perfetto_src_trace_processor_importers_systrace_systrace_line",
":perfetto_src_trace_processor_importers_systrace_systrace_parser",
":perfetto_src_trace_processor_importers_systrace_unittests",
+ ":perfetto_src_trace_processor_importers_zip_full",
":perfetto_src_trace_processor_lib",
":perfetto_src_trace_processor_metatrace",
":perfetto_src_trace_processor_metrics_metrics",
@@ -16052,6 +16063,7 @@
":perfetto_src_protozero_protozero",
":perfetto_src_trace_processor_containers_containers",
":perfetto_src_trace_processor_db_column_column",
+ ":perfetto_src_trace_processor_db_compare",
":perfetto_src_trace_processor_db_db",
":perfetto_src_trace_processor_db_minimal",
":perfetto_src_trace_processor_export_json",
@@ -16085,6 +16097,7 @@
":perfetto_src_trace_processor_importers_systrace_full",
":perfetto_src_trace_processor_importers_systrace_systrace_line",
":perfetto_src_trace_processor_importers_systrace_systrace_parser",
+ ":perfetto_src_trace_processor_importers_zip_full",
":perfetto_src_trace_processor_lib",
":perfetto_src_trace_processor_metatrace",
":perfetto_src_trace_processor_metrics_metrics",
@@ -16288,6 +16301,7 @@
":perfetto_src_protozero_protozero",
":perfetto_src_trace_processor_containers_containers",
":perfetto_src_trace_processor_db_column_column",
+ ":perfetto_src_trace_processor_db_compare",
":perfetto_src_trace_processor_db_minimal",
":perfetto_src_trace_processor_importers_common_common",
":perfetto_src_trace_processor_importers_common_parser_types",
@@ -16447,6 +16461,7 @@
":perfetto_src_protozero_protozero",
":perfetto_src_trace_processor_containers_containers",
":perfetto_src_trace_processor_db_column_column",
+ ":perfetto_src_trace_processor_db_compare",
":perfetto_src_trace_processor_db_db",
":perfetto_src_trace_processor_db_minimal",
":perfetto_src_trace_processor_export_json",
@@ -16480,6 +16495,7 @@
":perfetto_src_trace_processor_importers_systrace_full",
":perfetto_src_trace_processor_importers_systrace_systrace_line",
":perfetto_src_trace_processor_importers_systrace_systrace_parser",
+ ":perfetto_src_trace_processor_importers_zip_full",
":perfetto_src_trace_processor_lib",
":perfetto_src_trace_processor_metatrace",
":perfetto_src_trace_processor_metrics_metrics",
diff --git a/BUILD b/BUILD
index fa45cad..537ccc4 100644
--- a/BUILD
+++ b/BUILD
@@ -217,6 +217,7 @@
":src_kernel_utils_syscall_table",
":src_protozero_proto_ring_buffer",
":src_trace_processor_db_column_column",
+ ":src_trace_processor_db_compare",
":src_trace_processor_db_db",
":src_trace_processor_db_minimal",
":src_trace_processor_export_json",
@@ -250,6 +251,7 @@
":src_trace_processor_importers_systrace_full",
":src_trace_processor_importers_systrace_systrace_line",
":src_trace_processor_importers_systrace_systrace_parser",
+ ":src_trace_processor_importers_zip_full",
":src_trace_processor_lib",
":src_trace_processor_metatrace",
":src_trace_processor_metrics_metrics",
@@ -1425,6 +1427,14 @@
],
)
+# GN target: //src/trace_processor/db:compare
+perfetto_filegroup(
+ name = "src_trace_processor_db_compare",
+ srcs = [
+ "src/trace_processor/db/compare.h",
+ ],
+)
+
# GN target: //src/trace_processor/db:db
perfetto_filegroup(
name = "src_trace_processor_db_db",
@@ -2004,6 +2014,15 @@
],
)
+# GN target: //src/trace_processor/importers/zip:full
+perfetto_filegroup(
+ name = "src_trace_processor_importers_zip_full",
+ srcs = [
+ "src/trace_processor/importers/zip/zip_trace_reader.cc",
+ "src/trace_processor/importers/zip/zip_trace_reader.h",
+ ],
+)
+
# GN target: //src/trace_processor/metrics/sql/android:android
perfetto_filegroup(
name = "src_trace_processor_metrics_sql_android_android",
@@ -5969,6 +5988,7 @@
srcs = [
":src_kernel_utils_syscall_table",
":src_trace_processor_db_column_column",
+ ":src_trace_processor_db_compare",
":src_trace_processor_db_db",
":src_trace_processor_db_minimal",
":src_trace_processor_export_json",
@@ -6002,6 +6022,7 @@
":src_trace_processor_importers_systrace_full",
":src_trace_processor_importers_systrace_systrace_line",
":src_trace_processor_importers_systrace_systrace_parser",
+ ":src_trace_processor_importers_zip_full",
":src_trace_processor_lib",
":src_trace_processor_metatrace",
":src_trace_processor_metrics_metrics",
@@ -6147,6 +6168,7 @@
":src_profiling_symbolizer_symbolizer",
":src_protozero_proto_ring_buffer",
":src_trace_processor_db_column_column",
+ ":src_trace_processor_db_compare",
":src_trace_processor_db_db",
":src_trace_processor_db_minimal",
":src_trace_processor_export_json",
@@ -6180,6 +6202,7 @@
":src_trace_processor_importers_systrace_full",
":src_trace_processor_importers_systrace_systrace_line",
":src_trace_processor_importers_systrace_systrace_parser",
+ ":src_trace_processor_importers_zip_full",
":src_trace_processor_lib",
":src_trace_processor_metatrace",
":src_trace_processor_metrics_metrics",
@@ -6385,6 +6408,7 @@
":src_profiling_symbolizer_symbolizer",
":src_protozero_proto_ring_buffer",
":src_trace_processor_db_column_column",
+ ":src_trace_processor_db_compare",
":src_trace_processor_db_db",
":src_trace_processor_db_minimal",
":src_trace_processor_export_json",
@@ -6418,6 +6442,7 @@
":src_trace_processor_importers_systrace_full",
":src_trace_processor_importers_systrace_systrace_line",
":src_trace_processor_importers_systrace_systrace_parser",
+ ":src_trace_processor_importers_zip_full",
":src_trace_processor_lib",
":src_trace_processor_metatrace",
":src_trace_processor_metrics_metrics",
diff --git a/CHANGELOG b/CHANGELOG
index 70cb79c..8aae7f5 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,10 +1,13 @@
Unreleased:
Tracing service and probes:
- *
+ *
SQL Standard library:
* Added megacycles support to CPU package. Added tables:
`cpu_cycles_per_process`, `cpu_cycles_per_thread` and
`cpu_cycles_per_cpu`.
+ * Improved `memory` package. Added `memory.linux.process`,
+ `memory.linux.high_watermark` and `memory.android.gpu` modules.
+ * Created `gpu` package with `gpu.frequency` module.
* Migrated `sched.utilization` package to `cpu.utilization`.
Trace Processor:
* Added "time to initial display" and "time to full display" metrics to
diff --git a/protos/perfetto/metrics/android/startup_metric.proto b/protos/perfetto/metrics/android/startup_metric.proto
index 66f01ec..4a93ebc 100644
--- a/protos/perfetto/metrics/android/startup_metric.proto
+++ b/protos/perfetto/metrics/android/startup_metric.proto
@@ -342,7 +342,7 @@
optional uint32 thread_utid = 3;
}
- // Next id: 24
+ // Next id: 25
message Startup {
// Random id uniquely identifying an app startup in this trace.
optional uint32 startup_id = 1;
@@ -424,9 +424,16 @@
// Optional.
repeated string slow_start_reason = 17;
- // Same as slow_start_reason, but with more detailed information.
+ // Same as slow_start_reason, but with more detailed information, obsolete.
repeated SlowStartReasonDetailed slow_start_reason_detailed = 21;
+ // Similar to slow_start_reason_detailed, but with much more comprehensive info.
+ // such as expected threshold, actual value and threads/slices to inspect.
+ // slow_start_reason will be obsolete once slow_start_reason_with_details is completed
+ // since slow_start_reason_with_details contains all the data in slow_start_reason
+ // and more.
+ repeated SlowStartReason slow_start_reason_with_details = 24;
+
reserved 10;
}
diff --git a/protos/perfetto/metrics/perfetto_merged_metrics.proto b/protos/perfetto/metrics/perfetto_merged_metrics.proto
index bffca26..ad2d6b2 100644
--- a/protos/perfetto/metrics/perfetto_merged_metrics.proto
+++ b/protos/perfetto/metrics/perfetto_merged_metrics.proto
@@ -2458,7 +2458,7 @@
optional uint32 thread_utid = 3;
}
- // Next id: 24
+ // Next id: 25
message Startup {
// Random id uniquely identifying an app startup in this trace.
optional uint32 startup_id = 1;
@@ -2540,9 +2540,16 @@
// Optional.
repeated string slow_start_reason = 17;
- // Same as slow_start_reason, but with more detailed information.
+ // Same as slow_start_reason, but with more detailed information, obsolete.
repeated SlowStartReasonDetailed slow_start_reason_detailed = 21;
+ // Similar to slow_start_reason_detailed, but with much more comprehensive info.
+ // such as expected threshold, actual value and threads/slices to inspect.
+ // slow_start_reason will be obsolete once slow_start_reason_with_details is completed
+ // since slow_start_reason_with_details contains all the data in slow_start_reason
+ // and more.
+ repeated SlowStartReason slow_start_reason_with_details = 24;
+
reserved 10;
}
diff --git a/python/perfetto/trace_processor/metrics.descriptor b/python/perfetto/trace_processor/metrics.descriptor
index ef13816..f212c89 100644
--- a/python/perfetto/trace_processor/metrics.descriptor
+++ b/python/perfetto/trace_processor/metrics.descriptor
Binary files differ
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index 2a61ec6..802f9bc 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -177,6 +177,7 @@
"importers/proto:full",
"importers/proto:minimal",
"importers/systrace:full",
+ "importers/zip:full",
"metrics",
"perfetto_sql/engine",
"perfetto_sql/intrinsics/functions",
diff --git a/src/trace_processor/containers/string_pool.h b/src/trace_processor/containers/string_pool.h
index 75a57a2..2e03592 100644
--- a/src/trace_processor/containers/string_pool.h
+++ b/src/trace_processor/containers/string_pool.h
@@ -17,21 +17,25 @@
#ifndef SRC_TRACE_PROCESSOR_CONTAINERS_STRING_POOL_H_
#define SRC_TRACE_PROCESSOR_CONTAINERS_STRING_POOL_H_
-#include <stddef.h>
-#include <stdint.h>
-
+#include <cstddef>
+#include <cstdint>
+#include <functional>
#include <limits>
+#include <memory>
#include <optional>
+#include <string>
+#include <utility>
#include <vector>
+#include "perfetto/base/logging.h"
#include "perfetto/ext/base/flat_hash_map.h"
#include "perfetto/ext/base/hash.h"
#include "perfetto/ext/base/paged_memory.h"
+#include "perfetto/ext/base/string_view.h"
#include "perfetto/protozero/proto_utils.h"
#include "src/trace_processor/containers/null_term_string_view.h"
-namespace perfetto {
-namespace trace_processor {
+namespace perfetto::trace_processor {
// Interns strings in a string pool and hands out compact StringIds which can
// be used to retrieve the string in O(1).
@@ -40,34 +44,38 @@
struct Id {
Id() = default;
- bool operator==(const Id& other) const { return other.id == id; }
- bool operator!=(const Id& other) const { return !(other == *this); }
- bool operator<(const Id& other) const { return id < other.id; }
+ constexpr bool operator==(const Id& other) const { return other.id == id; }
+ constexpr bool operator!=(const Id& other) const {
+ return !(other == *this);
+ }
+ constexpr bool operator<(const Id& other) const { return id < other.id; }
- bool is_null() const { return id == 0u; }
+ constexpr bool is_null() const { return id == 0u; }
- bool is_large_string() const { return id & kLargeStringFlagBitMask; }
+ constexpr bool is_large_string() const {
+ return id & kLargeStringFlagBitMask;
+ }
- uint32_t block_offset() const { return id & kBlockOffsetBitMask; }
+ constexpr uint32_t block_offset() const { return id & kBlockOffsetBitMask; }
- uint32_t block_index() const {
+ constexpr uint32_t block_index() const {
return (id & kBlockIndexBitMask) >> kNumBlockOffsetBits;
}
- uint32_t large_string_index() const {
+ constexpr uint32_t large_string_index() const {
PERFETTO_DCHECK(is_large_string());
return id & ~kLargeStringFlagBitMask;
}
- uint32_t raw_id() const { return id; }
+ constexpr uint32_t raw_id() const { return id; }
- static Id LargeString(size_t index) {
+ static constexpr Id LargeString(size_t index) {
PERFETTO_DCHECK(index <= static_cast<uint32_t>(index));
PERFETTO_DCHECK(!(index & kLargeStringFlagBitMask));
return Id(kLargeStringFlagBitMask | static_cast<uint32_t>(index));
}
- static Id BlockString(size_t index, uint32_t offset) {
+ static constexpr Id BlockString(size_t index, uint32_t offset) {
PERFETTO_DCHECK(index < (1u << (kNumBlockIndexBits + 1)));
PERFETTO_DCHECK(offset < (1u << (kNumBlockOffsetBits + 1)));
return Id(~kLargeStringFlagBitMask &
@@ -80,7 +88,7 @@
static constexpr Id Null() { return Id(0u); }
private:
- constexpr Id(uint32_t i) : id(i) {}
+ constexpr explicit Id(uint32_t i) : id(i) {}
uint32_t id;
};
@@ -88,7 +96,7 @@
// Iterator over the strings in the pool.
class Iterator {
public:
- Iterator(const StringPool*);
+ explicit Iterator(const StringPool*);
explicit operator bool() const;
Iterator& operator++();
@@ -278,7 +286,7 @@
static NullTermStringView GetFromBlockPtr(const uint8_t* ptr) {
uint32_t size = 0;
const uint8_t* str_ptr = ReadSize(ptr, &size);
- return NullTermStringView(reinterpret_cast<const char*>(str_ptr), size);
+ return {reinterpret_cast<const char*>(str_ptr), size};
}
// Lookup a string in the |large_strings_| vector. |id| should have the MSB
@@ -288,7 +296,7 @@
size_t index = id.large_string_index();
PERFETTO_DCHECK(index < large_strings_.size());
const std::string* str = large_strings_[index].get();
- return NullTermStringView(str->c_str(), str->size());
+ return {str->c_str(), str->size()};
}
// The actual memory storing the strings.
@@ -308,8 +316,7 @@
string_index_{/*initial_capacity=*/4096u};
};
-} // namespace trace_processor
-} // namespace perfetto
+} // namespace perfetto::trace_processor
template <>
struct std::hash<::perfetto::trace_processor::StringPool::Id> {
diff --git a/src/trace_processor/db/column/BUILD.gn b/src/trace_processor/db/column/BUILD.gn
index d53e7ae..f77887f 100644
--- a/src/trace_processor/db/column/BUILD.gn
+++ b/src/trace_processor/db/column/BUILD.gn
@@ -43,6 +43,7 @@
"utils.h",
]
deps = [
+ "..:compare",
"../..:metatrace",
"../../../../gn:default_deps",
"../../../../include/perfetto/trace_processor",
diff --git a/src/trace_processor/db/column/arrangement_overlay.cc b/src/trace_processor/db/column/arrangement_overlay.cc
index c31d7c2..e7bc8ff 100644
--- a/src/trace_processor/db/column/arrangement_overlay.cc
+++ b/src/trace_processor/db/column/arrangement_overlay.cc
@@ -68,12 +68,21 @@
if (does_arrangement_order_storage_ && op != FilterOp::kGlob &&
op != FilterOp::kRegex) {
- Range inner_res = inner_->OrderedIndexSearchValidated(
- op, sql_val,
- OrderedIndices{arrangement_->data() + in.start, in.size(),
- arrangement_state_});
+ OrderedIndices indices{arrangement_->data() + in.start, in.size(),
+ arrangement_state_};
+ if (op == FilterOp::kNe) {
+ // Do an equality search and "invert" the range.
+ Range inner_res =
+ inner_->OrderedIndexSearchValidated(FilterOp::kEq, sql_val, indices);
+ BitVector bv(in.start);
+ bv.Resize(in.start + inner_res.start, true);
+ bv.Resize(in.start + inner_res.end, false);
+ bv.Resize(in.end, true);
+ return RangeOrBitVector(std::move(bv));
+ }
+ Range inner_res = inner_->OrderedIndexSearchValidated(op, sql_val, indices);
return RangeOrBitVector(
- Range(inner_res.start + in.start, inner_res.end + in.start));
+ Range(in.start + inner_res.start, in.start + inner_res.end));
}
const auto& arrangement = *arrangement_;
@@ -196,6 +205,11 @@
return inner_->MinElement(indices);
}
+SqlValue ArrangementOverlay::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return inner_->Get_AvoidUsingBecauseSlow((*arrangement_)[index]);
+}
+
void ArrangementOverlay::ChainImpl::Serialize(StorageProto* storage) const {
auto* arrangement_overlay = storage->set_arrangement_overlay();
arrangement_overlay->set_values(
diff --git a/src/trace_processor/db/column/arrangement_overlay.h b/src/trace_processor/db/column/arrangement_overlay.h
index 3656b07..85b9ce4 100644
--- a/src/trace_processor/db/column/arrangement_overlay.h
+++ b/src/trace_processor/db/column/arrangement_overlay.h
@@ -19,10 +19,10 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include <vector>
-#include "perfetto/base/logging.h"
#include "perfetto/trace_processor/basic_types.h"
#include "src/trace_processor/db/column/data_layer.h"
#include "src/trace_processor/db/column/types.h"
@@ -61,13 +61,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override {
- PERFETTO_FATAL(
- "OrderedIndexSearch can't be called on ArrangementOverlay");
- }
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -78,6 +71,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override {
diff --git a/src/trace_processor/db/column/arrangement_overlay_unittest.cc b/src/trace_processor/db/column/arrangement_overlay_unittest.cc
index 55f310b..8820336 100644
--- a/src/trace_processor/db/column/arrangement_overlay_unittest.cc
+++ b/src/trace_processor/db/column/arrangement_overlay_unittest.cc
@@ -108,15 +108,20 @@
}
TEST(ArrangementOverlay, OrderingSearch) {
- std::vector<uint32_t> arrangement{0, 2, 4, 1, 3};
- auto fake = FakeStorageChain::SearchSubset(5, BitVector({0, 1, 0, 1, 0}));
+ std::vector<uint32_t> numeric_data{0, 1, 2, 0, 1, 0};
+ NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false);
+
+ std::vector<uint32_t> arrangement{0, 3, 5, 1, 4, 2};
ArrangementOverlay storage(&arrangement, Indices::State::kNonmonotonic);
- auto chain =
- storage.MakeChain(std::move(fake), DataLayer::ChainCreationArgs(true));
+ auto chain = storage.MakeChain(numeric.MakeChain(),
+ DataLayer::ChainCreationArgs(true));
RangeOrBitVector res =
- chain->Search(FilterOp::kGe, SqlValue::Long(0u), Range(0, 5));
+ chain->Search(FilterOp::kGe, SqlValue::Long(1u), Range(0, 5));
ASSERT_THAT(utils::ToIndexVectorForTests(res), ElementsAre(3, 4));
+
+ res = chain->Search(FilterOp::kNe, SqlValue::Long(1u), Range(1, 6));
+ ASSERT_THAT(utils::ToIndexVectorForTests(res), ElementsAre(1, 2, 5));
}
TEST(ArrangementOverlay, StableSort) {
diff --git a/src/trace_processor/db/column/data_layer.cc b/src/trace_processor/db/column/data_layer.cc
index 634cbce..a570415 100644
--- a/src/trace_processor/db/column/data_layer.cc
+++ b/src/trace_processor/db/column/data_layer.cc
@@ -16,12 +16,15 @@
#include "src/trace_processor/db/column/data_layer.h"
+#include <algorithm>
#include <cstdint>
+#include <iterator>
#include <memory>
#include <utility>
#include <vector>
#include "perfetto/base/logging.h"
+#include "perfetto/trace_processor/basic_types.h"
#include "src/trace_processor/containers/bit_vector.h"
#include "src/trace_processor/containers/string_pool.h"
#include "src/trace_processor/db/column/arrangement_overlay.h"
@@ -35,6 +38,7 @@
#include "src/trace_processor/db/column/set_id_storage.h"
#include "src/trace_processor/db/column/string_storage.h"
#include "src/trace_processor/db/column/types.h"
+#include "src/trace_processor/db/compare.h"
namespace perfetto::trace_processor::column {
@@ -113,6 +117,53 @@
PERFETTO_FATAL("For GCC");
}
+Range DataLayerChain::OrderedIndexSearchValidated(
+ FilterOp op,
+ SqlValue value,
+ const OrderedIndices& indices) const {
+ auto lb = [&]() {
+ return static_cast<uint32_t>(std::distance(
+ indices.data,
+ std::lower_bound(indices.data, indices.data + indices.size, value,
+ [this](uint32_t idx, const SqlValue& v) {
+ return compare::SqlValueComparator(
+ Get_AvoidUsingBecauseSlow(idx), v);
+ })));
+ };
+ auto ub = [&]() {
+ return static_cast<uint32_t>(std::distance(
+ indices.data,
+ std::upper_bound(indices.data, indices.data + indices.size, value,
+ [this](const SqlValue& v, uint32_t idx) {
+ return compare::SqlValueComparator(
+ v, Get_AvoidUsingBecauseSlow(idx));
+ })));
+ };
+ switch (op) {
+ case FilterOp::kEq:
+ return {lb(), ub()};
+ case FilterOp::kLe:
+ return {0, ub()};
+ case FilterOp::kLt:
+ return {0, lb()};
+ case FilterOp::kGe:
+ return {lb(), indices.size};
+ case FilterOp::kGt:
+ return {ub(), indices.size};
+ case FilterOp::kIsNull:
+ PERFETTO_CHECK(value.is_null());
+ return {0, ub()};
+ case FilterOp::kIsNotNull:
+ PERFETTO_CHECK(value.is_null());
+ return {ub(), indices.size};
+ case FilterOp::kNe:
+ case FilterOp::kGlob:
+ case FilterOp::kRegex:
+ PERFETTO_FATAL("Wrong filtering operation");
+ }
+ PERFETTO_FATAL("For GCC");
+}
+
ArrangementOverlay::ArrangementOverlay(
const std::vector<uint32_t>* arrangement,
DataLayerChain::Indices::State arrangement_state)
diff --git a/src/trace_processor/db/column/data_layer.h b/src/trace_processor/db/column/data_layer.h
index 5e34539..b6f9382 100644
--- a/src/trace_processor/db/column/data_layer.h
+++ b/src/trace_processor/db/column/data_layer.h
@@ -321,9 +321,26 @@
// Post-validated implementation of |OrderedIndexSearch|. See
// |OrderedIndexSearch|'s documentation.
- virtual Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const = 0;
+ Range OrderedIndexSearchValidated(FilterOp op,
+ SqlValue value,
+ const OrderedIndices& indices) const;
+
+ // Returns the SqlValue representing the value at a given index.
+ //
+ // This function might be very tempting to use as it appears cheap on the
+ // surface but because of how DataLayerChains might be layered on top of each
+ // other, this might require *several* virtual function calls per index.
+ // If you're tempted to use this, please consider instead create a new
+ // "vectorized" function instead and only using this as a last resort.
+ //
+ // The correct "class" of algorithms to use this function are cases where you
+ // have a set of indices you want to lookup and based on the value returned
+ // you will only use a fraction of them. In this case, it might be worth
+ // paying the non-vectorized lookup to vastly reduce how many indices need
+ // to be translated.
+ //
+ // An example of such an algorithm is binary search on indices.
+ virtual SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const = 0;
};
} // namespace perfetto::trace_processor::column
diff --git a/src/trace_processor/db/column/dense_null_overlay.cc b/src/trace_processor/db/column/dense_null_overlay.cc
index 79ca3c0..426c6d2 100644
--- a/src/trace_processor/db/column/dense_null_overlay.cc
+++ b/src/trace_processor/db/column/dense_null_overlay.cc
@@ -219,50 +219,6 @@
inner_->IndexSearchValidated(op, sql_val, indices);
}
-Range DenseNullOverlay::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- // For NOT EQUAL the further analysis needs to be done by the caller.
- PERFETTO_CHECK(op != FilterOp::kNe);
-
- PERFETTO_TP_TRACE(metatrace::Category::DB,
- "DenseNullOverlay::ChainImpl::OrderedIndexSearch");
-
- // We assume all NULLs are ordered to be in the front. We are looking for the
- // first index that points to non NULL value.
- const uint32_t* first_non_null =
- std::partition_point(indices.data, indices.data + indices.size,
- [this](uint32_t i) { return !non_null_->IsSet(i); });
-
- auto non_null_offset =
- static_cast<uint32_t>(std::distance(indices.data, first_non_null));
- auto non_null_size = static_cast<uint32_t>(
- std::distance(first_non_null, indices.data + indices.size));
-
- if (op == FilterOp::kIsNull) {
- return {0, non_null_offset};
- }
-
- if (op == FilterOp::kIsNotNull) {
- switch (inner_->ValidateSearchConstraints(op, sql_val)) {
- case SearchValidationResult::kNoData:
- return {};
- case SearchValidationResult::kAllData:
- return {non_null_offset, indices.size};
- case SearchValidationResult::kOk:
- break;
- }
- }
-
- Range inner_range = inner_->OrderedIndexSearchValidated(
- op, sql_val,
- OrderedIndices{first_non_null, non_null_size,
- Indices::State::kNonmonotonic});
- return {inner_range.start + non_null_offset,
- inner_range.end + non_null_offset};
-}
-
void DenseNullOverlay::ChainImpl::StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const {
@@ -314,6 +270,12 @@
: *first_null_it;
}
+SqlValue DenseNullOverlay::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return non_null_->IsSet(index) ? inner_->Get_AvoidUsingBecauseSlow(index)
+ : SqlValue();
+}
+
void DenseNullOverlay::ChainImpl::Serialize(StorageProto* storage) const {
auto* null_overlay = storage->set_dense_null_overlay();
non_null_->Serialize(null_overlay->set_bit_vector());
diff --git a/src/trace_processor/db/column/dense_null_overlay.h b/src/trace_processor/db/column/dense_null_overlay.h
index e253400..d0a95d2 100644
--- a/src/trace_processor/db/column/dense_null_overlay.h
+++ b/src/trace_processor/db/column/dense_null_overlay.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -56,10 +57,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -70,6 +67,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override { return non_null_->size(); }
diff --git a/src/trace_processor/db/column/dense_null_overlay_unittest.cc b/src/trace_processor/db/column/dense_null_overlay_unittest.cc
index 24b1eee..7cd74f6 100644
--- a/src/trace_processor/db/column/dense_null_overlay_unittest.cc
+++ b/src/trace_processor/db/column/dense_null_overlay_unittest.cc
@@ -129,11 +129,12 @@
}
TEST(DenseNullOverlay, OrderedIndexSearch) {
- auto fake = FakeStorageChain::SearchSubset(6, BitVector({0, 1, 0, 1, 0, 1}));
+ std::vector<uint32_t> numeric_data{0, 1, 0, 1, 0, 1};
+ NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false);
BitVector bv{0, 1, 0, 1, 0, 1};
DenseNullOverlay storage(&bv);
- auto chain = storage.MakeChain(std::move(fake));
+ auto chain = storage.MakeChain(numeric.MakeChain());
std::vector<uint32_t> indices_vec({0, 2, 4, 1, 3, 5});
OrderedIndices indices{indices_vec.data(), 6, Indices::State::kNonmonotonic};
@@ -146,25 +147,25 @@
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 6u);
- res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(3), indices);
+ res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(1), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 6u);
- res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(3), indices);
+ res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(0), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 6u);
- res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(3), indices);
+ res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(1), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 6u);
- res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(3), indices);
- ASSERT_EQ(res.start, 3u);
- ASSERT_EQ(res.end, 6u);
+ res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(1), indices);
+ ASSERT_EQ(res.start, 0u);
+ ASSERT_EQ(res.end, 3u);
- res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(3), indices);
- ASSERT_EQ(res.start, 3u);
- ASSERT_EQ(res.end, 6u);
+ res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(0), indices);
+ ASSERT_EQ(res.start, 0u);
+ ASSERT_EQ(res.end, 3u);
}
TEST(DenseNullOverlay, SingleSearch) {
diff --git a/src/trace_processor/db/column/dummy_storage.cc b/src/trace_processor/db/column/dummy_storage.cc
index 631cbb7..ef9cacf 100644
--- a/src/trace_processor/db/column/dummy_storage.cc
+++ b/src/trace_processor/db/column/dummy_storage.cc
@@ -17,6 +17,7 @@
#include "src/trace_processor/db/column/dummy_storage.h"
#include <cstdint>
+#include <optional>
#include "perfetto/base/logging.h"
#include "perfetto/trace_processor/basic_types.h"
@@ -49,13 +50,6 @@
PERFETTO_FATAL("Shouldn't be called");
}
-Range DummyStorage::ChainImpl::OrderedIndexSearchValidated(
- FilterOp,
- SqlValue,
- const OrderedIndices&) const {
- PERFETTO_FATAL("Shouldn't be called");
-}
-
void DummyStorage::ChainImpl::StableSort(SortToken*,
SortToken*,
SortDirection) const {
@@ -78,6 +72,10 @@
PERFETTO_FATAL("Shouldn't be called");
}
+SqlValue DummyStorage::ChainImpl::Get_AvoidUsingBecauseSlow(uint32_t) const {
+ PERFETTO_FATAL("Shouldn't be called");
+}
+
void DummyStorage::ChainImpl::Serialize(StorageProto*) const {
PERFETTO_FATAL("Shouldn't be called");
}
diff --git a/src/trace_processor/db/column/dummy_storage.h b/src/trace_processor/db/column/dummy_storage.h
index ffa0c73..66f2497 100644
--- a/src/trace_processor/db/column/dummy_storage.h
+++ b/src/trace_processor/db/column/dummy_storage.h
@@ -18,6 +18,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -45,10 +46,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -59,6 +56,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override;
diff --git a/src/trace_processor/db/column/fake_storage.cc b/src/trace_processor/db/column/fake_storage.cc
index 5a26961..5015bf3 100644
--- a/src/trace_processor/db/column/fake_storage.cc
+++ b/src/trace_processor/db/column/fake_storage.cc
@@ -19,6 +19,7 @@
#include <algorithm>
#include <cstdint>
#include <iterator>
+#include <optional>
#include <utility>
#include "perfetto/base/logging.h"
@@ -112,43 +113,6 @@
PERFETTO_FATAL("For GCC");
}
-Range FakeStorageChain::OrderedIndexSearchValidated(
- FilterOp,
- SqlValue,
- const OrderedIndices& indices) const {
- switch (strategy_) {
- case kAll:
- return {0, indices.size};
- case kNone:
- return {};
- case kRange: {
- // We are looking at intersection of |range_| and |indices_|.
- const uint32_t* first_in_range = std::partition_point(
- indices.data, indices.data + indices.size,
- [this](uint32_t i) { return !range_.Contains(i); });
- const uint32_t* first_outside_range = std::partition_point(
- first_in_range, indices.data + indices.size,
- [this](uint32_t i) { return range_.Contains(i); });
- return {
- static_cast<uint32_t>(std::distance(indices.data, first_in_range)),
- static_cast<uint32_t>(
- std::distance(indices.data, first_outside_range))};
- }
- case kBitVector:
- // We are looking at intersection of |range_| and |bit_vector_|.
- const uint32_t* first_set = std::partition_point(
- indices.data, indices.data + indices.size,
- [this](uint32_t i) { return !bit_vector_.IsSet(i); });
- const uint32_t* first_non_set = std::partition_point(
- first_set, indices.data + indices.size,
- [this](uint32_t i) { return bit_vector_.IsSet(i); });
- return {
- static_cast<uint32_t>(std::distance(indices.data, first_set)),
- static_cast<uint32_t>(std::distance(indices.data, first_non_set))};
- }
- PERFETTO_FATAL("For GCC");
-}
-
void FakeStorageChain::Distinct(Indices&) const {
// Fake storage shouldn't implement Distinct as it's not a binary (this index
// passes or not) operation on a column.
@@ -166,6 +130,10 @@
PERFETTO_FATAL("Not implemented");
}
+SqlValue FakeStorageChain::Get_AvoidUsingBecauseSlow(uint32_t) const {
+ PERFETTO_FATAL("Not implemented");
+}
+
void FakeStorageChain::Serialize(StorageProto*) const {
// FakeStorage doesn't really make sense to serialize.
PERFETTO_FATAL("Not implemented");
diff --git a/src/trace_processor/db/column/fake_storage.h b/src/trace_processor/db/column/fake_storage.h
index 7adf786..980a951 100644
--- a/src/trace_processor/db/column/fake_storage.h
+++ b/src/trace_processor/db/column/fake_storage.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include <utility>
#include <vector>
@@ -84,10 +85,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -95,8 +92,11 @@
void Distinct(Indices&) const override;
std::optional<Token> MaxElement(Indices&) const override;
+
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override { return size_; }
diff --git a/src/trace_processor/db/column/fake_storage_unittest.cc b/src/trace_processor/db/column/fake_storage_unittest.cc
index 908b7e0..28c497f 100644
--- a/src/trace_processor/db/column/fake_storage_unittest.cc
+++ b/src/trace_processor/db/column/fake_storage_unittest.cc
@@ -43,7 +43,6 @@
using testing::IsEmpty;
using Indices = DataLayerChain::Indices;
-using OrderedIndices = DataLayerChain::OrderedIndices;
TEST(FakeStorage, ValidateSearchConstraints) {
{
@@ -197,48 +196,6 @@
}
}
-TEST(FakeStorage, OrderedIndexSearchValidated) {
- std::vector<uint32_t> table_idx{4, 3, 2, 1};
- OrderedIndices indices{table_idx.data(), uint32_t(table_idx.size()),
- Indices::State::kNonmonotonic};
- {
- // All passes
- auto fake = FakeStorageChain::SearchAll(5);
- Range ret =
- fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices);
- EXPECT_EQ(ret, Range(0, 4));
- }
- {
- // None passes
- auto fake = FakeStorageChain::SearchNone(5);
- Range ret =
- fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices);
- EXPECT_EQ(ret, Range(0, 0));
- }
- {
- // BitVector
- auto fake = FakeStorageChain::SearchSubset(5, BitVector{0, 0, 1, 1, 1});
- Range ret =
- fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices);
- EXPECT_EQ(ret, Range(0, 3));
- }
- {
- // Index vector
- auto fake =
- FakeStorageChain::SearchSubset(5, std::vector<uint32_t>{1, 2, 3});
- Range ret =
- fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices);
- EXPECT_EQ(ret, Range(1, 4));
- }
- {
- // Range
- auto fake = FakeStorageChain::SearchSubset(5, Range(1, 4));
- Range ret =
- fake->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(0u), indices);
- EXPECT_EQ(ret, Range(1, 4));
- }
-}
-
} // namespace
} // namespace column
} // namespace perfetto::trace_processor
diff --git a/src/trace_processor/db/column/id_storage.cc b/src/trace_processor/db/column/id_storage.cc
index ea1e909..1ebe459 100644
--- a/src/trace_processor/db/column/id_storage.cc
+++ b/src/trace_processor/db/column/id_storage.cc
@@ -246,47 +246,6 @@
PERFETTO_FATAL("FilterOp not matched");
}
-Range IdStorage::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- PERFETTO_DCHECK(op != FilterOp::kNe);
-
- PERFETTO_TP_TRACE(
- metatrace::Category::DB, "IdStorage::ChainImpl::OrderedIndexSearch",
- [indices, op](metatrace::Record* r) {
- r->AddArg("Count", std::to_string(indices.size));
- r->AddArg("Op", std::to_string(static_cast<uint32_t>(op)));
- });
-
- // It's a valid filter operation if |sql_val| is a double, although it
- // requires special logic.
- if (sql_val.type == SqlValue::kDouble) {
- switch (utils::CompareIntColumnWithDouble(op, &sql_val)) {
- case SearchValidationResult::kOk:
- break;
- case SearchValidationResult::kAllData:
- return {0, indices.size};
- case SearchValidationResult::kNoData:
- return {};
- }
- }
- auto val = static_cast<uint32_t>(sql_val.AsLong());
-
- // OrderedIndices are monotonic non contiguous values if OrderedIndexSearch
- // was called. Look for the first and last index and find the result of
- // looking for this range in IdStorage.
- Range indices_range(indices.data[0], indices.data[indices.size - 1] + 1);
- Range bin_search_ret = BinarySearchIntrinsic(op, val, indices_range);
-
- const auto* start_ptr = std::lower_bound(
- indices.data, indices.data + indices.size, bin_search_ret.start);
- const auto* end_ptr = std::lower_bound(start_ptr, indices.data + indices.size,
- bin_search_ret.end);
- return {static_cast<uint32_t>(std::distance(indices.data, start_ptr)),
- static_cast<uint32_t>(std::distance(indices.data, end_ptr))};
-}
-
Range IdStorage::ChainImpl::BinarySearchIntrinsic(FilterOp op,
Id val,
Range range) {
@@ -363,6 +322,10 @@
return *tok;
}
+SqlValue IdStorage::ChainImpl::Get_AvoidUsingBecauseSlow(uint32_t index) const {
+ return SqlValue::Long(index);
+}
+
void IdStorage::ChainImpl::Serialize(StorageProto* storage) const {
storage->set_id_storage();
}
diff --git a/src/trace_processor/db/column/id_storage.h b/src/trace_processor/db/column/id_storage.h
index 420568d..c3aeadf 100644
--- a/src/trace_processor/db/column/id_storage.h
+++ b/src/trace_processor/db/column/id_storage.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <limits>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -55,10 +56,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -69,6 +66,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override {
diff --git a/src/trace_processor/db/column/null_overlay.cc b/src/trace_processor/db/column/null_overlay.cc
index 128d3ff..372d3b0 100644
--- a/src/trace_processor/db/column/null_overlay.cc
+++ b/src/trace_processor/db/column/null_overlay.cc
@@ -256,56 +256,6 @@
inner_->IndexSearchValidated(op, sql_val, indices);
}
-Range NullOverlay::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- // For NOT EQUAL the translation or results from EQUAL needs to be done by the
- // caller.
- PERFETTO_CHECK(op != FilterOp::kNe);
-
- PERFETTO_TP_TRACE(metatrace::Category::DB,
- "NullOverlay::ChainImpl::OrderedIndexSearch");
-
- // We assume all NULLs are ordered to be in the front. We are looking for the
- // first index that points to non NULL value.
- const uint32_t* first_non_null =
- std::partition_point(indices.data, indices.data + indices.size,
- [this](uint32_t i) { return !non_null_->IsSet(i); });
- auto non_null_offset =
- static_cast<uint32_t>(std::distance(indices.data, first_non_null));
- auto non_null_size = static_cast<uint32_t>(
- std::distance(first_non_null, indices.data + indices.size));
-
- if (op == FilterOp::kIsNull) {
- return {0, non_null_offset};
- }
-
- if (op == FilterOp::kIsNotNull) {
- switch (inner_->ValidateSearchConstraints(op, sql_val)) {
- case SearchValidationResult::kNoData:
- return {};
- case SearchValidationResult::kAllData:
- return {non_null_offset, indices.size};
- case SearchValidationResult::kOk:
- break;
- }
- }
-
- std::vector<uint32_t> storage_iv;
- storage_iv.reserve(non_null_size);
- for (const uint32_t* it = first_non_null;
- it != first_non_null + non_null_size; it++) {
- storage_iv.push_back(non_null_->CountSetBits(*it));
- }
-
- Range inner_range = inner_->OrderedIndexSearchValidated(
- op, sql_val,
- OrderedIndices{storage_iv.data(), non_null_size, indices.state});
- return {inner_range.start + non_null_offset,
- inner_range.end + non_null_offset};
-}
-
void NullOverlay::ChainImpl::StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const {
@@ -369,6 +319,13 @@
return inner_->MinElement(indices);
}
+SqlValue NullOverlay::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return non_null_->IsSet(index)
+ ? inner_->Get_AvoidUsingBecauseSlow(non_null_->CountSetBits(index))
+ : SqlValue();
+}
+
void NullOverlay::ChainImpl::Serialize(StorageProto* storage) const {
auto* null_storage = storage->set_null_overlay();
non_null_->Serialize(null_storage->set_bit_vector());
diff --git a/src/trace_processor/db/column/null_overlay.h b/src/trace_processor/db/column/null_overlay.h
index 187c056..a74f211 100644
--- a/src/trace_processor/db/column/null_overlay.h
+++ b/src/trace_processor/db/column/null_overlay.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -55,10 +56,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -69,6 +66,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override { return non_null_->size(); }
diff --git a/src/trace_processor/db/column/null_overlay_unittest.cc b/src/trace_processor/db/column/null_overlay_unittest.cc
index 981c7bd..9cff095 100644
--- a/src/trace_processor/db/column/null_overlay_unittest.cc
+++ b/src/trace_processor/db/column/null_overlay_unittest.cc
@@ -200,15 +200,15 @@
}
TEST(NullOverlay, OrderedIndexSearch) {
+ std::vector<uint32_t> numeric_data{1, 0, 1, 0};
+ NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false);
+
BitVector bv{0, 1, 1, 1, 0, 1};
- // Passing values in final storage (on normal operations)
- // 0, 1, 0, 1, 0, 0
- auto fake = FakeStorageChain::SearchSubset(4, BitVector{1, 0, 1, 0});
NullOverlay storage(&bv);
- auto chain = storage.MakeChain(std::move(fake));
+ auto chain = storage.MakeChain(numeric.MakeChain());
// Passing values on final data
- // NULL, NULL, 0, 1, 1
+ // NULL, NULL, 0, 0, 1, 1
std::vector<uint32_t> table_idx{0, 4, 5, 1, 3};
OrderedIndices indices{table_idx.data(), uint32_t(table_idx.size()),
Indices::State::kNonmonotonic};
@@ -218,28 +218,28 @@
ASSERT_EQ(res.end, 2u);
res = chain->OrderedIndexSearch(FilterOp::kIsNotNull, SqlValue(), indices);
+ ASSERT_EQ(res.start, 2u);
+ ASSERT_EQ(res.end, 5u);
+
+ res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(1), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 5u);
- res = chain->OrderedIndexSearch(FilterOp::kEq, SqlValue::Long(3), indices);
+ res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(0), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 5u);
- res = chain->OrderedIndexSearch(FilterOp::kGt, SqlValue::Long(3), indices);
+ res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(1), indices);
ASSERT_EQ(res.start, 3u);
ASSERT_EQ(res.end, 5u);
- res = chain->OrderedIndexSearch(FilterOp::kGe, SqlValue::Long(3), indices);
- ASSERT_EQ(res.start, 3u);
- ASSERT_EQ(res.end, 5u);
+ res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(1), indices);
+ ASSERT_EQ(res.start, 0u);
+ ASSERT_EQ(res.end, 3u);
- res = chain->OrderedIndexSearch(FilterOp::kLt, SqlValue::Long(3), indices);
- ASSERT_EQ(res.start, 3u);
- ASSERT_EQ(res.end, 5u);
-
- res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(3), indices);
- ASSERT_EQ(res.start, 3u);
- ASSERT_EQ(res.end, 5u);
+ res = chain->OrderedIndexSearch(FilterOp::kLe, SqlValue::Long(0), indices);
+ ASSERT_EQ(res.start, 0u);
+ ASSERT_EQ(res.end, 3u);
}
TEST(NullOverlay, StableSort) {
diff --git a/src/trace_processor/db/column/numeric_storage.cc b/src/trace_processor/db/column/numeric_storage.cc
index b39b4cb..3612356 100644
--- a/src/trace_processor/db/column/numeric_storage.cc
+++ b/src/trace_processor/db/column/numeric_storage.cc
@@ -138,60 +138,6 @@
}
template <typename T>
-uint32_t TypedLowerBoundExtrinsic(T val,
- const T* data,
- OrderedIndices indices) {
- const auto* lower = std::lower_bound(
- indices.data, indices.data + indices.size, val,
- [data](uint32_t index, T value) { return data[index] < value; });
- return static_cast<uint32_t>(std::distance(indices.data, lower));
-}
-
-uint32_t LowerBoundExtrinsic(const void* vector_ptr,
- NumericValue val,
- OrderedIndices indices) {
- if (const auto* u32 = std::get_if<uint32_t>(&val)) {
- const auto* start =
- static_cast<const std::vector<uint32_t>*>(vector_ptr)->data();
- return TypedLowerBoundExtrinsic(*u32, start, indices);
- }
- if (const auto* i64 = std::get_if<int64_t>(&val)) {
- const auto* start =
- static_cast<const std::vector<int64_t>*>(vector_ptr)->data();
- return TypedLowerBoundExtrinsic(*i64, start, indices);
- }
- if (const auto* i32 = std::get_if<int32_t>(&val)) {
- const auto* start =
- static_cast<const std::vector<int32_t>*>(vector_ptr)->data();
- return TypedLowerBoundExtrinsic(*i32, start, indices);
- }
- if (const auto* db = std::get_if<double>(&val)) {
- const auto* start =
- static_cast<const std::vector<double>*>(vector_ptr)->data();
- return TypedLowerBoundExtrinsic(*db, start, indices);
- }
- PERFETTO_FATAL("Type not handled");
-}
-
-uint32_t UpperBoundExtrinsic(const void* vector_ptr,
- NumericValue val,
- OrderedIndices indices) {
- return std::visit(
- [vector_ptr, indices](auto val_data) {
- using T = decltype(val_data);
- const T* typed_start =
- static_cast<const std::vector<T>*>(vector_ptr)->data();
- const auto* upper =
- std::upper_bound(indices.data, indices.data + indices.size,
- val_data, [typed_start](T value, uint32_t index) {
- return value < typed_start[index];
- });
- return static_cast<uint32_t>(std::distance(indices.data, upper));
- },
- val);
-}
-
-template <typename T>
void TypedLinearSearch(T typed_val,
const T* start,
FilterOp op,
@@ -504,79 +450,6 @@
val);
}
-Range NumericStorageBase::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- PERFETTO_TP_TRACE(
- metatrace::Category::DB, "NumericStorage::ChainImpl::OrderedIndexSearch",
- [indices, op](metatrace::Record* r) {
- r->AddArg("Count", std::to_string(indices.size));
- r->AddArg("Op", std::to_string(static_cast<uint32_t>(op)));
- });
-
- // Mismatched types - value is double and column is int.
- if (sql_val.type == SqlValue::kDouble &&
- storage_type_ != ColumnType::kDouble) {
- if (auto ret = utils::CanReturnEarly(IntColumnWithDouble(op, &sql_val),
- indices.size);
- ret) {
- return *ret;
- }
- }
-
- // Mismatched types - column is double and value is int.
- if (sql_val.type != SqlValue::kDouble &&
- storage_type_ == ColumnType::kDouble) {
- if (auto ret = utils::CanReturnEarly(DoubleColumnWithInt(op, &sql_val),
- indices.size);
- ret) {
- return *ret;
- }
- }
-
- NumericValue val;
- switch (storage_type_) {
- case ColumnType::kDouble:
- val = sql_val.AsDouble();
- break;
- case ColumnType::kInt64:
- val = sql_val.AsLong();
- break;
- case ColumnType::kInt32:
- val = static_cast<int32_t>(sql_val.AsLong());
- break;
- case ColumnType::kUint32:
- val = static_cast<uint32_t>(sql_val.AsLong());
- break;
- case ColumnType::kString:
- case ColumnType::kDummy:
- case ColumnType::kId:
- PERFETTO_FATAL("Invalid type");
- }
-
- switch (op) {
- case FilterOp::kEq:
- return {LowerBoundExtrinsic(vector_ptr_, val, indices),
- UpperBoundExtrinsic(vector_ptr_, val, indices)};
- case FilterOp::kLe:
- return {0, UpperBoundExtrinsic(vector_ptr_, val, indices)};
- case FilterOp::kLt:
- return {0, LowerBoundExtrinsic(vector_ptr_, val, indices)};
- case FilterOp::kGe:
- return {LowerBoundExtrinsic(vector_ptr_, val, indices), indices.size};
- case FilterOp::kGt:
- return {UpperBoundExtrinsic(vector_ptr_, val, indices), indices.size};
- case FilterOp::kNe:
- case FilterOp::kIsNull:
- case FilterOp::kIsNotNull:
- case FilterOp::kGlob:
- case FilterOp::kRegex:
- PERFETTO_FATAL("Wrong filtering operation");
- }
- PERFETTO_FATAL("For GCC");
-}
-
BitVector NumericStorageBase::ChainImpl::LinearSearchInternal(
FilterOp op,
NumericValue val,
diff --git a/src/trace_processor/db/column/numeric_storage.h b/src/trace_processor/db/column/numeric_storage.h
index fd27d26..3c4416b 100644
--- a/src/trace_processor/db/column/numeric_storage.h
+++ b/src/trace_processor/db/column/numeric_storage.h
@@ -23,6 +23,7 @@
#include <optional>
#include <set>
#include <string>
+#include <type_traits>
#include <unordered_set>
#include <variant>
#include <vector>
@@ -49,10 +50,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void Serialize(StorageProto*) const override;
std::string DebugString() const override { return "NumericStorage"; }
@@ -145,6 +142,13 @@
return *tok;
}
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override {
+ if constexpr (std::is_same_v<T, double>) {
+ return SqlValue::Double((*vector_)[index]);
+ }
+ return SqlValue::Long((*vector_)[index]);
+ }
+
void StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const override {
diff --git a/src/trace_processor/db/column/range_overlay.cc b/src/trace_processor/db/column/range_overlay.cc
index 81bb7ae..dee7838 100644
--- a/src/trace_processor/db/column/range_overlay.cc
+++ b/src/trace_processor/db/column/range_overlay.cc
@@ -18,6 +18,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <utility>
#include <vector>
@@ -118,22 +119,6 @@
inner_->IndexSearchValidated(op, sql_val, indices);
}
-Range RangeOverlay::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- PERFETTO_TP_TRACE(metatrace::Category::DB, "RangeOverlay::IndexSearch");
-
- // Should be SIMD optimized.
- std::vector<uint32_t> storage_iv(indices.size);
- for (uint32_t i = 0; i < indices.size; ++i) {
- storage_iv[i] = indices.data[i] + range_->start;
- }
- return inner_->OrderedIndexSearchValidated(
- op, sql_val,
- OrderedIndices{storage_iv.data(), indices.size, indices.state});
-}
-
void RangeOverlay::ChainImpl::StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const {
@@ -160,6 +145,11 @@
return inner_->MaxElement(indices);
}
+SqlValue RangeOverlay::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return inner_->Get_AvoidUsingBecauseSlow(index + range_->start);
+}
+
std::optional<Token> RangeOverlay::ChainImpl::MinElement(
Indices& indices) const {
PERFETTO_TP_TRACE(metatrace::Category::DB, "RangeOverlay::MinElement");
diff --git a/src/trace_processor/db/column/range_overlay.h b/src/trace_processor/db/column/range_overlay.h
index f15e8dc..a58df40 100644
--- a/src/trace_processor/db/column/range_overlay.h
+++ b/src/trace_processor/db/column/range_overlay.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -52,10 +53,6 @@
void IndexSearchValidated(FilterOp p, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -66,6 +63,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override { return range_->size(); }
diff --git a/src/trace_processor/db/column/selector_overlay.cc b/src/trace_processor/db/column/selector_overlay.cc
index b2cbc43..9e957cb 100644
--- a/src/trace_processor/db/column/selector_overlay.cc
+++ b/src/trace_processor/db/column/selector_overlay.cc
@@ -16,9 +16,9 @@
#include "src/trace_processor/db/column/selector_overlay.h"
-#include <algorithm>
#include <cstdint>
#include <memory>
+#include <optional>
#include <utility>
#include <vector>
@@ -35,7 +35,7 @@
namespace perfetto::trace_processor::column {
namespace {
-static constexpr uint32_t kIndexOfNthSetRatio = 32;
+constexpr uint32_t kIndexOfNthSetRatio = 32;
} // namespace
@@ -97,23 +97,6 @@
return inner_->IndexSearchValidated(op, sql_val, indices);
}
-Range SelectorOverlay::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- // To go from TableIndexVector to StorageIndexVector we need to find index in
- // |selector_| by looking only into set bits.
- std::vector<uint32_t> inner_indices(indices.size);
- for (uint32_t i = 0; i < indices.size; ++i) {
- inner_indices[i] = selector_->IndexOfNthSet(indices.data[i]);
- }
- return inner_->OrderedIndexSearchValidated(
- op, sql_val,
- OrderedIndices{inner_indices.data(),
- static_cast<uint32_t>(inner_indices.size()),
- indices.state});
-}
-
void SelectorOverlay::ChainImpl::StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const {
@@ -148,6 +131,11 @@
return inner_->MinElement(indices);
}
+SqlValue SelectorOverlay::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return inner_->Get_AvoidUsingBecauseSlow(selector_->IndexOfNthSet(index));
+}
+
void SelectorOverlay::ChainImpl::Serialize(StorageProto* storage) const {
auto* selector_overlay = storage->set_selector_overlay();
inner_->Serialize(selector_overlay->set_storage());
@@ -164,15 +152,15 @@
for (auto& token : indices.tokens) {
token.index = selector_->IndexOfNthSet(token.index);
}
- } else {
- // TODO(mayzner): once we have a reverse index for IndexOfNthSet in
- // BitVector, this should no longer be necessary.
- std::vector<uint32_t> lookup = selector_->GetSetBitIndices();
- for (auto& token : indices.tokens) {
- token.index = lookup[token.index];
- }
+ return;
}
- return;
+
+ // TODO(mayzner): once we have a reverse index for IndexOfNthSet in
+ // BitVector, this should no longer be necessary.
+ std::vector<uint32_t> lookup = selector_->GetSetBitIndices();
+ for (auto& token : indices.tokens) {
+ token.index = lookup[token.index];
+ }
}
} // namespace perfetto::trace_processor::column
diff --git a/src/trace_processor/db/column/selector_overlay.h b/src/trace_processor/db/column/selector_overlay.h
index d54bdf5..3726f9f 100644
--- a/src/trace_processor/db/column/selector_overlay.h
+++ b/src/trace_processor/db/column/selector_overlay.h
@@ -19,6 +19,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include "perfetto/trace_processor/basic_types.h"
@@ -56,10 +57,6 @@
void IndexSearchValidated(FilterOp p, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection) const override;
@@ -70,6 +67,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override { return selector_->size(); }
diff --git a/src/trace_processor/db/column/selector_overlay_unittest.cc b/src/trace_processor/db/column/selector_overlay_unittest.cc
index 0abae4c..def2a77 100644
--- a/src/trace_processor/db/column/selector_overlay_unittest.cc
+++ b/src/trace_processor/db/column/selector_overlay_unittest.cc
@@ -17,6 +17,7 @@
#include "src/trace_processor/db/column/selector_overlay.h"
#include <cstdint>
+#include <utility>
#include <vector>
#include "data_layer.h"
@@ -103,35 +104,23 @@
ASSERT_THAT(utils::ExtractPayloadForTesting(indices), ElementsAre(1u));
}
-TEST(SelectorOverlay, OrderedIndexSearchTrivial) {
+TEST(SelectorOverlay, OrderedIndexSearch) {
+ std::vector<uint32_t> numeric_data{1, 0, 0, 1, 1};
+ NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false);
+
BitVector selector{1, 0, 1, 0, 1};
- auto fake = FakeStorageChain::SearchAll(5);
SelectorOverlay storage(&selector);
- auto chain = storage.MakeChain(std::move(fake));
+ auto chain = storage.MakeChain(numeric.MakeChain());
std::vector<uint32_t> table_idx{1u, 0u, 2u};
Range res = chain->OrderedIndexSearch(
- FilterOp::kGe, SqlValue::Long(0u),
+ FilterOp::kGe, SqlValue::Long(1u),
OrderedIndices{table_idx.data(), static_cast<uint32_t>(table_idx.size()),
Indices::State::kNonmonotonic});
- ASSERT_EQ(res.start, 0u);
+ ASSERT_EQ(res.start, 1u);
ASSERT_EQ(res.end, 3u);
}
-TEST(SelectorOverlay, OrderedIndexSearchNone) {
- BitVector selector{1, 0, 1, 0, 1};
- auto fake = FakeStorageChain::SearchNone(5);
- SelectorOverlay storage(&selector);
- auto chain = storage.MakeChain(std::move(fake));
-
- std::vector<uint32_t> table_idx{1u, 0u, 2u};
- Range res = chain->OrderedIndexSearch(
- FilterOp::kGe, SqlValue::Long(0u),
- OrderedIndices{table_idx.data(), static_cast<uint32_t>(table_idx.size()),
- Indices::State::kNonmonotonic});
- ASSERT_EQ(res.size(), 0u);
-}
-
TEST(SelectorOverlay, StableSort) {
std::vector<uint32_t> numeric_data{3, 1, 0, 0, 2, 4, 3, 4};
NumericStorage<uint32_t> numeric(&numeric_data, ColumnType::kUint32, false);
diff --git a/src/trace_processor/db/column/set_id_storage.cc b/src/trace_processor/db/column/set_id_storage.cc
index 207b649..42ab4ea 100644
--- a/src/trace_processor/db/column/set_id_storage.cc
+++ b/src/trace_processor/db/column/set_id_storage.cc
@@ -19,10 +19,7 @@
#include <algorithm>
#include <cstdint>
#include <functional>
-#include <iterator>
-#include <limits>
-#include <memory>
-#include <set>
+#include <optional>
#include <string>
#include <unordered_set>
#include <utility>
@@ -250,38 +247,17 @@
}
}
-Range SetIdStorage::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- PERFETTO_TP_TRACE(metatrace::Category::DB,
- "SetIdStorage::ChainImpl::OrderedIndexSearch");
- // OrderedIndices are monotonic non-contiguous values.
- auto res = SearchValidated(
- op, sql_val, Range(indices.data[0], indices.data[indices.size - 1] + 1));
- PERFETTO_CHECK(res.IsRange());
- Range res_range = std::move(res).TakeIfRange();
-
- const auto* start_ptr = std::lower_bound(
- indices.data, indices.data + indices.size, res_range.start);
- const auto* end_ptr =
- std::lower_bound(start_ptr, indices.data + indices.size, res_range.end);
-
- return {static_cast<uint32_t>(std::distance(indices.data, start_ptr)),
- static_cast<uint32_t>(std::distance(indices.data, end_ptr))};
-}
-
Range SetIdStorage::ChainImpl::BinarySearchIntrinsic(FilterOp op,
SetId val,
Range range) const {
switch (op) {
case FilterOp::kEq: {
- if (values_->data()[val] != val) {
- return Range();
+ if ((*values_)[val] != val) {
+ return {};
}
uint32_t start = std::max(val, range.start);
uint32_t end = UpperBoundIntrinsic(values_->data(), val, range);
- return Range(std::min(start, end), end);
+ return {std::min(start, end), end};
}
case FilterOp::kLe: {
return {range.start, UpperBoundIntrinsic(values_->data(), val, range)};
@@ -370,6 +346,11 @@
return *tok;
}
+SqlValue SetIdStorage::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ return SqlValue::Long((*values_)[index]);
+}
+
void SetIdStorage::ChainImpl::Serialize(StorageProto* msg) const {
auto* vec_msg = msg->set_set_id_storage();
vec_msg->set_values(reinterpret_cast<const uint8_t*>(values_->data()),
diff --git a/src/trace_processor/db/column/set_id_storage.h b/src/trace_processor/db/column/set_id_storage.h
index 2fa6c81..e459106 100644
--- a/src/trace_processor/db/column/set_id_storage.h
+++ b/src/trace_processor/db/column/set_id_storage.h
@@ -18,6 +18,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include <vector>
@@ -54,10 +55,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const override;
@@ -70,6 +67,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
uint32_t size() const override {
return static_cast<uint32_t>(values_->size());
}
diff --git a/src/trace_processor/db/column/string_storage.cc b/src/trace_processor/db/column/string_storage.cc
index adf3eda..1d25dbb 100644
--- a/src/trace_processor/db/column/string_storage.cc
+++ b/src/trace_processor/db/column/string_storage.cc
@@ -163,36 +163,6 @@
return static_cast<uint32_t>(std::distance(data, upper));
}
-uint32_t LowerBoundExtrinsic(StringPool* pool,
- const StringPool::Id* data,
- NullTermStringView val,
- const uint32_t* indices,
- uint32_t indices_count,
- uint32_t offset) {
- Less comp{pool};
- const auto* lower =
- std::lower_bound(indices + offset, indices + indices_count, val,
- [comp, data](uint32_t index, NullTermStringView val) {
- return comp(data[index], val);
- });
- return static_cast<uint32_t>(std::distance(indices, lower));
-}
-
-uint32_t UpperBoundExtrinsic(StringPool* pool,
- const StringPool::Id* data,
- NullTermStringView val,
- const uint32_t* indices,
- uint32_t indices_count,
- uint32_t offset) {
- Greater comp{pool};
- const auto* upper =
- std::upper_bound(indices + offset, indices + indices_count, val,
- [comp, data](NullTermStringView val, uint32_t index) {
- return comp(data[index], val);
- });
- return static_cast<uint32_t>(std::distance(indices, upper));
-}
-
} // namespace
StringStorage::ChainImpl::ChainImpl(StringPool* string_pool,
@@ -518,64 +488,6 @@
return std::move(builder).Build();
}
-Range StringStorage::ChainImpl::OrderedIndexSearchValidated(
- FilterOp op,
- SqlValue sql_val,
- const OrderedIndices& indices) const {
- PERFETTO_TP_TRACE(metatrace::Category::DB,
- "StringStorage::ChainImpl::OrderedIndexSearch");
- StringPool::Id val =
- (op == FilterOp::kIsNull || op == FilterOp::kIsNotNull)
- ? StringPool::Id::Null()
- : string_pool_->InternString(base::StringView(sql_val.AsString()));
- NullTermStringView val_str = string_pool_->Get(val);
-
- auto first_non_null = static_cast<uint32_t>(std::distance(
- indices.data,
- std::partition_point(indices.data, indices.data + indices.size,
- [this](uint32_t i) {
- return (*data_)[i] == StringPool::Id::Null();
- })));
-
- switch (op) {
- case FilterOp::kEq:
- return {LowerBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null),
- UpperBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null)};
- case FilterOp::kLe:
- return {first_non_null,
- UpperBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null)};
- case FilterOp::kLt:
- return {first_non_null,
- LowerBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null)};
- case FilterOp::kGe:
- return {LowerBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null),
- indices.size};
- case FilterOp::kGt:
- return {UpperBoundExtrinsic(string_pool_, data_->data(), val_str,
- indices.data, indices.size, first_non_null),
- indices.size};
- case FilterOp::kIsNull: {
- // Assuming nulls are at the front.
- return Range(0, first_non_null);
- }
- case FilterOp::kIsNotNull: {
- // Assuming nulls are at the front.
- return Range(first_non_null, indices.size);
- }
-
- case FilterOp::kNe:
- case FilterOp::kGlob:
- case FilterOp::kRegex:
- PERFETTO_FATAL("Not supported for OrderedIndexSearch");
- }
- PERFETTO_FATAL("For GCC");
-}
-
Range StringStorage::ChainImpl::BinarySearchIntrinsic(
FilterOp op,
SqlValue sql_val,
@@ -715,6 +627,14 @@
return *tok;
}
+SqlValue StringStorage::ChainImpl::Get_AvoidUsingBecauseSlow(
+ uint32_t index) const {
+ StringPool::Id id = (*data_)[index];
+ return id == StringPool::Id::Null()
+ ? SqlValue()
+ : SqlValue::String(string_pool_->Get(id).c_str());
+}
+
void StringStorage::ChainImpl::Serialize(StorageProto* msg) const {
auto* string_storage_msg = msg->set_string_storage();
string_storage_msg->set_is_sorted(is_sorted_);
diff --git a/src/trace_processor/db/column/string_storage.h b/src/trace_processor/db/column/string_storage.h
index 470ccd1..1c05dd7 100644
--- a/src/trace_processor/db/column/string_storage.h
+++ b/src/trace_processor/db/column/string_storage.h
@@ -18,6 +18,7 @@
#include <cstdint>
#include <memory>
+#include <optional>
#include <string>
#include <vector>
@@ -57,10 +58,6 @@
void IndexSearchValidated(FilterOp, SqlValue, Indices&) const override;
- Range OrderedIndexSearchValidated(FilterOp,
- SqlValue,
- const OrderedIndices&) const override;
-
void StableSort(SortToken* start,
SortToken* end,
SortDirection direction) const override;
@@ -71,6 +68,8 @@
std::optional<Token> MinElement(Indices&) const override;
+ SqlValue Get_AvoidUsingBecauseSlow(uint32_t index) const override;
+
void Serialize(StorageProto*) const override;
uint32_t size() const override {
diff --git a/src/trace_processor/db/column/string_storage_unittest.cc b/src/trace_processor/db/column/string_storage_unittest.cc
index 4474ea9..7e7cadb 100644
--- a/src/trace_processor/db/column/string_storage_unittest.cc
+++ b/src/trace_processor/db/column/string_storage_unittest.cc
@@ -383,12 +383,12 @@
op = FilterOp::kLt;
res = chain->OrderedIndexSearch(op, val, indices);
- ASSERT_EQ(res.start, 1u);
+ ASSERT_EQ(res.start, 0u);
ASSERT_EQ(res.end, 4u);
op = FilterOp::kLe;
res = chain->OrderedIndexSearch(op, val, indices);
- ASSERT_EQ(res.start, 1u);
+ ASSERT_EQ(res.start, 0u);
ASSERT_EQ(res.end, 5u);
op = FilterOp::kGt;
@@ -402,6 +402,7 @@
ASSERT_EQ(res.end, 6u);
op = FilterOp::kIsNull;
+ val = SqlValue();
res = chain->OrderedIndexSearch(op, val, indices);
ASSERT_EQ(res.start, 0u);
ASSERT_EQ(res.end, 1u);
diff --git a/src/trace_processor/db/query_executor_benchmark.cc b/src/trace_processor/db/query_executor_benchmark.cc
index d6cc57f..dcd75f0 100644
--- a/src/trace_processor/db/query_executor_benchmark.cc
+++ b/src/trace_processor/db/query_executor_benchmark.cc
@@ -513,6 +513,29 @@
}
BENCHMARK(BM_QEFilterOrderedArrangement);
+void BM_QEFilterNullOrderedArrangement(benchmark::State& state) {
+ SliceTableForBenchmark table(state);
+ Order order{table.table_.parent_id().index_in_table(), false};
+ Table slice_sorted_with_parent_id = table.table_.Sort({order});
+
+ Constraint c{table.table_.parent_id().index_in_table(), FilterOp::kGt,
+ SqlValue::Long(26091)};
+ Query q;
+ q.constraints = {c};
+ for (auto _ : state) {
+ benchmark::DoNotOptimize(slice_sorted_with_parent_id.QueryToRowMap(q));
+ }
+ state.counters["s/row"] = benchmark::Counter(
+ static_cast<double>(slice_sorted_with_parent_id.row_count()),
+ benchmark::Counter::kIsIterationInvariantRate |
+ benchmark::Counter::kInvert);
+ state.counters["s/out"] = benchmark::Counter(
+ static_cast<double>(table.table_.QueryToRowMap(q).size()),
+ benchmark::Counter::kIsIterationInvariantRate |
+ benchmark::Counter::kInvert);
+}
+BENCHMARK(BM_QEFilterNullOrderedArrangement);
+
void BM_QESliceFilterIndexSearchOneElement(benchmark::State& state) {
SliceTableForBenchmark table(state);
BenchmarkSliceTableFilter(
diff --git a/src/trace_processor/forwarding_trace_parser.cc b/src/trace_processor/forwarding_trace_parser.cc
index 369917d..a3b6a7f 100644
--- a/src/trace_processor/forwarding_trace_parser.cc
+++ b/src/trace_processor/forwarding_trace_parser.cc
@@ -29,6 +29,7 @@
#include "src/trace_processor/trace_reader_registry.h"
#include "src/trace_processor/types/trace_processor_context.h"
#include "src/trace_processor/util/status_macros.h"
+#include "src/trace_processor/util/trace_type.h"
namespace perfetto {
namespace trace_processor {
@@ -53,7 +54,7 @@
case kSystraceTraceType:
case kGzipTraceType:
case kCtraceTraceType:
- case kAndroidBugreportTraceType:
+ case kZipFile:
return std::nullopt;
case kPerfDataTraceType:
@@ -102,26 +103,11 @@
reader_ = std::move(*reader_or);
PERFETTO_DLOG("%s detected", ToString(trace_type));
- std::optional<TraceSorter::SortingMode> minimum_sorting_mode =
- GetMinimumSortingMode(trace_type, *context_);
+ UpdateSorterForTraceType(trace_type);
- if (minimum_sorting_mode.has_value()) {
- if (!context_->sorter) {
- context_->sorter.reset(new TraceSorter(context_, *minimum_sorting_mode));
- }
-
- switch (context_->sorter->sorting_mode()) {
- case TraceSorter::SortingMode::kDefault:
- PERFETTO_CHECK(minimum_sorting_mode ==
- TraceSorter::SortingMode::kDefault);
- break;
- case TraceSorter::SortingMode::kFullSort:
- break;
- }
- }
-
- // TODO(carlscab) Make sure kProtoTraceType and kSystraceTraceType are parsed
- // first so that we do not get issues with SetPidZeroIsUpidZeroIdleProcess()
+ // TODO(b/334978369) Make sure kProtoTraceType and kSystraceTraceType are
+ // parsed first so that we do not get issues with
+ // SetPidZeroIsUpidZeroIdleProcess()
if (trace_type == kProtoTraceType || trace_type == kSystraceTraceType) {
context_->process_tracker->SetPidZeroIsUpidZeroIdleProcess();
}
@@ -129,6 +115,27 @@
return base::OkStatus();
}
+void ForwardingTraceParser::UpdateSorterForTraceType(TraceType trace_type) {
+ std::optional<TraceSorter::SortingMode> minimum_sorting_mode =
+ GetMinimumSortingMode(trace_type, *context_);
+ if (!minimum_sorting_mode.has_value()) {
+ return;
+ }
+
+ if (!context_->sorter) {
+ context_->sorter.reset(new TraceSorter(context_, *minimum_sorting_mode));
+ }
+
+ switch (context_->sorter->sorting_mode()) {
+ case TraceSorter::SortingMode::kDefault:
+ PERFETTO_CHECK(minimum_sorting_mode ==
+ TraceSorter::SortingMode::kDefault);
+ break;
+ case TraceSorter::SortingMode::kFullSort:
+ break;
+ }
+}
+
base::Status ForwardingTraceParser::Parse(TraceBlobView blob) {
// If this is the first Parse() call, guess the trace type and create the
// appropriate parser.
diff --git a/src/trace_processor/forwarding_trace_parser.h b/src/trace_processor/forwarding_trace_parser.h
index 1f32938..12fe447 100644
--- a/src/trace_processor/forwarding_trace_parser.h
+++ b/src/trace_processor/forwarding_trace_parser.h
@@ -20,6 +20,7 @@
#include "perfetto/base/status.h"
#include "perfetto/trace_processor/trace_blob_view.h"
#include "src/trace_processor/importers/common/chunked_trace_reader.h"
+#include "src/trace_processor/util/trace_type.h"
namespace perfetto {
namespace trace_processor {
@@ -37,6 +38,7 @@
private:
base::Status Init(const TraceBlobView&);
+ void UpdateSorterForTraceType(TraceType trace_type);
TraceProcessorContext* const context_;
std::unique_ptr<ChunkedTraceReader> reader_;
};
diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc
index 87aa0f5..1d46efc 100644
--- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc
+++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.cc
@@ -17,54 +17,90 @@
#include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h"
#include <algorithm>
+#include <cstddef>
#include <optional>
+#include <string>
+#include <vector>
#include "perfetto/base/logging.h"
+#include "perfetto/base/status.h"
#include "perfetto/ext/base/string_utils.h"
-#include "perfetto/trace_processor/trace_blob.h"
-#include "perfetto/trace_processor/trace_blob_view.h"
+#include "protos/perfetto/common/builtin_clock.pbzero.h"
#include "src/trace_processor/importers/android_bugreport/android_log_parser.h"
#include "src/trace_processor/importers/common/clock_tracker.h"
#include "src/trace_processor/importers/common/process_tracker.h"
-#include "src/trace_processor/importers/common/track_tracker.h"
#include "src/trace_processor/types/trace_processor_context.h"
#include "src/trace_processor/util/zip_reader.h"
-#include "protos/perfetto/common/builtin_clock.pbzero.h"
-
namespace perfetto {
namespace trace_processor {
+namespace {
+const util::ZipFile* FindBugReportFile(
+ const std::vector<util::ZipFile>& zip_file_entries) {
+ for (const auto& zf : zip_file_entries) {
+ if (base::StartsWith(zf.name(), "bugreport-") &&
+ base::EndsWith(zf.name(), ".txt")) {
+ return &zf;
+ }
+ }
+ return nullptr;
+}
-AndroidBugreportParser::AndroidBugreportParser(TraceProcessorContext* ctx)
- : context_(ctx), zip_reader_(new util::ZipReader()) {}
+std::optional<int32_t> ExtractYearFromBugReportFilename(
+ const std::string& filename) {
+ // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt".
+ auto year_str =
+ filename.substr(filename.size() - strlen("2022-12-31-23-59-00.txt"), 4);
+ return base::StringToInt32(year_str);
+}
+
+} // namespace
+
+// static
+bool AndroidBugreportParser::IsAndroidBugReport(
+ const std::vector<util::ZipFile>& zip_file_entries) {
+ if (const util::ZipFile* file = FindBugReportFile(zip_file_entries);
+ file != nullptr) {
+ return ExtractYearFromBugReportFilename(file->name()).has_value();
+ }
+
+ return false;
+}
+
+// static
+util::Status AndroidBugreportParser::Parse(
+ TraceProcessorContext* context,
+ std::vector<util::ZipFile> zip_file_entries) {
+ return AndroidBugreportParser(context, std::move(zip_file_entries))
+ .ParseImpl();
+}
+
+AndroidBugreportParser::AndroidBugreportParser(
+ TraceProcessorContext* context,
+ std::vector<util::ZipFile> zip_file_entries)
+ : context_(context), zip_file_entries_(std::move(zip_file_entries)) {}
AndroidBugreportParser::~AndroidBugreportParser() = default;
-util::Status AndroidBugreportParser::Parse(TraceBlobView tbv) {
- if (!first_chunk_seen_) {
- first_chunk_seen_ = true;
- // All logs in Android bugreports use wall time (which creates problems
- // in case of early boot events before NTP kicks in, which get emitted as
- // 1970), but that is the state of affairs.
- context_->clock_tracker->SetTraceTimeClock(
- protos::pbzero::BUILTIN_CLOCK_REALTIME);
- }
-
- return zip_reader_->Parse(tbv.data(), tbv.size());
-}
-
-void AndroidBugreportParser::NotifyEndOfFile() {
+util::Status AndroidBugreportParser::ParseImpl() {
+ // All logs in Android bugreports use wall time (which creates problems
+ // in case of early boot events before NTP kicks in, which get emitted as
+ // 1970), but that is the state of affairs.
+ context_->clock_tracker->SetTraceTimeClock(
+ protos::pbzero::BUILTIN_CLOCK_REALTIME);
if (!DetectYearAndBrFilename()) {
context_->storage->IncrementStats(stats::android_br_parse_errors);
- return;
+ return base::ErrStatus("Zip file does not contain bugreport file.");
}
ParsePersistentLogcat();
ParseDumpstateTxt();
SortAndStoreLogcat();
+ return base::OkStatus();
}
void AndroidBugreportParser::ParseDumpstateTxt() {
+ PERFETTO_CHECK(dumpstate_file_);
// Dumpstate is organized in a two level hierarchy, beautifully flattened into
// one text file with load bearing ----- markers:
// 1. Various dumpstate sections, examples:
@@ -95,80 +131,81 @@
// Here we put each line in a dedicated table, android_dumpstate, keeping
// track of the dumpstate `section` and dumpsys `service`.
AndroidLogParser log_parser(br_year_, context_->storage.get());
- util::ZipFile* zf = zip_reader_->Find(dumpstate_fname_);
StringId section_id = StringId::Null(); // The current dumpstate section.
StringId service_id = StringId::Null(); // The current dumpsys service.
static constexpr size_t npos = base::StringView::npos;
enum { OTHER = 0, DUMPSYS, LOG } cur_sect = OTHER;
- zf->DecompressLines([&](const std::vector<base::StringView>& lines) {
- // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new vector
- // on every line.
- std::vector<base::StringView> log_line(1);
- for (const base::StringView& line : lines) {
- if (line.StartsWith("------ ") && line.EndsWith(" ------")) {
- // These lines mark the beginning and end of dumpstate sections:
- // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------
- // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------
- base::StringView section = line.substr(7);
- section = section.substr(0, section.size() - 7);
- bool end_marker = section.find("was the duration of") != npos;
- service_id = StringId::Null();
- if (end_marker) {
- section_id = StringId::Null();
- } else {
- section_id = context_->storage->InternString(section);
- cur_sect = OTHER;
- if (section.StartsWith("DUMPSYS")) {
- cur_sect = DUMPSYS;
- } else if (section.StartsWith("SYSTEM LOG") ||
- section.StartsWith("EVENT LOG") ||
- section.StartsWith("RADIO LOG")) {
- // KERNEL LOG is deliberately omitted because SYSTEM LOG is a
- // superset. KERNEL LOG contains all dupes.
- cur_sect = LOG;
- } else if (section.StartsWith("BLOCK STAT")) {
- // Coalesce all the block stats into one section. Otherwise they
- // pollute the table with one section per block device.
- section_id = context_->storage->InternString("BLOCK STAT");
+ dumpstate_file_->DecompressLines(
+ [&](const std::vector<base::StringView>& lines) {
+ // Optimization for ParseLogLines() below. Avoids ctor/dtor-ing a new
+ // vector on every line.
+ std::vector<base::StringView> log_line(1);
+ for (const base::StringView& line : lines) {
+ if (line.StartsWith("------ ") && line.EndsWith(" ------")) {
+ // These lines mark the beginning and end of dumpstate sections:
+ // ------ DUMPSYS CRITICAL (/system/bin/dumpsys) ------
+ // ------ 0.356s was the duration of 'DUMPSYS CRITICAL' ------
+ base::StringView section = line.substr(7);
+ section = section.substr(0, section.size() - 7);
+ bool end_marker = section.find("was the duration of") != npos;
+ service_id = StringId::Null();
+ if (end_marker) {
+ section_id = StringId::Null();
+ } else {
+ section_id = context_->storage->InternString(section);
+ cur_sect = OTHER;
+ if (section.StartsWith("DUMPSYS")) {
+ cur_sect = DUMPSYS;
+ } else if (section.StartsWith("SYSTEM LOG") ||
+ section.StartsWith("EVENT LOG") ||
+ section.StartsWith("RADIO LOG")) {
+ // KERNEL LOG is deliberately omitted because SYSTEM LOG is a
+ // superset. KERNEL LOG contains all dupes.
+ cur_sect = LOG;
+ } else if (section.StartsWith("BLOCK STAT")) {
+ // Coalesce all the block stats into one section. Otherwise they
+ // pollute the table with one section per block device.
+ section_id = context_->storage->InternString("BLOCK STAT");
+ }
+ }
+ continue;
}
+ // Skip end marker lines for dumpsys sections.
+ if (cur_sect == DUMPSYS && line.StartsWith("--------- ") &&
+ line.find("was the duration of dumpsys") != npos) {
+ service_id = StringId::Null();
+ continue;
+ }
+ if (cur_sect == DUMPSYS && service_id.is_null() &&
+ line.StartsWith(
+ "----------------------------------------------")) {
+ continue;
+ }
+ if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) {
+ // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName:
+ base::StringView svc = line.substr(line.rfind(' ') + 1);
+ svc = svc.substr(0, svc.size() - 1);
+ service_id = context_->storage->InternString(svc);
+ } else if (cur_sect == LOG) {
+ // Parse the non-persistent logcat and append to `log_events_`,
+ // together with the persistent one previously parsed by
+ // ParsePersistentLogcat(). Skips entries that are already seen in
+ // the persistent logcat, handling us vs ms truncation.
+ PERFETTO_DCHECK(log_line.size() == 1);
+ log_line[0] = line;
+ log_parser.ParseLogLines(log_line, &log_events_,
+ log_events_last_sorted_idx_);
+ }
+
+ if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) {
+ build_fpr_ = line.substr(20, line.size() - 20).ToStdString();
+ }
+
+ // Append the line to the android_dumpstate table.
+ context_->storage->mutable_android_dumpstate_table()->Insert(
+ {section_id, service_id, context_->storage->InternString(line)});
}
- continue;
- }
- // Skip end marker lines for dumpsys sections.
- if (cur_sect == DUMPSYS && line.StartsWith("--------- ") &&
- line.find("was the duration of dumpsys") != npos) {
- service_id = StringId::Null();
- continue;
- }
- if (cur_sect == DUMPSYS && service_id.is_null() &&
- line.StartsWith("----------------------------------------------")) {
- continue;
- }
- if (cur_sect == DUMPSYS && line.StartsWith("DUMP OF SERVICE")) {
- // DUMP OF SERVICE [CRITICAL|HIGH] ServiceName:
- base::StringView svc = line.substr(line.rfind(' ') + 1);
- svc = svc.substr(0, svc.size() - 1);
- service_id = context_->storage->InternString(svc);
- } else if (cur_sect == LOG) {
- // Parse the non-persistent logcat and append to `log_events_`, together
- // with the persistent one previously parsed by ParsePersistentLogcat().
- // Skips entries that are already seen in the persistent logcat,
- // handling us vs ms truncation.
- PERFETTO_DCHECK(log_line.size() == 1);
- log_line[0] = line;
- log_parser.ParseLogLines(log_line, &log_events_,
- log_events_last_sorted_idx_);
- }
-
- if (build_fpr_.empty() && line.StartsWith("Build fingerprint:")) {
- build_fpr_ = line.substr(20, line.size() - 20).ToStdString();
- }
-
- // Append the line to the android_dumpstate table.
- context_->storage->mutable_android_dumpstate_table()->Insert(
- {section_id, service_id, context_->storage->InternString(line)});
- }
- });
+ });
}
void AndroidBugreportParser::ParsePersistentLogcat() {
@@ -182,22 +219,23 @@
// Sort files to ease the job of the subsequent line-based sort. Unfortunately
// lines within each file are not 100% timestamp-ordered, due to things like
// kernel messages where log time != event time.
- std::vector<std::pair<uint64_t, std::string>> log_paths;
- for (const util::ZipFile& zf : zip_reader_->files()) {
+ std::vector<std::pair<uint64_t, const util::ZipFile*>> log_files;
+ for (const util::ZipFile& zf : zip_file_entries_) {
if (base::StartsWith(zf.name(), "FS/data/misc/logd/logcat") &&
!base::EndsWith(zf.name(), "logcat.id")) {
- log_paths.emplace_back(std::make_pair(zf.GetDatetime(), zf.name()));
+ log_files.push_back(std::make_pair(zf.GetDatetime(), &zf));
}
}
- std::sort(log_paths.begin(), log_paths.end());
+
+ std::sort(log_files.begin(), log_files.end());
// Push all events into the AndroidLogParser. It will take care of string
// interning into the pool. Appends entries into `log_events`.
- for (const auto& kv : log_paths) {
- util::ZipFile* zf = zip_reader_->Find(kv.second);
- zf->DecompressLines([&](const std::vector<base::StringView>& lines) {
- log_parser.ParseLogLines(lines, &log_events_);
- });
+ for (const auto& log_file : log_files) {
+ log_file.second->DecompressLines(
+ [&](const std::vector<base::StringView>& lines) {
+ log_parser.ParseLogLines(lines, &log_events_);
+ });
}
// Do an initial sorting pass. This is not the final sorting because we
@@ -230,30 +268,20 @@
// This is obviously bugged for cases of bugreports collected across new year
// but we'll live with that.
bool AndroidBugreportParser::DetectYearAndBrFilename() {
- const util::ZipFile* br_file = nullptr;
- for (const auto& zf : zip_reader_->files()) {
- if (base::StartsWith(zf.name(), "bugreport-") &&
- base::EndsWith(zf.name(), ".txt")) {
- br_file = &zf;
- break;
- }
- }
-
+ const util::ZipFile* br_file = FindBugReportFile(zip_file_entries_);
if (!br_file) {
PERFETTO_ELOG("Could not find bugreport-*.txt in the zip file");
return false;
}
- // Typical name: "bugreport-product-TP1A.220623.001-2022-06-24-16-24-37.txt".
- auto year_str = br_file->name().substr(
- br_file->name().size() - strlen("2022-12-31-23-59-00.txt"), 4);
- std::optional<int32_t> year = base::StringToInt32(year_str);
+ std::optional<int32_t> year =
+ ExtractYearFromBugReportFilename(br_file->name());
if (!year.has_value()) {
PERFETTO_ELOG("Could not parse the year from %s", br_file->name().c_str());
return false;
}
br_year_ = *year;
- dumpstate_fname_ = br_file->name();
+ dumpstate_file_ = br_file;
return true;
}
diff --git a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h
index 9969159..a9ca322 100644
--- a/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h
+++ b/src/trace_processor/importers/android_bugreport/android_bugreport_parser.h
@@ -17,8 +17,11 @@
#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_
#define SRC_TRACE_PROCESSOR_IMPORTERS_ANDROID_BUGREPORT_ANDROID_BUGREPORT_PARSER_H_
-#include "src/trace_processor/importers/common/chunked_trace_reader.h"
-#include "src/trace_processor/storage/trace_storage.h"
+#include <cstddef>
+#include <vector>
+
+#include "perfetto/trace_processor/status.h"
+#include "src/trace_processor/util/zip_reader.h"
namespace perfetto {
namespace trace_processor {
@@ -31,16 +34,19 @@
class TraceProcessorContext;
// Trace importer for Android bugreport.zip archives.
-class AndroidBugreportParser : public ChunkedTraceReader {
+class AndroidBugreportParser {
public:
- explicit AndroidBugreportParser(TraceProcessorContext*);
- ~AndroidBugreportParser() override;
-
- // ChunkedTraceReader implementation.
- util::Status Parse(TraceBlobView) override;
- void NotifyEndOfFile() override;
+ static bool IsAndroidBugReport(
+ const std::vector<util::ZipFile>& zip_file_entries);
+ static util::Status Parse(TraceProcessorContext* context,
+ std::vector<util::ZipFile> zip_file_entries);
private:
+ AndroidBugreportParser(TraceProcessorContext* context,
+ std::vector<util::ZipFile> zip_file_entries);
+ ~AndroidBugreportParser();
+ util::Status ParseImpl();
+
bool DetectYearAndBrFilename();
void ParsePersistentLogcat();
void ParseDumpstateTxt();
@@ -48,11 +54,11 @@
void SortLogEvents();
TraceProcessorContext* const context_;
+ std::vector<util::ZipFile> zip_file_entries_;
int br_year_ = 0; // The year when the bugreport has been taken.
- std::string dumpstate_fname_; // The name of bugreport-xxx-2022-08-04....txt
+ const util::ZipFile* dumpstate_file_ =
+ nullptr; // The bugreport-xxx-2022-08-04....txt file
std::string build_fpr_;
- bool first_chunk_seen_ = false;
- std::unique_ptr<util::ZipReader> zip_reader_;
std::vector<AndroidLogEvent> log_events_;
size_t log_events_last_sorted_idx_ = 0;
};
diff --git a/src/trace_processor/importers/proto/gpu_event_parser.cc b/src/trace_processor/importers/proto/gpu_event_parser.cc
index d08d946..5664fba 100644
--- a/src/trace_processor/importers/proto/gpu_event_parser.cc
+++ b/src/trace_processor/importers/proto/gpu_event_parser.cc
@@ -359,6 +359,16 @@
}
}
+ if (event.has_context()) {
+ uint64_t context_id = event.context();
+ auto* decoder = sequence_state->LookupInternedMessage<
+ protos::pbzero::InternedData::kGraphicsContextsFieldNumber,
+ protos::pbzero::InternedGraphicsContext>(context_id);
+ if (decoder) {
+ pid = decoder->pid();
+ }
+ }
+
auto args_callback = [this, &event,
sequence_state](ArgsTracker::BoundInserter* inserter) {
if (event.has_stage_iid()) {
diff --git a/src/trace_processor/importers/zip/BUILD.gn b/src/trace_processor/importers/zip/BUILD.gn
new file mode 100644
index 0000000..a938fdd
--- /dev/null
+++ b/src/trace_processor/importers/zip/BUILD.gn
@@ -0,0 +1,33 @@
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source_set("full") {
+ sources = [
+ "zip_trace_reader.cc",
+ "zip_trace_reader.h",
+ ]
+ deps = [
+ "../../../../gn:default_deps",
+ "../../../../gn:zlib",
+ "../../../../include/perfetto/ext/base:base",
+ "../../../trace_processor:storage_minimal",
+ "../../types",
+ "../../util:trace_type",
+ "../../util:util",
+ "../../util:zip_reader",
+ "../android_bugreport",
+ "../common",
+ "../proto:minimal",
+ ]
+}
diff --git a/src/trace_processor/importers/zip/zip_trace_reader.cc b/src/trace_processor/importers/zip/zip_trace_reader.cc
new file mode 100644
index 0000000..20f019f
--- /dev/null
+++ b/src/trace_processor/importers/zip/zip_trace_reader.cc
@@ -0,0 +1,158 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/importers/zip/zip_trace_reader.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstdint>
+#include <cstring>
+#include <memory>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/base/status.h"
+#include "perfetto/ext/base/status_or.h"
+#include "perfetto/trace_processor/trace_blob.h"
+#include "perfetto/trace_processor/trace_blob_view.h"
+#include "src/trace_processor/forwarding_trace_parser.h"
+#include "src/trace_processor/importers/android_bugreport/android_bugreport_parser.h"
+#include "src/trace_processor/importers/proto/proto_trace_tokenizer.h"
+#include "src/trace_processor/types/trace_processor_context.h"
+#include "src/trace_processor/util/status_macros.h"
+#include "src/trace_processor/util/trace_type.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace {
+
+// Proto traces should always parsed first as they might contains clock sync
+// data needed to correctly parse other traces.
+// The rest of the types are sorted by position in the enum but this is not
+// something users should rely on.
+// TODO(carlscab): Proto traces with just ModuleSymbols packets should be an
+// exception. We actually need those are the very end (once whe have all the
+// Frames). Alternatively we could build a map address -> symbol during
+// tokenization and use this during parsing to resolve symbols.
+bool CompareTraceType(TraceType lhs, TraceType rhs) {
+ if (rhs == TraceType::kProtoTraceType) {
+ return false;
+ }
+ if (lhs == TraceType::kProtoTraceType) {
+ return true;
+ }
+ return lhs < rhs;
+}
+
+bool HasSymbols(const TraceBlobView& blob) {
+ bool has_symbols = false;
+ ProtoTraceTokenizer().Tokenize(blob.copy(), [&](TraceBlobView raw) {
+ protos::pbzero::TracePacket::Decoder packet(raw.data(), raw.size());
+ has_symbols = packet.has_module_symbols();
+ return base::ErrStatus("break");
+ });
+ return has_symbols;
+}
+
+} // namespace
+
+ZipTraceReader::ZipTraceReader(TraceProcessorContext* context)
+ : context_(context) {}
+ZipTraceReader::~ZipTraceReader() = default;
+
+bool ZipTraceReader::Entry::operator<(const Entry& rhs) const {
+ // Traces with symbols should be the last ones to be read.
+ if (has_symbols) {
+ return false;
+ }
+ if (rhs.has_symbols) {
+ return true;
+ }
+ if (CompareTraceType(trace_type, rhs.trace_type)) {
+ return true;
+ }
+ if (CompareTraceType(rhs.trace_type, trace_type)) {
+ return false;
+ }
+ return std::tie(name, index) < std::tie(rhs.name, rhs.index);
+}
+
+util::Status ZipTraceReader::Parse(TraceBlobView blob) {
+ zip_reader_.Parse(blob.data(), blob.size());
+ return base::OkStatus();
+}
+
+void ZipTraceReader::NotifyEndOfFile() {
+ base::Status status = NotifyEndOfFileImpl();
+ if (!status.ok()) {
+ PERFETTO_ELOG("ZipTraceReader failed: %s", status.c_message());
+ }
+}
+
+base::Status ZipTraceReader::NotifyEndOfFileImpl() {
+ std::vector<util::ZipFile> files = zip_reader_.TakeFiles();
+
+ // Android bug reports are ZIP files and its files do not get handled
+ // separately.
+ if (AndroidBugreportParser::IsAndroidBugReport(files)) {
+ return AndroidBugreportParser::Parse(context_, std::move(files));
+ }
+
+ base::StatusOr<std::vector<Entry>> entries = ExtractEntries(std::move(files));
+ if (!entries.ok()) {
+ return entries.status();
+ }
+ std::sort(entries->begin(), entries->end());
+
+ for (Entry& e : *entries) {
+ parsers_.push_back(std::make_unique<ForwardingTraceParser>(context_));
+ auto& parser = *parsers_.back();
+ RETURN_IF_ERROR(parser.Parse(std::move(e.uncompressed_data)));
+ parser.NotifyEndOfFile();
+ }
+ return base::OkStatus();
+}
+
+base::StatusOr<std::vector<ZipTraceReader::Entry>>
+ZipTraceReader::ExtractEntries(std::vector<util::ZipFile> files) const {
+ // TODO(carlsacab): There is a lot of unnecessary copying going on here.
+ // ZipTraceReader can directly parse the ZIP file and given that we know the
+ // decompressed size we could directly decompress into TraceBlob chunks and
+ // send them to the tokenizer.
+ std::vector<Entry> entries;
+ std::vector<uint8_t> buffer;
+ for (size_t i = 0; i < files.size(); ++i) {
+ const util::ZipFile& zip_file = files[i];
+ Entry entry;
+ entry.name = zip_file.name();
+ entry.index = i;
+ RETURN_IF_ERROR(files[i].Decompress(&buffer));
+ entry.uncompressed_data =
+ TraceBlobView(TraceBlob::CopyFrom(buffer.data(), buffer.size()));
+ entry.trace_type = GuessTraceType(entry.uncompressed_data.data(),
+ entry.uncompressed_data.size());
+ entry.has_symbols = entry.trace_type == TraceType::kProtoTraceType &&
+ HasSymbols(entry.uncompressed_data);
+ entries.push_back(std::move(entry));
+ }
+ return std::move(entries);
+}
+
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/importers/zip/zip_trace_reader.h b/src/trace_processor/importers/zip/zip_trace_reader.h
new file mode 100644
index 0000000..d4f3d88
--- /dev/null
+++ b/src/trace_processor/importers/zip/zip_trace_reader.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_
+#define SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_
+
+#include <cstddef>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "perfetto/base/status.h"
+#include "perfetto/ext/base/status_or.h"
+#include "perfetto/trace_processor/trace_blob_view.h"
+#include "src/trace_processor/importers/common/chunked_trace_reader.h"
+#include "src/trace_processor/util/trace_type.h"
+#include "src/trace_processor/util/zip_reader.h"
+
+namespace perfetto::trace_processor {
+
+class ForwardingTraceParser;
+class TraceProcessorContext;
+
+// Forwards files contained in a ZIP to the appropiate ChunkedTraceReader. It is
+// guaranteed that proto traces will be parsed first.
+class ZipTraceReader : public ChunkedTraceReader {
+ public:
+ explicit ZipTraceReader(TraceProcessorContext* context);
+ ~ZipTraceReader() override;
+
+ // ChunkedTraceReader implementation
+ util::Status Parse(TraceBlobView) override;
+ void NotifyEndOfFile() override;
+
+ private:
+ // Represents a file in the ZIP file. Used to sort them before sending the
+ // files one by one to a `ForwardingTraceParser` instance.
+ struct Entry {
+ // File name. Used to break ties.
+ std::string name;
+ // Position in the zip file. Used to break ties.
+ size_t index;
+ // Trace type. This is the main attribute traces are ordered by. Proto
+ // traces are always parsed first as they might contains clock sync
+ // data needed to correctly parse other traces.
+ TraceType trace_type;
+ TraceBlobView uncompressed_data;
+ // True for proto trace_types whose fist message is a ModuleSymbols packet
+ bool has_symbols = false;
+ // Comparator used to determine the order in which files in the ZIP will be
+ // read.
+ bool operator<(const Entry& rhs) const;
+ };
+
+ base::Status NotifyEndOfFileImpl();
+ base::StatusOr<std::vector<Entry>> ExtractEntries(
+ std::vector<util::ZipFile> files) const;
+ base::Status ParseEntry(Entry entry);
+
+ TraceProcessorContext* const context_;
+ util::ZipReader zip_reader_;
+ // For every file in the ZIP we will create a `ForwardingTraceParser`instance
+ // and send that file to it for tokenization. The instances are kept around
+ // here as some tokenizers might keep state that is later needed after
+ // sorting.
+ std::vector<std::unique_ptr<ForwardingTraceParser>> parsers_;
+};
+
+} // namespace perfetto::trace_processor
+
+#endif // SRC_TRACE_PROCESSOR_IMPORTERS_ZIP_ZIP_TRACE_READER_H_
diff --git a/src/trace_processor/metrics/sql/android/android_startup.sql b/src/trace_processor/metrics/sql/android/android_startup.sql
index 95d050b..2e2e48e 100644
--- a/src/trace_processor/metrics/sql/android/android_startup.sql
+++ b/src/trace_processor/metrics/sql/android/android_startup.sql
@@ -514,7 +514,7 @@
)
),
- 'slow_start_reason_detailed', get_slow_start_reason_detailed(launches.startup_id)
+ 'slow_start_reason_with_details', get_slow_start_reason_with_details(launches.startup_id)
) AS startup
FROM android_startups launches;
diff --git a/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql b/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql
index 2a673c0..8108fa6 100644
--- a/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql
+++ b/src/trace_processor/metrics/sql/android/startup/slow_start_reasons.sql
@@ -201,110 +201,130 @@
ORDER BY slice_dur DESC
LIMIT 1;
-
-CREATE OR REPLACE PERFETTO FUNCTION get_slow_start_reason_detailed(startup_id LONG)
+CREATE OR REPLACE PERFETTO FUNCTION get_slow_start_reason_with_details(startup_id LONG)
RETURNS PROTO AS
- SELECT RepeatedField(AndroidStartupMetric_SlowStartReasonDetailed(
+ SELECT RepeatedField(AndroidStartupMetric_SlowStartReason(
+ 'reason_id', reason_id,
'reason', slow_cause,
- 'details', details))
+ 'launch_dur', launch_dur))
FROM (
SELECT 'No baseline or cloud profiles' AS slow_cause,
- get_missing_baseline_profile_for_launch(launch.startup_id, launch.package) as details
+ launch.dur as launch_dur,
+ 'NO_BASELINE_OR_CLOUD_PROFILES' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND missing_baseline_profile_for_launch(launch.startup_id, launch.package)
UNION ALL
- SELECT 'Optimized artifacts missing, run from apk' as slow_cause, NULL as details
+ SELECT 'Optimized artifacts missing, run from apk' as slow_cause,
+ launch.dur as launch_dur,
+ 'RUN_FROM_APK' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
- AND run_from_apk_for_launch(launch.startup_id)
+ AND run_from_apk_for_launch(launch.startup_id)
UNION ALL
- SELECT 'Unlock running during launch' as slow_cause, NULL as details
+ SELECT 'Unlock running during launch' as slow_cause,
+ launch.dur as launch_dur,
+ 'UNLOCK_RUNNING' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND is_unlock_running_during_launch(launch.startup_id)
UNION ALL
- SELECT 'App in debuggable mode' as slow_cause, NULL as details
- FROM android_startups launch
+ SELECT 'App in debuggable mode' as slow_cause,
+ launch.dur as launch_dur,
+ 'APP_IN_DEBUGGABLE_MODE' as reason_id
+ FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND is_process_debuggable(launch.package)
UNION ALL
- SELECT 'GC Activity' as slow_cause, NULL as details
+ SELECT 'GC Activity' as slow_cause,
+ launch.dur as launch_dur,
+ 'GC_ACTIVITY' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND total_gc_time_by_launch(launch.startup_id) > 0
UNION ALL
- SELECT 'dex2oat running during launch' AS slow_cause, NULL as details
+ SELECT 'dex2oat running during launch' AS slow_cause,
+ launch.dur as launch_dur,
+ 'DEX2OAT_RUNNING' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id AND
dur_of_process_running_concurrent_to_launch(launch.startup_id, '*dex2oat64') > 0
UNION ALL
- SELECT 'installd running during launch' AS slow_cause, NULL as details
+ SELECT 'installd running during launch' AS slow_cause,
+ launch.dur as launch_dur,
+ 'INSTALLD_RUNNING' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id AND
dur_of_process_running_concurrent_to_launch(launch.startup_id, '*installd') > 0
UNION ALL
SELECT 'Main Thread - Time spent in Runnable state' as slow_cause,
- get_main_thread_time_for_launch_in_runnable_state(
- launch.startup_id, launch.dur) as details
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_TIME_SPENT_IN_RUNNABLE' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND main_thread_time_for_launch_in_runnable_state(launch.startup_id) > launch.dur * 0.15
UNION ALL
- SELECT 'Main Thread - Time spent in interruptible sleep state'
- AS slow_cause, NULL as details
+ SELECT 'Main Thread - Time spent in interruptible sleep state' as slow_cause,
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_TIME_SPENT_IN_INTERRUPTIBLE_SLEEP' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND main_thread_time_for_launch_and_state(launch.startup_id, 'S') > 2900e6
UNION ALL
- SELECT 'Main Thread - Time spent in Blocking I/O' as slow_cause, NULL as details
+ SELECT 'Main Thread - Time spent in Blocking I/O' as slow_cause,
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_TIME_SPENT_IN_BLOCKING_IO' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND main_thread_time_for_launch_state_and_io_wait(launch.startup_id, 'D*', TRUE) > 450e6
UNION ALL
SELECT 'Main Thread - Time spent in OpenDexFilesFromOat*' as slow_cause,
- get_android_sum_dur_on_main_thread_for_startup_and_slice(
- launch.startup_id, 'OpenDexFilesFromOat*', launch.dur) as details
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_TIME_SPENT_IN_OPEN_DEX_FILES_FROM_OAT' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id AND
android_sum_dur_on_main_thread_for_startup_and_slice(
launch.startup_id, 'OpenDexFilesFromOat*') > launch.dur * 0.2
UNION ALL
- SELECT 'Time spent in bindApplication'
- AS slow_cause, NULL as details
+ SELECT 'Time spent in bindApplication' as slow_cause,
+ launch.dur as launch_dur,
+ 'TIME_SPENT_IN_BIND_APPLICATION' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND android_sum_dur_for_startup_and_slice(launch.startup_id, 'bindApplication') > 1250e6
UNION ALL
- SELECT 'Time spent in view inflation' as slow_cause, NULL as details
+ SELECT 'Time spent in view inflation' as slow_cause,
+ launch.dur as launch_dur,
+ 'TIME_SPENT_IN_VIEW_INFLATION' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND android_sum_dur_for_startup_and_slice(launch.startup_id, 'inflate') > 450e6
UNION ALL
SELECT 'Time spent in ResourcesManager#getResources' as slow_cause,
- get_android_sum_dur_for_startup_and_slice(
- launch.startup_id, 'ResourcesManager#getResources', 130) as details
+ launch.dur as launch_dur,
+ 'TIME_SPENT_IN_RESOURCES_MANAGER_GET_RESOURCES' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND android_sum_dur_for_startup_and_slice(
launch.startup_id, 'ResourcesManager#getResources') > 130e6
UNION ALL
- SELECT 'Time spent verifying classes'
- AS slow_cause, NULL as details
+ SELECT 'Time spent verifying classes' as slow_cause,
+ launch.dur as launch_dur,
+ 'TIME_SPENT_VERIFYING_CLASSES' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id AND
android_sum_dur_for_startup_and_slice(launch.startup_id, 'VerifyClass*')
@@ -312,7 +332,8 @@
UNION ALL
SELECT 'Potential CPU contention with another process' AS slow_cause,
- get_potential_cpu_contention_with_another_process(launch.startup_id) as details
+ launch.dur as launch_dur,
+ 'POTENTIAL_CPU_CONTENTION_WITH_ANOTHER_PROCESS' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id AND
main_thread_time_for_launch_in_runnable_state(launch.startup_id) > 100e6 AND
@@ -320,7 +341,8 @@
UNION ALL
SELECT 'JIT Activity' as slow_cause,
- get_jit_activity(launch.startup_id) as details
+ launch.dur as launch_dur,
+ 'JIT_ACTIVITY' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND thread_time_for_launch_state_and_thread(
@@ -330,8 +352,9 @@
) > 100e6
UNION ALL
- SELECT 'Main Thread - Lock contention'
- AS slow_cause, NULL as details
+ SELECT 'Main Thread - Lock contention' as slow_cause,
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_LOCK_CONTENTION' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND android_sum_dur_on_main_thread_for_startup_and_slice(
@@ -340,8 +363,9 @@
) > launch.dur * 0.2
UNION ALL
- SELECT 'Main Thread - Monitor contention'
- AS slow_cause, NULL as details
+ SELECT 'Main Thread - Monitor contention' as slow_cause,
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_MONITOR_CONTENTION' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND android_sum_dur_on_main_thread_for_startup_and_slice(
@@ -350,51 +374,55 @@
) > launch.dur * 0.15
UNION ALL
- SELECT 'JIT compiled methods' as slow_cause, NULL as details
+ SELECT 'JIT compiled methods' as slow_cause,
+ launch.dur as launch_dur,
+ 'JIT_COMPILED_METHODS' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND (
SELECT COUNT(1)
FROM ANDROID_SLICES_FOR_STARTUP_AND_SLICE_NAME(launch.startup_id, 'JIT compiling*')
- WHERE thread_name = 'Jit thread pool'
- ) > 65
+ WHERE thread_name = 'Jit thread pool') > 65
UNION ALL
- SELECT 'Broadcast dispatched count' as slow_cause, NULL as details
+ SELECT 'Broadcast dispatched count' as slow_cause,
+ launch.dur as launch_dur,
+ 'BROADCAST_DISPATCHED_COUNT' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND count_slices_concurrent_to_launch(
launch.startup_id,
- 'Broadcast dispatched*'
- ) > 15
+ 'Broadcast dispatched*') > 15
UNION ALL
- SELECT 'Broadcast received count' as slow_cause, NULL as details
+ SELECT 'Broadcast received count' as slow_cause,
+ launch.dur as launch_dur,
+ 'BROADCAST_RECEIVED_COUNT' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND count_slices_concurrent_to_launch(
launch.startup_id,
- 'broadcastReceiveReg*'
- ) > 50
+ 'broadcastReceiveReg*') > 50
UNION ALL
- SELECT 'Startup running concurrent to launch' as slow_cause, NULL as details
+ SELECT 'Startup running concurrent to launch' as slow_cause,
+ launch.dur as launch_dur,
+ 'STARTUP_RUNNING_CONCURRENT' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND EXISTS(
SELECT package
FROM android_startups l
WHERE l.startup_id != launch.startup_id
- AND _is_spans_overlapping(l.ts, l.ts_end, launch.ts, launch.ts_end)
- )
+ AND _is_spans_overlapping(l.ts, l.ts_end, launch.ts, launch.ts_end))
UNION ALL
SELECT 'Main Thread - Binder transactions blocked' as slow_cause,
- get_main_thread_binder_transactions_blocked(launch.startup_id, 2e7) as details
+ launch.dur as launch_dur,
+ 'MAIN_THREAD_BINDER_TRANSCATIONS_BLOCKED' as reason_id
FROM android_startups launch
WHERE launch.startup_id = $startup_id
AND (
SELECT COUNT(1)
- FROM BINDER_TRANSACTION_REPLY_SLICES_FOR_LAUNCH(launch.startup_id, 2e7)
- ) > 0
- );
+ FROM BINDER_TRANSACTION_REPLY_SLICES_FOR_LAUNCH(launch.startup_id, 2e7)) > 0
+ );
diff --git a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc
index 8f163a0..3048e68 100644
--- a/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc
+++ b/src/trace_processor/perfetto_sql/engine/perfetto_sql_engine.cc
@@ -32,6 +32,7 @@
#include "perfetto/base/logging.h"
#include "perfetto/base/status.h"
+#include "perfetto/ext/base/flat_hash_map.h"
#include "perfetto/ext/base/status_or.h"
#include "perfetto/ext/base/string_utils.h"
#include "perfetto/ext/base/string_view.h"
@@ -848,7 +849,18 @@
const std::vector<std::string>& column_names,
const std::vector<sql_argument::ArgumentDefinition>& schema,
const char* tag) {
- // If the user has not provided a schema, we have nothing to validate.
+ std::vector<std::string> duplicate_columns;
+ for (auto it = column_names.begin(); it != column_names.end(); ++it) {
+ if (std::count(it + 1, column_names.end(), *it) > 0) {
+ duplicate_columns.push_back(*it);
+ }
+ }
+ if (!duplicate_columns.empty()) {
+ return base::ErrStatus("%s: multiple columns are named: %s", tag,
+ base::Join(duplicate_columns, ", ").c_str());
+ }
+
+ // If the user has not provided a schema, we have nothing further to validate.
if (schema.empty()) {
return base::OkStatus();
}
diff --git a/src/trace_processor/trace_database_integrationtest.cc b/src/trace_processor/trace_database_integrationtest.cc
index d09c8a7..731f4e3 100644
--- a/src/trace_processor/trace_database_integrationtest.cc
+++ b/src/trace_processor/trace_database_integrationtest.cc
@@ -14,17 +14,23 @@
* limitations under the License.
*/
-#include <algorithm>
+#include <cstdint>
#include <cstdio>
-#include <map>
-#include <optional>
+#include <cstring>
+#include <memory>
#include <random>
#include <string>
+#include <utility>
+#include <vector>
+#include "perfetto/base/build_config.h"
#include "perfetto/base/logging.h"
+#include "perfetto/base/status.h"
#include "perfetto/ext/base/scoped_file.h"
#include "perfetto/ext/base/string_utils.h"
#include "perfetto/trace_processor/basic_types.h"
+#include "perfetto/trace_processor/iterator.h"
+#include "perfetto/trace_processor/status.h"
#include "perfetto/trace_processor/trace_processor.h"
#include "protos/perfetto/common/descriptor.pbzero.h"
#include "protos/perfetto/trace_processor/trace_processor.pbzero.h"
@@ -32,11 +38,12 @@
#include "src/base/test/utils.h"
#include "test/gtest_and_gmock.h"
-namespace perfetto {
-namespace trace_processor {
+namespace perfetto::trace_processor {
namespace {
-constexpr size_t kMaxChunkSize = 4 * 1024 * 1024;
+using testing::HasSubstr;
+
+constexpr size_t kMaxChunkSize = 4ul * 1024 * 1024;
TEST(TraceProcessorCustomConfigTest, SkipInternalMetricsMatchingMountPath) {
auto config = Config();
@@ -97,12 +104,13 @@
: processor_(TraceProcessor::CreateInstance(Config())) {}
protected:
- util::Status LoadTrace(const char* name,
+ base::Status LoadTrace(const char* name,
size_t min_chunk_size = 512,
size_t max_chunk_size = kMaxChunkSize) {
EXPECT_LE(min_chunk_size, max_chunk_size);
- base::ScopedFstream f(fopen(
- base::GetTestDataPath(std::string("test/data/") + name).c_str(), "rb"));
+ base::ScopedFstream f(
+ fopen(base::GetTestDataPath(std::string("test/data/") + name).c_str(),
+ "rbe"));
std::minstd_rand0 rnd_engine(0);
std::uniform_int_distribution<size_t> dist(min_chunk_size, max_chunk_size);
while (!feof(*f)) {
@@ -118,7 +126,7 @@
}
Iterator Query(const std::string& query) {
- return processor_->ExecuteQuery(query.c_str());
+ return processor_->ExecuteQuery(query);
}
TraceProcessor* Processor() { return processor_.get(); }
@@ -280,7 +288,7 @@
TEST_F(TraceProcessorIntegrationTest, ComputeMetricsFormattedExtension) {
std::string metric_output;
- util::Status status = Processor()->ComputeMetricText(
+ base::Status status = Processor()->ComputeMetricText(
std::vector<std::string>{"test_chrome_metric"},
TraceProcessor::MetricResultFormat::kProtoText, &metric_output);
ASSERT_TRUE(status.ok());
@@ -293,7 +301,7 @@
TEST_F(TraceProcessorIntegrationTest, ComputeMetricsFormattedNoExtension) {
std::string metric_output;
- util::Status status = Processor()->ComputeMetricText(
+ base::Status status = Processor()->ComputeMetricText(
std::vector<std::string>{"trace_metadata"},
TraceProcessor::MetricResultFormat::kProtoText, &metric_output);
ASSERT_TRUE(status.ok());
@@ -321,21 +329,21 @@
}
TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14762) {
- ASSERT_TRUE(LoadTrace("clusterfuzz_14762", 4096 * 1024).ok());
+ ASSERT_TRUE(LoadTrace("clusterfuzz_14762", 4096ul * 1024).ok());
auto it = Query("select sum(value) from stats where severity = 'error';");
ASSERT_TRUE(it.Next());
ASSERT_GT(it.Get(0).long_value, 0);
}
TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14767) {
- ASSERT_TRUE(LoadTrace("clusterfuzz_14767", 4096 * 1024).ok());
+ ASSERT_TRUE(LoadTrace("clusterfuzz_14767", 4096ul * 1024).ok());
auto it = Query("select sum(value) from stats where severity = 'error';");
ASSERT_TRUE(it.Next());
ASSERT_GT(it.Get(0).long_value, 0);
}
TEST_F(TraceProcessorIntegrationTest, Clusterfuzz14799) {
- ASSERT_TRUE(LoadTrace("clusterfuzz_14799", 4096 * 1024).ok());
+ ASSERT_TRUE(LoadTrace("clusterfuzz_14799", 4096ul * 1024).ok());
auto it = Query("select sum(value) from stats where severity = 'error';");
ASSERT_TRUE(it.Next());
ASSERT_GT(it.Get(0).long_value, 0);
@@ -793,6 +801,15 @@
ASSERT_TRUE(it.Status().ok());
}
+TEST_F(TraceProcessorIntegrationTest, CreateTableDuplicateNames) {
+ auto it = Query(
+ "create perfetto table foo select 1 as duplicate_a, 2 as duplicate_a, 3 "
+ "as duplicate_b, 4 as duplicate_b");
+ ASSERT_FALSE(it.Next());
+ ASSERT_FALSE(it.Status().ok());
+ ASSERT_THAT(it.Status().message(), HasSubstr("duplicate_a"));
+ ASSERT_THAT(it.Status().message(), HasSubstr("duplicate_b"));
+}
+
} // namespace
-} // namespace trace_processor
-} // namespace perfetto
+} // namespace perfetto::trace_processor
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc
index fb9b294..117a737 100644
--- a/src/trace_processor/trace_processor_impl.cc
+++ b/src/trace_processor/trace_processor_impl.cc
@@ -59,6 +59,7 @@
#include "src/trace_processor/importers/proto/additional_modules.h"
#include "src/trace_processor/importers/proto/content_analyzer.h"
#include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
+#include "src/trace_processor/importers/zip/zip_trace_reader.h"
#include "src/trace_processor/iterator_impl.h"
#include "src/trace_processor/metrics/all_chrome_metrics.descriptor.h"
#include "src/trace_processor/metrics/all_webview_metrics.descriptor.h"
@@ -121,6 +122,7 @@
#include "src/trace_processor/util/regex.h"
#include "src/trace_processor/util/sql_modules.h"
#include "src/trace_processor/util/status_macros.h"
+#include "src/trace_processor/util/trace_type.h"
#include "protos/perfetto/common/builtin_clock.pbzero.h"
#include "protos/perfetto/trace/clock_snapshot.pbzero.h"
@@ -308,8 +310,8 @@
return "ctrace";
case kNinjaLogTraceType:
return "ninja_log";
- case kAndroidBugreportTraceType:
- return "android_bugreport";
+ case kZipFile:
+ return "zip";
case kPerfDataTraceType:
return "perf_data";
}
@@ -362,8 +364,7 @@
kGzipTraceType);
context_.reader_registry->RegisterTraceReader<GzipTraceParser>(
kCtraceTraceType);
- context_.reader_registry->RegisterTraceReader<AndroidBugreportParser>(
- kAndroidBugreportTraceType);
+ context_.reader_registry->RegisterTraceReader<ZipTraceReader>(kZipFile);
}
if (json::IsJsonSupported()) {
diff --git a/src/trace_processor/trace_processor_storage_impl.cc b/src/trace_processor/trace_processor_storage_impl.cc
index 32f2452..09d5386 100644
--- a/src/trace_processor/trace_processor_storage_impl.cc
+++ b/src/trace_processor/trace_processor_storage_impl.cc
@@ -107,6 +107,8 @@
return;
Flush();
context_.chunk_reader->NotifyEndOfFile();
+ // NotifyEndOfFile might have pushed packets to the sorter.
+ Flush();
for (std::unique_ptr<ProtoImporterModule>& module : context_.modules) {
module->NotifyEndOfFile();
}
diff --git a/src/trace_processor/trace_reader_registry.cc b/src/trace_processor/trace_reader_registry.cc
index 09c5c7b..065f6f6 100644
--- a/src/trace_processor/trace_reader_registry.cc
+++ b/src/trace_processor/trace_reader_registry.cc
@@ -30,7 +30,7 @@
switch (type) {
case kGzipTraceType:
case kCtraceTraceType:
- case kAndroidBugreportTraceType:
+ case kZipFile:
return true;
case kNinjaLogTraceType:
diff --git a/src/trace_processor/util/trace_type.cc b/src/trace_processor/util/trace_type.cc
index d265b90..ee5a56a 100644
--- a/src/trace_processor/util/trace_type.cc
+++ b/src/trace_processor/util/trace_type.cc
@@ -58,8 +58,8 @@
return "gzip trace";
case kCtraceTraceType:
return "ctrace trace";
- case kAndroidBugreportTraceType:
- return "Android Bugreport";
+ case kZipFile:
+ return "ZIP file";
case kPerfDataTraceType:
return "perf data";
case kUnknownTraceType:
@@ -124,12 +124,8 @@
if (base::StartsWith(start, "\x0a"))
return kProtoTraceType;
- // Android bugreport.zip
- // TODO(primiano). For now we assume any .zip file is a bugreport. In future,
- // if we want to support different trace formats based on a .zip arachive we
- // will need an extra layer similar to what we did kGzipTraceType.
if (base::StartsWith(start, "PK\x03\x04"))
- return kAndroidBugreportTraceType;
+ return kZipFile;
return kUnknownTraceType;
}
diff --git a/src/trace_processor/util/trace_type.h b/src/trace_processor/util/trace_type.h
index fec7a74..2d40d83 100644
--- a/src/trace_processor/util/trace_type.h
+++ b/src/trace_processor/util/trace_type.h
@@ -31,7 +31,7 @@
kGzipTraceType,
kCtraceTraceType,
kNinjaLogTraceType,
- kAndroidBugreportTraceType,
+ kZipFile,
kPerfDataTraceType,
};
diff --git a/src/trace_redaction/redact_sched_switch.cc b/src/trace_redaction/redact_sched_switch.cc
index dc8898d..d2ce751 100644
--- a/src/trace_redaction/redact_sched_switch.cc
+++ b/src/trace_redaction/redact_sched_switch.cc
@@ -106,7 +106,7 @@
// collection of ftrace event messages) because data in a sched_switch message
// is needed in order to know if the event should be added to the bundle.
-SchedSwitchTransform::~SchedSwitchTransform() = default;
+RedactSchedSwitchHarness::Modifier::~Modifier() = default;
base::Status RedactSchedSwitchHarness::Transform(const Context& context,
std::string* packet) const {
@@ -188,14 +188,29 @@
for (auto field = decoder.ReadField(); field.valid();
field = decoder.ReadField()) {
- if (field.id() == protos::pbzero::FtraceEvent::kSchedSwitchFieldNumber) {
- protos::pbzero::SchedSwitchFtraceEvent::Decoder sched_switch(
- field.as_bytes());
- RETURN_IF_ERROR(TransformFtraceEventSchedSwitch(
- context, ts.as_uint64(), cpu, sched_switch, &scratch_str,
- message->set_sched_switch()));
- } else {
- proto_util::AppendField(field, message);
+ switch (field.id()) {
+ case protos::pbzero::FtraceEvent::kSchedSwitchFieldNumber: {
+ protos::pbzero::SchedSwitchFtraceEvent::Decoder sched_switch(
+ field.as_bytes());
+ RETURN_IF_ERROR(TransformFtraceEventSchedSwitch(
+ context, ts.as_uint64(), cpu, sched_switch, &scratch_str,
+ message->set_sched_switch()));
+ break;
+ }
+
+ case protos::pbzero::FtraceEvent::kSchedWakingFieldNumber: {
+ protos::pbzero::SchedWakingFtraceEvent::Decoder sched_waking(
+ field.as_bytes());
+ RETURN_IF_ERROR(TransformFtraceEventSchedWaking(
+ context, ts.as_uint64(), cpu, sched_waking, &scratch_str,
+ message->set_sched_waking()));
+ break;
+ }
+
+ default: {
+ proto_util::AppendField(field, message);
+ break;
+ }
}
}
@@ -209,6 +224,10 @@
protos::pbzero::SchedSwitchFtraceEvent::Decoder& sched_switch,
std::string* scratch_str,
protos::pbzero::SchedSwitchFtraceEvent* message) const {
+ PERFETTO_DCHECK(modifier_);
+ PERFETTO_DCHECK(scratch_str);
+ PERFETTO_DCHECK(message);
+
auto has_fields = {
sched_switch.has_prev_comm(), sched_switch.has_prev_pid(),
sched_switch.has_prev_prio(), sched_switch.has_prev_state(),
@@ -233,10 +252,7 @@
scratch_str->assign(prev_comm.data, prev_comm.size);
- for (const auto& transform : transforms_) {
- RETURN_IF_ERROR(
- transform->Transform(context, ts, cpu, &prev_pid, scratch_str));
- }
+ RETURN_IF_ERROR(modifier_->Modify(context, ts, cpu, &prev_pid, scratch_str));
message->set_prev_comm(*scratch_str); // FieldNumber = 1
message->set_prev_pid(prev_pid); // FieldNumber = 2
@@ -245,10 +261,7 @@
scratch_str->assign(next_comm.data, next_comm.size);
- for (const auto& transform : transforms_) {
- RETURN_IF_ERROR(
- transform->Transform(context, ts, cpu, &next_pid, scratch_str));
- }
+ RETURN_IF_ERROR(modifier_->Modify(context, ts, cpu, &next_pid, scratch_str));
message->set_next_comm(*scratch_str); // FieldNumber = 5
message->set_next_pid(next_pid); // FieldNumber = 6
@@ -257,6 +270,47 @@
return base::OkStatus();
}
+base::Status RedactSchedSwitchHarness::TransformFtraceEventSchedWaking(
+ const Context& context,
+ uint64_t ts,
+ int32_t cpu,
+ protos::pbzero::SchedWakingFtraceEvent::Decoder& sched_waking,
+ std::string* scratch_str,
+ protos::pbzero::SchedWakingFtraceEvent* message) const {
+ PERFETTO_DCHECK(modifier_);
+ PERFETTO_DCHECK(scratch_str);
+ PERFETTO_DCHECK(message);
+
+ auto has_fields = {sched_waking.has_comm(), sched_waking.has_pid(),
+ sched_waking.has_prio(), sched_waking.has_success(),
+ sched_waking.has_target_cpu()};
+
+ if (!std::all_of(has_fields.begin(), has_fields.end(), IsTrue)) {
+ return base::ErrStatus(
+ "RedactSchedSwitchHarness: missing required SchedWakingFtraceEvent "
+ "field.");
+ }
+
+ auto pid = sched_waking.pid();
+ auto comm = sched_waking.comm();
+
+ // There are 5 values in a sched switch message. Since 2 of the 5 can be
+ // replaced, it is easier/cleaner to go value-by-value. Go in proto-defined
+ // order.
+
+ scratch_str->assign(comm.data, comm.size);
+
+ RETURN_IF_ERROR(modifier_->Modify(context, ts, cpu, &pid, scratch_str));
+
+ message->set_comm(*scratch_str); // FieldNumber = 1
+ message->set_pid(pid); // FieldNumber = 2
+ message->set_prio(sched_waking.prio()); // FieldNumber = 3
+ message->set_success(sched_waking.success()); // FieldNumber = 4
+ message->set_target_cpu(sched_waking.target_cpu()); // FieldNumber = 5
+
+ return base::OkStatus();
+}
+
base::Status RedactSchedSwitchHarness::TransformCompSched(
const Context& context,
int32_t cpu,
@@ -314,6 +368,9 @@
protos::pbzero::FtraceEventBundle::CompactSched::Decoder& comp_sched,
InternTable* intern_table,
protos::pbzero::FtraceEventBundle::CompactSched* message) const {
+ PERFETTO_DCHECK(modifier_);
+ PERFETTO_DCHECK(message);
+
auto has_fields = {
comp_sched.has_intern_table(),
comp_sched.has_switch_timestamp(),
@@ -359,9 +416,7 @@
scratch_str.assign(comm);
- for (const auto& transform : transforms_) {
- transform->Transform(context, ts, cpu, &pid, &scratch_str);
- }
+ RETURN_IF_ERROR(modifier_->Modify(context, ts, cpu, &pid, &scratch_str));
auto found = intern_table->Push(scratch_str.data(), scratch_str.size());
@@ -429,11 +484,11 @@
// Switch event transformation: Clear the comm value if the thread/process is
// not part of the target packet.
-base::Status ClearComms::Transform(const Context& context,
- uint64_t ts,
- int32_t,
- int32_t* pid,
- std::string* comm) const {
+base::Status ClearComms::Modify(const Context& context,
+ uint64_t ts,
+ int32_t,
+ int32_t* pid,
+ std::string* comm) const {
PERFETTO_DCHECK(pid);
PERFETTO_DCHECK(comm);
diff --git a/src/trace_redaction/redact_sched_switch.h b/src/trace_redaction/redact_sched_switch.h
index 36d6762..906eb11 100644
--- a/src/trace_redaction/redact_sched_switch.h
+++ b/src/trace_redaction/redact_sched_switch.h
@@ -44,25 +44,28 @@
std::vector<std::string_view> interned_comms_;
};
-class SchedSwitchTransform {
- public:
- virtual ~SchedSwitchTransform();
- virtual base::Status Transform(const Context& context,
- uint64_t ts,
- int32_t cpu,
- int32_t* pid,
- std::string* comm) const = 0;
-};
-
-// Goes through all sched switch events are modifies them.
+// TODO(vaage): Rename this class. When it was first created, it only handled
+// switch events, so having "switch" in the name sense. Now that it is
+// expanding to include waking events, a more general name is needed (e.g.
+// scheduling covers both switch and waking events).
class RedactSchedSwitchHarness : public TransformPrimitive {
public:
+ class Modifier {
+ public:
+ virtual ~Modifier();
+ virtual base::Status Modify(const Context& context,
+ uint64_t ts,
+ int32_t cpu,
+ int32_t* pid,
+ std::string* comm) const = 0;
+ };
+
base::Status Transform(const Context& context,
std::string* packet) const override;
template <class Transform>
void emplace_transform() {
- transforms_.emplace_back(new Transform());
+ modifier_ = std::make_unique<Transform>();
}
private:
@@ -86,6 +89,14 @@
std::string* scratch_str,
protos::pbzero::SchedSwitchFtraceEvent* message) const;
+ base::Status TransformFtraceEventSchedWaking(
+ const Context& context,
+ uint64_t ts,
+ int32_t cpu,
+ protos::pbzero::SchedWakingFtraceEvent::Decoder& sched_waking,
+ std::string* scratch_str,
+ protos::pbzero::SchedWakingFtraceEvent* message) const;
+
base::Status TransformCompSched(
const Context& context,
int32_t cpu,
@@ -99,15 +110,15 @@
InternTable* intern_table,
protos::pbzero::FtraceEventBundle::CompactSched* message) const;
- std::vector<std::unique_ptr<SchedSwitchTransform>> transforms_;
+ std::unique_ptr<Modifier> modifier_;
};
-class ClearComms : public SchedSwitchTransform {
- base::Status Transform(const Context& context,
- uint64_t ts,
- int32_t cpu,
- int32_t* pid,
- std::string* comm) const override;
+class ClearComms : public RedactSchedSwitchHarness::Modifier {
+ base::Status Modify(const Context& context,
+ uint64_t ts,
+ int32_t cpu,
+ int32_t* pid,
+ std::string* comm) const override;
};
} // namespace perfetto::trace_redaction
diff --git a/src/trace_redaction/redact_sched_switch_unittest.cc b/src/trace_redaction/redact_sched_switch_unittest.cc
index e2374d1..cd05794 100644
--- a/src/trace_redaction/redact_sched_switch_unittest.cc
+++ b/src/trace_redaction/redact_sched_switch_unittest.cc
@@ -35,25 +35,33 @@
constexpr int32_t kNoParent = 10;
constexpr int32_t kPidA = 11;
constexpr int32_t kPidB = 12;
+constexpr int32_t kPidC = 13;
constexpr int32_t kCpuA = 0;
+constexpr int32_t kCpuB = 1;
+constexpr int32_t kCpuC = 2;
constexpr uint64_t kFullStep = 1000;
constexpr uint64_t kTimeA = 0;
constexpr uint64_t kTimeB = kFullStep;
+constexpr uint64_t kTimeC = kFullStep * 2;
constexpr auto kCommA = "comm-a";
constexpr auto kCommB = "comm-b";
+constexpr auto kCommC = "comm-c";
constexpr auto kCommNone = "";
-class NegatePid : public SchedSwitchTransform {
+class ChangePidToMax : public RedactSchedSwitchHarness::Modifier {
public:
- base::Status Transform(const Context&,
- uint64_t,
- int32_t,
- int32_t* pid,
- std::string*) const override {
- *pid = -(*pid);
+ base::Status Modify(const Context& context,
+ uint64_t ts,
+ int32_t,
+ int32_t* pid,
+ std::string*) const override {
+ if (!context.timeline->PidConnectsToUid(ts, *pid, *context.package_uid)) {
+ *pid = std::numeric_limits<int32_t>::max();
+ }
+
return base::OkStatus();
}
};
@@ -326,12 +334,13 @@
AddSwitchEvent(kTimeA, kPidA, 0, 0, kCommIndexA);
AddSwitchEvent(kTimeB, kPidB, 0, 0, kCommIndexB);
- context_.package_uid = kUidA; // This value is not needed to use NegatePid.
+ // Because the target is package A, PidA should be remain. PidB should change.
+ context_.package_uid = kUidA;
auto packet_buffer = packet_.SerializeAsString();
RedactSchedSwitchHarness redact;
- redact.emplace_transform<NegatePid>();
+ redact.emplace_transform<ChangePidToMax>();
ASSERT_OK(redact.Transform(context_, &packet_buffer));
@@ -347,8 +356,173 @@
ASSERT_EQ(compact_sched.intern_table_size(), 2);
ASSERT_EQ(compact_sched.switch_next_pid_size(), 2);
- ASSERT_EQ(compact_sched.switch_next_pid().at(0), -kPidA);
- ASSERT_EQ(compact_sched.switch_next_pid().at(1), -kPidB);
+ ASSERT_EQ(compact_sched.switch_next_pid().at(0), kPidA);
+ ASSERT_NE(compact_sched.switch_next_pid().at(1), kPidB);
+}
+
+class RedactSchedWakingFtraceEventTest : public testing::Test {
+ protected:
+ void SetUp() override {
+ // Create a packet where two pids are swapping back-and-forth.
+ auto* bundle = packet_.mutable_ftrace_events();
+ bundle->set_cpu(kCpuA);
+
+ // Pid A wakes up Pid B at time Time B
+ {
+ auto* event = bundle->add_event();
+
+ event->set_timestamp(kTimeB);
+ event->set_pid(kPidA);
+
+ auto* sched_waking = event->mutable_sched_waking();
+ sched_waking->set_comm(kCommB);
+ sched_waking->set_pid(kPidB);
+ sched_waking->set_prio(0);
+ sched_waking->set_success(true);
+ sched_waking->set_target_cpu(kCpuB);
+ }
+
+ // Pid A wakes up Pid C at time Time C.
+ {
+ auto* event = bundle->add_event();
+
+ event->set_timestamp(kTimeC);
+ event->set_pid(kPidA);
+
+ auto* sched_waking = event->mutable_sched_waking();
+ sched_waking->set_comm(kCommC);
+ sched_waking->set_pid(kPidC);
+ sched_waking->set_prio(0);
+ sched_waking->set_success(true);
+ sched_waking->set_target_cpu(kCpuC);
+ }
+
+ // PID A and PID B need to be attached to different packages (UID) so that
+ // its possible to include one but not the other.
+ context_.timeline = std::make_unique<ProcessThreadTimeline>();
+ context_.timeline->Append(
+ ProcessThreadTimeline::Event::Open(kTimeA, kPidA, kNoParent, kUidA));
+ context_.timeline->Append(
+ ProcessThreadTimeline::Event::Open(kTimeA, kPidB, kNoParent, kUidB));
+ context_.timeline->Append(
+ ProcessThreadTimeline::Event::Open(kTimeA, kPidC, kNoParent, kUidC));
+ context_.timeline->Sort();
+ }
+
+ protos::gen::TracePacket packet_;
+ Context context_;
+};
+
+TEST_F(RedactSchedWakingFtraceEventTest, WakeeKeepsCommWhenConnectedToPackage) {
+ RedactSchedSwitchHarness redact;
+ redact.emplace_transform<ClearComms>();
+
+ context_.package_uid = kUidB;
+
+ auto packet_buffer = packet_.SerializeAsString();
+
+ ASSERT_OK(redact.Transform(context_, &packet_buffer));
+
+ protos::gen::TracePacket packet;
+ ASSERT_TRUE(packet.ParseFromString(packet_buffer));
+
+ const auto& bundle = packet.ftrace_events();
+ const auto& events = bundle.event();
+
+ ASSERT_EQ(events.size(), 2u);
+
+ ASSERT_EQ(events.front().sched_waking().comm(), kCommB);
+ ASSERT_EQ(events.back().sched_waking().comm(), kCommNone);
+}
+
+TEST_F(RedactSchedWakingFtraceEventTest,
+ WakeeLosesCommWhenNotConnectedToPackage) {
+ RedactSchedSwitchHarness redact;
+ redact.emplace_transform<ClearComms>();
+
+ context_.package_uid = kUidA;
+
+ auto packet_buffer = packet_.SerializeAsString();
+
+ ASSERT_OK(redact.Transform(context_, &packet_buffer));
+
+ protos::gen::TracePacket packet;
+ ASSERT_TRUE(packet.ParseFromString(packet_buffer));
+
+ const auto& bundle = packet.ftrace_events();
+ const auto& events = bundle.event();
+
+ ASSERT_EQ(events.size(), 2u);
+
+ ASSERT_EQ(events.front().sched_waking().comm(), kCommNone);
+ ASSERT_EQ(events.back().sched_waking().comm(), kCommNone);
+}
+
+TEST_F(RedactSchedWakingFtraceEventTest, WakeeKeepsPidWhenConnectedToPackage) {
+ RedactSchedSwitchHarness redact;
+ redact.emplace_transform<ChangePidToMax>();
+
+ context_.package_uid = kUidB;
+
+ auto packet_buffer = packet_.SerializeAsString();
+
+ ASSERT_OK(redact.Transform(context_, &packet_buffer));
+
+ protos::gen::TracePacket packet;
+ ASSERT_TRUE(packet.ParseFromString(packet_buffer));
+
+ const auto& bundle = packet.ftrace_events();
+ const auto& events = bundle.event();
+
+ ASSERT_EQ(events.size(), 2u);
+
+ ASSERT_EQ(events.front().sched_waking().pid(), kPidB);
+ ASSERT_NE(events.back().sched_waking().pid(), kPidC);
+}
+
+TEST_F(RedactSchedWakingFtraceEventTest,
+ WakeeLosesPidWhenNotConnectedToPackage) {
+ RedactSchedSwitchHarness redact;
+ redact.emplace_transform<ChangePidToMax>();
+
+ context_.package_uid = kUidA;
+
+ auto packet_buffer = packet_.SerializeAsString();
+
+ ASSERT_OK(redact.Transform(context_, &packet_buffer));
+
+ protos::gen::TracePacket packet;
+ ASSERT_TRUE(packet.ParseFromString(packet_buffer));
+
+ const auto& bundle = packet.ftrace_events();
+ const auto& events = bundle.event();
+
+ ASSERT_EQ(events.size(), 2u);
+
+ ASSERT_NE(events.front().sched_waking().pid(), kPidB);
+ ASSERT_NE(events.back().sched_waking().pid(), kPidC);
+}
+
+TEST_F(RedactSchedWakingFtraceEventTest, WakerPidIsLeftUnaffected) {
+ RedactSchedSwitchHarness redact;
+ redact.emplace_transform<ChangePidToMax>();
+
+ context_.package_uid = kUidB;
+
+ auto packet_buffer = packet_.SerializeAsString();
+
+ ASSERT_OK(redact.Transform(context_, &packet_buffer));
+
+ protos::gen::TracePacket packet;
+ ASSERT_TRUE(packet.ParseFromString(packet_buffer));
+
+ const auto& bundle = packet.ftrace_events();
+ const auto& events = bundle.event();
+
+ ASSERT_EQ(events.size(), 2u);
+
+ ASSERT_EQ(events.front().pid(), static_cast<uint32_t>(kPidA));
+ ASSERT_EQ(events.back().pid(), static_cast<uint32_t>(kPidA));
}
} // namespace perfetto::trace_redaction
diff --git a/src/trace_redaction/redact_task_newtask.cc b/src/trace_redaction/redact_task_newtask.cc
index 7a0fbeb..0286df2 100644
--- a/src/trace_redaction/redact_task_newtask.cc
+++ b/src/trace_redaction/redact_task_newtask.cc
@@ -46,7 +46,7 @@
const protos::pbzero::FtraceEventBundle::Decoder& bundle,
protozero::ProtoDecoder& event,
protos::pbzero::FtraceEvent* event_message) const {
- PERFETTO_DCHECK(transform_);
+ PERFETTO_DCHECK(modifier_);
if (!context.package_uid.has_value()) {
return base::ErrStatus("RedactTaskNewTask: missing package uid");
@@ -84,8 +84,8 @@
auto cpu = static_cast<int32_t>(bundle.cpu());
- RETURN_IF_ERROR(transform_->Transform(context, timestamp_field.as_uint64(),
- cpu, &pid, &comm));
+ RETURN_IF_ERROR(modifier_->Modify(context, timestamp_field.as_uint64(), cpu,
+ &pid, &comm));
auto* new_task_message = event_message->set_task_newtask();
new_task_message->set_pid(pid);
diff --git a/src/trace_redaction/redact_task_newtask.h b/src/trace_redaction/redact_task_newtask.h
index 5b589fc..61c8b52 100644
--- a/src/trace_redaction/redact_task_newtask.h
+++ b/src/trace_redaction/redact_task_newtask.h
@@ -36,13 +36,13 @@
protozero::ProtoDecoder& event,
protos::pbzero::FtraceEvent* event_message) const override;
- template <class Transform>
+ template <class Modifier>
void emplace_back() {
- transform_ = std::make_unique<Transform>();
+ modifier_ = std::make_unique<Modifier>();
}
public:
- std::unique_ptr<SchedSwitchTransform> transform_;
+ std::unique_ptr<RedactSchedSwitchHarness::Modifier> modifier_;
};
} // namespace perfetto::trace_redaction
diff --git a/src/traceconv/symbolize_profile.cc b/src/traceconv/symbolize_profile.cc
index 00cc7b9..4efaa10 100644
--- a/src/traceconv/symbolize_profile.cc
+++ b/src/traceconv/symbolize_profile.cc
@@ -55,12 +55,12 @@
PERFETTO_FATAL("Failed to read trace.");
tp->Flush();
+ tp->NotifyEndOfFile();
SymbolizeDatabase(
tp.get(), symbolizer.get(),
[output](const std::string& trace_proto) { *output << trace_proto; });
- tp->NotifyEndOfFile();
return 0;
}
diff --git a/test/.gitignore b/test/.gitignore
index 369c917..47a77bd 100644
--- a/test/.gitignore
+++ b/test/.gitignore
@@ -22,3 +22,7 @@
!/data/simpleperf
/data/simpleperf/*
!/data/simpleperf/*.sha256
+
+!/data/zip
+/data/zip/*
+!/data/zip/*.sha256
\ No newline at end of file
diff --git a/test/data/zip/perf_track_sym.zip.sha256 b/test/data/zip/perf_track_sym.zip.sha256
new file mode 100644
index 0000000..53dfcb3
--- /dev/null
+++ b/test/data/zip/perf_track_sym.zip.sha256
@@ -0,0 +1 @@
+146b1d8f48743323e91fd63d6ab5a1f6d8fcca8c5ea8455ebe8f087667a23c3f
\ No newline at end of file
diff --git a/test/trace_processor/diff_tests/include_index.py b/test/trace_processor/diff_tests/include_index.py
index 33ea8c2..e70dfa3 100644
--- a/test/trace_processor/diff_tests/include_index.py
+++ b/test/trace_processor/diff_tests/include_index.py
@@ -96,6 +96,7 @@
from diff_tests.parser.track_event.tests import TrackEvent
from diff_tests.parser.translated_args.tests import TranslatedArgs
from diff_tests.parser.ufs.tests import Ufs
+from diff_tests.parser.zip.tests import Zip
from diff_tests.stdlib.android.frames_tests import Frames
from diff_tests.stdlib.android.startups_tests import Startups
from diff_tests.stdlib.android.tests import AndroidStdlib
@@ -220,6 +221,7 @@
*FtraceCrop(index_path, 'parser/ftrace', 'FtraceCrop').fetch(),
*ParsingTracedStats(index_path, 'parser/parsing',
'ParsingTracedStats').fetch(),
+ *Zip(index_path, 'parser/zip', 'Zip').fetch(),
]
metrics_tests = [
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup.out b/test/trace_processor/diff_tests/metrics/startup/android_startup.out
index 9dfc289..9928efb 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup.out
@@ -67,10 +67,11 @@
installd_dur_ns: 0
dex2oat_dur_ns: 0
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_RUNNABLE
reason: "Main Thread - Time spent in Runnable state"
- details: " target 15% actual 74.07% [ longest_chunk: start_s 3.0e-08 dur_ms 8.0e-0 thread_id 3 thread_name com.google.android.calendar ] [ extra_info: launches_dur_ms 0.0001 runnable_dur_ms 8.0e-0 R_sum_dur_ms 8.0e-0 R+(Preempted)_sum_dur_ms 0.0 ]"
- }
+ launch_dur: 108
+ }
startup_type: "warm"
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution.out
index 9b6e742..30a1b23 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution.out
@@ -134,14 +134,20 @@
slow_start_reason: "GC Activity"
slow_start_reason: "Main Thread - Time spent in OpenDexFilesFromOat*"
slow_start_reason: "Main Thread - Binder transactions blocked"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: GC_ACTIVITY
reason: "GC Activity"
+ launch_dur: 999999900
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_OPEN_DEX_FILES_FROM_OAT
reason: "Main Thread - Time spent in OpenDexFilesFromOat*"
+ launch_dur: 999999900
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_BINDER_TRANSCATIONS_BLOCKED
reason: "Main Thread - Binder transactions blocked"
+ launch_dur: 999999900
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution_slow.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution_slow.out
index 3e8ff27..cd305f1 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution_slow.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_attribution_slow.out
@@ -98,15 +98,20 @@
slow_start_reason: "GC Activity"
slow_start_reason: "JIT Activity"
slow_start_reason: "JIT compiled methods"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: GC_ACTIVITY
reason: "GC Activity"
+ launch_dur: 999999900000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: JIT_ACTIVITY
reason: "JIT Activity"
- details: " target 100ms actual 20000.ms [ longest_chunk: start_s 154.99 dur_ms 10000. thread_id 4 thread_name Jit thread pool ]"
+ launch_dur: 999999900000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: JIT_COMPILED_METHODS
reason: "JIT compiled methods"
+ launch_dur: 999999900000000000
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown.out
index f40578d..e91c756 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown.out
@@ -119,26 +119,35 @@
slow_start_reason: "Time spent in bindApplication"
slow_start_reason: "Time spent in view inflation"
slow_start_reason: "Time spent in ResourcesManager#getResources"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: NO_BASELINE_OR_CLOUD_PROFILES
reason: "No baseline or cloud profiles"
- details: " target FALSE actual TRUE [ longest_chunk: start_s 154.0 dur_ms 1000.0 thread_id -1 thread_name com.google.android.calendar ] [ extra_info: slice_name location=/system/framework/oat/arm/com.google.android.calendar.odex status=up-to-date filter=speed reason=install-dm ]"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: RUN_FROM_APK
reason: "Optimized artifacts missing, run from apk"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_BIND_APPLICATION
reason: "Time spent in bindApplication"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_VIEW_INFLATION
reason: "Time spent in view inflation"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_RESOURCES_MANAGER_GET_RESOURCES
reason: "Time spent in ResourcesManager#getResources"
- details: " target 130ms actual 1000.0ms [ longest_chunk: start_s 138.0 dur_ms 1000.0 thread_id 3 thread_name com.google.android.calendar ]"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: POTENTIAL_CPU_CONTENTION_WITH_ANOTHER_PROCESS
reason: "Potential CPU contention with another process"
- details: " target 100ms actual 5000.0ms most_active_process_for_launch init [ longest_chunk: start_s 155.0 dur_ms 5000.0 thread_id 3 thread_name com.google.android.calendar ] [ extra_info: runnable_dur_ms 5000.0 R_sum_dur_ms 5000.0 R+(Preempted)_sum_dur 0 ]"
+ launch_dur: 108000000000
}
startup_type: "cold"
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown_slow.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown_slow.out
index 891060b..bb11660 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown_slow.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_breakdown_slow.out
@@ -118,22 +118,30 @@
slow_start_reason: "Time spent in bindApplication"
slow_start_reason: "Time spent in view inflation"
slow_start_reason: "Time spent in ResourcesManager#getResources"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: RUN_FROM_APK
reason: "Optimized artifacts missing, run from apk"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_BIND_APPLICATION
reason: "Time spent in bindApplication"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_VIEW_INFLATION
reason: "Time spent in view inflation"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_RESOURCES_MANAGER_GET_RESOURCES
reason: "Time spent in ResourcesManager#getResources"
- details: " target 130ms actual 5000.0ms [ longest_chunk: start_s 137.0 dur_ms 5000.0 thread_id 3 thread_name com.google.android.calendar ]"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: POTENTIAL_CPU_CONTENTION_WITH_ANOTHER_PROCESS
reason: "Potential CPU contention with another process"
- details: " target 100ms actual 5000.0ms most_active_process_for_launch init [ longest_chunk: start_s 155.0 dur_ms 5000.0 thread_id 3 thread_name com.google.android.calendar ] [ extra_info: runnable_dur_ms 5000.0 R_sum_dur_ms 5000.0 R+(Preempted)_sum_dur 0 ]"
+ launch_dur: 108000000000
}
startup_type: "cold"
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_broadcast_multiple.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_broadcast_multiple.out
index 02e142c..7f7d68c 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_broadcast_multiple.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_broadcast_multiple.out
@@ -31,11 +31,15 @@
}
slow_start_reason: "Broadcast dispatched count"
slow_start_reason: "Broadcast received count"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: BROADCAST_DISPATCHED_COUNT
reason: "Broadcast dispatched count"
+ launch_dur: 100
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: BROADCAST_RECEIVED_COUNT
reason: "Broadcast received count"
+ launch_dur: 100
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat.out
index 1fd7f2e..488e597 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat.out
@@ -62,8 +62,10 @@
dex2oat_dur_ns: 5
}
slow_start_reason: "dex2oat running during launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: DEX2OAT_RUNNING
reason: "dex2oat running during launch"
+ launch_dur: 100
}
}
startup {
@@ -98,8 +100,10 @@
dex2oat_dur_ns: 0
}
slow_start_reason: "installd running during launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: INSTALLD_RUNNING
reason: "installd running during launch"
+ launch_dur: 100
}
}
startup {
@@ -136,11 +140,15 @@
}
slow_start_reason: "dex2oat running during launch"
slow_start_reason: "installd running during launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: DEX2OAT_RUNNING
reason: "dex2oat running during launch"
+ launch_dur: 100
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: INSTALLD_RUNNING
reason: "installd running during launch"
+ launch_dur: 100
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat_slow.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat_slow.out
index 91e0d5a..2cc2b16 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat_slow.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_installd_dex2oat_slow.out
@@ -62,8 +62,10 @@
dex2oat_dur_ns: 5000000000
}
slow_start_reason: "dex2oat running during launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: DEX2OAT_RUNNING
reason: "dex2oat running during launch"
+ launch_dur: 100000000000
}
}
startup {
@@ -101,14 +103,20 @@
slow_start_reason: "dex2oat running during launch"
slow_start_reason: "installd running during launch"
slow_start_reason: "Startup running concurrent to launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: DEX2OAT_RUNNING
reason: "dex2oat running during launch"
+ launch_dur: 250000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: INSTALLD_RUNNING
reason: "installd running during launch"
+ launch_dur: 250000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: STARTUP_RUNNING_CONCURRENT
reason: "Startup running concurrent to launch"
+ launch_dur: 250000000000
}
startup_concurrent_to_launch: "com.google.android.gm"
}
@@ -145,14 +153,20 @@
slow_start_reason: "dex2oat running during launch"
slow_start_reason: "installd running during launch"
slow_start_reason: "Startup running concurrent to launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: DEX2OAT_RUNNING
reason: "dex2oat running during launch"
+ launch_dur: 100000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: INSTALLD_RUNNING
reason: "installd running during launch"
+ launch_dur: 100000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: STARTUP_RUNNING_CONCURRENT
reason: "Startup running concurrent to launch"
+ launch_dur: 100000000000
}
startup_concurrent_to_launch: "com.google.android.deskclock"
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_lock_contention_slow.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_lock_contention_slow.out
index 3e5f6a6..05fc7d1 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_lock_contention_slow.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_lock_contention_slow.out
@@ -72,14 +72,20 @@
slow_start_reason: "Time spent in bindApplication"
slow_start_reason: "Main Thread - Lock contention"
slow_start_reason: "Main Thread - Monitor contention"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: TIME_SPENT_IN_BIND_APPLICATION
reason: "Time spent in bindApplication"
+ launch_dur: 100000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_LOCK_CONTENTION
reason: "Main Thread - Lock contention"
+ launch_dur: 100000000000
}
- slow_start_reason_detailed {
- reason: "Main Thread - Monitor contention"
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_MONITOR_CONTENTION
+ reason: "Main Thread - Monitor contention"
+ launch_dur: 100000000000
}
startup_type: "cold"
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_minsdk33.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_minsdk33.out
index e56ae6f..1ee29d4 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_minsdk33.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_minsdk33.out
@@ -30,8 +30,10 @@
dex2oat_dur_ns: 0
}
slow_start_reason: "App in debuggable mode"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: APP_IN_DEBUGGABLE_MODE
reason: "App in debuggable mode"
+ launch_dur: 100
}
}
startup {
@@ -86,8 +88,10 @@
}
startup_type: "hot"
slow_start_reason: "App in debuggable mode"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: APP_IN_DEBUGGABLE_MODE
reason: "App in debuggable mode"
+ launch_dur: 10
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_process_track.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_process_track.out
index fb10a32..efd407a 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_process_track.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_process_track.out
@@ -66,9 +66,10 @@
dex2oat_dur_ns: 0
}
startup_type: "cold"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_RUNNABLE
reason: "Main Thread - Time spent in Runnable state"
- details: " target 15% actual 57.14% [ longest_chunk: start_s 3.0e-09 dur_ms 4.0e-0 thread_id 3 thread_name com.google.android.calendar ] [ extra_info: launches_dur_ms 7.0e-0 runnable_dur_ms 4.0e-0 R_sum_dur_ms 4.0e-0 R+(Preempted)_sum_dur_ms 0.0 ]"
+ launch_dur: 7
}
}
startup {
@@ -139,9 +140,10 @@
dex2oat_dur_ns: 0
}
startup_type: "cold"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_RUNNABLE
reason: "Main Thread - Time spent in Runnable state"
- details: " target 15% actual 57.14% [ longest_chunk: start_s 1.03e-07 dur_ms 4.0e-0 thread_id 4 thread_name com.google.android.calendar ] [ extra_info: launches_dur_ms 7.0e-0 runnable_dur_ms 4.0e-0 R_sum_dur_ms 4.0e-0 R+(Preempted)_sum_dur_ms 0.0 ]"
+ launch_dur: 7
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_slow.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_slow.out
index 4cdc5e9..1b038e6 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_slow.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_slow.out
@@ -70,19 +70,25 @@
startup_type: "warm"
slow_start_reason: "Main Thread - Time spent in interruptible sleep state"
slow_start_reason: "Main Thread - Time spent in Blocking I/O"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_RUNNABLE
reason: "Main Thread - Time spent in Runnable state"
- details: " target 15% actual 74.07% [ longest_chunk: start_s 30.0 dur_ms 80000. thread_id 3 thread_name com.google.android.calendar ] [ extra_info: launches_dur_ms 108000 runnable_dur_ms 80000. R_sum_dur_ms 80000. R+(Preempted)_sum_dur_ms 0.0 ]"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_INTERRUPTIBLE_SLEEP
reason: "Main Thread - Time spent in interruptible sleep state"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: MAIN_THREAD_TIME_SPENT_IN_BLOCKING_IO
reason: "Main Thread - Time spent in Blocking I/O"
+ launch_dur: 108000000000
}
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: POTENTIAL_CPU_CONTENTION_WITH_ANOTHER_PROCESS
reason: "Potential CPU contention with another process"
- details: " target 100ms actual 80000.ms most_active_process_for_launch init [ longest_chunk: start_s 30.0 dur_ms 80000. thread_id 3 thread_name com.google.android.calendar ] [ extra_info: runnable_dur_ms 80000. R_sum_dur_ms 80000. R+(Preempted)_sum_dur 0 ]"
+ launch_dur: 108000000000
}
}
}
diff --git a/test/trace_processor/diff_tests/metrics/startup/android_startup_unlock.out b/test/trace_processor/diff_tests/metrics/startup/android_startup_unlock.out
index 9686456..cf87a52 100644
--- a/test/trace_processor/diff_tests/metrics/startup/android_startup_unlock.out
+++ b/test/trace_processor/diff_tests/metrics/startup/android_startup_unlock.out
@@ -30,8 +30,10 @@
dex2oat_dur_ns: 0
}
slow_start_reason: "Unlock running during launch"
- slow_start_reason_detailed {
+ slow_start_reason_with_details {
+ reason_id: UNLOCK_RUNNING
reason: "Unlock running during launch"
+ launch_dur: 100
}
}
}
diff --git a/test/trace_processor/diff_tests/parser/simpleperf/clocks_align_test.sql b/test/trace_processor/diff_tests/parser/simpleperf/clocks_align_test.sql
new file mode 100644
index 0000000..f5a65ee
--- /dev/null
+++ b/test/trace_processor/diff_tests/parser/simpleperf/clocks_align_test.sql
@@ -0,0 +1,99 @@
+--
+-- Copyright 2024 The Android Open Source Project
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- https://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+CREATE PERFETTO VIEW perf_sample_in(ts INT, dur INT)
+AS
+SELECT ts, 0 AS dur FROM perf_sample;
+
+CREATE VIRTUAL TABLE span
+USING
+ SPAN_JOIN(perf_sample_in, slice PARTITIONED depth);
+
+CREATE PERFETTO TABLE slice_stack
+AS
+WITH
+ tmp AS (
+ SELECT
+ ts,
+ parent_stack_id,
+ string_AGG(IIF(name = 'Main loop', 'main', name), ',')
+ OVER (
+ PARTITION BY ts
+ ORDER BY depth ASC
+ RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ ) AS stack
+ FROM span
+ )
+SELECT ts, stack FROM tmp WHERE parent_stack_id = 0 ORDER BY TS ASC;
+
+CREATE PERFETTO TABLE perf_stack
+AS
+WITH
+ symbol AS (
+ SELECT
+ id,
+ symbol_set_id,
+ replace(replace(name, '(anonymous namespace)::', ''), '()', '') AS name
+ FROM stack_profile_symbol
+ ),
+ symbol_agg AS (
+ SELECT
+ id,
+ symbol_set_id,
+ string_agg(name, ',')
+ OVER (
+ PARTITION BY symbol_set_id
+ ORDER BY id DESC
+ RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ ) AS name
+ FROM symbol
+ WHERE name IN ('main', 'A', 'B', 'C', 'D', 'E')
+ ),
+ inline AS (
+ SELECT symbol_set_id, name FROM symbol_agg WHERE id = symbol_set_id
+ ),
+ frame AS (
+ SELECT f.id AS frame_id, i.name
+ FROM STACK_PROFILE_FRAME f, inline i
+ USING (symbol_set_id)
+ ),
+ child AS (
+ SELECT
+ s.ts,
+ spc.id,
+ spc.parent_id,
+ name
+ FROM perf_sample s, stack_profile_callsite spc
+ ON (s.callsite_id = spc.id),
+ frame USING (frame_id)
+ UNION ALL
+ SELECT
+ child.ts,
+ parent.id,
+ parent.parent_id,
+ COALESCE(f.name || ',', '') || child.name AS name
+ FROM child, stack_profile_callsite parent
+ ON (child.parent_id = parent.id)
+ LEFT JOIN frame f
+ USING (frame_id)
+ )
+SELECT ts, name AS stack FROM child WHERE parent_id IS NULL ORDER BY ts ASC;
+
+SELECT COUNT(*) AS misaligned_count
+FROM slice_stack s
+FULL JOIN perf_stack p
+ USING (ts)
+WHERE s.stack <> p.stack;
diff --git a/test/trace_processor/diff_tests/parser/simpleperf/tests.py b/test/trace_processor/diff_tests/parser/simpleperf/tests.py
index c98239d..4f5a22b 100644
--- a/test/trace_processor/diff_tests/parser/simpleperf/tests.py
+++ b/test/trace_processor/diff_tests/parser/simpleperf/tests.py
@@ -142,3 +142,12 @@
"0b12a384a9f4a3f3659b7171ca615dbec3a81f71","/t1"
"0b12a384a9f4a3f3659b7171ca615dbec3a81f71","/t2"
'''))
+
+ def test_clocks_align(self):
+ return DiffTestBlueprint(
+ trace=DataPath('zip/perf_track_sym.zip'),
+ query=Path('clocks_align_test.sql'),
+ out=Csv('''
+ "misaligned_count"
+ 0
+ '''))
\ No newline at end of file
diff --git a/test/trace_processor/diff_tests/parser/zip/stacks_test.sql b/test/trace_processor/diff_tests/parser/zip/stacks_test.sql
new file mode 100644
index 0000000..68543f1
--- /dev/null
+++ b/test/trace_processor/diff_tests/parser/zip/stacks_test.sql
@@ -0,0 +1,48 @@
+WITH
+ symbol AS (
+ SELECT
+ id,
+ symbol_set_id,
+ replace(replace(name, '(anonymous namespace)::', ''), '()', '') AS name
+ FROM stack_profile_symbol
+ ),
+ symbol_agg AS (
+ SELECT
+ id,
+ symbol_set_id,
+ string_agg(name, ',')
+ OVER (
+ PARTITION BY symbol_set_id
+ ORDER BY id DESC
+ RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+ ) AS name
+ FROM symbol
+ WHERE name IN ('main', 'A', 'B', 'C', 'D', 'E')
+ ),
+ inline AS (
+ SELECT symbol_set_id, name FROM symbol_agg WHERE id = symbol_set_id
+ ),
+ frame AS (
+ SELECT f.id AS frame_id, i.name
+ FROM STACK_PROFILE_FRAME f, inline i
+ USING (symbol_set_id)
+ ),
+ child AS (
+ SELECT
+ spc.id,
+ spc.parent_id,
+ name
+ FROM perf_sample s, stack_profile_callsite spc
+ ON (s.callsite_id = spc.id),
+ frame USING (frame_id)
+ UNION ALL
+ SELECT
+ parent.id,
+ parent.parent_id,
+ COALESCE(f.name || ',', '') || child.name AS name
+ FROM child, stack_profile_callsite parent
+ ON (child.parent_id = parent.id)
+ LEFT JOIN frame f
+ USING (frame_id)
+ )
+SELECT DISTINCT name FROM child WHERE parent_id IS NULL ORDER BY name
\ No newline at end of file
diff --git a/test/trace_processor/diff_tests/parser/zip/tests.py b/test/trace_processor/diff_tests/parser/zip/tests.py
new file mode 100644
index 0000000..9827072
--- /dev/null
+++ b/test/trace_processor/diff_tests/parser/zip/tests.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License a
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from python.generators.diff_tests.testing import Csv, Path, DataPath
+from python.generators.diff_tests.testing import DiffTestBlueprint
+from python.generators.diff_tests.testing import TestSuite
+
+
+class Zip(TestSuite):
+
+ def test_perf_proto_sym(self):
+ return DiffTestBlueprint(
+ trace=DataPath('zip/perf_track_sym.zip'),
+ query=Path('stacks_test.sql'),
+ out=Csv('''
+ "name"
+ "main,A"
+ "main,A,B"
+ "main,A,B,C"
+ "main,A,B,C,D"
+ "main,A,B,C,D,E"
+ "main,A,B,C,E"
+ "main,A,B,D"
+ "main,A,B,D,E"
+ "main,A,B,E"
+ "main,A,C"
+ "main,A,C,D"
+ "main,A,C,D,E"
+ "main,A,C,E"
+ "main,A,D"
+ "main,A,D,E"
+ "main,A,E"
+ "main,B"
+ "main,B,C"
+ "main,B,C,D"
+ "main,B,C,D,E"
+ "main,B,C,E"
+ "main,B,D"
+ "main,B,D,E"
+ "main,B,E"
+ "main,C"
+ "main,C,D"
+ "main,C,D,E"
+ "main,C,E"
+ "main,D"
+ "main,D,E"
+ "main,E"
+ '''))
diff --git a/ui/src/core/default_plugins.ts b/ui/src/core/default_plugins.ts
index 4f5460e..44b4e21 100644
--- a/ui/src/core/default_plugins.ts
+++ b/ui/src/core/default_plugins.ts
@@ -58,4 +58,5 @@
'perfetto.VisualisedArgs',
'org.kernel.LinuxKernelDevices',
'perfetto.TrackUtils',
+ 'com.google.PixelMemory',
];
diff --git a/ui/src/plugins/com.google.PixelMemory/OWNERS b/ui/src/plugins/com.google.PixelMemory/OWNERS
new file mode 100644
index 0000000..249152a
--- /dev/null
+++ b/ui/src/plugins/com.google.PixelMemory/OWNERS
@@ -0,0 +1 @@
+liumartin@google.com
diff --git a/ui/src/plugins/com.google.PixelMemory/index.ts b/ui/src/plugins/com.google.PixelMemory/index.ts
new file mode 100644
index 0000000..c79faf2
--- /dev/null
+++ b/ui/src/plugins/com.google.PixelMemory/index.ts
@@ -0,0 +1,63 @@
+// Copyright (C) 2024 The Android Open Source Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {Plugin, PluginContextTrace, PluginDescriptor} from '../../public';
+
+import {addDebugCounterTrack} from '../../frontend/debug_tracks';
+
+class PixelMemory implements Plugin {
+ async onTraceLoad(ctx: PluginContextTrace): Promise<void> {
+ ctx.registerCommand({
+ id: 'dev.perfetto.PixelMemory#ShowTotalMemory',
+ name: 'Add tracks: show a process total memory',
+ callback: async (pid) => {
+ if (pid === undefined) {
+ pid = prompt('Enter a process pid', '');
+ if (pid === null) return;
+ }
+ const RSS_ALL = `
+ INCLUDE PERFETTO MODULE memory.linux.process;
+ INCLUDE PERFETTO MODULE memory.android.gpu;
+
+ DROP TABLE IF EXISTS process_mem_rss_anon_file_shmem_swap_gpu;
+
+ CREATE VIRTUAL TABLE process_mem_rss_anon_file_shmem_swap_gpu
+ USING
+ SPAN_OUTER_JOIN(
+ memory_gpu_per_process PARTITIONED upid, memory_rss_and_swap_per_process PARTITIONED upid);
+ `;
+ await ctx.engine.query(RSS_ALL);
+ await addDebugCounterTrack(
+ {
+ sqlSource: `
+ SELECT
+ ts,
+ COALESCE(rss_and_swap, 0) + COALESCE(gpu_memory, 0) AS value
+ FROM process_mem_rss_anon_file_shmem_swap_gpu
+ WHERE pid = ${pid}
+ `,
+ columns: ['ts', 'value'],
+ },
+ pid + '_rss_anon_file_swap_shmem_gpu',
+ {ts: 'ts', value: 'value'},
+ );
+ },
+ });
+ }
+}
+
+export const plugin: PluginDescriptor = {
+ pluginId: 'com.google.PixelMemory',
+ plugin: PixelMemory,
+};
