Merge "CI: remove android-arm-asan bot" into main
diff --git a/infra/ui.perfetto.dev/appengine/main.py b/infra/ui.perfetto.dev/appengine/main.py
index 0fac701..acca00d 100644
--- a/infra/ui.perfetto.dev/appengine/main.py
+++ b/infra/ui.perfetto.dev/appengine/main.py
@@ -19,8 +19,9 @@
REQ_HEADERS = [
'Accept',
- 'Accept-Encoding',
- 'Cache-Control',
+ # TODO(primiano): re-enable once the gzip handling outage fixed.
+ # 'Accept-Encoding',
+ # 'Cache-Control',
]
RESP_HEADERS = [
diff --git a/infra/ui.perfetto.dev/appengine/requirements.txt b/infra/ui.perfetto.dev/appengine/requirements.txt
index 1faca3d..9aa5a27 100644
--- a/infra/ui.perfetto.dev/appengine/requirements.txt
+++ b/infra/ui.perfetto.dev/appengine/requirements.txt
@@ -1,2 +1,3 @@
-Flask==1.1.2
+Flask==2.2.5
+Jinja2==3.0.3
requests
diff --git a/src/trace_processor/db/column.h b/src/trace_processor/db/column.h
index 262b346..d6e21c5 100644
--- a/src/trace_processor/db/column.h
+++ b/src/trace_processor/db/column.h
@@ -207,7 +207,7 @@
// Creates a Column which returns the index as the value of the row.
static ColumnLegacy IdColumn(uint32_t col_idx_in_table,
- uint32_t row_map_idx,
+ uint32_t overlay_idx,
const char* name = "id",
uint32_t flags = kIdFlags);
diff --git a/src/trace_processor/db/runtime_table.cc b/src/trace_processor/db/runtime_table.cc
index 38938b8..0de15c3 100644
--- a/src/trace_processor/db/runtime_table.cc
+++ b/src/trace_processor/db/runtime_table.cc
@@ -19,25 +19,34 @@
#include <algorithm>
#include <cinttypes>
#include <cstdint>
+#include <functional>
+#include <limits>
#include <memory>
#include <optional>
+#include <set>
#include <string>
#include <utility>
#include <variant>
#include <vector>
+#include "perfetto/base/compiler.h"
#include "perfetto/base/logging.h"
#include "perfetto/base/status.h"
#include "perfetto/ext/base/status_or.h"
+#include "perfetto/trace_processor/iterator.h"
#include "perfetto/trace_processor/ref_counted.h"
+#include "src/trace_processor/containers/bit_vector.h"
#include "src/trace_processor/containers/string_pool.h"
#include "src/trace_processor/db/column.h"
#include "src/trace_processor/db/column/data_layer.h"
#include "src/trace_processor/db/column/id_storage.h"
#include "src/trace_processor/db/column/null_overlay.h"
#include "src/trace_processor/db/column/numeric_storage.h"
+#include "src/trace_processor/db/column/range_overlay.h"
+#include "src/trace_processor/db/column/selector_overlay.h"
#include "src/trace_processor/db/column/string_storage.h"
#include "src/trace_processor/db/column/types.h"
+#include "src/trace_processor/db/column_storage.h"
#include "src/trace_processor/db/column_storage_overlay.h"
namespace perfetto::trace_processor {
@@ -62,6 +71,60 @@
std::get_if<RuntimeTable::DoubleStorage>(&col) == nullptr;
}
+void CreateNonNullableIntsColumn(
+ uint32_t col_idx,
+ const char* col_name,
+ ColumnStorage<int64_t>* ints_storage,
+ std::vector<RefPtr<column::DataLayer>>& storage_layers,
+ std::vector<RefPtr<column::DataLayer>>& overlay_layers,
+ std::vector<ColumnLegacy>& legacy_columns,
+ std::vector<ColumnStorageOverlay>& legacy_overlays) {
+ const std::vector<int64_t>& values = ints_storage->vector();
+
+ // Looking for the iterator to the first value that is less or equal to the
+ // previous value. The values before are therefore strictly monotonic - each
+ // is greater than the previous one.
+ bool is_monotonic = true;
+ bool is_sorted = true;
+ for (uint32_t i = 1; i < values.size() && is_sorted; i++) {
+ is_monotonic = is_monotonic && values[i - 1] < values[i];
+ is_sorted = values[i - 1] <= values[i];
+ }
+
+ // The special treatement for Id columns makes no sense for empty or
+ // single element indices. Those should be treated as standard int
+ // column.
+ // We are checking if the first value is not too big - we will later
+ // create a BitVector which will skip all of the bits until the front
+ // of the index vector, so we are creating a cutoff to prevent
+ // unreasonable memory usage.
+ bool is_id = is_monotonic && values.size() > 1 && values.front() < 1 << 20 &&
+ values.front() >= std::numeric_limits<uint32_t>::min() &&
+ values.back() < std::numeric_limits<uint32_t>::max();
+
+ if (is_id) {
+ // The column is an Id column.
+ storage_layers[col_idx].reset(new column::IdStorage());
+
+ legacy_overlays.emplace_back(BitVector::FromSortedIndexVector(values));
+ overlay_layers.emplace_back().reset(new column::SelectorOverlay(
+ legacy_overlays.back().row_map().GetIfBitVector()));
+
+ legacy_columns.push_back(ColumnLegacy::IdColumn(
+ col_idx, static_cast<uint32_t>(legacy_overlays.size() - 1), col_name,
+ ColumnLegacy::kIdFlags));
+ return;
+ }
+
+ uint32_t flags =
+ is_sorted ? ColumnLegacy::Flag::kNonNull | ColumnLegacy::Flag::kSorted
+ : ColumnLegacy::Flag::kNonNull;
+
+ legacy_columns.emplace_back(col_name, ints_storage, flags, col_idx, 0);
+ storage_layers[col_idx].reset(new column::NumericStorage<int64_t>(
+ &values, ColumnType::kInt64, is_sorted));
+}
+
} // namespace
RuntimeTable::RuntimeTable(
@@ -179,7 +242,19 @@
uint32_t rows) && {
std::vector<RefPtr<column::DataLayer>> storage_layers(col_names_.size() + 1);
std::vector<RefPtr<column::DataLayer>> null_layers(col_names_.size() + 1);
- std::vector<ColumnLegacy> columns;
+
+ std::vector<ColumnLegacy> legacy_columns;
+ std::vector<ColumnStorageOverlay> legacy_overlays;
+
+ // |overlay_layers| might use the RowMaps used by |legacy_overlays| and access
+ // them by fetching the pointer to the RowMap inside overlay. We need to make
+ // sure that those pointers will not change, hence we need to make sure that
+ // the vector will not resize. In the current implementation there is at most
+ // one overlay per column.
+ legacy_overlays.reserve(col_names_.size() + 1);
+ legacy_overlays.emplace_back(rows);
+ std::vector<RefPtr<column::DataLayer>> overlay_layers(1);
+
for (uint32_t i = 0; i < col_names_.size(); ++i) {
auto* col = storage_[i].get();
std::unique_ptr<column::DataLayerChain> chain;
@@ -188,38 +263,34 @@
PERFETTO_CHECK(*leading_nulls == rows);
*col = Fill<NullIntStorage>(*leading_nulls, std::nullopt);
}
+
if (auto* ints = std::get_if<NullIntStorage>(col)) {
+ // The `ints` column
PERFETTO_CHECK(ints->size() == rows);
- // Check if the column is nullable.
+
if (ints->non_null_size() == ints->size()) {
+ // The column doesn't have any nulls so we construct a new nonnullable
+ // column.
*col = IntStorage::CreateFromAssertNonNull(std::move(*ints));
- auto* non_null_ints = std::get_if<IntStorage>(col);
- bool is_sorted = std::is_sorted(non_null_ints->vector().begin(),
- non_null_ints->vector().end());
- uint32_t flags = is_sorted ? ColumnLegacy::Flag::kNonNull |
- ColumnLegacy::Flag::kSorted
- : ColumnLegacy::Flag::kNonNull;
- columns.emplace_back(col_names_[i].c_str(), non_null_ints, flags, i, 0);
- storage_layers[i].reset(new column::NumericStorage<int64_t>(
- &non_null_ints->vector(), ColumnType::kInt64, is_sorted));
+ CreateNonNullableIntsColumn(
+ i, col_names_[i].c_str(), std::get_if<IntStorage>(col),
+ storage_layers, overlay_layers, legacy_columns, legacy_overlays);
} else {
- columns.emplace_back(col_names_[i].c_str(), ints,
- ColumnLegacy::Flag::kNoFlag, i, 0);
+ // Nullable ints column.
+ legacy_columns.emplace_back(col_names_[i].c_str(), ints,
+ ColumnLegacy::Flag::kNoFlag, i, 0);
storage_layers[i].reset(new column::NumericStorage<int64_t>(
&ints->non_null_vector(), ColumnType::kInt64, false));
null_layers[i].reset(
new column::NullOverlay(&ints->non_null_bit_vector()));
}
- } else if (auto* strings = std::get_if<StringStorage>(col)) {
- PERFETTO_CHECK(strings->size() == rows);
- columns.emplace_back(col_names_[i].c_str(), strings,
- ColumnLegacy::Flag::kNonNull, i, 0);
- storage_layers[i].reset(
- new column::StringStorage(string_pool_, &strings->vector()));
+
+ // The doubles column.
} else if (auto* doubles = std::get_if<NullDoubleStorage>(col)) {
PERFETTO_CHECK(doubles->size() == rows);
- // Check if the column is nullable.
+
if (doubles->non_null_size() == doubles->size()) {
+ // The column is not nullable.
*col = DoubleStorage::CreateFromAssertNonNull(std::move(*doubles));
auto* non_null_doubles = std::get_if<DoubleStorage>(col);
@@ -228,34 +299,40 @@
uint32_t flags = is_sorted ? ColumnLegacy::Flag::kNonNull |
ColumnLegacy::Flag::kSorted
: ColumnLegacy::Flag::kNonNull;
- columns.emplace_back(col_names_[i].c_str(), non_null_doubles, flags, i,
- 0);
+ legacy_columns.emplace_back(col_names_[i].c_str(), non_null_doubles,
+ flags, i, 0);
storage_layers[i].reset(new column::NumericStorage<double>(
&non_null_doubles->vector(), ColumnType::kDouble, is_sorted));
+
} else {
- columns.emplace_back(col_names_[i].c_str(), doubles,
- ColumnLegacy::Flag::kNoFlag, i, 0);
+ // The column is nullable.
+ legacy_columns.emplace_back(col_names_[i].c_str(), doubles,
+ ColumnLegacy::Flag::kNoFlag, i, 0);
storage_layers[i].reset(new column::NumericStorage<double>(
&doubles->non_null_vector(), ColumnType::kDouble, false));
null_layers[i].reset(
new column::NullOverlay(&doubles->non_null_bit_vector()));
}
+
+ } else if (auto* strings = std::get_if<StringStorage>(col)) {
+ // The `strings` column.
+ PERFETTO_CHECK(strings->size() == rows);
+ legacy_columns.emplace_back(col_names_[i].c_str(), strings,
+ ColumnLegacy::Flag::kNonNull, i, 0);
+ storage_layers[i].reset(
+ new column::StringStorage(string_pool_, &strings->vector()));
+
} else {
PERFETTO_FATAL("Unexpected column type");
}
}
- columns.push_back(ColumnLegacy::IdColumn(
- static_cast<uint32_t>(columns.size()), 0, "_auto_id",
+ legacy_columns.push_back(ColumnLegacy::IdColumn(
+ static_cast<uint32_t>(legacy_columns.size()), 0, "_auto_id",
ColumnLegacy::kIdFlags | ColumnLegacy::Flag::kHidden));
storage_layers.back().reset(new column::IdStorage());
- std::vector<ColumnStorageOverlay> overlays;
- overlays.emplace_back(rows);
-
- std::vector<RefPtr<column::DataLayer>> overlay_layers(1);
-
auto table = std::make_unique<RuntimeTable>(
- string_pool_, rows, std::move(columns), std::move(overlays),
+ string_pool_, rows, std::move(legacy_columns), std::move(legacy_overlays),
std::move(storage_layers), std::move(null_layers),
std::move(overlay_layers));
table->storage_ = std::move(storage_);
diff --git a/src/trace_processor/metrics/sql/android/android_boot.sql b/src/trace_processor/metrics/sql/android/android_boot.sql
index a81f28a..1a9adba 100644
--- a/src/trace_processor/metrics/sql/android/android_boot.sql
+++ b/src/trace_processor/metrics/sql/android/android_boot.sql
@@ -56,7 +56,7 @@
'full_trace_process_start_aggregation', (
SELECT NULL_IF_EMPTY(AndroidBootMetric_ProcessStartAggregation(
'total_start_sum', (SELECT SUM(total_dur) FROM android_app_process_starts),
- 'num_of_processes', (SELECT COUNT(process_name) FROM android_app_process_starts GROUP BY process_name),
+ 'num_of_processes', (SELECT COUNT(*) FROM android_app_process_starts),
'average_start_time', (SELECT AVG(total_dur) FROM android_app_process_starts)))
FROM android_app_process_starts),
'post_boot_process_start_aggregation', (
@@ -66,11 +66,10 @@
FROM thread_slice WHERE name GLOB "*android.intent.action.USER_UNLOCKED*"
ORDER BY ts ASC LIMIT 1 )
),
- 'num_of_processes', (SELECT COUNT(process_name) FROM android_app_process_starts
+ 'num_of_processes', (SELECT COUNT(*) FROM android_app_process_starts
WHERE proc_start_ts > (SELECT COALESCE(MIN(ts), 0) FROM thread_slice
WHERE name GLOB "*android.intent.action.USER_UNLOCKED*" ORDER BY ts
ASC LIMIT 1 )
- GROUP BY process_name
),
'average_start_time', (SELECT AVG(total_dur) FROM android_app_process_starts
WHERE proc_start_ts > (SELECT COALESCE(MIN(ts), 0) FROM thread_slice
diff --git a/src/trace_redaction/trace_redactor_integrationtest.cc b/src/trace_redaction/trace_redactor_integrationtest.cc
index 5bc962a..8e10095 100644
--- a/src/trace_redaction/trace_redactor_integrationtest.cc
+++ b/src/trace_redaction/trace_redactor_integrationtest.cc
@@ -34,14 +34,12 @@
using TracePacket = protos::pbzero::TracePacket;
constexpr std::string_view kTracePath =
- "test/data/trace_redaction_jank_high_cpu.pftrace";
+ "test/data/trace-redaction-general.pftrace";
-// "com.google.android.settings.intelligence" will have one package, but two
-// processes will reference it. When doing so, they will use two different
-// uids (multiples of 1,000,000).
constexpr std::string_view kPackageName =
- "com.google.android.settings.intelligence";
-constexpr uint64_t kPackageUid = 10118;
+ "com.Unity.com.unity.multiplayer.samples.coop";
+
+constexpr uint64_t kPackageUid = 10252;
class TraceRedactorIntegrationTest : public testing::Test {
public:
@@ -58,13 +56,30 @@
const std::string& dest_trace() const { return dest_trace_->path(); }
+ std::vector<protozero::ConstBytes> GetPackageInfos(
+ const Trace::Decoder& trace) const {
+ std::vector<protozero::ConstBytes> infos;
+
+ for (auto packet_it = trace.packet(); packet_it; ++packet_it) {
+ TracePacket::Decoder packet_decoder(*packet_it);
+ if (packet_decoder.has_packages_list()) {
+ PackagesList::Decoder list_it(packet_decoder.packages_list());
+ for (auto info_it = list_it.packages(); info_it; ++info_it) {
+ PackageInfo::Decoder info(*info_it);
+ infos.push_back(*info_it);
+ }
+ }
+ }
+
+ return infos;
+ }
+
private:
std::string src_trace_;
std::unique_ptr<base::TempFile> dest_trace_;
};
-TEST_F(TraceRedactorIntegrationTest,
- DISABLED_FindsPackageAndFiltersPackageList) {
+TEST_F(TraceRedactorIntegrationTest, FindsPackageAndFiltersPackageList) {
TraceRedactor redaction;
redaction.collectors()->emplace_back(new FindPackageUid());
redaction.transformers()->emplace_back(new PrunePackageList());
@@ -79,33 +94,27 @@
std::string redacted_buffer;
ASSERT_TRUE(base::ReadFile(dest_trace(), &redacted_buffer));
- // Collect package info from the trace.
- std::vector<protozero::ConstBytes> infos;
+ Trace::Decoder redacted_trace(redacted_buffer);
+ std::vector<protozero::ConstBytes> infos = GetPackageInfos(redacted_trace);
- Trace::Decoder trace_decoder(redacted_buffer);
+ // It is possible for two packages_list to appear in the trace. The
+ // find_package_uid will stop after the first one is found. Package uids are
+ // appear as n * 1,000,000 where n is some integer. It is also possible for
+ // two packages_list to contain copies of each other - for example
+ // "com.Unity.com.unity.multiplayer.samples.coop" appears in both
+ // packages_list.
+ ASSERT_GE(infos.size(), 1u);
- for (auto packet_it = trace_decoder.packet(); packet_it; ++packet_it) {
- TracePacket::Decoder packet_decoder(*packet_it);
+ for (const auto& info_buffer : infos) {
+ PackageInfo::Decoder info(info_buffer);
- if (packet_decoder.has_packages_list()) {
- PackagesList::Decoder list_it(packet_decoder.packages_list());
+ ASSERT_TRUE(info.has_name());
+ ASSERT_EQ(info.name().ToStdString(), kPackageName);
- for (auto info_it = list_it.packages(); info_it; ++info_it) {
- infos.push_back(*info_it);
- }
- }
+ ASSERT_TRUE(info.has_uid());
+ ASSERT_EQ(NormalizeUid(info.uid()), NormalizeUid(kPackageUid));
}
- ASSERT_EQ(infos.size(), 1u);
-
- PackageInfo::Decoder info(infos[0]);
-
- ASSERT_TRUE(info.has_name());
- ASSERT_EQ(info.name().ToStdString(), kPackageName);
-
- ASSERT_TRUE(info.has_uid());
- ASSERT_EQ(NormalizeUid(info.uid()), NormalizeUid(kPackageUid));
-
ASSERT_TRUE(context.package_uid.has_value());
ASSERT_EQ(NormalizeUid(context.package_uid.value()),
NormalizeUid(kPackageUid));
diff --git a/test/data/trace-redaction-general.pftrace.sha256 b/test/data/trace-redaction-general.pftrace.sha256
new file mode 100644
index 0000000..9e7d405
--- /dev/null
+++ b/test/data/trace-redaction-general.pftrace.sha256
@@ -0,0 +1 @@
+dd5f14e36a23158cf740e146cbdd57816056c70ed73b6038a5f317dab28477fe
\ No newline at end of file
diff --git a/test/trace_processor/diff_tests/metrics/android/tests.py b/test/trace_processor/diff_tests/metrics/android/tests.py
index 4f15081..bb7dcbc 100644
--- a/test/trace_processor/diff_tests/metrics/android/tests.py
+++ b/test/trace_processor/diff_tests/metrics/android/tests.py
@@ -213,12 +213,12 @@
}
full_trace_process_start_aggregation {
total_start_sum: 10678297679
- num_of_processes: 1
+ num_of_processes: 29
average_start_time: 368217161.3448276
}
post_boot_process_start_aggregation {
total_start_sum: 6112984648
- num_of_processes: 1
+ num_of_processes: 21
average_start_time: 291094507.04761904
}
}
diff --git a/test/trace_processor/diff_tests/syntax/table_tests.py b/test/trace_processor/diff_tests/syntax/table_tests.py
index 0f56881..10ec9ab 100644
--- a/test/trace_processor/diff_tests/syntax/table_tests.py
+++ b/test/trace_processor/diff_tests/syntax/table_tests.py
@@ -130,3 +130,19 @@
"sorted"
1
"""))
+
+ def test_create_perfetto_table_id_column(self):
+ return DiffTestBlueprint(
+ trace=DataPath('android_boot.pftrace'),
+ query="""
+ CREATE PERFETTO TABLE foo AS
+ SELECT dur FROM slice WHERE dur != -1
+ GROUP BY 1 ORDER BY 1;
+
+ SELECT col_type FROM perfetto_table_info('foo')
+ WHERE name = 'dur';
+ """,
+ out=Csv("""
+ "col_type"
+ "id"
+ """))
diff --git a/ui/release/build_all_channels.py b/ui/release/build_all_channels.py
index 9aa9c33..5549421 100755
--- a/ui/release/build_all_channels.py
+++ b/ui/release/build_all_channels.py
@@ -26,6 +26,7 @@
import sys
from os.path import dirname
+
pjoin = os.path.join
BUCKET_NAME = 'ui.perfetto.dev'
@@ -127,10 +128,10 @@
print('===================================================================')
print('Uploading to gs://%s' % BUCKET_NAME)
print('===================================================================')
- cp_cmd = [
- 'gsutil', '-m', '-h', 'Cache-Control:public, max-age=3600', 'cp', '-z',
- 'html,js,css,wasm,map'
- ]
+ # TODO(primiano): re-enable caching once the gzip-related outage is restored.
+ # cache_hdr = 'Cache-Control:public, max-age=3600'
+ cache_hdr = 'Cache-Control:no-cache'
+ cp_cmd = ['gsutil', '-m', '-h', cache_hdr, 'cp', '-j', 'html,js,css,wasm,map']
for name in os.listdir(merged_dist_dir):
path = pjoin(merged_dist_dir, name)
if os.path.isdir(path):
