stdlib: Expose app startup breakdown module
This is a similar approach to the existing binder_breakdowns added
in aosp/3208231.
Test: tools/diff_test_trace_processor.py out/android/trace_processor_shell
Change-Id: Ia6eaff5797b6c2e7315a499bdfc16331770bcca7
diff --git a/Android.bp b/Android.bp
index d19109a..9415383 100644
--- a/Android.bp
+++ b/Android.bp
@@ -13565,6 +13565,7 @@
"src/trace_processor/perfetto_sql/stdlib/android/screenshots.sql",
"src/trace_processor/perfetto_sql/stdlib/android/services.sql",
"src/trace_processor/perfetto_sql/stdlib/android/slices.sql",
+ "src/trace_processor/perfetto_sql/stdlib/android/startup/startup_breakdowns.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startup_events.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startups.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startups_maxsdk28.sql",
diff --git a/BUILD b/BUILD
index c856483..f69117f 100644
--- a/BUILD
+++ b/BUILD
@@ -2784,6 +2784,7 @@
perfetto_filegroup(
name = "src_trace_processor_perfetto_sql_stdlib_android_startup_startup",
srcs = [
+ "src/trace_processor/perfetto_sql/stdlib/android/startup/startup_breakdowns.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startup_events.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startups.sql",
"src/trace_processor/perfetto_sql/stdlib/android/startup/startups_maxsdk28.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/startup/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/android/startup/BUILD.gn
index c495f08..bf7a41a 100644
--- a/src/trace_processor/perfetto_sql/stdlib/android/startup/BUILD.gn
+++ b/src/trace_processor/perfetto_sql/stdlib/android/startup/BUILD.gn
@@ -16,6 +16,7 @@
perfetto_sql_source_set("startup") {
sources = [
+ "startup_breakdowns.sql",
"startup_events.sql",
"startups.sql",
"startups_maxsdk28.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/startup/startup_breakdowns.sql b/src/trace_processor/perfetto_sql/stdlib/android/startup/startup_breakdowns.sql
new file mode 100644
index 0000000..c6b972a
--- /dev/null
+++ b/src/trace_processor/perfetto_sql/stdlib/android/startup/startup_breakdowns.sql
@@ -0,0 +1,153 @@
+--
+-- Copyright 2024 The Android Open Source Project
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+-- https://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+INCLUDE PERFETTO MODULE android.startup.startups;
+INCLUDE PERFETTO MODULE intervals.overlap;
+INCLUDE PERFETTO MODULE slices.hierarchy;
+INCLUDE PERFETTO MODULE slices.with_context;
+
+-- Maps slice names with common prefixes to a static string key.
+-- Returns NULL if there's no mapping.
+CREATE PERFETTO FUNCTION _normalize_android_string(name STRING)
+RETURNS STRING
+AS
+SELECT
+ CASE
+ WHEN $name = 'mm_vmscan_direct_reclaim' THEN 'kernel_memory_reclaim'
+ WHEN $name GLOB 'GC: Wait For*' THEN 'userspace_memory_reclaim'
+ WHEN ($name GLOB 'monitor contention*' OR $name GLOB 'Lock contention on a monitor lock*')
+ THEN 'monitor_contention'
+ WHEN $name GLOB 'Lock contention*' THEN 'art_lock_contention'
+ WHEN ($name = 'binder transaction' OR $name = 'binder reply') THEN 'binder'
+ WHEN $name = 'Contending for pthread mutex' THEN 'mutex_contention'
+ WHEN $name GLOB 'dlopen*' THEN 'dlopen'
+ WHEN $name GLOB 'VerifyClass*' THEN 'verify_class'
+ WHEN $name = 'inflate' THEN 'inflate'
+ WHEN $name GLOB 'Choreographer#doFrame*' THEN 'choreographer_do_frame'
+ WHEN $name GLOB 'OpenDexFilesFromOat*' THEN 'open_dex_files_from_oat'
+ WHEN $name = 'ResourcesManager#getResources' THEN 'resources_manager_get_resources'
+ WHEN $name = 'bindApplication' THEN 'bind_application'
+ WHEN $name = 'activityStart' THEN 'activity_start'
+ WHEN $name = 'activityResume' THEN 'activity_resume'
+ WHEN $name = 'activityRestart' THEN 'activity_restart'
+ WHEN $name = 'clientTransactionExecuted' THEN 'client_transaction_executed'
+ ELSE NULL
+ END name;
+
+-- Derives a startup reason from a slice name and some thread_state columns.
+CREATE PERFETTO FUNCTION _startup_breakdown_reason(
+ name STRING,
+ state STRING,
+ io_wait INT,
+ irq_context INT)
+RETURNS STRING
+AS
+SELECT
+ CASE
+ WHEN $io_wait = 1 THEN 'io'
+ WHEN $name IS NOT NULL THEN $name
+ WHEN $irq_contex = 1 THEN 'irq'
+ ELSE $state
+ END name;
+
+-- List of startups with unique ids for each possible upid. The existing
+-- startup_ids are not necessarily unique (because of multiuser).
+CREATE PERFETTO TABLE _startup_root_slices
+AS
+SELECT
+ (SELECT MAX(id) FROM slice) + row_number() OVER () AS id,
+ android_startups.dur AS dur,
+ android_startups.ts AS ts,
+ android_startups.startup_id,
+ android_startups.startup_type,
+ process.name AS process_name,
+ thread.utid AS utid
+FROM android_startup_processes startup
+JOIN android_startups
+ USING (startup_id)
+JOIN thread
+ ON thread.upid = process.upid AND thread.is_main_thread
+JOIN process
+ ON process.upid = startup.upid
+WHERE android_startups.dur > 0
+ORDER BY ts;
+
+-- All slices normalized with _normalize_android_string.
+CREATE PERFETTO TABLE _startup_normalized_slices
+AS
+SELECT p.id, p.parent_id, p.depth, p.name, thread_slice.ts, thread_slice.dur, thread_slice.utid
+FROM
+ _slice_remove_nulls_and_reparent
+ !((SELECT id, parent_id, depth, _normalize_android_string(name) AS name FROM slice WHERE dur > 0), name)
+ p
+JOIN thread_slice
+ USING (id);
+
+-- Subset of _startup_normalized_slices that occurred during any app startups on the main thread.
+-- Their timestamps and durations are chopped to fit within the respective app startup duration.
+CREATE PERFETTO TABLE _startup_slices_breakdown
+AS
+SELECT *
+FROM _intervals_merge_root_and_children_by_intersection !(_startup_root_slices, _startup_normalized_slices, utid);
+
+-- Flattened slice version of _startup_slices_breakdown. This selects the leaf slice at every region
+-- of the slice stack.
+CREATE PERFETTO TABLE _startup_flat_slices_breakdown
+AS
+SELECT i.ts, i.dur, i.root_id, s.id AS slice_id, s.name FROM _intervals_flatten !(_startup_slices_breakdown) i
+JOIN _startup_normalized_slices s USING (id);
+
+-- Subset of thread_states that occurred during any app startups on the main thread.
+CREATE PERFETTO TABLE _startup_thread_states_breakdown
+AS
+SELECT i.ts, i.dur, i.root_id, t.id AS thread_state_id, t.state, t.io_wait, t.irq_context
+ FROM _intervals_merge_root_and_children_by_intersection!(_startup_root_slices,
+ (SELECT *, NULL AS parent_id FROM thread_state),
+ utid) i
+JOIN thread_state t USING(id);
+
+-- Intersection of _startup_flat_slices_breakdown and _startup_thread_states_breakdown.
+-- A left intersection is used since some parts of the slice stack may not have any slices
+-- but will have thread states.
+CREATE VIRTUAL TABLE _startup_thread_states_and_slices_breakdown_sp
+USING
+ SPAN_LEFT_JOIN(
+ _startup_thread_states_breakdown PARTITIONED root_id,
+ _startup_flat_slices_breakdown PARTITIONED root_id);
+
+-- Blended thread state and slice breakdown blocking app startups.
+--
+-- Each row blames a unique period during an app startup with a reason
+-- derived from the slices and thread states on the main thread.
+--
+-- Some helpful events to enables are binder transactions, ART, am and view.
+CREATE PERFETTO TABLE android_startup_opinionated_breakdown(
+ -- Startup id. Alias of `slice.id`
+ startup_id INT,
+ -- Id of relevant slice blocking startup. Alias of `slice.id`.
+ slice_id INT,
+ -- Id of thread_state blocking startup. Alias of `thread_state.id`.
+ thread_state_id INT,
+ -- Timestamp of an exclusive interval during the app startup with a single latency reason.
+ ts INT,
+ -- Duration of an exclusive interval during the app startup with a single latency reason.
+ dur INT,
+ -- Cause of delay during an exclusive interval of the app startup.
+ reason STRING
+)
+AS
+SELECT b.ts, b.dur, startup.startup_id, b.slice_id, b.thread_state_id, _startup_breakdown_reason(name, state, io_wait, irq_context) AS reason
+FROM _startup_thread_states_and_slices_breakdown_sp b
+JOIN _startup_root_slices startup ON startup.id = b.root_id;
diff --git a/src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql b/src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql
index cd84164..356ec31 100644
--- a/src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql
+++ b/src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql
@@ -13,6 +13,8 @@
-- See the License for the specific language governing permissions and
-- limitations under the License.
+INCLUDE PERFETTO MODULE intervals.intersect;
+
-- Compute the distribution of the overlap of the given intervals over time.
--
-- Each interval is a (ts, dur) pair and the overlap represented as a (ts, value)
@@ -94,6 +96,8 @@
-- Merges a |roots_table| and |children_table| into one table. See _intervals_flatten
-- that accepts the output of this macro to flatten intervals.
+
+-- See: _intervals_merge_root_and_children_by_intersection.
CREATE PERFETTO MACRO _intervals_merge_root_and_children(
-- Table or subquery containing all the root intervals: (id, ts, dur).
-- Note that parent_id is not necessary in this table as it will be NULL anyways.
@@ -141,6 +145,46 @@
JOIN _roots USING(root_id)
);
+-- Merges a |roots_table| and |children_table| into one table. See _intervals_flatten
+-- that accepts the output of this macro to flatten intervals.
+
+-- This is very similar to _intervals_merge_root_and_children but there is no explicit
+-- root_id shared between the root and the children. Instead an _interval_intersect is
+-- used to derive the root and child relationships.
+CREATE PERFETTO MACRO _intervals_merge_root_and_children_by_intersection(
+ -- Table or subquery containing all the root intervals: (id, ts, dur).
+ -- Note that parent_id is not necessary in this table as it will be NULL anyways.
+ roots_table TableOrSubquery,
+ -- Table or subquery containing all the child intervals:
+ -- (root_id, id, parent_id, ts, dur)
+ children_table TableOrSubquery,
+ -- intersection key used in deriving the root child relationships.
+ key ColumnName)
+RETURNS TableOrSubQuery
+AS (
+ WITH
+ _roots AS (
+ SELECT * FROM $roots_table WHERE dur > 0 ORDER BY ts
+ ),
+ _children AS (
+ SELECT * FROM $children_table WHERE dur > 0 ORDER BY ts
+ )
+ SELECT
+ ii.ts,
+ ii.dur,
+ _children.id,
+ IIF(_children.parent_id IS NULL, id_1, _children.parent_id) AS parent_id,
+ _roots.id AS root_id,
+ _roots.ts AS root_ts,
+ _roots.dur AS root_dur,
+ ii.$key
+ FROM _interval_intersect!((_children, _roots), ($key)) ii
+ JOIN _children
+ ON _children.id = id_0
+ JOIN _roots
+ ON _roots.id = id_1
+);
+
-- Partition and flatten a hierarchy of intervals into non-overlapping intervals where
-- each resulting interval is the leaf in the hierarchy at any given time. The result also
-- denotes the 'self-time' of each interval.
diff --git a/src/trace_processor/perfetto_sql/stdlib/slices/hierarchy.sql b/src/trace_processor/perfetto_sql/stdlib/slices/hierarchy.sql
index dff6475..137a123 100644
--- a/src/trace_processor/perfetto_sql/stdlib/slices/hierarchy.sql
+++ b/src/trace_processor/perfetto_sql/stdlib/slices/hierarchy.sql
@@ -91,4 +91,41 @@
UNION ALL
SELECT
id, type, ts, dur, track_id, category, name, depth, parent_id, arg_set_id, thread_ts, thread_dur
-FROM descendant_slice($slice_id);
\ No newline at end of file
+FROM descendant_slice($slice_id);
+
+-- Delete rows from |slice_table| where the |column_name| value is NULL.
+--
+-- The |parent_id| of the remaining rows are adjusted to point to the closest
+-- ancestor remaining. This keeps the trees as connected as possible,
+-- allowing further graph analysis.
+CREATE PERFETTO MACRO _slice_remove_nulls_and_reparent(
+ -- Table or subquery containing a subset of the slice table. Required columns are
+ -- (id INT64, parent_id INT64, depth UINT32, <column_name>).
+ slice_table TableOrSubQuery,
+ -- Column name for which a NULL value indicates the row will be deleted.
+ column_name ColumnName)
+ -- The returned table has the schema (id INT64, parent_id INT64, depth UINT32, <column_name>).
+RETURNS TableOrSubQuery
+AS (
+ WITH _slice AS (
+ SELECT * FROM $slice_table WHERE $column_name IS NOT NULL
+ )
+ SELECT
+ id,
+ parent_id,
+ depth,
+ $column_name
+ FROM _slice
+ WHERE depth = 0
+ UNION ALL
+ SELECT
+ child.id,
+ anc.id AS parent_id,
+ MAX(IIF(parent.$column_name IS NULL, 0, anc.depth)) AS depth,
+ child.$column_name
+ FROM _slice child
+ JOIN ancestor_slice(child.id) anc
+ LEFT JOIN _slice parent
+ ON parent.id = anc.id
+ GROUP BY child.id
+);
diff --git a/test/trace_processor/diff_tests/stdlib/android/startups_tests.py b/test/trace_processor/diff_tests/stdlib/android/startups_tests.py
index c4f5a2e..acce2d7 100644
--- a/test/trace_processor/diff_tests/stdlib/android/startups_tests.py
+++ b/test/trace_processor/diff_tests/stdlib/android/startups_tests.py
@@ -168,3 +168,34 @@
"startup_id","time_to_initial_display","time_to_full_display","ttid_frame_id","ttfd_frame_id","upid"
0,143980066,620815843,5873276,5873353,229
"""))
+
+ def test_android_startup_breakdown(self):
+ return DiffTestBlueprint(
+ trace=DataPath('api31_startup_cold.perfetto-trace'),
+ query="""
+ INCLUDE PERFETTO MODULE android.startup.startup_breakdowns;
+ SELECT
+ SUM(dur) AS dur,
+ reason
+ FROM android_startup_opinionated_breakdown
+ GROUP BY reason ORDER BY dur DESC;
+ """,
+ out=Csv("""
+ "dur","reason"
+ 28663023,"choreographer_do_frame"
+ 22564487,"binder"
+ 16351925,"Running"
+ 13212137,"activity_start"
+ 10264635,"io"
+ 6779947,"inflate"
+ 6240207,"bind_application"
+ 5214375,"R+"
+ 3072397,"resources_manager_get_resources"
+ 2722869,"D"
+ 2574273,"open_dex_files_from_oat"
+ 2392761,"S"
+ 2353124,"activity_resume"
+ 1325727,"R"
+ 43698,"art_lock_contention"
+ 5573,"verify_class"
+ """))
diff --git a/test/trace_processor/diff_tests/stdlib/intervals/tests.py b/test/trace_processor/diff_tests/stdlib/intervals/tests.py
index cf48cdc..2712599 100644
--- a/test/trace_processor/diff_tests/stdlib/intervals/tests.py
+++ b/test/trace_processor/diff_tests/stdlib/intervals/tests.py
@@ -118,3 +118,72 @@
8,1,0,0
9,1,1,1
"""))
+
+ def test_intervals_flatten_by_intersection(self):
+ return DiffTestBlueprint(
+ trace=TextProto(""),
+ query="""
+ INCLUDE PERFETTO MODULE intervals.overlap;
+
+ CREATE PERFETTO TABLE foo AS
+ WITH roots_data (id, ts, dur, utid) AS (
+ VALUES
+ (0, 0, 9, 0),
+ (0, 0, 9, 1),
+ (1, 9, 1, 2)
+ ), children_data (id, parent_id, ts, dur, utid) AS (
+ VALUES
+ (2, 0, 1, 3, 0),
+ (3, 0, 5, 1, 0),
+ (4, 0, 6, 1, 0),
+ (5, 0, 7, 0, 0),
+ (6, 0, 7, 1, 0),
+ (7, 2, 2, 1, 0)
+ )
+ SELECT *
+ FROM _intervals_merge_root_and_children_by_intersection!(roots_data, children_data, utid);
+
+ SELECT ts, dur, id, root_id FROM _intervals_flatten!(foo) ORDER BY ts;
+ """,
+ out=Csv("""
+ "ts","dur","id","root_id"
+ 0,1,0,0
+ 1,1,2,0
+ 2,1,7,0
+ 3,1,2,0
+ 4,1,0,0
+ 5,1,3,0
+ 6,1,4,0
+ 7,1,6,0
+ 8,1,0,0
+ """))
+
+ def test_intervals_flatten_by_intersection_no_matching_key(self):
+ return DiffTestBlueprint(
+ trace=TextProto(""),
+ query="""
+ INCLUDE PERFETTO MODULE intervals.overlap;
+
+ CREATE PERFETTO TABLE foo AS
+ WITH roots_data (id, ts, dur, utid) AS (
+ VALUES
+ (0, 0, 9, 1),
+ (0, 0, 9, 2),
+ (1, 9, 1, 3)
+ ), children_data (id, parent_id, ts, dur, utid) AS (
+ VALUES
+ (2, 0, 1, 3, 0),
+ (3, 0, 5, 1, 0),
+ (4, 0, 6, 1, 0),
+ (5, 0, 7, 0, 0),
+ (6, 0, 7, 1, 0),
+ (7, 2, 2, 1, 0)
+ )
+ SELECT *
+ FROM _intervals_merge_root_and_children_by_intersection!(roots_data, children_data, utid);
+
+ SELECT ts, dur, id, root_id FROM _intervals_flatten!(foo) ORDER BY ts;
+ """,
+ out=Csv("""
+ "ts","dur","id","root_id"
+ """))
diff --git a/test/trace_processor/diff_tests/stdlib/slices/tests.py b/test/trace_processor/diff_tests/stdlib/slices/tests.py
index fa326d0..625659e 100644
--- a/test/trace_processor/diff_tests/stdlib/slices/tests.py
+++ b/test/trace_processor/diff_tests/stdlib/slices/tests.py
@@ -104,6 +104,26 @@
"NestedThreadSlice",6,1,1
"""))
+ def test_slice_remove_nulls_and_reparent(self):
+ return DiffTestBlueprint(
+ trace=Path('trace.py'),
+ query="""
+ INCLUDE PERFETTO MODULE slices.hierarchy;
+
+ SELECT id, parent_id, name, depth
+ FROM _slice_remove_nulls_and_reparent!(
+ (SELECT id, parent_id, depth, IIF(name = 'ProcessSlice', NULL, name) AS name
+ FROM slice),
+ name
+ ) LIMIT 10;
+ """,
+ out=Csv("""
+ "id","parent_id","name","depth"
+ 0,"[NULL]","AsyncSlice",0
+ 2,"[NULL]","ThreadSlice",0
+ 3,2,"NestedThreadSlice",0
+ """))
+
# Common functions
def test_slice_flattened(self):
@@ -156,4 +176,4 @@
7,33333
8,46926
9,17865
- """))
\ No newline at end of file
+ """))
