stdlib: fill gaps with INTERVAL FILL GAPS (#8163) **Summary** Replaces hand-written gap filling in Wattson and the Android state modules with INTERVAL FILL GAPS, and removes the _intervals_fill_gaps! macro. **Why** Each module filled gaps its own way, with UNION ALLs of leading and missing rows or the macro's NULL-group trick, and they disagreed at the edges. One operator states the intent directly and behaves the same everywhere. **Changes** - Wattson's CPU hotplug, CPU frequency, GPU frequency, GPU power state and TPU frequency tables are filled over the trace bounds, per CPU. - android_suspend_state, android_screen_state and android_charging_states fill each machine between its rows, and the primary machine even without data. - The _intervals_fill_gaps! macro and its tests are removed. - _adjusted_deep_idle keeps its prefix: its rows are already contiguous, and filling would re-sort ~400K rows for nothing. - _tpu_requests_count is unchanged: pipes cannot yet clamp counts. Wattson's estimates are unchanged on every test trace. Outputs differ only where the old code was inconsistent: no zero-width leading rows, a CPU without frequency events carries its policy, an empty GPU table is one NULL row over the trace, and a trace without screen or charging data has its 'unknown' row on machine 0 rather than a NULL machine. **Benchmarks** Query time against main (trace load excluded), release build, median of 5 alternating runs. Output is identical in every case. | Trace | Wattson estimate | Changed tables | | ----------------------- | ---------------- | -------------- | | wattson_tk4_pcmark | 1297.0 → 1296.5 ms | 343.3 → 343.8 ms | | wattson_syscore_suspend | 744.8 → 745.2 ms | 166.2 → 166.5 ms | | wattson_eos_suspend | 285.5 → 284.3 ms | 121.5 → 121.3 ms | | wattson_jank_cuj | 281.8 → 281.5 ms | 39.9 → 40.4 ms |
diff --git a/Android.bp b/Android.bp index 8ed2200..6ae0b48 100644 --- a/Android.bp +++ b/Android.bp
@@ -19449,7 +19449,6 @@ name: "perfetto_src_trace_processor_perfetto_sql_stdlib_intervals_intervals", srcs: [ "src/trace_processor/perfetto_sql/stdlib/intervals/create_intervals.sql", - "src/trace_processor/perfetto_sql/stdlib/intervals/fill_gaps.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/mipmap.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql",
diff --git a/BUILD b/BUILD index e147bed..70f067a 100644 --- a/BUILD +++ b/BUILD
@@ -4280,7 +4280,6 @@ name = "src_trace_processor_perfetto_sql_stdlib_intervals_intervals", srcs = [ "src/trace_processor/perfetto_sql/stdlib/intervals/create_intervals.sql", - "src/trace_processor/perfetto_sql/stdlib/intervals/fill_gaps.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/intersect.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/mipmap.sql", "src/trace_processor/perfetto_sql/stdlib/intervals/overlap.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/battery/charging_states.sql b/src/trace_processor/perfetto_sql/stdlib/android/battery/charging_states.sql index cf92a09..1f0d027 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/battery/charging_states.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/battery/charging_states.sql
@@ -15,7 +15,56 @@ INCLUDE PERFETTO MODULE counters.intervals; -INCLUDE PERFETTO MODULE intervals.fill_gaps; +-- Charging state intervals per machine, NULL where the state is unknown: every +-- machine with data is filled between its rows, and the primary machine is +-- filled even without data, since the trace says it was there. Other machines +-- without data may not have emitted any, which is not the same as their state +-- being unknown. +CREATE PERFETTO TABLE _android_charging_states_filled AS +FROM ( + WITH + _counter AS ( + SELECT counter.id, ts, counter.track_id, value + FROM counter + JOIN counter_track + ON counter_track.id = counter.track_id + WHERE + counter_track.name = 'BatteryStatus' + ), + _intervals AS ( + SELECT + intervals.id, + intervals.ts, + intervals.dur, + counter_track.machine_id, + CASE intervals.value + WHEN 2 THEN 'charging' + WHEN 3 THEN 'discharging' + WHEN 4 THEN 'not_charging' + WHEN 5 THEN 'full' + END AS short_charging_state, + CASE intervals.value + -- 0 and 1 are both 'Unknown' + WHEN 2 THEN 'Charging' + WHEN 3 THEN 'Discharging' + -- special case when charger is present but battery isn't charging + WHEN 4 THEN 'Not charging' + WHEN 5 THEN 'Full' + END AS charging_state + FROM counter_leading_intervals!(_counter) AS intervals + JOIN counter_track + ON counter_track.id = intervals.track_id + WHERE + dur > 0 + ) + -- A charging state value which is not a valid enum value is unknown. + SELECT ts, dur, machine_id, short_charging_state, charging_state + FROM _intervals +) +|> INTERVAL FILL GAPS WITH (SELECT trace_start() AS ts, trace_dur() AS dur) PER machine_id +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, 0 AS machine_id +) PER machine_id; -- Device charging states. CREATE PERFETTO TABLE android_charging_states( @@ -35,56 +84,16 @@ charging_state STRING ) AS --- Either the first statement is populated or the select statement after the --- union is populated but not both. -WITH - _counter AS ( - SELECT counter.id, ts, counter.track_id, value - FROM counter - JOIN counter_track - ON counter_track.id = counter.track_id - WHERE - counter_track.name = 'BatteryStatus' - ), - _intervals AS ( - SELECT - intervals.id, - intervals.ts, - intervals.dur, - counter_track.machine_id, - CASE intervals.value - WHEN 2 THEN 'charging' - WHEN 3 THEN 'discharging' - WHEN 4 THEN 'not_charging' - WHEN 5 THEN 'full' - END AS short_charging_state, - CASE intervals.value - -- 0 and 1 are both 'Unknown' - WHEN 2 THEN 'Charging' - WHEN 3 THEN 'Discharging' - -- special case when charger is present but battery isn't charging - WHEN 4 THEN 'Not charging' - WHEN 5 THEN 'Full' - END AS charging_state - FROM counter_leading_intervals!(_counter) AS intervals - JOIN counter_track - ON counter_track.id = intervals.track_id - WHERE - dur > 0 - ) --- When the trace does not have a slice in the charging state track or when --- the charging state value is not a valid enum value, assume the charging --- state for the entire trace is unknown. The use of _intervals_fill_gaps --- ensures we still have data for the entirety of the trace. +-- Where the charging state is not known, it is 'unknown' for the entirety of +-- the trace. SELECT - ROW_NUMBER() OVER () AS id, + ROW_NUMBER() OVER (ORDER BY ts, machine_id) AS id, ts, dur, machine_id, COALESCE(short_charging_state, 'unknown') AS short_charging_state, COALESCE(charging_state, 'Unknown') AS charging_state -FROM _intervals_fill_gaps!( - (machine_id), - (short_charging_state, charging_state), - _intervals -); +FROM _android_charging_states_filled +ORDER BY + ts, + machine_id;
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/screen_state.sql b/src/trace_processor/perfetto_sql/stdlib/android/screen_state.sql index 0cef8dc..2195c8e 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/screen_state.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/screen_state.sql
@@ -14,7 +14,94 @@ INCLUDE PERFETTO MODULE counters.intervals; -INCLUDE PERFETTO MODULE intervals.fill_gaps; +-- Screen state intervals per machine, NULL where the state is unknown: every +-- machine with data is filled between its rows, and the primary machine is +-- filled even without data, since the trace says it was there. Other machines +-- without data may not have emitted any, which is not the same as their state +-- being unknown. +CREATE PERFETTO TABLE _android_screen_state_filled AS +FROM ( + WITH + screen_state_span AS ( + SELECT * + FROM counter_leading_intervals!( + ( + SELECT counter.id, ts, counter.track_id, value + FROM counter + JOIN counter_track ON counter_track.id = counter.track_id + WHERE + name = 'ScreenState' + ) + ) + ), + mapped_names AS ( + SELECT + screen_state_span.id, + screen_state_span.ts, + screen_state_span.dur, + counter_track.machine_id, + -- Should be kept in sync with the enums in Display.java + CASE screen_state_span.value + -- Display.STATE_OFF + WHEN 1 THEN 'off' + -- Display.STATE_ON + WHEN 2 THEN 'on' + -- Display.STATE_DOZE + WHEN 3 THEN 'doze' + -- Display.STATE_DOZE_SUSPEND + WHEN 4 THEN 'doze' + -- Display.STATE_VR + WHEN 5 THEN 'on' + -- Display.STATE_ON_SUSPEND + WHEN 6 THEN 'on' + END AS simple_screen_state, + CASE screen_state_span.value + -- Display.STATE_OFF + WHEN 1 THEN 'off' + -- Display.STATE_ON + WHEN 2 THEN 'on' + -- Display.STATE_DOZE + WHEN 3 THEN 'doze' + -- Display.STATE_DOZE_SUSPEND + WHEN 4 THEN 'doze-suspend' + -- Display.STATE_VR + WHEN 5 THEN 'on-vr' + -- Display.STATE_ON_SUSPEND + WHEN 6 THEN 'on-suspend' + END AS short_screen_state, + CASE screen_state_span.value + -- Display.STATE_OFF + WHEN 1 THEN 'Screen off' + -- Display.STATE_ON + WHEN 2 THEN 'Screen on' + -- Display.STATE_DOZE + WHEN 3 THEN 'Always-on display (doze)' + -- Display.STATE_DOZE_SUSPEND + WHEN 4 THEN 'Always-on display (doze-suspend)' + -- Display.STATE_VR + WHEN 5 THEN 'Screen on (VR)' + -- Display.STATE_ON_SUSPEND + WHEN 6 THEN 'Screen on (suspend)' + END AS screen_state + FROM screen_state_span + JOIN counter_track + ON counter_track.id = screen_state_span.track_id + WHERE + dur > 0 + ) + SELECT + ts, + dur, + machine_id, + simple_screen_state, + short_screen_state, + screen_state + FROM mapped_names +) +|> INTERVAL FILL GAPS WITH (SELECT trace_start() AS ts, trace_dur() AS dur) PER machine_id +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, 0 AS machine_id +) PER machine_id; -- Table of the screen state - on, off or doze (always on display). CREATE PERFETTO TABLE android_screen_state( @@ -34,84 +121,15 @@ screen_state STRING ) AS -WITH - screen_state_span AS ( - SELECT * - FROM counter_leading_intervals!( - ( - SELECT counter.id, ts, counter.track_id, value - FROM counter - JOIN counter_track ON counter_track.id = counter.track_id - WHERE - name = 'ScreenState' - ) - ) - ), - mapped_names AS ( - SELECT - screen_state_span.id, - screen_state_span.ts, - screen_state_span.dur, - counter_track.machine_id, - -- Should be kept in sync with the enums in Display.java - CASE screen_state_span.value - -- Display.STATE_OFF - WHEN 1 THEN 'off' - -- Display.STATE_ON - WHEN 2 THEN 'on' - -- Display.STATE_DOZE - WHEN 3 THEN 'doze' - -- Display.STATE_DOZE_SUSPEND - WHEN 4 THEN 'doze' - -- Display.STATE_VR - WHEN 5 THEN 'on' - -- Display.STATE_ON_SUSPEND - WHEN 6 THEN 'on' - END AS simple_screen_state, - CASE screen_state_span.value - -- Display.STATE_OFF - WHEN 1 THEN 'off' - -- Display.STATE_ON - WHEN 2 THEN 'on' - -- Display.STATE_DOZE - WHEN 3 THEN 'doze' - -- Display.STATE_DOZE_SUSPEND - WHEN 4 THEN 'doze-suspend' - -- Display.STATE_VR - WHEN 5 THEN 'on-vr' - -- Display.STATE_ON_SUSPEND - WHEN 6 THEN 'on-suspend' - END AS short_screen_state, - CASE screen_state_span.value - -- Display.STATE_OFF - WHEN 1 THEN 'Screen off' - -- Display.STATE_ON - WHEN 2 THEN 'Screen on' - -- Display.STATE_DOZE - WHEN 3 THEN 'Always-on display (doze)' - -- Display.STATE_DOZE_SUSPEND - WHEN 4 THEN 'Always-on display (doze-suspend)' - -- Display.STATE_VR - WHEN 5 THEN 'Screen on (VR)' - -- Display.STATE_ON_SUSPEND - WHEN 6 THEN 'Screen on (suspend)' - END AS screen_state - FROM screen_state_span - JOIN counter_track - ON counter_track.id = screen_state_span.track_id - WHERE - dur > 0 - ) SELECT - ROW_NUMBER() OVER () AS id, + ROW_NUMBER() OVER (ORDER BY ts, machine_id) AS id, ts, dur, machine_id, COALESCE(simple_screen_state, 'unknown') AS simple_screen_state, COALESCE(short_screen_state, 'unknown') AS short_screen_state, COALESCE(screen_state, 'Unknown') AS screen_state -FROM _intervals_fill_gaps!( - (machine_id), - (simple_screen_state, short_screen_state, screen_state), - mapped_names -); +FROM _android_screen_state_filled +ORDER BY + ts, + machine_id;
diff --git a/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql b/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql index 6fe9c96..f5fbf21 100644 --- a/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql +++ b/src/trace_processor/perfetto_sql/stdlib/android/suspend.sql
@@ -14,13 +14,14 @@ -- limitations under the License. -- -INCLUDE PERFETTO MODULE intervals.fill_gaps; - -- Table of suspended and awake slices. -- -- Selects either the minimal or full ftrace source depending on what's -- available, marks suspended periods, and complements them to give awake --- periods. +-- periods: every machine with suspend slices is awake between them. The +-- primary machine is awake throughout if it never suspended, since the trace +-- starting and ending at all says it was awake. Other machines may have been +-- asleep throughout, so nothing is known of them without suspend slices. CREATE PERFETTO TABLE android_suspend_state( -- Timestamp ts TIMESTAMP, @@ -32,79 +33,74 @@ machine_id JOINID(machine.id) ) AS -WITH - suspend_slice_from_minimal AS ( - SELECT - ts, - dur, - coalesce( - lead(ts) OVER (PARTITION BY t.machine_id ORDER BY ts), - trace_end() - ) - - ts - - dur AS duration_gap, - t.machine_id - FROM track AS t - JOIN slice AS s - ON s.track_id = t.id - WHERE - t.name = 'Suspend/Resume Minimal' - ), - suspend_slice_latency AS ( - SELECT - ts, - dur, - coalesce( - lead(ts) OVER (PARTITION BY track.machine_id ORDER BY ts), - trace_end() - ) - - ts - - dur AS duration_gap, - track.machine_id - FROM slice - JOIN track - ON slice.track_id = track.id - WHERE - track.name = 'Suspend/Resume Latency' - AND (slice.name = 'syscore_resume(0)' - OR slice.name = 'timekeeping_freeze(0)') - AND dur != -1 - AND NOT EXISTS ( - SELECT * - FROM suspend_slice_from_minimal - WHERE - suspend_slice_from_minimal.machine_id = track.machine_id - ) - ), - suspend_slice_pre_filter AS ( - SELECT ts, dur, duration_gap, machine_id FROM suspend_slice_from_minimal - UNION ALL - SELECT ts, dur, duration_gap, machine_id FROM suspend_slice_latency - ), - suspend_slice AS ( - -- Filter out all the slices that overlapped with the following slices. - -- This happens with data loss where we lose start and end slices for suspends. - SELECT ts, dur, machine_id, 'suspended' AS power_state - FROM suspend_slice_pre_filter - WHERE - duration_gap >= 0 - UNION ALL - -- This guarantees that if machine 0 has no suspend slices in the trace, - -- that _intervals_fill_gaps will add an awake slice for the trace bounds. - -- This only works for the primary machine because we know from the trace - -- starting and ending at all that the device was awake. However, for other - -- machines, we don't actually know the suspend state as it's very possible - -- they were just asleep throughout the trace. - SELECT NULL, NULL, 0, NULL - ) -SELECT - ts, - dur, - COALESCE(machine_id, 0) AS machine_id, - COALESCE(power_state, 'awake') AS power_state -FROM _intervals_fill_gaps!((machine_id), (power_state), suspend_slice) -ORDER BY - ts; +FROM ( + WITH + suspend_slice_from_minimal AS ( + SELECT + ts, + dur, + coalesce( + lead(ts) OVER (PARTITION BY t.machine_id ORDER BY ts), + trace_end() + ) + - ts + - dur AS duration_gap, + t.machine_id + FROM track AS t + JOIN slice AS s + ON s.track_id = t.id + WHERE + t.name = 'Suspend/Resume Minimal' + ), + suspend_slice_latency AS ( + SELECT + ts, + dur, + coalesce( + lead(ts) OVER (PARTITION BY track.machine_id ORDER BY ts), + trace_end() + ) + - ts + - dur AS duration_gap, + track.machine_id + FROM slice + JOIN track + ON slice.track_id = track.id + WHERE + track.name = 'Suspend/Resume Latency' + AND (slice.name = 'syscore_resume(0)' + OR slice.name = 'timekeeping_freeze(0)') + AND dur != -1 + AND NOT EXISTS ( + SELECT * + FROM suspend_slice_from_minimal + WHERE + suspend_slice_from_minimal.machine_id = track.machine_id + ) + ), + suspend_slice_pre_filter AS ( + SELECT ts, dur, duration_gap, machine_id FROM suspend_slice_from_minimal + UNION ALL + SELECT ts, dur, duration_gap, machine_id FROM suspend_slice_latency + ) + -- Filter out all the slices that overlapped with the following slices. + -- This happens with data loss where we lose start and end slices for suspends. + SELECT ts, dur, machine_id, 'suspended' AS power_state + FROM suspend_slice_pre_filter + WHERE + duration_gap >= 0 +) +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, 'awake' AS power_state +) PER machine_id +|> INTERVAL FILL GAPS WITH ( + SELECT + trace_start() AS ts, + trace_dur() AS dur, + 0 AS machine_id, + 'awake' AS power_state +) PER machine_id +|> ORDER BY ts; -- Order by will cause Perfetto table to index by ts.;
diff --git a/src/trace_processor/perfetto_sql/stdlib/intervals/BUILD.gn b/src/trace_processor/perfetto_sql/stdlib/intervals/BUILD.gn index b6ecfb9..065d5f1 100644 --- a/src/trace_processor/perfetto_sql/stdlib/intervals/BUILD.gn +++ b/src/trace_processor/perfetto_sql/stdlib/intervals/BUILD.gn
@@ -17,7 +17,6 @@ perfetto_sql_source_set("intervals") { sources = [ "create_intervals.sql", - "fill_gaps.sql", "intersect.sql", "mipmap.sql", "overlap.sql",
diff --git a/src/trace_processor/perfetto_sql/stdlib/intervals/fill_gaps.sql b/src/trace_processor/perfetto_sql/stdlib/intervals/fill_gaps.sql deleted file mode 100644 index a288c0c..0000000 --- a/src/trace_processor/perfetto_sql/stdlib/intervals/fill_gaps.sql +++ /dev/null
@@ -1,118 +0,0 @@ --- --- Copyright 2026 The Android Open Source Project --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- https://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. - -INCLUDE PERFETTO MODULE std.metasql.unparenthesize; - --- Helper for __intrinsic_token_apply to fill in NULLs for a ColumnNameList. -CREATE PERFETTO MACRO __ifg_null(x Expr) -RETURNS Expr -AS NULL; - --- Fills in gaps within a table of slices, so data is present from trace_start --- to trace_end for each group of columns. --- --- Given a table of slices (`ts`, `dur`, `group_cols`, `data_cols`), this macro --- returns a table where every distinct `group_cols` has contiguous data from --- trace_start to trace_end. The `data_cols` are carried through from the source --- and filled with nulls when gaps are found. The input must not have overlaps --- within a set of `group_cols` --- --- Notes: --- * If grouping is not required, you may pass `(NULL)` for group_cols. --- * To guarantee a group is present, it is safe to `UNION` a row with the --- correct `group_cols` and all other columns as NULL. --- --- Example: --- ``` --- _intervals_fill_gaps!( --- (uid, package_name), --- (process_state), --- process_state_table --- ) --- ``` -CREATE PERFETTO MACRO _intervals_fill_gaps( - -- A parenthesized list of column names to group by, or (NULL) to not group. - group_cols ColumnNameList, - -- A parethensizes list of column names to carry through. - data_cols ColumnNameList, - -- The source data, with columns ts, dur, group_cols, and data_cols. - data TableOrSubquery -) -RETURNS TableOrSubquery -AS ( - WITH SourceData AS ( - SELECT - ts, dur, - metasql_unparenthesize_exprlist!($group_cols), - metasql_unparenthesize_exprlist!($data_cols) - FROM $data - ), - Bounds AS ( - SELECT - metasql_unparenthesize_exprlist!($group_cols), - MIN(ts) AS min_ts, - MAX(ts+dur) AS max_ts - FROM SourceData - GROUP BY metasql_unparenthesize_exprlist!($group_cols) - ), - Parts AS ( - -- Part 1: the base data from the table. - SELECT - ts, dur, - metasql_unparenthesize_exprlist!($group_cols), - metasql_unparenthesize_exprlist!($data_cols) - FROM SourceData - UNION ALL - -- Part 2: the time from the trace start, to first slice. - SELECT - trace_start(), min_ts - trace_start(), - metasql_unparenthesize_exprlist!($group_cols), - __intrinsic_token_apply!(__ifg_null, $data_cols) - FROM Bounds - UNION ALL - -- Part 3: the time from the last slice, to trace end. - SELECT - max_ts, trace_end() - max_ts, - metasql_unparenthesize_exprlist!($group_cols), - __intrinsic_token_apply!(__ifg_null, $data_cols) - FROM Bounds - UNION ALL - -- Part 4.a: when there is no data for a group, return one row for the whole trace. - SELECT - trace_start(), trace_dur(), - metasql_unparenthesize_exprlist!($group_cols), - __intrinsic_token_apply!(__ifg_null, $data_cols) - FROM Bounds - WHERE min_ts IS NULL AND max_ts IS NULL - UNION ALL - -- Part 4.b: when there is no data at all, return null group_cols for the whole trace. - SELECT - trace_start(), trace_dur(), - __intrinsic_token_apply!(__ifg_null, $group_cols), - __intrinsic_token_apply!(__ifg_null, $data_cols) - WHERE NOT EXISTS(SELECT * FROM SourceData) - UNION ALL - -- Part 5: the time between slices (from when one ends, to next start). - SELECT - ts+dur, LEAD(ts) OVER (PARTITION BY metasql_unparenthesize_exprlist!($group_cols) ORDER BY ts) - (ts+dur), - metasql_unparenthesize_exprlist!($group_cols), - __intrinsic_token_apply!(__ifg_null, $data_cols) - FROM SourceData - ) - SELECT * - FROM Parts - WHERE IFNULL(dur, 0) > 0 - ORDER BY ts -);
diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/freq.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/freq.sql index 4afcbae..6002494 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/freq.sql +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/freq.sql
@@ -17,41 +17,17 @@ INCLUDE PERFETTO MODULE wattson.device_infos; +-- Every physically present CPU's frequency from trace start to trace end, NULL +-- where no freq event defines it, so that interval_intersect doesn't remove the +-- undefined time region. CREATE PERFETTO TABLE _adjusted_cpu_freq AS -WITH - _cpu_freq AS ( - SELECT ts, dur, freq, cf.ucpu AS cpu, d_map.policy - FROM cpu_frequency_counters AS cf - JOIN _dev_cpu_policy_map AS d_map - ON cf.ucpu = d_map.cpu - ), - -- Get first freq transition per CPU - first_cpu_freq_slices AS ( - SELECT min(ts) AS ts, cpu FROM _cpu_freq GROUP BY cpu - ) --- Prepend NULL slices up to first freq events on a per CPU basis -SELECT - -- Construct slices from first cpu ts up to first freq event for each cpu - trace_start() AS ts, - first_slices.ts - trace_start() AS dur, - NULL AS freq, - first_slices.cpu, - d_map.policy -FROM first_cpu_freq_slices AS first_slices -JOIN _dev_cpu_policy_map AS d_map - ON first_slices.cpu = d_map.cpu -UNION ALL -SELECT ts, dur, freq, cpu, policy FROM _cpu_freq -UNION ALL --- Add empty cpu freq counters for CPUs that are physically present, but did not --- have a single freq event register. The time region needs to be defined so --- that interval_intersect doesn't remove the undefined time region. -SELECT - trace_start() AS ts, - trace_dur() AS dur, - NULL AS freq, - cpu, - NULL AS policy -FROM _dev_cpu_policy_map -WHERE - NOT (cpu IN (SELECT cpu FROM first_cpu_freq_slices)); +FROM ( + SELECT ts, dur, freq, cf.ucpu AS cpu, d_map.policy + FROM cpu_frequency_counters AS cf + JOIN _dev_cpu_policy_map AS d_map + ON cf.ucpu = d_map.cpu +) +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, cpu, policy + FROM _dev_cpu_policy_map +) PER cpu;
diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/hotplug.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/hotplug.sql index a0478fb..0257b1f 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/hotplug.sql +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/cpu/hotplug.sql
@@ -69,34 +69,18 @@ iif(is_different_cpu AND dur < time_from_us(100), FALSE, is_different_cpu) AS is_different_cpu FROM transitions_dur; --- Fill gaps from beginning of trace to end of trace so that this table can be --- used by interval_intersect(). +-- Every CPU's hotplug state from trace start to trace end, online wherever +-- cpuhp says nothing, so that this table can be used by interval_intersect(): +-- a CPU which is never offline is online throughout. CREATE PERFETTO TABLE _gapless_hotplug_slices AS -WITH - filled_gaps AS ( - -- First slice from trace_start() to first offline slice per CPU - SELECT - cpu, - trace_start() AS ts, - min(ts) - trace_start() AS dur, - FALSE AS offline - FROM _cpu_hotplug_offline - GROUP BY - cpu - UNION ALL - -- All online and offline regions as defined by cpuhp. This will have - -- continuous slices from somewhere in the middle to the end of the trace. - SELECT cpu, ts, dur, is_different_cpu AS offline FROM _cpu_hotplug_offline - UNION ALL - -- Creates a single online slice spanning the entire trace for CPUs that are - -- never offline. This is needed for interval_intersect() to not delete - -- undefined time periods - SELECT cpu, trace_start() AS ts, trace_dur() AS dur, FALSE AS offline - FROM _dev_cpu_policy_map - WHERE - NOT (cpu IN (SELECT cpu FROM _cpu_hotplug_offline)) - ) -SELECT ts, dur, cpu, offline FROM filled_gaps ORDER BY cpu, ts; +FROM ( + SELECT ts, dur, cpu, is_different_cpu AS offline FROM _cpu_hotplug_offline +) +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, cpu, FALSE AS offline + FROM _dev_cpu_policy_map +) PER cpu +|> ORDER BY cpu, ts; -- Nominal suspend table with gapless suspended state. CREATE PERFETTO TABLE _gapless_suspend_slices AS
diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/gpu/freq_idle.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/gpu/freq_idle.sql index 46b8656..2d9286c 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/gpu/freq_idle.sql +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/gpu/freq_idle.sql
@@ -40,46 +40,28 @@ WHERE t.name = 'powervr_gpu_power_state'; --- Gapless time slices of GPU freq from trace_start() to trace_end() +-- Gapless time slices of GPU freq from trace_start() to trace_end(), NULL where +-- no freq event defines it. CREATE PERFETTO TABLE _gapless_gpu_freq AS -WITH - nominal_freqs AS ( - -- Prepend NULL slices up to first freq events - SELECT - trace_start() AS ts, - min(ts) - trace_start() AS dur, - NULL AS freq, - NULL AS prev_freq, - next_gpu_freq AS next_freq, - gpu_id - FROM android_gpu_frequency - WHERE - gpu_id - = ( - SELECT gpu_id - FROM _gpuid_map - JOIN _wattson_device - ON _gpuid_map.device = _wattson_device.name - ) - UNION ALL - SELECT - ts, - dur, - gpu_freq AS freq, - prev_gpu_freq AS prev_freq, - next_gpu_freq AS next_freq, - gpu_id - FROM android_gpu_frequency - WHERE - gpu_id - = ( - SELECT gpu_id - FROM _gpuid_map - JOIN _wattson_device - ON _gpuid_map.device = _wattson_device.name - ) - ) -SELECT * FROM nominal_freqs; +FROM ( + SELECT + ts, + dur, + gpu_freq AS freq, + prev_gpu_freq AS prev_freq, + next_gpu_freq AS next_freq, + gpu_id + FROM android_gpu_frequency + WHERE + gpu_id + = ( + SELECT gpu_id + FROM _gpuid_map + JOIN _wattson_device + ON _gpuid_map.device = _wattson_device.name + ) +) +|> INTERVAL FILL GAPS WITH (SELECT trace_start() AS ts, trace_dur() AS dur); -- A single source for GPU power state information -- from either mali or pvr which are mutually exclusive @@ -88,20 +70,11 @@ UNION ALL SELECT ts, dur, power_state FROM _pvr_gpu_power_state; --- Gapless time slices of GPU idle from trace_start() to trace_end() +-- Gapless time slices of GPU idle from trace_start() to trace_end(), NULL where +-- no power state event defines it. CREATE PERFETTO TABLE _gapless_gpu_power_state AS --- Prepend NULL slices up to first idle events -WITH - nominal_power_states AS ( - SELECT - trace_start() AS ts, - min(ts) - trace_start() AS dur, - NULL AS power_state - FROM _gpu_power_state - UNION ALL - SELECT ts, dur, power_state FROM _gpu_power_state - ) -SELECT * FROM nominal_power_states; +FROM _gpu_power_state +|> INTERVAL FILL GAPS WITH (SELECT trace_start() AS ts, trace_dur() AS dur); -- GPU frequency and power state over the regions where both are defined. CREATE PERFETTO TABLE _gpu_freq_idle_raw AS
diff --git a/src/trace_processor/perfetto_sql/stdlib/wattson/tpu/freq_idle.sql b/src/trace_processor/perfetto_sql/stdlib/wattson/tpu/freq_idle.sql index 14abee9..900d2bc 100644 --- a/src/trace_processor/perfetto_sql/stdlib/wattson/tpu/freq_idle.sql +++ b/src/trace_processor/perfetto_sql/stdlib/wattson/tpu/freq_idle.sql
@@ -14,47 +14,41 @@ -- limitations under the License. -- Creates TPU freq counter and cluster based off of slices --- (gapless from trace_start to trace_end) +-- (gapless from trace_start to trace_end, 0 where no DVFS event defines it) +-- A device without a TPU has no DVFS events, and so no TPU frequency at all. CREATE PERFETTO TABLE _tpu_freq AS -WITH - raw_freqs AS ( - SELECT - s.ts, - iif(s.dur = -1, trace_end() - s.ts, s.dur) AS dur, - CAST(s.name AS INTEGER) AS freq, - CASE - WHEN s.name GLOB '*Cluster Both*' THEN 2 - WHEN s.name GLOB '*Cluster 0*' THEN 0 - WHEN s.name GLOB '*Cluster 1*' THEN 1 - ELSE 2 - END AS cluster - FROM slice AS s - JOIN track AS t - ON t.id = s.track_id - WHERE - t.name = 'DVFS' - ), - all_slices AS ( - -- Initial gap before first DVFS event - SELECT - trace_start() AS ts, - min(ts) - trace_start() AS dur, - 0 AS freq, - 0 AS cluster - FROM raw_freqs - UNION ALL - -- Active DVFS events - SELECT ts, dur, freq, cluster FROM raw_freqs - UNION ALL - -- Gaps between consecutive DVFS events and after the last DVFS event - SELECT - ts + dur AS ts, - lead(ts, 1, trace_end()) OVER (ORDER BY ts) - (ts + dur) AS dur, - 0 AS freq, - 0 AS cluster - FROM raw_freqs - ) -SELECT * FROM all_slices WHERE dur > 0 ORDER BY ts; +FROM ( + SELECT + s.ts, + iif(s.dur = -1, trace_end() - s.ts, s.dur) AS dur, + CAST(s.name AS INTEGER) AS freq, + CASE + WHEN s.name GLOB '*Cluster Both*' THEN 2 + WHEN s.name GLOB '*Cluster 0*' THEN 0 + WHEN s.name GLOB '*Cluster 1*' THEN 1 + ELSE 2 + END AS cluster + FROM slice AS s + JOIN track AS t + ON t.id = s.track_id + WHERE + t.name = 'DVFS' + AND s.dur != 0 +) +|> INTERVAL FILL GAPS WITH ( + SELECT trace_start() AS ts, trace_dur() AS dur, 0 AS freq, 0 AS cluster + WHERE + EXISTS ( + SELECT 1 + FROM slice AS s + JOIN track AS t + ON t.id = s.track_id + WHERE + t.name = 'DVFS' + AND s.dur != 0 + ) +) +|> ORDER BY ts; -- Number of parallel TPU requests at any point in time. The weight 0 -- background row spanning the whole trace makes INTERVAL FLATTEN also emit
diff --git a/test/trace_processor/diff_tests/include_index.py b/test/trace_processor/diff_tests/include_index.py index ad86ec7..f9aa447 100644 --- a/test/trace_processor/diff_tests/include_index.py +++ b/test/trace_processor/diff_tests/include_index.py
@@ -163,7 +163,6 @@ from diff_tests.stdlib.graphs.scan_tests import GraphScanTests from diff_tests.stdlib.graphs.search_tests import GraphSearchTests from diff_tests.stdlib.intervals.create_intervals_tests import CreateIntervals -from diff_tests.stdlib.intervals.fill_gaps_tests import IntervalsFillGaps from diff_tests.stdlib.intervals.intersect_duration_tests import IntervalsIntersectDurations from diff_tests.stdlib.intervals.intersect_tests import IntervalsIntersect from diff_tests.stdlib.intervals.tests import StdlibIntervals @@ -386,7 +385,6 @@ SpanJoinSmoke, Stacks, CreateIntervals, - IntervalsFillGaps, IntervalsIntersect, IntervalsIntersectDurations, StdlibIntervals,
diff --git a/test/trace_processor/diff_tests/stdlib/intervals/fill_gaps_tests.py b/test/trace_processor/diff_tests/stdlib/intervals/fill_gaps_tests.py deleted file mode 100644 index e2175fe..0000000 --- a/test/trace_processor/diff_tests/stdlib/intervals/fill_gaps_tests.py +++ /dev/null
@@ -1,244 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (C) 2026 The Android Open Source Project -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from python.generators.diff_tests.testing import Path, DataPath, Metric -from python.generators.diff_tests.testing import Csv, TextProto, DiffTestBlueprint, TestSuite - - -def make_trace_bounds(start: int, end: int) -> TextProto: - return TextProto(f""" - packet {{ - ftrace_events {{ - cpu: 0 - event {{ - timestamp: {start} - pid: 10 - print {{ - buf: "B|10|Start" - }} - }} - event {{ - timestamp: {end} - pid: 10 - print {{ - buf: "E|10" - }} - }} - }} - }} - """) - - -class IntervalsFillGaps(TestSuite): - - def test_fill_gaps_single_group(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, grp, val) AS ( - VALUES - (200, 100, 'A', 42), - (500, 200, 'A', 43) - ) - SELECT * FROM data; - - SELECT ts, dur, grp, val - FROM _intervals_fill_gaps!((grp), (val), data_table) - ORDER BY ts; - """, - out=Csv(""" - "ts","dur","grp","val" - 100,100,"A","[NULL]" - 200,100,"A",42 - 300,200,"A","[NULL]" - 500,200,"A",43 - 700,300,"A","[NULL]" - """)) - - def test_fill_gaps_multiple_groups(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, grp, val) AS ( - VALUES - (200, 100, 'A', 42), - (500, 200, 'A', 43), - (300, 150, 'B', 99) - ) - SELECT * FROM data; - - SELECT ts, dur, grp, val - FROM _intervals_fill_gaps!((grp), (val), data_table) - ORDER BY grp, ts; - """, - out=Csv(""" - "ts","dur","grp","val" - 100,100,"A","[NULL]" - 200,100,"A",42 - 300,200,"A","[NULL]" - 500,200,"A",43 - 700,300,"A","[NULL]" - 100,200,"B","[NULL]" - 300,150,"B",99 - 450,550,"B","[NULL]" - """)) - - def test_fill_gaps_empty_input(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE empty_table AS - SELECT CAST(NULL AS INT) AS ts, CAST(NULL AS INT) AS dur, CAST(NULL AS TEXT) AS grp, CAST(NULL AS INT) AS val LIMIT 0; - - SELECT ts, dur, grp, val - FROM _intervals_fill_gaps!((grp), (val), empty_table); - """, - out=Csv(""" - "ts","dur","grp","val" - 100,900,"[NULL]","[NULL]" - """)) - - def test_fill_gaps_no_valid_slices(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, grp, val) AS ( - VALUES - (NULL, NULL, 'A', NULL) - ) - SELECT * FROM data; - - SELECT ts, dur, grp, val - FROM _intervals_fill_gaps!((grp), (val), data_table); - """, - out=Csv(""" - "ts","dur","grp","val" - 100,900,"A","[NULL]" - """)) - - def test_fill_gaps_multiple_group_cols(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, grp1, grp2, val) AS ( - VALUES - (200, 100, 'A', 'X', 42), - (500, 200, 'A', 'Y', 43) - ) - SELECT * FROM data; - - SELECT ts, dur, grp1, grp2, val - FROM _intervals_fill_gaps!((grp1, grp2), (val), data_table) - ORDER BY grp1, grp2, ts; - """, - out=Csv(""" - "ts","dur","grp1","grp2","val" - 100,100,"A","X","[NULL]" - 200,100,"A","X",42 - 300,700,"A","X","[NULL]" - 100,400,"A","Y","[NULL]" - 500,200,"A","Y",43 - 700,300,"A","Y","[NULL]" - """)) - - def test_fill_gaps_multiple_data_cols(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, grp, val1, val2) AS ( - VALUES - (200, 100, 'A', 42, 'hello'), - (500, 200, 'A', 43, 'world') - ) - SELECT * FROM data; - - SELECT ts, dur, grp, val1, val2 - FROM _intervals_fill_gaps!((grp), (val1, val2), data_table) - ORDER BY ts; - """, - out=Csv(""" - "ts","dur","grp","val1","val2" - 100,100,"A","[NULL]","[NULL]" - 200,100,"A",42,"hello" - 300,200,"A","[NULL]","[NULL]" - 500,200,"A",43,"world" - 700,300,"A","[NULL]","[NULL]" - """)) - - def test_fill_gaps_union_guarantees_group(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, uid, val) AS ( - VALUES - (200, 100, 1234, 42), - (NULL, NULL, 1234, NULL) - ) - SELECT * FROM data; - - SELECT ts, dur, uid, val - FROM _intervals_fill_gaps!((uid), (val), data_table) - ORDER BY uid, ts; - """, - out=Csv(""" - "ts","dur","uid","val" - 100,100,1234,"[NULL]" - 200,100,1234,42 - 300,700,1234,"[NULL]" - """)) - - def test_fill_gaps_without_partition(self): - return DiffTestBlueprint( - trace=make_trace_bounds(100, 1000), - query=""" - INCLUDE PERFETTO MODULE intervals.fill_gaps; - - CREATE PERFETTO TABLE data_table AS - WITH data(ts, dur, val) AS ( - VALUES - (200, 100, 42) - ) - SELECT * FROM data; - - SELECT ts, dur, val - FROM _intervals_fill_gaps!((NULL), (val), data_table) - ORDER BY ts; - """, - out=Csv(""" - "ts","dur","val" - 100,100,"[NULL]" - 200,100,42 - 300,700,"[NULL]" - """)) \ No newline at end of file
diff --git a/test/trace_processor/diff_tests/syntax/pipeline_tests.py b/test/trace_processor/diff_tests/syntax/pipeline_tests.py index 17a37fb..52785ec 100644 --- a/test/trace_processor/diff_tests/syntax/pipeline_tests.py +++ b/test/trace_processor/diff_tests/syntax/pipeline_tests.py
@@ -1034,13 +1034,13 @@ 10,20,0,1 """)) - # Matches the stdlib macro it replaces, on rows which never overlap. - def test_interval_fill_gaps_matches_the_macro(self): + # Rows which never overlap within a cpu, filled over the trace, tile the + # trace exactly on every cpu. + def test_interval_fill_gaps_tiles_the_trace(self): return DiffTestBlueprint( trace=DataPath('android_boot.pftrace'), query=""" PERFETTO PRAGMA pipelines = 1; - INCLUDE PERFETTO MODULE intervals.fill_gaps; CREATE PERFETTO TABLE sched_rows AS SELECT ts, dur, cpu, utid FROM sched WHERE dur > 0 AND utid != 0; @@ -1051,27 +1051,26 @@ SELECT trace_start() AS ts, trace_dur() AS dur ) PER cpu; - CREATE PERFETTO TABLE macro AS - SELECT ts, dur, cpu, utid - FROM _intervals_fill_gaps!((cpu), (utid), sched_rows); - SELECT (SELECT count(*) FROM piped WHERE utid IS NULL) AS fillers, ( SELECT count(*) FROM ( - SELECT ts, dur, cpu, utid FROM piped - EXCEPT SELECT ts, dur, cpu, utid FROM macro + SELECT cpu FROM piped GROUP BY cpu HAVING sum(dur) != trace_dur() ) - ) + ( + ) AS untiled_cpus, + ( SELECT count(*) FROM ( - SELECT ts, dur, cpu, utid FROM macro - EXCEPT SELECT ts, dur, cpu, utid FROM piped + SELECT + ts + dur > lead(ts) OVER (PARTITION BY cpu ORDER BY ts) + AS overlaps + FROM piped ) - ) AS mismatches; + WHERE overlaps + ) AS overlaps; """, out=Csv(""" - "fillers","mismatches" - 76331,0 + "fillers","untiled_cpus","overlaps" + 76331,0,0 """)) # Without PER, a filler takes the background's columns by name and its own