Merge "trace_processor: Remove out of date TODO"
diff --git a/docs/contributing/getting-started.md b/docs/contributing/getting-started.md
index 8cb27ec..9696e7e 100644
--- a/docs/contributing/getting-started.md
+++ b/docs/contributing/getting-started.md
@@ -47,8 +47,6 @@
## Community
You can reach us on our [Discord channel](https://discord.gg/35ShE3A).
-If you prefer using IRC we have an experimental Discord <> IRC bridge
-synced with `#perfetto-dev` on [Freenode](https://webchat.freenode.net/).
Mailing list: https://groups.google.com/forum/#!forum/perfetto-dev
diff --git a/infra/discord-irc-bridge/Dockerfile b/infra/discord-irc-bridge/Dockerfile
deleted file mode 100644
index 5591da2..0000000
--- a/infra/discord-irc-bridge/Dockerfile
+++ /dev/null
@@ -1,25 +0,0 @@
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM node:latest
-
-RUN useradd -m discord_irc
-RUN npm install -g discord-irc
-RUN apt-get -y update && apt-get -y install supervisor
-COPY supervisord.conf /etc/
-COPY discord-irc.json /etc/
-COPY start.py /
-USER discord_irc
-
-CMD ["python", "start.py"]
\ No newline at end of file
diff --git a/infra/discord-irc-bridge/README.md b/infra/discord-irc-bridge/README.md
deleted file mode 100644
index 9c79d2d..0000000
--- a/infra/discord-irc-bridge/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# IRC <> Discord bridge
-
-This directory contains the docker image for the discord<>IRC bot.
-The docker container is built and pushed running:
-
-```bash
-docker build -t gcr.io/perfetto-irc/discord-irc-bridge .
-docker push gcr.io/perfetto-irc/discord-irc-bridge
-```
-
-The docker container requires two environment variables to be set (see below).
-These are set at the GCE project level (project: perfetto-irc).
-There is a VM template in the GCE project which has the right env vars set.
-If a VM restart is required use the template, don't create the VM from scratch.
-
-NICKNAME: This must be set to perfetto_discord:password. The password can be
- obtained on the usual internal website for passwords. Look for the account
- "perfetto_discord@freenode".
-
-DISCORD_TOKEN: This must be set to the Discord token for the bot. Look for
- the account "perfetto-discord-bot-token" in the internal password website.
diff --git a/infra/discord-irc-bridge/discord-irc.json b/infra/discord-irc-bridge/discord-irc.json
deleted file mode 100644
index bf71f2a..0000000
--- a/infra/discord-irc-bridge/discord-irc.json
+++ /dev/null
@@ -1,23 +0,0 @@
-[
- {
- "server": "irc.freenode.org",
- "autoSendCommands": [
- ],
- "channelMapping": {
- "629013441096450058": "#perfetto-dev"
- },
- "ircOptions": {
- "port": "6697",
- "secure": true
- },
- "ircNickColor": false,
- "ircPreventMention": true,
- "commandCharacters": [],
- "ignoreUsers": {
- "irc": [],
- "discord": []
- },
- "webhooks": {
- }
- }
-]
diff --git a/infra/discord-irc-bridge/start.py b/infra/discord-irc-bridge/start.py
deleted file mode 100644
index 35bbb93..0000000
--- a/infra/discord-irc-bridge/start.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import json
-import os
-import sys
-
-with open("/etc/discord-irc.json") as f:
- cfg = json.load(f)
-
-cfg[0]["nickname"] = os.getenv("NICKNAME")
-cfg[0]["discordToken"] = os.getenv("DISCORD_TOKEN")
-
-if cfg[0]["nickname"] is None:
- sys.stderr.write("NICKNAME env var not set\n")
- sys.exit(1)
-
-if cfg[0]["discordToken"] is None:
- sys.stderr.write("DISCORD_TOKEN env var not set\n")
- sys.exit(1)
-
-with open("/tmp/discord-irc-merged.json", "w") as f:
- json.dump(cfg, f)
-
-os.execl("/usr/bin/supervisord", "supervisord")
diff --git a/infra/discord-irc-bridge/supervisord.conf b/infra/discord-irc-bridge/supervisord.conf
deleted file mode 100644
index 713b332..0000000
--- a/infra/discord-irc-bridge/supervisord.conf
+++ /dev/null
@@ -1,13 +0,0 @@
-[supervisord]
-nodaemon=true
-logfile=/dev/stdout
-logfile_maxbytes=0
-
-[program:discord_irc]
-command=discord-irc --config /tmp/discord-irc-merged.json
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stdout_logfile_backups=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-stderr_logfile_backups=0
diff --git a/src/trace_processor/metrics/sql/chrome/scroll_jank_cause_queuing_delay.sql b/src/trace_processor/metrics/sql/chrome/scroll_jank_cause_queuing_delay.sql
index d5424bd..d3762ef 100644
--- a/src/trace_processor/metrics/sql/chrome/scroll_jank_cause_queuing_delay.sql
+++ b/src/trace_processor/metrics/sql/chrome/scroll_jank_cause_queuing_delay.sql
@@ -30,15 +30,45 @@
slice.name != "ThreadController active" AND
(slice.depth = 0 OR ancestor.name = "ThreadController active");
+-- Sort track ids to optimize joining with slices
+-- as engine doesn't do the sort to join in O(LogN)
+-- per row by default
+DROP VIEW IF EXISTS chrome_annotated_threads_and_processes;
+CREATE VIEW chrome_annotated_threads_and_processes AS
+ SELECT
+ thread_track.id AS track_id,
+ chrome_thread.canonical_name AS thread_name,
+ chrome_process.process_type AS process_name
+ FROM
+ thread_track JOIN
+ chrome_thread JOIN
+ chrome_process ON
+ thread_track.utid = chrome_thread.utid AND
+ chrome_thread.upid = chrome_process.upid
+ ORDER BY
+ track_id ASC;
+
+-- See b/166441398 & crbug/1094361 for why we remove threadpool (originally
+-- the -to-End step). In essence -to-End is often reported on the ThreadPool
+-- after the fact with explicit timestamps so it being blocked isn't noteworthy.
+DROP VIEW IF EXISTS blocking_chrome_tasks_without_threadpool;
+CREATE VIEW blocking_chrome_tasks_without_threadpool AS
+ SELECT
+ slice.*,
+ annotations.thread_name AS thread_name,
+ annotations.process_name AS process_name
+ FROM
+ blocking_tasks_no_threadcontroller_active AS slice JOIN
+ chrome_annotated_threads_and_processes AS annotations ON
+ annotations.track_id = slice.track_id
+ WHERE
+ NOT (thread_name GLOB "*ThreadPool*");
+
-- This view grabs any slice that could have prevented any GestureScrollUpdate
-- flow event from being run (queuing delays). For RunTask we know that its
-- generic (and thus hard to figure out whats the cause) so we grab the src
-- location to make it more meaningful.
--
--- See b/166441398 & crbug/1094361 for why we remove the -to-End step. In
--- essence -to-End is often reported on the ThreadPool after the fact with
--- explicit timestamps so it being blocked isn't noteworthy.
---
-- See b/184134310 for why we allow depth == 1 and ancestor.id is null (which
-- implies its a "ThreadController active" slice because we removed it
-- previously).
@@ -72,14 +102,10 @@
slice.*
FROM
scroll_flow_event_queuing_delay queuing JOIN
- blocking_tasks_no_threadcontroller_active AS slice ON
+ blocking_chrome_tasks_without_threadpool AS slice ON
slice.ts + slice.dur > queuing.ancestor_end AND
queuing.maybe_next_ancestor_ts > slice.ts AND
- slice.track_id = queuing.next_track_id AND
- queuing.description NOT GLOB
- "InputLatency.LatencyInfo.*ank.STEP_DRAW_AND_SWAP-to-End" AND
- queuing.description NOT GLOB
- "InputLatency.LatencyInfo.*ank.STEP_FINISHED_SWAP_BUFFERS-to-End"
+ slice.track_id = queuing.next_track_id
WHERE
queuing_time_ns IS NOT NULL AND
queuing_time_ns > 0;
@@ -208,6 +234,8 @@
dur_overlapping_ns,
description,
replace(file, rtrim(file, replace(file, '/', '')), '') AS file,
+ thread_name,
+ process_name,
function,
GROUP_CONCAT(
CASE WHEN descendant_depth < invalid_depth OR descendant_major_slice THEN
@@ -252,7 +280,7 @@
, "-") AS descendant_cpu_time
FROM
blocking_tasks_queuing_delay_with_invalid_depth
- GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+ GROUP BY 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17
ORDER BY descendant_cpu_percentage DESC;
-- Create a common name for each "cause" based on the slice stack we found.
@@ -281,8 +309,8 @@
-- Join every row (jank and non-jank with the average non-jank time for the
-- given metric_name).
-DROP VIEW IF EXISTS scroll_jank_cause_queuing_delay_unannotated;
-CREATE VIEW scroll_jank_cause_queuing_delay_unannotated AS
+DROP VIEW IF EXISTS scroll_jank_cause_queuing_delay;
+CREATE VIEW scroll_jank_cause_queuing_delay AS
SELECT
base.*,
'InputLatency.LatencyInfo.Flow.QueuingDelay.' ||
@@ -293,15 +321,4 @@
FROM
scroll_jank_cause_queuing_delay_temp base LEFT JOIN
scroll_jank_cause_queuing_delay_average_no_jank_time avg_no_jank ON
- base.location = avg_no_jank.location;
-
--- Annotate with process and thread names.
-DROP VIEW IF EXISTS scroll_jank_cause_queuing_delay;
-CREATE VIEW scroll_jank_cause_queuing_delay AS
-SELECT p.process_type AS process_name, ct.canonical_name AS thread_name, s.*
-FROM scroll_jank_cause_queuing_delay_unannotated s,
- thread_track tt, chrome_thread ct,
- chrome_process p
-WHERE s.track_id = tt.id
- AND tt.utid = ct.utid
- AND ct.upid = p.upid;
+ base.location = avg_no_jank.location;
\ No newline at end of file
diff --git a/src/trace_processor/python/perfetto/trace_processor/api.py b/src/trace_processor/python/perfetto/trace_processor/api.py
index 56d9fc9..ce06edf 100644
--- a/src/trace_processor/python/perfetto/trace_processor/api.py
+++ b/src/trace_processor/python/perfetto/trace_processor/api.py
@@ -66,10 +66,23 @@
# contents into lists based on the type of the batch
batch_index = 0
while True:
+ # It's possible on some occasions that there are non UTF-8 characters
+ # in the string_cells field. If this is the case, string_cells is
+ # a bytestring which needs to be decoded (but passing ignore so that
+ # we don't fail in decoding).
+ strings_batch_str = batches[batch_index].string_cells
+ try:
+ strings_batch_str = strings_batch_str.decode('utf-8', 'ignore')
+ except AttributeError:
+ # AttributeError can occur when |strings_batch_str| is an str which
+ # happens when everything in it is UTF-8 (protobuf automatically
+ # does the conversion if it can).
+ pass
+
# Null-terminated strings in a batch are concatenated
# into a single large byte array, so we split on the
# null-terminator to get the individual strings
- strings_batch = batches[batch_index].string_cells.split('\0')[:-1]
+ strings_batch = strings_batch_str.split('\0')[:-1]
self.__data_lists[TraceProcessor.QUERY_CELL_STRING_FIELD_ID].extend(
strings_batch)
self.__data_lists[TraceProcessor.QUERY_CELL_VARINT_FIELD_ID].extend(
diff --git a/test/trace_processor/chrome/scroll_jank_cause_queuing_delay.out b/test/trace_processor/chrome/scroll_jank_cause_queuing_delay.out
index eedf5a2..a54d9b9 100644
--- a/test/trace_processor/chrome/scroll_jank_cause_queuing_delay.out
+++ b/test/trace_processor/chrome/scroll_jank_cause_queuing_delay.out
@@ -8,6 +8,7 @@
"Renderer","Compositor",2918,0,7000,"InputLatency.LatencyInfo.Flow.QueuingDelay.NoJank.BlockingTasksUs.ScrollPredictor::ResampleScrollEvents"
"Renderer","Compositor",2918,0,25000,"InputLatency.LatencyInfo.Flow.QueuingDelay.NoJank.BlockingTasksUs.InputHandlerProxy::HandleGestureScrollUpdate-DeltaUnits"
"Renderer","Compositor",2918,0,6000,"InputLatency.LatencyInfo.Flow.QueuingDelay.NoJank.BlockingTasksUs.LatencyInfo.Flow"
+"Gpu","VizCompositorThread",2918,0,10000,"InputLatency.LatencyInfo.Flow.QueuingDelay.NoJank.BlockingTasksUs.LatencyInfo.Flow"
"Browser","CrProcessMain",2926,1,52000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.InputRouterImpl::GestureEventHandled-GestureEventQueue::ProcessGestureAck"
"Browser","CrProcessMain",2926,1,17000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.GestureProvider::OnTouchEvent"
"Renderer","Compositor",2926,1,1208,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.WidgetInputHandlerImpl::DispatchNonBlockingEvent-LatencyInfo.Flow"
@@ -19,3 +20,6 @@
"Gpu","VizCompositorThread",2926,1,2000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
"Gpu","VizCompositorThread",2926,1,5000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
"Gpu","VizCompositorThread",2926,1,8000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
+"Gpu","VizCompositorThread",2926,1,2000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
+"Gpu","VizCompositorThread",2926,1,8000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
+"Gpu","VizCompositorThread",2926,1,2000,"InputLatency.LatencyInfo.Flow.QueuingDelay.Jank.BlockingTasksUs.LatencyInfo.Flow"
diff --git a/test/trace_processor/chrome/scroll_jank_cause_queuing_delay_general_validation.out b/test/trace_processor/chrome/scroll_jank_cause_queuing_delay_general_validation.out
index 7e3393f..daf9c40 100644
--- a/test/trace_processor/chrome/scroll_jank_cause_queuing_delay_general_validation.out
+++ b/test/trace_processor/chrome/scroll_jank_cause_queuing_delay_general_validation.out
@@ -1,3 +1,3 @@
"total","janky_latency_info_non_jank_avg_dur","non_janky_latency_info_non_jank_avg_dur"
-139,6358.208955,6358.208955
+139,6185.000000,6185.000000
