blob: f30bd1499e01f6a42ca09cc312fb05f81c5e8b06 [file]
#!/usr/bin/env python3
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs tracing with CPU profiling enabled, and symbolizes traces if requested.
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DO NOT EDIT. Auto-generated by tools/gen_amalgamated_python_tools
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
For usage instructions, please see:
https://perfetto.dev/docs/quickstart/callstack-sampling
Adapted in large part from `heap_profile`.
"""
import argparse
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import textwrap
import time
import uuid
# ----- Amalgamator: begin of python/perfetto/prebuilts/manifests/traceconv.py
# This file has been generated by: tools/release/roll-prebuilts v56.1
TRACECONV_MANIFEST = [{
'arch':
'mac-amd64',
'file_name':
'traceconv',
'file_size':
11839400,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/mac-amd64/traceconv',
'sha256':
'7b696e45600d03fe95121cc5710852080331874396f17297610e829887300d7f',
'platform':
'darwin',
'machine': ['x86_64']
}, {
'arch':
'mac-arm64',
'file_name':
'traceconv',
'file_size':
10959176,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/mac-arm64/traceconv',
'sha256':
'99dd2ebf488af5591fb5d4be0aa1d84ac173ceed62b077dd73bfa0cff8531be4',
'platform':
'darwin',
'machine': ['arm64']
}, {
'arch':
'linux-amd64',
'file_name':
'traceconv',
'file_size':
12057312,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/linux-amd64/traceconv',
'sha256':
'0d91b41016b86733e56ed641b8bf88ee4192e01470e7e5d5feb1ecb00401cdbc',
'platform':
'linux',
'machine': ['x86_64']
}, {
'arch':
'linux-arm',
'file_name':
'traceconv',
'file_size':
9103156,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/linux-arm/traceconv',
'sha256':
'b68f21399e4e49b81631f0e720dd1e626562020ca25b3220df4f88ba96c73eea',
'platform':
'linux',
'machine': ['armv6l', 'armv7l', 'armv8l']
}, {
'arch':
'linux-arm64',
'file_name':
'traceconv',
'file_size':
11435112,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/linux-arm64/traceconv',
'sha256':
'6641337ce8fb865a85f9538c52ab563f57ed18e62b4efbe670fca484589b4c8a',
'platform':
'linux',
'machine': ['aarch64']
}, {
'arch':
'android-arm',
'file_name':
'traceconv',
'file_size':
9076156,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/android-arm/traceconv',
'sha256':
'501ef91a96d5ff2bb49df967ad169ed6f7959f4c4c910ac782672a8bf6a25cc1'
}, {
'arch':
'android-arm64',
'file_name':
'traceconv',
'file_size':
11292248,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/android-arm64/traceconv',
'sha256':
'c0488a20b6da2533557b78d10cb69d503345c3a2660460453dee5853fe27cd08'
}, {
'arch':
'android-x86',
'file_name':
'traceconv',
'file_size':
12661180,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/android-x86/traceconv',
'sha256':
'0ebd3d1a0c4c15ea32b0d15f3def3152c14967234a9ea86d6efb060a235983ed'
}, {
'arch':
'android-x64',
'file_name':
'traceconv',
'file_size':
11835984,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/android-x64/traceconv',
'sha256':
'8242ee7c63709a170c0ca3584bc1d814a420807eb1b095b224d6d48ea5bd3c31'
}, {
'arch':
'windows-amd64',
'file_name':
'traceconv.exe',
'file_size':
11760640,
'url':
'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v56.1/windows-amd64/traceconv.exe',
'sha256':
'10d420057affdfad9aaae34ade221a96740af233f03b9e5e387b7ae0f5c0ecab',
'platform':
'win32',
'machine': ['amd64']
}]
# ----- Amalgamator: end of python/perfetto/prebuilts/manifests/traceconv.py
# ----- Amalgamator: begin of python/perfetto/prebuilts/perfetto_prebuilts.py
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Functions to fetch pre-pinned Perfetto prebuilts.
This function is used in different places:
- Into the //tools/{trace_processor, traceconv} scripts, which are just plain
wrappers around executables.
- Into the //tools/{heap_profiler, record_android_trace} scripts, which contain
some other hand-written python code.
The manifest argument looks as follows:
TRACECONV_MANIFEST = [
{
'arch': 'mac-amd64',
'file_name': 'traceconv',
'file_size': 7087080,
'url': https://commondatastorage.googleapis.com/.../trace_to_text',
'sha256': 7d957c005b0dc130f5bd855d6cec27e060d38841b320d04840afc569f9087490',
'platform': 'darwin',
'machine': 'x86_64'
},
...
]
The intended usage is:
from perfetto.prebuilts.manifests.traceconv import TRACECONV_MANIFEST
bin_path = get_perfetto_prebuilt(TRACECONV_MANIFEST)
subprocess.call(bin_path, ...)
"""
import hashlib
import os
import platform
import random
import subprocess
import sys
def download_or_get_cached(file_name, url, sha256):
""" Downloads a prebuilt or returns a cached version
The first time this is invoked, it downloads the |url| and caches it into
~/.local/share/perfetto/prebuilts/$tool_name-$sha256. On subsequent
invocations it just runs the cached version.
The (short) SHA-256 is embedded in the cached file name so that several
versions of the same tool can coexist on the same machine. This matters when
e.g. two virtualenvs pin different Perfetto releases, or the //tools wrappers
and the Python API request different versions: without the SHA in the name
they would all map to the same path, clobber each other and trigger a
re-download on every switch.
"""
dir = os.path.join(
os.path.expanduser('~'), '.local', 'share', 'perfetto', 'prebuilts')
os.makedirs(dir, exist_ok=True)
# Embed the SHA in the file name, preserving any extension (e.g. .exe) as the
# last component since callers (and the OS, on Windows) rely on it.
root, ext = os.path.splitext(file_name)
bin_path = os.path.join(dir, '%s-%s%s' % (root, sha256[:16], ext))
# The cached file is only ever created via an atomic rename after the SHA-256
# has been verified, so if a file at this (SHA-named) path exists we can trust
# it without recomputing the hash on every invocation.
if os.path.exists(bin_path):
return bin_path
# Use a unique random file to guard against concurrent executions.
# See https://github.com/google/perfetto/issues/786 .
tmp_path = '%s.%d.tmp' % (bin_path, random.randint(0, 100000))
print('Downloading ' + url)
subprocess.check_call(['curl', '-f', '-L', '-#', '-o', tmp_path, url])
with open(tmp_path, 'rb') as fd:
actual_sha256 = hashlib.sha256(fd.read()).hexdigest()
if actual_sha256 != sha256:
raise Exception('Checksum mismatch for %s (actual: %s, expected: %s)' %
(url, actual_sha256, sha256))
os.chmod(tmp_path, 0o755)
os.replace(tmp_path, bin_path)
return bin_path
def get_perfetto_prebuilt(manifest, soft_fail=False, arch=None):
""" Downloads the prebuilt, if necessary, and returns its path on disk. """
plat = sys.platform.lower()
machine = platform.machine().lower()
manifest_entry = None
for entry in manifest:
# If the caller overrides the arch, just match that (for Android prebuilts).
if arch:
if entry.get('arch') == arch:
manifest_entry = entry
break
continue
# Otherwise guess the local machine arch.
if entry.get('platform') == plat and machine in entry.get('machine', []):
manifest_entry = entry
break
if manifest_entry is None:
if soft_fail:
return None
raise Exception(
('No prebuilts available for %s-%s\n' % (plat, machine)) +
'See https://perfetto.dev/docs/contributing/build-instructions')
# Placeholder entries (e.g. before a release has been rolled) have an empty
# URL. Treat them the same as a missing entry when soft_fail is set.
if not manifest_entry.get('url'):
if soft_fail:
return None
raise Exception('No prebuilt URL available for %s on %s-%s. '
'The prebuilt may not have been rolled yet.' %
(manifest_entry.get('file_name', '?'), plat, machine))
return download_or_get_cached(
file_name=manifest_entry['file_name'],
url=manifest_entry['url'],
sha256=manifest_entry['sha256'])
def run_perfetto_prebuilt(manifest):
bin_path = get_perfetto_prebuilt(manifest)
if sys.platform.lower() == 'win32':
sys.exit(subprocess.check_call([bin_path, *sys.argv[1:]]))
os.execv(bin_path, [bin_path] + sys.argv[1:])
# ----- Amalgamator: end of python/perfetto/prebuilts/perfetto_prebuilts.py
# Used for creating directories, etc.
UUID = str(uuid.uuid4())[-6:]
# See `sigint_handler` below.
IS_INTERRUPTED = False
def sigint_handler(signal, frame):
"""Useful for cleanly interrupting tracing."""
global IS_INTERRUPTED
IS_INTERRUPTED = True
def exit_with_no_profile():
sys.exit("No profiles generated.")
def exit_with_bug_report(error):
sys.exit(
"{}\n\n If this is unexpected, please consider filing a bug at: \n"
"https://perfetto.dev/docs/contributing/getting-started#bugs.".format(
error))
def adb_check_output(command):
"""Runs an `adb` command and returns its output."""
try:
return subprocess.check_output(command).decode('utf-8')
except FileNotFoundError:
sys.exit("`adb` not found: Is it installed or on PATH?")
except subprocess.CalledProcessError as error:
sys.exit("`adb` error: Are any (or multiple) devices connected?\n"
"If multiple devices are connected, please select one by "
"setting `ANDROID_SERIAL=device_id`.\n"
"{}".format(error))
except Exception as error:
exit_with_bug_report(error)
def parse_and_validate_args():
"""Parses, validates, and returns command-line arguments for this script."""
DESCRIPTION = """Runs tracing with CPU profiling enabled, and symbolizes
traces if requested.
For usage instructions, please see:
https://perfetto.dev/docs/quickstart/callstack-sampling
"""
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument(
"-f",
"--frequency",
help="Sampling frequency (Hz). "
"Default: 100 Hz.",
metavar="FREQUENCY",
type=int,
default=100)
parser.add_argument(
"-d",
"--duration",
help="Duration of profile (ms). 0 to run until interrupted. "
"Default: until interrupted by user.",
metavar="DURATION",
type=int,
default=0)
# Profiling using hardware counters.
parser.add_argument(
"-e",
"--event",
help="Use the specified hardware counter event for sampling.",
metavar="EVENT",
action="append",
# See: '//perfetto/protos/perfetto/trace/perfetto_trace.proto'.
choices=[
'HW_CPU_CYCLES', 'HW_INSTRUCTIONS', 'HW_CACHE_REFERENCES',
'HW_CACHE_MISSES', 'HW_BRANCH_INSTRUCTIONS', 'HW_BRANCH_MISSES',
'HW_BUS_CYCLES', 'HW_STALLED_CYCLES_FRONTEND',
'HW_STALLED_CYCLES_BACKEND'
],
default=[])
parser.add_argument(
"-k",
"--kernel-frames",
help="Collect kernel frames. Default: false.",
action="store_true",
default=False)
parser.add_argument(
"-n",
"--name",
help="Comma-separated list of names of processes to be profiled.",
metavar="NAMES",
default=None)
parser.add_argument(
"-p",
"--partial-matching",
help="If set, enables \"partial matching\" on the strings in --names/-n."
"Processes that are already running when profiling is started, and whose "
"names include any of the values in --names/-n as substrings will be "
"profiled.",
action="store_true")
parser.add_argument(
"-c",
"--config",
help="A custom configuration file, if any, to be used for profiling. "
"If provided, --frequency/-f, --duration/-d, and --name/-n are not used.",
metavar="CONFIG",
default=None)
parser.add_argument(
"--no-annotations",
help="Do not suffix the pprof function names with Android ART mode "
"annotations such as [jit].",
action="store_true")
parser.add_argument(
"--print-config",
action="store_true",
help="Print config instead of running. For debugging.")
parser.add_argument(
"-o",
"--output",
help="Output directory for recorded trace.",
metavar="DIRECTORY",
default=None)
args = parser.parse_args()
if args.config is not None:
if args.name is not None:
sys.exit("--name/-n should not be specified with --config/-c.")
elif args.event:
sys.exit("-e/--event should not be specified with --config/-c.")
elif args.config is None and args.name is None:
sys.exit("One of --names/-n or --config/-c is required.")
return args
def get_matching_processes(args, names_to_match):
"""Returns a list of currently-running processes whose names match
`names_to_match`.
Args:
args: The command-line arguments provided to this script.
names_to_match: The list of process names provided by the user.
"""
# Returns names as they are.
if not args.partial_matching:
return names_to_match
# Attempt to match names to names of currently running processes.
PS_PROCESS_OFFSET = 8
matching_processes = []
for line in adb_check_output(['adb', 'shell', 'ps', '-A']).splitlines():
line_split = line.split()
if len(line_split) <= PS_PROCESS_OFFSET:
continue
process = line_split[PS_PROCESS_OFFSET]
for name in names_to_match:
if name in process:
matching_processes.append(process)
break
return matching_processes
def get_perfetto_config(args):
"""Returns a Perfetto config with CPU profiling enabled for the selected
processes.
Args:
args: The command-line arguments provided to this script.
"""
if args.config is not None:
try:
with open(args.config, 'r') as config_file:
return config_file.read()
except IOError as error:
sys.exit("Unable to read config file: {}".format(error))
CONFIG_INDENT = ' '
CONFIG = textwrap.dedent('''\
buffers {{
size_kb: 2048
}}
buffers {{
size_kb: 63488
}}
data_sources {{
config {{
name: "linux.process_stats"
target_buffer: 0
process_stats_config {{
proc_stats_poll_ms: 100
}}
}}
}}
duration_ms: {duration}
write_into_file: true
flush_timeout_ms: 30000
flush_period_ms: 604800000
''')
matching_processes = []
if args.name is not None:
names_to_match = [name.strip() for name in args.name.split(',')]
matching_processes = get_matching_processes(args, names_to_match)
if not matching_processes:
sys.exit("No running processes matched for profiling.")
target_config = "\n".join(
[f'{CONFIG_INDENT}target_cmdline: "{p}"' for p in matching_processes])
events = args.event or ['SW_CPU_CLOCK']
for event in events:
CONFIG += (
textwrap.dedent('''
data_sources {{
config {{
name: "linux.perf"
target_buffer: 1
perf_event_config {{
timebase {{
counter: %s
frequency: {frequency}
timestamp_clock: PERF_CLOCK_MONOTONIC
}}
callstack_sampling {{
scope {{
{target_config}
}}
kernel_frames: {kernel_config}
}}
}}
}}
}}
''') % (event))
if args.kernel_frames:
kernel_config = "true"
else:
kernel_config = "false"
if not args.print_config:
print("Configured profiling for these processes:\n")
for matching_process in matching_processes:
print(matching_process)
print()
config = CONFIG.format(
frequency=args.frequency,
duration=args.duration,
target_config=target_config,
kernel_config=kernel_config)
return config
def release_or_newer(release):
"""Returns whether a new enough Android release is being used."""
SDK = {'T': 33}
sdk = int(
adb_check_output(
['adb', 'shell', 'getprop', 'ro.system.build.version.sdk']).strip())
if sdk >= SDK[release]:
return True
codename = adb_check_output(
['adb', 'shell', 'getprop', 'ro.build.version.codename']).strip()
return codename == release
def get_and_prepare_profile_target(args):
"""Returns the target where the trace/profile will be output. Creates a
new directory if necessary.
Args:
args: The command-line arguments provided to this script.
"""
profile_target = os.path.join(tempfile.gettempdir(), UUID)
if args.output is not None:
profile_target = args.output
else:
os.makedirs(profile_target, exist_ok=True)
if not os.path.isdir(profile_target):
sys.exit("Output directory {} not found.".format(profile_target))
if os.listdir(profile_target):
sys.exit("Output directory {} not empty.".format(profile_target))
return profile_target
def record_trace(config, profile_target):
"""Runs Perfetto with the provided configuration to record a trace.
Args:
config: The Perfetto config to be used for tracing/profiling.
profile_target: The directory where the recorded trace is output.
"""
NULL = open(os.devnull)
NO_OUT = {
'stdout': NULL,
'stderr': NULL,
}
if not release_or_newer('T'):
sys.exit("This tool requires Android T+ to run.")
# Push configuration to the device.
# On Windows, temp files cannot be accessed by external processes while open
# due to file locking, so we must close before adb push and manually cleanup.
tf = tempfile.NamedTemporaryFile(delete=False)
try:
tf.write(config.encode('utf-8'))
tf.flush()
tf.close()
profile_config_path = '/data/misc/perfetto-configs/config-' + UUID
adb_check_output(['adb', 'push', tf.name, profile_config_path])
finally:
os.remove(tf.name)
profile_device_path = '/data/misc/perfetto-traces/profile-' + UUID
perfetto_command = ('perfetto --txt -c {} -o {} -d')
try:
perfetto_pid = int(
adb_check_output([
'adb', 'exec-out',
perfetto_command.format(profile_config_path, profile_device_path)
]).strip())
except ValueError as error:
sys.exit("Unable to start profiling: {}".format(error))
print("Profiling active. Press Ctrl+C to terminate.")
old_handler = signal.signal(signal.SIGINT, sigint_handler)
perfetto_alive = True
while perfetto_alive and not IS_INTERRUPTED:
perfetto_alive = subprocess.call(
['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)], **NO_OUT) == 0
time.sleep(0.25)
print("Finishing profiling and symbolization...")
if IS_INTERRUPTED:
adb_check_output(['adb', 'shell', 'kill', '-INT', str(perfetto_pid)])
# Restore old handler.
signal.signal(signal.SIGINT, old_handler)
while perfetto_alive:
perfetto_alive = subprocess.call(
['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)]) == 0
time.sleep(0.25)
profile_host_path = os.path.join(profile_target, 'raw-trace')
adb_check_output(['adb', 'pull', profile_device_path, profile_host_path])
adb_check_output(['adb', 'shell', 'rm', profile_config_path])
adb_check_output(['adb', 'shell', 'rm', profile_device_path])
def get_traceconv():
"""Sets up and returns the path to `traceconv`."""
try:
traceconv = get_perfetto_prebuilt(TRACECONV_MANIFEST, soft_fail=True)
except Exception as error:
exit_with_bug_report(error)
if traceconv is None:
exit_with_bug_report(
"Unable to download `traceconv` for symbolizing profiles.")
return traceconv
def concatenate_files(files_to_concatenate, output_file):
"""Concatenates files.
Args:
files_to_concatenate: Paths for input files to concatenate.
output_file: Path to the resultant output file.
"""
with open(output_file, 'wb') as output:
for file in files_to_concatenate:
with open(file, 'rb') as input:
shutil.copyfileobj(input, output)
def symbolize_trace(traceconv, profile_target):
"""Attempts symbolization of the recorded trace/profile, if symbols are
available.
Args:
traceconv: The path to the `traceconv` binary used for symbolization.
profile_target: The directory where the recorded trace was output.
Returns:
The path to the symbolized trace file if symbolization was completed,
and the original trace file, if it was not.
"""
binary_path = os.getenv('PERFETTO_BINARY_PATH')
trace_file = os.path.join(profile_target, 'raw-trace')
files_to_concatenate = [trace_file]
if binary_path is not None:
try:
with open(os.path.join(profile_target, 'symbols'), 'w') as symbols_file:
return_code = subprocess.call([traceconv, 'symbolize', trace_file],
env=dict(
os.environ,
PERFETTO_BINARY_PATH=binary_path),
stdout=symbols_file)
except IOError as error:
sys.exit("Unable to write symbols to disk: {}".format(error))
if return_code == 0:
files_to_concatenate.append(os.path.join(profile_target, 'symbols'))
else:
print("Failed to symbolize. Continuing without symbols.", file=sys.stderr)
if len(files_to_concatenate) > 1:
trace_file = os.path.join(profile_target, 'symbolized-trace')
try:
concatenate_files(files_to_concatenate, trace_file)
except Exception as error:
sys.exit("Unable to write symbolized profile to disk: {}".format(error))
return trace_file
def generate_pprof_profiles(traceconv, trace_file, args):
"""Generates pprof profiles from the recorded trace.
Args:
traceconv: The path to the `traceconv` binary used for generating profiles.
trace_file: The oath to the recorded and potentially symbolized trace file.
Returns:
The directory where pprof profiles are output.
"""
try:
conversion_args = [traceconv, 'profile', '--perf'] + (
['--no-annotations'] if args.no_annotations else []) + [trace_file]
traceconv_output = subprocess.check_output(conversion_args)
except Exception as error:
exit_with_bug_report(
"Unable to extract profiles from trace: {}".format(error))
profiles_output_directory = None
for word in traceconv_output.decode('utf-8').split():
if 'perf_profile-' in word:
profiles_output_directory = word
if profiles_output_directory is None:
exit_with_no_profile()
return profiles_output_directory
def copy_profiles_to_destination(profile_target, profile_path):
"""Copies recorded profiles to `profile_target` from `profile_path`."""
profile_files = os.listdir(profile_path)
if not profile_files:
exit_with_no_profile()
try:
for profile_file in profile_files:
shutil.copy(os.path.join(profile_path, profile_file), profile_target)
except Exception as error:
sys.exit("Unable to copy profiles to {}: {}".format(profile_target, error))
print("Wrote profiles to {}".format(profile_target))
def main(argv):
args = parse_and_validate_args()
profile_target = get_and_prepare_profile_target(args)
trace_config = get_perfetto_config(args)
if args.print_config:
print(trace_config)
return 0
record_trace(trace_config, profile_target)
traceconv = get_traceconv()
trace_file = symbolize_trace(traceconv, profile_target)
copy_profiles_to_destination(
profile_target, generate_pprof_profiles(traceconv, trace_file, args))
return 0
if __name__ == '__main__':
sys.exit(main(sys.argv))