tools/cpu_profile

#!/usr/bin/env python3
# Copyright (C) 2022 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Runs tracing with CPU profiling enabled, and symbolizes traces if requested.

# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DO NOT EDIT. Auto-generated by tools/gen_amalgamated_python_tools
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

For usage instructions, please see:
https://perfetto.dev/docs/quickstart/callstack-sampling

Adapted in large part from `heap_profile`.
"""

import argparse
import os
import shutil
import signal
import subprocess
import sys
import tempfile
import textwrap
import time
import uuid


# ----- Amalgamator: begin of python/perfetto/prebuilts/manifests/traceconv.py
# This file has been generated by: /usr/local/google/home/lalitm/perfetto/tools/roll-prebuilts v31.0
TRACECONV_MANIFEST = [{
    'arch':
        'mac-amd64',
    'file_name':
        'traceconv',
    'file_size':
        7772872,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/mac-amd64/traceconv',
    'sha256':
        'a987cfd2e722994a5911032d046ec2d0b912845a83b093226c3fccd5e316ed01',
    'platform':
        'darwin',
    'machine': ['x86_64']
}, {
    'arch':
        'mac-arm64',
    'file_name':
        'traceconv',
    'file_size':
        6554552,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/mac-arm64/traceconv',
    'sha256':
        '972610d249990d4b03de50bece4a7adf03ebd6b3cc3c2c10feb7bb6561c1fce1',
    'platform':
        'darwin',
    'machine': ['arm64']
}, {
    'arch':
        'linux-amd64',
    'file_name':
        'traceconv',
    'file_size':
        8073432,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/linux-amd64/traceconv',
    'sha256':
        'e34f6ddd91ad0de62bfbef76303f2a94136470fe5a4f679af2f5b8898548ac13',
    'platform':
        'linux',
    'machine': ['x86_64']
}, {
    'arch':
        'linux-arm',
    'file_name':
        'traceconv',
    'file_size':
        6677612,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/linux-arm/traceconv',
    'sha256':
        '55c9cd6e5c5bc65210068a173b0c16ad3323faaba0fccec20247dca5bd3b913d',
    'platform':
        'linux',
    'machine': ['armv6l', 'armv7l', 'armv8l']
}, {
    'arch':
        'linux-arm64',
    'file_name':
        'traceconv',
    'file_size':
        7543648,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/linux-arm64/traceconv',
    'sha256':
        '2477857470f613410f6151acb06b6b6a067ef8612d619eee86dd2a3b8aac2a3e',
    'platform':
        'linux',
    'machine': ['aarch64']
}, {
    'arch':
        'android-arm',
    'file_name':
        'traceconv',
    'file_size':
        5360020,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/android-arm/traceconv',
    'sha256':
        '60cecbd8d9b6357bb89929fa5afa1eba6c1d46d621e62ee558509424d88bd53d'
}, {
    'arch':
        'android-arm64',
    'file_name':
        'traceconv',
    'file_size':
        6769184,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/android-arm64/traceconv',
    'sha256':
        'e373cbed4502977290abe3e246132ced538fca33881058e6049676ef9c613dd9'
}, {
    'arch':
        'android-x86',
    'file_name':
        'traceconv',
    'file_size':
        7682412,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/android-x86/traceconv',
    'sha256':
        'dcaa5d156247433537d59ad33238feb23dc0f97ba64c51581360f25ac0fc16a8'
}, {
    'arch':
        'android-x64',
    'file_name':
        'traceconv',
    'file_size':
        7903832,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/android-x64/traceconv',
    'sha256':
        'b73b8d398019b2a349d3748b4fe4096a069c6362697375d3c2668cf79ef1bb38'
}, {
    'arch':
        'windows-amd64',
    'file_name':
        'traceconv.exe',
    'file_size':
        7188992,
    'url':
        'https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v31.0/windows-amd64/traceconv.exe',
    'sha256':
        '9cbdfcad3b5f2164d0c9e79d4f8b749db693814ae97fba038385d2e6f0111f72',
    'platform':
        'win32',
    'machine': ['amd64']
}]

# ----- Amalgamator: end of python/perfetto/prebuilts/manifests/traceconv.py

# ----- Amalgamator: begin of python/perfetto/prebuilts/perfetto_prebuilts.py
# Copyright (C) 2021 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Functions to fetch pre-pinned Perfetto prebuilts.

This function is used in different places:
- Into the //tools/{trace_processor, traceconv} scripts, which are just plain
  wrappers around executables.
- Into the //tools/{heap_profiler, record_android_trace} scripts, which contain
  some other hand-written python code.

The manifest argument looks as follows:
TRACECONV_MANIFEST = [
  {
    'arch': 'mac-amd64',
    'file_name': 'traceconv',
    'file_size': 7087080,
    'url': https://commondatastorage.googleapis.com/.../trace_to_text',
    'sha256': 7d957c005b0dc130f5bd855d6cec27e060d38841b320d04840afc569f9087490',
    'platform': 'darwin',
    'machine': 'x86_64'
  },
  ...
]

The intended usage is:

  from perfetto.prebuilts.manifests.traceconv import TRACECONV_MANIFEST
  bin_path = get_perfetto_prebuilt(TRACECONV_MANIFEST)
  subprocess.call(bin_path, ...)
"""

import hashlib
import os
import platform
import subprocess
import sys


def download_or_get_cached(file_name, url, sha256):
  """ Downloads a prebuilt or returns a cached version

  The first time this is invoked, it downloads the |url| and caches it into
  ~/.local/share/perfetto/prebuilts/$tool_name. On subsequent invocations it
  just runs the cached version.
  """
  dir = os.path.join(
      os.path.expanduser('~'), '.local', 'share', 'perfetto', 'prebuilts')
  os.makedirs(dir, exist_ok=True)
  bin_path = os.path.join(dir, file_name)
  sha256_path = os.path.join(dir, file_name + '.sha256')
  needs_download = True

  # Avoid recomputing the SHA-256 on each invocation. The SHA-256 of the last
  # download is cached into file_name.sha256, just check if that matches.
  if os.path.exists(bin_path) and os.path.exists(sha256_path):
    with open(sha256_path, 'rb') as f:
      digest = f.read().decode()
      if digest == sha256:
        needs_download = False

  if needs_download:
    # Either the filed doesn't exist or the SHA256 doesn't match.
    tmp_path = bin_path + '.tmp'
    print('Downloading ' + url)
    subprocess.check_call(['curl', '-f', '-L', '-#', '-o', tmp_path, url])
    with open(tmp_path, 'rb') as fd:
      actual_sha256 = hashlib.sha256(fd.read()).hexdigest()
    if actual_sha256 != sha256:
      raise Exception('Checksum mismatch for %s (actual: %s, expected: %s)' %
                      (url, actual_sha256, sha256))
    os.chmod(tmp_path, 0o755)
    os.replace(tmp_path, bin_path)
    with open(sha256_path, 'w') as f:
      f.write(sha256)
  return bin_path


def get_perfetto_prebuilt(manifest, soft_fail=False, arch=None):
  """ Downloads the prebuilt, if necessary, and returns its path on disk. """
  plat = sys.platform.lower()
  machine = platform.machine().lower()
  manifest_entry = None
  for entry in manifest:
    # If the caller overrides the arch, just match that (for Android prebuilts).
    if arch:
      if entry.get('arch') == arch:
        manifest_entry = entry
        break
      continue
    # Otherwise guess the local machine arch.
    if entry.get('platform') == plat and machine in entry.get('machine', []):
      manifest_entry = entry
      break
  if manifest_entry is None:
    if soft_fail:
      return None
    raise Exception(
        ('No prebuilts available for %s-%s\n' % (plat, machine)) +
        'See https://perfetto.dev/docs/contributing/build-instructions')

  return download_or_get_cached(
      file_name=manifest_entry['file_name'],
      url=manifest_entry['url'],
      sha256=manifest_entry['sha256'])


def run_perfetto_prebuilt(manifest):
  bin_path = get_perfetto_prebuilt(manifest)
  if sys.platform.lower() == 'win32':
    sys.exit(subprocess.check_call([bin_path, *sys.argv[1:]]))
  os.execv(bin_path, [bin_path] + sys.argv[1:])

# ----- Amalgamator: end of python/perfetto/prebuilts/perfetto_prebuilts.py

# Used for creating directories, etc.
UUID = str(uuid.uuid4())[-6:]

# See `sigint_handler` below.
IS_INTERRUPTED = False


def sigint_handler(signal, frame):
  """Useful for cleanly interrupting tracing."""
  global IS_INTERRUPTED
  IS_INTERRUPTED = True


def exit_with_no_profile():
  sys.exit("No profiles generated.")


def exit_with_bug_report(error):
  sys.exit(
      "{}\n\n If this is unexpected, please consider filing a bug at: \n"
      "https://perfetto.dev/docs/contributing/getting-started#bugs.".format(
          error))


def adb_check_output(command):
  """Runs an `adb` command and returns its output."""
  try:
    return subprocess.check_output(command).decode('utf-8')
  except FileNotFoundError:
    sys.exit("`adb` not found: Is it installed or on PATH?")
  except subprocess.CalledProcessError as error:
    sys.exit("`adb` error: Are any (or multiple) devices connected?\n"
             "If multiple devices are connected, please select one by "
             "setting `ANDROID_SERIAL=device_id`.\n"
             "{}".format(error))
  except Exception as error:
    exit_with_bug_report(error)


def parse_and_validate_args():
  """Parses, validates, and returns command-line arguments for this script."""
  DESCRIPTION = """Runs tracing with CPU profiling enabled, and symbolizes
  traces if requested.

  For usage instructions, please see:
  https://perfetto.dev/docs/quickstart/cpu-profiling
  """
  parser = argparse.ArgumentParser(description=DESCRIPTION)
  parser.add_argument(
      "-f",
      "--frequency",
      help="Sampling frequency (Hz). "
      "Default: 100 Hz.",
      metavar="FREQUENCY",
      type=int,
      default=100)
  parser.add_argument(
      "-d",
      "--duration",
      help="Duration of profile (ms). 0 to run until interrupted. "
      "Default: until interrupted by user.",
      metavar="DURATION",
      type=int,
      default=0)
  parser.add_argument(
      "-n",
      "--name",
      help="Comma-separated list of names of processes to be profiled.",
      metavar="NAMES",
      default=None)
  parser.add_argument(
      "-p",
      "--partial-matching",
      help="If set, enables \"partial matching\" on the strings in --names/-n."
      "Processes that are already running when profiling is started, and whose "
      "names include any of the values in --names/-n as substrings will be "
      "profiled.",
      action="store_true")
  parser.add_argument(
      "-c",
      "--config",
      help="A custom configuration file, if any, to be used for profiling. "
      "If provided, --frequency/-f, --duration/-d, and --name/-n are not used.",
      metavar="CONFIG",
      default=None)
  parser.add_argument(
      "--no-annotations",
      help="Do not suffix the pprof function names with Android ART mode "
      "annotations such as [jit].",
      action="store_true")
  parser.add_argument(
      "--print-config",
      action="store_true",
      help="Print config instead of running. For debugging.")
  parser.add_argument(
      "-o",
      "--output",
      help="Output directory for recorded trace.",
      metavar="DIRECTORY",
      default=None)

  args = parser.parse_args()
  if args.config is not None and args.name is not None:
    sys.exit("--name/-n should not be provided when --config/-c is provided.")
  elif args.config is None and args.name is None:
    sys.exit("One of --names/-n or --config/-c is required.")

  return args


def get_matching_processes(args, names_to_match):
  """Returns a list of currently-running processes whose names match
  `names_to_match`.

  Args:
    args: The command-line arguments provided to this script.
    names_to_match: The list of process names provided by the user.
  """
  # Returns names as they are.
  if not args.partial_matching:
    return names_to_match

  # Attempt to match names to names of currently running processes.
  PS_PROCESS_OFFSET = 8
  matching_processes = []
  for line in adb_check_output(['adb', 'shell', 'ps', '-A']).splitlines():
    line_split = line.split()
    if len(line_split) <= PS_PROCESS_OFFSET:
      continue
    process = line_split[PS_PROCESS_OFFSET]
    for name in names_to_match:
      if name in process:
        matching_processes.append(process)
        break

  return matching_processes


def get_perfetto_config(args):
  """Returns a Perfetto config with CPU profiling enabled for the selected
  processes.

  Args:
    args: The command-line arguments provided to this script.
  """
  if args.config is not None:
    try:
      with open(args.config, 'r') as config_file:
        return config_file.read()
    except IOError as error:
      sys.exit("Unable to read config file: {}".format(error))

  CONFIG_INDENT = '      '
  CONFIG = textwrap.dedent('''\
  buffers {{
    size_kb: 2048
  }}

  buffers {{
    size_kb: 63488
  }}

  data_sources {{
    config {{
      name: "linux.process_stats"
      target_buffer: 0
      process_stats_config {{
        proc_stats_poll_ms: 100
      }}
    }}
  }}

  data_sources {{
    config {{
      name: "linux.perf"
      target_buffer: 1
      perf_event_config {{
        all_cpus: true
        sampling_frequency: {frequency}
  {target_config}
      }}
    }}
  }}

  duration_ms: {duration}
  write_into_file: true
  flush_timeout_ms: 30000
  flush_period_ms: 604800000
  ''')

  matching_processes = []
  if args.name is not None:
    names_to_match = [name.strip() for name in args.name.split(',')]
    matching_processes = get_matching_processes(args, names_to_match)

  if not matching_processes:
    sys.exit("No running processes matched for profiling.")

  target_config = "\n".join(
      [f'{CONFIG_INDENT}target_cmdline: "{p}"' for p in matching_processes])

  if not args.print_config:
    print("Configured profiling for these processes:\n")
    for matching_process in matching_processes:
      print(matching_process)
    print()

  config = CONFIG.format(
      frequency=args.frequency,
      duration=args.duration,
      target_config=target_config)

  return config


def release_or_newer(release):
  """Returns whether a new enough Android release is being used."""
  SDK = {'R': 30}
  sdk = int(
      adb_check_output(
          ['adb', 'shell', 'getprop', 'ro.system.build.version.sdk']).strip())
  if sdk >= SDK[release]:
    return True

  codename = adb_check_output(
      ['adb', 'shell', 'getprop', 'ro.build.version.codename']).strip()
  return codename == release


def get_and_prepare_profile_target(args):
  """Returns the target where the trace/profile will be output.  Creates a
  new directory if necessary.

  Args:
    args: The command-line arguments provided to this script.
  """
  profile_target = os.path.join(tempfile.gettempdir(), UUID)
  if args.output is not None:
    profile_target = args.output
  else:
    os.makedirs(profile_target, exist_ok=True)
  if not os.path.isdir(profile_target):
    sys.exit("Output directory {} not found.".format(profile_target))
  if os.listdir(profile_target):
    sys.exit("Output directory {} not empty.".format(profile_target))

  return profile_target


def record_trace(config, profile_target):
  """Runs Perfetto with the provided configuration to record a trace.

  Args:
    config: The Perfetto config to be used for tracing/profiling.
    profile_target: The directory where the recorded trace is output.
  """
  NULL = open(os.devnull)
  NO_OUT = {
      'stdout': NULL,
      'stderr': NULL,
  }
  if not release_or_newer('R'):
    sys.exit("This tool requires Android R+ to run.")

  # Push configuration to the device.
  tf = tempfile.NamedTemporaryFile()
  tf.file.write(config.encode('utf-8'))
  tf.file.flush()
  profile_config_path = '/data/misc/perfetto-configs/config-' + UUID
  adb_check_output(['adb', 'push', tf.name, profile_config_path])
  tf.close()


  profile_device_path = '/data/misc/perfetto-traces/profile-' + UUID
  perfetto_command = ('perfetto --txt -c {} -o {} -d')
  try:
    perfetto_pid = int(
        adb_check_output([
            'adb', 'exec-out',
            perfetto_command.format(profile_config_path, profile_device_path)
        ]).strip())
  except ValueError as error:
    sys.exit("Unable to start profiling: {}".format(error))

  print("Profiling active. Press Ctrl+C to terminate.")

  old_handler = signal.signal(signal.SIGINT, sigint_handler)

  perfetto_alive = True
  while perfetto_alive and not IS_INTERRUPTED:
    perfetto_alive = subprocess.call(
        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)], **NO_OUT) == 0
    time.sleep(0.25)

  print("Finishing profiling and symbolization...")

  if IS_INTERRUPTED:
    adb_check_output(['adb', 'shell', 'kill', '-INT', str(perfetto_pid)])

  # Restore old handler.
  signal.signal(signal.SIGINT, old_handler)

  while perfetto_alive:
    perfetto_alive = subprocess.call(
        ['adb', 'shell', '[ -d /proc/{} ]'.format(perfetto_pid)]) == 0
    time.sleep(0.25)

  profile_host_path = os.path.join(profile_target, 'raw-trace')
  adb_check_output(['adb', 'pull', profile_device_path, profile_host_path])
  adb_check_output(['adb', 'shell', 'rm', profile_config_path])
  adb_check_output(['adb', 'shell', 'rm', profile_device_path])


def get_traceconv():
  """Sets up and returns the path to `traceconv`."""
  try:
    traceconv = get_perfetto_prebuilt(TRACECONV_MANIFEST, soft_fail=True)
  except Exception as error:
    exit_with_bug_report(error)
  if traceconv is None:
    exit_with_bug_report(
        "Unable to download `traceconv` for symbolizing profiles.")

  return traceconv


def concatenate_files(files_to_concatenate, output_file):
  """Concatenates files.

  Args:
    files_to_concatenate: Paths for input files to concatenate.
    output_file: Path to the resultant output file.
  """
  with open(output_file, 'wb') as output:
    for file in files_to_concatenate:
      with open(file, 'rb') as input:
        shutil.copyfileobj(input, output)


def symbolize_trace(traceconv, profile_target):
  """Attempts symbolization of the recorded trace/profile, if symbols are
  available.

  Args:
    traceconv: The path to the `traceconv` binary used for symbolization.
    profile_target: The directory where the recorded trace was output.

  Returns:
    The path to the symbolized trace file if symbolization was completed,
    and the original trace file, if it was not.
  """
  binary_path = os.getenv('PERFETTO_BINARY_PATH')
  trace_file = os.path.join(profile_target, 'raw-trace')
  files_to_concatenate = [trace_file]

  if binary_path is not None:
    try:
      with open(os.path.join(profile_target, 'symbols'), 'w') as symbols_file:
        return_code = subprocess.call([traceconv, 'symbolize', trace_file],
                                      env=dict(
                                          os.environ,
                                          PERFETTO_BINARY_PATH=binary_path),
                                      stdout=symbols_file)
    except IOError as error:
      sys.exit("Unable to write symbols to disk: {}".format(error))
    if return_code == 0:
      files_to_concatenate.append(os.path.join(profile_target, 'symbols'))
    else:
      print("Failed to symbolize. Continuing without symbols.", file=sys.stderr)

  if len(files_to_concatenate) > 1:
    trace_file = os.path.join(profile_target, 'symbolized-trace')
    try:
      concatenate_files(files_to_concatenate, trace_file)
    except Exception as error:
      sys.exit("Unable to write symbolized profile to disk: {}".format(error))

  return trace_file


def generate_pprof_profiles(traceconv, trace_file, args):
  """Generates pprof profiles from the recorded trace.

  Args:
    traceconv: The path to the `traceconv` binary used for generating profiles.
    trace_file: The oath to the recorded and potentially symbolized trace file.

  Returns:
    The directory where pprof profiles are output.
  """
  try:
    conversion_args = [traceconv, 'profile', '--perf'] + (
        ['--no-annotations'] if args.no_annotations else []) + [trace_file]
    traceconv_output = subprocess.check_output(conversion_args)
  except Exception as error:
    exit_with_bug_report(
        "Unable to extract profiles from trace: {}".format(error))

  profiles_output_directory = None
  for word in traceconv_output.decode('utf-8').split():
    if 'perf_profile-' in word:
      profiles_output_directory = word
  if profiles_output_directory is None:
    exit_with_no_profile()
  return profiles_output_directory


def copy_profiles_to_destination(profile_target, profile_path):
  """Copies recorded profiles to `profile_target` from `profile_path`."""
  profile_files = os.listdir(profile_path)
  if not profile_files:
    exit_with_no_profile()

  try:
    for profile_file in profile_files:
      shutil.copy(os.path.join(profile_path, profile_file), profile_target)
  except Exception as error:
    sys.exit("Unable to copy profiles to {}: {}".format(profile_target, error))

  print("Wrote profiles to {}".format(profile_target))


def main(argv):
  args = parse_and_validate_args()
  profile_target = get_and_prepare_profile_target(args)
  trace_config = get_perfetto_config(args)
  if args.print_config:
    print(trace_config)
    return 0
  record_trace(trace_config, profile_target)
  traceconv = get_traceconv()
  trace_file = symbolize_trace(traceconv, profile_target)
  copy_profiles_to_destination(
      profile_target, generate_pprof_profiles(traceconv, trace_file, args))
  return 0


if __name__ == '__main__':
  sys.exit(main(sys.argv))