tools/gen_android_bp

#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This tool translates a collection of BUILD.gn files into a mostly equivalent
# Android.bp file for the Android Soong build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
#   gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Android.bp
# build rules. The dependencies for the GN labels are squashed to the generated
# Android.bp target, except for actions which get their own genrule. Some
# libraries are also mapped to their Android equivalents -- see |builtin_deps|.

import argparse
import json
import os
import re
import sys
from typing import Dict
from typing import List
from typing import Optional

import gn_utils
from gn_utils import GnParser

from compat import itervalues

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

# Arguments for the GN output directory.
gn_args = ' '.join([
    'is_debug=false',
    'is_perfetto_build_generator=true',
    'perfetto_build_with_android=true',
    'target_cpu="arm"',
    'target_os="android"',
])

# Default targets to translate to the blueprint file.
default_targets = [
    '//:libperfetto_client_experimental',
    '//:libperfetto',
    '//:perfetto_integrationtests',
    '//:perfetto_unittests',
    '//protos/perfetto/trace:perfetto_trace_protos',
    '//protos/perfetto/trace/android:perfetto_winscope_extensions_zero',
    '//src/android_internal:libperfetto_android_internal',
    '//src/base:perfetto_base_default_platform',
    '//src/shared_lib:libperfetto_c',
    '//src/perfetto_cmd:perfetto',
    '//src/perfetto_cmd:trigger_perfetto',
    '//src/profiling/memory:heapprofd_client',
    '//src/profiling/memory:heapprofd_client_api',
    '//src/profiling/memory:heapprofd_api_noop',
    '//src/profiling/memory:heapprofd',
    '//src/profiling/memory:heapprofd_standalone_client',
    '//src/profiling/perf:traced_perf',
    '//src/traced/probes:traced_probes',
    '//src/traced/service:traced',
    '//src/traced_relay:traced_relay',
    '//src/trace_processor:trace_processor_shell',
    '//src/trace_redaction:trace_redactor',
    '//test/cts:perfetto_cts_deps',
    '//test/cts:perfetto_cts_jni_deps',
    '//test:perfetto_gtest_logcat_printer',
    '//test:perfetto_end_to_end_integrationtests',
    '//test/vts:perfetto_vts_deps',
]

# Host targets
ipc_plugin = '//src/ipc/protoc_plugin:ipc_plugin(%s)' % gn_utils.HOST_TOOLCHAIN
protozero_plugin = '//src/protozero/protoc_plugin:protozero_plugin(%s)' % (
    gn_utils.HOST_TOOLCHAIN)
cppgen_plugin = '//src/protozero/protoc_plugin:cppgen_plugin(%s)' % (
    gn_utils.HOST_TOOLCHAIN)

default_targets += [
    '//src/traceconv:traceconv(%s)' % gn_utils.HOST_TOOLCHAIN,
    protozero_plugin,
    ipc_plugin,
]

# Defines a custom init_rc argument to be applied to the corresponding output
# blueprint target.
target_initrc = {
    '//src/traced/service:traced': {'perfetto.rc'},
    '//src/profiling/memory:heapprofd': {'heapprofd.rc'},
    '//src/profiling/perf:traced_perf': {'traced_perf.rc'},
}

target_host_supported = [
    '//:libperfetto',
    '//:libperfetto_client_experimental',
    '//protos/perfetto/trace:perfetto_trace_protos',
    '//protos/perfetto/trace/android:perfetto_winscope_extensions_zero',
    '//src/shared_lib:libperfetto_c',
    '//src/trace_processor:demangle',
    '//src/trace_processor:trace_processor_shell',
    '//src/traced/probes:traced_probes',
    '//src/traced/service:traced',
]

target_vendor_available = [
    '//:libperfetto_client_experimental',
]

target_product_available = [
    '//:libperfetto_client_experimental',
]

# Proto target groups which will be made public.
proto_groups = {
    'trace': {
        'types': ['lite'],
        'targets': [
            '//protos/perfetto/trace:non_minimal_source_set',
            '//protos/perfetto/trace:minimal_source_set',
        ]
    },
    'winscope': {
        'types': ['filegroup'],
        'targets': [
            '//protos/perfetto/trace:non_minimal_source_set',
            '//protos/perfetto/trace/android:winscope_extensions_source_set',
        ]
    },
    'config': {
        'types': ['filegroup'],
        'targets': [
            '//protos/perfetto/config:source_set',
        ]
    },
}

needs_libfts = [
    '//:perfetto_unittests',
    '//src/trace_processor:trace_processor_shell',
    '//src/traceconv:traceconv',
]

# All module names are prefixed with this string to avoid collisions.
module_prefix = 'perfetto_'

# Shared libraries which are directly translated to Android system equivalents.
shared_library_allowlist = [
    'android',
    'android.hardware.atrace@1.0',
    'android.hardware.health@2.0',
    'android.hardware.health-V2-ndk',
    'android.hardware.power.stats@1.0',
    'android.hardware.power.stats-V1-cpp',
    'base',
    'binder',
    'binder_ndk',
    'cutils',
    'hidlbase',
    'hidltransport',
    'hwbinder',
    'incident',
    'log',
    'services',
    'statssocket',
    'tracingproxy',
    'utils',
    'statspull',
]

# Static libraries which are directly translated to Android system equivalents.
static_library_allowlist = [
    'statslog_perfetto',
]

# Name of the module which settings such as compiler flags for all other
# modules.
defaults_module = module_prefix + 'defaults'

# Location of the project in the Android source tree.
tree_path = 'external/perfetto'

# Path for the protobuf sources in the standalone build.
buildtools_protobuf_src = '//buildtools/protobuf/src'

# Location of the protobuf src dir in the Android source tree.
android_protobuf_src = 'external/protobuf/src'

# Compiler flags which are passed through to the blueprint.
cflag_allowlist = r'^-DPERFETTO.*$'

# Compiler defines which are passed through to the blueprint.
define_allowlist = r'^(GOOGLE_PROTO.*)|(ZLIB_.*)|(USE_MMAP)$'

# The directory where the generated perfetto_build_flags.h will be copied into.
buildflags_dir = 'include/perfetto/base/build_configs/android_tree'

def enumerate_data_deps():
  with open(os.path.join(ROOT_DIR, 'tools', 'test_data.txt')) as f:
    lines = f.readlines()
  for line in (line.strip() for line in lines if not line.startswith('#')):
    assert os.path.exists(line), 'file %s should exist' % line
    if line.startswith('test/data/'):
      # Skip test data files that require GCS. They are only for benchmarks.
      # We don't run benchmarks in the android tree.
      continue
    if line.endswith('/.'):
      yield line[:-1] + '**/*'
    else:
      yield line


# Additional arguments to apply to Android.bp rules.
additional_args = {
    'heapprofd_client_api': [
        ('static_libs', {'libasync_safe'}),
        # heapprofd_client_api MUST NOT have global constructors. Because it
        # is loaded in an __attribute__((constructor)) of libc, we cannot
        # guarantee that the global constructors get run before it is used.
        ('cflags', {'-Wglobal-constructors', '-Werror=global-constructors'}),
        ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
        ('stubs', {
            'versions': ['S'],
            'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt',
        }),
        ('export_include_dirs', {'src/profiling/memory/include'}),
    ],
    'heapprofd_api_noop': [
        ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
        ('stubs', {
            'versions': ['S'],
            'symbol_file': 'src/profiling/memory/heapprofd_client_api.map.txt',
        }),
        ('export_include_dirs', {'src/profiling/memory/include'}),
    ],
    'heapprofd_client': [
        ('include_dirs', {'bionic/libc'}),
        ('static_libs', {'libasync_safe'}),
    ],
    'heapprofd_standalone_client': [
        ('static_libs', {'libasync_safe'}),
        ('version_script', 'src/profiling/memory/heapprofd_client_api.map.txt'),
        ('export_include_dirs', {'src/profiling/memory/include'}),
        ('stl', 'libc++_static'),
    ],
    'perfetto_unittests': [
        ('data', set(enumerate_data_deps())),
        ('include_dirs', {'bionic/libc/kernel'}),
    ],
    'perfetto_integrationtests': [
        ('test_suites', {'general-tests'}),
        ('test_config', 'PerfettoIntegrationTests.xml'),
    ],
    'libperfetto_android_internal': [('static_libs', {'libhealthhalutils'}),],
    'trace_processor_shell': [
        ('strip', {
            'all': True
        }),
        ('host', {
            'stl': 'libc++_static',
            'dist': {
                'targets': ['sdk_repo']
            },
        }),
    ],
    'libperfetto_client_experimental': [
        ('apex_available', {
            '//apex_available:platform', '//apex_available:anyapex'
        }),
        ('min_sdk_version', '30'),
        ('shared_libs', {'liblog'}),
        ('export_include_dirs', {'include', buildflags_dir}),
    ],
    'libperfetto_c': [
        ('min_sdk_version', '30'),
        ('export_include_dirs', {'include'}),
        ('cflags', {'-DPERFETTO_SHLIB_SDK_IMPLEMENTATION'}),
    ],
    'perfetto_trace_protos': [
        ('apex_available', {
            '//apex_available:platform', 'com.android.art',
            'com.android.art.debug'
        }),
        ('min_sdk_version', 'S'),
    ],
    'libperfetto': [('export_include_dirs', {'include', buildflags_dir}),],
    'perfetto': [('required', {'perfetto_persistent_cfg.pbtxt'}),],
}


def enable_base_platform(module):
  module.srcs.add(':perfetto_base_default_platform')


def enable_gtest_and_gmock(module):
  module.static_libs.add('libgmock')
  module.static_libs.add('libgtest')
  if module.name != 'perfetto_gtest_logcat_printer':
    module.whole_static_libs.add('perfetto_gtest_logcat_printer')


def enable_protobuf_full(module):
  if module.type == 'cc_binary_host':
    module.static_libs.add('libprotobuf-cpp-full')
  elif module.host_supported:
    module.host.static_libs.add('libprotobuf-cpp-full')
    module.android.shared_libs.add('libprotobuf-cpp-full')
  else:
    module.shared_libs.add('libprotobuf-cpp-full')


def enable_protobuf_lite(module):
  module.shared_libs.add('libprotobuf-cpp-lite')


def enable_protoc_lib(module):
  if module.type == 'cc_binary_host':
    module.static_libs.add('libprotoc')
  else:
    module.shared_libs.add('libprotoc')


def enable_libunwindstack(module):
  if module.name != 'heapprofd_standalone_client':
    module.shared_libs.add('libunwindstack')
    module.shared_libs.add('libprocinfo')
    module.shared_libs.add('libbase')
  else:
    module.static_libs.add('libunwindstack')
    module.static_libs.add('libprocinfo')
    module.static_libs.add('libbase')
    module.static_libs.add('liblzma')
    module.static_libs.add('libdexfile_support')
    module.runtime_libs.add('libdexfile')  # libdexfile_support dependency


def enable_libunwind(module):
  # libunwind is disabled on Darwin so we cannot depend on it.
  pass


def enable_sqlite(module):
  if module.type == 'cc_binary_host':
    module.static_libs.add('libsqlite_static_noicu')
    module.static_libs.add('sqlite_ext_percentile')
  elif module.host_supported:
    # Copy what the sqlite3 command line tool does.
    module.android.shared_libs.add('libsqlite')
    module.android.shared_libs.add('libicu')
    module.android.shared_libs.add('liblog')
    module.android.shared_libs.add('libutils')
    module.android.static_libs.add('sqlite_ext_percentile')
    module.host.static_libs.add('libsqlite_static_noicu')
    module.host.static_libs.add('sqlite_ext_percentile')
  else:
    module.shared_libs.add('libsqlite')
    module.shared_libs.add('libicu')
    module.shared_libs.add('liblog')
    module.shared_libs.add('libutils')
    module.static_libs.add('sqlite_ext_percentile')


def enable_zlib(module):
  if module.type == 'cc_binary_host':
    module.static_libs.add('libz')
  elif module.host_supported:
    module.android.shared_libs.add('libz')
    module.host.static_libs.add('libz')
  else:
    module.shared_libs.add('libz')


def enable_uapi_headers(module):
  module.include_dirs.add('bionic/libc/kernel')


def enable_bionic_libc_platform_headers_on_android(module):
  module.header_libs.add('bionic_libc_platform_headers')


# Android equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
    '//gn:default_deps':
        lambda x: None,
    '//gn:gtest_main':
        lambda x: None,
    '//gn:protoc':
        lambda x: None,
    '//gn:base_platform':
        enable_base_platform,
    '//gn:gtest_and_gmock':
        enable_gtest_and_gmock,
    '//gn:libunwind':
        enable_libunwind,
    '//gn:protobuf_full':
        enable_protobuf_full,
    '//gn:protobuf_lite':
        enable_protobuf_lite,
    '//gn:protoc_lib':
        enable_protoc_lib,
    '//gn:libunwindstack':
        enable_libunwindstack,
    '//gn:sqlite':
        enable_sqlite,
    '//gn:zlib':
        enable_zlib,
    '//gn:bionic_kernel_uapi_headers':
        enable_uapi_headers,
    '//src/profiling/memory:bionic_libc_platform_headers_on_android':
        enable_bionic_libc_platform_headers_on_android,
}

# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------


class Error(Exception):
  pass


class ThrowingArgumentParser(argparse.ArgumentParser):

  def __init__(self, context):
    super(ThrowingArgumentParser, self).__init__()
    self.context = context

  def error(self, message):
    raise Error('%s: %s' % (self.context, message))


def write_blueprint_key_value(output, name, value, sort=True):
  """Writes a Blueprint key-value pair to the output"""

  if isinstance(value, bool):
    if value:
      output.append('    %s: true,' % name)
    else:
      output.append('    %s: false,' % name)
    return
  if not value:
    return
  if isinstance(value, set):
    value = sorted(value)
  if isinstance(value, list):
    output.append('    %s: [' % name)
    for item in sorted(value) if sort else value:
      output.append('        "%s",' % item)
    output.append('    ],')
    return
  if isinstance(value, Target):
    value.to_string(output)
    return
  if isinstance(value, dict):
    kv_output = []
    for k, v in value.items():
      write_blueprint_key_value(kv_output, k, v)

    output.append('    %s: {' % name)
    for line in kv_output:
      output.append('    %s' % line)
    output.append('    },')
    return
  output.append('    %s: "%s",' % (name, value))


class Target(object):
  """A target-scoped part of a module"""

  def __init__(self, name):
    self.name = name
    self.shared_libs = set()
    self.static_libs = set()
    self.whole_static_libs = set()
    self.cflags = set()
    self.dist = dict()
    self.strip = dict()
    self.stl = None

  def to_string(self, output):
    nested_out = []
    self._output_field(nested_out, 'shared_libs')
    self._output_field(nested_out, 'static_libs')
    self._output_field(nested_out, 'whole_static_libs')
    self._output_field(nested_out, 'cflags')
    self._output_field(nested_out, 'stl')
    self._output_field(nested_out, 'dist')
    self._output_field(nested_out, 'strip')

    if nested_out:
      output.append('    %s: {' % self.name)
      for line in nested_out:
        output.append('    %s' % line)
      output.append('    },')

  def _output_field(self, output, name, sort=True):
    value = getattr(self, name)
    return write_blueprint_key_value(output, name, value, sort)


class Module(object):
  """A single module (e.g., cc_binary, cc_test) in a blueprint."""

  def __init__(self, mod_type, name, gn_target):
    assert (gn_target)
    self.type = mod_type
    self.gn_target = gn_target
    self.name = name
    self.srcs = set()
    self.main: Optional[str] = None
    self.comment = 'GN: ' + gn_utils.label_without_toolchain(gn_target)
    self.shared_libs = set()
    self.static_libs = set()
    self.whole_static_libs = set()
    self.runtime_libs = set()
    self.tools = set()
    self.cmd: Optional[str] = None
    self.host_supported = False
    self.vendor_available = False
    self.product_available = False
    self.init_rc = set()
    self.out = set()
    self.export_include_dirs = set()
    self.generated_headers = set()
    self.export_generated_headers = set()
    self.defaults = set()
    self.cflags = set()
    self.include_dirs = set()
    self.header_libs = set()
    self.required = set()
    self.user_debug_flag = False
    self.tool_files: Optional[List[str]] = None
    self.android = Target('android')
    self.host = Target('host')
    self.musl = Target('musl')
    self.lto: Optional[bool] = None
    self.stl = None
    self.dist = dict()
    self.strip = dict()
    self.data = set()
    self.apex_available = set()
    self.min_sdk_version = None
    self.proto = dict()
    self.output_extension: Optional[str] = None
    # The genrule_XXX below are properties that must to be propagated back
    # on the module(s) that depend on the genrule.
    self.genrule_headers = set()
    self.genrule_srcs = set()
    self.genrule_shared_libs = set()
    self.version_script = None
    self.test_suites = set()
    self.test_config = None
    self.stubs = {}

  def to_string(self, output):
    if self.comment:
      output.append('// %s' % self.comment)
    output.append('%s {' % self.type)
    self._output_field(output, 'name')
    self._output_field(output, 'srcs')
    self._output_field(output, 'shared_libs')
    self._output_field(output, 'static_libs')
    self._output_field(output, 'whole_static_libs')
    self._output_field(output, 'runtime_libs')
    self._output_field(output, 'tools')
    self._output_field(output, 'cmd', sort=False)
    if self.host_supported:
      self._output_field(output, 'host_supported')
    if self.vendor_available:
      self._output_field(output, 'vendor_available')
    if self.product_available:
      self._output_field(output, 'product_available')
    self._output_field(output, 'init_rc')
    self._output_field(output, 'out')
    self._output_field(output, 'export_include_dirs')
    self._output_field(output, 'generated_headers')
    self._output_field(output, 'export_generated_headers')
    self._output_field(output, 'defaults')
    self._output_field(output, 'cflags')
    self._output_field(output, 'include_dirs')
    self._output_field(output, 'header_libs')
    self._output_field(output, 'required')
    self._output_field(output, 'dist')
    self._output_field(output, 'strip')
    self._output_field(output, 'tool_files')
    self._output_field(output, 'data')
    self._output_field(output, 'stl')
    self._output_field(output, 'apex_available')
    self._output_field(output, 'min_sdk_version')
    self._output_field(output, 'version_script')
    self._output_field(output, 'test_suites')
    self._output_field(output, 'test_config')
    self._output_field(output, 'stubs')
    self._output_field(output, 'proto')
    self._output_field(output, 'main')
    self._output_field(output, 'output_extension')

    target_out = []
    self._output_field(target_out, 'android')
    self._output_field(target_out, 'host')
    self._output_field(target_out, 'musl')
    if target_out:
      output.append('    target: {')
      for line in target_out:
        output.append('    %s' % line)
      output.append('    },')

    if self.user_debug_flag:
      output.append('    product_variables: {')
      output.append('        debuggable: {')
      output.append(
          '            cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],')
      output.append('        },')
      output.append('    },')
    if self.lto is not None:
      output.append('    target: {')
      output.append('        android: {')
      output.append('            lto: {')
      output.append('                thin: %s,' %
                    'true' if self.lto else 'false')
      output.append('            },')
      output.append('        },')
      output.append('    },')
    output.append('}')
    output.append('')

  def add_android_static_lib(self, lib):
    if self.type == 'cc_binary_host':
      raise Exception('Adding Android static lib for host tool is unsupported')
    elif self.host_supported:
      self.android.static_libs.add(lib)
    else:
      self.static_libs.add(lib)

  def add_android_shared_lib(self, lib):
    if self.type == 'cc_binary_host':
      raise Exception('Adding Android shared lib for host tool is unsupported')
    elif self.host_supported:
      self.android.shared_libs.add(lib)
    else:
      self.shared_libs.add(lib)

  def _output_field(self, output, name, sort=True):
    value = getattr(self, name)
    return write_blueprint_key_value(output, name, value, sort)


class Blueprint(object):
  """In-memory representation of an Android.bp file."""

  def __init__(self):
    self.modules: Dict[str, Module] = {}
    self.gn_target_to_module: Dict[str, Module] = {}

  def add_module(self, module: Module):
    """Adds a new module to the blueprint, replacing any existing module
        with the same name.

        Args:
            module: Module instance.
        """
    self.modules[module.name] = module
    self.gn_target_to_module[module.gn_target] = module

  def to_string(self, output):
    for m in sorted(itervalues(self.modules), key=lambda m: m.name):
      m.to_string(output)


def label_to_module_name(label: str):
  """Turn a GN label (e.g., //:perfetto_tests) into a module name."""
  # If the label is explicibly listed in the default target list, don't prefix
  # its name and return just the target name. This is so tools like
  # "traceconv" stay as such in the Android tree.
  label_without_toolchain = gn_utils.label_without_toolchain(label)
  if label in default_targets or label_without_toolchain in default_targets:
    return label_without_toolchain.split(':')[-1]

  module = re.sub(r'^//:?', '', label_without_toolchain)
  module = re.sub(r'[^a-zA-Z0-9_]', '_', module)
  if not module.startswith(module_prefix):
    return module_prefix + module
  return module


def is_supported_source_file(name: str):
  """Returns True if |name| can appear in a 'srcs' list."""
  return os.path.splitext(name)[1] in ['.c', '.cc', '.proto']


def create_proto_modules(blueprint: Blueprint, gn: GnParser,
                         target: GnParser.Target):
  """Generate genrules for a proto GN target.

    GN actions are used to dynamically generate files during the build. The
    Soong equivalent is a genrule. This function turns a specific kind of
    genrule which turns .proto files into source and header files into a pair
    equivalent genrules.

    Args:
        blueprint: Blueprint instance which is being generated.
        target: gn_utils.Target object.

    Returns:
        The source_genrule module.
    """
  assert (target.type == 'proto_library')

  # We don't generate any targets for source_set proto modules because
  # they will be inlined into other modules if required.
  if target.proto_plugin == 'source_set':
    return None

  tools = {'aprotoc'}
  cpp_out_dir = '$(genDir)/%s/' % tree_path
  target_module_name = label_to_module_name(target.name)

  # In GN builds the proto path is always relative to the output directory
  # (out/tmp.xxx).
  cmd = ['mkdir -p %s &&' % cpp_out_dir, '$(location aprotoc)']
  cmd += ['--proto_path=%s' % tree_path]

  tool_files = set()
  if buildtools_protobuf_src in target.proto_paths:
    cmd += ['--proto_path=%s' % android_protobuf_src]
    # Add `google/protobuf/descriptor.proto` to implicit deps
    tool_files.add(':libprotobuf-internal-descriptor-proto')

  # Descriptor targets only generate a single target.
  if target.proto_plugin == 'descriptor':
    out = '{}.bin'.format(target_module_name)

    cmd += ['--descriptor_set_out=$(out)']
    cmd += ['$(in)']

    descriptor_module = Module('genrule', target_module_name, target.name)
    descriptor_module.cmd = ' '.join(cmd)
    descriptor_module.out.add(out)
    descriptor_module.tools = tools
    blueprint.add_module(descriptor_module)

    # Recursively extract the .proto files of all the dependencies and
    # add them to srcs.
    descriptor_module.srcs.update(
        gn_utils.label_to_path(src) for src in target.sources)
    # Add the tool_files to srcs so that they get copied if this action is
    # sandboxed in Soong.
    # Add to `srcs` instead of `tool_files` (the latter will require a
    # --proto_path that depends on Soong's sandbox implementation.)
    descriptor_module.srcs.update(
        src for src in tool_files)
    for dep in target.transitive_proto_deps():
      current_target = gn.get_target(dep.name)
      descriptor_module.srcs.update(
          gn_utils.label_to_path(src) for src in current_target.sources)

    return descriptor_module

  # We create two genrules for each proto target: one for the headers and
  # another for the sources. This is because the module that depends on the
  # generated files needs to declare two different types of dependencies --
  # source files in 'srcs' and headers in 'generated_headers' -- and it's not
  # valid to generate .h files from a source dependency and vice versa.
  #
  # We create an additional filegroup for .proto
  # The .proto filegroup will be added to `tool_files` of rdeps so that the
  # genrules can be sandboxed.

  for proto_dep in target.proto_deps().union(target.transitive_proto_deps()):
    tool_files.add(":" + label_to_module_name(proto_dep.name))

  filegroup_module = Module('filegroup', target_module_name, target.name)
  filegroup_module.srcs.update(
      gn_utils.label_to_path(src) for src in target.sources)
  blueprint.add_module(filegroup_module)

  source_module_name = target_module_name + '_gen'
  source_module = Module('genrule', source_module_name, target.name)
  # Add the "root" .proto filegroup to srcs
  source_module.srcs = set([':' + target_module_name])
  # Add the tool_files to srcs so that they get copied if this action is
  # sandboxed in Soong.
  # Add to `srcs` instead of `tool_files` (the latter will require a
  # --proto_path that depends on Soong's sandbox implementation.)
  source_module.srcs.update(
      src for src in tool_files)
  blueprint.add_module(source_module)

  header_module = Module('genrule', source_module_name + '_headers',
                         target.name)
  blueprint.add_module(header_module)
  header_module.srcs = set(source_module.srcs)

  # TODO(primiano): at some point we should remove this. This was introduced
  # by aosp/1108421 when adding "protos/" to .proto include paths, in order to
  # avoid doing multi-repo changes and allow old clients in the android tree
  # to still do the old #include "perfetto/..." rather than
  # #include "protos/perfetto/...".
  header_module.export_include_dirs = {'.', 'protos'}

  source_module.genrule_srcs.add(':' + source_module.name)
  source_module.genrule_headers.add(header_module.name)

  if target.proto_plugin == 'proto':
    suffixes = ['pb']
    source_module.genrule_shared_libs.add('libprotobuf-cpp-lite')
    cmd += ['--cpp_out=lite=true:' + cpp_out_dir]
  elif target.proto_plugin == 'protozero':
    suffixes = ['pbzero']
    plugin = create_modules_from_target(blueprint, gn, protozero_plugin)
    assert (plugin)
    tools.add(plugin.name)
    cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
    cmd += ['--plugin_out=wrapper_namespace=pbzero:' + cpp_out_dir]
  elif target.proto_plugin == 'cppgen':
    suffixes = ['gen']
    plugin = create_modules_from_target(blueprint, gn, cppgen_plugin)
    assert (plugin)
    tools.add(plugin.name)
    cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
    cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir]
  elif target.proto_plugin == 'ipc':
    suffixes = ['ipc']
    plugin = create_modules_from_target(blueprint, gn, ipc_plugin)
    assert (plugin)
    tools.add(plugin.name)
    cmd += ['--plugin=protoc-gen-plugin=$(location %s)' % plugin.name]
    cmd += ['--plugin_out=wrapper_namespace=gen:' + cpp_out_dir]
  else:
    raise Error('Unsupported proto plugin: %s' % target.proto_plugin)

  cmd += ['$(locations :%s)' % target_module_name]
  source_module.cmd = ' '.join(cmd)
  header_module.cmd = source_module.cmd
  source_module.tools = tools
  header_module.tools = tools


  for sfx in suffixes:
    source_module.out.update('%s/%s' %
                             (tree_path, src.replace('.proto', '.%s.cc' % sfx))
                             for src in filegroup_module.srcs)
    header_module.out.update('%s/%s' %
                             (tree_path, src.replace('.proto', '.%s.h' % sfx))
                             for src in filegroup_module.srcs)
  return source_module


def create_tp_tables_module(blueprint: Blueprint, gn: GnParser,
                            target: GnParser.Target):
  bp_module_name = label_to_module_name(target.name)
  bp_binary_module_name = f'{bp_module_name}_binary'
  srcs = [gn_utils.label_to_path(src) for src in target.sources]

  binary_module = Module('python_binary_host', bp_binary_module_name,
                         target.name)
  for dep in target.non_proto_or_source_set_deps():
    binary_module.srcs.update([
        gn_utils.label_to_path(src) for src in gn.get_target(dep.name).sources
    ])
  binary_module.srcs.update(srcs)
  binary_module.srcs.add('tools/gen_tp_table_headers.py')
  binary_module.main = 'tools/gen_tp_table_headers.py'
  blueprint.add_module(binary_module)

  module = Module('genrule', bp_module_name, target.name)
  module.tools = set([
      bp_binary_module_name,
  ])
  module.cmd = ' '.join([
      f'$(location {bp_binary_module_name})',
      '--gen-dir=$(genDir)',
      '--relative-input-dir=external/perfetto',
      '--inputs',
      '$(in)',
  ])
  module.out.update(target.outputs)
  module.genrule_headers.add(module.name)
  module.srcs.update(srcs)
  blueprint.add_module(module)

  return module


def create_amalgamated_sql_module(blueprint: Blueprint, gn: GnParser,
                                  target: GnParser.Target):
  bp_module_name = label_to_module_name(target.name)

  def find_arg(name):
    for i, arg in enumerate(target.args):
      if arg.startswith(f'--{name}'):
        return target.args[i + 1]

  namespace = find_arg('namespace')

  module = Module('genrule', bp_module_name, target.name)
  module.tool_files = [
      'tools/gen_amalgamated_sql.py',
  ]
  module.cmd = ' '.join([
      '$(location tools/gen_amalgamated_sql.py)',
      f'--namespace={namespace}',
      '--cpp-out=$(out)',
      '$(in)',
  ])
  module.genrule_headers.add(module.name)
  module.out.update(target.outputs)

  for dep in target.transitive_deps:
    # Use globs for the chrome stdlib so the file does not need to be
    # regenerated in autoroll CLs.
    if dep.name.startswith(
          '//src/trace_processor/perfetto_sql/stdlib/chrome:chrome_sql'):
      module.srcs.add('src/trace_processor/perfetto_sql/stdlib/chrome/**/*.sql')
      continue
    module.srcs.update(
        [gn_utils.label_to_path(src) for src in gn.get_target(dep.name).inputs])
  blueprint.add_module(module)
  return module


def create_cc_proto_descriptor_module(blueprint: Blueprint,
                                      target: GnParser.Target):
  bp_module_name = label_to_module_name(target.name)
  module = Module('genrule', bp_module_name, target.name)
  module.tool_files = [
      'tools/gen_cc_proto_descriptor.py',
  ]
  module.cmd = ' '.join([
      '$(location tools/gen_cc_proto_descriptor.py)', '--gen_dir=$(genDir)',
      '--cpp_out=$(out)', '$(in)'
  ])
  module.genrule_headers.add(module.name)
  module.srcs.update(
      ':' + label_to_module_name(dep.name) for dep in target.proto_deps())
  module.srcs.update(
      gn_utils.label_to_path(src)
      for src in target.inputs
      if "tmp.gn_utils" not in src)
  module.out.update(target.outputs)
  blueprint.add_module(module)
  return module


def create_gen_version_module(blueprint: Blueprint, target: GnParser.Target,
                              bp_module_name: str):
  module = Module('genrule', bp_module_name, gn_utils.GEN_VERSION_TARGET)
  script_path = gn_utils.label_to_path(target.script)
  module.genrule_headers.add(bp_module_name)
  module.tool_files = [script_path]
  module.out.update(target.outputs)
  module.srcs.update(gn_utils.label_to_path(src) for src in target.inputs)
  module.cmd = ' '.join([
      'python3 $(location %s)' % script_path, '--no_git',
      '--changelog=$(location CHANGELOG)', '--cpp_out=$(out)'
  ])
  blueprint.add_module(module)
  return module


def create_proto_group_modules(blueprint, gn: GnParser, module_name: str,
                               group):
  target_names = group['targets']
  module_types = group['types']
  module_sources = set()

  for name in target_names:
    target = gn.get_target(name)
    module_sources.update(gn_utils.label_to_path(src) for src in target.sources)
    for dep_label in target.transitive_proto_deps():
      dep = gn.get_target(dep_label.name)
      module_sources.update(gn_utils.label_to_path(src) for src in dep.sources)

  for type in module_types:
    if type == 'filegroup':
      name = label_to_module_name(module_name) + '_filegroup_proto'
      module = Module('filegroup', name, name)
      module.comment = f'''GN: [{', '.join(target_names)}]'''
      module.srcs = module_sources
      blueprint.add_module(module)
    elif type == 'lite':
      name = label_to_module_name(module_name) + '_java_protos'
      module = Module('java_library', name, name)
      module.comment = f'''GN: [{', '.join(target_names)}]'''
      module.proto = {'type': 'lite', 'canonical_path_from_root': False}
      module.srcs = module_sources
      blueprint.add_module(module)
    else:
      raise Error('Unhandled proto group type: {}'.format(group.type))


def _get_cflags(target: GnParser.Target):
  cflags = {flag for flag in target.cflags if re.match(cflag_allowlist, flag)}
  cflags |= set("-D%s" % define
                for define in target.defines
                if re.match(define_allowlist, define))
  return cflags


def create_modules_from_target(blueprint: Blueprint, gn: GnParser,
                               gn_target_name: str) -> Optional[Module]:
  """Generate module(s) for a given GN target.

    Given a GN target name, generate one or more corresponding modules into a
    blueprint. The only case when this generates >1 module is proto libraries.

    Args:
        blueprint: Blueprint instance which is being generated.
        gn: gn_utils.GnParser object.
        gn_target_name: GN target for module generation.
    """
  if gn_target_name in blueprint.gn_target_to_module:
    return blueprint.gn_target_to_module[gn_target_name]

  target = gn.get_target(gn_target_name)
  bp_module_name = label_to_module_name(gn_target_name)
  name_without_toolchain = gn_utils.label_without_toolchain(target.name)
  if target.type == 'executable':
    if target.toolchain == gn_utils.HOST_TOOLCHAIN:
      module_type = 'cc_binary_host'
    elif target.testonly:
      module_type = 'cc_test'
    else:
      module_type = 'cc_binary'
    module = Module(module_type, bp_module_name, gn_target_name)
  elif target.type == 'static_library':
    module = Module('cc_library_static', bp_module_name, gn_target_name)
  elif target.type == 'shared_library':
    module = Module('cc_library_shared', bp_module_name, gn_target_name)
  elif target.type == 'source_set':
    module = Module('filegroup', bp_module_name, gn_target_name)
  elif target.type == 'group':
    # "group" targets are resolved recursively by gn_utils.get_target().
    # There's nothing we need to do at this level for them.
    return None
  elif target.type == 'proto_library':
    module = create_proto_modules(blueprint, gn, target)
    if module is None:
      return None
  elif target.type == 'action':
    if target.custom_action_type == 'sql_amalgamation':
      return create_amalgamated_sql_module(blueprint, gn, target)
    if target.custom_action_type == 'tp_tables':
      return create_tp_tables_module(blueprint, gn, target)

    if target.custom_action_type == 'cc_proto_descriptor':
      module = create_cc_proto_descriptor_module(blueprint, target)
    elif name_without_toolchain == gn_utils.GEN_VERSION_TARGET:
      module = create_gen_version_module(blueprint, target, bp_module_name)
    else:
      raise Error('Unhandled action: {}'.format(target.name))
  else:
    raise Error('Unknown target %s (%s)' % (target.name, target.type))

  blueprint.add_module(module)
  module.host_supported = (name_without_toolchain in target_host_supported)
  module.vendor_available = (name_without_toolchain in target_vendor_available)
  module.product_available = (name_without_toolchain in target_product_available)
  module.init_rc.update(target_initrc.get(target.name, []))
  if target.type != 'proto_library':
    # proto_library embeds a "root" filegroup in its srcs.
    # Skip to prevent adding dups
    module.srcs.update(
      gn_utils.label_to_path(src)
      for src in target.sources
      if is_supported_source_file(src))

  if name_without_toolchain in needs_libfts:
    module.musl.static_libs.add('libfts')

  if target.type in gn_utils.LINKER_UNIT_TYPES:
    module.cflags.update(_get_cflags(target))

  module_is_compiled = module.type not in ('genrule', 'filegroup')
  if module_is_compiled:
    # Don't try to inject library/source dependencies into genrules or
    # filegroups because they are not compiled in the traditional sense.
    module.defaults.update([defaults_module])
    for lib in target.libs:
      # Generally library names should be mangled as 'libXXX', unless they
      # are HAL libraries (e.g., android.hardware.health@2.0) or AIDL c++ / NDK
      # libraries (e.g. "android.hardware.power.stats-V1-cpp")
      android_lib = lib if '@' in lib or "-cpp" in lib or "-ndk" in lib \
        else 'lib' + lib
      if lib in shared_library_allowlist:
        module.add_android_shared_lib(android_lib)
      if lib in static_library_allowlist:
        module.add_android_static_lib(android_lib)

  # If the module is a static library, export all the generated headers.
  if module.type == 'cc_library_static':
    module.export_generated_headers = module.generated_headers

  # Merge in additional hardcoded arguments.
  for key, add_val in additional_args.get(module.name, []):
    curr = getattr(module, key)
    if add_val and isinstance(add_val, set) and isinstance(curr, set):
      curr.update(add_val)
    elif isinstance(add_val, str) and (not curr or isinstance(curr, str)):
      setattr(module, key, add_val)
    elif isinstance(add_val, bool) and (not curr or isinstance(curr, bool)):
      setattr(module, key, add_val)
    elif isinstance(add_val, dict) and isinstance(curr, dict):
      curr.update(add_val)
    elif isinstance(add_val, dict) and isinstance(curr, Target):
      curr.__dict__.update(add_val)
    else:
      raise Error('Unimplemented type %r of additional_args: %r' %
                  (type(add_val), key))

  # dep_name is an unmangled GN target name (e.g. //foo:bar(toolchain)).
  all_deps = target.non_proto_or_source_set_deps()
  all_deps |= target.transitive_source_set_deps()
  all_deps |= target.transitive_proto_deps()
  for dep in all_deps:
    # If the dependency refers to a library which we can replace with an
    # Android equivalent, stop recursing and patch the dependency in.
    # Don't recurse into //buildtools, builtin_deps are intercepted at
    # the //gn:xxx level.
    dep_name = dep.name
    if dep_name.startswith('//buildtools'):
      continue

    # Ignore the dependency on the gen_buildflags genrule. That is run
    # separately in this generator and the generated file is copied over
    # into the repo (see usage of |buildflags_dir| in this script).
    if dep_name.startswith(gn_utils.BUILDFLAGS_TARGET):
      continue

    dep_module = create_modules_from_target(blueprint, gn, dep_name)

    # For filegroups and genrule, recurse but don't apply the deps.
    if not module_is_compiled:
      continue

    # |builtin_deps| override GN deps with Android-specific ones. See the
    # config in the top of this file.
    if gn_utils.label_without_toolchain(dep_name) in builtin_deps:
      builtin_deps[gn_utils.label_without_toolchain(dep_name)](module)
      continue

    # Don't recurse in any other //gn dep if not handled by builtin_deps.
    if dep_name.startswith('//gn:'):
      continue

    if dep_module is None:
      continue
    if dep_module.type == 'cc_library_shared':
      module.shared_libs.add(dep_module.name)
    elif dep_module.type == 'cc_library_static':
      module.static_libs.add(dep_module.name)
    elif dep_module.type == 'filegroup':
      module.srcs.add(':' + dep_module.name)
    elif dep_module.type == 'genrule':
      module.generated_headers.update(dep_module.genrule_headers)
      module.srcs.update(dep_module.genrule_srcs)
      module.shared_libs.update(dep_module.genrule_shared_libs)
    elif dep_module.type == 'cc_binary':
      continue  # Ignore executables deps (used by cmdline integration tests).
    else:
      raise Error('Unknown dep %s (%s) for target %s' %
                  (dep_module.name, dep_module.type, module.name))

  return module


def create_blueprint_for_targets(gn: GnParser, targets: List[str]):
  """Generate a blueprint for a list of GN targets."""
  blueprint = Blueprint()

  # Default settings used by all modules.
  defaults = Module('cc_defaults', defaults_module, '//gn:default_deps')

  # We have to use include_dirs passing the path relative to the android tree.
  # This is because: (i) perfetto_cc_defaults is used also by
  # test/**/Android.bp; (ii) if we use local_include_dirs instead, paths
  # become relative to the Android.bp that *uses* cc_defaults (not the one
  # that defines it).s
  defaults.include_dirs = {
      tree_path, tree_path + '/include', tree_path + '/' + buildflags_dir,
      tree_path + '/src/profiling/memory/include'
  }
  defaults.cflags.update([
      '-Wno-error=return-type',
      '-Wno-sign-compare',
      '-Wno-sign-promo',
      '-Wno-unused-parameter',
      '-fvisibility=hidden',
      '-O2',
  ])
  defaults.user_debug_flag = True
  defaults.lto = True

  blueprint.add_module(defaults)
  for target in targets:
    create_modules_from_target(blueprint, gn, target)
  return blueprint


def main():
  parser = argparse.ArgumentParser(
      description='Generate Android.bp from a GN description.')
  parser.add_argument(
      '--check-only',
      help='Don\'t keep the generated files',
      action='store_true')
  parser.add_argument(
      '--desc',
      help='GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
  )
  parser.add_argument(
      '--extras',
      help='Extra targets to include at the end of the Blueprint file',
      default=os.path.join(gn_utils.repo_root(), 'Android.bp.extras'),
  )
  parser.add_argument(
      '--output',
      help='Blueprint file to create',
      default=os.path.join(gn_utils.repo_root(), 'Android.bp'),
  )
  parser.add_argument(
      'targets',
      nargs=argparse.REMAINDER,
      help='Targets to include in the blueprint (e.g., "//:perfetto_tests")')
  args = parser.parse_args()

  if args.desc:
    with open(args.desc) as f:
      desc = json.load(f)
  else:
    desc = gn_utils.create_build_description(gn_args)

  gn = gn_utils.GnParser(desc)
  blueprint = create_blueprint_for_targets(gn, args.targets or default_targets)
  project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
  tool_name = os.path.relpath(os.path.abspath(__file__), project_root)

  # TODO(primiano): enable this on Android after the TODO in
  # perfetto_component.gni is fixed.
  # Check for ODR violations
  # for target_name in default_targets:
  # checker = gn_utils.ODRChecker(gn, target_name)

  # Add any proto groups to the blueprint.
  for name, group in proto_groups.items():
    create_proto_group_modules(blueprint, gn, name, group)

  output = [
      """// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file is automatically generated by %s. Do not edit.
""" % (tool_name)
  ]
  blueprint.to_string(output)
  with open(args.extras, 'r') as r:
    for line in r:
      output.append(line.rstrip("\n\r"))

  out_files = []

  # Generate the Android.bp file.
  out_files.append(args.output + '.swp')
  with open(out_files[-1], 'w') as f:
    f.write('\n'.join(output))
    # Text files should have a trailing EOL.
    f.write('\n')

  # Generate the perfetto_build_flags.h file.
  out_files.append(os.path.join(buildflags_dir, 'perfetto_build_flags.h.swp'))
  gn_utils.gen_buildflags(gn_args, out_files[-1])

  # Either check the contents or move the files to their final destination.
  return gn_utils.check_or_commit_generated_files(out_files, args.check_only)


if __name__ == '__main__':
  sys.exit(main())