Primiano Tucci | 34bc559 | 2021-02-19 17:53:36 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 2 | # Copyright (C) 2019 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | # This tool uses a collection of BUILD.gn files and build targets to generate |
| 17 | # an "amalgamated" C++ header and source file pair which compiles to an |
| 18 | # equivalent program. The tool also outputs the necessary compiler and linker |
| 19 | # flags needed to compile the resulting source code. |
| 20 | |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 21 | from __future__ import print_function |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 22 | import argparse |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 23 | import os |
| 24 | import re |
| 25 | import shutil |
| 26 | import subprocess |
| 27 | import sys |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 28 | import tempfile |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 29 | |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 30 | import gn_utils |
| 31 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 32 | # Default targets to include in the result. |
Primiano Tucci | 75ae50e | 2019-08-28 13:09:55 +0200 | [diff] [blame] | 33 | # TODO(primiano): change this script to recurse into target deps when generating |
| 34 | # headers, but only for proto targets. .pbzero.h files don't include each other |
| 35 | # and we need to list targets here individually, which is unmaintainable. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 36 | default_targets = [ |
Primiano Tucci | 658e2d6 | 2019-06-14 10:03:32 +0100 | [diff] [blame] | 37 | '//:libperfetto_client_experimental', |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 38 | '//include/perfetto/protozero:protozero', |
| 39 | '//protos/perfetto/config:zero', |
| 40 | '//protos/perfetto/trace:zero', |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 41 | ] |
| 42 | |
| 43 | # Arguments for the GN output directory (unless overridden from the command |
| 44 | # line). |
Primiano Tucci | 9c41165 | 2019-08-27 07:13:59 +0200 | [diff] [blame] | 45 | gn_args = ' '.join([ |
| 46 | 'is_debug=false', |
Primiano Tucci | 7e05fc1 | 2019-08-27 17:29:47 +0200 | [diff] [blame] | 47 | 'is_perfetto_build_generator=true', |
| 48 | 'is_perfetto_embedder=true', |
| 49 | 'use_custom_libcxx=false', |
| 50 | 'enable_perfetto_ipc=true', |
Primiano Tucci | 9c41165 | 2019-08-27 07:13:59 +0200 | [diff] [blame] | 51 | ]) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 52 | |
Primiano Tucci | cb05065 | 2019-08-29 01:10:34 +0200 | [diff] [blame] | 53 | # By default, the amalgamated .h only recurses in #includes but not in the |
| 54 | # target deps. In the case of protos we want to follow deps even in lieu of |
| 55 | # direct #includes. This is because, by design, protozero headers don't |
Sami Kyostila | 02fccc1 | 2020-07-14 19:33:15 +0100 | [diff] [blame] | 56 | # include each other but rely on forward declarations. The alternative would |
Primiano Tucci | cb05065 | 2019-08-29 01:10:34 +0200 | [diff] [blame] | 57 | # be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero), |
Sami Kyostila | 02fccc1 | 2020-07-14 19:33:15 +0100 | [diff] [blame] | 58 | # but doing that is unmaintainable. We also do this for cpp bindings since some |
| 59 | # tracing SDK functions depend on them (and the system tracing IPC mechanism |
| 60 | # does so too). |
| 61 | recurse_in_header_deps = '^//protos/.*(cpp|zero)$' |
Primiano Tucci | cb05065 | 2019-08-29 01:10:34 +0200 | [diff] [blame] | 62 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 63 | # Compiler flags which aren't filtered out. |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 64 | cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 65 | |
| 66 | # Linker flags which aren't filtered out. |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 67 | ldflag_allowlist = r'^-()$' |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 68 | |
| 69 | # Libraries which are filtered out. |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 70 | lib_denylist = r'^(c|gcc_eh)$' |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 71 | |
| 72 | # Macros which aren't filtered out. |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 73 | define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 74 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 75 | # Includes which will be removed from the generated source. |
| 76 | includes_to_remove = r'^(gtest).*$' |
| 77 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 78 | default_cflags = [ |
| 79 | # Since we're expanding header files into the generated source file, some |
| 80 | # constant may remain unused. |
| 81 | '-Wno-unused-const-variable' |
| 82 | ] |
| 83 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 84 | # Build flags to satisfy a protobuf (lite or full) dependency. |
| 85 | protobuf_cflags = [ |
| 86 | # Note that these point to the local copy of protobuf in buildtools. In |
| 87 | # reality the user of the amalgamated result will have to provide a path to |
| 88 | # an installed copy of the exact same version of protobuf which was used to |
| 89 | # generate the amalgamated build. |
| 90 | '-isystembuildtools/protobuf/src', |
| 91 | '-Lbuildtools/protobuf/src/.libs', |
| 92 | # We also need to disable some warnings for protobuf. |
| 93 | '-Wno-missing-prototypes', |
| 94 | '-Wno-missing-variable-declarations', |
| 95 | '-Wno-sign-conversion', |
| 96 | '-Wno-unknown-pragmas', |
| 97 | '-Wno-unused-macros', |
| 98 | ] |
| 99 | |
| 100 | # A mapping of dependencies to system libraries. Libraries in this map will not |
| 101 | # be built statically but instead added as dependencies of the amalgamated |
| 102 | # project. |
| 103 | system_library_map = { |
| 104 | '//buildtools:protobuf_full': { |
| 105 | 'libs': ['protobuf'], |
| 106 | 'cflags': protobuf_cflags, |
| 107 | }, |
| 108 | '//buildtools:protobuf_lite': { |
| 109 | 'libs': ['protobuf-lite'], |
| 110 | 'cflags': protobuf_cflags, |
| 111 | }, |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 112 | '//buildtools:protoc_lib': { |
| 113 | 'libs': ['protoc'] |
| 114 | }, |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 115 | } |
| 116 | |
| 117 | # ---------------------------------------------------------------------------- |
| 118 | # End of configuration. |
| 119 | # ---------------------------------------------------------------------------- |
| 120 | |
| 121 | tool_name = os.path.basename(__file__) |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 122 | project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 123 | preamble = """// Copyright (C) 2019 The Android Open Source Project |
| 124 | // |
| 125 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 126 | // you may not use this file except in compliance with the License. |
| 127 | // You may obtain a copy of the License at |
| 128 | // |
| 129 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 130 | // |
| 131 | // Unless required by applicable law or agreed to in writing, software |
| 132 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 133 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 134 | // See the License for the specific language governing permissions and |
| 135 | // limitations under the License. |
| 136 | // |
| 137 | // This file is automatically generated by %s. Do not edit. |
| 138 | """ % tool_name |
| 139 | |
| 140 | |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 141 | def apply_denylist(denylist, items): |
| 142 | return [item for item in items if not re.match(denylist, item)] |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 143 | |
| 144 | |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 145 | def apply_allowlist(allowlist, items): |
| 146 | return [item for item in items if re.match(allowlist, item)] |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 147 | |
| 148 | |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 149 | def normalize_path(path): |
| 150 | path = os.path.relpath(path, project_root) |
| 151 | path = re.sub(r'^out/[^/]+/', '', path) |
| 152 | return path |
| 153 | |
| 154 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 155 | class Error(Exception): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 156 | pass |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 157 | |
| 158 | |
| 159 | class DependencyNode(object): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 160 | """A target in a GN build description along with its dependencies.""" |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 161 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 162 | def __init__(self, target_name): |
| 163 | self.target_name = target_name |
| 164 | self.dependencies = set() |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 165 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 166 | def add_dependency(self, target_node): |
| 167 | if target_node in self.dependencies: |
| 168 | return |
| 169 | self.dependencies.add(target_node) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 170 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 171 | def iterate_depth_first(self): |
| 172 | for node in sorted(self.dependencies, key=lambda n: n.target_name): |
| 173 | for node in node.iterate_depth_first(): |
| 174 | yield node |
| 175 | if self.target_name: |
| 176 | yield self |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 177 | |
| 178 | |
| 179 | class DependencyTree(object): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 180 | """A tree of GN build target dependencies.""" |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 181 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 182 | def __init__(self): |
| 183 | self.target_to_node_map = {} |
| 184 | self.root = self._get_or_create_node(None) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 185 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 186 | def _get_or_create_node(self, target_name): |
| 187 | if target_name in self.target_to_node_map: |
| 188 | return self.target_to_node_map[target_name] |
| 189 | node = DependencyNode(target_name) |
| 190 | self.target_to_node_map[target_name] = node |
| 191 | return node |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 192 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 193 | def add_dependency(self, from_target, to_target): |
| 194 | from_node = self._get_or_create_node(from_target) |
| 195 | to_node = self._get_or_create_node(to_target) |
| 196 | assert from_node is not to_node |
| 197 | from_node.add_dependency(to_node) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 198 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 199 | def iterate_depth_first(self): |
| 200 | for node in self.root.iterate_depth_first(): |
| 201 | yield node |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 202 | |
| 203 | |
| 204 | class AmalgamatedProject(object): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 205 | """In-memory representation of an amalgamated source/header pair.""" |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 206 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 207 | def __init__(self, desc, source_deps, compute_deps_only=False): |
| 208 | """Constructor. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 209 | |
| 210 | Args: |
| 211 | desc: JSON build description. |
| 212 | source_deps: A map of (source file, [dependency header]) which is |
| 213 | to detect which header files are included by each source file. |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 214 | compute_deps_only: If True, the project will only be used to compute |
| 215 | dependency information. Use |get_source_files()| to retrieve |
| 216 | the result. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 217 | """ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 218 | self.desc = desc |
| 219 | self.source_deps = source_deps |
| 220 | self.header = [] |
| 221 | self.source = [] |
| 222 | self.source_defines = [] |
| 223 | # Note that we don't support multi-arg flags. |
| 224 | self.cflags = set(default_cflags) |
| 225 | self.ldflags = set() |
| 226 | self.defines = set() |
| 227 | self.libs = set() |
| 228 | self._dependency_tree = DependencyTree() |
| 229 | self._processed_sources = set() |
| 230 | self._processed_headers = set() |
| 231 | self._processed_header_deps = set() |
| 232 | self._processed_source_headers = set() # Header files included from .cc |
| 233 | self._include_re = re.compile(r'#include "(.*)"') |
| 234 | self._compute_deps_only = compute_deps_only |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 235 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 236 | def add_target(self, target_name): |
| 237 | """Include |target_name| in the amalgamated result.""" |
| 238 | self._dependency_tree.add_dependency(None, target_name) |
| 239 | self._add_target_dependencies(target_name) |
| 240 | self._add_target_flags(target_name) |
| 241 | self._add_target_headers(target_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 242 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 243 | # Recurse into target deps, but only for protos. This generates headers |
Sami Kyostila | 02fccc1 | 2020-07-14 19:33:15 +0100 | [diff] [blame] | 244 | # for all the .{pbzero,gen}.h files, even if they don't #include each other. |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 245 | for _, dep in self._iterate_dep_edges(target_name): |
| 246 | if (dep not in self._processed_header_deps and |
| 247 | re.match(recurse_in_header_deps, dep)): |
| 248 | self._processed_header_deps.add(dep) |
| 249 | self.add_target(dep) |
Primiano Tucci | cb05065 | 2019-08-29 01:10:34 +0200 | [diff] [blame] | 250 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 251 | def _iterate_dep_edges(self, target_name): |
| 252 | target = self.desc[target_name] |
| 253 | for dep in target.get('deps', []): |
| 254 | # Ignore system libraries since they will be added as build-time |
| 255 | # dependencies. |
| 256 | if dep in system_library_map: |
| 257 | continue |
| 258 | # Don't descend into build action dependencies. |
| 259 | if self.desc[dep]['type'] == 'action': |
| 260 | continue |
| 261 | for sub_target, sub_dep in self._iterate_dep_edges(dep): |
| 262 | yield sub_target, sub_dep |
| 263 | yield target_name, dep |
Primiano Tucci | cb05065 | 2019-08-29 01:10:34 +0200 | [diff] [blame] | 264 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 265 | def _iterate_target_and_deps(self, target_name): |
| 266 | yield target_name |
| 267 | for _, dep in self._iterate_dep_edges(target_name): |
| 268 | yield dep |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 269 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 270 | def _add_target_dependencies(self, target_name): |
| 271 | for target, dep in self._iterate_dep_edges(target_name): |
| 272 | self._dependency_tree.add_dependency(target, dep) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 273 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 274 | def process_dep(dep): |
| 275 | if dep in system_library_map: |
| 276 | self.libs.update(system_library_map[dep].get('libs', [])) |
| 277 | self.cflags.update(system_library_map[dep].get('cflags', [])) |
| 278 | self.defines.update(system_library_map[dep].get('defines', [])) |
| 279 | return True |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 280 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 281 | def walk_all_deps(target_name): |
| 282 | target = self.desc[target_name] |
| 283 | for dep in target.get('deps', []): |
| 284 | if process_dep(dep): |
| 285 | return |
| 286 | walk_all_deps(dep) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 287 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 288 | walk_all_deps(target_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 289 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 290 | def _filter_cflags(self, cflags): |
| 291 | # Since we want to deduplicate flags, combine two-part switches (e.g., |
| 292 | # "-foo bar") into one value ("-foobar") so we can store the result as |
| 293 | # a set. |
| 294 | result = [] |
| 295 | for flag in cflags: |
| 296 | if flag.startswith('-'): |
| 297 | result.append(flag) |
| 298 | else: |
| 299 | result[-1] += flag |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 300 | return apply_allowlist(cflag_allowlist, result) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 301 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 302 | def _add_target_flags(self, target_name): |
| 303 | for target_name in self._iterate_target_and_deps(target_name): |
| 304 | target = self.desc[target_name] |
| 305 | self.cflags.update(self._filter_cflags(target.get('cflags', []))) |
| 306 | self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) |
| 307 | self.ldflags.update( |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 308 | apply_allowlist(ldflag_allowlist, target.get('ldflags', []))) |
| 309 | self.libs.update(apply_denylist(lib_denylist, target.get('libs', []))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 310 | self.defines.update( |
Primiano Tucci | a364520 | 2020-08-03 16:28:18 +0200 | [diff] [blame] | 311 | apply_allowlist(define_allowlist, target.get('defines', []))) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 312 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 313 | def _add_target_headers(self, target_name): |
| 314 | target = self.desc[target_name] |
| 315 | if not 'sources' in target: |
| 316 | return |
| 317 | headers = [ |
| 318 | gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h') |
| 319 | ] |
| 320 | for header in headers: |
| 321 | self._add_header(target_name, header) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 322 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 323 | def _get_include_dirs(self, target_name): |
| 324 | include_dirs = set() |
| 325 | for target_name in self._iterate_target_and_deps(target_name): |
| 326 | target = self.desc[target_name] |
| 327 | if 'include_dirs' in target: |
| 328 | include_dirs.update( |
| 329 | [gn_utils.label_to_path(d) for d in target['include_dirs']]) |
| 330 | return include_dirs |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 331 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 332 | def _add_source_included_header(self, include_dirs, allowed_files, |
| 333 | header_name): |
Sami Kyostila | 02fccc1 | 2020-07-14 19:33:15 +0100 | [diff] [blame] | 334 | if header_name in self._processed_headers: |
| 335 | return |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 336 | if header_name in self._processed_source_headers: |
| 337 | return |
| 338 | self._processed_source_headers.add(header_name) |
| 339 | for include_dir in include_dirs: |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 340 | rel_path = os.path.join(include_dir, header_name) |
| 341 | full_path = os.path.join(gn_utils.repo_root(), rel_path) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 342 | if os.path.exists(full_path): |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 343 | if not rel_path in allowed_files: |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 344 | return |
| 345 | with open(full_path) as f: |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 346 | self.source.append( |
| 347 | '// %s begin header: %s' % (tool_name, normalize_path(full_path))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 348 | self.source.extend( |
| 349 | self._process_source_includes(include_dirs, allowed_files, f)) |
| 350 | return |
| 351 | if self._compute_deps_only: |
| 352 | return |
| 353 | msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) |
| 354 | raise Error('Header file %s not found. %s' % (header_name, msg)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 355 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 356 | def _add_source(self, target_name, source_name): |
| 357 | if source_name in self._processed_sources: |
| 358 | return |
| 359 | self._processed_sources.add(source_name) |
| 360 | include_dirs = self._get_include_dirs(target_name) |
| 361 | deps = self.source_deps[source_name] |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 362 | full_path = os.path.join(gn_utils.repo_root(), source_name) |
| 363 | if not os.path.exists(full_path): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 364 | raise Error('Source file %s not found' % source_name) |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 365 | with open(full_path) as f: |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 366 | self.source.append( |
| 367 | '// %s begin source: %s' % (tool_name, normalize_path(full_path))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 368 | try: |
| 369 | self.source.extend( |
| 370 | self._patch_source( |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 371 | source_name, self._process_source_includes( |
| 372 | include_dirs, deps, f))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 373 | except Error as e: |
| 374 | raise Error('Failed adding source %s: %s' % (source_name, e.message)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 375 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 376 | def _add_header_included_header(self, include_dirs, header_name): |
| 377 | if header_name in self._processed_headers: |
| 378 | return |
| 379 | self._processed_headers.add(header_name) |
| 380 | for include_dir in include_dirs: |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 381 | full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 382 | if os.path.exists(full_path): |
| 383 | with open(full_path) as f: |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 384 | self.header.append( |
| 385 | '// %s begin header: %s' % (tool_name, normalize_path(full_path))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 386 | self.header.extend(self._process_header_includes(include_dirs, f)) |
| 387 | return |
| 388 | if self._compute_deps_only: |
| 389 | return |
| 390 | msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) |
| 391 | raise Error('Header file %s not found. %s' % (header_name, msg)) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 392 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 393 | def _add_header(self, target_name, header_name): |
| 394 | if header_name in self._processed_headers: |
| 395 | return |
| 396 | self._processed_headers.add(header_name) |
| 397 | include_dirs = self._get_include_dirs(target_name) |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 398 | full_path = os.path.join(gn_utils.repo_root(), header_name) |
| 399 | if not os.path.exists(full_path): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 400 | if self._compute_deps_only: |
| 401 | return |
| 402 | raise Error('Header file %s not found' % header_name) |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 403 | with open(full_path) as f: |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 404 | self.header.append( |
| 405 | '// %s begin header: %s' % (tool_name, normalize_path(full_path))) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 406 | try: |
| 407 | self.header.extend(self._process_header_includes(include_dirs, f)) |
| 408 | except Error as e: |
| 409 | raise Error('Failed adding header %s: %s' % (header_name, e.message)) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 410 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 411 | def _patch_source(self, source_name, lines): |
| 412 | result = [] |
| 413 | namespace = re.sub(r'[^a-z]', '_', |
| 414 | os.path.splitext(os.path.basename(source_name))[0]) |
| 415 | for line in lines: |
| 416 | # Protobuf generates an identical anonymous function into each |
| 417 | # message description. Rename all but the first occurrence to avoid |
| 418 | # duplicate symbol definitions. |
| 419 | line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) |
| 420 | result.append(line) |
| 421 | return result |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 422 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 423 | def _process_source_includes(self, include_dirs, allowed_files, file): |
| 424 | result = [] |
| 425 | for line in file: |
| 426 | line = line.rstrip('\n') |
| 427 | m = self._include_re.match(line) |
| 428 | if not m: |
| 429 | result.append(line) |
| 430 | continue |
| 431 | elif re.match(includes_to_remove, m.group(1)): |
| 432 | result.append('// %s removed: %s' % (tool_name, line)) |
| 433 | else: |
| 434 | result.append('// %s expanded: %s' % (tool_name, line)) |
| 435 | self._add_source_included_header(include_dirs, allowed_files, |
| 436 | m.group(1)) |
| 437 | return result |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 438 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 439 | def _process_header_includes(self, include_dirs, file): |
| 440 | result = [] |
| 441 | for line in file: |
| 442 | line = line.rstrip('\n') |
| 443 | m = self._include_re.match(line) |
| 444 | if not m: |
| 445 | result.append(line) |
| 446 | continue |
| 447 | elif re.match(includes_to_remove, m.group(1)): |
| 448 | result.append('// %s removed: %s' % (tool_name, line)) |
| 449 | else: |
| 450 | result.append('// %s expanded: %s' % (tool_name, line)) |
| 451 | self._add_header_included_header(include_dirs, m.group(1)) |
| 452 | return result |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 453 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 454 | def generate(self): |
| 455 | """Prepares the output for this amalgamated project. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 456 | |
| 457 | Call save() to persist the result. |
| 458 | """ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 459 | assert not self._compute_deps_only |
| 460 | self.source_defines.append('// %s: predefined macros' % tool_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 461 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 462 | def add_define(name): |
| 463 | # Valued macros aren't supported for now. |
| 464 | assert '=' not in name |
| 465 | self.source_defines.append('#if !defined(%s)' % name) |
| 466 | self.source_defines.append('#define %s' % name) |
| 467 | self.source_defines.append('#endif') |
| 468 | |
| 469 | for name in self.defines: |
| 470 | add_define(name) |
| 471 | for target_name, source_name in self.get_source_files(): |
| 472 | self._add_source(target_name, source_name) |
| 473 | |
| 474 | def get_source_files(self): |
| 475 | """Return a list of (target, [source file]) that describes the source |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 476 | files pulled in by each target which is a dependency of this project. |
| 477 | """ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 478 | source_files = [] |
| 479 | for node in self._dependency_tree.iterate_depth_first(): |
| 480 | target = self.desc[node.target_name] |
| 481 | if not 'sources' in target: |
| 482 | continue |
| 483 | sources = [(node.target_name, gn_utils.label_to_path(s)) |
| 484 | for s in target['sources'] |
| 485 | if s.endswith('.cc')] |
| 486 | source_files.extend(sources) |
| 487 | return source_files |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 488 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 489 | def _get_nice_path(self, prefix, format): |
| 490 | basename = os.path.basename(prefix) |
| 491 | return os.path.join( |
| 492 | os.path.relpath(os.path.dirname(prefix)), format % basename) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 493 | |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 494 | def _make_directories(self, directory): |
| 495 | if not os.path.isdir(directory): |
| 496 | os.makedirs(directory) |
| 497 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 498 | def save(self, output_prefix): |
| 499 | """Save the generated header and source file pair. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 500 | |
| 501 | Returns a message describing the output with build instructions. |
| 502 | """ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 503 | header_file = self._get_nice_path(output_prefix, '%s.h') |
| 504 | source_file = self._get_nice_path(output_prefix, '%s.cc') |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 505 | self._make_directories(os.path.dirname(header_file)) |
| 506 | self._make_directories(os.path.dirname(source_file)) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 507 | with open(header_file, 'w') as f: |
| 508 | f.write('\n'.join([preamble] + self.header + ['\n'])) |
| 509 | with open(source_file, 'w') as f: |
| 510 | include_stmt = '#include "%s"' % os.path.basename(header_file) |
| 511 | f.write('\n'.join([preamble] + self.source_defines + [include_stmt] + |
| 512 | self.source + ['\n'])) |
| 513 | build_cmd = self.get_build_command(output_prefix) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 514 | return """Amalgamated project written to %s and %s. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 515 | |
| 516 | Build settings: |
| 517 | - cflags: %s |
| 518 | - ldflags: %s |
| 519 | - libs: %s |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 520 | |
| 521 | Example build command: |
| 522 | |
| 523 | %s |
Primiano Tucci | e22ffbd | 2020-01-17 01:11:31 +0000 | [diff] [blame] | 524 | """ % (header_file, source_file, ' '.join(self.cflags), ' '.join(self.ldflags), |
| 525 | ' '.join(self.libs), ' '.join(build_cmd)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 526 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 527 | def get_build_command(self, output_prefix): |
| 528 | """Returns an example command line for building the output source.""" |
| 529 | source = self._get_nice_path(output_prefix, '%s.cc') |
| 530 | library = self._get_nice_path(output_prefix, 'lib%s.so') |
Lalit Maganti | 54afee6 | 2020-07-16 19:28:04 +0100 | [diff] [blame] | 531 | |
| 532 | if sys.platform.startswith('linux'): |
| 533 | llvm_script = os.path.join(gn_utils.repo_root(), 'gn', |
| 534 | 'standalone', 'toolchain', |
| 535 | 'linux_find_llvm.py') |
Primiano Tucci | 34bc559 | 2021-02-19 17:53:36 +0100 | [diff] [blame] | 536 | cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode() |
Lalit Maganti | 54afee6 | 2020-07-16 19:28:04 +0100 | [diff] [blame] | 537 | else: |
| 538 | cxx = 'clang++' |
| 539 | |
| 540 | build_cmd = [cxx, source, '-o', library, '-shared'] + \ |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 541 | sorted(self.cflags) + sorted(self.ldflags) |
| 542 | for lib in sorted(self.libs): |
| 543 | build_cmd.append('-l%s' % lib) |
| 544 | return build_cmd |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 545 | |
| 546 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 547 | def main(): |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 548 | parser = argparse.ArgumentParser( |
| 549 | description='Generate an amalgamated header/source pair from a GN ' |
| 550 | 'build description.') |
| 551 | parser.add_argument( |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 552 | '--out', |
| 553 | help='The name of the temporary build folder in \'out\'', |
| 554 | default='tmp.gen_amalgamated.%u' % os.getpid()) |
| 555 | parser.add_argument( |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 556 | '--output', |
| 557 | help='Base name of files to create. A .cc/.h extension will be added', |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 558 | default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto')) |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 559 | parser.add_argument( |
| 560 | '--gn_args', |
| 561 | help='GN arguments used to prepare the output directory', |
| 562 | default=gn_args) |
| 563 | parser.add_argument( |
| 564 | '--keep', |
| 565 | help='Don\'t delete the GN output directory at exit', |
| 566 | action='store_true') |
| 567 | parser.add_argument( |
| 568 | '--build', help='Also compile the generated files', action='store_true') |
| 569 | parser.add_argument( |
| 570 | '--check', help='Don\'t keep the generated files', action='store_true') |
| 571 | parser.add_argument('--quiet', help='Only report errors', action='store_true') |
| 572 | parser.add_argument( |
| 573 | '--dump-deps', |
| 574 | help='List all source files that the amalgamated output depends on', |
| 575 | action='store_true') |
| 576 | parser.add_argument( |
| 577 | 'targets', |
| 578 | nargs=argparse.REMAINDER, |
| 579 | help='Targets to include in the output (e.g., "//:libperfetto")') |
| 580 | args = parser.parse_args() |
| 581 | targets = args.targets or default_targets |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 582 | |
Hector Dearman | 762c18f | 2021-05-24 11:10:52 +0100 | [diff] [blame] | 583 | # The CHANGELOG mtime triggers the perfetto_version.gen.h genrule. This is |
Primiano Tucci | ec59013 | 2020-11-16 14:16:44 +0100 | [diff] [blame] | 584 | # to avoid emitting a stale version information in the remote case of somebody |
| 585 | # running gen_amalgamated incrementally after having moved to another commit. |
| 586 | changelog_path = os.path.join(project_root, 'CHANGELOG') |
| 587 | assert(os.path.exists(changelog_path)) |
| 588 | subprocess.check_call(['touch', '-c', changelog_path]) |
| 589 | |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 590 | output = args.output |
| 591 | if args.check: |
| 592 | output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') |
| 593 | |
Matthew Clarkson | 58a3ebb | 2019-10-11 11:29:18 +0100 | [diff] [blame] | 594 | out = gn_utils.prepare_out_directory(args.gn_args, args.out) |
Matthew Clarkson | a990d95 | 2019-10-08 14:52:12 +0100 | [diff] [blame] | 595 | if not args.quiet: |
| 596 | print('Building project...') |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 597 | try: |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 598 | desc = gn_utils.load_build_description(out) |
| 599 | |
| 600 | # We need to build everything first so that the necessary header |
| 601 | # dependencies get generated. However if we are just dumping dependency |
| 602 | # information this can be skipped, allowing cross-platform operation. |
| 603 | if not args.dump_deps: |
| 604 | gn_utils.build_targets(out, targets) |
| 605 | source_deps = gn_utils.compute_source_dependencies(out) |
| 606 | project = AmalgamatedProject( |
| 607 | desc, source_deps, compute_deps_only=args.dump_deps) |
| 608 | |
| 609 | for target in targets: |
| 610 | project.add_target(target) |
| 611 | |
| 612 | if args.dump_deps: |
| 613 | source_files = [ |
| 614 | source_file for _, source_file in project.get_source_files() |
| 615 | ] |
| 616 | print('\n'.join(sorted(set(source_files)))) |
| 617 | return |
| 618 | |
| 619 | project.generate() |
| 620 | result = project.save(output) |
| 621 | if not args.quiet: |
| 622 | print(result) |
| 623 | if args.build: |
| 624 | if not args.quiet: |
| 625 | sys.stdout.write('Building amalgamated project...') |
| 626 | sys.stdout.flush() |
| 627 | subprocess.check_call(project.get_build_command(output)) |
| 628 | if not args.quiet: |
| 629 | print('done') |
| 630 | finally: |
| 631 | if not args.keep: |
| 632 | shutil.rmtree(out) |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 633 | if args.check: |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 634 | shutil.rmtree(os.path.dirname(output)) |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 635 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 636 | |
| 637 | if __name__ == '__main__': |
Primiano Tucci | 834fdc7 | 2019-10-04 11:33:44 +0100 | [diff] [blame] | 638 | sys.exit(main()) |