blob: 2fe68a6558ef771b1743c2d8e2fdc0d0e259d003 [file] [log] [blame]
Primiano Tucci34bc5592021-02-19 17:53:36 +01001#!/usr/bin/env python3
Sami Kyostila0a34b032019-05-16 18:28:48 +01002# Copyright (C) 2019 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# This tool uses a collection of BUILD.gn files and build targets to generate
17# an "amalgamated" C++ header and source file pair which compiles to an
18# equivalent program. The tool also outputs the necessary compiler and linker
19# flags needed to compile the resulting source code.
20
Sami Kyostila3c88a1d2019-05-22 18:29:42 +010021from __future__ import print_function
Sami Kyostila0a34b032019-05-16 18:28:48 +010022import argparse
Sami Kyostila0a34b032019-05-16 18:28:48 +010023import os
24import re
25import shutil
26import subprocess
27import sys
Sami Kyostila468e61d2019-05-23 15:54:01 +010028import tempfile
Sami Kyostila0a34b032019-05-16 18:28:48 +010029
Sami Kyostila3c88a1d2019-05-22 18:29:42 +010030import gn_utils
31
Sami Kyostila0a34b032019-05-16 18:28:48 +010032# Default targets to include in the result.
Primiano Tucci75ae50e2019-08-28 13:09:55 +020033# TODO(primiano): change this script to recurse into target deps when generating
34# headers, but only for proto targets. .pbzero.h files don't include each other
35# and we need to list targets here individually, which is unmaintainable.
Sami Kyostila0a34b032019-05-16 18:28:48 +010036default_targets = [
Primiano Tucci658e2d62019-06-14 10:03:32 +010037 '//:libperfetto_client_experimental',
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +010038 '//include/perfetto/protozero:protozero',
39 '//protos/perfetto/config:zero',
40 '//protos/perfetto/trace:zero',
Sami Kyostila0a34b032019-05-16 18:28:48 +010041]
42
43# Arguments for the GN output directory (unless overridden from the command
44# line).
Ching-lin Yuae24f9b2022-10-11 14:05:31 +000045gn_args = ' '.join([
Primiano Tucciadb22cf2021-11-02 22:14:39 +000046 'enable_perfetto_ipc=true',
Daniele Di Proietto2789b5b2023-04-14 09:53:03 +000047 'enable_perfetto_zlib=false',
Primiano Tucci9c411652019-08-27 07:13:59 +020048 'is_debug=false',
Primiano Tucci7e05fc12019-08-27 17:29:47 +020049 'is_perfetto_build_generator=true',
50 'is_perfetto_embedder=true',
Primiano Tucciadb22cf2021-11-02 22:14:39 +000051 'perfetto_enable_git_rev_version_header=true',
Primiano Tucci7e05fc12019-08-27 17:29:47 +020052 'use_custom_libcxx=false',
Primiano Tucci9c411652019-08-27 07:13:59 +020053])
Sami Kyostila0a34b032019-05-16 18:28:48 +010054
Primiano Tuccicb050652019-08-29 01:10:34 +020055# By default, the amalgamated .h only recurses in #includes but not in the
56# target deps. In the case of protos we want to follow deps even in lieu of
57# direct #includes. This is because, by design, protozero headers don't
Sami Kyostila02fccc12020-07-14 19:33:15 +010058# include each other but rely on forward declarations. The alternative would
Primiano Tuccicb050652019-08-29 01:10:34 +020059# be adding each proto sub-target individually (e.g. //proto/trace/gpu:zero),
Sami Kyostila02fccc12020-07-14 19:33:15 +010060# but doing that is unmaintainable. We also do this for cpp bindings since some
61# tracing SDK functions depend on them (and the system tracing IPC mechanism
62# does so too).
63recurse_in_header_deps = '^//protos/.*(cpp|zero)$'
Primiano Tuccicb050652019-08-29 01:10:34 +020064
Sami Kyostila0a34b032019-05-16 18:28:48 +010065# Compiler flags which aren't filtered out.
Primiano Tuccia3645202020-08-03 16:28:18 +020066cflag_allowlist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$'
Sami Kyostila0a34b032019-05-16 18:28:48 +010067
68# Linker flags which aren't filtered out.
Primiano Tuccia3645202020-08-03 16:28:18 +020069ldflag_allowlist = r'^-()$'
Sami Kyostila0a34b032019-05-16 18:28:48 +010070
71# Libraries which are filtered out.
Primiano Tuccia3645202020-08-03 16:28:18 +020072lib_denylist = r'^(c|gcc_eh)$'
Sami Kyostila0a34b032019-05-16 18:28:48 +010073
74# Macros which aren't filtered out.
Primiano Tuccia3645202020-08-03 16:28:18 +020075define_allowlist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$'
Sami Kyostila0a34b032019-05-16 18:28:48 +010076
Sami Kyostila0a34b032019-05-16 18:28:48 +010077# Includes which will be removed from the generated source.
78includes_to_remove = r'^(gtest).*$'
79
Sami Kyostilabc0cd792023-07-07 08:42:10 +000080# From //gn:default_config (since "gn desc" doesn't describe configs).
81default_includes = [
82 'include',
83]
84
Sami Kyostila7e8509f2019-05-29 12:36:24 +010085default_cflags = [
86 # Since we're expanding header files into the generated source file, some
87 # constant may remain unused.
88 '-Wno-unused-const-variable'
89]
90
Sami Kyostila0a34b032019-05-16 18:28:48 +010091# Build flags to satisfy a protobuf (lite or full) dependency.
92protobuf_cflags = [
93 # Note that these point to the local copy of protobuf in buildtools. In
94 # reality the user of the amalgamated result will have to provide a path to
95 # an installed copy of the exact same version of protobuf which was used to
96 # generate the amalgamated build.
97 '-isystembuildtools/protobuf/src',
98 '-Lbuildtools/protobuf/src/.libs',
99 # We also need to disable some warnings for protobuf.
100 '-Wno-missing-prototypes',
101 '-Wno-missing-variable-declarations',
102 '-Wno-sign-conversion',
103 '-Wno-unknown-pragmas',
104 '-Wno-unused-macros',
105]
106
107# A mapping of dependencies to system libraries. Libraries in this map will not
108# be built statically but instead added as dependencies of the amalgamated
109# project.
110system_library_map = {
111 '//buildtools:protobuf_full': {
112 'libs': ['protobuf'],
113 'cflags': protobuf_cflags,
114 },
115 '//buildtools:protobuf_lite': {
116 'libs': ['protobuf-lite'],
117 'cflags': protobuf_cflags,
118 },
Primiano Tucci834fdc72019-10-04 11:33:44 +0100119 '//buildtools:protoc_lib': {
120 'libs': ['protoc']
121 },
Sami Kyostila0a34b032019-05-16 18:28:48 +0100122}
123
124# ----------------------------------------------------------------------------
125# End of configuration.
126# ----------------------------------------------------------------------------
127
128tool_name = os.path.basename(__file__)
Primiano Tuccie22ffbd2020-01-17 01:11:31 +0000129project_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
Sami Kyostila0a34b032019-05-16 18:28:48 +0100130preamble = """// Copyright (C) 2019 The Android Open Source Project
131//
132// Licensed under the Apache License, Version 2.0 (the "License");
133// you may not use this file except in compliance with the License.
134// You may obtain a copy of the License at
135//
136// http://www.apache.org/licenses/LICENSE-2.0
137//
138// Unless required by applicable law or agreed to in writing, software
139// distributed under the License is distributed on an "AS IS" BASIS,
140// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
141// See the License for the specific language governing permissions and
142// limitations under the License.
143//
144// This file is automatically generated by %s. Do not edit.
145""" % tool_name
146
147
Primiano Tuccia3645202020-08-03 16:28:18 +0200148def apply_denylist(denylist, items):
149 return [item for item in items if not re.match(denylist, item)]
Sami Kyostila0a34b032019-05-16 18:28:48 +0100150
151
Primiano Tuccia3645202020-08-03 16:28:18 +0200152def apply_allowlist(allowlist, items):
153 return [item for item in items if re.match(allowlist, item)]
Sami Kyostila0a34b032019-05-16 18:28:48 +0100154
155
Primiano Tuccie22ffbd2020-01-17 01:11:31 +0000156def normalize_path(path):
157 path = os.path.relpath(path, project_root)
158 path = re.sub(r'^out/[^/]+/', '', path)
159 return path
160
161
Sami Kyostila0a34b032019-05-16 18:28:48 +0100162class Error(Exception):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100163 pass
Sami Kyostila0a34b032019-05-16 18:28:48 +0100164
165
166class DependencyNode(object):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100167 """A target in a GN build description along with its dependencies."""
Sami Kyostila0a34b032019-05-16 18:28:48 +0100168
Primiano Tucci834fdc72019-10-04 11:33:44 +0100169 def __init__(self, target_name):
170 self.target_name = target_name
171 self.dependencies = set()
Sami Kyostila0a34b032019-05-16 18:28:48 +0100172
Primiano Tucci834fdc72019-10-04 11:33:44 +0100173 def add_dependency(self, target_node):
174 if target_node in self.dependencies:
175 return
176 self.dependencies.add(target_node)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100177
Primiano Tucci834fdc72019-10-04 11:33:44 +0100178 def iterate_depth_first(self):
179 for node in sorted(self.dependencies, key=lambda n: n.target_name):
180 for node in node.iterate_depth_first():
181 yield node
182 if self.target_name:
183 yield self
Sami Kyostila0a34b032019-05-16 18:28:48 +0100184
185
186class DependencyTree(object):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100187 """A tree of GN build target dependencies."""
Sami Kyostila0a34b032019-05-16 18:28:48 +0100188
Primiano Tucci834fdc72019-10-04 11:33:44 +0100189 def __init__(self):
190 self.target_to_node_map = {}
191 self.root = self._get_or_create_node(None)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100192
Primiano Tucci834fdc72019-10-04 11:33:44 +0100193 def _get_or_create_node(self, target_name):
194 if target_name in self.target_to_node_map:
195 return self.target_to_node_map[target_name]
196 node = DependencyNode(target_name)
197 self.target_to_node_map[target_name] = node
198 return node
Sami Kyostila0a34b032019-05-16 18:28:48 +0100199
Primiano Tucci834fdc72019-10-04 11:33:44 +0100200 def add_dependency(self, from_target, to_target):
201 from_node = self._get_or_create_node(from_target)
202 to_node = self._get_or_create_node(to_target)
203 assert from_node is not to_node
204 from_node.add_dependency(to_node)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100205
Primiano Tucci834fdc72019-10-04 11:33:44 +0100206 def iterate_depth_first(self):
207 for node in self.root.iterate_depth_first():
208 yield node
Sami Kyostila0a34b032019-05-16 18:28:48 +0100209
210
211class AmalgamatedProject(object):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100212 """In-memory representation of an amalgamated source/header pair."""
Sami Kyostila0a34b032019-05-16 18:28:48 +0100213
Primiano Tucci834fdc72019-10-04 11:33:44 +0100214 def __init__(self, desc, source_deps, compute_deps_only=False):
215 """Constructor.
Sami Kyostila0a34b032019-05-16 18:28:48 +0100216
217 Args:
218 desc: JSON build description.
219 source_deps: A map of (source file, [dependency header]) which is
220 to detect which header files are included by each source file.
Sami Kyostilad4c857e2019-08-08 18:44:33 +0100221 compute_deps_only: If True, the project will only be used to compute
222 dependency information. Use |get_source_files()| to retrieve
223 the result.
Sami Kyostila0a34b032019-05-16 18:28:48 +0100224 """
Primiano Tucci834fdc72019-10-04 11:33:44 +0100225 self.desc = desc
226 self.source_deps = source_deps
227 self.header = []
228 self.source = []
229 self.source_defines = []
230 # Note that we don't support multi-arg flags.
231 self.cflags = set(default_cflags)
232 self.ldflags = set()
233 self.defines = set()
234 self.libs = set()
235 self._dependency_tree = DependencyTree()
236 self._processed_sources = set()
237 self._processed_headers = set()
238 self._processed_header_deps = set()
239 self._processed_source_headers = set() # Header files included from .cc
240 self._include_re = re.compile(r'#include "(.*)"')
241 self._compute_deps_only = compute_deps_only
Sami Kyostila0a34b032019-05-16 18:28:48 +0100242
Primiano Tucci834fdc72019-10-04 11:33:44 +0100243 def add_target(self, target_name):
244 """Include |target_name| in the amalgamated result."""
245 self._dependency_tree.add_dependency(None, target_name)
246 self._add_target_dependencies(target_name)
247 self._add_target_flags(target_name)
248 self._add_target_headers(target_name)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100249
Primiano Tucci834fdc72019-10-04 11:33:44 +0100250 # Recurse into target deps, but only for protos. This generates headers
Sami Kyostila02fccc12020-07-14 19:33:15 +0100251 # for all the .{pbzero,gen}.h files, even if they don't #include each other.
Primiano Tucci834fdc72019-10-04 11:33:44 +0100252 for _, dep in self._iterate_dep_edges(target_name):
253 if (dep not in self._processed_header_deps and
254 re.match(recurse_in_header_deps, dep)):
255 self._processed_header_deps.add(dep)
256 self.add_target(dep)
Primiano Tuccicb050652019-08-29 01:10:34 +0200257
Primiano Tucci834fdc72019-10-04 11:33:44 +0100258 def _iterate_dep_edges(self, target_name):
259 target = self.desc[target_name]
260 for dep in target.get('deps', []):
261 # Ignore system libraries since they will be added as build-time
262 # dependencies.
263 if dep in system_library_map:
264 continue
265 # Don't descend into build action dependencies.
266 if self.desc[dep]['type'] == 'action':
267 continue
268 for sub_target, sub_dep in self._iterate_dep_edges(dep):
269 yield sub_target, sub_dep
270 yield target_name, dep
Primiano Tuccicb050652019-08-29 01:10:34 +0200271
Primiano Tucci834fdc72019-10-04 11:33:44 +0100272 def _iterate_target_and_deps(self, target_name):
273 yield target_name
274 for _, dep in self._iterate_dep_edges(target_name):
275 yield dep
Sami Kyostila0a34b032019-05-16 18:28:48 +0100276
Primiano Tucci834fdc72019-10-04 11:33:44 +0100277 def _add_target_dependencies(self, target_name):
278 for target, dep in self._iterate_dep_edges(target_name):
279 self._dependency_tree.add_dependency(target, dep)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100280
Primiano Tucci834fdc72019-10-04 11:33:44 +0100281 def process_dep(dep):
282 if dep in system_library_map:
283 self.libs.update(system_library_map[dep].get('libs', []))
284 self.cflags.update(system_library_map[dep].get('cflags', []))
285 self.defines.update(system_library_map[dep].get('defines', []))
286 return True
Sami Kyostila0a34b032019-05-16 18:28:48 +0100287
Primiano Tucci834fdc72019-10-04 11:33:44 +0100288 def walk_all_deps(target_name):
289 target = self.desc[target_name]
290 for dep in target.get('deps', []):
291 if process_dep(dep):
292 return
293 walk_all_deps(dep)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100294
Primiano Tucci834fdc72019-10-04 11:33:44 +0100295 walk_all_deps(target_name)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100296
Primiano Tucci834fdc72019-10-04 11:33:44 +0100297 def _filter_cflags(self, cflags):
298 # Since we want to deduplicate flags, combine two-part switches (e.g.,
299 # "-foo bar") into one value ("-foobar") so we can store the result as
300 # a set.
301 result = []
302 for flag in cflags:
303 if flag.startswith('-'):
304 result.append(flag)
305 else:
306 result[-1] += flag
Primiano Tuccia3645202020-08-03 16:28:18 +0200307 return apply_allowlist(cflag_allowlist, result)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100308
Primiano Tucci834fdc72019-10-04 11:33:44 +0100309 def _add_target_flags(self, target_name):
310 for target_name in self._iterate_target_and_deps(target_name):
311 target = self.desc[target_name]
312 self.cflags.update(self._filter_cflags(target.get('cflags', [])))
313 self.cflags.update(self._filter_cflags(target.get('cflags_cc', [])))
314 self.ldflags.update(
Primiano Tuccia3645202020-08-03 16:28:18 +0200315 apply_allowlist(ldflag_allowlist, target.get('ldflags', [])))
316 self.libs.update(apply_denylist(lib_denylist, target.get('libs', [])))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100317 self.defines.update(
Primiano Tuccia3645202020-08-03 16:28:18 +0200318 apply_allowlist(define_allowlist, target.get('defines', [])))
Sami Kyostila0a34b032019-05-16 18:28:48 +0100319
Primiano Tucci834fdc72019-10-04 11:33:44 +0100320 def _add_target_headers(self, target_name):
321 target = self.desc[target_name]
322 if not 'sources' in target:
323 return
324 headers = [
325 gn_utils.label_to_path(s) for s in target['sources'] if s.endswith('.h')
326 ]
327 for header in headers:
328 self._add_header(target_name, header)
Sami Kyostila7e8509f2019-05-29 12:36:24 +0100329
Primiano Tucci834fdc72019-10-04 11:33:44 +0100330 def _get_include_dirs(self, target_name):
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000331 include_dirs = set(default_includes)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100332 for target_name in self._iterate_target_and_deps(target_name):
333 target = self.desc[target_name]
334 if 'include_dirs' in target:
335 include_dirs.update(
336 [gn_utils.label_to_path(d) for d in target['include_dirs']])
337 return include_dirs
Sami Kyostila0a34b032019-05-16 18:28:48 +0100338
Primiano Tucci834fdc72019-10-04 11:33:44 +0100339 def _add_source_included_header(self, include_dirs, allowed_files,
340 header_name):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100341 for include_dir in include_dirs:
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100342 rel_path = os.path.join(include_dir, header_name)
343 full_path = os.path.join(gn_utils.repo_root(), rel_path)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100344 if os.path.exists(full_path):
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100345 if not rel_path in allowed_files:
Primiano Tucci834fdc72019-10-04 11:33:44 +0100346 return
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000347 if full_path in self._processed_headers:
348 return
349 if full_path in self._processed_source_headers:
350 return
351 self._processed_source_headers.add(full_path)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100352 with open(full_path) as f:
Hector Dearmanb1989b02022-07-05 20:11:07 +0100353 self.source.append('// %s begin header: %s' %
354 (tool_name, normalize_path(full_path)))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100355 self.source.extend(
356 self._process_source_includes(include_dirs, allowed_files, f))
357 return
358 if self._compute_deps_only:
359 return
360 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs)
361 raise Error('Header file %s not found. %s' % (header_name, msg))
Sami Kyostila0a34b032019-05-16 18:28:48 +0100362
Primiano Tucci834fdc72019-10-04 11:33:44 +0100363 def _add_source(self, target_name, source_name):
364 if source_name in self._processed_sources:
365 return
366 self._processed_sources.add(source_name)
367 include_dirs = self._get_include_dirs(target_name)
368 deps = self.source_deps[source_name]
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100369 full_path = os.path.join(gn_utils.repo_root(), source_name)
370 if not os.path.exists(full_path):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100371 raise Error('Source file %s not found' % source_name)
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100372 with open(full_path) as f:
Hector Dearmanb1989b02022-07-05 20:11:07 +0100373 self.source.append('// %s begin source: %s' %
374 (tool_name, normalize_path(full_path)))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100375 try:
376 self.source.extend(
377 self._patch_source(
Hector Dearmanb1989b02022-07-05 20:11:07 +0100378 source_name,
379 self._process_source_includes(include_dirs, deps, f)))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100380 except Error as e:
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000381 raise Error('Failed adding source %s: %s' % (source_name, e))
Sami Kyostila0a34b032019-05-16 18:28:48 +0100382
Primiano Tucci834fdc72019-10-04 11:33:44 +0100383 def _add_header_included_header(self, include_dirs, header_name):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100384 for include_dir in include_dirs:
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100385 full_path = os.path.join(gn_utils.repo_root(), include_dir, header_name)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100386 if os.path.exists(full_path):
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000387 if full_path in self._processed_headers:
388 return
389 self._processed_headers.add(full_path)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100390 with open(full_path) as f:
Hector Dearmanb1989b02022-07-05 20:11:07 +0100391 self.header.append('// %s begin header: %s' %
392 (tool_name, normalize_path(full_path)))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100393 self.header.extend(self._process_header_includes(include_dirs, f))
394 return
395 if self._compute_deps_only:
396 return
397 msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs)
398 raise Error('Header file %s not found. %s' % (header_name, msg))
Sami Kyostila7e8509f2019-05-29 12:36:24 +0100399
Primiano Tucci834fdc72019-10-04 11:33:44 +0100400 def _add_header(self, target_name, header_name):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100401 include_dirs = self._get_include_dirs(target_name)
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100402 full_path = os.path.join(gn_utils.repo_root(), header_name)
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000403 if full_path in self._processed_headers:
404 return
405 self._processed_headers.add(full_path)
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100406 if not os.path.exists(full_path):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100407 if self._compute_deps_only:
408 return
409 raise Error('Header file %s not found' % header_name)
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100410 with open(full_path) as f:
Hector Dearmanb1989b02022-07-05 20:11:07 +0100411 self.header.append('// %s begin header: %s' %
412 (tool_name, normalize_path(full_path)))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100413 try:
414 self.header.extend(self._process_header_includes(include_dirs, f))
415 except Error as e:
Sami Kyostilabc0cd792023-07-07 08:42:10 +0000416 raise Error('Failed adding header %s: %s' % (header_name, e))
Sami Kyostila7e8509f2019-05-29 12:36:24 +0100417
Primiano Tucci834fdc72019-10-04 11:33:44 +0100418 def _patch_source(self, source_name, lines):
419 result = []
420 namespace = re.sub(r'[^a-z]', '_',
421 os.path.splitext(os.path.basename(source_name))[0])
422 for line in lines:
423 # Protobuf generates an identical anonymous function into each
424 # message description. Rename all but the first occurrence to avoid
425 # duplicate symbol definitions.
426 line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace)
427 result.append(line)
428 return result
Sami Kyostila0a34b032019-05-16 18:28:48 +0100429
Primiano Tucci834fdc72019-10-04 11:33:44 +0100430 def _process_source_includes(self, include_dirs, allowed_files, file):
431 result = []
432 for line in file:
433 line = line.rstrip('\n')
434 m = self._include_re.match(line)
435 if not m:
436 result.append(line)
437 continue
438 elif re.match(includes_to_remove, m.group(1)):
439 result.append('// %s removed: %s' % (tool_name, line))
440 else:
441 result.append('// %s expanded: %s' % (tool_name, line))
442 self._add_source_included_header(include_dirs, allowed_files,
443 m.group(1))
444 return result
Sami Kyostila7e8509f2019-05-29 12:36:24 +0100445
Primiano Tucci834fdc72019-10-04 11:33:44 +0100446 def _process_header_includes(self, include_dirs, file):
447 result = []
448 for line in file:
449 line = line.rstrip('\n')
450 m = self._include_re.match(line)
451 if not m:
452 result.append(line)
453 continue
454 elif re.match(includes_to_remove, m.group(1)):
455 result.append('// %s removed: %s' % (tool_name, line))
456 else:
457 result.append('// %s expanded: %s' % (tool_name, line))
458 self._add_header_included_header(include_dirs, m.group(1))
459 return result
Sami Kyostila0a34b032019-05-16 18:28:48 +0100460
Primiano Tucci834fdc72019-10-04 11:33:44 +0100461 def generate(self):
462 """Prepares the output for this amalgamated project.
Sami Kyostila0a34b032019-05-16 18:28:48 +0100463
464 Call save() to persist the result.
465 """
Primiano Tucci834fdc72019-10-04 11:33:44 +0100466 assert not self._compute_deps_only
467 self.source_defines.append('// %s: predefined macros' % tool_name)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100468
Primiano Tucci834fdc72019-10-04 11:33:44 +0100469 def add_define(name):
470 # Valued macros aren't supported for now.
471 assert '=' not in name
472 self.source_defines.append('#if !defined(%s)' % name)
473 self.source_defines.append('#define %s' % name)
474 self.source_defines.append('#endif')
475
476 for name in self.defines:
477 add_define(name)
478 for target_name, source_name in self.get_source_files():
479 self._add_source(target_name, source_name)
480
481 def get_source_files(self):
482 """Return a list of (target, [source file]) that describes the source
Sami Kyostilad4c857e2019-08-08 18:44:33 +0100483 files pulled in by each target which is a dependency of this project.
484 """
Primiano Tucci834fdc72019-10-04 11:33:44 +0100485 source_files = []
486 for node in self._dependency_tree.iterate_depth_first():
487 target = self.desc[node.target_name]
488 if not 'sources' in target:
489 continue
490 sources = [(node.target_name, gn_utils.label_to_path(s))
491 for s in target['sources']
492 if s.endswith('.cc')]
493 source_files.extend(sources)
494 return source_files
Sami Kyostila0a34b032019-05-16 18:28:48 +0100495
Primiano Tucci834fdc72019-10-04 11:33:44 +0100496 def _get_nice_path(self, prefix, format):
497 basename = os.path.basename(prefix)
498 return os.path.join(
499 os.path.relpath(os.path.dirname(prefix)), format % basename)
Sami Kyostila0a34b032019-05-16 18:28:48 +0100500
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100501 def _make_directories(self, directory):
502 if not os.path.isdir(directory):
503 os.makedirs(directory)
504
Chinglin Yu2e958a22022-09-28 14:45:07 +0800505 def save(self, output_prefix, system_buildtools=False):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100506 """Save the generated header and source file pair.
Sami Kyostila0a34b032019-05-16 18:28:48 +0100507
508 Returns a message describing the output with build instructions.
509 """
Primiano Tucci834fdc72019-10-04 11:33:44 +0100510 header_file = self._get_nice_path(output_prefix, '%s.h')
511 source_file = self._get_nice_path(output_prefix, '%s.cc')
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100512 self._make_directories(os.path.dirname(header_file))
513 self._make_directories(os.path.dirname(source_file))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100514 with open(header_file, 'w') as f:
515 f.write('\n'.join([preamble] + self.header + ['\n']))
516 with open(source_file, 'w') as f:
517 include_stmt = '#include "%s"' % os.path.basename(header_file)
518 f.write('\n'.join([preamble] + self.source_defines + [include_stmt] +
519 self.source + ['\n']))
Chinglin Yu2e958a22022-09-28 14:45:07 +0800520 build_cmd = self.get_build_command(output_prefix, system_buildtools)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100521 return """Amalgamated project written to %s and %s.
Sami Kyostila0a34b032019-05-16 18:28:48 +0100522
523Build settings:
524 - cflags: %s
525 - ldflags: %s
526 - libs: %s
Sami Kyostila0a34b032019-05-16 18:28:48 +0100527
528Example build command:
529
530%s
Hector Dearmanb1989b02022-07-05 20:11:07 +0100531""" % (header_file, source_file, ' '.join(self.cflags), ' '.join(
532 self.ldflags), ' '.join(self.libs), ' '.join(build_cmd))
Sami Kyostila0a34b032019-05-16 18:28:48 +0100533
Chinglin Yu2e958a22022-09-28 14:45:07 +0800534 def get_build_command(self, output_prefix, system_buildtools=False):
Primiano Tucci834fdc72019-10-04 11:33:44 +0100535 """Returns an example command line for building the output source."""
536 source = self._get_nice_path(output_prefix, '%s.cc')
537 library = self._get_nice_path(output_prefix, 'lib%s.so')
Lalit Maganti54afee62020-07-16 19:28:04 +0100538
Chinglin Yu2e958a22022-09-28 14:45:07 +0800539 if sys.platform.startswith('linux') and not system_buildtools:
Hector Dearmanb1989b02022-07-05 20:11:07 +0100540 llvm_script = os.path.join(gn_utils.repo_root(), 'gn', 'standalone',
541 'toolchain', 'linux_find_llvm.py')
Primiano Tucci34bc5592021-02-19 17:53:36 +0100542 cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode()
Lalit Maganti54afee62020-07-16 19:28:04 +0100543 else:
544 cxx = 'clang++'
545
546 build_cmd = [cxx, source, '-o', library, '-shared'] + \
Primiano Tucci834fdc72019-10-04 11:33:44 +0100547 sorted(self.cflags) + sorted(self.ldflags)
548 for lib in sorted(self.libs):
549 build_cmd.append('-l%s' % lib)
550 return build_cmd
Sami Kyostila0a34b032019-05-16 18:28:48 +0100551
552
Sami Kyostila0a34b032019-05-16 18:28:48 +0100553def main():
Primiano Tucci834fdc72019-10-04 11:33:44 +0100554 parser = argparse.ArgumentParser(
555 description='Generate an amalgamated header/source pair from a GN '
556 'build description.')
557 parser.add_argument(
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100558 '--out',
559 help='The name of the temporary build folder in \'out\'',
560 default='tmp.gen_amalgamated.%u' % os.getpid())
561 parser.add_argument(
Primiano Tucci834fdc72019-10-04 11:33:44 +0100562 '--output',
563 help='Base name of files to create. A .cc/.h extension will be added',
Matthew Clarkson58a3ebb2019-10-11 11:29:18 +0100564 default=os.path.join(gn_utils.repo_root(), 'out/amalgamated/perfetto'))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100565 parser.add_argument(
566 '--gn_args',
Ching-lin Yuae24f9b2022-10-11 14:05:31 +0000567 help='GN arguments used to prepare the output directory',
568 default=gn_args)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100569 parser.add_argument(
570 '--keep',
571 help='Don\'t delete the GN output directory at exit',
572 action='store_true')
573 parser.add_argument(
574 '--build', help='Also compile the generated files', action='store_true')
575 parser.add_argument(
576 '--check', help='Don\'t keep the generated files', action='store_true')
577 parser.add_argument('--quiet', help='Only report errors', action='store_true')
578 parser.add_argument(
579 '--dump-deps',
580 help='List all source files that the amalgamated output depends on',
581 action='store_true')
582 parser.add_argument(
Chinglin Yu2e958a22022-09-28 14:45:07 +0800583 '--system_buildtools',
584 help='Use the buildtools (e.g. gn) preinstalled in the system instead '
585 'of the hermetic ones',
586 action='store_true')
587 parser.add_argument(
Primiano Tucci834fdc72019-10-04 11:33:44 +0100588 'targets',
589 nargs=argparse.REMAINDER,
590 help='Targets to include in the output (e.g., "//:libperfetto")')
591 args = parser.parse_args()
592 targets = args.targets or default_targets
Sami Kyostila0a34b032019-05-16 18:28:48 +0100593
Hector Dearman762c18f2021-05-24 11:10:52 +0100594 # The CHANGELOG mtime triggers the perfetto_version.gen.h genrule. This is
Primiano Tucciec590132020-11-16 14:16:44 +0100595 # to avoid emitting a stale version information in the remote case of somebody
596 # running gen_amalgamated incrementally after having moved to another commit.
597 changelog_path = os.path.join(project_root, 'CHANGELOG')
Hector Dearmanb1989b02022-07-05 20:11:07 +0100598 assert (os.path.exists(changelog_path))
Primiano Tucciec590132020-11-16 14:16:44 +0100599 subprocess.check_call(['touch', '-c', changelog_path])
600
Primiano Tucci834fdc72019-10-04 11:33:44 +0100601 output = args.output
602 if args.check:
603 output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated')
604
Ching-lin Yuae24f9b2022-10-11 14:05:31 +0000605 out = gn_utils.prepare_out_directory(args.gn_args,
Chinglin Yu2e958a22022-09-28 14:45:07 +0800606 args.out,
607 system_buildtools=args.system_buildtools)
Matthew Clarksona990d952019-10-08 14:52:12 +0100608 if not args.quiet:
609 print('Building project...')
Primiano Tucci834fdc72019-10-04 11:33:44 +0100610 try:
Chinglin Yu2e958a22022-09-28 14:45:07 +0800611 desc = gn_utils.load_build_description(out, args.system_buildtools)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100612
613 # We need to build everything first so that the necessary header
614 # dependencies get generated. However if we are just dumping dependency
615 # information this can be skipped, allowing cross-platform operation.
616 if not args.dump_deps:
Chinglin Yu2e958a22022-09-28 14:45:07 +0800617 gn_utils.build_targets(out, targets,
618 system_buildtools=args.system_buildtools)
619 source_deps = gn_utils.compute_source_dependencies(out,
620 args.system_buildtools)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100621 project = AmalgamatedProject(
622 desc, source_deps, compute_deps_only=args.dump_deps)
623
624 for target in targets:
625 project.add_target(target)
626
627 if args.dump_deps:
628 source_files = [
629 source_file for _, source_file in project.get_source_files()
630 ]
631 print('\n'.join(sorted(set(source_files))))
632 return
633
634 project.generate()
Chinglin Yu2e958a22022-09-28 14:45:07 +0800635 result = project.save(output, args.system_buildtools)
Primiano Tucci834fdc72019-10-04 11:33:44 +0100636 if not args.quiet:
637 print(result)
638 if args.build:
639 if not args.quiet:
640 sys.stdout.write('Building amalgamated project...')
641 sys.stdout.flush()
Chinglin Yu2e958a22022-09-28 14:45:07 +0800642 subprocess.check_call(project.get_build_command(output,
643 args.system_buildtools))
Primiano Tucci834fdc72019-10-04 11:33:44 +0100644 if not args.quiet:
645 print('done')
646 finally:
647 if not args.keep:
648 shutil.rmtree(out)
Sami Kyostila468e61d2019-05-23 15:54:01 +0100649 if args.check:
Primiano Tucci834fdc72019-10-04 11:33:44 +0100650 shutil.rmtree(os.path.dirname(output))
Sami Kyostila468e61d2019-05-23 15:54:01 +0100651
Sami Kyostila0a34b032019-05-16 18:28:48 +0100652
653if __name__ == '__main__':
Primiano Tucci834fdc72019-10-04 11:33:44 +0100654 sys.exit(main())