blob: 6eb7c7be29d8cccb8806817997e29c94ef702df0 [file] [log] [blame]
#!/usr/bin/env python
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import hashlib
import logging
import os
import shutil
import subprocess
import sys
import tempfile
import zipfile
from collections import namedtuple
from platform import system
# The format for the deps below is the following:
# (target_folder, source_url, sha1, target_platform)
# |source_url| can be either a git repo or a http url.
# If a git repo, |checksum| is the SHA1 committish that will be checked out.
# If a http url, |checksum| is the SHA256 of the downloaded file.
# If the url is a .zip or .tgz file it will be automatically deflated under
# |target_folder|, taking care of stripping the root folder if it's a single
# root (to avoid ending up with buildtools/protobuf/protobuf-1.2.3/... and have
# instead just buildtools/protobuf).
# |target_platform| is either 'darwin', 'linux' or 'all' and applies the dep
# only on the given platform
Dependency = namedtuple(
'Dependency',
['target_folder', 'source_url', 'checksum', 'target_platform'])
# Dependencies required to build code on the host or when targeting desktop OS.
BUILD_DEPS_HOST = [
# GN. From https://chrome-infra-packages.appspot.com/dl/gn/gn/.
# git_revision:83dad00afb232d7235dd70dff1ee90292d72a01e .
Dependency(
'buildtools/mac/gn',
'https://storage.googleapis.com/perfetto/gn-mac-1695-83dad00a',
'513d3adeb56b745e62af4e3ccb76b76f023c6aaa25d6a2be9a89e44cd10a4c1a',
'darwin'),
Dependency(
'buildtools/linux64/gn',
'https://storage.googleapis.com/perfetto/gn-linux64-1695-83dad00a',
'4f589364153f182b05cd845e93407489d6ce8acc03290c897928a7bd22b20cce',
'linux'),
Dependency(
'buildtools/win/gn.exe',
'https://storage.googleapis.com/perfetto/gn-win-1695-83dad00a',
'908c29556539292203d2952ebf55df03697cbc7cf526a3e295f31ba2576e4cac',
'windows'),
# clang-format
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/mac/clang-format.sha1
Dependency(
'buildtools/mac/clang-format',
'https://storage.googleapis.com/chromium-clang-format/62bde1baa7196ad9df969fc1f06b66360b1a927b',
'6df686a937443cbe6efc013467a7ba5f98d3f187eb7765bb7abc6ce47626cf66',
'darwin'),
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/linux64/clang-format.sha1
Dependency(
'buildtools/linux64/clang-format',
'https://storage.googleapis.com/chromium-clang-format/1baf0089e895c989a311b6a38ed94d0e8be4c0a7',
'd02a97a87e8c28898033aaf5986967b24dc47ebd5b376e1cd93e5009f22cd75e',
'linux'),
# From https://chromium.googlesource.com/chromium/src/buildtools/+/refs/heads/master/win/clang-format.exe.sha1
Dependency(
'buildtools/win/clang-format.exe',
'https://storage.googleapis.com/chromium-clang-format/d4afd4eba27022f5f6d518133aebde57281677c9',
'2ba1b4d3ade90ea80316890b598ab5fc16777572be26afec6ce23117da121b80',
'windows'),
# Keep the SHA1 in sync with |clang_format_rev| in chromium //buildtools/DEPS.
Dependency(
'buildtools/clang_format/script',
'https://chromium.googlesource.com/chromium/llvm-project/cfe/tools/clang-format.git',
'96636aa0e9f047f17447f2d45a094d0b59ed7917', 'all'),
# Ninja
Dependency(
'buildtools/mac/ninja',
'https://storage.googleapis.com/perfetto/ninja-mac-c15b0698da038b2bd2e8970c14c75fadc06b1add',
'4224b90734590b0148ad8ee63ee7b295e88e0652e4d1f4271ef2b91d880b0e19',
'darwin'),
Dependency(
'buildtools/linux64/ninja',
'https://storage.googleapis.com/perfetto/ninja-linux64-c866952bda50c29a669222477309287119bbb7e8',
'54ac6a01362190aaabf4cf276f9c8982cdf11b225438940fdde3339be0f2ecdc',
'linux'),
Dependency(
'buildtools/win/ninja.exe',
'https://storage.googleapis.com/perfetto/ninja-win-4a5f05c24afef05ef03329a1bbfedee0678b524a',
'6f8af488be74ed8787d04e107080d05330587a4198ba047bd5b7f5b0c3150d61',
'windows'),
# Keep in sync with Android's //external/googletest/README.version.
Dependency(
'buildtools/googletest',
'https://android.googlesource.com/platform/external/googletest.git',
'3f05f651ae3621db58468153e32016bc1397800b', 'all'),
# Keep in sync with Chromium's //third_party/protobuf.
Dependency(
'buildtools/protobuf',
'https://chromium.googlesource.com/external/github.com/google/protobuf.git',
'6a59a2ad1f61d9696092f79b6d74368b4d7970a3', # refs/tags/v3.9.0
'all'),
# libc++, libc++abi and libunwind for Linux where we need to rebuild the C++
# lib from sources. Keep the SHA1s in sync with Chrome's src/buildtools/DEPS.
Dependency(
'buildtools/libcxx',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxx.git',
'd9040c75cfea5928c804ab7c235fed06a63f743a', 'all'),
Dependency(
'buildtools/libcxxabi',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libcxxabi.git',
'196ba1aaa8ac285d94f4ea8d9836390a45360533', 'all'),
Dependency(
'buildtools/libunwind',
'https://chromium.googlesource.com/external/github.com/llvm/llvm-project/libunwind.git',
'd999d54f4bca789543a2eb6c995af2d9b5a1f3ed', 'all'),
# Keep the revision in sync with Chrome's PACKAGE_VERSION in
# tools/clang/scripts/update.py.
Dependency(
'buildtools/clang.tgz',
'https://commondatastorage.googleapis.com/chromium-browser-clang/Linux_x64/clang-llvmorg-12-init-5035-gd0abc757-3.tgz',
'b0c3015209b6d624844ad230064eb5c9b4429a2eafd4854981e73217c563d93d',
'linux'),
Dependency(
'buildtools/clang.tgz',
'https://commondatastorage.googleapis.com/chromium-browser-clang/Win/clang-llvmorg-12-init-5035-gd0abc757-3.tgz',
'b2854d871a466e3a060469b5edb24ca355ef64576d38778f64acbd3c6d7cf530',
'windows'),
# Keep in sync with chromium DEPS.
Dependency(
'buildtools/libfuzzer',
'https://chromium.googlesource.com/chromium/llvm-project/compiler-rt/lib/fuzzer.git',
'debe7d2d1982e540fbd6bd78604bf001753f9e74', 'linux'),
# Benchmarking tool.
Dependency(
'buildtools/benchmark',
'https://chromium.googlesource.com/external/github.com/google/benchmark.git',
'090faecb454fbd6e6e17a75ef8146acb037118d4', 'all'),
# Libbacktrace, for stacktraces in Linux/Android debug builds.
# From https://github.com/ianlancetaylor/libbacktrace/archive/177940370e4a6b2509e92a0aaa9749184e64af43.zip
Dependency(
'buildtools/libbacktrace.zip',
'https://storage.googleapis.com/perfetto/libbacktrace-177940370e4a6b2509e92a0aaa9749184e64af43.zip',
'21ac9a4209f7aeef766c482be53a7fa365063c031c7077e2070b491202983b31',
'all'),
# Sqlite for the trace processing library.
# This is the amalgamated source whose compiled output is meant to be faster.
# We still pull the full source for the extensions (which are not available
# in the amalgamation).
Dependency(
'buildtools/sqlite.zip',
'https://storage.googleapis.com/perfetto/sqlite-amalgamation-3320300.zip',
'e9cec01d4519e2d49b3810615237325263fe1feaceae390ee12b4a29bd73dbe2',
'all'),
Dependency(
'buildtools/sqlite_src',
'https://chromium.googlesource.com/external/github.com/sqlite/sqlite.git',
'ee3686eb50c0e3dbb087c9a0976f7e37e1b014ae', # refs/tags/version-3.32.3.
'all'),
# JsonCpp for legacy json import. Used only by the trace processor in
# standalone builds.
Dependency(
'buildtools/jsoncpp',
'https://chromium.googlesource.com/external/github.com/open-source-parsers/jsoncpp.git',
'6aba23f4a8628d599a9ef7fa4811c4ff6e4070e2', # refs/tags/1.9.3.
'all'),
# These dependencies are for libunwindstack, which is used by src/profiling.
Dependency('buildtools/android-core',
'https://android.googlesource.com/platform/system/core.git',
'9e6cef7f07d8c11b3ea820938aeb7ff2e9dbaa52', 'all'),
Dependency('buildtools/android-unwinding',
'https://android.googlesource.com/platform/system/unwinding.git',
'4af25d50dd646368577b57bb18a9224ae202fe64', 'all'),
Dependency('buildtools/android-logging',
'https://android.googlesource.com/platform/system/logging.git',
'7b36b566c9113fc703d68f76e8f40c0c2432481c', 'all'),
Dependency('buildtools/android-libbase',
'https://android.googlesource.com/platform/system/libbase.git',
'78f1c2f83e625bdf66d55b48bdb3a301c20d2fb3', 'all'),
Dependency('buildtools/android-libprocinfo',
'https://android.googlesource.com/platform/system/libprocinfo.git',
'bd752195f52109fb0dca525236ab9dec74b986e1', 'all'),
Dependency('buildtools/lzma',
'https://android.googlesource.com/platform/external/lzma.git',
'7851dce6f4ca17f5caa1c93a4e0a45686b1d56c3', 'all'),
Dependency('buildtools/zlib',
'https://android.googlesource.com/platform/external/zlib.git',
'5c85a2da4c13eda07f69d81a1579a5afddd35f59', 'all'),
Dependency('buildtools/bionic',
'https://android.googlesource.com/platform/bionic.git',
'a60488109cda997dfd83832731c8527feaa2825e', 'all'),
# Example traces for regression tests.
Dependency(
'buildtools/test_data.zip',
'https://storage.googleapis.com/perfetto/test-data-20201207-131703.zip',
'2c894eee8e19cb3247dd0aff14e7c4b14c09c305988e901f852fb004ea2e6716',
'all',
),
# Linenoise, used only by trace_processor in standalone builds.
Dependency('buildtools/linenoise',
'https://fuchsia.googlesource.com/third_party/linenoise.git',
'c894b9e59f02203dbe4e2be657572cf88c4230c3', 'all'),
]
# Dependencies required to build Android code.
# URLs and SHA1s taken from:
# - https://dl.google.com/android/repository/repository-11.xml
# - https://dl.google.com/android/repository/sys-img/android/sys-img.xml
BUILD_DEPS_ANDROID = [
# Android NDK
Dependency(
'buildtools/ndk.zip',
'https://dl.google.com/android/repository/android-ndk-r17b-darwin-x86_64.zip',
'd21072c04ffcf8a723a4dba3837c886bd30c18c0623a4d0ddc53850e2222d27f',
'darwin'),
Dependency(
'buildtools/ndk.zip',
'https://dl.google.com/android/repository/android-ndk-r17b-linux-x86_64.zip',
'5dfbbdc2d3ba859fed90d0e978af87c71a91a5be1f6e1c40ba697503d48ccecd',
'linux'),
]
# Dependencies required to run Android tests.
TEST_DEPS_ANDROID = [
# Android emulator images.
Dependency(
'buildtools/aosp-arm.zip',
'https://storage.googleapis.com/perfetto/aosp-02022018-arm.zip',
'f5c7a3a22ad7aa0bd14ba467e8697e1e917d306699bd25622aa4419a413b9b67',
'all'),
# platform-tools.zip contains adb binaries.
Dependency(
'buildtools/android_sdk/platform-tools.zip',
'https://dl.google.com/android/repository/platform-tools_r26.0.0-darwin.zip',
'98d392cbd21ca20d643c7e1605760cc49075611e317c534096b5564053f4ac8e',
'darwin'),
Dependency(
'buildtools/android_sdk/platform-tools.zip',
'https://dl.google.com/android/repository/platform-tools_r26.0.0-linux.zip',
'90208207521d85abf0d46e3374aa4e04b7aff74e4f355c792ac334de7a77e50b',
'linux'),
# Android emulator binaries.
Dependency(
'buildtools/emulator',
'https://android.googlesource.com/platform/prebuilts/android-emulator.git',
'4b260028dc27bc92c39bee9129cb2ba839970956', 'all'),
]
# This variable is updated by tools/roll-catapult-trace-viewer.
CATAPULT_SHA256 = 'b30108e05268ce6c65bb4126b65f6bfac165d17f5c1fd285046e7e6fd76c209f'
TYPEFACES_SHA256 = 'b3f0f14eeecd4555ae94f897ec246b2c6e046ce0ea417407553f5767e7812575'
UI_DEPS = [
Dependency(
'buildtools/nodejs.tgz',
'https://storage.googleapis.com/perfetto/node-v12.18.3-darwin-x64.tar.gz',
'af376caf114bdd5d7e566dbf7590e9077ffc01f9b2692eb2651f31d7219a30bb',
'darwin'),
Dependency(
'buildtools/nodejs.tgz',
'https://storage.googleapis.com/perfetto/node-v12.18.3-linux-x64.tar.gz',
'0be428afce5b24f799f3b1ab8902d9d91094b929ac58d2b1cec29436ea2d742c',
'linux'),
Dependency(
'buildtools/emsdk/emscripten.tgz',
'https://storage.googleapis.com/perfetto/emscripten-1.37.40.tar.gz',
'40d8a095c510e5e5e2032131e0dc5d38f996172deb0415588b702146d1ea8bd5',
'all'),
Dependency(
'buildtools/emsdk/llvm.tgz',
'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-darwin.tar.gz',
'ed51f3a467c0a5af5365abb448794734affcc2932fa19d62be233dee435c89ea',
'darwin'),
Dependency(
'buildtools/emsdk/llvm.tgz',
'https://storage.googleapis.com/perfetto/emscripten-llvm-e1.37.40-static-linux.tar.gz',
'257da419f92d305025777f340b79ee9f3d3077af38f5e7efb0fac1060d3ea6d9',
'linux'),
Dependency(
'buildtools/catapult_trace_viewer.tgz',
'https://storage.googleapis.com/perfetto/catapult_trace_viewer-%s.tar.gz'
% CATAPULT_SHA256, CATAPULT_SHA256, 'all'),
Dependency(
'buildtools/typefaces.tgz',
'https://storage.googleapis.com/perfetto/typefaces-%s.tar.gz' %
TYPEFACES_SHA256, TYPEFACES_SHA256, 'all')
]
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
UI_DIR = os.path.join(ROOT_DIR, 'ui')
NODE_MODULES_STATUS_FILE = os.path.join(UI_DIR, 'node_modules', '.last_install')
def DownloadURL(url, out_file):
subprocess.check_call(['curl', '-L', '-#', '-o', out_file, url])
def ReadFile(path):
if not os.path.exists(path):
return None
with open(path) as f:
return f.read().strip()
def MkdirRecursive(path):
# Works with both relative and absolute paths
cwd = '/' if path.startswith('/') else ROOT_DIR
for part in path.split('/'):
cwd = os.path.join(cwd, part)
if not os.path.exists(cwd):
os.makedirs(cwd)
else:
assert (os.path.isdir(cwd))
def HashLocalFile(path):
if not os.path.exists(path):
return None
with open(path, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
def ExtractZipfilePreservePermissions(zf, info, path):
zf.extract(info.filename, path=path)
target_path = os.path.join(path, info.filename)
min_acls = 0o755 if info.filename.endswith('/') else 0o644
os.chmod(target_path, (info.external_attr >> 16) | min_acls)
def IsGitRepoCheckoutOutAtRevision(path, revision):
return ReadFile(os.path.join(path, '.git', 'HEAD')) == revision
def CheckoutGitRepo(path, git_url, revision, check_only):
if IsGitRepoCheckoutOutAtRevision(path, revision):
return False
if check_only:
return True
path = path.replace('/', os.sep)
if os.path.exists(path):
shutil.rmtree(path)
MkdirRecursive(path)
logging.info('Fetching %s @ %s into %s', git_url, revision, path)
subprocess.check_call(['git', 'init', path], cwd=path)
subprocess.check_call(
['git', 'fetch', '--quiet', '--depth', '1', git_url, revision], cwd=path)
subprocess.check_call(['git', 'checkout', revision, '--quiet'], cwd=path)
assert (IsGitRepoCheckoutOutAtRevision(path, revision))
return True
def InstallNodeModules(force_clean=False):
if force_clean:
node_modules = os.path.join(UI_DIR, 'node_modules')
logging.info('Clearing %s', node_modules)
subprocess.check_call(['git', 'clean', '-qxffd', node_modules],
cwd=ROOT_DIR)
logging.info("Running npm install in {0}".format(UI_DIR))
subprocess.check_call([os.path.join(UI_DIR, 'npm'), 'install', '--no-save'],
cwd=UI_DIR)
with open(NODE_MODULES_STATUS_FILE, 'w') as f:
f.write(HashLocalFile(os.path.join(UI_DIR, 'package-lock.json')))
def CheckNodeModules():
"""Returns True if the modules are up-to-date.
There doesn't seem to be an easy way to check node modules versions. Instead
just check if package-lock.json changed since the last `npm install` call.
"""
if not os.path.exists(NODE_MODULES_STATUS_FILE):
return False
with open(NODE_MODULES_STATUS_FILE, 'r') as f:
actual = f.read()
expected = HashLocalFile(os.path.join(UI_DIR, 'package-lock.json'))
return expected == actual
def CheckHashes():
for deps in [BUILD_DEPS_HOST, BUILD_DEPS_ANDROID, TEST_DEPS_ANDROID, UI_DEPS]:
for dep in deps:
if dep.source_url.endswith('.git'):
continue
logging.info('Downloading %s from %s', dep.target_platform,
dep.source_url)
with tempfile.NamedTemporaryFile(delete=False) as f:
f.close()
DownloadURL(dep.source_url, f.name)
actual_checksum = HashLocalFile(f.name)
os.unlink(f.name)
if (actual_checksum != dep.checksum):
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
dep.source_url, dep.checksum, actual_checksum))
def Main():
parser = argparse.ArgumentParser()
parser.add_argument('--android', action='store_true')
parser.add_argument('--ui', action='store_true')
parser.add_argument('--check-only')
parser.add_argument('--verify', help='Check all URLs', action='store_true')
args = parser.parse_args()
if args.verify:
CheckHashes()
return 0
deps = BUILD_DEPS_HOST
if args.android:
deps += BUILD_DEPS_ANDROID + TEST_DEPS_ANDROID
if args.ui:
deps += UI_DEPS
deps_updated = False
nodejs_updated = False
for dep in deps:
if (dep.target_platform != 'all' and
dep.target_platform != system().lower()):
continue
local_path = os.path.join(ROOT_DIR, dep.target_folder)
if dep.source_url.endswith('.git'):
deps_updated |= CheckoutGitRepo(local_path, dep.source_url, dep.checksum,
args.check_only)
continue
is_zip = local_path.endswith('.zip') or local_path.endswith('.tgz')
zip_target_dir = local_path[:-4] if is_zip else None
zip_dir_stamp = os.path.join(zip_target_dir, '.stamp') if is_zip else None
if ((not is_zip and HashLocalFile(local_path) == dep.checksum) or
(is_zip and ReadFile(zip_dir_stamp) == dep.checksum)):
continue
deps_updated = True
if args.check_only:
continue
MkdirRecursive(os.path.dirname(dep.target_folder))
if HashLocalFile(local_path) != dep.checksum:
download_path = local_path + '.tmp'
logging.info('Downloading %s from %s', local_path, dep.source_url)
DownloadURL(dep.source_url, download_path)
os.chmod(download_path, 0o755)
actual_checksum = HashLocalFile(download_path)
if (actual_checksum != dep.checksum):
os.remove(download_path)
logging.fatal('SHA-256 mismatch for {} expected {} was {}'.format(
download_path, dep.checksum, actual_checksum))
return 1
shutil.move(download_path, local_path)
if 'nodejs' in dep.target_folder:
nodejs_updated = True
assert (HashLocalFile(local_path) == dep.checksum)
if is_zip:
logging.info('Extracting %s into %s' % (local_path, zip_target_dir))
assert (os.path.commonprefix((ROOT_DIR, zip_target_dir)) == ROOT_DIR)
if os.path.exists(zip_target_dir):
logging.info('Deleting stale dir %s' % zip_target_dir)
shutil.rmtree(zip_target_dir)
# Decompress the archive.
if local_path.endswith('.tgz'):
MkdirRecursive(zip_target_dir)
subprocess.check_call(['tar', '-xf', local_path], cwd=zip_target_dir)
elif local_path.endswith('.zip'):
with zipfile.ZipFile(local_path, 'r') as zf:
for info in zf.infolist():
ExtractZipfilePreservePermissions(zf, info, zip_target_dir)
# If the zip contains one root folder, rebase one level up moving all
# its sub files and folders inside |target_dir|.
subdir = os.listdir(zip_target_dir)
if len(subdir) == 1:
subdir = os.path.join(zip_target_dir, subdir[0])
if os.path.isdir(subdir):
for subf in os.listdir(subdir):
shutil.move(os.path.join(subdir, subf), zip_target_dir)
os.rmdir(subdir)
# Create stamp and remove the archive.
with open(zip_dir_stamp, 'w') as stamp_file:
stamp_file.write(dep.checksum)
os.remove(local_path)
if args.ui:
# Needs to happen after nodejs is installed above.
if args.check_only:
deps_updated = not CheckNodeModules()
else:
InstallNodeModules(force_clean=nodejs_updated)
if args.check_only:
if not deps_updated:
with open(args.check_only, 'w') as f:
f.write('OK') # The content is irrelevant, just keep GN happy.
return 0
argz = ' '.join([x for x in sys.argv[1:] if '--check-only' not in x])
sys.stderr.write('\033[91mBuild deps are stale. ' +
'Please run tools/install-build-deps %s\033[0m' % argz)
return 1
if deps_updated:
# Stale binary files may be compiled against old sysroot headers that aren't
# tracked by gn.
logging.warning('Remember to run "gn clean <output_directory>" ' +
'to avoid stale binary files.')
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
sys.exit(Main())