blob: 689f0466a2daeb33c2200aba2bca2ab31565d9c0 [file] [log] [blame]
Primiano Tucci33a94e12021-07-21 22:19:21 +01001#!/usr/bin/env python3
2# Copyright (C) 2021 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
Primiano Tucci11d94e12022-08-02 17:44:33 +010015"""Updates the python scripts in python/perfetto/prebuilts/manifests
Primiano Tucci33a94e12021-07-21 22:19:21 +010016
Primiano Tucci11d94e12022-08-02 17:44:33 +010017This script does the following, for each entry in MANIFESTS_TO_UPDATE:
Primiano Tucci33a94e12021-07-21 22:19:21 +010018 - Downloads the artifact by the LUCI infrastructure, one for each arch.
19 - Computes the SHA-256 of each artifact.
20 - Generates a manifest with URL, SHA-256 and other details.
Primiano Tucci1fe41c82021-07-28 20:52:40 +010021 - Merges get_perfetto_prebuilt.py with the manifest and writes tools/xxx.
Primiano Tucci33a94e12021-07-21 22:19:21 +010022
23This script is supposed to be run by Perfetto OWNERS after every monthly release
24after the LUCI jobs have completed.
25"""
26
27import argparse
28import hashlib
29import logging
30import os
31import subprocess
32import sys
33
34from concurrent.futures import ThreadPoolExecutor
35
36GCS_URL = 'https://commondatastorage.googleapis.com/perfetto-luci-artifacts'
37
38ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
Primiano Tucci11d94e12022-08-02 17:44:33 +010039MANIFESTS_DIR = os.path.join(ROOT_DIR, 'python/perfetto/prebuilts/manifests')
Primiano Tucci33a94e12021-07-21 22:19:21 +010040
Lalit Maganti973d7c32022-05-30 17:49:53 +010041UNIX_ARCHS = [
Hector Dearmanb1989b02022-07-05 20:11:07 +010042 'mac-amd64',
43 'mac-arm64',
44 'linux-amd64',
45 'linux-arm',
46 'linux-arm64',
47 'android-arm',
48 'android-arm64',
49 'android-x86',
50 'android-x64',
Lalit Maganti973d7c32022-05-30 17:49:53 +010051]
52ALL_ARCHS = UNIX_ARCHS + ['windows-amd64']
53
Primiano Tucci11d94e12022-08-02 17:44:33 +010054MANIFESTS_TO_UPDATE = [
Primiano Tucci33a94e12021-07-21 22:19:21 +010055 {
Lalit Maganti973d7c32022-05-30 17:49:53 +010056 'tool': 'trace_processor_shell',
57 'archs': ALL_ARCHS
Primiano Tucci33a94e12021-07-21 22:19:21 +010058 },
59 {
Hector Dearmana9545e52022-05-17 12:23:25 +010060 'tool': 'traceconv',
Lalit Maganti973d7c32022-05-30 17:49:53 +010061 'archs': ALL_ARCHS
Primiano Tucci33a94e12021-07-21 22:19:21 +010062 },
Primiano Tucci1fe41c82021-07-28 20:52:40 +010063 {
Primiano Tucci1fe41c82021-07-28 20:52:40 +010064 'tool': 'tracebox',
Lalit Maganti973d7c32022-05-30 17:49:53 +010065 'archs': UNIX_ARCHS
Primiano Tucci1fe41c82021-07-28 20:52:40 +010066 },
Primiano Tucci33a94e12021-07-21 22:19:21 +010067]
68
Primiano Tuccid3e40bd2021-08-03 10:52:39 +010069# Maps a 'os-arch' string (were arch follows LUCI conventions) into
70# corresponding tuples that match against python's platform / machine API
71# (see get_perfetto_prebuilt.py for usage).
Primiano Tucci33a94e12021-07-21 22:19:21 +010072ARCH_TO_PYTHON = {
73 'mac-amd64': {
74 'platform': 'darwin',
Primiano Tuccid3e40bd2021-08-03 10:52:39 +010075 'machine': ['x86_64'],
Primiano Tucci33a94e12021-07-21 22:19:21 +010076 },
Primiano Tucci06dab692022-03-15 15:32:33 +000077 'mac-arm64': {
78 'platform': 'darwin',
79 'machine': ['arm64'],
80 },
Primiano Tucci33a94e12021-07-21 22:19:21 +010081 'windows-amd64': {
82 'platform': 'win32',
Primiano Tuccid3e40bd2021-08-03 10:52:39 +010083 'machine': ['amd64'],
84 },
85 'linux-amd64': {
86 'platform': 'linux',
87 'machine': ['x86_64'],
88 },
89 'linux-arm': {
90 'platform': 'linux',
91 'machine': ['armv6l', 'armv7l', 'armv8l'],
92 },
93 'linux-arm64': {
94 'platform': 'linux',
95 'machine': ['aarch64'],
Primiano Tucci33a94e12021-07-21 22:19:21 +010096 },
97}
98
99
100def make_manifest(git_revision, tool, arch):
101 ext = '.exe' if arch.startswith('windows') else ''
102 file_name = tool + ext
103 url = '%s/%s/%s/%s' % (GCS_URL, git_revision, arch, file_name)
104 logging.info('Downloading %s', url)
105 data = subprocess.check_output(['curl', '-fsL', '-o', '-', url])
106 manifest = {
Primiano Tucci33a94e12021-07-21 22:19:21 +0100107 'arch': arch,
108 'file_name': file_name,
109 'file_size': len(data),
110 'url': url,
111 'sha256': hashlib.sha256(data).hexdigest()
112 }
113 manifest.update(ARCH_TO_PYTHON.get(arch, {}))
114 return manifest
115
116
Primiano Tucci11d94e12022-08-02 17:44:33 +0100117def update_manifest(git_revision, tool_name, archs):
Primiano Tucci33a94e12021-07-21 22:19:21 +0100118 with ThreadPoolExecutor(max_workers=8) as executor:
119 manifests = list(
120 executor.map(lambda arch: make_manifest(git_revision, tool_name, arch),
121 archs))
Primiano Tucci11d94e12022-08-02 17:44:33 +0100122 out_file = os.path.join(MANIFESTS_DIR, tool_name + '.py')
Primiano Tucci33a94e12021-07-21 22:19:21 +0100123
Primiano Tucci11d94e12022-08-02 17:44:33 +0100124 content = '# This file has been generated by: {script} {git_revision}\n'
125 content += '{tool_uppercase}_MANIFEST = {manifests}\n'
Primiano Tucci1fe41c82021-07-28 20:52:40 +0100126 content = content.format(
Lalit Magantidda6e852023-07-07 00:39:52 +0100127 script=os.path.relpath(__file__),
Primiano Tucci11d94e12022-08-02 17:44:33 +0100128 tool_uppercase=tool_name.upper(),
Primiano Tucci1fe41c82021-07-28 20:52:40 +0100129 git_revision=git_revision,
Primiano Tucci11d94e12022-08-02 17:44:33 +0100130 manifests=str(manifests))
Primiano Tucci1fe41c82021-07-28 20:52:40 +0100131
132 with open(out_file + '.tmp', 'w') as f:
Primiano Tucci11d94e12022-08-02 17:44:33 +0100133 f.write(content)
Primiano Tucci1fe41c82021-07-28 20:52:40 +0100134 subprocess.check_call(['yapf', '-i', out_file + '.tmp'])
135 os.rename(out_file + '.tmp', out_file)
136 os.chmod(out_file, 0o755)
Primiano Tucci33a94e12021-07-21 22:19:21 +0100137
138
139def main():
Primiano Tucci5df05572021-10-05 11:15:51 +0100140 usage = '%s v20.0 | 0a1b2c3d\n\n' % __file__
141 usage += 'To list available revisions run\n'
142 usage += 'gsutil ls gs://perfetto-luci-artifacts/\n'
143 usage += 'or visit https://chrome-infra-packages.appspot.com/p/perfetto\n'
144 parser = argparse.ArgumentParser(usage=usage)
145 parser.add_argument('version')
Primiano Tucci33a94e12021-07-21 22:19:21 +0100146 args = parser.parse_args()
147
Primiano Tucci5df05572021-10-05 11:15:51 +0100148 git_revision = args.version
Primiano Tucci11d94e12022-08-02 17:44:33 +0100149 for spec in MANIFESTS_TO_UPDATE:
150 logging.info('Updating %s', spec['tool'])
151 update_manifest(git_revision, spec['tool'], spec['archs'])
Primiano Tucci33a94e12021-07-21 22:19:21 +0100152
153
154if __name__ == '__main__':
155 logging.basicConfig(level=logging.INFO)
156 sys.exit(main())