Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # Copyright (C) 2021 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 15 | """Updates the python scripts in python/perfetto/prebuilts/manifests |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 16 | |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 17 | This script does the following, for each entry in MANIFESTS_TO_UPDATE: |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 18 | - Downloads the artifact by the LUCI infrastructure, one for each arch. |
| 19 | - Computes the SHA-256 of each artifact. |
| 20 | - Generates a manifest with URL, SHA-256 and other details. |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 21 | - Merges get_perfetto_prebuilt.py with the manifest and writes tools/xxx. |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 22 | |
| 23 | This script is supposed to be run by Perfetto OWNERS after every monthly release |
| 24 | after the LUCI jobs have completed. |
| 25 | """ |
| 26 | |
| 27 | import argparse |
| 28 | import hashlib |
| 29 | import logging |
| 30 | import os |
| 31 | import subprocess |
| 32 | import sys |
| 33 | |
| 34 | from concurrent.futures import ThreadPoolExecutor |
| 35 | |
| 36 | GCS_URL = 'https://commondatastorage.googleapis.com/perfetto-luci-artifacts' |
| 37 | |
| 38 | ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 39 | MANIFESTS_DIR = os.path.join(ROOT_DIR, 'python/perfetto/prebuilts/manifests') |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 40 | |
Lalit Maganti | 973d7c3 | 2022-05-30 17:49:53 +0100 | [diff] [blame] | 41 | UNIX_ARCHS = [ |
Hector Dearman | b1989b0 | 2022-07-05 20:11:07 +0100 | [diff] [blame] | 42 | 'mac-amd64', |
| 43 | 'mac-arm64', |
| 44 | 'linux-amd64', |
| 45 | 'linux-arm', |
| 46 | 'linux-arm64', |
| 47 | 'android-arm', |
| 48 | 'android-arm64', |
| 49 | 'android-x86', |
| 50 | 'android-x64', |
Lalit Maganti | 973d7c3 | 2022-05-30 17:49:53 +0100 | [diff] [blame] | 51 | ] |
| 52 | ALL_ARCHS = UNIX_ARCHS + ['windows-amd64'] |
| 53 | |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 54 | MANIFESTS_TO_UPDATE = [ |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 55 | { |
Lalit Maganti | 973d7c3 | 2022-05-30 17:49:53 +0100 | [diff] [blame] | 56 | 'tool': 'trace_processor_shell', |
| 57 | 'archs': ALL_ARCHS |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 58 | }, |
| 59 | { |
Hector Dearman | a9545e5 | 2022-05-17 12:23:25 +0100 | [diff] [blame] | 60 | 'tool': 'traceconv', |
Lalit Maganti | 973d7c3 | 2022-05-30 17:49:53 +0100 | [diff] [blame] | 61 | 'archs': ALL_ARCHS |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 62 | }, |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 63 | { |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 64 | 'tool': 'tracebox', |
Lalit Maganti | 973d7c3 | 2022-05-30 17:49:53 +0100 | [diff] [blame] | 65 | 'archs': UNIX_ARCHS |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 66 | }, |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 67 | ] |
| 68 | |
Primiano Tucci | d3e40bd | 2021-08-03 10:52:39 +0100 | [diff] [blame] | 69 | # Maps a 'os-arch' string (were arch follows LUCI conventions) into |
| 70 | # corresponding tuples that match against python's platform / machine API |
| 71 | # (see get_perfetto_prebuilt.py for usage). |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 72 | ARCH_TO_PYTHON = { |
| 73 | 'mac-amd64': { |
| 74 | 'platform': 'darwin', |
Primiano Tucci | d3e40bd | 2021-08-03 10:52:39 +0100 | [diff] [blame] | 75 | 'machine': ['x86_64'], |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 76 | }, |
Primiano Tucci | 06dab69 | 2022-03-15 15:32:33 +0000 | [diff] [blame] | 77 | 'mac-arm64': { |
| 78 | 'platform': 'darwin', |
| 79 | 'machine': ['arm64'], |
| 80 | }, |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 81 | 'windows-amd64': { |
| 82 | 'platform': 'win32', |
Primiano Tucci | d3e40bd | 2021-08-03 10:52:39 +0100 | [diff] [blame] | 83 | 'machine': ['amd64'], |
| 84 | }, |
| 85 | 'linux-amd64': { |
| 86 | 'platform': 'linux', |
| 87 | 'machine': ['x86_64'], |
| 88 | }, |
| 89 | 'linux-arm': { |
| 90 | 'platform': 'linux', |
| 91 | 'machine': ['armv6l', 'armv7l', 'armv8l'], |
| 92 | }, |
| 93 | 'linux-arm64': { |
| 94 | 'platform': 'linux', |
| 95 | 'machine': ['aarch64'], |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 96 | }, |
| 97 | } |
| 98 | |
| 99 | |
| 100 | def make_manifest(git_revision, tool, arch): |
| 101 | ext = '.exe' if arch.startswith('windows') else '' |
| 102 | file_name = tool + ext |
| 103 | url = '%s/%s/%s/%s' % (GCS_URL, git_revision, arch, file_name) |
| 104 | logging.info('Downloading %s', url) |
| 105 | data = subprocess.check_output(['curl', '-fsL', '-o', '-', url]) |
| 106 | manifest = { |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 107 | 'arch': arch, |
| 108 | 'file_name': file_name, |
| 109 | 'file_size': len(data), |
| 110 | 'url': url, |
| 111 | 'sha256': hashlib.sha256(data).hexdigest() |
| 112 | } |
| 113 | manifest.update(ARCH_TO_PYTHON.get(arch, {})) |
| 114 | return manifest |
| 115 | |
| 116 | |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 117 | def update_manifest(git_revision, tool_name, archs): |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 118 | with ThreadPoolExecutor(max_workers=8) as executor: |
| 119 | manifests = list( |
| 120 | executor.map(lambda arch: make_manifest(git_revision, tool_name, arch), |
| 121 | archs)) |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 122 | out_file = os.path.join(MANIFESTS_DIR, tool_name + '.py') |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 123 | |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 124 | content = '# This file has been generated by: {script} {git_revision}\n' |
| 125 | content += '{tool_uppercase}_MANIFEST = {manifests}\n' |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 126 | content = content.format( |
Lalit Maganti | dda6e85 | 2023-07-07 00:39:52 +0100 | [diff] [blame] | 127 | script=os.path.relpath(__file__), |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 128 | tool_uppercase=tool_name.upper(), |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 129 | git_revision=git_revision, |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 130 | manifests=str(manifests)) |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 131 | |
| 132 | with open(out_file + '.tmp', 'w') as f: |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 133 | f.write(content) |
Primiano Tucci | 1fe41c8 | 2021-07-28 20:52:40 +0100 | [diff] [blame] | 134 | subprocess.check_call(['yapf', '-i', out_file + '.tmp']) |
| 135 | os.rename(out_file + '.tmp', out_file) |
| 136 | os.chmod(out_file, 0o755) |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 137 | |
| 138 | |
| 139 | def main(): |
Primiano Tucci | 5df0557 | 2021-10-05 11:15:51 +0100 | [diff] [blame] | 140 | usage = '%s v20.0 | 0a1b2c3d\n\n' % __file__ |
| 141 | usage += 'To list available revisions run\n' |
| 142 | usage += 'gsutil ls gs://perfetto-luci-artifacts/\n' |
| 143 | usage += 'or visit https://chrome-infra-packages.appspot.com/p/perfetto\n' |
| 144 | parser = argparse.ArgumentParser(usage=usage) |
| 145 | parser.add_argument('version') |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 146 | args = parser.parse_args() |
| 147 | |
Primiano Tucci | 5df0557 | 2021-10-05 11:15:51 +0100 | [diff] [blame] | 148 | git_revision = args.version |
Primiano Tucci | 11d94e1 | 2022-08-02 17:44:33 +0100 | [diff] [blame] | 149 | for spec in MANIFESTS_TO_UPDATE: |
| 150 | logging.info('Updating %s', spec['tool']) |
| 151 | update_manifest(git_revision, spec['tool'], spec['archs']) |
Primiano Tucci | 33a94e1 | 2021-07-21 22:19:21 +0100 | [diff] [blame] | 152 | |
| 153 | |
| 154 | if __name__ == '__main__': |
| 155 | logging.basicConfig(level=logging.INFO) |
| 156 | sys.exit(main()) |