tools: python amalgamation for tools/tracebox etc
Before this change:
the business logic for tools/{tracebox, record_android_trace, ...}
was in the file itself and the manifest was replaced in-place
when invoking tools/roll-prebuilts.
This still made it impossible to share code between tools
without copy/pasting.
With this change:
- Move the business logic to python/xxx
- Add an amalgamator that follows includes. Only the form
'from perfetto.xxx import yyy' is supported.
- Keep the amalgamated files in tools/traceconv
No code sharing / major refactorings are made by this change.
They can happen as a follow-up though
Change-Id: I7420387881e6ef1e109abae6380dde7c06ac1b27diff --git a/tools/roll-prebuilts b/tools/roll-prebuilts
index 50ef65d..08666ad 100755
--- a/tools/roll-prebuilts
+++ b/tools/roll-prebuilts
@@ -12,9 +12,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Updates the python scripts in tools/{trace_processor, traceconv, tracebox}
+"""Updates the python scripts in python/perfetto/prebuilts/manifests
-This script does the following, for each entry in SCRIPTS_TO_UPDATE:
+This script does the following, for each entry in MANIFESTS_TO_UPDATE:
- Downloads the artifact by the LUCI infrastructure, one for each arch.
- Computes the SHA-256 of each artifact.
- Generates a manifest with URL, SHA-256 and other details.
@@ -36,7 +36,7 @@
GCS_URL = 'https://commondatastorage.googleapis.com/perfetto-luci-artifacts'
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-TOOLS_DIR = os.path.join(ROOT_DIR, 'tools')
+MANIFESTS_DIR = os.path.join(ROOT_DIR, 'python/perfetto/prebuilts/manifests')
UNIX_ARCHS = [
'mac-amd64',
@@ -51,39 +51,16 @@
]
ALL_ARCHS = UNIX_ARCHS + ['windows-amd64']
-SCRIPTS_TO_UPDATE = [
- # Scripts using trace_processor_shell.
+MANIFESTS_TO_UPDATE = [
{
- 'script': 'trace_processor',
'tool': 'trace_processor_shell',
'archs': ALL_ARCHS
},
-
- # Scripts using traceconv.
{
- 'script': 'traceconv',
'tool': 'traceconv',
'archs': ALL_ARCHS
},
{
- 'script': 'heap_profile',
- 'tool': 'traceconv',
- 'archs': ALL_ARCHS
- },
- {
- 'script': 'cpu_profile',
- 'tool': 'traceconv',
- 'archs': ALL_ARCHS
- },
-
- # Scripts using tracebox.
- {
- 'script': 'tracebox',
- 'tool': 'tracebox',
- 'archs': UNIX_ARCHS
- },
- {
- 'script': 'record_android_trace',
'tool': 'tracebox',
'archs': UNIX_ARCHS
},
@@ -127,7 +104,6 @@
logging.info('Downloading %s', url)
data = subprocess.check_output(['curl', '-fsL', '-o', '-', url])
manifest = {
- 'tool': tool,
'arch': arch,
'file_name': file_name,
'file_size': len(data),
@@ -138,41 +114,23 @@
return manifest
-# Returns the section of get_perfetto_prebuilt.py which should be copy/pasted
-# in the various scripts.
-def read_get_perfetto_prebuilt_script():
- in_file = os.path.join(TOOLS_DIR, 'get_perfetto_prebuilt.py')
- with open(in_file, 'r') as f:
- contents = f.read()
- return contents.split('COPIED_SECTION_START_MARKER')[1]
-
-
-def update_script(git_revision, tool_name, script_name, archs):
+def update_manifest(git_revision, tool_name, archs):
with ThreadPoolExecutor(max_workers=8) as executor:
manifests = list(
executor.map(lambda arch: make_manifest(git_revision, tool_name, arch),
archs))
- out_file = os.path.join(TOOLS_DIR, script_name)
- with open(out_file) as f:
- script = f.read()
+ out_file = os.path.join(MANIFESTS_DIR, tool_name + '.py')
- begin_marker = '\n# BEGIN_SECTION_GENERATED_BY(roll-prebuilts)\n'
- end_marker = '\n# END_SECTION_GENERATED_BY(roll-prebuilts)\n'
- before = script.partition(begin_marker)[0]
- after = script.partition(end_marker)[2]
-
- content = '# Revision: {git_revision}\n'
- content += 'PERFETTO_PREBUILT_MANIFEST = {manifests}\n'
- content += '{fn_body}\n'
+ content = '# This file has been generated by: {script} {git_revision}\n'
+ content += '{tool_uppercase}_MANIFEST = {manifests}\n'
content = content.format(
+ script=__file__,
+ tool_uppercase=tool_name.upper(),
git_revision=git_revision,
- manifests=str(manifests),
- fn_body=read_get_perfetto_prebuilt_script())
-
- script = before + begin_marker + content + end_marker + after
+ manifests=str(manifests))
with open(out_file + '.tmp', 'w') as f:
- f.write(script)
+ f.write(content)
subprocess.check_call(['yapf', '-i', out_file + '.tmp'])
os.rename(out_file + '.tmp', out_file)
os.chmod(out_file, 0o755)
@@ -188,9 +146,9 @@
args = parser.parse_args()
git_revision = args.version
- for spec in SCRIPTS_TO_UPDATE:
- logging.info('Updating %s', spec['script'])
- update_script(git_revision, spec['tool'], spec['script'], spec['archs'])
+ for spec in MANIFESTS_TO_UPDATE:
+ logging.info('Updating %s', spec['tool'])
+ update_manifest(git_revision, spec['tool'], spec['archs'])
if __name__ == '__main__':
