Move scripts to python3
Today we are in a limbo where some scripts are run with
python (which defaults to python2 on some systems) some
other with python3. This normalizes everything* to python3.
In the process of doing so:
- Fixed a sorting issue in gen_bazel and gen_android_bp,
as it was causing different outputs when running with
the two different versions of python.
- Got rid of some vestigial scripts we no longer use.
- Got rid of the shebang from some scripts that are executed
only by GN. Having a shebang is misleading because in reality
GN is the one deciding which version of python to run.
I left only few others that are too expensive to test
right now like some infra bots and the heap profile tools.
Change-Id: Iab5b28196c5a92603745a0dccad17831c43ee590
diff --git a/tools/build_all_configs.py b/tools/build_all_configs.py
index 8f2245c..3318daf 100755
--- a/tools/build_all_configs.py
+++ b/tools/build_all_configs.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/check_include_violations b/tools/check_include_violations
index 9b68c7b..d86931b 100755
--- a/tools/check_include_violations
+++ b/tools/check_include_violations
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/check_proto_comments b/tools/check_proto_comments
index d56fea4..6e7392f 100755
--- a/tools/check_proto_comments
+++ b/tools/check_proto_comments
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/compat.py b/tools/compat.py
index cdf5afa..44b0c02 100644
--- a/tools/compat.py
+++ b/tools/compat.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/dev_server b/tools/dev_server
deleted file mode 100755
index 331720e..0000000
--- a/tools/dev_server
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/usr/bin/env python
-# Copyright (C) 2019 The Android Open Source Project
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-import sys
-import os
-import time
-import argparse
-import socket
-import subprocess
-import threading
-
-try:
- import socketserver
- from http.server import SimpleHTTPRequestHandler
-except ImportError:
- import SocketServer as socketserver
- import SimpleHTTPServer
- SimpleHTTPRequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
-
-
-class TCPServer(socketserver.TCPServer):
-
- def server_bind(self):
- self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- self.socket.bind(self.server_address)
-
-
-class Server(object):
-
- def __init__(self, port, directory, rebuilder):
- self.port = port
- self.directory = directory
- self.rebuilder = rebuilder
- self.lock = threading.Lock()
-
- def serve(self):
- this = self
-
- class Handler(SimpleHTTPRequestHandler):
-
- def __init__(self, *args, **kwargs):
- SimpleHTTPRequestHandler.__init__(self, *args, **kwargs)
- self.extensions_map['.wasm'] = 'application/wasm'
-
- def translate_path(self, path):
- path = SimpleHTTPRequestHandler.translate_path(self, path)
- relpath = os.path.relpath(path, os.getcwd())
- fullpath = os.path.join(this.directory, relpath)
- return fullpath
-
- def do_GET(self):
- try:
- with this.lock:
- this.rebuilder.rebuild_if_needed()
- except RebuildFailed as e:
- self.send_response(200)
- self.send_header("Content-type", "text/html")
- self.end_headers()
- self.wfile.write("<pre>")
- self.wfile.write(e.stdout_and_stderr)
- return
- return SimpleHTTPRequestHandler.do_GET(self)
-
- print('Starting server at http://localhost:{}'.format(self.port))
- httpd = TCPServer(('', self.port), Handler)
- try:
- httpd.serve_forever()
- except KeyboardInterrupt:
- httpd.shutdown()
- httpd.server_close()
-
-
-class RebuildFailed(Exception):
-
- def __init__(self, stdout_and_stderr):
- self.stdout_and_stderr = stdout_and_stderr
-
-
-class Rebuilder(object):
-
- def __init__(self, command, ignored_paths):
- self.command = command
- self.ignored_paths = ignored_paths
- self.last_disk_state = None
- self.abs_ignored_paths = [os.path.abspath(p) for p in self.ignored_paths]
-
- def rebuild_if_needed(self):
- if self.last_disk_state == self.last_modified_time():
- return
- stdout_and_stderr, success = self.rebuild()
- if not success:
- message = b"Failed to build! Command output:\n\n" + stdout_and_stderr
- raise RebuildFailed(message)
- self.last_disk_state = self.last_modified_time()
-
- def last_modified_time(self):
- start_time = time.time()
- max_mtime = 0
- for dirpath, dirnames, filenames in os.walk(
- os.path.abspath('.'), topdown=True):
- dirnames[:] = [
- n for n in dirnames
- if not self.should_ignore_path(os.path.join(dirpath, n))
- ]
- for filename in filenames:
- path = os.path.join(dirpath, filename)
- if self.should_ignore_path(path):
- continue
- mtime = os.stat(path).st_mtime
- max_mtime = max(max_mtime, mtime)
- print(' scanned in {:.03f}s'.format(time.time() - start_time).rjust(
- 80, '='))
- return max_mtime
-
- def should_ignore_path(self, path):
- return path in self.abs_ignored_paths
-
- def rebuild(self):
- print('Running command: {}'.format(self.command))
- print(' begin build'.rjust(80, '='))
- start_time = time.time()
- status = 0
- try:
- stdout_and_stderr = subprocess.check_output(
- self.command, shell=True, stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError as e:
- status = e.returncode
- stdout_and_stderr = e.output
- print(stdout_and_stderr.decode('utf8'))
- print(' built in {:.03f}s'.format(time.time() - start_time).rjust(80, '='))
- return stdout_and_stderr, status == 0
-
-
-def main(argv):
- parser = argparse.ArgumentParser(description='HTTP server for UI development')
- parser.add_argument(
- '-p',
- '--port',
- help='port number (default: 3000)',
- type=int,
- default=3000)
- parser.add_argument(
- '-i', '--ignore', default=[], action='append', help='Ignore this path')
- parser.add_argument(
- '-s',
- '--serve',
- default=os.getcwd(),
- help='Serve this directory (default: current directory)')
- parser.add_argument('command', default='', nargs='?', help='Command to run')
-
- args = parser.parse_args(argv)
-
- rebuilder = Rebuilder(args.command, args.ignore)
- server = Server(args.port, args.serve, rebuilder)
- server.serve()
- return 0
-
-
-if __name__ == '__main__':
- sys.exit(main(sys.argv[1:]))
diff --git a/tools/extract_linux_syscall_tables b/tools/extract_linux_syscall_tables
index d1753bf..0f2d2a6 100755
--- a/tools/extract_linux_syscall_tables
+++ b/tools/extract_linux_syscall_tables
@@ -1,18 +1,19 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
import re
import sys
-import urllib2
+
+from urllib.request import urlopen
syscalls = {}
def print_table(name):
tab = syscalls[name]
- print '\n\n----------------- BEGIN OF %s -----------------' % name
- for i in xrange(max(tab.keys()) + 1):
- print '"%s", // %d' % (tab.get(i, ''), i)
- print '----------------- END OF %s -----------------' % name
+ print('\n\n----------------- BEGIN OF %s -----------------' % name)
+ for i in range(max(tab.keys()) + 1):
+ print('"%s", // %d' % (tab.get(i, ''), i))
+ print('----------------- END OF %s -----------------' % name)
# Parses a .tbl file (new format).
@@ -41,24 +42,24 @@
def Main():
KSRC = 'https://raw.githubusercontent.com/torvalds/linux/v4.20/'
- response = urllib2.urlopen(KSRC + 'arch/x86/entry/syscalls/syscall_64.tbl')
- syscalls['x86_64'] = parse_tlb(response.read())
+ response = urlopen(KSRC + 'arch/x86/entry/syscalls/syscall_64.tbl')
+ syscalls['x86_64'] = parse_tlb(response.read().decode())
- response = urllib2.urlopen(KSRC + 'arch/x86/entry/syscalls/syscall_32.tbl')
- syscalls['x86'] = parse_tlb(response.read())
+ response = urlopen(KSRC + 'arch/x86/entry/syscalls/syscall_32.tbl')
+ syscalls['x86'] = parse_tlb(response.read().decode())
- response = urllib2.urlopen(KSRC + 'arch/arm/tools/syscall.tbl')
- syscalls['armeabi'] = parse_tlb(response.read())
+ response = urlopen(KSRC + 'arch/arm/tools/syscall.tbl')
+ syscalls['armeabi'] = parse_tlb(response.read().decode())
- response = urllib2.urlopen(KSRC + 'arch/arm64/include/asm/unistd32.h')
- syscalls['aarch32'] = parse_def(response.read())
+ response = urlopen(KSRC + 'arch/arm64/include/asm/unistd32.h')
+ syscalls['aarch32'] = parse_def(response.read().decode())
# From:
# arch/arm64/include/asm/unistd.h
# -> arch/arm64/include/uapi/asm/unistd.h
# -> include/uapi/asm-generic/unistd.h
- response = urllib2.urlopen(KSRC + 'include/uapi/asm-generic/unistd.h')
- syscalls['aarch64'] = parse_def(response.read())
+ response = urlopen(KSRC + 'include/uapi/asm-generic/unistd.h')
+ syscalls['aarch64'] = parse_def(response.read().decode())
print_table('x86_64')
print_table('x86')
@@ -68,4 +69,4 @@
if __name__ == '__main__':
- sys.exit(Main())
+ sys.exit(Main())
\ No newline at end of file
diff --git a/tools/fix_include_guards b/tools/fix_include_guards
index 4f08b41..2799b39 100755
--- a/tools/fix_include_guards
+++ b/tools/fix_include_guards
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/gen_all b/tools/gen_all
index d96148b..b065236 100755
--- a/tools/gen_all
+++ b/tools/gen_all
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/gen_amalgamated b/tools/gen_amalgamated
index 95e9e97..bff4641 100755
--- a/tools/gen_amalgamated
+++ b/tools/gen_amalgamated
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -511,7 +511,6 @@
f.write('\n'.join([preamble] + self.source_defines + [include_stmt] +
self.source + ['\n']))
build_cmd = self.get_build_command(output_prefix)
-
return """Amalgamated project written to %s and %s.
Build settings:
@@ -534,7 +533,7 @@
llvm_script = os.path.join(gn_utils.repo_root(), 'gn',
'standalone', 'toolchain',
'linux_find_llvm.py')
- cxx = subprocess.check_output([llvm_script]).splitlines()[2]
+ cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode()
else:
cxx = 'clang++'
diff --git a/tools/gen_android_bp b/tools/gen_android_bp
index 0d5ba7b..5026370 100755
--- a/tools/gen_android_bp
+++ b/tools/gen_android_bp
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/gen_bazel b/tools/gen_bazel
index 1674c0f..48eb2df 100755
--- a/tools/gen_bazel
+++ b/tools/gen_bazel
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -213,7 +213,7 @@
indent = ' '
else:
indent = ' '
- for entry in v:
+ for entry in sorted(v):
if entry.startswith('PERFETTO_CONFIG.'):
res += '%s %s,\n' % (indent, entry)
else:
diff --git a/tools/gen_binary_descriptors b/tools/gen_binary_descriptors
index 7fcd714..0027c27 100755
--- a/tools/gen_binary_descriptors
+++ b/tools/gen_binary_descriptors
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/gen_merged_protos b/tools/gen_merged_protos
index 07964b6..8286cc5 100755
--- a/tools/gen_merged_protos
+++ b/tools/gen_merged_protos
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/install-build-deps b/tools/install-build-deps
index 0a7ff82..ed1eb72 100755
--- a/tools/install-build-deps
+++ b/tools/install-build-deps
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/protoc_helper.py b/tools/protoc_helper.py
old mode 100755
new mode 100644
index 6861009..a5f937b
--- a/tools/protoc_helper.py
+++ b/tools/protoc_helper.py
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
# Copyright (C) 2018 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/pull_ftrace_format_files.py b/tools/pull_ftrace_format_files.py
index be934bc..9f33df9 100755
--- a/tools/pull_ftrace_format_files.py
+++ b/tools/pull_ftrace_format_files.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/run_android_emulator b/tools/run_android_emulator
index 7852919..6f6ffcf 100755
--- a/tools/run_android_emulator
+++ b/tools/run_android_emulator
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,7 +15,6 @@
import argparse
import os
-import shutil
import sys
@@ -55,8 +54,8 @@
os.path.join(aosp_path, 'vendor-qemu.img'), '-data',
os.path.join(aosp_path, 'userdata-qemu.img')
]
- print '\n'.join('='.join(x) for x in env.items())
- print ' '.join([emulator_bin] + emulator_args)
+ print('\n'.join('='.join(x) for x in env.items()))
+ print(' '.join([emulator_bin] + emulator_args))
if args.pid:
with open(args.pid, 'w') as f:
f.write(str(os.getpid()))
diff --git a/tools/run_android_test b/tools/run_android_test
index 7a69e9e..00fc5c2 100755
--- a/tools/run_android_test
+++ b/tools/run_android_test
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,12 +15,10 @@
import argparse
import os
-import re
import functools
import logging
import subprocess
import sys
-import tempfile
import time
""" Runs a test executable on Android.
@@ -85,7 +83,7 @@
def AdbCall(*args):
cmd = [ADB_PATH] + list(args)
- print '> adb ' + ' '.join(args)
+ print('> adb ' + ' '.join(args))
return subprocess.check_call(cmd)
@@ -93,18 +91,18 @@
if not os.path.exists(host):
logging.fatal('Cannot find %s. Was it built?', host)
cmd = [ADB_PATH, 'push', host, device]
- print '> adb push ' + ' '.join(cmd[2:])
- with open(os.devnull) as devnull:
+ print('> adb push ' + ' '.join(cmd[2:]))
+ with open(os.devnull, 'wb') as devnull:
return subprocess.check_call(cmd, stdout=devnull)
def GetProp(prop):
cmd = [ADB_PATH, 'shell', 'getprop', prop]
- print '> adb ' + ' '.join(cmd)
- output = subprocess.check_output(cmd)
+ print('> adb ' + ' '.join(cmd))
+ output = subprocess.check_output(cmd).decode()
lines = output.splitlines()
assert len(lines) == 1, 'Expected output to have one line: {}'.format(output)
- print lines[0]
+ print(lines[0])
return lines[0]
@@ -139,7 +137,7 @@
test_bin = os.path.join(args.out_dir, args.test_name)
assert os.path.exists(test_bin)
- print 'Waiting for device ...'
+ print('Waiting for device ...')
AdbCall('wait-for-device')
# WaitForBootCompletion()
AdbCall('root')
@@ -183,14 +181,14 @@
if args.cmd_args:
actual_args = [arg.replace(args.test_name, binary) for arg in args.cmd_args]
cmd += ' ' + ' '.join(actual_args)
- print cmd
- retcode = subprocess.call([ADB_PATH, 'shell', cmd])
+ print(cmd)
+ retcode = subprocess.call([ADB_PATH, 'shell', '-tt', cmd])
if not args.no_cleanup:
AdbCall('shell', 'rm -rf "%s"' % target_dir)
# Smoke test that adb shell is actually propagating retcode. adb has a history
# of breaking this.
- test_code = subprocess.call([ADB_PATH, 'shell', 'echo Done; exit 42'])
+ test_code = subprocess.call([ADB_PATH, 'shell', '-tt', 'echo Done; exit 42'])
if test_code != 42:
logging.fatal('adb is incorrectly propagating the exit code')
return 1
diff --git a/tools/test_gen_amalgamated.py b/tools/test_gen_amalgamated.py
index a4239bf..aa99bc4 100755
--- a/tools/test_gen_amalgamated.py
+++ b/tools/test_gen_amalgamated.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -63,7 +63,7 @@
if sys.platform.startswith('linux'):
llvm_script = os.path.join(ROOT_DIR, 'gn', 'standalone', 'toolchain',
'linux_find_llvm.py')
- cxx = subprocess.check_output([llvm_script]).splitlines()[2]
+ cxx = subprocess.check_output([llvm_script]).splitlines()[2].decode()
else:
cxx = 'clang++'
call(cxx, *args)
diff --git a/tools/trace_processor b/tools/trace_processor
index f788c48..590a269 100755
--- a/tools/trace_processor
+++ b/tools/trace_processor
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/traceconv b/tools/traceconv
index 243e222..a1a8619 100755
--- a/tools/traceconv
+++ b/tools/traceconv
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
diff --git a/tools/write_version_header.py b/tools/write_version_header.py
index 028ad2a..0325f6f 100755
--- a/tools/write_version_header.py
+++ b/tools/write_version_header.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
# Copyright (C) 2020 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
