| #!/usr/bin/env python2 |
| # Copyright (C) 2019 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| import argparse |
| import httplib2 |
| import logging |
| import mimetypes |
| import mmap |
| import os |
| import subprocess |
| import signal |
| import sys |
| import threading |
| import time |
| |
| from common_utils import init_logging |
| from config import GCS_ARTIFACTS |
| from multiprocessing.pool import ThreadPool |
| from oauth2client.client import GoogleCredentials |
| |
| CUR_DIR = os.path.dirname(__file__) |
| RESCAN_PERIOD_SEC = 5 # Scan for new artifact directories every X seconds. |
| WATCHDOG_SEC = 60 * 6 # Self kill after 5 minutes |
| |
| tls = threading.local() |
| '''Polls for new directories under ARTIFACTS_DIR and uploads them to GCS''' |
| |
| |
| def get_http_obj(): |
| http = getattr(tls, 'http', None) |
| if http is not None: |
| return http |
| tls.http = httplib2.Http() |
| scopes = ['https://www.googleapis.com/auth/cloud-platform'] |
| creds = GoogleCredentials.get_application_default().create_scoped(scopes) |
| creds.authorize(tls.http) |
| return tls.http |
| |
| |
| def upload_one_file(fpath): |
| http = get_http_obj() |
| relpath = os.path.relpath(fpath, os.getenv('ARTIFACTS_DIR')) |
| logging.debug('Uploading %s', relpath) |
| assert (os.path.exists(fpath)) |
| fsize = os.path.getsize(fpath) |
| mime_type = mimetypes.guess_type(fpath)[0] or 'application/octet-stream' |
| mm = '' |
| hdr = {'Content-Length': fsize, 'Content-type': mime_type} |
| if fsize > 0: |
| with open(fpath, 'rb') as f: |
| mm = mmap.mmap(f.fileno(), fsize, access=mmap.ACCESS_READ) |
| uri = 'https://%s.storage.googleapis.com/%s' % (GCS_ARTIFACTS, relpath) |
| resp, res = http.request(uri, method='PUT', headers=hdr, body=mm) |
| if fsize > 0: |
| mm.close() |
| if resp.status != 200: |
| logging.error('HTTP request failed with code %d : %s', resp.status, res) |
| return -1 |
| return fsize |
| |
| |
| def upload_one_file_with_retries(fpath): |
| for retry in [0.5, 1.5, 3]: |
| res = upload_one_file(fpath) |
| if res >= 0: |
| return res |
| logging.warn('Upload of %s failed, retrying in %s seconds', fpath, retry) |
| time.sleep(retry) |
| |
| |
| def list_files(path): |
| for root, _, files in os.walk(path): |
| for fname in files: |
| fpath = os.path.join(root, fname) |
| if os.path.isfile(fpath): |
| yield fpath |
| |
| |
| def scan_and_upload_perf_folder(job_id, dirpath): |
| perf_folder = os.path.join(dirpath, 'perf') |
| if not os.path.isdir(perf_folder): |
| return |
| uploader = os.path.join(CUR_DIR, 'perf_metrics_uploader.py') |
| for path in list_files(perf_folder): |
| subprocess.call([uploader, '--job-id', job_id, path]) |
| |
| |
| def main(): |
| init_logging() |
| signal.alarm(WATCHDOG_SEC) |
| mimetypes.add_type('application/wasm', '.wasm') |
| |
| parser = argparse.ArgumentParser() |
| parser.add_argument('--rm', action='store_true', help='Removes the directory') |
| parser.add_argument( |
| '--job-id', |
| type=str, |
| required=True, |
| help='The Perfetto CI job ID to tie this upload to') |
| args = parser.parse_args() |
| job_id = args.job_id |
| dirpath = os.path.join(os.getenv('ARTIFACTS_DIR', default=os.curdir), job_id) |
| if not os.path.isdir(dirpath): |
| logging.error('Directory not found: %s', dirpath) |
| return 1 |
| |
| total_size = 0 |
| uploads = 0 |
| failures = 0 |
| files = list_files(dirpath) |
| pool = ThreadPool(processes=10) |
| for upl_size in pool.imap_unordered(upload_one_file_with_retries, files): |
| uploads += 1 if upl_size >= 0 else 0 |
| failures += 1 if upl_size < 0 else 0 |
| total_size += max(upl_size, 0) |
| |
| logging.info('Uploaded artifacts for %s: %d files, %s failures, %d KB', |
| job_id, uploads, failures, total_size / 1e3) |
| |
| scan_and_upload_perf_folder(job_id, dirpath) |
| |
| if args.rm: |
| subprocess.call(['sudo', 'rm', '-rf', dirpath]) |
| |
| return 0 |
| |
| |
| if __name__ == '__main__': |
| sys.exit(main()) |