| # Copyright 2020 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine import recipe_api |
| |
| INFRA_BUCKET_NAME = 'flutter_infra_release' |
| |
| |
| class BucketUtilApi(recipe_api.RecipeApi): |
| """Utility functions to upload files to cloud buckets. |
| |
| Properties: |
| upload_packages: (bool) Whether to upload the packages to the bucket. |
| force_upload: |
| (bool) Whether a file should be re-uploaded if it exists in the bucket. |
| """ |
| |
| def should_upload_packages(self): |
| return self.m.properties.get('upload_packages', False) |
| |
| def upload_folder( |
| self, |
| dir_label, |
| parent_directory, |
| folder_name, |
| zip_name, |
| platform=None, |
| bucket_name=INFRA_BUCKET_NAME |
| ): |
| """Uploads a folder to the cloud bucket |
| |
| Args: |
| dir_label: (str) A label to append to the step that creates a temporary directory. |
| parent_directory: (str) Parent directory of folder_name. |
| folder_name: (str) Folder to upload. |
| zip_name: (str) Name of the zip file in the cloud bucket. |
| platform: (str) Directory name to add the zip file to. |
| bucket_name: (str) The bucket name. Defaults to flutter_infra. |
| """ |
| self.upload_folder_and_files( |
| dir_label, |
| parent_directory, |
| folder_name, |
| zip_name, |
| platform=platform, |
| bucket_name=bucket_name |
| ) |
| |
| def upload_folder_and_files( |
| self, |
| dir_label, |
| parent_directory, |
| folder_name, |
| zip_name, |
| platform=None, |
| file_paths=None, |
| bucket_name=INFRA_BUCKET_NAME |
| ): |
| """Uploads a folder and or files to the cloud bucket |
| |
| Args: |
| dir_label: (str) A label to append to the step that creates a temporary directory. |
| parent_directory: (str) Parent directory of folder_name and/or file_paths. |
| folder_name: (str) Folder to upload. |
| zip_name: (str) Name of the zip file in the cloud bucket. |
| platform: (str) directory name to add the zip file to. |
| file_paths: (list) A list of string with the filenames to upload. |
| bucket_name: (str) The bucket name. Defaults to flutter_infra. |
| """ |
| with self.m.os_utils.make_temp_directory(dir_label) as temp_dir: |
| remote_name = '%s/%s' % (platform, zip_name) if platform else zip_name |
| local_zip = temp_dir.join(zip_name) |
| remote_zip = self.get_cloud_path(remote_name) |
| if isinstance(parent_directory, str): |
| parent_directory = self.m.path['cache'].join('builder', parent_directory) |
| pkg = self.m.zip.make_package(parent_directory, local_zip) |
| pkg.add_directory(parent_directory.join(folder_name)) |
| |
| if file_paths is not None: |
| self.add_files(pkg, file_paths) |
| |
| pkg.zip('Zip %s' % folder_name) |
| if self.should_upload_packages(): |
| self.safe_upload(local_zip, remote_zip, bucket_name=bucket_name) |
| |
| def safe_upload( |
| self, |
| local_path, |
| remote_path, |
| bucket_name=INFRA_BUCKET_NAME, |
| args=[], |
| skip_on_duplicate=False, |
| dry_run=False, |
| add_mock=True, |
| ): |
| """Upload a file if it doesn't already exist, fail job otherwise. |
| |
| The check can be overridden with the `force_upload` property. |
| |
| This method will check whether the file exists first and fail if it does |
| not. If should_upload_packages returns false, it still will assert the file |
| exists. |
| |
| Args: |
| local_path: (str) The local path to upload. |
| remote_path: (str) The remove path in the cloud bucket. |
| bucket_name: (str) The bucket name. Defaults to flutter_infra. |
| args: (list) Arguments to pass to gsutil.upload step. |
| skip_on_duplicate: (bool) |
| Whether to avoid uploading to an already existing path in the bucket. |
| dry_run: (bool) Whether to skip the upload even if should_upload_packages |
| returns true. |
| add_mock: (bool) Whether to call path.add_mock_file for the local_path. |
| |
| Returns: |
| The result of the gsutil.upload step. Useful for verifying the exit code. |
| """ |
| if add_mock: |
| self.m.path.mock_add_file(local_path) |
| |
| if not self.m.path.exists(local_path): |
| with self.m.step.nest('%s not found' % local_path) as presentation: |
| parent_dir = self.m.path.abs_to_path(self.m.path.dirname(local_path)) |
| self.m.file.listdir('Files in parent directory of safe_upload request', parent_dir) |
| presentation.status = self.m.step.FAILURE |
| raise AssertionError('File not found %s' % local_path) |
| |
| if not self.should_upload_packages() or dry_run: |
| return |
| |
| experimental = self.m.runtime.is_experimental |
| force_upload = self.m.properties.get('force_upload', False) |
| # Experimental builds go to a different bucket, duplicates allowed |
| if not experimental and not force_upload: |
| cloud_path = 'gs://%s/%s' % (bucket_name, remote_path) |
| result = self.m.step( |
| 'Ensure %s does not already exist on cloud storage' % remote_path, |
| [ |
| 'python', |
| self.m.depot_tools.gsutil_py_path, |
| 'stat', |
| cloud_path, |
| ], |
| ok_ret='all', |
| ) |
| # A return value of 0 means the file ALREADY exists on cloud storage |
| if result.exc_result.retcode == 0: |
| if skip_on_duplicate: |
| # This file already exists, but we shouldn't fail the build |
| return |
| raise AssertionError('%s already exists on cloud storage' % cloud_path) |
| |
| return self.m.gsutil.upload( |
| local_path, |
| bucket_name, |
| remote_path, |
| args=args, |
| name='upload "%s"' % remote_path |
| ) |
| |
| def add_files(self, pkg, relative_paths=[]): |
| """Adds files to the package. |
| |
| Args: |
| pkg: (package) The package that contains the files. |
| relative_paths: |
| (list) The relative_paths parameter is a list of strings and pairs of |
| strings. If the path is a string, then it will be used as the source |
| filename, and its basename will be used as the destination filename |
| in the archive. If the path is a pair, then the first element will be |
| used as the source filename, and the second element will be used as the |
| destination filename in the archive. |
| """ |
| for path in relative_paths: |
| pkg.add_file(pkg.root.join(path), archive_name=self.m.path.basename(path)) |
| |
| def add_directories(self, pkg, relative_paths=[]): |
| """Adds directories to the package. |
| |
| Args: |
| pkg: (package) The package that contains the files. |
| relative_paths: |
| (list) The relative_paths parameter is a list of directories to add to |
| the archive. |
| """ |
| for path in relative_paths: |
| pkg.add_directory(pkg.root.join(path)) |
| |
| def get_cloud_path(self, path): |
| """Gets the path in the cloud bucket. |
| |
| Args: |
| path: (str) Path to append after the commit hash. |
| |
| Returns: |
| The path formed by `flutter/<commit-hash|uuid>/<path>`. |
| """ |
| git_hash = self.m.buildbucket.gitiles_commit.id |
| # gitiles_commit is only populated on post-submits. |
| # UUID is used in LED and try jobs. |
| uuid = self.m.uuid.random() |
| invocation_id = git_hash if git_hash else uuid |
| |
| if self.m.runtime.is_experimental: |
| return 'flutter/experimental/%s/%s' % (invocation_id, path) |
| return 'flutter/%s/%s' % (invocation_id, path) |