| # Copyright 2022 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import attr |
| import re |
| |
| from recipe_engine import recipe_api |
| |
| |
| @attr.s |
| class ArchivePaths(object): |
| """Paths for an archive config.""" |
| local = attr.ib(type=str) |
| remote = attr.ib(type=str) |
| |
| |
| ANDROID_ARTIFACTS_BUCKET = 'download.flutter.io' |
| |
| # Monorepo constant. |
| MONOREPO = 'monorepo' |
| MONOREPO_TRY_BUCKET = 'monorepo-try-bucket' |
| |
| # Used for mock paths |
| DIRECTORY = 'DIRECTORY' |
| |
| # Relative paths used to mock paths for testing. |
| MOCK_JAR_PATH = ( |
| 'io/flutter/x86_debug/' |
| '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/' |
| 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar' |
| ) |
| MOCK_POM_PATH = ( |
| 'io/flutter/x86_debug/' |
| '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/' |
| 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom' |
| ) |
| |
| # Bucket + initial prefix for artifact destination. |
| LUCI_TO_GCS_PREFIX = { |
| 'flutter_EXPERIMENTAL': ('flutter_infra_release', ''), |
| 'flutter_PRODUCTION': ('flutter_infra_release', ''), |
| '%s_EXPERIMENTAL' % MONOREPO: |
| ('flutter_archives_v2/monorepo', 'flutter_infra_release'), |
| '%s_PRODUCTION' % MONOREPO: |
| ('flutter_archives_v2/monorepo', 'flutter_infra_release'), |
| '%s_EXPERIMENTAL' % MONOREPO_TRY_BUCKET: |
| ('flutter_archives_v2/monorepo_try', 'flutter_infra_release'), |
| '%s_PRODUCTION' % MONOREPO_TRY_BUCKET: |
| ('flutter_archives_v2/monorepo_try', 'flutter_infra_release'), |
| 'prod_PRODUCTION': ('flutter_infra_release', ''), |
| 'prod_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'staging_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'staging_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'try_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'try_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'try.shadow_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'try.shadow_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'prod.shadow_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), |
| 'prod.shadow_EXPERIMENTAL': |
| ('flutter_archives_v2', 'flutter_infra_release') |
| } |
| |
| # Bucket + initial prefix for artifact destination. |
| LUCI_TO_ANDROID_GCS_PREFIX = { |
| 'flutter_EXPERIMENTAL': '', |
| 'flutter_PRODUCTION': '', |
| '%s_EXPERIMENTAL' % MONOREPO: 'flutter_archives_v2/monorepo', |
| '%s_PRODUCTION' % MONOREPO: 'flutter_archives_v2/monorepo', |
| '%s_EXPERIMENTAL' % MONOREPO_TRY_BUCKET: 'flutter_archives_v2/monorepo_try', |
| '%s_PRODUCTION' % MONOREPO_TRY_BUCKET: 'flutter_archives_v2/monorepo_try', |
| 'prod_EXPERIMENTAL': 'flutter_archives_v2', |
| 'prod_PRODUCTION': '', |
| 'staging_EXPERIMENTAL': 'flutter_archives_v2', |
| 'staging_PRODUCTION': 'flutter_archives_v2', |
| 'try_EXPERIMENTAL': 'flutter_archives_v2', |
| 'try_PRODUCTION': 'flutter_archives_v2', |
| 'try.shadow_EXPERIMENTAL': 'flutter_archives_v2', |
| 'try.shadow_PRODUCTION': 'flutter_archives_v2' |
| } |
| |
| # Subpath for realms. A realm is used to separate file destinations |
| # within the same configuration. E.g. production environment with |
| # an experimental realm and production environment with a production realm. |
| REALM_TO_PATH = {'production': '', 'experimental': 'experimental'} |
| |
| |
| class ArchivesApi(recipe_api.RecipeApi): |
| """Api to handle archives from engine_v2 recipes.""" |
| |
| def _full_path_list(self, checkout, archive_config): |
| """Calculates the local paths using an archive_config. |
| |
| Args: |
| checkout: (Path) the checkout path of the engine repository. |
| archive_config: (dict) a dictionary with the archive files generated by |
| a given build. |
| |
| Returns: |
| A list of strings with the expected local files as described |
| by the archive configuration. |
| """ |
| results = [] |
| self.m.path.mock_add_directory( |
| self.m.path['start_dir'] |
| .join('out/android_profile/zip_archives/download.flutter.io') |
| ) |
| for include_path in archive_config.get('include_paths', []): |
| full_include_path = self.m.path.abspath(checkout.join(include_path)) |
| if self.m.path.isdir(full_include_path): |
| test_data = [] |
| paths = self.m.file.listdir( |
| 'Expand directory', |
| checkout.join(include_path), |
| recursive=True, |
| test_data=(MOCK_JAR_PATH, MOCK_POM_PATH) |
| ) |
| paths = [self.m.path.abspath(p) for p in paths] |
| results.extend(paths) |
| else: |
| results.append(full_include_path) |
| return results |
| |
| def _split_dst_parts(self, dst): |
| """Splits gsutil uri into a bucket and path sections. |
| |
| Args: |
| dst: (str) a gcs path like gs://bucket/a/b/c. |
| |
| Returns: |
| A tuple with the bucket as the first item and the path to the |
| object as the second parameter. |
| """ |
| |
| matches = re.match('gs://([\w.]+)/(.+)', dst) |
| return (matches.group(1), matches.group(2)) |
| |
| def upload_artifact(self, src, dst, metadata=None): |
| """Uploads a local object to a gcs destination. |
| |
| This method also ensures the directoy structure is recreated in the |
| destination. |
| |
| Args: |
| src: (str) a string with the object local path. |
| dst: (str) a string with the destination path in gcs. |
| metadata: (dict) a dictionary with the header as key and its content as value. |
| """ |
| bucket, path = self._split_dst_parts(dst) |
| dir_part = self.m.path.dirname(path) |
| archive_dir = self.m.path.mkdtemp() |
| local_dst_tree = archive_dir.join(*dir_part.split('/')) |
| self.m.file.ensure_directory('Ensure %s' % dir_part, local_dst_tree) |
| self.m.file.copy('Copy %s to tmp location' % src, src, local_dst_tree) |
| self.m.gsutil.upload( |
| name='Upload %s to %s' % (src, dst), |
| source='%s/*' % archive_dir, |
| bucket=bucket, |
| dest='', |
| args=['-r'], |
| metadata=metadata, |
| ) |
| |
| def download(self, src, dst): |
| """Downloads a file from GCS. |
| |
| Args: |
| src: A string with gcs uri to download. |
| dst: A string with the local destination for the file. |
| """ |
| bucket, path = self._split_dst_parts(src) |
| self.m.gsutil.download(bucket, path, dst, name="download %s" % src) |
| |
| def engine_v2_gcs_paths(self, checkout, archive_config): |
| """Calculates engine v2 GCS paths from an archive config. |
| |
| Args: |
| checkout: (Path) the engine repository checkout folder. |
| archive_config: (dict) the archive configuration for a recipes v2 build. |
| |
| Returns: |
| A list of ArchivePaths with expected local and remote locations for the |
| generated artifacts. |
| """ |
| results = [] |
| build_id = '' |
| realm = archive_config.get('realm', 'experimental').upper() |
| # Weather to include build_id as part of the namespace or not. |
| include_build_id = True |
| file_list = self._full_path_list(checkout, archive_config) |
| if self.m.monorepo.is_monorepo_try_build: |
| commit = self.m.monorepo.build_identifier |
| bucket = MONOREPO_TRY_BUCKET |
| elif self.m.monorepo.is_monorepo_ci_build: |
| commit = self.m.repo_util.get_commit(checkout.join('../../monorepo')) |
| bucket = MONOREPO |
| else: |
| commit = self.m.repo_util.get_commit(checkout.join('flutter')) |
| bucket = self.m.buildbucket.build.builder.bucket |
| if self.m.flutter_bcid.is_official_build(): |
| include_build_id = False |
| elif self.m.flutter_bcid.is_prod_build() and realm == 'PRODUCTION': |
| include_build_id = False |
| build_id = self.m.monorepo.build_identifier if include_build_id else '' |
| bucket_plus_realm = '_'.join(filter(None, (bucket, realm))) |
| for include_path in file_list: |
| is_android_artifact = ANDROID_ARTIFACTS_BUCKET in include_path |
| dir_part = self.m.path.dirname(include_path) |
| full_base_path = self.m.path.abspath( |
| checkout.join(archive_config.get('base_path', '')) |
| ) |
| rel_path = self.m.path.relpath(dir_part, full_base_path) |
| rel_path = '' if rel_path == '.' else rel_path |
| base_name = self.m.path.basename(include_path) |
| |
| if is_android_artifact: |
| # We are not using a slash in the first parameter becase artifact_prefix |
| # already includes the slash. |
| artifact_path = '%s/%s' % (rel_path, base_name) |
| # Replace ANDROID_ARTIFACTS_BUCKET to include the realm. |
| old_location = '/'.join([ANDROID_ARTIFACTS_BUCKET, 'io', 'flutter']) |
| new_location = '/'.join( |
| filter(bool, [ANDROID_ARTIFACTS_BUCKET, 'io', 'flutter']) |
| ) |
| artifact_path = artifact_path.replace(old_location, new_location) |
| bucket_and_prefix = LUCI_TO_ANDROID_GCS_PREFIX.get(bucket_plus_realm) |
| artifact_path = '/'.join( |
| filter(bool, [bucket_and_prefix, build_id, artifact_path]) |
| ) |
| else: |
| gcs_bucket, bucket_postfix = LUCI_TO_GCS_PREFIX.get(bucket_plus_realm) |
| artifact_path = '/'.join( |
| filter( |
| bool, [ |
| gcs_bucket, build_id, bucket_postfix, 'flutter', commit, |
| rel_path, base_name |
| ] |
| ) |
| ) |
| |
| results.append(ArchivePaths(include_path, 'gs://%s' % artifact_path)) |
| return results |
| |
| def global_generator_paths(self, checkout, archives): |
| """Calculates the global generator paths for an archive config. |
| |
| Args: |
| checkout: (Path) the engine repository checkout folder. |
| archives: (list) list of dictionaries source and destination path |
| of files relative to the gclient checkout. |
| |
| Returns: |
| A list of ArchivePaths with expected local and remote locations for the |
| generated artifacts. |
| """ |
| results = [] |
| build_id = '' |
| |
| # Calculate prefix and commit. |
| if self.m.monorepo.is_monorepo_try_build: |
| commit = self.m.monorepo.build_identifier |
| bucket = MONOREPO_TRY_BUCKET |
| elif self.m.monorepo.is_monorepo_ci_build: |
| commit = self.m.repo_util.get_commit(checkout.join('../../monorepo')) |
| bucket = MONOREPO |
| else: |
| commit = self.m.repo_util.get_commit(checkout.join('flutter')) |
| bucket = self.m.buildbucket.build.builder.bucket |
| |
| for archive in archives: |
| realm = archive.get('realm', 'experimental').upper() |
| bucket_plus_realm = '_'.join(filter(None, (bucket, realm))) |
| include_build_id = True |
| if self.m.flutter_bcid.is_official_build(): |
| include_build_id = False |
| if self.m.flutter_bcid.is_prod_build() and realm == 'PRODUCTION': |
| include_build_id = False |
| build_id = self.m.monorepo.build_identifier if include_build_id else '' |
| gcs_bucket, bucket_postfix = LUCI_TO_GCS_PREFIX.get(bucket_plus_realm) |
| source = checkout.join(archive.get('source')) |
| artifact_path = '/'.join( |
| filter( |
| bool, [ |
| gcs_bucket, build_id, bucket_postfix, 'flutter', commit, |
| archive.get('destination') |
| ] |
| ) |
| ) |
| dst = 'gs://%s' % artifact_path |
| results.append(ArchivePaths(self.m.path.abspath(source), dst)) |
| return results |
