Module to calculate archive locations and upload to GCS.
This module has methods to flatten out the archives generated by builds
to ensure a single file is passed to the provenance generation at the
time.
Bug: https://github.com/flutter/flutter/issues/113193
Change-Id: Ic3c6319a6d2663e3c487ccae69aaa9d04acb1cf0
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/35162
Reviewed-by: Yusuf Mohsinally <mohsinally@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/archives/__init__.py b/recipe_modules/archives/__init__.py
new file mode 100644
index 0000000..35fefba
--- /dev/null
+++ b/recipe_modules/archives/__init__.py
@@ -0,0 +1,11 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+ 'depot_tools/gsutil',
+ 'flutter/repo_util',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/file',
+ 'recipe_engine/path',
+]
diff --git a/recipe_modules/archives/api.py b/recipe_modules/archives/api.py
new file mode 100644
index 0000000..f3590eb
--- /dev/null
+++ b/recipe_modules/archives/api.py
@@ -0,0 +1,150 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import attr
+import re
+
+from recipe_engine import recipe_api
+
+
+@attr.s
+class ArchivePaths(object):
+ """Paths for an archive config."""
+ local = attr.ib(type=str)
+ remote = attr.ib(type=str)
+
+
+DEFAULT_BUCKET = 'flutter_archives_v2'
+ANDROID_ARTIFACTS_BUCKET = 'download.flutter.io'
+# Used for mock paths
+DIRECTORY = 'DIRECTORY'
+
+
+class ArchivesApi(recipe_api.RecipeApi):
+ """Api to handle archives from engine_v2 recipes."""
+
+ def _full_path_list(self, checkout, archive_config):
+ """Calculates the local paths using an archive_config.
+
+ Args:
+ checkout: (Path) the checkout path of the engine repository.
+ archive_config: (dict) a dictionary with the archive files generated by
+ a given build.
+
+ Returns:
+ A list of strings with the expected local files as described
+ by the archive configuration.
+ """
+ results = []
+ self.m.path.mock_add_paths(
+ self.m.path['start_dir'].join(
+ 'out/android_profile/zip_archives/download.flutter.io'),
+ DIRECTORY
+ )
+ for include_path in archive_config.get('include_paths', []):
+ full_include_path = self.m.path.abspath(checkout.join(include_path))
+ if self.m.path.isdir(full_include_path):
+ test_data = [
+ (
+ 'io/flutter/x86_debug/'
+ '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/'
+ 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar'),
+ (
+ 'io/flutter/x86_debug/'
+ '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/'
+ 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom'),
+
+ ]
+ paths = self.m.file.glob_paths(
+ 'Expand directory', checkout.join(include_path),
+ '**', test_data=test_data
+ )
+ paths = [self.m.path.abspath(p) for p in paths]
+ results.extend(paths)
+ else:
+ results.append(full_include_path)
+ return results
+
+ def _split_dst_parts(self, dst):
+ """Splits gsutil uri into a bucket and path sections.
+
+ Args:
+ dst: (str) a gcs path like gs://bucket/a/b/c.
+
+ Returns:
+ A tuple with the bucket as the first item and the path to the
+ object as the second parameter.
+ """
+ matches = re.match('gs://(\w+)/(.+)', dst)
+ return (matches.group(1), matches.group(2))
+
+ def upload_artifact(self, src, dst):
+ """Uploads a local object to a gcs destination.
+
+ This method also ensures the directoy structure is recreated in the
+ destination.
+
+ Args:
+ src: (str) a string with the object local path.
+ dst: (str) a string with the destination path in gcs.
+ """
+ bucket, path = self._split_dst_parts(dst)
+ dir_part = self.m.path.dirname(path)
+ archive_dir = self.m.path.mkdtemp()
+ self.m.file.ensure_directory('Ensure %s' % dir_part, archive_dir.join(dir_part))
+ self.m.file.copy('Copy %s' % dst, src, archive_dir.join(dir_part))
+ self.m.gsutil.upload(
+ source='%s/*' % archive_dir,
+ bucket=bucket,
+ dest='',
+ args=['-r'],
+ name=path,
+ )
+
+ def engine_v2_gcs_paths(self, checkout, archive_config, bucket=DEFAULT_BUCKET):
+ """Calculates engine v2 GCS paths from an archive config.
+
+ Args:
+ checkout: (Path) the engine repository checkout folder.
+ archive_config: (dict) the archive configuration for a recipes v2 build.
+ bucket: (str) the bucket used to calculate the object destination.
+
+ Returns:
+ A list of ArchivePaths with expected local and remote locations for the
+ generated artifacts.
+ """
+ results = []
+ file_list = self._full_path_list(checkout, archive_config)
+ for include_path in file_list:
+ is_android_artifact = ANDROID_ARTIFACTS_BUCKET in include_path
+ dir_part = self.m.path.dirname(include_path)
+ full_base_path = self.m.path.abspath(checkout.join(archive_config.get('base_path','')))
+ rel_path = self.m.path.relpath(dir_part, full_base_path)
+ rel_path = '' if rel_path == '.' else rel_path
+ base_name = self.m.path.basename(include_path)
+ is_monorepo = self.m.buildbucket.gitiles_commit.project == 'monorepo'
+
+ if is_monorepo:
+ commit = self.m.repo_util.get_commit(checkout.join('../../monorepo'))
+ artifact_prefix = 'monorepo/'
+ else:
+ commit = self.m.repo_util.get_commit(checkout.join('flutter'))
+ artifact_prefix = ''
+
+ if is_android_artifact:
+ # We are not using a slash in the first parameter becase artifact_prefix
+ # already includes the slash.
+ artifact_path = '%s%s/%s' % (
+ artifact_prefix, rel_path, base_name)
+ else:
+ artifact_path = '%sflutter_infra_release/flutter%s/%s/%s' % (
+ artifact_prefix, commit, rel_path, base_name)
+
+ results.append(
+ ArchivePaths(
+ include_path,
+ 'gs://%s/%s' % (bucket, artifact_path)
+ )
+ )
+ return results
diff --git a/recipe_modules/archives/examples/full.expected/basic.json b/recipe_modules/archives/examples/full.expected/basic.json
new file mode 100644
index 0000000..715e413
--- /dev/null
+++ b/recipe_modules/archives/examples/full.expected/basic.json
@@ -0,0 +1,122 @@
+[
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/out/android_profile/zip_archives/download.flutter.io",
+ "**"
+ ],
+ "infra_step": true,
+ "name": "Expand directory",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "git rev-parse"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "git rev-parse (2)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "git rev-parse (3)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "git rev-parse (4)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "git rev-parse (5)"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ ],
+ "infra_step": true,
+ "name": "Ensure flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ ],
+ "infra_step": true,
+ "name": "Copy gs://flutter_archives_v2/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "----",
+ "cp",
+ "-r",
+ "[CLEANUP]/tmp_tmp_1/*",
+ "gs://flutter_archives_v2/"
+ ],
+ "infra_step": true,
+ "name": "gsutil flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip",
+ "~followup_annotations": [
+ "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/monorepo_gcs.json b/recipe_modules/archives/examples/full.expected/monorepo_gcs.json
new file mode 100644
index 0000000..ca03f20
--- /dev/null
+++ b/recipe_modules/archives/examples/full.expected/monorepo_gcs.json
@@ -0,0 +1,230 @@
+[
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[START_DIR]/out/android_profile/zip_archives/download.flutter.io",
+ "**"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Expand directory",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@",
+ "@@@STEP_LOG_LINE@glob@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/../../monorepo",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "git rev-parse"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/../../monorepo",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "git rev-parse (2)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/../../monorepo",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "git rev-parse (3)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/../../monorepo",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "git rev-parse (4)"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/../../monorepo",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "git rev-parse (5)"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/tmp_tmp_1/monorepo/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Ensure monorepo/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip",
+ "[CLEANUP]/tmp_tmp_1/monorepo/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Copy gs://flutter_archives_v2/monorepo/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "----",
+ "cp",
+ "-r",
+ "[CLEANUP]/tmp_tmp_1/*",
+ "gs://flutter_archives_v2/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gsutil monorepo/flutter_infra_release/flutter12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip",
+ "~followup_annotations": [
+ "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.py b/recipe_modules/archives/examples/full.py
new file mode 100644
index 0000000..a712ced
--- /dev/null
+++ b/recipe_modules/archives/examples/full.py
@@ -0,0 +1,54 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine.post_process import DoesNotRun, Filter, StatusFailure
+
+DEPS = [
+ 'flutter/archives',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/path',
+ 'recipe_engine/raw_io',
+]
+
+
+def RunSteps(api):
+ checkout = api.path['start_dir']
+ config = {
+ "name": "android_profile",
+ "type": "gcs",
+ "base_path": "out/android_profile/zip_archives/",
+ "include_paths": [
+ "out/android_profile/zip_archives/android-arm-profile/artifacts.zip",
+ "out/android_profile/zip_archives/android-arm-profile/linux-x64.zip",
+ "out/android_profile/zip_archives/android-arm-profile/symbols.zip",
+ "out/android_profile/zip_archives/download.flutter.io"
+ ]
+ }
+ results = api.archives.engine_v2_gcs_paths(checkout, config)
+ api.archives.upload_artifact(results[0].local, results[0].remote)
+
+
+def GenTests(api):
+ yield api.test(
+ 'basic',
+ api.step_data(
+ 'git rev-parse',
+ stdout=api.raw_io
+ .output_text('12345abcde12345abcde12345abcde12345abcde\n')
+ )
+ )
+ yield api.test(
+ 'monorepo_gcs',
+ api.buildbucket.ci_build(
+ project='dart',
+ bucket='ci.sandbox',
+ git_repo='https://dart.googlesource.com/monorepo',
+ git_ref='refs/heads/main'
+ ),
+ api.step_data(
+ 'git rev-parse',
+ stdout=api.raw_io
+ .output_text('12345abcde12345abcde12345abcde12345abcde\n')
+ )
+ )
