Changing call to flutter_bcid,upload_provenance to single files.
The upload_provenance function does not handle glob patterns, so
adding documentation to it to clarify requirements, and changed
calls to it to only send in single files and not patterns/directories.
Bug:b/232552448
Change-Id: Icd429c2003588a12c6aa8f5e690b8a67f35ba4ab
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/35160
Commit-Queue: Yusuf Mohsinally <mohsinally@google.com>
Reviewed-by: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/flutter_bcid/api.py b/recipe_modules/flutter_bcid/api.py
index c60fc11..d15891b 100644
--- a/recipe_modules/flutter_bcid/api.py
+++ b/recipe_modules/flutter_bcid/api.py
@@ -18,6 +18,16 @@
self.m.bcid_reporter.report_stage(stage)
def upload_provenance(self, local_artifact_path, remote_artifact_path):
+ """Generate provenance for given artifact.
+
+ This function acts on one specific local file and one specific
+ remote file location. It does not accept glob patterns or
+ directories.
+
+ parmeters:
+ local_artifact_path: (str) path and filename of a specific file.
+ remote_artifact_path: (str) path and filename of a specific file.
+ """
if self._is_official_build():
sha256 = self.m.file.file_hash(local_artifact_path)
self.m.bcid_reporter.report_gcs(
diff --git a/recipes/engine_v2/builder.expected/basic_gcs.json b/recipes/engine_v2/builder.expected/basic_gcs.json
index 5c35624..d6e751a 100644
--- a/recipes/engine_v2/builder.expected/basic_gcs.json
+++ b/recipes/engine_v2/builder.expected/basic_gcs.json
@@ -672,6 +672,43 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[CLEANUP]/host_debug_unopt_tmp_1",
+ "*"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Generate provenance",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
"python3",
"-u",
"RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
@@ -709,6 +746,43 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[CLEANUP]/host_debug_unopt_tmp_1",
+ "download.flutter.io/*"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Generate provenance (2)",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
diff --git a/recipes/engine_v2/builder.expected/monorepo_gcs.json b/recipes/engine_v2/builder.expected/monorepo_gcs.json
index d890feb..4bc7e63 100644
--- a/recipes/engine_v2/builder.expected/monorepo_gcs.json
+++ b/recipes/engine_v2/builder.expected/monorepo_gcs.json
@@ -1007,6 +1007,55 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[CLEANUP]/host_debug_unopt_tmp_1",
+ "*"
+ ],
+ "cwd": "[CACHE]/builder/engine/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Generate provenance",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [
"python3",
"-u",
"RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
@@ -1056,6 +1105,55 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[CLEANUP]/host_debug_unopt_tmp_1",
+ "download.flutter.io/*"
+ ],
+ "cwd": "[CACHE]/builder/engine/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Generate provenance (2)",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py
index 3133505..6fa16c3 100644
--- a/recipes/engine_v2/builder.py
+++ b/recipes/engine_v2/builder.py
@@ -176,9 +176,9 @@
args=['-r'],
name=archive_config['name'],
)
- api.flutter_bcid.upload_provenance(
- '%s/*' % archive_dir, 'gs://%s/%s' % (bucket, artifact_path)
- )
+ for local_filepath in api.file.glob_paths('Generate provenance', archive_dir, '*', test_data=('file.txt',)):
+ remote_filepath = 'gs://%s/%s/%s' % (bucket, artifact_path, api.path.basename(str(local_filepath)))
+ api.flutter_bcid.upload_provenance(local_filepath, remote_filepath)
# Jar and pom files are uploaded to download.flutter.io while all the other artifacts
# are uploaded to flutter_infra_release. If we override paths artifacts need to be organized
# as gs://<overriden_bucket>/flutter_infra_release for non android artifacts and
@@ -192,10 +192,11 @@
args=['-r'],
name=archive_config['name'],
)
- api.flutter_bcid.upload_provenance(
- '%s/download.flutter.io/' % archive_dir,
- 'gs://%s/%s' % (bucket, android_artifact_path)
- )
+ for local_filepath in api.file.glob_paths('Generate provenance', archive_dir,
+ 'download.flutter.io/*', test_data=('file.txt',)):
+ remote_filepath = 'gs://%s/%s/%s' % (bucket, artifact_path,
+ api.path.basename(str(local_filepath)))
+ api.flutter_bcid.upload_provenance(local_filepath, remote_filepath)
return 'gs://%s/%s/%s' % ( bucket, artifact_path, api.path.basename(archive_dir))
# Archive using CAS by default
return api.cas_util.upload(archive_dir, step_name='Archive %s' % archive_config['name'])
