Changing call to flutter_bcid,upload_provenance to single files. The upload_provenance function does not handle glob patterns, so adding documentation to it to clarify requirements, and changed calls to it to only send in single files and not patterns/directories. Bug:b/232552448 Change-Id: Icd429c2003588a12c6aa8f5e690b8a67f35ba4ab Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/35160 Commit-Queue: Yusuf Mohsinally <mohsinally@google.com> Reviewed-by: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/flutter_bcid/api.py b/recipe_modules/flutter_bcid/api.py index c60fc11..d15891b 100644 --- a/recipe_modules/flutter_bcid/api.py +++ b/recipe_modules/flutter_bcid/api.py
@@ -18,6 +18,16 @@ self.m.bcid_reporter.report_stage(stage) def upload_provenance(self, local_artifact_path, remote_artifact_path): + """Generate provenance for given artifact. + + This function acts on one specific local file and one specific + remote file location. It does not accept glob patterns or + directories. + + parmeters: + local_artifact_path: (str) path and filename of a specific file. + remote_artifact_path: (str) path and filename of a specific file. + """ if self._is_official_build(): sha256 = self.m.file.file_hash(local_artifact_path) self.m.bcid_reporter.report_gcs(
diff --git a/recipes/engine_v2/builder.expected/basic_gcs.json b/recipes/engine_v2/builder.expected/basic_gcs.json index 5c35624..d6e751a 100644 --- a/recipes/engine_v2/builder.expected/basic_gcs.json +++ b/recipes/engine_v2/builder.expected/basic_gcs.json
@@ -672,6 +672,43 @@ }, { "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "glob", + "[CLEANUP]/host_debug_unopt_tmp_1", + "*" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", + "ENGINE_PATH": "[CACHE]/builder", + "GIT_BRANCH": "", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux", + "REVISION": "" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "infra_step": true, + "name": "Generate provenance", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@", + "@@@STEP_LOG_END@glob@@@" + ] + }, + { + "cmd": [ "python3", "-u", "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", @@ -709,6 +746,43 @@ ] }, { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "glob", + "[CLEANUP]/host_debug_unopt_tmp_1", + "download.flutter.io/*" + ], + "cwd": "[CACHE]/builder/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", + "ENGINE_PATH": "[CACHE]/builder", + "GIT_BRANCH": "", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux", + "REVISION": "" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "infra_step": true, + "name": "Generate provenance (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@", + "@@@STEP_LOG_END@glob@@@" + ] + }, + { "cmd": [], "name": "Set output properties", "~followup_annotations": [
diff --git a/recipes/engine_v2/builder.expected/monorepo_gcs.json b/recipes/engine_v2/builder.expected/monorepo_gcs.json index d890feb..4bc7e63 100644 --- a/recipes/engine_v2/builder.expected/monorepo_gcs.json +++ b/recipes/engine_v2/builder.expected/monorepo_gcs.json
@@ -1007,6 +1007,55 @@ }, { "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "glob", + "[CLEANUP]/host_debug_unopt_tmp_1", + "*" + ], + "cwd": "[CACHE]/builder/engine/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", + "ENGINE_PATH": "[CACHE]/builder", + "GIT_BRANCH": "", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux", + "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "dart:ci.sandbox" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Generate provenance", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@", + "@@@STEP_LOG_END@glob@@@" + ] + }, + { + "cmd": [ "python3", "-u", "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", @@ -1056,6 +1105,55 @@ ] }, { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "glob", + "[CLEANUP]/host_debug_unopt_tmp_1", + "download.flutter.io/*" + ], + "cwd": "[CACHE]/builder/engine/src/flutter", + "env": { + "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", + "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", + "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", + "ENGINE_PATH": "[CACHE]/builder", + "GIT_BRANCH": "", + "LUCI_BRANCH": "", + "LUCI_CI": "True", + "LUCI_PR": "", + "LUCI_WORKDIR": "[START_DIR]", + "OS": "linux", + "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" + ] + }, + "infra_step": true, + "luci_context": { + "realm": { + "name": "dart:ci.sandbox" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Generate provenance (2)", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@glob@[CLEANUP]/host_debug_unopt_tmp_1/file.txt@@@", + "@@@STEP_LOG_END@glob@@@" + ] + }, + { "cmd": [], "name": "Set output properties", "~followup_annotations": [
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py index 3133505..6fa16c3 100644 --- a/recipes/engine_v2/builder.py +++ b/recipes/engine_v2/builder.py
@@ -176,9 +176,9 @@ args=['-r'], name=archive_config['name'], ) - api.flutter_bcid.upload_provenance( - '%s/*' % archive_dir, 'gs://%s/%s' % (bucket, artifact_path) - ) + for local_filepath in api.file.glob_paths('Generate provenance', archive_dir, '*', test_data=('file.txt',)): + remote_filepath = 'gs://%s/%s/%s' % (bucket, artifact_path, api.path.basename(str(local_filepath))) + api.flutter_bcid.upload_provenance(local_filepath, remote_filepath) # Jar and pom files are uploaded to download.flutter.io while all the other artifacts # are uploaded to flutter_infra_release. If we override paths artifacts need to be organized # as gs://<overriden_bucket>/flutter_infra_release for non android artifacts and @@ -192,10 +192,11 @@ args=['-r'], name=archive_config['name'], ) - api.flutter_bcid.upload_provenance( - '%s/download.flutter.io/' % archive_dir, - 'gs://%s/%s' % (bucket, android_artifact_path) - ) + for local_filepath in api.file.glob_paths('Generate provenance', archive_dir, + 'download.flutter.io/*', test_data=('file.txt',)): + remote_filepath = 'gs://%s/%s/%s' % (bucket, artifact_path, + api.path.basename(str(local_filepath))) + api.flutter_bcid.upload_provenance(local_filepath, remote_filepath) return 'gs://%s/%s/%s' % ( bucket, artifact_path, api.path.basename(archive_dir)) # Archive using CAS by default return api.cas_util.upload(archive_dir, step_name='Archive %s' % archive_config['name'])