Add artifact destination for shadow builds Bug: https://github.com/flutter/flutter/issues/149341 Change-Id: I2dedc120f272ab832e7220683180359ccad72e1e Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/57840 Reviewed-by: Christopher Fujino <fujino@google.com> Commit-Queue: Victoria Ashworth <vashworth@google.com>
diff --git a/recipe_modules/archives/api.py b/recipe_modules/archives/api.py index 5327157..4d2b9df 100644 --- a/recipe_modules/archives/api.py +++ b/recipe_modules/archives/api.py
@@ -56,6 +56,10 @@ 'try_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), 'try.shadow_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), 'try.shadow_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release'), + 'staging.shadow_PRODUCTION': + ('flutter_archives_v2', 'flutter_infra_release'), + 'staging.shadow_EXPERIMENTAL': + ('flutter_archives_v2', 'flutter_infra_release'), 'prod.shadow_PRODUCTION': ('flutter_archives_v2', 'flutter_infra_release'), 'prod.shadow_EXPERIMENTAL': ('flutter_archives_v2', 'flutter_infra_release') @@ -76,7 +80,11 @@ 'try_EXPERIMENTAL': 'flutter_archives_v2', 'try_PRODUCTION': 'flutter_archives_v2', 'try.shadow_EXPERIMENTAL': 'flutter_archives_v2', - 'try.shadow_PRODUCTION': 'flutter_archives_v2' + 'try.shadow_PRODUCTION': 'flutter_archives_v2', + 'staging.shadow_EXPERIMENTAL': 'flutter_archives_v2', + 'staging.shadow_PRODUCTION': 'flutter_archives_v2', + 'prod.shadow_EXPERIMENTAL': 'flutter_archives_v2', + 'prod.shadow_PRODUCTION': 'flutter_archives_v2' } # Subpath for realms. A realm is used to separate file destinations
diff --git a/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_experimental_realm.json b/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_experimental_realm.json new file mode 100644 index 0000000..d41e994 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_experimental_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_production_realm.json b/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_production_realm.json new file mode 100644 index 0000000..d41e994 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/prod_pool_shadow_bucket_production_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_experimental_realm.json b/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_experimental_realm.json new file mode 100644 index 0000000..26669a2 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_experimental_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_production_realm.json b/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_production_realm.json new file mode 100644 index 0000000..26669a2 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/staging_pool_shadow_bucket_production_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:staging.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_experimental_realm.json b/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_experimental_realm.json new file mode 100644 index 0000000..d41e994 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_experimental_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_production_realm.json b/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_production_realm.json new file mode 100644 index 0000000..d41e994 --- /dev/null +++ b/recipe_modules/archives/examples/full.expected/try_pool_shadow_bucket_production_realm.json
@@ -0,0 +1,176 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [], + "name": "get buildbucket id", + "~followup_annotations": [ + "@@@STEP_TEXT@98765@@@" + ] + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0o777", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure 98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip", + "[CLEANUP]/tmp_tmp_1/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload [START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip to gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip", + "[START_DIR]/out/android_profile/zip_archives/android-arm-profile/artifacts.zip" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try.shadow" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:98765", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil download gs://flutter_archives_v2/98765/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/full.py b/recipe_modules/archives/examples/full.py index 82e472a..64bbdac 100644 --- a/recipe_modules/archives/examples/full.py +++ b/recipe_modules/archives/examples/full.py
@@ -98,6 +98,54 @@ ) ) + # Try shadow LUCI pool with "production" realm in build configuration file. + try_pool_shadow_bucket_production_realm = try_pool_production_realm + yield api.test( + 'try_pool_shadow_bucket_production_realm', + api.properties( + config=archive_config, + expected_destinations=try_pool_shadow_bucket_production_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='try.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + + # Try shadow LUCI pool with "experimental" realm in build configuration file. + try_pool_shadow_bucket_experimental_realm = try_pool_production_realm + try_pool_shadow_bucket_experimental_realm_config = copy.deepcopy( + archive_config + ) + try_pool_shadow_bucket_experimental_realm_config['realm'] = 'experimental' + yield api.test( + 'try_pool_shadow_bucket_experimental_realm', + api.properties( + config=try_pool_shadow_bucket_experimental_realm_config, + expected_destinations=try_pool_shadow_bucket_experimental_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='try.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + # Prod LUCI pool with "production" realm in build configuration file. prod_pool_production_realm = [ 'gs://flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip', @@ -150,6 +198,54 @@ ) ) + # Prod shadow LUCI pool with "production" realm in build configuration file. + prod_pool_shadow_bucket_production_realm = try_pool_production_realm + yield api.test( + 'prod_pool_shadow_bucket_production_realm', + api.properties( + config=archive_config, + expected_destinations=prod_pool_shadow_bucket_production_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='try.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + + # Prod shadow LUCI pool with "experimental" realm in build configuration file. + prod_pool_shadow_bucket_experimental_realm = try_pool_production_realm + prod_pool_shadow_bucket_experimental_realm_config = copy.deepcopy( + archive_config + ) + prod_pool_shadow_bucket_experimental_realm_config['realm'] = 'experimental' + yield api.test( + 'prod_pool_shadow_bucket_experimental_realm', + api.properties( + config=prod_pool_shadow_bucket_experimental_realm_config, + expected_destinations=prod_pool_shadow_bucket_experimental_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='try.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + # Flutter LUCI pool with "production" realm in build configuration file. flutter_pool_production_realm = prod_pool_production_realm yield api.test( @@ -240,6 +336,54 @@ ) ) + # Staging shadow LUCI pool with "production" realm in build configuration file. + staging_pool_shadow_bucket_production_realm = try_pool_production_realm + yield api.test( + 'staging_pool_shadow_bucket_production_realm', + api.properties( + config=archive_config, + expected_destinations=staging_pool_shadow_bucket_production_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='staging.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + + # Staging shadow LUCI pool with "experimental" realm in build configuration file. + staging_pool_shadow_bucket_experimental_realm = try_pool_production_realm + staging_pool_shadow_bucket_experimental_realm_config = copy.deepcopy( + archive_config + ) + staging_pool_shadow_bucket_experimental_realm_config['realm'] = 'experimental' + yield api.test( + 'staging_pool_shadow_bucket_experimental_realm', + api.properties( + config=staging_pool_shadow_bucket_experimental_realm_config, + expected_destinations=staging_pool_shadow_bucket_experimental_realm + ), + api.buildbucket.ci_build( + project='flutter', + bucket='staging.shadow', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main', + build_id=98765 + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + # Monorepo ci with "production" realm in build configuration file. monorepo_production_realm = [ 'gs://flutter_archives_v2/monorepo/123/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip',