Allow try uploads. This will allow artifact creation and upload validation on presubmit. These artifacts will be uploaded to a try bucket. Bug: https://github.com/flutter/flutter/issues/124063 Change-Id: If40dfaea93f4d788a917a2bc90c6a32421e0e9d2 Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/41323 Reviewed-by: Ricardo Amador <ricardoamador@google.com> Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/archives/__init__.py b/recipe_modules/archives/__init__.py index 35fefba..7bd1c65 100644 --- a/recipe_modules/archives/__init__.py +++ b/recipe_modules/archives/__init__.py
@@ -4,6 +4,7 @@ DEPS = [ 'depot_tools/gsutil', + 'flutter/monorepo', 'flutter/repo_util', 'recipe_engine/buildbucket', 'recipe_engine/file',
diff --git a/recipe_modules/archives/api.py b/recipe_modules/archives/api.py index 31c38ed..263c986 100644 --- a/recipe_modules/archives/api.py +++ b/recipe_modules/archives/api.py
@@ -42,7 +42,8 @@ MONOREPO: 'flutter_archives_v2/monorepo/flutter_infra_release', 'prod': 'flutter_infra_release', 'staging': 'flutter_archives_v2/flutter_infra_release', - 'try': 'flutter_archives_v2/flutter_infra_release' + 'try': 'flutter_archives_v2/flutter_infra_release', + 'try.shadow': 'flutter_archives_v2/flutter_infra_release' } # Bucket + initial prefix for artifact destination. @@ -51,7 +52,8 @@ MONOREPO: 'flutter_archives_v2/monorepo', 'prod': '', 'staging': 'flutter_archives_v2', - 'try': 'flutter_archives_v2' + 'try': 'flutter_archives_v2', + 'try.shadow': 'flutter_archives_v2' } # Subpath for realms. A realm is used to separate file destinations @@ -167,9 +169,8 @@ # artifacts to the same bucket but different path when the build configurations use an experimental # realm. Defaults to experimental. artifact_realm = REALM_TO_PATH.get(archive_config.get('realm', ''), 'experimental') - # Do not archive if the build is a try build or has no input commit - if (self.m.buildbucket.build.input.gerrit_changes or - not self.m.buildbucket.gitiles_commit.project): + # Do not archive if this is a monorepo try build. + if self.m.monorepo.is_monorepo_try_build: return results # Calculate prefix and commit. @@ -228,9 +229,8 @@ """ results = [] - # Do not archive if the build is a try build or has no input commit - if (self.m.buildbucket.build.input.gerrit_changes or - not self.m.buildbucket.gitiles_commit.project): + # Do not archive if this is a monorepo try build. + if self.m.monorepo.is_monorepo_try_build: return results # Calculate prefix and commit.
diff --git a/recipe_modules/archives/examples/engine_v2_gcs_paths.expected/basic.json b/recipe_modules/archives/examples/engine_v2_gcs_paths.expected/prod.json similarity index 100% rename from recipe_modules/archives/examples/engine_v2_gcs_paths.expected/basic.json rename to recipe_modules/archives/examples/engine_v2_gcs_paths.expected/prod.json
diff --git a/recipe_modules/archives/examples/engine_v2_gcs_paths.expected/try.json b/recipe_modules/archives/examples/engine_v2_gcs_paths.expected/try.json new file mode 100644 index 0000000..5b0bf0d --- /dev/null +++ b/recipe_modules/archives/examples/engine_v2_gcs_paths.expected/try.json
@@ -0,0 +1,58 @@ +[ + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "listdir", + "[START_DIR]/out/android_profile/zip_archives/download.flutter.io", + "--recursive" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Expand directory", + "~followup_annotations": [ + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@", + "@@@STEP_LOG_LINE@listdir@[START_DIR]/out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@", + "@@@STEP_LOG_END@listdir@@@" + ] + }, + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/engine_v2_gcs_paths.py b/recipe_modules/archives/examples/engine_v2_gcs_paths.py index 524478d..712a653 100644 --- a/recipe_modules/archives/examples/engine_v2_gcs_paths.py +++ b/recipe_modules/archives/examples/engine_v2_gcs_paths.py
@@ -12,6 +12,7 @@ 'recipe_engine/assertions', 'recipe_engine/buildbucket', 'recipe_engine/path', + 'recipe_engine/properties', 'recipe_engine/raw_io', ] @@ -33,7 +34,7 @@ ] } results = api.archives.engine_v2_gcs_paths(checkout, config) - expected_results = [ + expected_prod_results = [ ArchivePaths( local=str(api.path['start_dir'].join('out/android_profile/zip_archives/artifact1.zip')), remote='gs://flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/artifact1.zip' @@ -63,18 +64,69 @@ remote='gs://flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/sky_engine.zip' ) ] - api.assertions.assertListEqual(expected_results, results) + expected_try_results = [ + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/artifact1.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/artifact1.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/android-arm-profile/artifacts.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/artifacts.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/android-arm-profile/linux-x64.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/linux-x64.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/android-arm-profile/symbols.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-arm-profile/symbols.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar')), + remote='gs://flutter_archives_v2/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom')), + remote='gs://flutter_archives_v2/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('out/android_profile/zip_archives/sky_engine.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/sky_engine.zip' + ) + ] + config = api.properties.get('config') + expected_results = { + 'prod': expected_prod_results, + 'try': expected_try_results + } + api.assertions.assertListEqual(expected_results[config], results) def GenTests(api): yield api.test( - 'basic', + 'prod', api.buildbucket.ci_build( project='flutter', bucket='prod', git_repo='https://flutter.googlesource.com/mirrors/engine', git_ref='refs/heads/main' ), + api.properties(config='prod'), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + ) + yield api.test( + 'try', + api.buildbucket.ci_build( + project='flutter', + bucket='try', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main' + ), + api.properties(config='try'), api.step_data( 'git rev-parse', stdout=api.raw_io
diff --git a/recipe_modules/archives/examples/global_generator_paths.expected/try.json b/recipe_modules/archives/examples/global_generator_paths.expected/try.json new file mode 100644 index 0000000..74fb50b --- /dev/null +++ b/recipe_modules/archives/examples/global_generator_paths.expected/try.json
@@ -0,0 +1,27 @@ +[ + { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[START_DIR]/src/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/archives/examples/global_generator_paths.py b/recipe_modules/archives/examples/global_generator_paths.py index 037214c..cb51250 100644 --- a/recipe_modules/archives/examples/global_generator_paths.py +++ b/recipe_modules/archives/examples/global_generator_paths.py
@@ -61,10 +61,25 @@ remote='gs://flutter_archives_v2/monorepo/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/ios-release/Flutter.dSYM.zip' ) ] + expected_try_results = [ + ArchivePaths( + local=str(api.path['start_dir'].join('src/out/debug/artifacts.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/ios/artifacts.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('src/out/release-nobitcode/Flutter.dSYM.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/ios-release-nobitcode/Flutter.dSYM.zip' + ), + ArchivePaths( + local=str(api.path['start_dir'].join('src/out/release/Flutter.dSYM.zip')), + remote='gs://flutter_archives_v2/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/ios-release/Flutter.dSYM.zip' + ) + ] env_to_results = { 'production': expected_results, 'monorepo': expected_monorepo_results, - 'monorepo_try': [] + 'monorepo_try': [], + 'try': expected_try_results } config = api.properties.get('config') results = api.archives.global_generator_paths(checkout, archives) @@ -97,3 +112,18 @@ api.properties(config='monorepo_try'), api.monorepo.try_build(), ) + yield api.test( + 'try', + api.properties(config='try'), + api.buildbucket.ci_build( + project='flutter', + bucket='try', + git_repo='https://flutter.googlesource.com/mirrors/engine', + git_ref='refs/heads/main' + ), + api.step_data( + 'git rev-parse', + stdout=api.raw_io + .output_text('12345abcde12345abcde12345abcde12345abcde\n') + ) + )
diff --git a/recipes/engine_v2/engine_v2.expected/config_from_file.json b/recipes/engine_v2/engine_v2.expected/config_from_file.json index 33c48df..3117f07 100644 --- a/recipes/engine_v2/engine_v2.expected/config_from_file.json +++ b/recipes/engine_v2/engine_v2.expected/config_from_file.json
@@ -842,6 +842,112 @@ ] }, { + "cmd": [ + "git", + "rev-parse", + "HEAD" + ], + "cwd": "[CACHE]/builder/src/flutter", + "infra_step": true, + "luci_context": { + "realm": { + "name": "proj:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "git rev-parse" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "ensure-directory", + "--mode", + "0777", + "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/bucket" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "proj:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Ensure flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/bucket" + }, + { + "cmd": [ + "vpython3", + "-u", + "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", + "--json-output", + "/path/to/tmp/json", + "copy", + "/a/b/c.txt", + "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/bucket" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "proj:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Copy /a/b/c.txt to tmp location" + }, + { + "cmd": [ + "python3", + "-u", + "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", + "--", + "RECIPE_REPO[depot_tools]/gsutil.py", + "----", + "cp", + "-r", + "[CLEANUP]/tmp_tmp_1/*", + "gs://flutter_archives_v2/" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "proj:try" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "gsutil Upload /a/b/c.txt to gs://flutter_archives_v2/flutter_infra_release/flutter/experimental/12345abcde12345abcde12345abcde12345abcde/bucket/c.txt", + "~followup_annotations": [ + "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@" + ] + }, + { "cmd": [], "name": "launch tests" },