Add support to archive artifacts to GCS.
Archives artifacts in GCS or CAS depending depending on the type
parameter of the archive configuration.
Bug: https://github.com/flutter/flutter/issues/81855
Change-Id: I0e2f322214efa8d1a5cccfd58a81ebeaa467487c
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/31100
Reviewed-by: Xilai Zhang <xilaizhang@google.com>
Reviewed-by: Keyong Han <keyonghan@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipes/engine_v2/builder.expected/basic_gcs.json b/recipes/engine_v2/builder.expected/basic_gcs.json
new file mode 100644
index 0000000..bd110b8
--- /dev/null
+++ b/recipes/engine_v2/builder.expected/basic_gcs.json
@@ -0,0 +1,918 @@
+[
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "rmtree",
+ "[CACHE]/builder/src/out"
+ ],
+ "infra_step": true,
+ "name": "Clobber build output"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/builder"
+ ],
+ "infra_step": true,
+ "name": "Ensure checkout cache"
+ },
+ {
+ "cmd": [],
+ "name": "Checkout source code"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "rmtree",
+ "[CACHE]/builder"
+ ],
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "Checkout source code.Clobber cache",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "rmtree",
+ "[CACHE]/git"
+ ],
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "Checkout source code.Clobber git cache",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/builder"
+ ],
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "Checkout source code.Ensure checkout cache",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
+ "--spec-path",
+ "cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src/flutter', 'url': 'https://flutter.googlesource.com/mirrors/engine'}]",
+ "--revision_mapping_file",
+ "{\"got_engine_revision\": \"src/flutter\"}",
+ "--git-cache-dir",
+ "[CACHE]/git",
+ "--cleanup-dir",
+ "[CLEANUP]/bot_update",
+ "--output_json",
+ "/path/to/tmp/json",
+ "--revision",
+ "src/flutter@HEAD"
+ ],
+ "cwd": "[CACHE]/builder",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "DEPOT_TOOLS_COLLECT_METRICS": "0",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "GIT_HTTP_LOW_SPEED_LIMIT": "102400",
+ "GIT_HTTP_LOW_SPEED_TIME": "1800",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0",
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]",
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "Checkout source code.bot_update",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@Some step text@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"did_run\": true, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"fixed_revisions\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": \"HEAD\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"manifest\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"repository\": \"https://fake.org/src/flutter.git\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"patch_failure\": false, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"patch_root\": \"src/flutter\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision\": \"9221bca00ddbd888260084def81f09543281b952\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"got_engine_revision_cp\": \"refs/heads/main@{#84512}\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"got_revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"root\": \"src/flutter\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"source_manifest\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"directories\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"src/flutter\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"git_checkout\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"repo_url\": \"https://fake.org/src/flutter.git\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"revision\": \"9221bca00ddbd888260084def81f09543281b952\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"version\": 0@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"step_text\": \"Some step text\"@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@SET_BUILD_PROPERTY@got_engine_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@",
+ "@@@SET_BUILD_PROPERTY@got_engine_revision_cp@\"refs/heads/main@{#84512}\"@@@",
+ "@@@SET_BUILD_PROPERTY@got_revision@\"9221bca00ddbd888260084def81f09543281b952\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_REPO[depot_tools]/gclient.py",
+ "runhooks"
+ ],
+ "cwd": "[CACHE]/builder",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]",
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "name": "Checkout source code.gclient runhooks",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "ensure goma"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/goma/client",
+ "-ensure-file",
+ "fuchsia/third_party/goma/client/${platform} integration",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "ensure goma.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-integration-----\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"fuchsia/third_party/goma/client/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[CACHE]/builder/src/flutter/tools/gn",
+ "--goma",
+ "--ios"
+ ],
+ "env": {
+ "GOMA_DIR": "[CACHE]/goma/client"
+ },
+ "name": "gn --ios"
+ },
+ {
+ "cmd": [],
+ "name": "setup goma"
+ },
+ {
+ "cmd": [],
+ "name": "setup goma.ensure cpython3",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[fuchsia::python3]/resources/tool_manifest.json",
+ "/path/to/tmp/json"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "setup goma.ensure cpython3.read manifest",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@{@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@ \"path\": \"path/to/cpython3\",@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@ \"version\": \"version:pinned-version\"@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@}@@@",
+ "@@@STEP_LOG_END@tool_manifest.json@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "setup goma.ensure cpython3.install path/to/cpython3",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "setup goma.ensure cpython3.install path/to/cpython3.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version",
+ "-ensure-file",
+ "path/to/cpython3 version:pinned-version",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "setup goma.ensure cpython3.install path/to/cpython3.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:pinned-v\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"path/to/cpython3\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version/bin/python3",
+ "[CACHE]/goma/client/goma_ctl.py",
+ "restart"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "setup goma.start goma",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "RECIPE_REPO[depot_tools]/ninja",
+ "-j",
+ "100",
+ "-C",
+ "[CACHE]/builder/src/out/ios_debug"
+ ],
+ "env": {
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "name": "build ios_debug"
+ },
+ {
+ "cmd": [],
+ "name": "teardown goma"
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version/bin/python3",
+ "[CACHE]/goma/client/goma_ctl.py",
+ "jsonstatus",
+ "[CACHE]/goma/client/jsonstatus"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "name": "teardown goma.goma jsonstatus",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"notice\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"infra_status\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"num_user_error\": 0, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"ping_status_code\": 200@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version/bin/python3",
+ "[CACHE]/goma/client/goma_ctl.py",
+ "stat"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "name": "teardown goma.goma stats",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/path/to/cpython3/version%3Apinned-version/bin/python3",
+ "[CACHE]/goma/client/goma_ctl.py",
+ "stop"
+ ],
+ "env": {
+ "GLOG_log_dir": "[CLEANUP]",
+ "GOMA_CACHE_DIR": "[CACHE]/goma",
+ "GOMA_DEPS_CACHE_FILE": "goma_deps_cache",
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_DUMP_STATS_FILE": "[CACHE]/goma/client/goma_stats.json",
+ "GOMA_LOCAL_OUTPUT_CACHE_DIR": "[CACHE]/goma/localoutputcache",
+ "GOMA_MAX_SUM_OUTPUT_SIZE_IN_MB": "256",
+ "GOMA_SERVER_HOST": "rbe-prod1.endpoints.fuchsia-infra-goma-prod.cloud.goog",
+ "GOMA_STORE_LOCAL_RUN_OUTPUT": "True",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "name": "teardown goma.stop goma",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/compiler_proxy.WARNING",
+ "/path/to/tmp/"
+ ],
+ "env": {
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "teardown goma.read goma_client warning log",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@compiler_proxy.WARNING@test log@@@",
+ "@@@STEP_LOG_END@compiler_proxy.WARNING@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CACHE]/goma/client/goma_stats.json",
+ "/path/to/tmp/"
+ ],
+ "env": {
+ "GOMA_DIR": "[CACHE]/goma/client",
+ "GOMA_TMP_DIR": "[CLEANUP]/goma",
+ "GOMA_USE_LOCAL": "False"
+ },
+ "infra_step": true,
+ "name": "teardown goma.read goma_stats.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "dart",
+ "pub",
+ "get"
+ ],
+ "cwd": "[CACHE]/builder/src",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin",
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "name": "dart pub get"
+ },
+ {
+ "cmd": [
+ "[CACHE]/builder/src/script1.sh",
+ "[CACHE]/builder/src/dev/felt.dart",
+ "--argument1"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "name": "generator1"
+ },
+ {
+ "cmd": [
+ "[CACHE]/builder/src/myscript.sh",
+ "param1",
+ "param2"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "name": "mytest"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "rmtree",
+ "[CACHE]/builder/src/out/host_debug_unopt/obj"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Remove out/host_debug_unopt/obj"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "rmtree",
+ "[CACHE]/builder/src/out/host_debug_unopt/stripped.exe"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Remove out/host_debug_unopt/stripped.exe"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/src/out/host_debug_unopt",
+ "[CLEANUP]/host_debug_unopt_tmp_1/host_debug_unopt"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Copy out/host_debug_unopt/"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/host_debug_unopt_tmp_1/out/host_debug_unopt"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Ensuring [CLEANUP]/host_debug_unopt_tmp_1/out/host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CACHE]/builder/src/out/host_debug_unopt/file.zip",
+ "[CLEANUP]/host_debug_unopt_tmp_1/out/host_debug_unopt/file.zip"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "Copy out/host_debug_unopt/file.zip"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[CACHE]/builder/src",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "git rev-parse"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "----",
+ "cp",
+ "-r",
+ "[CLEANUP]/host_debug_unopt_tmp_1/*",
+ "gs://flutter_archives_v2/"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "LUCI_WORKDIR": "[START_DIR]",
+ "OS": "linux",
+ "REVISION": ""
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "gsutil host_debug_unopt",
+ "~followup_annotations": [
+ "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "Set output properties",
+ "~followup_annotations": [
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"gs://flutter_archives_v2/host_debug_unopt_tmp_1\"}@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py
index 28840d0..cced2eb 100644
--- a/recipes/engine_v2/builder.py
+++ b/recipes/engine_v2/builder.py
@@ -25,6 +25,7 @@
}
}
"""
+import copy
from contextlib import contextmanager
@@ -36,6 +37,7 @@
PYTHON_VERSION_COMPATIBILITY = 'PY3'
DEPS = [
+ 'depot_tools/gsutil',
'flutter/build_util',
'flutter/flutter_deps',
'flutter/os_utils',
@@ -128,6 +130,14 @@
base_name = api.path.basename(full_include_path)
api.file.ensure_directory('Ensuring %s' % archive_dir.join(rel_path), archive_dir.join(rel_path))
api.file.copy('Copy %s' % include_path, full_include_path, archive_dir.join(rel_path, base_name))
+ if archive_config.get('type') == 'gcs':
+ commit = api.repo_util.get_commit(checkout)
+ api.gsutil.upload(
+ source='%s/*' % archive_dir, bucket='flutter_archives_v2',
+ dest=commit, args=['-r'],
+ name=archive_config['name'],)
+ return 'gs://flutter_archives_v2/%s' % api.path.basename(archive_dir)
+ # Archive using CAS by default
return api.cas_util.upload(archive_dir, step_name='Archive %s' % archive_config['name'])
@@ -164,6 +174,7 @@
"archives": [
{
"name": "host_debug_unopt",
+ "type": "cas",
"include_paths": ['out/host_debug_unopt/', 'out/host_debug_unopt/file.zip'],
"exclude_paths": ['out/host_debug_unopt/obj', 'out/host_debug_unopt/stripped.exe']
}
@@ -201,3 +212,7 @@
yield api.test(
'basic_custom_vars', api.properties(build=build_custom, goma_jobs="100")
)
+ # gcs archives
+ build_gcs = copy.deepcopy(build)
+ build_gcs['archives'][0]['type'] = 'gcs'
+ yield api.test('basic_gcs', api.properties(build=build_gcs, goma_jobs="100"))
