Implement global build archives.
Mac and fuchsia require the generation of artifacts using files from
multiple sub-builds. This change adds functionality to archive full
subbuilds to CAS and download them at the end of the process.
Bug: https://github.com/flutter/flutter/issues/81855
Change-Id: I54f3e67c63a931b48799faefc180aecad7145a17
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/32280
Reviewed-by: Keyong Han <keyonghan@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
Reviewed-by: Ricardo Amador <ricardoamador@google.com>
diff --git a/recipe_modules/shard_util_v2/__init__.py b/recipe_modules/shard_util_v2/__init__.py
index 8237219..7cf75f7 100644
--- a/recipe_modules/shard_util_v2/__init__.py
+++ b/recipe_modules/shard_util_v2/__init__.py
@@ -5,7 +5,9 @@
PYTHON_VERSION_COMPATIBILITY = 'PY3'
DEPS = [
+ 'fuchsia/cas_util',
'recipe_engine/buildbucket',
+ 'recipe_engine/cas',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/led',
diff --git a/recipe_modules/shard_util_v2/api.py b/recipe_modules/shard_util_v2/api.py
index 325311e..d563dc1 100644
--- a/recipe_modules/shard_util_v2/api.py
+++ b/recipe_modules/shard_util_v2/api.py
@@ -380,3 +380,41 @@
build_name=build_id_to_name[int(build_id)]
)
return builds
+
+ def download_full_builds(self, build_results, out_build_paths):
+ """Downloads intermediate builds from CAS.
+
+ Args:
+ build_results (dict(int, SubbuildResult)): A dictionary with the subbuild
+ result and the build id as key.
+
+ Mac and fuchsia use artifacts from different sub-builds to generate the final artifacts.
+ Calls to this API will happen most likely after all the subbuilds have been completed and
+ only if global generators will be executed.
+ """
+ for build_id in build_results:
+ build_props = build_results[build_id].build_proto.output.properties
+ if 'cas_output_hash' in build_props:
+ cas_out_dict = build_props['cas_output_hash']
+ build_name = build_results[build_id].build_name
+ if 'full_build' in cas_out_dict:
+ self.m.cas.download(
+ 'Download for build %s and cas key %s' % (build_id, build_name),
+ cas_out_dict['full_build'],
+ out_build_paths.join(build_name)
+ )
+
+ def archive_full_build(self, build_dir, target):
+ """Archives a full build in cas.
+
+ Args:
+ build_dir: The path to the build output folder.
+ target(str): The name of the build we are archiving.
+
+ Returns:
+ A string with the hash of the cas archive.
+ """
+ cas_dir = self.m.path.mkdtemp('out-cas-directory')
+ cas_engine = cas_dir.join(target)
+ self.m.file.copytree('Copy host_debug_unopt', build_dir, cas_engine)
+ return self.m.cas_util.upload(cas_dir, step_name='Archive full build for %s' % target)
diff --git a/recipe_modules/shard_util_v2/examples/full.expected/presubmit_led.json b/recipe_modules/shard_util_v2/examples/full.expected/presubmit_led.json
index e5231ee..f5215e1 100644
--- a/recipe_modules/shard_util_v2/examples/full.expected/presubmit_led.json
+++ b/recipe_modules/shard_util_v2/examples/full.expected/presubmit_led.json
@@ -1025,6 +1025,7 @@
"@@@STEP_LOG_LINE@build.proto.json@ \"output\": {@@@",
"@@@STEP_LOG_LINE@build.proto.json@ \"properties\": {@@@",
"@@@STEP_LOG_LINE@build.proto.json@ \"cas_output_hash\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"full_build\": \"123\",@@@",
"@@@STEP_LOG_LINE@build.proto.json@ \"ios_debug\": \"bcd\",@@@",
"@@@STEP_LOG_LINE@build.proto.json@ \"web_tests\": \"abc\"@@@",
"@@@STEP_LOG_LINE@build.proto.json@ }@@@",
@@ -1116,6 +1117,206 @@
]
},
{
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[START_DIR]/out/host_debug",
+ "[CLEANUP]/out-cas-directory_tmp_1/host_debug"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.Copy host_debug_unopt",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect builds.install infra/tools/luci/cas",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.Archive full build for host_debug",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "download",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-digest",
+ "123",
+ "-dir",
+ "[CLEANUP]/out/builder-subbuild2"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "collect builds.Download for build fake-task-id-2 and cas key builder-subbuild2",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
"cmd": [],
"name": "launch builds (2)"
},
diff --git a/recipe_modules/shard_util_v2/examples/full.py b/recipe_modules/shard_util_v2/examples/full.py
index 57be634..195f323 100644
--- a/recipe_modules/shard_util_v2/examples/full.py
+++ b/recipe_modules/shard_util_v2/examples/full.py
@@ -12,6 +12,7 @@
'flutter/shard_util_v2',
'fuchsia/buildbucket_util',
'recipe_engine/buildbucket',
+ 'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/step',
@@ -28,6 +29,8 @@
for build in builds.values():
if build.build_proto.status != common_pb2.SUCCESS:
raise api.step.StepFailure("build %s failed" % build.build_id)
+ api.shard_util_v2.archive_full_build(api.path['start_dir'].join('out', 'host_debug'), 'host_debug')
+ api.shard_util_v2.download_full_builds(builds, api.path['cleanup'].join('out'))
with api.step.nest("launch builds") as presentation:
reqs = api.shard_util_v2.schedule_tests(test_configs, builds, presentation)
@@ -38,7 +41,7 @@
builder="ios_debug",
input_props={'task_name': 'mytask'},
output_props={
- "cas_output_hash": {"web_tests": "abc", "ios_debug": "bcd"}
+ "cas_output_hash": {"web_tests": "abc", "ios_debug": "bcd", "full_build": "123"}
},
status="SUCCESS",
)
diff --git a/recipes/engine_v2/builder.expected/basic.json b/recipes/engine_v2/builder.expected/basic.json
index d5487c4..0c27dc2 100644
--- a/recipes/engine_v2/builder.expected/basic.json
+++ b/recipes/engine_v2/builder.expected/basic.json
@@ -35,80 +35,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src/flutter', 'url': 'https://flutter.googlesource.com/mirrors/engine'}]",
@@ -595,6 +521,109 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -908,133 +937,6 @@
},
{
"cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
- "/path/to/tmp/"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read infra revision",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
- "@@@STEP_LOG_END@infra.sha1@@@"
- ]
- },
- {
- "cmd": [],
- "name": "install infra/tools/luci/cas"
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure package directory",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "cipd",
- "ensure",
- "-root",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
- "-ensure-file",
- "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
- "-max-threads",
- "0",
- "-json-output",
- "/path/to/tmp/json"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure_installed",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ ]@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
- "cmd": [
"[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
"archive",
"-cas-instance",
@@ -1075,7 +977,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.expected/basic_custom_vars.json b/recipes/engine_v2/builder.expected/basic_custom_vars.json
index cc17762..51a5d5c 100644
--- a/recipes/engine_v2/builder.expected/basic_custom_vars.json
+++ b/recipes/engine_v2/builder.expected/basic_custom_vars.json
@@ -35,80 +35,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src/flutter', 'url': 'https://flutter.googlesource.com/mirrors/engine'}]",
@@ -595,6 +521,109 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -881,133 +910,6 @@
},
{
"cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
- "/path/to/tmp/"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read infra revision",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
- "@@@STEP_LOG_END@infra.sha1@@@"
- ]
- },
- {
- "cmd": [],
- "name": "install infra/tools/luci/cas"
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure package directory",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "cipd",
- "ensure",
- "-root",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
- "-ensure-file",
- "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
- "-max-threads",
- "0",
- "-json-output",
- "/path/to/tmp/json"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure_installed",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ ]@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
- "cmd": [
"[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
"archive",
"-cas-instance",
@@ -1048,7 +950,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.expected/basic_gcs.json b/recipes/engine_v2/builder.expected/basic_gcs.json
index 61dd68f..705dd5d 100644
--- a/recipes/engine_v2/builder.expected/basic_gcs.json
+++ b/recipes/engine_v2/builder.expected/basic_gcs.json
@@ -35,80 +35,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src/flutter', 'url': 'https://flutter.googlesource.com/mirrors/engine'}]",
@@ -595,6 +521,109 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -1014,7 +1043,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde//host_debug_unopt_tmp_1\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde//host_debug_unopt_tmp_1\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.expected/mac.json b/recipes/engine_v2/builder.expected/mac.json
index 7a84a2b..ec83778 100644
--- a/recipes/engine_v2/builder.expected/mac.json
+++ b/recipes/engine_v2/builder.expected/mac.json
@@ -35,80 +35,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'src/flutter', 'url': 'https://flutter.googlesource.com/mirrors/engine'}]",
@@ -669,6 +595,109 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -982,133 +1011,6 @@
},
{
"cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
- "/path/to/tmp/"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "darwin",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read infra revision",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
- "@@@STEP_LOG_END@infra.sha1@@@"
- ]
- },
- {
- "cmd": [],
- "name": "install infra/tools/luci/cas"
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "darwin",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure package directory",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "cipd",
- "ensure",
- "-root",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
- "-ensure-file",
- "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
- "-max-threads",
- "0",
- "-json-output",
- "/path/to/tmp/json"
- ],
- "cwd": "[CACHE]/builder/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "darwin",
- "REVISION": ""
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "install infra/tools/luci/cas.ensure_installed",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ ]@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
- "cmd": [
"[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
"archive",
"-cas-instance",
@@ -1158,7 +1060,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.expected/monorepo.json b/recipes/engine_v2/builder.expected/monorepo.json
index 48bdb16..6cb8aae 100644
--- a/recipes/engine_v2/builder.expected/monorepo.json
+++ b/recipes/engine_v2/builder.expected/monorepo.json
@@ -59,116 +59,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'monorepo', 'url': 'https://dart.googlesource.com/monorepo'}]",
@@ -1055,6 +945,169 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/engine/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -1488,169 +1541,6 @@
},
{
"cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
- "/path/to/tmp/"
- ],
- "cwd": "[CACHE]/builder/engine/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789"
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "read infra revision",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
- "@@@STEP_LOG_END@infra.sha1@@@"
- ]
- },
- {
- "cmd": [],
- "name": "install infra/tools/luci/cas"
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
- ],
- "cwd": "[CACHE]/builder/engine/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789"
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "install infra/tools/luci/cas.ensure package directory",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "cipd",
- "ensure",
- "-root",
- "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
- "-ensure-file",
- "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
- "-max-threads",
- "0",
- "-json-output",
- "/path/to/tmp/json"
- ],
- "cwd": "[CACHE]/builder/engine/src/flutter",
- "env": {
- "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "ENGINE_CHECKOUT_PATH": "[CACHE]/builder/src",
- "ENGINE_PATH": "[CACHE]/builder",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "LUCI_WORKDIR": "[START_DIR]",
- "OS": "linux",
- "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789"
- },
- "env_prefixes": {
- "PATH": [
- "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "install infra/tools/luci/cas.ensure_installed",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ ]@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
- "cmd": [
"[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
"archive",
"-cas-instance",
@@ -1703,7 +1593,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.expected/monorepo_gcs.json b/recipes/engine_v2/builder.expected/monorepo_gcs.json
index 3909317..4e00745 100644
--- a/recipes/engine_v2/builder.expected/monorepo_gcs.json
+++ b/recipes/engine_v2/builder.expected/monorepo_gcs.json
@@ -59,116 +59,6 @@
"cmd": [
"vpython3",
"-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmcontents",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Clobber cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "rmtree",
- "[CACHE]/git"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Clobber git cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[CACHE]/builder"
- ],
- "env_suffixes": {
- "DEPOT_TOOLS_UPDATE": [
- "0"
- ],
- "PATH": [
- "RECIPE_REPO[depot_tools]"
- ]
- },
- "infra_step": true,
- "luci_context": {
- "realm": {
- "name": "dart:ci.sandbox"
- },
- "resultdb": {
- "current_invocation": {
- "name": "invocations/build:8945511751514863184",
- "update_token": "token"
- },
- "hostname": "rdbhost"
- }
- },
- "name": "Checkout source code.Ensure checkout cache",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
"RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
"--spec-path",
"cache_dir = '[CACHE]/git'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': False, 'name': 'monorepo', 'url': 'https://dart.googlesource.com/monorepo'}]",
@@ -1055,6 +945,169 @@
},
{
"cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copytree",
+ "[CACHE]/builder/engine/src/out",
+ "[CLEANUP]/out-cas-directory_tmp_1"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Copy host_debug_unopt"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[recipe_engine::cas]/resources/infra.sha1",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "read infra revision",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@infra.sha1@git_revision:mock_infra_git_revision@@@",
+ "@@@STEP_LOG_END@infra.sha1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "install infra/tools/luci/cas"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "install infra/tools/luci/cas.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision",
+ "-ensure-file",
+ "infra/tools/luci/cas/${platform} git_revision:mock_infra_git_revision",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "install infra/tools/luci/cas.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-git_revision:moc\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/cas/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/infra/tools/luci/cas/git_revision%3Amock_infra_git_revision/cas",
+ "archive",
+ "-cas-instance",
+ "projects/example-cas-server/instances/default_instance",
+ "-dump-digest",
+ "/path/to/tmp/",
+ "-log-level",
+ "debug",
+ "-paths",
+ "[CLEANUP]/out-cas-directory_tmp_1:."
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "dart:ci.sandbox"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Archive full build for None",
+ "~followup_annotations": [
+ "@@@STEP_LINK@CAS UI@https://cas-viewer.appspot.com/projects/example-cas-server/instances/default_instance/blobs/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0/tree@@@"
+ ]
+ },
+ {
+ "cmd": [
"dart",
"pub",
"get"
@@ -1630,7 +1683,7 @@
"cmd": [],
"name": "Set output properties",
"~followup_annotations": [
- "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"host_debug_unopt\": \"gs://flutter_archives_v2/monorepo/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde//host_debug_unopt_tmp_1\"}@@@"
+ "@@@SET_BUILD_PROPERTY@cas_output_hash@{\"full_build\": \"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/0\", \"host_debug_unopt\": \"gs://flutter_archives_v2/monorepo/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde//host_debug_unopt_tmp_1\"}@@@"
]
},
{
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py
index e47b3cd..1bd5bfd 100644
--- a/recipes/engine_v2/builder.py
+++ b/recipes/engine_v2/builder.py
@@ -44,6 +44,7 @@
'flutter/osx_sdk',
'flutter/repo_util',
'flutter/retry',
+ 'flutter/shard_util_v2',
'flutter/test_utils',
'fuchsia/cas_util',
'recipe_engine/buildbucket',
@@ -73,6 +74,10 @@
ninja = build.get('ninja')
ninja_tool[ninja.get('tool', 'ninja')
](ninja.get('config'), checkout, ninja.get('targets'))
+ # Archive full build. This is inneficient but necessary for global generators.
+ full_build_hash = api.shard_util_v2.archive_full_build(
+ checkout.join('out', build.get('name')), build.get('name'))
+ outputs['full_build'] = full_build_hash
generator_tasks = build.get('generators', {}).get('tasks', [])
pub_dirs = build.get('generators', {}).get('pub_dirs', [])
archives = build.get('archives', [])
@@ -200,14 +205,15 @@
env['ENGINE_PATH'] = api.path['cache'].join('builder')
custom_vars = api.properties.get('gclient_custom_vars', {})
+ clobber = api.properties.get('clobber', False)
if api.buildbucket.gitiles_commit.project == 'monorepo':
api.repo_util.monorepo_checkout(
- cache_root, env, env_prefixes, custom_vars=custom_vars
+ cache_root, env, env_prefixes, clobber=clobber, custom_vars=custom_vars
)
checkout = api.path['cache'].join('builder', 'engine', 'src')
else:
api.repo_util.engine_checkout(
- cache_root, env, env_prefixes, custom_vars=custom_vars
+ cache_root, env, env_prefixes, clobber=clobber, custom_vars=custom_vars
)
outputs = {}
if api.platform.is_mac:
diff --git a/recipes/engine_v2/engine_v2.py b/recipes/engine_v2/engine_v2.py
index d9bbbb8..a4e5f05 100644
--- a/recipes/engine_v2/engine_v2.py
+++ b/recipes/engine_v2/engine_v2.py
@@ -61,11 +61,11 @@
with api.step.nest('launch builds') as presentation:
tasks = api.shard_util_v2.schedule_builds(builds, presentation)
with api.step.nest('collect builds') as presentation:
- results = api.shard_util_v2.collect(tasks, presentation)
+ build_results = api.shard_util_v2.collect(tasks, presentation)
api.display_util.display_builds(
step_name='display builds',
- builds=[b.build_proto for b in results.values()],
+ builds=[b.build_proto for b in build_results.values()],
raise_on_failure=True,
)
@@ -78,16 +78,20 @@
step_test_data=lambda: api.json.test_api.output({})
).json.output.get('tests', [])
with api.step.nest('launch tests') as presentation:
- tasks = api.shard_util_v2.schedule_tests(tests, results, presentation)
+ tasks = api.shard_util_v2.schedule_tests(tests, build_results, presentation)
with api.step.nest('collect tests') as presentation:
- results = api.shard_util_v2.collect(tasks, presentation)
+ test_results = api.shard_util_v2.collect(tasks, presentation)
api.display_util.display_builds(
step_name='display tests',
- builds=[b.build_proto for b in results.values()],
+ builds=[b.build_proto for b in test_results.values()],
raise_on_failure=True,
)
+ # Download sub-builds
+ out_builds_path = api.path['cleanup'].join('out')
+ api.shard_util_v2.download_full_builds(build_results, out_builds_path)
+
def GenTests(api):
try_subbuild1 = api.shard_util_v2.try_build_message(
