Revert "Upload engine artifact VSAs" This reverts commit 0a821eeda018550936f6da33ffd6e5f81cf1defd. Reason for revert: https://github.com/flutter/flutter/issues/133206, this is breaking engine artifact uploads Original change's description: > Upload engine artifact VSAs > > Pull the VSA information directly from the BCID call, write it to a file, and upload it next to the provenance and artifact in GCS > > Change-Id: Ibff96f767706a122f1f298d1515fbf6427e2d2d7 > Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/49380 > Reviewed-by: Jesse Seales <jseales@google.com> > Commit-Queue: Drew Roen <drewroen@google.com> # Not skipping CQ checks because original CL landed > 1 day ago. Change-Id: Ibdea8686ba16f714ad21085ac377c6c8ef04f0d2 Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/50101 Reviewed-by: Drew Roen <drewroen@google.com> Commit-Queue: Drew Roen <drewroen@google.com> Reviewed-by: Jesús Guerrero <jsguerrero@google.com> (cherry picked from commit 6ecdbfdf6c93abd610beb782725d2f1d2b462582) Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/50121 Commit-Queue: Jesús Guerrero <jsguerrero@google.com> (cherry picked from commit d77f73b30c038568d317c0679200b144a7ad7c6a) Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/50140
diff --git a/recipe_modules/flutter_bcid/__init__.py b/recipe_modules/flutter_bcid/__init__.py index 033f58b..c790b26 100644 --- a/recipe_modules/flutter_bcid/__init__.py +++ b/recipe_modules/flutter_bcid/__init__.py
@@ -3,12 +3,9 @@ # found in the LICENSE file. DEPS = [ - 'dart/dart', - 'depot_tools/gsutil', 'flutter/repo_util', 'recipe_engine/bcid_reporter', 'recipe_engine/buildbucket', 'recipe_engine/file', - 'recipe_engine/path', 'recipe_engine/platform', ]
diff --git a/recipe_modules/flutter_bcid/api.py b/recipe_modules/flutter_bcid/api.py index 9481d1a..3ab2a52 100644 --- a/recipe_modules/flutter_bcid/api.py +++ b/recipe_modules/flutter_bcid/api.py
@@ -47,36 +47,3 @@ if self.is_official_build(): sha256 = self.m.file.file_hash(local_artifact_path) self.m.bcid_reporter.report_gcs(sha256, remote_artifact_path) - - def download_and_verify_provenance( - self, filename, bucket, gcs_path_without_bucket - ): - """Downloads and verifies provenance for a specified artifact. - - This method downloads an artifact and associated provenance from GCS, - verifies it. If verification fails, an error is raised. - - parameters: - filename: (str) the name of the file, eg: "flutter_artifact.zip" - bucket: (str) the GCS bucket, eg: "flutter_infra_release" - gcs_path_without_bucket: (str) the GCS path, excluding gs://{bucket}/ - eg: "flutter/004d0bdf6721bc65cdb9a558908b2de4cfac97c5/sky_engine.zip" - """ - verify_temp_path = self.m.path.mkdtemp("verify") - download_path = download_path = verify_temp_path.join(filename) - bcid_response = self.m.dart.download_and_verify( - filename, bucket, gcs_path_without_bucket, download_path, - 'misc_software://flutter/engine' - ) - - artifact_vsa = bcid_response['verificationSummary'] - vsa_local_path = f'{download_path}.vsa.intoto.jsonl' - self.m.file.write_text( - f'write {filename}.vsa.intoto.jsonl', vsa_local_path, artifact_vsa - ) - self.m.gsutil.upload( - vsa_local_path, - bucket, - gcs_path_without_bucket, - name='upload "%s"' % gcs_path_without_bucket - )
diff --git a/recipe_modules/flutter_bcid/examples/full.expected/basic.json b/recipe_modules/flutter_bcid/examples/full.expected/basic.json index 321de6d..42423f8 100644 --- a/recipe_modules/flutter_bcid/examples/full.expected/basic.json +++ b/recipe_modules/flutter_bcid/examples/full.expected/basic.json
@@ -112,336 +112,6 @@ "name": "snoop: report_gcs" }, { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "gs://flutter_infra/release_artifacts/artifacts.zip", - "[CLEANUP]/verify_tmp_1/artifact.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil download artifact.zip" - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "gs://flutter_infra/release_artifacts/artifacts.zip.intoto.jsonl", - "[CLEANUP]/verify_tmp_1/artifact.zip.intoto.jsonl" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil download artifact.zip provenance" - }, - { - "cmd": [ - "luci-auth", - "token", - "-scopes", - "https://www.googleapis.com/auth/bcid_verify https://www.googleapis.com/auth/cloud-platform", - "-lifetime", - "3m" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "get access token for default account" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "file_hash", - "[CLEANUP]/verify_tmp_1/artifact.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Compute file hash (2)", - "~followup_annotations": [ - "@@@STEP_TEXT@Hash calculated: 3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[CLEANUP]/verify_tmp_1/artifact.zip.intoto.jsonl", - "/path/to/tmp/" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "read artifact.zip provenance", - "~followup_annotations": [ - "@@@STEP_LOG_END@artifact.zip.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "Authorization: Bearer extra.secret.token.should.not.be.logged\n", - "[CLEANUP]/authorization" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write authorization" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "{\"resourceToVerify\": \"misc_software://flutter/engine\", \"artifactInfo\": {\"digests\": {\"sha256\": \"3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90\"}, \"attestations\": [\"\"]}}", - "[CLEANUP]/request" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write request", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@request@{\"resourceToVerify\": \"misc_software://flutter/engine\", \"artifactInfo\": {\"digests\": {\"sha256\": \"3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90\"}, \"attestations\": [\"\"]}}@@@", - "@@@STEP_LOG_END@request@@@" - ] - }, - { - "cmd": [ - "curl", - "-H", - "@[CLEANUP]/authorization", - "-H", - "Content-Type: application/json", - "-d", - "@[CLEANUP]/request", - "https://bcidsoftwareverifier-pa.googleapis.com/v1/software-artifact-verification-requests" - ], - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "verify artifact.zip provenance", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true, \"verificationSummary\": \"This artifact is definitely legitimate!\"}@@@", - "@@@STEP_LOG_END@raw_io.output_text@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "remove", - "[CLEANUP]/authorization" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "delete authorization" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "remove", - "[CLEANUP]/request" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "delete request" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "This artifact is definitely legitimate!", - "[CLEANUP]/verify_tmp_1/artifact.zip.vsa.intoto.jsonl" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write artifact.zip.vsa.intoto.jsonl", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@artifact.zip.vsa.intoto.jsonl@This artifact is definitely legitimate!@@@", - "@@@STEP_LOG_END@artifact.zip.vsa.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[CLEANUP]/verify_tmp_1/artifact.zip.vsa.intoto.jsonl", - "gs://flutter_infra/release_artifacts/artifacts.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil upload \"release_artifacts/artifacts.zip\"", - "~followup_annotations": [ - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/flutter_infra/release_artifacts/artifacts.zip@@@" - ] - }, - { "name": "$result" } ] \ No newline at end of file
diff --git a/recipe_modules/flutter_bcid/examples/full.expected/prod_build.json b/recipe_modules/flutter_bcid/examples/full.expected/prod_build.json index 321de6d..42423f8 100644 --- a/recipe_modules/flutter_bcid/examples/full.expected/prod_build.json +++ b/recipe_modules/flutter_bcid/examples/full.expected/prod_build.json
@@ -112,336 +112,6 @@ "name": "snoop: report_gcs" }, { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "gs://flutter_infra/release_artifacts/artifacts.zip", - "[CLEANUP]/verify_tmp_1/artifact.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil download artifact.zip" - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "gs://flutter_infra/release_artifacts/artifacts.zip.intoto.jsonl", - "[CLEANUP]/verify_tmp_1/artifact.zip.intoto.jsonl" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil download artifact.zip provenance" - }, - { - "cmd": [ - "luci-auth", - "token", - "-scopes", - "https://www.googleapis.com/auth/bcid_verify https://www.googleapis.com/auth/cloud-platform", - "-lifetime", - "3m" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "get access token for default account" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "file_hash", - "[CLEANUP]/verify_tmp_1/artifact.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Compute file hash (2)", - "~followup_annotations": [ - "@@@STEP_TEXT@Hash calculated: 3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "[CLEANUP]/verify_tmp_1/artifact.zip.intoto.jsonl", - "/path/to/tmp/" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "read artifact.zip provenance", - "~followup_annotations": [ - "@@@STEP_LOG_END@artifact.zip.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "Authorization: Bearer extra.secret.token.should.not.be.logged\n", - "[CLEANUP]/authorization" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write authorization" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "{\"resourceToVerify\": \"misc_software://flutter/engine\", \"artifactInfo\": {\"digests\": {\"sha256\": \"3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90\"}, \"attestations\": [\"\"]}}", - "[CLEANUP]/request" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write request", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@request@{\"resourceToVerify\": \"misc_software://flutter/engine\", \"artifactInfo\": {\"digests\": {\"sha256\": \"3038cc85aa9c41479c21791a47b1af8f38a422a73f61553b320b1411018a4c90\"}, \"attestations\": [\"\"]}}@@@", - "@@@STEP_LOG_END@request@@@" - ] - }, - { - "cmd": [ - "curl", - "-H", - "@[CLEANUP]/authorization", - "-H", - "Content-Type: application/json", - "-d", - "@[CLEANUP]/request", - "https://bcidsoftwareverifier-pa.googleapis.com/v1/software-artifact-verification-requests" - ], - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "verify artifact.zip provenance", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true, \"verificationSummary\": \"This artifact is definitely legitimate!\"}@@@", - "@@@STEP_LOG_END@raw_io.output_text@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "remove", - "[CLEANUP]/authorization" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "delete authorization" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "remove", - "[CLEANUP]/request" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "delete request" - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "This artifact is definitely legitimate!", - "[CLEANUP]/verify_tmp_1/artifact.zip.vsa.intoto.jsonl" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "write artifact.zip.vsa.intoto.jsonl", - "~followup_annotations": [ - "@@@STEP_LOG_LINE@artifact.zip.vsa.intoto.jsonl@This artifact is definitely legitimate!@@@", - "@@@STEP_LOG_END@artifact.zip.vsa.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[CLEANUP]/verify_tmp_1/artifact.zip.vsa.intoto.jsonl", - "gs://flutter_infra/release_artifacts/artifacts.zip" - ], - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "gsutil upload \"release_artifacts/artifacts.zip\"", - "~followup_annotations": [ - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/flutter_infra/release_artifacts/artifacts.zip@@@" - ] - }, - { "name": "$result" } ] \ No newline at end of file
diff --git a/recipe_modules/flutter_bcid/examples/full.py b/recipe_modules/flutter_bcid/examples/full.py index 428cfee..d423d09 100644 --- a/recipe_modules/flutter_bcid/examples/full.py +++ b/recipe_modules/flutter_bcid/examples/full.py
@@ -6,7 +6,6 @@ 'flutter/flutter_bcid', 'recipe_engine/buildbucket', 'recipe_engine/path', - 'recipe_engine/raw_io', ] @@ -17,14 +16,9 @@ ) api.flutter_bcid.is_official_build() api.flutter_bcid.is_prod_build() - api.flutter_bcid.download_and_verify_provenance( - "artifact.zip", "flutter_infra", "release_artifacts/artifacts.zip" - ) def GenTests(api): - fake_bcid_response_success = '{"allowed": true, "verificationSummary": "This artifact is definitely legitimate!"}' - artifacts_location = 'artifact.zip' yield api.test( 'basic', api.buildbucket.ci_build( @@ -33,10 +27,6 @@ git_repo='https://dart.googlesource.com/monorepo', git_ref='refs/heads/main' ), - api.step_data( - 'verify %s provenance' % artifacts_location, - stdout=api.raw_io.output_text(fake_bcid_response_success) - ), ) yield api.test( @@ -47,8 +37,4 @@ git_repo='https://dart.googlesource.com/monorepo', git_ref='refs/heads/main' ), - api.step_data( - 'verify %s provenance' % artifacts_location, - stdout=api.raw_io.output_text(fake_bcid_response_success) - ), )
diff --git a/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json b/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json index 68ff16c..c2c9972 100644 --- a/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json +++ b/recipes/engine_v2/builder.expected/dart-internal-flutter-success.json
@@ -2859,7 +2859,7 @@ "name": "Verify provenance.verify artifacts.zip provenance", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true, \"verificationSummary\": \"This artifact is definitely legitimate!\"}@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true}@@@", "@@@STEP_LOG_END@raw_io.output_text@@@" ] }, @@ -2979,126 +2979,6 @@ }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "This artifact is definitely legitimate!", - "[CLEANUP]/verify_tmp_1/artifacts.zip.vsa.intoto.jsonl" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.write artifacts.zip.vsa.intoto.jsonl", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@artifacts.zip.vsa.intoto.jsonl@This artifact is definitely legitimate!@@@", - "@@@STEP_LOG_END@artifacts.zip.vsa.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[CLEANUP]/verify_tmp_1/artifacts.zip.vsa.intoto.jsonl", - "gs://flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-x86-jit-release/artifacts.zip" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.gsutil upload \"flutter/12345abcde12345abcde12345abcde12345abcde/android-x86-jit-release/artifacts.zip\"", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/android-x86-jit-release/artifacts.zip@@@" - ] - }, - { - "cmd": [ "python3", "-u", "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", @@ -3561,7 +3441,7 @@ "name": "Verify provenance.verify x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar provenance", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true, \"verificationSummary\": \"This artifact is definitely legitimate!\"}@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true}@@@", "@@@STEP_LOG_END@raw_io.output_text@@@" ] }, @@ -3681,126 +3561,6 @@ }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "This artifact is definitely legitimate!", - "[CLEANUP]/verify_tmp_2/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar.vsa.intoto.jsonl" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.write x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar.vsa.intoto.jsonl", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar.vsa.intoto.jsonl@This artifact is definitely legitimate!@@@", - "@@@STEP_LOG_END@x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar.vsa.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[CLEANUP]/verify_tmp_2/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar.vsa.intoto.jsonl", - "gs://download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.gsutil upload \"io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar\"", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar@@@" - ] - }, - { - "cmd": [ "python3", "-u", "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", @@ -4263,7 +4023,7 @@ "name": "Verify provenance.verify x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom provenance", "~followup_annotations": [ "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true, \"verificationSummary\": \"This artifact is definitely legitimate!\"}@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@{\"allowed\": true}@@@", "@@@STEP_LOG_END@raw_io.output_text@@@" ] }, @@ -4388,126 +4148,6 @@ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", "--json-output", "/path/to/tmp/json", - "copy", - "This artifact is definitely legitimate!", - "[CLEANUP]/verify_tmp_3/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom.vsa.intoto.jsonl" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.write x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom.vsa.intoto.jsonl", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LOG_LINE@x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom.vsa.intoto.jsonl@This artifact is definitely legitimate!@@@", - "@@@STEP_LOG_END@x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom.vsa.intoto.jsonl@@@" - ] - }, - { - "cmd": [ - "python3", - "-u", - "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py", - "--", - "RECIPE_REPO[depot_tools]/gsutil.py", - "----", - "cp", - "[CLEANUP]/verify_tmp_3/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom.vsa.intoto.jsonl", - "gs://download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom" - ], - "cwd": "[CACHE]/builder/src/flutter", - "env": { - "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk", - "ANDROID_SDK_HOME": "[CLEANUP]/tmp_tmp_1", - "ANDROID_USER_HOME": "[CLEANUP]/tmp_tmp_1/.android", - "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]", - "ENGINE_CHECKOUT_PATH": "[CACHE]/builder", - "ENGINE_PATH": "[CACHE]/builder", - "GIT_BRANCH": "", - "LUCI_BRANCH": "", - "LUCI_CI": "True", - "LUCI_PR": "", - "LUCI_WORKDIR": "[START_DIR]", - "OS": "linux", - "REVISION": "2d72510e447ab60a9728aeea2362d8be2cbd7789" - }, - "env_prefixes": { - "PATH": [ - "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin" - ] - }, - "env_suffixes": { - "DEPOT_TOOLS_UPDATE": [ - "0" - ], - "PATH": [ - "RECIPE_REPO[depot_tools]" - ] - }, - "infra_step": true, - "luci_context": { - "realm": { - "name": "dart-internal:flutter" - }, - "resultdb": { - "current_invocation": { - "name": "invocations/build:8945511751514863184", - "update_token": "token" - }, - "hostname": "rdbhost" - } - }, - "name": "Verify provenance.gsutil upload \"io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom\"", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@1@@@", - "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/download.flutter.io/io/flutter/x86_debug/1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", "copytree", "[CACHE]/builder/src/out", "[CLEANUP]/out-cas-directory_tmp_1"
diff --git a/recipes/engine_v2/builder.py b/recipes/engine_v2/builder.py index 6965dc6..877ad52 100644 --- a/recipes/engine_v2/builder.py +++ b/recipes/engine_v2/builder.py
@@ -33,7 +33,9 @@ from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2 DEPS = [ + 'dart/dart', 'depot_tools/depot_tools', + 'depot_tools/gsutil', 'flutter/archives', 'flutter/build_util', 'flutter/flutter_bcid', @@ -224,6 +226,7 @@ '(Non-blocking) Provenance verification failed - check step above', [] ) + continue # Archive full build. This is inefficient but necessary for global generators. if build.get('cas_archive', True): full_build_hash = api.shard_util_v2.archive_full_build( @@ -256,14 +259,17 @@ paths = api.archives.engine_v2_gcs_paths(checkout, archive_config) for path in paths: + verify_temp_path = api.path.mkdtemp("verify") gcs_path = path.remote gcs_path_without_prefix = str.lstrip(gcs_path, 'gs://') file = api.path.basename(gcs_path) bucket = gcs_path_without_prefix.split('/', maxsplit=1)[0] gcs_path_without_bucket = '/'.join(gcs_path_without_prefix.split('/')[1:]) + download_path = verify_temp_path.join(file) - api.flutter_bcid.download_and_verify_provenance( - file, bucket, gcs_path_without_bucket + api.dart.download_and_verify( + file, bucket, gcs_path_without_bucket, download_path, + 'misc_software://flutter/engine' ) @@ -387,8 +393,6 @@ api.monorepo.try_build(), ) - fake_bcid_response_success = '{"allowed": true, "verificationSummary": "This artifact is definitely legitimate!"}' - fake_bcid_response_failure = '{"rejectionMessage": "failed to validate!"}' build_custom = dict(build) build_custom["gclient_variables"] = {"example_custom_var": True} build_custom["tests"] = [] @@ -406,15 +410,15 @@ ), api.step_data( 'Verify provenance.verify %s provenance' % artifacts_location, - stdout=api.raw_io.output_text(fake_bcid_response_success) + stdout=api.raw_io.output_text('{"allowed": true}') ), api.step_data( 'Verify provenance.verify %s provenance' % jar_location, - stdout=api.raw_io.output_text(fake_bcid_response_success) + stdout=api.raw_io.output_text('{"allowed": true}') ), api.step_data( 'Verify provenance.verify %s provenance' % pom_location, - stdout=api.raw_io.output_text(fake_bcid_response_success) + stdout=api.raw_io.output_text('{"allowed": true}') ), ) yield api.test( @@ -428,6 +432,7 @@ ), api.step_data( 'Verify provenance.verify %s provenance' % artifacts_location, - stdout=api.raw_io.output_text(fake_bcid_response_failure) + stdout=api.raw_io + .output_text('{"rejectionMessage": "failed to validate!"}') ), )