Reland: Generate provenance for global archives.
The global archives are used to generate fat binaries from architecture
specific builds. This CL adds provenance generation steps for global
generators. This is the second and final part to enabling provenance
generation for the flutte engine artifacts.
Bug: https://github.com/flutter/flutter/issues/113193
Change-Id: I44b8cbef5f73711952de2fb8130e93d67d6b9c7e
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/35321
Reviewed-by: Yusuf Mohsinally <mohsinally@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/flutter_bcid/api.py b/recipe_modules/flutter_bcid/api.py
index d15891b..e3806b9 100644
--- a/recipe_modules/flutter_bcid/api.py
+++ b/recipe_modules/flutter_bcid/api.py
@@ -3,9 +3,21 @@
# found in the LICENSE file.
import re
+from enum import Enum
+
from recipe_engine import recipe_api
+class BcidStage(Enum):
+ """Enum representing valid bcis stages."""
+ START='start',
+ FETCH='fetch',
+ COMPILE='compile',
+ UPLOAD='upload',
+ UPLOAD_COMPLETE='upload-complete',
+ TEST='test'
+
+
class FlutterBcidApi(recipe_api.RecipeApi):
def _is_official_build(self):
diff --git a/recipes/engine_v2/engine_v2.expected/basic_mac.json b/recipes/engine_v2/engine_v2.expected/basic_mac.json
index 07ce18b..0647b21 100644
--- a/recipes/engine_v2/engine_v2.expected/basic_mac.json
+++ b/recipes/engine_v2/engine_v2.expected/basic_mac.json
@@ -1067,16 +1067,15 @@
},
{
"cmd": [
- "python3",
+ "vpython3",
"-u",
- "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
- "--",
- "RECIPE_REPO[depot_tools]/gsutil.py",
- "----",
- "cp",
- "-r",
- "/a/b/c.txt",
- "gs://flutter_archives_v2/flutter_infra_release/flutter//bucket/c.txt"
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
],
"infra_step": true,
"luci_context": {
@@ -1091,9 +1090,63 @@
"hostname": "rdbhost"
}
},
- "name": "gsutil c.txt",
+ "name": "Ensure flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "/a/b/c.txt",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Copy gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket/c.txt"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "----",
+ "cp",
+ "-r",
+ "[CLEANUP]/tmp_tmp_1/*",
+ "gs://flutter_archives_v2/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gsutil flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket/c.txt",
"~followup_annotations": [
- "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/flutter_infra_release/flutter//bucket/c.txt@@@"
+ "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@"
]
},
{
diff --git a/recipes/engine_v2/engine_v2.expected/config_from_file.json b/recipes/engine_v2/engine_v2.expected/config_from_file.json
index 9464dbd..d80edac 100644
--- a/recipes/engine_v2/engine_v2.expected/config_from_file.json
+++ b/recipes/engine_v2/engine_v2.expected/config_from_file.json
@@ -929,16 +929,15 @@
},
{
"cmd": [
- "python3",
+ "vpython3",
"-u",
- "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
- "--",
- "RECIPE_REPO[depot_tools]/gsutil.py",
- "----",
- "cp",
- "-r",
- "/a/b/c.txt",
- "gs://flutter_archives_v2/flutter_infra_release/flutter//bucket/c.txt"
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
],
"infra_step": true,
"luci_context": {
@@ -953,9 +952,63 @@
"hostname": "rdbhost"
}
},
- "name": "gsutil c.txt",
+ "name": "Ensure flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "/a/b/c.txt",
+ "[CLEANUP]/tmp_tmp_1/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Copy gs://flutter_archives_v2/flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket/c.txt"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "----",
+ "cp",
+ "-r",
+ "[CLEANUP]/tmp_tmp_1/*",
+ "gs://flutter_archives_v2/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "proj:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "gsutil flutter_infra_release/flutter/12345abcde12345abcde12345abcde12345abcde/bucket/c.txt",
"~followup_annotations": [
- "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/flutter_infra_release/flutter//bucket/c.txt@@@"
+ "@@@STEP_LINK@gsutil.upload@https://console.cloud.google.com/storage/browser/flutter_archives_v2/@@@"
]
},
{
diff --git a/recipes/engine_v2/engine_v2.py b/recipes/engine_v2/engine_v2.py
index b9caf3a..f1cbce0 100644
--- a/recipes/engine_v2/engine_v2.py
+++ b/recipes/engine_v2/engine_v2.py
@@ -13,6 +13,7 @@
# engine_v2/tester.py.
from contextlib import contextmanager
+from google.protobuf import struct_pb2
from PB.recipes.flutter.engine.engine import InputProperties
from PB.recipes.flutter.engine.engine import EnvProperties
@@ -20,10 +21,13 @@
from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
from PB.go.chromium.org.luci.buildbucket.proto \
import builds_service as builds_service_pb2
-from google.protobuf import struct_pb2
+
+from RECIPE_MODULES.flutter.flutter_bcid.api import BcidStage
+
DEPS = [
- 'depot_tools/gsutil',
+ 'flutter/archives',
+ 'flutter/flutter_bcid',
'flutter/display_util',
'flutter/flutter_deps',
'flutter/repo_util',
@@ -36,6 +40,7 @@
'recipe_engine/path',
'recipe_engine/platform',
'recipe_engine/properties',
+ 'recipe_engine/raw_io',
'recipe_engine/step',
]
@@ -44,6 +49,7 @@
def RunSteps(api, properties, env_properties):
+ api.flutter_bcid.report_stage(BcidStage.START.value)
config_name = api.properties.get('config_name')
builds = api.properties.get('builds')
tests = api.properties.get('tests')
@@ -57,6 +63,7 @@
project = 'engine'
# Only check out the repository, not dependencies.
+ api.flutter_bcid.report_stage(BcidStage.FETCH.value)
checkout_path = api.path['start_dir'].join(project)
api.repo_util.checkout(
project,
@@ -130,6 +137,7 @@
api.shard_util_v2.download_full_builds(build_results, out_builds_path)
with api.step.nest('Global generators') as presentation:
if 'tasks' in generators:
+ api.flutter_bcid.report_stage(BcidStage.COMPILE.value)
for generator_task in generators['tasks']:
# Generators must run from inside flutter folder.
# If platform is mac we need to run the generator from an xcode context.
@@ -151,6 +159,7 @@
api.file.listdir('Final List checkout 2', full_engine_checkout.join('src', 'flutter', 'sky'), recursive=True)
# Global archives
if archives:
+ api.flutter_bcid.report_stage(BcidStage.UPLOAD.value)
# Global archives are stored in out folder from full_engine_checkout inside
# release, debug or profile depending on the runtime mode.
# So far we are uploading files only.
@@ -163,13 +172,10 @@
artifact_path = 'flutter_infra_release/flutter/%s/%s' % (
commit, archive.get('destination')
)
- api.gsutil.upload(
- source=source,
- bucket=bucket,
- dest=artifact_path,
- args=['-r'],
- name=archive.get('name'),
- )
+ dst = 'gs://%s/%s' % (bucket, artifact_path)
+ api.archives.upload_artifact(source, dst)
+ api.flutter_bcid.upload_provenance(source, dst)
+ api.flutter_bcid.report_stage(BcidStage.UPLOAD_COMPLETE.value)
def _run_global_generator(api, generator_task, full_engine_checkout, env, env_prefixes):
@@ -246,6 +252,11 @@
launch_step="launch builds",
collect_step="collect builds",
),
+ api.step_data(
+ 'git rev-parse',
+ stdout=api.raw_io
+ .output_text('12345abcde12345abcde12345abcde12345abcde\n')
+ )
)
yield api.test(
@@ -287,6 +298,11 @@
'Read build config file',
api.file.read_json({'builds': builds, 'archives': archives})
),
+ api.step_data(
+ 'git rev-parse',
+ stdout=api.raw_io
+ .output_text('12345abcde12345abcde12345abcde12345abcde\n')
+ )
)
yield api.test(
