Report dart-internal builds to the pubsub endpoint, so they can later be added to the datastore
Bug: https://github.com/flutter/flutter/issues/124627
Change-Id: I151fa6fcbd26b8e5f924e5fdf14fa5aba04a70fc
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/43360
Reviewed-by: Godofredo Contreras <godofredoc@google.com>
Commit-Queue: Drew Roen <drewroen@google.com>
diff --git a/recipe_modules/status_reporting/__init__.py b/recipe_modules/status_reporting/__init__.py
index f0997e7..12b04a3 100644
--- a/recipe_modules/status_reporting/__init__.py
+++ b/recipe_modules/status_reporting/__init__.py
@@ -3,7 +3,7 @@
# found in the LICENSE file.
DEPS = [
- 'fuchsia/gcloud',
+ 'flutter/gcloud',
'recipe_engine/file',
'recipe_engine/step',
]
diff --git a/recipe_modules/status_reporting/api.py b/recipe_modules/status_reporting/api.py
index 5630eaa..8956d2b 100644
--- a/recipe_modules/status_reporting/api.py
+++ b/recipe_modules/status_reporting/api.py
@@ -25,7 +25,8 @@
def publish_builds(
self,
subbuilds,
- topic='projects/flutter-dashboard/topics/luci-builds-prod'
+ topic='projects/flutter-dashboard/topics/luci-builds-prod',
+ only_publish_build_id=False
):
"""Publish builds to a pubsub topic.
@@ -33,11 +34,18 @@
subbuilds(dict): A dictionary with the build name as key and a value
of shard_util_v2.SubbuildResult as a value.
topic(str): (optional) gcloud topic to publish message to.
+ only_publish_build_id(bool): (optional) If True, only publish the build_id
+ of the shard_util_v2.SubbuildResult instead of the entire build json.
"""
with self.m.step.nest('Publish results') as presentation:
for id_name, build in subbuilds.items():
+ if only_publish_build_id is True:
+ message = build.build_id
+ else:
+ message = self.build_to_json(build.build_proto)
+
cmd = [
'pubsub', 'topics', 'publish', topic,
- '--message=\'%s\'' % self.build_to_json(build.build_proto)
+ '--message=\'%s\'' % message
]
self.m.gcloud(*cmd, infra_step=True)
diff --git a/recipe_modules/status_reporting/examples/full.expected/basic.json b/recipe_modules/status_reporting/examples/full.expected/basic.json
index b08e2a0..66a2da4 100644
--- a/recipe_modules/status_reporting/examples/full.expected/basic.json
+++ b/recipe_modules/status_reporting/examples/full.expected/basic.json
@@ -12,74 +12,26 @@
},
{
"cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "RECIPE_MODULE[fuchsia::gcloud]/resources/tool_manifest.json",
- "/path/to/tmp/json"
- ],
- "infra_step": true,
- "name": "Publish results.ensure gcloud.read manifest",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@",
- "@@@STEP_LOG_LINE@tool_manifest.json@{@@@",
- "@@@STEP_LOG_LINE@tool_manifest.json@ \"path\": \"path/to/gcloud\",@@@",
- "@@@STEP_LOG_LINE@tool_manifest.json@ \"version\": \"version:pinned-version\"@@@",
- "@@@STEP_LOG_LINE@tool_manifest.json@}@@@",
- "@@@STEP_LOG_END@tool_manifest.json@@@"
- ]
- },
- {
- "cmd": [],
- "name": "Publish results.ensure gcloud.install path/to/gcloud",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@2@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "ensure-directory",
- "--mode",
- "0777",
- "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version"
- ],
- "infra_step": true,
- "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure package directory",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@"
- ]
- },
- {
- "cmd": [
"cipd",
"ensure",
"-root",
- "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version",
+ "[START_DIR]/gcloud",
"-ensure-file",
- "path/to/gcloud version:pinned-version",
+ "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3",
"-max-threads",
"0",
"-json-output",
"/path/to/tmp/json"
],
- "infra_step": true,
- "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure_installed",
+ "name": "Publish results.ensure gcloud.ensure_installed",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_NEST_LEVEL@2@@@",
"@@@STEP_LOG_LINE@json.output@{@@@",
"@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
"@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
"@@@STEP_LOG_LINE@json.output@ {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:pinned-v\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"package\": \"path/to/gcloud\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
"@@@STEP_LOG_LINE@json.output@ ]@@@",
"@@@STEP_LOG_LINE@json.output@ }@@@",
@@ -89,7 +41,7 @@
},
{
"cmd": [
- "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/bin/gcloud",
+ "[START_DIR]/gcloud/bin/gcloud",
"pubsub",
"topics",
"publish",
@@ -103,6 +55,116 @@
]
},
{
+ "cmd": [],
+ "name": "Publish results (2)"
+ },
+ {
+ "cmd": [],
+ "name": "Publish results (2).ensure gcloud",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/gcloud",
+ "-ensure-file",
+ "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "Publish results (2).ensure gcloud.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/gcloud/bin/gcloud",
+ "pubsub",
+ "topics",
+ "publish",
+ "custom/pubsub/url",
+ "--message='{\n \"builder\": {\n \"project\": \"flutter\",\n \"bucket\": \"try\",\n \"builder\": \"mybuild\"\n }\n}'"
+ ],
+ "infra_step": true,
+ "name": "Publish results (2).gcloud pubsub",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "Publish results (3)"
+ },
+ {
+ "cmd": [],
+ "name": "Publish results (3).ensure gcloud",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/gcloud",
+ "-ensure-file",
+ "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "Publish results (3).ensure gcloud.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/gcloud/bin/gcloud",
+ "pubsub",
+ "topics",
+ "publish",
+ "custom/pubsub/url",
+ "--message='123'"
+ ],
+ "infra_step": true,
+ "name": "Publish results (3).gcloud pubsub",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
"name": "$result"
}
]
\ No newline at end of file
diff --git a/recipe_modules/status_reporting/examples/full.py b/recipe_modules/status_reporting/examples/full.py
index 6f74caf..f92e3bb 100644
--- a/recipe_modules/status_reporting/examples/full.py
+++ b/recipe_modules/status_reporting/examples/full.py
@@ -23,7 +23,15 @@
url='https://123',
build_proto=build
)
- api.status_reporting.publish_builds({'mybuild': result})
+ api.status_reporting.publish_builds(subbuilds={'mybuild': result})
+ api.status_reporting.publish_builds(
+ subbuilds={'mybuild': result}, topic='custom/pubsub/url'
+ )
+ api.status_reporting.publish_builds(
+ subbuilds={'mybuild': result},
+ topic='custom/pubsub/url',
+ only_publish_build_id=True
+ )
def GenTests(api):
diff --git a/recipes/engine_v2/engine_v2.expected/basic_mac.json b/recipes/engine_v2/engine_v2.expected/basic_mac.json
index c4dd16d..90f692e 100644
--- a/recipes/engine_v2/engine_v2.expected/basic_mac.json
+++ b/recipes/engine_v2/engine_v2.expected/basic_mac.json
@@ -275,6 +275,85 @@
},
{
"cmd": [],
+ "name": "Publish results"
+ },
+ {
+ "cmd": [],
+ "name": "Publish results.ensure gcloud",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/gcloud",
+ "-ensure-file",
+ "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "luci_context": {
+ "realm": {
+ "name": "flutter:prod"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Publish results.ensure gcloud.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/gcloud/bin/gcloud",
+ "pubsub",
+ "topics",
+ "publish",
+ "projects/flutter-dashboard/topics/dart-internal-build-results",
+ "--message='8945511751514863186'"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "flutter:prod"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Publish results.gcloud pubsub",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
"name": "display builds"
},
{
@@ -1044,6 +1123,10 @@
},
{
"cmd": [],
+ "name": "Publish results (2)"
+ },
+ {
+ "cmd": [],
"name": "display tests"
},
{
diff --git a/recipes/engine_v2/engine_v2.py b/recipes/engine_v2/engine_v2.py
index 686b893..adcf710 100644
--- a/recipes/engine_v2/engine_v2.py
+++ b/recipes/engine_v2/engine_v2.py
@@ -33,6 +33,7 @@
'flutter/flutter_deps',
'flutter/monorepo',
'flutter/repo_util',
+ 'flutter/status_reporting',
'flutter/osx_sdk',
'flutter/shard_util_v2',
'recipe_engine/buildbucket',
@@ -43,11 +44,13 @@
'recipe_engine/platform',
'recipe_engine/properties',
'recipe_engine/raw_io',
+ 'recipe_engine/runtime',
'recipe_engine/step',
]
PROPERTIES = InputProperties
ENV_PROPERTIES = EnvProperties
+BUILD_RESULT_PUBSUB_ENDPOINT = 'projects/flutter-dashboard/topics/dart-internal-build-results'
def RunSteps(api, properties, env_properties):
@@ -107,6 +110,11 @@
with api.step.nest('collect builds') as presentation:
build_results = api.shard_util_v2.collect(tasks)
+ if not api.runtime.is_experimental and api.flutter_bcid.is_prod_build():
+ api.status_reporting.publish_builds(
+ build_results, BUILD_RESULT_PUBSUB_ENDPOINT, True
+ )
+
api.display_util.display_subbuilds(
step_name='display builds',
subbuilds=build_results,
@@ -165,6 +173,11 @@
with api.step.nest('collect tests') as presentation:
test_results = api.shard_util_v2.collect(tasks)
+ if not api.runtime.is_experimental and api.flutter_bcid.is_prod_build():
+ api.status_reporting.publish_builds(
+ test_results, BUILD_RESULT_PUBSUB_ENDPOINT, True
+ )
+
api.display_util.display_subbuilds(
step_name='display tests',
subbuilds=test_results,
