Report dart-internal builds to the pubsub endpoint, so they can later be added to the datastore Bug: https://github.com/flutter/flutter/issues/124627 Change-Id: I151fa6fcbd26b8e5f924e5fdf14fa5aba04a70fc Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/43360 Reviewed-by: Godofredo Contreras <godofredoc@google.com> Commit-Queue: Drew Roen <drewroen@google.com>
diff --git a/recipe_modules/status_reporting/__init__.py b/recipe_modules/status_reporting/__init__.py index f0997e7..12b04a3 100644 --- a/recipe_modules/status_reporting/__init__.py +++ b/recipe_modules/status_reporting/__init__.py
@@ -3,7 +3,7 @@ # found in the LICENSE file. DEPS = [ - 'fuchsia/gcloud', + 'flutter/gcloud', 'recipe_engine/file', 'recipe_engine/step', ]
diff --git a/recipe_modules/status_reporting/api.py b/recipe_modules/status_reporting/api.py index 5630eaa..8956d2b 100644 --- a/recipe_modules/status_reporting/api.py +++ b/recipe_modules/status_reporting/api.py
@@ -25,7 +25,8 @@ def publish_builds( self, subbuilds, - topic='projects/flutter-dashboard/topics/luci-builds-prod' + topic='projects/flutter-dashboard/topics/luci-builds-prod', + only_publish_build_id=False ): """Publish builds to a pubsub topic. @@ -33,11 +34,18 @@ subbuilds(dict): A dictionary with the build name as key and a value of shard_util_v2.SubbuildResult as a value. topic(str): (optional) gcloud topic to publish message to. + only_publish_build_id(bool): (optional) If True, only publish the build_id + of the shard_util_v2.SubbuildResult instead of the entire build json. """ with self.m.step.nest('Publish results') as presentation: for id_name, build in subbuilds.items(): + if only_publish_build_id is True: + message = build.build_id + else: + message = self.build_to_json(build.build_proto) + cmd = [ 'pubsub', 'topics', 'publish', topic, - '--message=\'%s\'' % self.build_to_json(build.build_proto) + '--message=\'%s\'' % message ] self.m.gcloud(*cmd, infra_step=True)
diff --git a/recipe_modules/status_reporting/examples/full.expected/basic.json b/recipe_modules/status_reporting/examples/full.expected/basic.json index b08e2a0..66a2da4 100644 --- a/recipe_modules/status_reporting/examples/full.expected/basic.json +++ b/recipe_modules/status_reporting/examples/full.expected/basic.json
@@ -12,74 +12,26 @@ }, { "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "copy", - "RECIPE_MODULE[fuchsia::gcloud]/resources/tool_manifest.json", - "/path/to/tmp/json" - ], - "infra_step": true, - "name": "Publish results.ensure gcloud.read manifest", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@", - "@@@STEP_LOG_LINE@tool_manifest.json@{@@@", - "@@@STEP_LOG_LINE@tool_manifest.json@ \"path\": \"path/to/gcloud\",@@@", - "@@@STEP_LOG_LINE@tool_manifest.json@ \"version\": \"version:pinned-version\"@@@", - "@@@STEP_LOG_LINE@tool_manifest.json@}@@@", - "@@@STEP_LOG_END@tool_manifest.json@@@" - ] - }, - { - "cmd": [], - "name": "Publish results.ensure gcloud.install path/to/gcloud", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@2@@@" - ] - }, - { - "cmd": [ - "vpython3", - "-u", - "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py", - "--json-output", - "/path/to/tmp/json", - "ensure-directory", - "--mode", - "0777", - "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version" - ], - "infra_step": true, - "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure package directory", - "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@3@@@" - ] - }, - { - "cmd": [ "cipd", "ensure", "-root", - "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version", + "[START_DIR]/gcloud", "-ensure-file", - "path/to/gcloud version:pinned-version", + "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3", "-max-threads", "0", "-json-output", "/path/to/tmp/json" ], - "infra_step": true, - "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure_installed", + "name": "Publish results.ensure gcloud.ensure_installed", "~followup_annotations": [ - "@@@STEP_NEST_LEVEL@3@@@", + "@@@STEP_NEST_LEVEL@2@@@", "@@@STEP_LOG_LINE@json.output@{@@@", "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", "@@@STEP_LOG_LINE@json.output@ {@@@", - "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:pinned-v\", @@@", - "@@@STEP_LOG_LINE@json.output@ \"package\": \"path/to/gcloud\"@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@", "@@@STEP_LOG_LINE@json.output@ }@@@", "@@@STEP_LOG_LINE@json.output@ ]@@@", "@@@STEP_LOG_LINE@json.output@ }@@@", @@ -89,7 +41,7 @@ }, { "cmd": [ - "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/bin/gcloud", + "[START_DIR]/gcloud/bin/gcloud", "pubsub", "topics", "publish", @@ -103,6 +55,116 @@ ] }, { + "cmd": [], + "name": "Publish results (2)" + }, + { + "cmd": [], + "name": "Publish results (2).ensure gcloud", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[START_DIR]/gcloud", + "-ensure-file", + "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "name": "Publish results (2).ensure gcloud.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/gcloud/bin/gcloud", + "pubsub", + "topics", + "publish", + "custom/pubsub/url", + "--message='{\n \"builder\": {\n \"project\": \"flutter\",\n \"bucket\": \"try\",\n \"builder\": \"mybuild\"\n }\n}'" + ], + "infra_step": true, + "name": "Publish results (2).gcloud pubsub", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], + "name": "Publish results (3)" + }, + { + "cmd": [], + "name": "Publish results (3).ensure gcloud", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[START_DIR]/gcloud", + "-ensure-file", + "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "name": "Publish results (3).ensure gcloud.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/gcloud/bin/gcloud", + "pubsub", + "topics", + "publish", + "custom/pubsub/url", + "--message='123'" + ], + "infra_step": true, + "name": "Publish results (3).gcloud pubsub", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { "name": "$result" } ] \ No newline at end of file
diff --git a/recipe_modules/status_reporting/examples/full.py b/recipe_modules/status_reporting/examples/full.py index 6f74caf..f92e3bb 100644 --- a/recipe_modules/status_reporting/examples/full.py +++ b/recipe_modules/status_reporting/examples/full.py
@@ -23,7 +23,15 @@ url='https://123', build_proto=build ) - api.status_reporting.publish_builds({'mybuild': result}) + api.status_reporting.publish_builds(subbuilds={'mybuild': result}) + api.status_reporting.publish_builds( + subbuilds={'mybuild': result}, topic='custom/pubsub/url' + ) + api.status_reporting.publish_builds( + subbuilds={'mybuild': result}, + topic='custom/pubsub/url', + only_publish_build_id=True + ) def GenTests(api):
diff --git a/recipes/engine_v2/engine_v2.expected/basic_mac.json b/recipes/engine_v2/engine_v2.expected/basic_mac.json index c4dd16d..90f692e 100644 --- a/recipes/engine_v2/engine_v2.expected/basic_mac.json +++ b/recipes/engine_v2/engine_v2.expected/basic_mac.json
@@ -275,6 +275,85 @@ }, { "cmd": [], + "name": "Publish results" + }, + { + "cmd": [], + "name": "Publish results.ensure gcloud", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[START_DIR]/gcloud", + "-ensure-file", + "infra/3pp/tools/gcloud/${platform} version:2@428.0.0.chromium.3", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "luci_context": { + "realm": { + "name": "flutter:prod" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Publish results.ensure gcloud.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@2@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:2@428.0.\", @@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/3pp/tools/gcloud/resolved-platform\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [ + "[START_DIR]/gcloud/bin/gcloud", + "pubsub", + "topics", + "publish", + "projects/flutter-dashboard/topics/dart-internal-build-results", + "--message='8945511751514863186'" + ], + "infra_step": true, + "luci_context": { + "realm": { + "name": "flutter:prod" + }, + "resultdb": { + "current_invocation": { + "name": "invocations/build:8945511751514863184", + "update_token": "token" + }, + "hostname": "rdbhost" + } + }, + "name": "Publish results.gcloud pubsub", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [], "name": "display builds" }, { @@ -1044,6 +1123,10 @@ }, { "cmd": [], + "name": "Publish results (2)" + }, + { + "cmd": [], "name": "display tests" }, {
diff --git a/recipes/engine_v2/engine_v2.py b/recipes/engine_v2/engine_v2.py index 686b893..adcf710 100644 --- a/recipes/engine_v2/engine_v2.py +++ b/recipes/engine_v2/engine_v2.py
@@ -33,6 +33,7 @@ 'flutter/flutter_deps', 'flutter/monorepo', 'flutter/repo_util', + 'flutter/status_reporting', 'flutter/osx_sdk', 'flutter/shard_util_v2', 'recipe_engine/buildbucket', @@ -43,11 +44,13 @@ 'recipe_engine/platform', 'recipe_engine/properties', 'recipe_engine/raw_io', + 'recipe_engine/runtime', 'recipe_engine/step', ] PROPERTIES = InputProperties ENV_PROPERTIES = EnvProperties +BUILD_RESULT_PUBSUB_ENDPOINT = 'projects/flutter-dashboard/topics/dart-internal-build-results' def RunSteps(api, properties, env_properties): @@ -107,6 +110,11 @@ with api.step.nest('collect builds') as presentation: build_results = api.shard_util_v2.collect(tasks) + if not api.runtime.is_experimental and api.flutter_bcid.is_prod_build(): + api.status_reporting.publish_builds( + build_results, BUILD_RESULT_PUBSUB_ENDPOINT, True + ) + api.display_util.display_subbuilds( step_name='display builds', subbuilds=build_results, @@ -165,6 +173,11 @@ with api.step.nest('collect tests') as presentation: test_results = api.shard_util_v2.collect(tasks) + if not api.runtime.is_experimental and api.flutter_bcid.is_prod_build(): + api.status_reporting.publish_builds( + test_results, BUILD_RESULT_PUBSUB_ENDPOINT, True + ) + api.display_util.display_subbuilds( step_name='display tests', subbuilds=test_results,