Adds a results reporting module.
This module will be used to send build results data from dart-internal
to the flutter-dashboard.
Bug: https://github.com/flutter/flutter/issues/115490
Change-Id: I2beab566ba9970f14b3c352111f9c62adde72b68
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/36640
Reviewed-by: Yusuf Mohsinally <mohsinally@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/status_reporting/__init__.py b/recipe_modules/status_reporting/__init__.py
new file mode 100644
index 0000000..b37189a
--- /dev/null
+++ b/recipe_modules/status_reporting/__init__.py
@@ -0,0 +1,10 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+DEPS = [
+ 'fuchsia/gcloud',
+ 'recipe_engine/file',
+ 'recipe_engine/step',
+]
diff --git a/recipe_modules/status_reporting/api.py b/recipe_modules/status_reporting/api.py
new file mode 100644
index 0000000..7953045
--- /dev/null
+++ b/recipe_modules/status_reporting/api.py
@@ -0,0 +1,43 @@
+# Copyright 2022 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+from recipe_engine import recipe_api
+from google.protobuf import json_format
+
+from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2
+from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
+
+
+class StatusReportingApi(recipe_api.RecipeApi):
+
+ def build_to_json(self, build):
+ """Encodes a shard_util_v2.SubbuildResult to a json string.
+
+ Args:
+ build(build.Build): The build to encode.
+
+ Returns:
+ A string with the Build message encoded as Json.
+ """
+ return json_format.MessageToJson(build)
+
+ def publish_builds(
+ self,
+ subbuilds,
+ topic='projects/flutter-dashboard/topics/luci-builds-prod'):
+ """Publish builds to a pubsub topic.
+
+ Args:
+ subbuilds(dict): A dictionary with the build name as key and a value
+ of shard_util_v2.SubbuildResult as a value.
+ topic(str): (optional) gcloud topic to publish message to.
+ """
+ with self.m.step.nest('Publish results') as presentation:
+ for id_name, build in subbuilds.items():
+ cmd = [
+ 'pubsub', 'topics', 'publish', topic,
+ '--message=\'%s\'' % self.build_to_json(build.build_proto)
+ ]
+ self.m.gcloud(*cmd, infra_step=True)
diff --git a/recipe_modules/status_reporting/examples/full.expected/basic.json b/recipe_modules/status_reporting/examples/full.expected/basic.json
new file mode 100644
index 0000000..4b7ec7a
--- /dev/null
+++ b/recipe_modules/status_reporting/examples/full.expected/basic.json
@@ -0,0 +1,109 @@
+[
+ {
+ "cmd": [],
+ "name": "Publish results"
+ },
+ {
+ "cmd": [],
+ "name": "Publish results.ensure gcloud",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "RECIPE_MODULE[fuchsia::gcloud]/resources/tool_manifest.json",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "Publish results.ensure gcloud.read manifest",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@{@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@ \"path\": \"path/to/gcloud\",@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@ \"version\": \"version:pinned-version\"@@@",
+ "@@@STEP_LOG_LINE@tool_manifest.json@}@@@",
+ "@@@STEP_LOG_END@tool_manifest.json@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "Publish results.ensure gcloud.install path/to/gcloud",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version"
+ ],
+ "infra_step": true,
+ "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version",
+ "-ensure-file",
+ "path/to/gcloud version:pinned-version",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "Publish results.ensure gcloud.install path/to/gcloud.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@3@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-version:pinned-v\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"path/to/gcloud\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/lib/gcloud.py",
+ "pubsub",
+ "topics",
+ "publish",
+ "projects/flutter-dashboard/topics/luci-builds-prod",
+ "--message='{\n \"builder\": {\n \"project\": \"flutter\",\n \"bucket\": \"try\",\n \"builder\": \"mybuild\"\n }\n}'"
+ ],
+ "infra_step": true,
+ "name": "Publish results.gcloud pubsub",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/status_reporting/examples/full.py b/recipe_modules/status_reporting/examples/full.py
new file mode 100644
index 0000000..d7ccf56
--- /dev/null
+++ b/recipe_modules/status_reporting/examples/full.py
@@ -0,0 +1,27 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine.post_process import DoesNotRun, Filter, StatusFailure
+
+from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
+from PB.go.chromium.org.luci.buildbucket.proto import builder_common as builder_pb2
+from RECIPE_MODULES.flutter.shard_util_v2.api import SubbuildResult
+
+
+DEPS = [
+ 'flutter/status_reporting'
+]
+
+def RunSteps(api):
+ build = build_pb2.Build(
+ builder=builder_pb2.BuilderID(project='flutter', bucket='try', builder='mybuild')
+ )
+ result = SubbuildResult(
+ builder='mybuild', build_id=123, build_name='build_name',
+ url='https://123', build_proto=build)
+ api.status_reporting.publish_builds({'mybuild': result})
+
+
+def GenTests(api):
+ yield api.test('basic')
diff --git a/recipe_modules/status_reporting/resources/mac_result.json b/recipe_modules/status_reporting/resources/mac_result.json
new file mode 100644
index 0000000..fd6ec9e
--- /dev/null
+++ b/recipe_modules/status_reporting/resources/mac_result.json
@@ -0,0 +1,42 @@
+{
+ "build": {
+ "bucket": "luci.flutter.prod",
+ "canary": false,
+ "canary_preference": "PROD",
+ "completed_ts": "1669080231756573",
+ "created_by": "user:flutter-dashboard@appspot.gserviceaccount.com",
+ "created_ts": "1669079440844500",
+ "experimental": false,
+ "id": "8796837072196031249",
+ "parameters_json": "{\"builder_name\": \"Mac iOS Engine Release\", \"properties\": {\"$flutter/osx_sdk\": {\"sdk_version\": \"14a5294e\"}, \"bringup\": false, \"build_android_aot\": false, \"build_android_debug\": false, \"build_android_jit_release\": false, \"build_android_vulkan\": false, \"build_fuchsia\": false, \"build_host\": false, \"build_ios\": true, \"cpu\": \"x86\", \"dependencies\": [{\"dependency\": \"open_jdk\", \"version\": \"version:1.8.0u202-b08\"}], \"device_type\": \"none\", \"exe_cipd_version\": \"refs/heads/main\", \"gcs_goldens_bucket\": \"\", \"git_branch\": \"main\", \"ios_debug\": false, \"ios_profile\": false, \"ios_release\": true, \"jazzy_version\": \"0.14.1\", \"no_bitcode\": false, \"os\": \"Mac-12\", \"xcode\": \"14a5294e\"}}",
+ "project": "flutter",
+ "result": "SUCCESS",
+ "result_details_json": "{\"properties\": {\"got_engine_revision\": \"8dd8e092e43f1c3051239f0cf2b01e8caeca6b49\"}, \"swarming\": {\"bot_dimensions\": {\"bot_config\": [\"bot_config.py\"], \"caches\": [\"engine__builder\", \"engine__open_jdk_version_1_8_0u202_b08_legacy\", \"engine__xcode_14a5294e_legacy\", \"engine_main_builder\", \"engine_main_open_jdk_version_1_8_0u202_b08_legacy\", \"engine_main_xcode_14a5294e_legacy\", \"flutter_3_3_0_android_sdk_version_32v1_legacy\", \"flutter_3_3_0_open_jdk_11_legacy\", \"flutter__android_sdk_version_33v6_legacy\", \"flutter__chrome_and_driver_version_98_1_legacy\", \"flutter__open_jdk_version_11_legacy\", \"flutter__xcode_14a5294e_legacy\", \"git\", \"goma_v2\", \"plugins_main_xcode_14a5294e_legacy\", \"vpython\"], \"cores\": [\"12\"], \"cpu\": [\"x86\", \"x86-64\", \"x86-64-i7-8700B\"], \"device_os\": [\"none\"], \"device_type\": [\"none\"], \"display_attached\": [\"1\"], \"dut_state\": [\"ready\"], \"gce\": [\"0\"], \"gpu\": [\"8086\", \"8086:3e9b\"], \"hidpi\": [\"0\"], \"id\": [\"build805-m9\"], \"inside_docker\": [\"0\"], \"mac_model\": [\"Macmini8,1\"], \"machine_type\": [\"n1-standard-8\"], \"os\": [\"Mac\", \"Mac-12\", \"Mac-12.6\", \"Mac-12.6-21G115\"], \"pool\": [\"luci.flutter.prod\"], \"python\": [\"3\", \"3.8\", \"3.8.10+chromium.23\"], \"server_version\": [\"6898-ae55986\"], \"ssd\": [\"1\"], \"ufs_state\": [\"ready\"], \"ufs_zone\": [\"ZONE_ATL97\"], \"zone\": [\"us\", \"us-atl\", \"us-atl-golo\", \"us-atl-golo-m9\"]}}}",
+ "retry_of": "0",
+ "service_account": "flutter-prod-builder@chops-service-accounts.iam.gserviceaccount.com",
+ "started_ts": "1669079478114499",
+ "status": "COMPLETED",
+ "status_changed_ts": "1669080231756574",
+ "tags": [
+ "build_address:luci.flutter.prod/Mac iOS Engine Release/17453",
+ "builder:Mac iOS Engine Release",
+ "buildset:commit/git/8dd8e092e43f1c3051239f0cf2b01e8caeca6b49",
+ "buildset:commit/gitiles/flutter.googlesource.com/mirrors/engine/+/8dd8e092e43f1c3051239f0cf2b01e8caeca6b49",
+ "gitiles_ref:refs/heads/main",
+ "scheduler_job_id:flutter/Mac iOS Engine Release",
+ "swarming_hostname:chromium-swarm.appspot.com",
+ "swarming_tag:log_location:logdog://logs.chromium.org/flutter/buildbucket/cr-buildbucket/8796837072196031249/+/annotations",
+ "swarming_tag:luci_project:flutter",
+ "swarming_tag:os:Mac",
+ "swarming_tag:recipe_name:engine/engine",
+ "swarming_tag:recipe_package:flutter/recipe_bundles/flutter.googlesource.com/recipes",
+ "swarming_task_id:5eb5b4cc3bbbfe10",
+ "user_agent:flutter-cocoon"
+ ],
+ "updated_ts": "1669080231756573",
+ "url": "https://ci.chromium.org/b/8796837072196031249",
+ "utcnow_ts": "1669080231879293"
+ },
+ "hostname": "cr-buildbucket.appspot.com",
+ "user_data": "eyJjb21taXRfa2V5IjoiZmx1dHRlci9lbmdpbmUvbWFpbi84ZGQ4ZTA5MmU0M2YxYzMwNTEyMzlmMGNmMmIwMWU4Y2FlY2E2YjQ5IiwidGFza19rZXkiOiI1ODE1Njg2NTQ3NzY3Mjk2In0="
+}
diff --git a/recipe_modules/status_reporting/resources/mac_result2.json b/recipe_modules/status_reporting/resources/mac_result2.json
new file mode 100644
index 0000000..fd6ec9e
--- /dev/null
+++ b/recipe_modules/status_reporting/resources/mac_result2.json
@@ -0,0 +1,42 @@
+{
+ "build": {
+ "bucket": "luci.flutter.prod",
+ "canary": false,
+ "canary_preference": "PROD",
+ "completed_ts": "1669080231756573",
+ "created_by": "user:flutter-dashboard@appspot.gserviceaccount.com",
+ "created_ts": "1669079440844500",
+ "experimental": false,
+ "id": "8796837072196031249",
+ "parameters_json": "{\"builder_name\": \"Mac iOS Engine Release\", \"properties\": {\"$flutter/osx_sdk\": {\"sdk_version\": \"14a5294e\"}, \"bringup\": false, \"build_android_aot\": false, \"build_android_debug\": false, \"build_android_jit_release\": false, \"build_android_vulkan\": false, \"build_fuchsia\": false, \"build_host\": false, \"build_ios\": true, \"cpu\": \"x86\", \"dependencies\": [{\"dependency\": \"open_jdk\", \"version\": \"version:1.8.0u202-b08\"}], \"device_type\": \"none\", \"exe_cipd_version\": \"refs/heads/main\", \"gcs_goldens_bucket\": \"\", \"git_branch\": \"main\", \"ios_debug\": false, \"ios_profile\": false, \"ios_release\": true, \"jazzy_version\": \"0.14.1\", \"no_bitcode\": false, \"os\": \"Mac-12\", \"xcode\": \"14a5294e\"}}",
+ "project": "flutter",
+ "result": "SUCCESS",
+ "result_details_json": "{\"properties\": {\"got_engine_revision\": \"8dd8e092e43f1c3051239f0cf2b01e8caeca6b49\"}, \"swarming\": {\"bot_dimensions\": {\"bot_config\": [\"bot_config.py\"], \"caches\": [\"engine__builder\", \"engine__open_jdk_version_1_8_0u202_b08_legacy\", \"engine__xcode_14a5294e_legacy\", \"engine_main_builder\", \"engine_main_open_jdk_version_1_8_0u202_b08_legacy\", \"engine_main_xcode_14a5294e_legacy\", \"flutter_3_3_0_android_sdk_version_32v1_legacy\", \"flutter_3_3_0_open_jdk_11_legacy\", \"flutter__android_sdk_version_33v6_legacy\", \"flutter__chrome_and_driver_version_98_1_legacy\", \"flutter__open_jdk_version_11_legacy\", \"flutter__xcode_14a5294e_legacy\", \"git\", \"goma_v2\", \"plugins_main_xcode_14a5294e_legacy\", \"vpython\"], \"cores\": [\"12\"], \"cpu\": [\"x86\", \"x86-64\", \"x86-64-i7-8700B\"], \"device_os\": [\"none\"], \"device_type\": [\"none\"], \"display_attached\": [\"1\"], \"dut_state\": [\"ready\"], \"gce\": [\"0\"], \"gpu\": [\"8086\", \"8086:3e9b\"], \"hidpi\": [\"0\"], \"id\": [\"build805-m9\"], \"inside_docker\": [\"0\"], \"mac_model\": [\"Macmini8,1\"], \"machine_type\": [\"n1-standard-8\"], \"os\": [\"Mac\", \"Mac-12\", \"Mac-12.6\", \"Mac-12.6-21G115\"], \"pool\": [\"luci.flutter.prod\"], \"python\": [\"3\", \"3.8\", \"3.8.10+chromium.23\"], \"server_version\": [\"6898-ae55986\"], \"ssd\": [\"1\"], \"ufs_state\": [\"ready\"], \"ufs_zone\": [\"ZONE_ATL97\"], \"zone\": [\"us\", \"us-atl\", \"us-atl-golo\", \"us-atl-golo-m9\"]}}}",
+ "retry_of": "0",
+ "service_account": "flutter-prod-builder@chops-service-accounts.iam.gserviceaccount.com",
+ "started_ts": "1669079478114499",
+ "status": "COMPLETED",
+ "status_changed_ts": "1669080231756574",
+ "tags": [
+ "build_address:luci.flutter.prod/Mac iOS Engine Release/17453",
+ "builder:Mac iOS Engine Release",
+ "buildset:commit/git/8dd8e092e43f1c3051239f0cf2b01e8caeca6b49",
+ "buildset:commit/gitiles/flutter.googlesource.com/mirrors/engine/+/8dd8e092e43f1c3051239f0cf2b01e8caeca6b49",
+ "gitiles_ref:refs/heads/main",
+ "scheduler_job_id:flutter/Mac iOS Engine Release",
+ "swarming_hostname:chromium-swarm.appspot.com",
+ "swarming_tag:log_location:logdog://logs.chromium.org/flutter/buildbucket/cr-buildbucket/8796837072196031249/+/annotations",
+ "swarming_tag:luci_project:flutter",
+ "swarming_tag:os:Mac",
+ "swarming_tag:recipe_name:engine/engine",
+ "swarming_tag:recipe_package:flutter/recipe_bundles/flutter.googlesource.com/recipes",
+ "swarming_task_id:5eb5b4cc3bbbfe10",
+ "user_agent:flutter-cocoon"
+ ],
+ "updated_ts": "1669080231756573",
+ "url": "https://ci.chromium.org/b/8796837072196031249",
+ "utcnow_ts": "1669080231879293"
+ },
+ "hostname": "cr-buildbucket.appspot.com",
+ "user_data": "eyJjb21taXRfa2V5IjoiZmx1dHRlci9lbmdpbmUvbWFpbi84ZGQ4ZTA5MmU0M2YxYzMwNTEyMzlmMGNmMmIwMWU4Y2FlY2E2YjQ5IiwidGFza19rZXkiOiI1ODE1Njg2NTQ3NzY3Mjk2In0="
+}
