Manual cherrypick of 'Filter release_builder targets by enabled branches.'
PR: https://flutter-review.googlesource.com/c/recipes/+/45380
Change-Id: I76f61c654d2cff90653d2c8ae42f3b1a99bfed20
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/48420
Reviewed-by: Godofredo Contreras <godofredoc@google.com>
Commit-Queue: Drew Roen <drewroen@google.com>
Reviewed-by: Xilai Zhang <xilaizhang@google.com>
diff --git a/recipes/release/release_builder.expected/basic_linux_beta.json b/recipes/release/release_builder.expected/basic_linux_beta.json
index 94c5598..167fdbd 100644
--- a/recipes/release/release_builder.expected/basic_linux_beta.json
+++ b/recipes/release/release_builder.expected/basic_linux_beta.json
@@ -291,6 +291,62 @@
},
{
"cmd": [],
+ "name": "Identify branches"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git rev-parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "branch",
+ "-a",
+ "--contains",
+ "12345abcde12345abcde12345abcde12345abcde"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git branch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
"name": "launch builds"
},
{
diff --git a/recipes/release/release_builder.expected/basic_linux_main.json b/recipes/release/release_builder.expected/basic_linux_main.json
index 4c3646b..9574748 100644
--- a/recipes/release/release_builder.expected/basic_linux_main.json
+++ b/recipes/release/release_builder.expected/basic_linux_main.json
@@ -292,6 +292,62 @@
},
{
"cmd": [],
+ "name": "Identify branches"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git rev-parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "branch",
+ "-a",
+ "--contains",
+ "12345abcde12345abcde12345abcde12345abcde"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git branch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
"name": "launch builds"
},
{
diff --git a/recipes/release/release_builder.expected/filter_enabled_branches.json b/recipes/release/release_builder.expected/filter_enabled_branches.json
new file mode 100644
index 0000000..a715709
--- /dev/null
+++ b/recipes/release/release_builder.expected/filter_enabled_branches.json
@@ -0,0 +1,363 @@
+[
+ {
+ "cmd": [],
+ "name": "Checkout flutter/mirrors/engine"
+ },
+ {
+ "cmd": [
+ "python3",
+ "-u",
+ "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+ "--path",
+ "[START_DIR]/mirrors/engine",
+ "--url",
+ "https://flutter.googlesource.com/mirrors/engine"
+ ],
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.git setup",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "origin",
+ "--recurse-submodules",
+ "--progress",
+ "--tags"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "env": {
+ "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
+ },
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.git fetch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.git checkout",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.read revision",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@",
+ "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.git clean",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.submodule sync",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Checkout flutter/mirrors/engine.submodule update",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "read ci yaml",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@yaml@@@@",
+ "@@@STEP_LOG_END@yaml@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/mirrors/engine/.ci.yaml",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "read ci yaml.read",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_END@.ci.yaml@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython3",
+ "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
+ "--yaml_file",
+ "[START_DIR]/mirrors/engine/.ci.yaml",
+ "--json_file",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "read ci yaml.parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"targets\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"drone_dimensions\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"os=Linux\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ ], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"linux one\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"$flutter/osx_sdk\": \"{\\\"sdk_version\\\": \\\"14a5294e\\\"}\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"release_build\": true@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"recipe\": \"engine/something\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"drone_dimensions\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"os=Linux\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ ], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"enabled_branches\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"beta\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"main\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ ], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"linux packaging one\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"$flutter/osx_sdk\": \"{\\\"sdk_version\\\": \\\"14a5294e\\\"}\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"recipe\": \"release/something\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"scheduler\": \"release\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "Identify branches"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git rev-parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "branch",
+ "-a",
+ "--contains",
+ "12345abcde12345abcde12345abcde12345abcde"
+ ],
+ "cwd": "[START_DIR]/mirrors/engine",
+ "infra_step": true,
+ "luci_context": {
+ "realm": {
+ "name": "prod:try"
+ },
+ "resultdb": {
+ "current_invocation": {
+ "name": "invocations/build:8945511751514863184",
+ "update_token": "token"
+ },
+ "hostname": "rdbhost"
+ }
+ },
+ "name": "Identify branches.git branch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "launch builds"
+ },
+ {
+ "cmd": [],
+ "name": "collect builds"
+ },
+ {
+ "cmd": [],
+ "name": "display builds"
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/release/release_builder.py b/recipes/release/release_builder.py
index 5cfbcc1..dfd4a9b 100644
--- a/recipes/release/release_builder.py
+++ b/recipes/release/release_builder.py
@@ -9,6 +9,7 @@
import json
+import re
from contextlib import contextmanager
from PB.recipes.flutter.release.release import InputProperties
@@ -19,16 +20,17 @@
from google.protobuf import struct_pb2
DEPS = [
- 'flutter/yaml',
- 'flutter/display_util',
- 'flutter/repo_util',
- 'flutter/shard_util_v2',
- 'recipe_engine/buildbucket',
- 'recipe_engine/json',
- 'recipe_engine/path',
- 'recipe_engine/platform',
- 'recipe_engine/properties',
- 'recipe_engine/step',
+ 'flutter/yaml',
+ 'flutter/display_util',
+ 'flutter/repo_util',
+ 'flutter/shard_util_v2',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/json',
+ 'recipe_engine/path',
+ 'recipe_engine/platform',
+ 'recipe_engine/properties',
+ 'recipe_engine/raw_io',
+ 'recipe_engine/step',
]
PROPERTIES = InputProperties
@@ -37,8 +39,19 @@
RELEASE_CHANNELS = ('refs/heads/beta', 'refs/heads/stable')
-def ShouldRun(api, git_ref, target):
+def ShouldRun(api, git_ref, target, release_branch):
"""Validates if a target should run based on platform, channel and repo."""
+ # Enabled for current branch
+ enabled_branches = target.get('enabled_branches', [])
+ if enabled_branches:
+ for r in enabled_branches:
+ # Enabled branches is a list of regex
+ if re.match(r, release_branch):
+ break
+ else:
+ # Current branch didn't match any of the enabled branches.
+ return False
+
release_build = target.get('properties', {}).get('release_build', False)
for_this_platform = target['name'].lower().startswith(api.platform.name)
# Postsubmit for engine and flutter repositories.
@@ -70,13 +83,18 @@
ci_yaml_path = checkout_path.join('.ci.yaml')
ci_yaml = api.yaml.read('read ci yaml', ci_yaml_path, api.json.output())
+ # Get release branch.
+ branches = api.repo_util.current_commit_branches(checkout_path)
+ branches = [b for b in branches if b.startswith('flutter')]
+ release_branch = branches[0] if branches else 'main'
+
# Foreach target defined in .ci.yaml, if it contains
# release_build: True, then spawn a subbuild.
tasks = {}
build_results = []
with api.step.nest('launch builds') as presentation:
for target in ci_yaml.json.output['targets']:
- if ShouldRun(api, git_ref, target):
+ if ShouldRun(api, git_ref, target, release_branch):
target = api.shard_util_v2.pre_process_properties(target)
tasks.update(api.shard_util_v2.schedule(
[target, ], presentation))
@@ -113,21 +131,40 @@
}
for git_ref in ['main', 'beta']:
yield api.test(
- 'basic_linux_%s' % git_ref,
- api.platform.name('linux'),
+ 'basic_linux_%s' % git_ref,
+ api.platform.name('linux'),
+ api.properties(environment='Staging', repository='engine'),
+ api.buildbucket.try_build(
+ project='prod',
+ builder='try-builder',
+ git_repo='https://flutter.googlesource.com/mirrors/engine',
+ revision='a' * 40,
+ build_number=123,
+ git_ref='refs/heads/%s' % git_ref,
+ ),
+ api.shard_util_v2.child_build_steps(
+ subbuilds=[try_subbuild1],
+ launch_step="launch builds",
+ collect_step="collect builds",
+ ),
+ api.step_data('read ci yaml.parse', api.json.output(tasks_dict)),
+ )
+
+ yield api.test(
+ 'filter_enabled_branches',
api.properties(environment='Staging', repository='engine'),
api.buildbucket.try_build(
- project='prod',
- builder='try-builder',
- git_repo='https://flutter.googlesource.com/mirrors/engine',
- revision='a' * 40,
- build_number=123,
- git_ref='refs/heads/%s' % git_ref,
+ project='prod',
+ builder='try-builder',
+ git_repo='https://flutter.googlesource.com/mirrors/engine',
+ revision='a' * 40,
+ build_number=123,
+ git_ref='refs/heads/%s' % git_ref,
),
- api.shard_util_v2.child_build_steps(
- subbuilds=[try_subbuild1],
- launch_step="launch builds",
- collect_step="collect builds",
+ api.step_data('read ci yaml.parse', api.json.output(tasks_dict)),
+ api.step_data(
+ 'Identify branches.git branch',
+ stdout=api.raw_io
+ .output_text('branch1\nbranch2\nflutter-3.2-candidate.5')
),
- api.step_data('read ci yaml.parse', api.json.output(tasks_dict))
)
