recipes/release/release_builder.py

# Copyright 2022 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Orchestrator recipe that runs subbuilds required to release engine.
#
# This recipe reads <engine_checkout>/.ci_yaml, and for every target
# marked with release_build: true, and spawens a subbuild.

import re
from contextlib import contextmanager

from PB.recipes.flutter.release.release import InputProperties
from PB.recipes.flutter.release.release import EnvProperties

from RECIPE_MODULES.flutter.repo_util.api import REPOS

from google.protobuf import struct_pb2

import json

DEPS = [
    'flutter/display_util',
    'flutter/flutter_bcid',
    'flutter/os_utils',
    'flutter/repo_util',
    'flutter/shard_util_v2',
    'flutter/yaml',
    'recipe_engine/buildbucket',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/runtime',
    'recipe_engine/step',
]

PROPERTIES = InputProperties
ENV_PROPERTIES = EnvProperties

RELEASE_CHANNELS = ('refs/heads/beta', 'refs/heads/stable')


def ShouldRun(api, git_ref, target, release_branch, retry_override_list):
  """Validates if a target should run based on platform, channel and repo."""

  # If retry_override_list list of targets to retry has been provided,
  # skip the target if not specified on list.
  config_name = target.get('properties', {}).get('config_name', False)
  should_skip = retry_override_list and config_name not in retry_override_list
  if should_skip:
    # A list of targets to retry was provided
    # and this target was not on the list.
    return False

  # Enabled for current branch
  enabled_branches = target.get('enabled_branches', [])
  if enabled_branches and target.get('scheduler') != 'release':
    for r in enabled_branches:
      # Enabled branches is a list of regex
      if re.match(r, release_branch):
        break
    else:
      # Current branch didn't match any of the enabled branches.
      return False

  release_build = target.get('properties', {}).get('release_build', False)
  for_this_platform = target['name'].lower().startswith(api.platform.name)
  # Postsubmit for engine and flutter repositories.
  if (release_build and for_this_platform and
      (git_ref not in RELEASE_CHANNELS)):
    return True
  # Packaging for the flutter repository.
  if (target.get('scheduler') == 'release' and for_this_platform and
      (git_ref in RELEASE_CHANNELS) and
      git_ref.replace('refs/heads/', '') in target.get('enabled_branches', [])):
    return True
  return False


def RunSteps(api, properties, env_properties):
  api.os_utils.collect_os_info()
  repository = api.properties.get(
      'git_repo'
  ) or api.buildbucket.gitiles_commit.project
  repository_parts = repository.split('/')
  checkout_path = api.path['start_dir'].join(*repository_parts)
  git_ref = api.properties.get('git_ref') or api.buildbucket.gitiles_commit.ref
  git_url = api.properties.get('git_url') or REPOS[repository]
  api.repo_util.checkout(
      repository, checkout_path=checkout_path, url=git_url, ref=git_ref
  )

  # retry_override_list is optional and is a space separated string of
  # the config_name of targets to explitly retry
  retry_override_list = api.properties.get('retry_override_list', '').split()

  ci_yaml_path = checkout_path.join('.ci.yaml')
  ci_yaml = api.yaml.read('read ci yaml', ci_yaml_path, api.json.output())

  # Get release branch.
  branches = api.repo_util.current_commit_branches(checkout_path)
  branches = [b for b in branches if b.startswith('flutter')]
  release_branch = branches[0] if branches else 'main'

  # Foreach target defined in .ci.yaml, if it contains
  # release_build: True, then spawn a subbuild.
  tasks = {}
  build_results = []
  with api.step.nest('launch builds') as presentation:
    for target in ci_yaml.json.output['targets']:
      if ShouldRun(api, git_ref, target, release_branch, retry_override_list):
        target = api.shard_util_v2.pre_process_properties(target)
        tasks.update(
            api.shard_util_v2.schedule([
                target,
            ],
                                       presentation,
                                       branch=release_branch)
        )
  with api.step.nest('collect builds') as presentation:
    build_results = api.shard_util_v2.collect(tasks)

  api.display_util.display_subbuilds(
      step_name='display builds',
      subbuilds=build_results,
      raise_on_failure=True,
  )


def GenTests(api):
  try_subbuild1 = api.shard_util_v2.try_build_message(
      build_id=8945511751514863186,
      builder="builder-subbuild1",
      output_props={"test_orchestration_inputs_hash": "abc"},
      status="SUCCESS",
  )
  tasks_dict = {
      'targets': [{
          'name': 'linux one', 'recipe': 'engine/something', 'properties': {
              'release_build': True,
              '$flutter/osx_sdk': '{"sdk_version": "14a5294e"}'
          }, 'drone_dimensions': ['os=Linux']
      }, {
          'name': 'linux packaging one', 'recipe': 'release/something',
          'scheduler': 'release',
          'properties': {'$flutter/osx_sdk': '{"sdk_version": "14a5294e"}'},
          'enabled_branches': ['beta',
                               'main'], 'drone_dimensions': ['os=Linux']
      }]
  }
  for git_ref in ['main', 'beta']:
    yield api.test(
        'basic_linux_%s' % git_ref,
        api.platform.name('linux'),
        api.properties(environment='Staging', repository='engine'),
        api.buildbucket.try_build(
            project='prod',
            builder='try-builder',
            git_repo='https://flutter.googlesource.com/mirrors/engine',
            revision='a' * 40,
            build_number=123,
            git_ref='refs/heads/%s' % git_ref,
        ),
        api.shard_util_v2.child_build_steps(
            subbuilds=[try_subbuild1],
            launch_step="launch builds.schedule",
            collect_step="collect builds",
        ),
        api.step_data('read ci yaml.parse', api.json.output(tasks_dict)),
    )

  tasks_dict_scheduler = {
      'targets': [
          {
              'name': 'linux packaging one',
              'recipe': 'release/something',
              #'scheduler': 'release',
              'properties': {'$flutter/osx_sdk': '{"sdk_version": "14a5294e"}'},
              'enabled_branches': ['flutter-3.2-candidate.5'],
              'drone_dimensions': ['os=Linux']
          },
          {
              'name': 'linux packaging two',
              'recipe': 'release/something',
              #'scheduler': 'release',
              'properties': {'$flutter/osx_sdk': '{"sdk_version": "14a5294e"}'},
              'enabled_branches': ['beta', 'main'],
              'drone_dimensions': ['os=Linux']
          }
      ]
  }
  yield api.test(
      'filter_enabled_branches',
      api.properties(environment='Staging', repository='engine'),
      api.buildbucket.try_build(
          project='prod',
          builder='try-builder',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          revision='a' * 40,
          build_number=123,
          git_ref='refs/heads/%s' % git_ref,
      ),
      api.step_data(
          'read ci yaml.parse', api.json.output(tasks_dict_scheduler)
      ),
      api.step_data(
          'Identify branches.git branch',
          stdout=api.raw_io
          .output_text('branch1\nbranch2\nflutter-3.2-candidate.5')
      ),
  )
  for retry_list in ['skip_target', 'skip_target linux_target']:
    yield api.test(
        'retry_override_%s' % retry_list,
        api.properties(
            environment='Staging',
            repository='engine',
            retry_override_list=retry_list
        ),
        api.buildbucket.try_build(
            project='prod',
            builder='try-builder',
            git_repo='https://flutter.googlesource.com/mirrors/engine',
            revision='a' * 40,
            build_number=123,
            git_ref='refs/heads/%s' % git_ref
        ),
        api.step_data(
            'read ci yaml.parse', api.json.output(tasks_dict_scheduler)
        ),
    )

  yield api.test(
      'dart_internal',
      api.properties(environment='Staging', repository='engine'),
      api.buildbucket.try_build(
          project='dart-internal',
          bucket='flutter',
          builder='prod-builder',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          revision='a' * 40,
          build_number=123,
          git_ref='refs/heads/%s' % git_ref,
      ),
      api.step_data(
          'read ci yaml.parse', api.json.output(tasks_dict_scheduler)
      ),
      api.step_data(
          'Identify branches.git branch',
          stdout=api.raw_io
          .output_text('branch1\nbranch2\nflutter-3.2-candidate.5')
      ),
  )