config/engine_config.star

#!/usr/bin/env lucicfg
# Copyright 2020 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
Configurations for the flutter engine repository.

The schedulers pull commits indirectly from GoB repo
(https://chromium.googlesource.com/external/github.com/flutter/engine)
which is mirrored from https://github.com/flutter/engine.

Try jobs use github directly but are also registered with luci-cq
to take advantage of led recipe builder tests.
"""

load("//lib/ci_yaml/ci_yaml.star", "ci_yaml")
load("//lib/common.star", "common")
load("//lib/release_branches/release_branches.star", "release_branches")
load("//lib/repos.star", "repos")

def _engine_properties(gcs_goldens_bucket = ""):
    properties = {
        "use_cas": True,
        "gcs_goldens_bucket": gcs_goldens_bucket,
    }
    return properties

def _setup():
    """Default configurations for branches and repos."""
    release_triggering_policy = scheduler.greedy_batching(
        max_batch_size = 1,
        max_concurrent_invocations = 3,
    )
    notifies = [
        luci.notifier(
            name = "engine_build_failure_notification",
            on_occurrence = ["INFRA_FAILURE", "FAILURE"],
            failed_step_regexp = "(^build|.*test:).*",
            notify_emails = ["flutter-engine@grotations.appspotmail.com"],
            template = luci.notifier_template(
                name = "test_build_failure",
                body = io.read_file("luci-notify/email-templates/test_build_failure.template"),
            ),
        ),
        luci.notifier(
            name = "engine_build_flake_notification",
            on_occurrence = ["SUCCESS"],
            failed_step_regexp = "(.*test:).*",
            notify_emails = ["flutter-engine@grotations.appspotmail.com"],
            template = luci.notifier_template(
                name = "test_flake",
                body = io.read_file("luci-notify/email-templates/test_flake.template"),
            ),
        ),
    ]
    ci_yaml.generate(
        "engine",
        "main",
        release_branches.main.version,
        release_branches.main.testing_ref,
        {},
        _engine_properties(),
        notifies = notifies,
        recipes_ref = release_branches.main.recipes_ref,
    )
    _drone_prod_builders("", release_branches.main)
    for branch_name in release_branches.fuchsia.keys():
        branch = release_branches.fuchsia[branch_name]
        ci_yaml.generate(
            "engine",
            branch_name,
            branch.version,
            branch.testing_ref,
            {},
            _engine_properties(),
            release_triggering_policy,
            recipes_ref = branch.recipes_ref,
        )

    _drone_v2_builders()
    _drone_try_builders()

def _drone_prod_builders(prefix, branch):
    """Generate drone builders to help with subshards."""
    recipes = ["engine/engine_builder", "engine/web_engine_drone"]
    for recipe in recipes:
        luci.recipe(
            name = "%s%s" % (prefix, recipe),
            cipd_package = "flutter/recipe_bundles/flutter.googlesource.com/recipes",
            cipd_version = branch.recipes_ref,
            use_bbagent = True,
        )
    ci_yaml_config = ci_yaml.ci_yaml("engine", "main")
    platform_properties = ci_yaml.platform_properties(ci_yaml_config)
    common.linux_prod_builder(
        name = "%sLinux Engine Drone" % prefix,
        recipe = "engine/engine_builder",
        console_view_name = None,
        priority = 30,
        caches = _caches(platform_properties, "linux"),
        os = platform_properties["linux"]["os"],
    )
    common.linux_prod_builder(
        name = "%sLinux Web Drone" % prefix,
        recipe = "engine/web_engine_drone",
        properties = _engine_properties(gcs_goldens_bucket = "flutter_logs"),
        console_view_name = None,
        priority = 28,
        caches = _caches(platform_properties, "linux"),
        os = platform_properties["linux"]["os"],
    )
    common.mac_prod_builder(
        name = "%sMac Engine Drone" % prefix,
        recipe = "engine/engine_builder",
        console_view_name = None,
        priority = 30,
        caches = _caches(platform_properties, "mac"),
        os = platform_properties["mac"]["os"],
    )
    common.windows_prod_builder(
        name = "%sWindows Engine Drone" % prefix,
        recipe = "engine/engine_builder",
        console_view_name = None,
        priority = 30,
        caches = _caches(platform_properties, "windows"),
        os = platform_properties["windows"]["os"],
    )

def _caches(platform_properties, platform, enforce_xcode_cache = False):
    properties = {k: v for k, v in platform_properties[platform].items()}
    osx_sdk = properties.get("$flutter/osx_sdk", {})
    dependencies = {}
    if "dependencies" in properties:
        dependencies = properties["dependencies"]
    swarming_caches = ci_yaml.swarming_caches("engine", "main", dependencies, osx_sdk)
    if enforce_xcode_cache:
        swarming_caches.append(swarming.cache(name = "flutter_xcode", path = "osx_sdk"))
    return swarming_caches

def _drone_v2_builders():
    """Generate drone builders to help with subshards."""
    engine_v2_builder_recipe_name = "engine_v2/builder"
    recipes = [engine_v2_builder_recipe_name]
    for recipe in recipes:
        luci.recipe(
            name = recipe,
            cipd_package = "flutter/recipe_bundles/flutter.googlesource.com/recipes",
            cipd_version = "refs/heads/main",
            use_bbagent = True,
        )
    env_to_bucket = {"Staging": "staging", "Production": "prod"}
    ci_yaml_config = ci_yaml.ci_yaml("engine", "main")
    platform_properties = ci_yaml.platform_properties(ci_yaml_config)
    for environment in ("Staging", "Production"):
        common.linux_prod_builder(
            name = "Linux %s Engine Drone|drn" % environment,
            recipe = engine_v2_builder_recipe_name,
            repo = repos.GIT_REMOTE["engine"],
            bucket = env_to_bucket.get(environment),
            pool = "luci.flutter.%s" % env_to_bucket.get(environment),
            console_view_name = None,
            priority = 30,
            caches = _caches(platform_properties, "linux", enforce_xcode_cache = True),
            os = platform_properties["linux"]["os"],
        )
        common.mac_prod_builder(
            name = "Mac %s Engine Drone|drn" % environment,
            recipe = engine_v2_builder_recipe_name,
            repo = repos.GIT_REMOTE["engine"],
            bucket = env_to_bucket.get(environment),
            pool = "luci.flutter.%s" % env_to_bucket.get(environment),
            console_view_name = None,
            priority = 30,
            caches = _caches(platform_properties, "mac"),
            os = platform_properties["mac"]["os"],
        )
        common.windows_prod_builder(
            name = "Windows %s Engine Drone|drn" % environment,
            recipe = engine_v2_builder_recipe_name,
            repo = repos.GIT_REMOTE["engine"],
            bucket = env_to_bucket.get(environment),
            pool = "luci.flutter.%s" % env_to_bucket.get(environment),
            console_view_name = None,
            priority = 30,
            caches = _caches(platform_properties, "windows"),
            os = platform_properties["windows"]["os"],
        )

    # Temporarily add a stagigng builder for legacy builders.
    # TODO(godofredoc): https://github.com/flutter/flutter/issues/125924
    common.linux_prod_builder(
        name = "Linux Engine Drone|drn",
        recipe = "engine/engine_builder",
        repo = repos.GIT_REMOTE["engine"],
        bucket = env_to_bucket.get("Staging"),
        pool = "luci.flutter.staging",
        console_view_name = None,
        priority = 30,
        caches = _caches(platform_properties, "linux", enforce_xcode_cache = True),
        os = platform_properties["linux"]["os"],
    )

def _drone_try_builders():
    """Generate drone builders to help with subshards."""
    list_view_name = "engine-try"
    ci_yaml_config = ci_yaml.ci_yaml("engine", "main")
    platform_properties = ci_yaml.platform_properties(ci_yaml_config)
    common.linux_try_builder(
        name = "Linux Engine Drone|drn",
        recipe = "engine/engine_builder",
        repo = repos.GIT_REMOTE["engine"],
        list_view_name = list_view_name,
        caches = _caches(platform_properties, "linux"),
        os = platform_properties["linux"]["os"],
    )
    common.linux_try_builder(
        name = "Linux Web Drone|webdrn",
        recipe = "engine/web_engine_drone",
        repo = repos.GIT_REMOTE["engine"],
        list_view_name = list_view_name,
        properties = _engine_properties(
            gcs_goldens_bucket = "flutter_logs",
        ),
        caches = _caches(platform_properties, "linux"),
        os = platform_properties["linux"]["os"],
    )
    common.mac_try_builder(
        name = "Mac Engine Drone|drn",
        recipe = "engine/engine_builder",
        repo = repos.GIT_REMOTE["engine"],
        list_view_name = list_view_name,
        caches = _caches(platform_properties, "mac"),
        os = platform_properties["mac"]["os"],
    )
    common.windows_try_builder(
        name = "Windows Engine Drone|drn",
        recipe = "engine/engine_builder",
        repo = repos.GIT_REMOTE["engine"],
        list_view_name = list_view_name,
        caches = _caches(platform_properties, "windows"),
        os = platform_properties["windows"]["os"],
    )

engine_config = struct(setup = _setup)