| #!/usr/bin/env lucicfg |
| # Copyright 2020 The Flutter Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """ |
| Utility for loading Flutter infrastructure's yaml config into LUCI. |
| |
| See https://github.com/flutter/cocoon/blob/main/CI_YAML.md for more information. |
| """ |
| |
| proto.new_descriptor_set( |
| name = "scheduler", |
| blob = io.read_file("./scheduler.bin"), |
| ).register() |
| |
| load("@proto//scheduler.proto", _scheduler_pb = "scheduler") |
| load("//lib/common.star", "common") |
| load("//lib/copy.star", "copy") |
| load("//lib/helpers.star", "helpers") |
| load("//lib/repos.star", "repos") |
| |
| def _ci_yaml(repo, branch): |
| """Creates ci_yaml object. |
| |
| ci_yamls are expected to have a generated jspb file under |
| //config/generated/ci_yaml with the format `$repo_config.json` for tip of |
| tree or `$repo_$branch_config.json` for release branches. |
| |
| ci_yaml json files are autogenerated based on the .ci.yaml in $repo. |
| |
| Args: |
| repo(str): Name of the repository to follow. |
| branch(str): Branch name. |
| """ |
| version = "" if _is_default_branch(branch) else "_%s" % branch |
| return proto.from_jsonpb( |
| _scheduler_pb.SchedulerConfig, |
| io.read_file("./generated/ci_yaml/%s%s_config.json" % (repo, version)), |
| ) |
| |
| # lucicfg does not currently support enums from protos. |
| # To work around this, statically redeclare the values from: |
| # //config/lib/ci_yaml/scheduler.proto - SchedulerSytem |
| SCHEDULER_COCOON = 1 |
| SCHEDULER_LUCI = 2 |
| |
| # lucicfg is only intended to generate LUCI based targets. |
| SUPPORTED_SCHEDULERS = [SCHEDULER_COCOON, SCHEDULER_LUCI] |
| |
| def _recipes(repo, branch, version, recipes_ref): |
| """Return the set of recipes specified in ci.yaml.""" |
| recipe_names = {} |
| ci_yaml = _ci_yaml(repo, branch) |
| for target in ci_yaml.targets: |
| # Only LUCI based targets have recipes |
| if target.scheduler not in SUPPORTED_SCHEDULERS: |
| continue |
| if target.enabled_branches and branch not in target.enabled_branches: |
| continue |
| recipe_name = _full_recipe_name(target.recipe, version, recipes_ref) |
| if recipe_name not in recipe_names.keys(): |
| recipe_names[recipe_name] = target.recipe |
| for recipe_name, recipe in recipe_names.items(): |
| luci.recipe( |
| name = recipe_name, |
| recipe = recipe, |
| cipd_package = "flutter/recipe_bundles/flutter.googlesource.com/recipes", |
| cipd_version = recipes_ref, |
| use_bbagent = True, |
| ) |
| |
| def _add_recipes_cq(target): |
| if "add_recipes_cq" in target.properties: |
| return target.properties["add_recipes_cq"] == "true" |
| return False |
| |
| def _swarming_caches(ci_yaml, target, platform): |
| """Return the equivalent swarming caches for target. |
| |
| Targets inherit from their platform_properties and can |
| specify their own caches. |
| """ |
| swarming_caches = [] |
| caches = [] |
| cache_set = [] |
| |
| # Target specific caches |
| if target and "caches" in target.properties: |
| caches = json.decode(target.properties["caches"]) |
| |
| # Mark all in unique set to ensure platform does not override |
| for cache in caches: |
| cache_set.append(cache["name"]) |
| |
| # Platform wide caches |
| if "caches" in ci_yaml.platform_properties[platform].properties: |
| platform_caches = json.decode(ci_yaml.platform_properties[platform].properties["caches"]) |
| |
| # Ensure platform wide caches do not override target level caches |
| for platform_cache in platform_caches: |
| if platform_cache["name"] not in cache_set: |
| caches.append(platform_cache) |
| |
| # Generate LUCI config caches |
| for cache in caches: |
| swarming_caches.append(swarming.cache(name = cache["name"], path = cache["path"])) |
| return swarming_caches |
| |
| def _platform_properties(ci_yaml): |
| """Gets platform_properties from ci_yaml.""" |
| platform_properties = {} |
| for platform, map in dict(ci_yaml.platform_properties).items(): |
| platform_properties[platform] = {} |
| for k, v in dict(ci_yaml.platform_properties[platform].properties).items(): |
| if v == "true": |
| platform_properties[platform][k] = True |
| elif v == "false": |
| platform_properties[platform][k] = False |
| elif v.startswith("["): |
| platform_properties[platform][k] = json.decode(v) |
| else: |
| platform_properties[platform][k] = v |
| return platform_properties |
| |
| def _properties(ci_yaml, target, default_properties = {}, repo = None, branch = None): |
| """Creates builder properties based on ci_yaml. |
| |
| Args: |
| ci_yaml(SchedulerConfig): Config file that contains target. |
| target(Target): Configuration to generate properties for. |
| default_properties(dict): Repo level properties. |
| repo(string): Name of the repo. |
| branch(string): Name of the branch. |
| """ |
| properties = {} |
| |
| # Initialize with default repo properties |
| for k, v in default_properties.items(): |
| properties[k] = v |
| |
| # Update with platform specific properties |
| platform = _target_platform(target) |
| for k, v in _platform_properties(ci_yaml)[platform].items(): |
| properties[k] = v |
| |
| # Finally, update with target specific properties |
| json_list = ["android_sdk_license", "android_sdk_preview_license"] |
| for k, v in dict(target.properties).items(): |
| if k == "dependencies": |
| # Merge target dependencies with platform |
| dependencies = json.decode(v) |
| target_dependency_lookup_table = [] |
| for dep in dependencies: |
| target_dependency_lookup_table.append(dep["dependency"]) |
| |
| # Add platform dependencies that are not already defined |
| for dep in properties[k]: |
| if dep["dependency"] not in target_dependency_lookup_table: |
| dependencies.append(dep) |
| properties[k] = dependencies |
| elif v == "true": |
| properties[k] = True |
| elif v == "false": |
| properties[k] = False |
| elif k in json_list: |
| # Yaml -> JSON -> Here adds an extra string escape that we unescape here |
| properties[k] = v.replace("\\n", "\n") |
| elif v.startswith("["): |
| properties[k] = json.decode(v) |
| elif v.isdigit(): |
| properties[k] = int(v) |
| else: |
| properties[k] = v |
| |
| # Special case to pass xcode for engine |
| for xcode_platform in ["mac", "mac_ios", "mac_ios32"]: |
| if platform != xcode_platform: |
| continue |
| if "xcode" not in properties: |
| continue |
| xcode_version = { |
| "sdk_version": properties["xcode"], |
| } |
| |
| # DeviceLab bots are on a slower lab network. We avoid LUCI caches on those bots |
| if platform in ["mac_ios", "mac_ios32"]: |
| properties["$flutter/devicelab_osx_sdk"] = xcode_version |
| else: |
| properties["$flutter/osx_sdk"] = xcode_version |
| |
| # Special case to add runtimes for ios simultators |
| if platform == "mac" and "runtime_versions" in properties: |
| # Add to existing osx_sdk properties dict, or create new |
| osx_sdk = properties.get("$flutter/osx_sdk", {}) |
| osx_sdk["runtime_versions"] = properties["runtime_versions"] |
| properties["$flutter/osx_sdk"] = osx_sdk |
| |
| filter_list = ["caches"] |
| for filter in filter_list: |
| if filter in properties: |
| properties.pop(filter) |
| if repo: |
| properties["git_repo"] = repo |
| if branch: |
| properties["git_branch"] = branch |
| |
| # Add `bringup` to properties. |
| properties["bringup"] = target.bringup |
| |
| return properties |
| |
| def _full_recipe_name(recipe_name, version, recipes_ref): |
| """Creates a recipe name for recipe and version. |
| |
| Args: |
| recipe_name(str): This is a string with the recipe base name. |
| version(str): A string with the build version. E.g. dev, beta, stable. |
| recipes_ref(str): The git ref pointing to the recipes bundle to use. |
| |
| Returns: |
| A string with the recipe's full name. |
| """ |
| ref = recipes_ref.replace("refs/heads/", "") |
| return "%s-%s" % (ref, recipe_name) |
| |
| def _builder_name(builder_name, branch): |
| """Creates a builder name that is versioned to branch. |
| |
| Args: |
| builder_name(str): The original builder name. |
| branch(str): Branch this builder is for. |
| """ |
| if not _is_default_branch(branch): |
| parts = builder_name.split(" ") |
| parts.insert(1, branch) |
| builder_name = " ".join(parts) |
| return "%s|%s" % (builder_name, common.short_name(builder_name)) |
| |
| def _is_default_branch(branch): |
| """Returns whether the branch is a default branch.""" |
| return branch in ("main", "master") |
| |
| def _builders(repo, branch, version, testing_ref, dimensions, properties, triggering_policy, notifies = None, recipes_ref = "refs/heads/main"): |
| """Return the builders specified in ci.yaml. |
| |
| Targets defined in .ci.yaml are translated to their LUCI equivalents. |
| Mostly, this is forward fields to the LUCI equivalent, but there |
| are some fields that are special cased. |
| |
| Args: |
| repo(str): Name of the repo to create builders for. |
| branch(str): Name of the branch to create builders for. Either None:dev|beta|stable. |
| version(str): The branch number, following major_minor. |
| testing_ref(str): The git ref to trigger builds on. |
| dimensions(dict): Platform dimensions to pass. |
| properties(dict): Repo level properties to pass to every builder. |
| triggering_policy(scheduler): Scheduler policy to implement on postsubmit builds. |
| notifies(list): List of luci.notifier to send notifications on events from builders. |
| recipes_ref(str): The git ref pointing to the recipes bundle to use. |
| """ |
| ci_yaml = _ci_yaml(repo, branch) |
| |
| # Only generate presubmit and staging builds on tip of tree |
| staging_view_name = "%s_staging" % repo |
| if _is_default_branch(branch): |
| list_view_name = "%s-try" % repo |
| luci.list_view( |
| name = list_view_name, |
| title = "%s try builders" % repo.capitalize(), |
| ) |
| luci.console_view( |
| name = staging_view_name, |
| repo = repos.GIT_REMOTE[repo], |
| refs = [testing_ref], |
| ) |
| console_view_name = repo if _is_default_branch(branch) else "%s_%s" % (branch, repo) |
| luci.console_view( |
| name = console_view_name, |
| repo = repos.GIT_REMOTE[repo], |
| refs = [testing_ref], |
| ) |
| |
| # Defines prod schedulers |
| trigger_name = "%s-gitiles-trigger-%s" % (branch, repo) |
| luci.gitiles_poller( |
| name = trigger_name, |
| bucket = "prod", |
| repo = repos.GIT_REMOTE[repo], |
| refs = [testing_ref], |
| ) |
| |
| # Defines default triggering policy |
| if not triggering_policy: |
| triggering_policy = scheduler.greedy_batching( |
| max_batch_size = 1, |
| max_concurrent_invocations = 3, |
| ) |
| |
| for target in ci_yaml.targets: |
| # Not all targets in ci.yaml are LUCI based, skip those. |
| if target.scheduler not in SUPPORTED_SCHEDULERS: |
| continue |
| |
| # Flaky tests should not be on releases |
| if not _is_default_branch(branch) and target.bringup: |
| continue |
| platform = _target_platform(target) |
| merged_properties = _properties(ci_yaml, target, properties, repo, branch) |
| target_args = { |
| "caches": _swarming_caches(ci_yaml, target, platform), |
| "category": platform.capitalize(), |
| "dimensions": {}, |
| # On release branches, extend the timeout to handle goma misses. |
| # TODO(godofredoc): Remove after hotfix 2.5.3 is released. |
| "execution_timeout": target.timeout * time.minute * 3, |
| "name": _builder_name(target.name, branch), |
| "os": merged_properties["os"], |
| "recipe": _full_recipe_name(target.recipe, version, recipes_ref), |
| "repo": repos.GIT_REMOTE[repo], |
| "properties": merged_properties, |
| } |
| |
| # Set dimensions dimensions from properties. |
| if platform in dimensions: |
| target_args["dimensions"] = dimensions[platform] |
| dimension_key_list = ("device_type", "device_os", "mac_model", "cores") |
| for dimension_key in dimension_key_list: |
| if dimension_key in merged_properties: |
| target_args["dimensions"][dimension_key] = str(merged_properties[dimension_key]) |
| |
| # Try builds are generated for every target to enable led and presubmit runs |
| if _is_default_branch(branch): |
| presubmit_target_args = copy.deepcopy(target_args) |
| presubmit_target_args["list_view_name"] = list_view_name |
| if repo == "engine": |
| presubmit_target_args["properties"]["no_lto"] = True |
| if _add_recipes_cq(target): |
| presubmit_target_args["add_cq"] = True |
| common.common_try_builder(**presubmit_target_args) |
| |
| # Postsubmit only generates if the target is enabled for this branch, which defaults to true. |
| if target.postsubmit and (not target.enabled_branches or branch in target.enabled_branches): |
| # Set customized triggering policy for engine staging builders. |
| # https://github.com/flutter/flutter/issues/92947 |
| if target.bringup: |
| final_triggering_policy = scheduler.greedy_batching( |
| max_batch_size = 20, |
| max_concurrent_invocations = 1, |
| ) |
| else: |
| final_triggering_policy = triggering_policy |
| postsubmit_target_args = helpers.merge_dicts({ |
| "bucket": "staging" if target.bringup else "prod", |
| "console_view_name": staging_view_name if target.bringup else console_view_name, |
| "expiration_timeout": 24 * time.hour if target.bringup else None, |
| "pool": "luci.flutter.staging" if target.bringup else "luci.flutter.prod", |
| "priority": 30, |
| # Only LUCI targets need triggers to the LUCI scheduler. |
| "triggered_by": [trigger_name] if target.scheduler == SCHEDULER_LUCI else None, |
| "triggering_policy": final_triggering_policy if target.scheduler == SCHEDULER_LUCI else None, |
| "notifies": notifies, |
| }, target_args) |
| |
| # TODO(chillers): Remove when postsubmit properties are supported. https://github.com/flutter/flutter/issues/87675 |
| if target.recipe == "devicelab/devicelab_drone": |
| if _is_default_branch(branch): |
| postsubmit_target_args["properties"]["upload_metrics"] = True |
| |
| # Set a 2h expiration_timeout for non flaky devicelab builders. |
| if not target.bringup: |
| postsubmit_target_args["expiration_timeout"] = 120 * time.minute |
| common.common_prod_builder(**postsubmit_target_args) |
| |
| def _target_platform(target): |
| """Return the target platform based on the name.""" |
| platform = target.name.split(" ")[0] |
| return platform.lower() |
| |
| def _generate(repo, branch, version, testing_ref, dimensions = {}, properties = {}, triggering_policy = None, notifies = None, recipes_ref = "refs/heads/main"): |
| # CQ group configurations. Only FLUTTER_RECIPES is using |
| # LUCI CQ but we still need the CQ configurations for all |
| # the try configurations for led recipe tests. |
| if _is_default_branch(branch): |
| common.cq_group(repos.GIT_REMOTE[repo]) |
| _recipes(repo, branch, version, recipes_ref) |
| _builders(repo, branch, version, testing_ref, dimensions, properties, triggering_policy, notifies = notifies, recipes_ref = recipes_ref) |
| |
| ci_yaml = struct( |
| builder_name = _builder_name, |
| ci_yaml = _ci_yaml, |
| full_recipe_name = _full_recipe_name, |
| generate = _generate, |
| platform_properties = _platform_properties, |
| swarming_caches = _swarming_caches, |
| ) |