| # Copyright 2020 The Flutter Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """Utility methods to create builders.""" |
| |
| load("//lib/helpers.star", "helpers") |
| load("//lib/timeout.star", "timeout") |
| load("//lib/resultdb_config.star", "resultdb_config") |
| |
| # Regular expressions for files to skip CQ. |
| LOCATION_FILTER_MARKDOWN = cq.location_filter(path_regexp = r".*\.md", exclude = True) |
| LOCATION_FILTER_OWNERS = cq.location_filter(path_regexp = r"(.*/)?OWNERS", exclude = True) |
| |
| def _full_recipe_name(recipe_name, recipes_ref): |
| """Creates a recipe name for recipe and recipes_ref. |
| |
| Args: |
| recipe_name(str): This is a string with the recipe base name. |
| recipes_ref(str): The git ref pointing to the recipes bundle to use. |
| |
| Returns: |
| A string with the recipe's full name. |
| """ |
| ref = recipes_ref.replace("refs/heads/", "") |
| return "%s-%s" % (ref, recipe_name) |
| |
| def _repo_url_to_luci_object_name(repo_url): |
| """Takes a git repository URL and returns a name for a LUCI object. |
| |
| Examples: |
| https://foo.googlesource.com/bar/baz -> bar-baz |
| |
| Args: |
| repo_url(str): The repository url to transform. |
| |
| Returns: |
| A string with the repository name created from the url. |
| """ |
| domain_and_path = repo_url.split("://")[1].split("/") |
| path = domain_and_path[1:] |
| return "-".join(path).replace(".", "_") |
| |
| def _cq_group_name(repo_url): |
| """Returns the name to be passed to cq_group for a repo.""" |
| return _repo_url_to_luci_object_name(repo_url) |
| |
| def _ref_to_luci_object_name(ref): |
| """Takes a git ref and returns a name for a LUCI object. |
| |
| Strips out non-alphanumeric parts of the ref and replaces |
| slashes with dashes. |
| Examples: |
| refs/heads/master: refs-heads-master |
| refs/heads/sandbox/.+: refs-heads-sandbox |
| |
| Args: |
| ref(str): The git ref to transform. |
| |
| Returns: |
| A string with git ref parts joined with '-'. |
| """ |
| parts = ref.split("/") |
| char_parts = [] |
| for part in parts: |
| if part.isalnum(): |
| char_parts.append(part) |
| return "-".join(char_parts) |
| |
| def _cq_group(repo, tree_status_host = None): |
| luci.cq_group( |
| name = _cq_group_name(repo), |
| retry_config = cq.retry_config( |
| single_quota = 1, |
| global_quota = 2, |
| failure_weight = 2, |
| transient_failure_weight = 1, |
| timeout_weight = 1, |
| ), |
| tree_status_host = tree_status_host, |
| watch = cq.refset(repo, refs = ["refs/heads/.+"]), |
| ) |
| |
| def _poller_name(repo_url, poller_suffix, ref, path_regexps = None): |
| """Returns the name to be passed to gitiles_poller for a repo.""" |
| gitiles_poller_suffix = "-gitiles-trigger" |
| if path_regexps: |
| for regexp in path_regexps: |
| basename = regexp.split("/")[-1] |
| gitiles_poller_suffix = ("-" + basename.replace(".", "-") + |
| gitiles_poller_suffix) |
| if poller_suffix: |
| gitiles_poller_suffix = "-" + poller_suffix + gitiles_poller_suffix |
| if ref and ref != "refs/heads/master": |
| gitiles_poller_suffix += "-" + _ref_to_luci_object_name(ref) |
| return _repo_url_to_luci_object_name(repo_url) + gitiles_poller_suffix |
| |
| def _builder( |
| name, |
| builder_group, |
| executable, |
| properties, |
| execution_timeout, |
| caches = None, |
| console_category = None, |
| console_short_name = None, |
| cq_disable_reuse = False, |
| dimensions = None, |
| experiment_percentage = None, |
| experiments = None, |
| location_filters = None, |
| path_regexps = None, |
| notifies = None, |
| priority = None, |
| schedule = None, |
| disabled = False, |
| swarming_tags = None, |
| service_account = None, |
| triggered_by = None, |
| add_cq_tryjob_verifier = True, |
| cq_group = None, |
| resultdb_settings = None): |
| """Creates a builder, notifier, trigger, view entry. |
| |
| Args: |
| swarming_tags: Tags passed to luci build. |
| name: Passed through to luci.builder. |
| builder_group: struct from groups_lib. |
| executable: Passed through to luci builder. |
| properties: Passed through to luci builder, with some defaults applied. |
| execution_timeout: Passed through to luci builder. |
| caches: Passed through to luci builder. |
| Note, a number of global caches are configured by default: |
| https://chrome-internal.googlesource.com/infradata/config/+/master/configs/cr-buildbucket/settings.cfg |
| console_category: Passed through as category arg to luci.console_view_entry. |
| Must be set if console_short_name is set. |
| console_short_name: Passed through as short_name arg to luci.console_view_entry. |
| Must be set if console_category is set. |
| cq_disable_reuse: Passed through to luci.cq_tryjob_verifer. If true, |
| this builder will be triggered by every CQ run even if it already |
| passed on a previous recent CQ run on the same patchset. |
| dimensions: Passed through to luci.builder, with some defaults applied. |
| experiment_percentage: Passed through to luci.cq_tryjob_verifier. |
| experiments: Passed through to luci.builder. |
| location_filters: Passed through to luci.cq_tryjob_verifier. |
| path_regexps: Passed through to luci.gitiles_poller. |
| notifies: Passed through to luci.builder. |
| priority: Passed through to luci.builder, Overrides builder_group.priority. |
| schedule: Passed through to luci.builder. |
| disabled: If True, don't set up a schedule, gitiles_poller, or |
| cq_tryjob_verifier, but still create the builder so that it can |
| be triggered manually. |
| service_account: Passed through to luci.builder. |
| triggered_by: LUCI triggers that will start this builder. |
| add_cq_tryjob_verifier: If true a new tryjob verifier will be added. |
| cq_group: The luci.cq_group to be assigned to this builder. |
| resultdb_settings: Settings for enabling ResultDB builder integration. |
| """ |
| absolute_name = builder_group.bucket + "/" + name |
| is_try = builder_group.bucket.endswith("try") |
| final_properties = {} |
| |
| # "mastername" is the legacy property used to identify buildbot master, |
| # which is conveniently used by Chromium infrastructure products for data |
| # post processing such as Sheriff-o-Matic. |
| # |
| # We want to back-fill this information in order to surface the dashboard |
| # view group name (used by Milo) to BuildBucket (http://screen/1VJpRuGSf8D). |
| # This way we will be able to re-use Sheriff-o-Matic's build filter logic, |
| # that was originally created to filter buildbot. |
| if builder_group.views: |
| final_properties["mastername"] = ", ".join(builder_group.views) |
| final_properties.update(properties) |
| final_dimensions = { |
| "pool": builder_group.pool, |
| } |
| if dimensions: |
| final_dimensions.update(dimensions) |
| |
| resultdb_settings = resultdb_settings or resultdb_config.get(builder_group.bucket) |
| |
| luci.builder( |
| name = name, |
| bucket = builder_group.bucket, |
| caches = caches, |
| dimensions = final_dimensions, |
| executable = executable, |
| execution_timeout = execution_timeout, |
| experiments = experiments, |
| notifies = notifies, |
| priority = priority or builder_group.priority, |
| properties = final_properties, |
| triggering_policy = builder_group.triggering_policy, |
| triggered_by = triggered_by, |
| schedule = None if disabled else schedule, |
| task_template_canary_percentage = 0, |
| service_account = service_account or builder_group.account, |
| swarming_tags = swarming_tags, |
| resultdb_settings = resultdb_settings, |
| ) |
| triggering_repos = builder_group.triggering_repos |
| if disabled: |
| triggering_repos = [] |
| for repo in triggering_repos: |
| if is_try: |
| kwargs = { |
| "builder": absolute_name, |
| "cq_group": cq_group or _cq_group_name(repo), |
| "disable_reuse": cq_disable_reuse, |
| } |
| if experiment_percentage: |
| kwargs["experiment_percentage"] = experiment_percentage |
| if location_filters: |
| fail( |
| "location_filters cannot be used simultaneously with " + |
| "experiment_percentage", |
| ) |
| else: |
| kwargs["location_filters"] = location_filters |
| if add_cq_tryjob_verifier: |
| luci.cq_tryjob_verifier(**kwargs) |
| else: |
| for ref in builder_group.triggering_refs or ( |
| "refs/heads/master", |
| ): |
| luci.gitiles_poller( |
| name = _poller_name( |
| repo, |
| builder_group.poller_suffix, |
| ref, |
| path_regexps, |
| ), |
| bucket = builder_group.bucket, |
| path_regexps = path_regexps, |
| refs = [ref], |
| repo = repo, |
| triggers = [absolute_name], |
| ) |
| for view in builder_group.views: |
| if is_try or builder_group.bucket.endswith(("cron", "roller")): |
| luci.list_view_entry(builder = absolute_name, list_view = view) |
| elif console_short_name: |
| luci.console_view_entry( |
| builder = absolute_name, |
| category = console_category, |
| console_view = view, |
| short_name = console_short_name, |
| ) |
| |
| ############################# |
| def _flutter_builder( |
| bucket, |
| pool, |
| name, |
| recipe, |
| os, |
| properties = {}, |
| cores = None, |
| dimensions = None, |
| experiments = None, |
| **kwargs): |
| if dimensions: |
| dimensions = helpers.merge_dicts({ |
| "pool": pool, |
| "os": os, |
| }, dimensions) |
| else: |
| dimensions = { |
| "pool": pool, |
| "os": os, |
| } |
| if cores != None: |
| dimensions["cores"] = cores |
| name_parts = name.split("|") |
| exec_timeout = kwargs.pop("execution_timeout", None) |
| resultdb_settings = resultdb_config.get(bucket) if name_parts[0].startswith("Linux framework") else None |
| luci.builder( |
| name = name_parts[0], |
| bucket = bucket, |
| executable = recipe, |
| properties = properties, |
| service_account = "flutter-" + bucket + |
| "-builder@chops-service-accounts.iam.gserviceaccount.com", |
| execution_timeout = exec_timeout or timeout.MEDIUM, |
| experiments = experiments, |
| dimensions = dimensions, |
| build_numbers = True, |
| task_template_canary_percentage = 0, |
| resultdb_settings = resultdb_settings, |
| **kwargs |
| ) |
| |
| def _try_builder( |
| name, |
| list_view_name, |
| properties = {}, |
| pool = "luci.flutter.try", |
| **kwargs): |
| bucket = "try" |
| merged_properties = helpers.merge_dicts(properties, { |
| "gold_tryjob": True, |
| }) |
| name_parts = name.split("|") |
| |
| luci.list_view_entry( |
| builder = bucket + "/" + name_parts[0], |
| list_view = list_view_name, |
| ) |
| experiments = {"luci.recipes.use_python3": 100} |
| return _flutter_builder( |
| bucket, |
| pool, |
| name, |
| properties = merged_properties, |
| experiments = experiments, |
| **kwargs |
| ) |
| |
| def _prod_builder( |
| name, |
| console_view_name, |
| category, |
| properties = {}, |
| notifies = None, |
| **kwargs): |
| merged_properties = helpers.merge_dicts(properties, { |
| "upload_packages": True, |
| "gold_tryjob": False, |
| }) |
| bucket = kwargs.pop("bucket", "prod") |
| pool = kwargs.pop("pool", "luci.flutter.prod") |
| name_parts = name.split("|") |
| |
| # Handle case where gerrit builders are one word |
| short_name = name_parts[0] |
| if len(name_parts) > 1: |
| short_name = name_parts[1] |
| |
| if console_view_name: |
| luci.console_view_entry( |
| builder = bucket + "/" + name_parts[0], |
| console_view = console_view_name, |
| category = category, |
| short_name = short_name, |
| ) |
| |
| experiments = {"luci.recipes.use_python3": 100} |
| return _flutter_builder( |
| bucket, |
| pool, |
| name, |
| properties = merged_properties, |
| experiments = experiments, |
| notifies = notifies, |
| **kwargs |
| ) |
| |
| def _common_builder(**common_kwargs): |
| def prod_job(*args, **kwargs): |
| return _prod_builder( |
| *args, |
| **helpers.merge_dicts(common_kwargs, kwargs) |
| ) |
| |
| def try_job(*args, **kwargs): |
| cq_args = {} |
| cq_args["builder"] = "try/%s" % kwargs["name"].split("|")[0] |
| cq_args["cq_group"] = _cq_group_name(kwargs["repo"]) |
| new_common_kwargs = {k: v for k, v in common_kwargs.items() if k not in ["category"]} |
| if kwargs.get("add_cq"): |
| kwargs.pop("add_cq") |
| luci.cq_tryjob_verifier(**cq_args) |
| return _try_builder( |
| *args, |
| **helpers.merge_dicts(new_common_kwargs, kwargs) |
| ) |
| |
| return try_job, prod_job |
| |
| def _mac_builder(properties = {}, category = "Mac", os = None, **kwargs): |
| return _common_builder( |
| os = os, |
| properties = properties, |
| category = category, |
| **kwargs |
| ) |
| |
| def _linux_builder( |
| properties = {}, |
| category = "Linux", |
| os = None, |
| **kwargs): |
| return _common_builder( |
| os = os, |
| properties = properties, |
| category = category, |
| **kwargs |
| ) |
| |
| def _windows_builder( |
| properties = {}, |
| category = "Windows", |
| os = None, |
| **kwargs): |
| return _common_builder( |
| os = os, |
| properties = properties, |
| category = category, |
| **kwargs |
| ) |
| |
| _common_try_builder, _common_prod_builder = _common_builder() |
| _linux_try_builder, _linux_prod_builder = _linux_builder() |
| _mac_try_builder, _mac_prod_builder = _mac_builder() |
| _windows_try_builder, _windows_prod_builder = _windows_builder() |
| |
| def _short_name(task_name): |
| """Create a short name for task name.""" |
| task_name = task_name.replace("__", "_") |
| task_name = task_name.replace(" ", "_") |
| words = task_name.split("_") |
| return "".join([w[0] for w in words])[:5].lower() |
| |
| def _builder_with_subshards( |
| properties, |
| bucket, |
| os, |
| branch_name, |
| **kwargs): |
| """Create separate builder config for any subshard in properties["subshards"]. |
| |
| Args: |
| bucket(str): The pool to run builder: "try" or "prod". |
| os(str): The host os to run builder. |
| properties(dict): Properties passed through to luci builder. |
| branch_name(str): The branch for prod builders. |
| **kwargs: Other kwargs, like repo, caches, recipe, etc. |
| |
| Returns: |
| A list of builder configs for all subshards. |
| """ |
| |
| for subshard in properties.pop("subshards"): |
| buildername = "%s_%s" % (properties["shard"], subshard) |
| properties["subshard"] = subshard |
| if bucket == "try" and os.startswith("Linux"): |
| _linux_try_builder( |
| name = "Linux %s|%s" % (buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| elif bucket == "try" and os.startswith("Mac"): |
| _mac_try_builder( |
| name = "Mac %s|%s" % (buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| elif bucket == "try" and os.startswith("Windows"): |
| _windows_try_builder( |
| name = "Windows %s|%s" % (buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| elif bucket == "prod" and os.startswith("Linux"): |
| _linux_prod_builder( |
| name = "Linux%s %s|%s" % (branch_name, buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| elif bucket == "prod" and os.startswith("Mac"): |
| _mac_prod_builder( |
| name = "Mac%s %s|%s" % (branch_name, buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| elif bucket == "prod" and os.startswith("Windows"): |
| _windows_prod_builder( |
| name = "Windows%s %s|%s" % (branch_name, buildername, _short_name(buildername)), |
| properties = properties, |
| os = os, |
| **kwargs |
| ) |
| |
| common = struct( |
| builder = _builder, |
| LOCATION_FILTER_MARKDOWN = LOCATION_FILTER_MARKDOWN, |
| LOCATION_FILTER_OWNERS = LOCATION_FILTER_OWNERS, |
| common_prod_builder = _common_prod_builder, |
| common_try_builder = _common_try_builder, |
| cq_group = _cq_group, |
| cq_group_name = _cq_group_name, |
| poller_name = _poller_name, |
| TARGET_X64 = "x64", |
| builder_with_subshards = _builder_with_subshards, |
| linux_try_builder = _linux_try_builder, |
| linux_prod_builder = _linux_prod_builder, |
| mac_try_builder = _mac_try_builder, |
| mac_prod_builder = _mac_prod_builder, |
| short_name = _short_name, |
| windows_try_builder = _windows_try_builder, |
| windows_prod_builder = _windows_prod_builder, |
| try_builder = _try_builder, |
| full_recipe_name = _full_recipe_name, |
| ) |
