blob: b41f6a7358af257b3948842ad8a07f306fbf0857 [file] [log] [blame] [edit]
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from recipe_engine.recipe_api import Property
from PB.go.chromium.org.luci.led.job import job as job_pb2
from RECIPE_MODULES.flutter.swarming_retry import api as swarming_retry_api
DEPS = [
"flutter/swarming_retry",
"recipe_engine/buildbucket",
"recipe_engine/led",
"recipe_engine/properties",
"recipe_engine/step",
]
PROPERTIES = {
"full":
Property(
kind=bool,
default=False,
help="Whether to run six tasks or just one.",
),
"run_count":
Property(
kind=int,
default=1,
help="Number of successful runs we want for each task.",
),
"task_type":
Property(
kind=str,
default="test",
help="Type of tasks to create. Options: "
'"test", "internal_failure", "raising", "led", "triggered".',
),
"max_attempts":
Property(kind=int, default=2, help="Overall max attempts."),
"last_task_max_attempts":
Property(
kind=int,
default=None,
help="Override the overall max attempts by setting on "
"Task.max_attempts. Only set on last task.",
),
"abort_early":
Property(
kind=bool,
default=False,
help="Whether to run a task that will fail and abort early",
),
}
class Task(swarming_retry_api.Task):
"""Required subclass for testing swarming_retry.
Defined inside a function because base class is inside api object.
"""
def __init__(self, initial_task_id, *args, **kwargs):
"""Construct a Task object.
Args:
initial_task_id (int or str): integer decimal value (since this needs
to be incremented but is then used as a str later this method
accepts both int and str types to minimize confusion, so long as
int(initial_task_id) works)
"""
abort_early = kwargs.pop("abort_early", False)
super().__init__(*args, **kwargs)
self._next_task_id = int(initial_task_id)
self.abort_early_if_failed = abort_early
def launch(self, _):
kwargs = {
"task_id": str(self._next_task_id),
}
self._next_task_id += 1
# This looks funny but it's needed to ensure coverage of
# Attempt.task_ui_link.
if self._next_task_id % 2 == 0:
kwargs["host"] = "testhost"
else:
kwargs["task_ui_link"] = f"https://testhost/task?id={kwargs['task_id']}"
attempt = self._api.swarming_retry.Attempt(**kwargs)
self._api.step.empty(f"launch {self.name}", step_text=attempt.task_id)
return attempt
class FlakeTask(Task):
def process_result(self, attempt):
attempt.has_flakes = True
class InternalFailureTask(Task):
def process_result(self, attempt):
attempt.failure_reason = "internal failure"
class RaisingTask(Task):
def process_result(self, _):
self._api.step.empty("failing step", status=self._api.step.FAILURE)
class LedTask(swarming_retry_api.LedTask):
def __init__(self, initial_task_id, api, **kwargs):
ir = api.led("get-builder", "project/bucket:builder")
build_proto = ir.result.buildbucket.bbagent_args.build
build_proto.id = int(initial_task_id)
build_proto.infra.swarming.task_id = str(initial_task_id)
super().__init__(ir, api=api, **kwargs)
def launch(self, priority_boost_amount):
ret = super().launch(priority_boost_amount)
build_proto = self._led_data.result.buildbucket.bbagent_args.build
cur_id = int(build_proto.infra.swarming.task_id)
build_proto.infra.swarming.task_id = str(cur_id + 1)
build_proto.id = cur_id + 1
return ret
# pylint: disable=invalid-name
def RunSteps(
api,
full,
task_type,
max_attempts,
last_task_max_attempts,
run_count,
abort_early,
):
task_types = {
"test": Task,
"flake_task": FlakeTask,
"internal_failure": InternalFailureTask,
"raising": RaisingTask,
"led": LedTask,
#"triggered": TriggeredTask,
}
_create_task = task_types[task_type] # pylint: disable=invalid-name
if full:
tasks = [
_create_task(api=api, name="pass", initial_task_id=100),
_create_task(api=api, name="flake", initial_task_id=200),
_create_task(api=api, name="fail", initial_task_id=300),
_create_task(api=api, name="pass_long", initial_task_id=400),
_create_task(api=api, name="flake_long", initial_task_id=500),
_create_task(api=api, name="fail_long", initial_task_id=600),
]
else:
tasks = [_create_task(api=api, name="task", initial_task_id=100)]
if abort_early:
tasks.append(
_create_task(
api=api,
name="abort_early_task",
initial_task_id=700,
abort_early=True,
)
)
if last_task_max_attempts:
tasks[-1].max_attempts = last_task_max_attempts
api.swarming_retry.run_and_present_tasks(
tasks, max_attempts=max_attempts, run_count=run_count
)
def GenTests(api): # pylint: disable=invalid-name
test_api = api.swarming_retry
def led_build_data(priority=100):
build = api.buildbucket.ci_build_message(priority=priority)
job_def = job_pb2.Definition()
job_def.buildbucket.bbagent_args.build.CopyFrom(build)
return api.led.mock_get_builder(job_def)
yield (
api.test("full_test", status="FAILURE") + api.properties(full=True) +
test_api.collect_data(
[
test_api.passed_task("pass", 100),
test_api.failed_task("flake", 200),
test_api.failed_task("fail", 300),
],
iteration=0,
) + test_api.collect_data(
[
test_api.passed_task("flake", 201),
test_api.failed_task("fail", 301)
],
iteration=1,
) +
# `fail` task failed max times so remaining long tasks should only be run
# once.
test_api.collect_data(
[
test_api.incomplete_task("pass_long", 400),
test_api.incomplete_task("flake_long", 500),
test_api.incomplete_task("fail_long", 600),
],
iteration=2,
) + test_api.collect_data([], iteration=3) +
test_api.collect_data([test_api.passed_task("pass_long", 400)],
iteration=4) +
test_api.collect_data([test_api.failed_task("flake_long", 500)],
iteration=5) +
test_api.collect_data([test_api.failed_task("fail_long", 600)],
iteration=6)
)
yield (
api.test("timeout_then_pass") + api.properties(full=False) +
test_api.collect_data([test_api.timed_out_task("task", 100)]) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1)
)
yield (
api.test("internal_failure", status="FAILURE") +
api.properties(full=False, task_type="internal_failure") +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1)
)
yield (
api.test("raising_process_results", status="FAILURE") +
api.properties(full=False, task_type="raising") +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1)
)
yield (
api.test("led_task") + api.properties(full=False, task_type="led") +
led_build_data() +
test_api.collect_data([test_api.failed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1)
)
yield (
api.test("led_task_hardcoded_attempt") +
api.properties(full=False, task_type="led") + led_build_data()
)
yield (
api.test("max_attempts_three", status="FAILURE") +
api.properties(full=False, task_type="raising", max_attempts=3) +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1) +
test_api.collect_data([test_api.passed_task("task", 102)], iteration=2)
)
yield (
api.test("last_task_max_attempts_low", status="FAILURE") + api.properties(
full=False,
task_type="raising",
max_attempts=3,
last_task_max_attempts=1
) +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0)
)
yield (
api.test("last_task_max_attempts_high", status="FAILURE") +
api.properties(
full=False,
task_type="raising",
max_attempts=3,
last_task_max_attempts=5
) +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1) +
test_api.collect_data([test_api.passed_task("task", 102)], iteration=2) +
test_api.collect_data([test_api.passed_task("task", 103)], iteration=3) +
test_api.collect_data([test_api.passed_task("task", 104)], iteration=4)
)
# Test case where we want to get multiple successful runs of the same
# task (run_count > 1).
# Test the simple case where there are no failures of the task.
yield (
api.test("multirun_without_failures", status="SUCCESS") +
api.properties(run_count=2, task_type="led") +
# Enforce that both of these task attempt are launched in the first
# iteration. (This requires using task_type="triggered".)
#+ api.swarming_retry.trigger_data("task", 100, iteration=0, attempt=0) +
#api.swarming_retry.trigger_data("task", 101, iteration=0, attempt=1) +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=1)
)
# Test the case where a task must be retried (multiple times) but
# eventually passes.
yield (
api.test("multirun_retry_overall_pass", status="SUCCESS") +
api.properties(run_count=2, task_type="led") +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.failed_task("task", 101)], iteration=1) +
test_api.collect_data([test_api.failed_task("task", 102)], iteration=2) +
test_api.collect_data([test_api.passed_task("task", 103)], iteration=3)
)
# Test the case where a task is retried, but ultimately we do not get
# enough passes within the max_attempts retry limit.
yield (
api.test("multirun_retry_overall_fail", status="FAILURE") +
api.properties(run_count=2, task_type="led") +
test_api.collect_data([test_api.passed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.failed_task("task", 101)], iteration=1) +
test_api.collect_data([test_api.failed_task("task", 102)], iteration=2) +
test_api.collect_data([test_api.failed_task("task", 103)], iteration=3)
)
# If the last attempt in the list passed but the task failed overall,
# it should not be treated as passed.
#
# Also, if the last attempt in the list completes before other attempts
# have completed, the task should be treated as in-progress not
# finished.
yield (
api.test("multirun_fail_pass", status="FAILURE") +
api.properties(run_count=2, max_attempts=1, task_type="led") +
test_api.collect_data([test_api.passed_task("task", 101)], iteration=0) +
test_api.collect_data([test_api.failed_task("task", 100)], iteration=1)
)
# Test that the "no futile retries" strategy is applied: If all of the
# attempts in the first batch fail, there should be no retries.
yield (
api.test("multirun_no_futile_retries", status="FAILURE") +
api.properties(run_count=2, task_type="led") +
test_api.collect_data([test_api.failed_task("task", 100)], iteration=0) +
test_api.collect_data([test_api.failed_task("task", 101)], iteration=1)
)
yield (
api.test("no_collect_after_failed_abort_early_task", status="FAILURE") +
api.properties(full=True, abort_early=True, task_type="flake_task") +
test_api.collect_data(
[
test_api.failed_task("abort_early_task", 700),
test_api.passed_task("pass", 100),
test_api.failed_task("flake", 200),
test_api.failed_task("fail", 300),
],
iteration=0,
) + test_api.collect_data(
[test_api.failed_task("abort_early_task", 701)],
iteration=1,
)
)