[devicelab] Run tasks concurrently.
1. Add a job module to help create sub builds, which has a lot of customizations and simplifications than the fuchsia/subbuild module.
2. In the future we will add a builder that runs all devicelab tests for every flutter commit.
3. I have tried an optimization of isolating the flutter/flutter checkout to sub-jobs. It doesn't work because a sub-job needs 70 secs to download the isolate, while checking out flutter/flutter only takes 20 secs.
4: Example test: https://chromium-swarm.appspot.com/task?id=4d7414a646f94c10
Bug: https://github.com/flutter/flutter/issues/53789
Change-Id: I60c8da510384bfff3093580d90331ea57c0d8045
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/4263
Reviewed-by: Godofredo Contreras <godofredoc@google.com>
Commit-Queue: Tong Wu <wutong@google.com>
diff --git a/recipe_modules/job/__init__.py b/recipe_modules/job/__init__.py
new file mode 100644
index 0000000..5f3ac86
--- /dev/null
+++ b/recipe_modules/job/__init__.py
@@ -0,0 +1,14 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+DEPS = [
+ "recipe_engine/buildbucket",
+ "recipe_engine/file",
+ "recipe_engine/json",
+ "recipe_engine/led",
+ "recipe_engine/path",
+ "recipe_engine/properties",
+ "recipe_engine/step",
+ "recipe_engine/swarming",
+]
diff --git a/recipe_modules/job/api.py b/recipe_modules/job/api.py
new file mode 100644
index 0000000..1f2ab72
--- /dev/null
+++ b/recipe_modules/job/api.py
@@ -0,0 +1,133 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from google.protobuf import json_format
+from recipe_engine import recipe_api
+from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2
+
+
+class Job(object):
+ """Describes a job that could be launched via led.
+
+ Attributes:
+ name (str): Name of the job.
+ properties (dict): Properties as the arguments to `led edit -p`.
+ dimensions (dict): Dimensions as the arguments to `led edit -d`.
+ task_id (str): Id of the swarming task.
+ task_server (str): Host name of the swarming server, e.g.
+ "chromium-swarm.appspot.com".
+ task_result (api.swarming.TaskResult): Result of the swarming task.
+ build_proto(build_pb2.Build): Proto generated from a buildbucket build.
+ outcome (str): The outcome could be "success", "none" when task_result.state
+ is None, or TaskState in lower case.
+ """
+
+ def __init__(self, name):
+ # Metadata attached before execution.
+ assert isinstance(name, basestring)
+ self.name = name
+ self.properties = {'name': name}
+ self.dimensions = {}
+
+ # Metadata attached during execution.
+ self.task_id = None
+ self.task_server = None
+
+ # Metadata attached after execution.
+ self.task_result = None
+ self.build_proto = None
+ self.outcome = None
+
+ @property
+ def task_url(self):
+ """Returns the URL of the associated task in the Swarming UI."""
+ return "%s/task?id=%s" % (self.task_server, self.task_id)
+
+ @property
+ def milo_url(self):
+ """Returns the URL of the associated task in the Milo UI."""
+ return "https://ci.chromium.org/swarming/task/%s?server=%s" % (
+ self.task_id, self.task_server)
+
+
+class JobApi(recipe_api.RecipeApi):
+ """API for launching jobs and collecting the results."""
+
+ def __init__(self, *args, **kwargs):
+ super(JobApi, self).__init__(*args, **kwargs)
+
+ def new(self, job_name):
+ return Job(job_name)
+
+ def current_recipe(self):
+ return self.m.properties.get('recipe')
+
+ def launch(self, job, presentation):
+ """Launches a job with led.
+
+ Args:
+ job (Job): The job definition.
+ presentation (StepPresentation): The presentation to add logs to.
+
+ Returns:
+ The input job object with additional details about the execution.
+ """
+ current = self.m.buildbucket.build.builder
+ led_data = self.m.led(
+ "get-builder",
+ "luci.%s.%s:%s" % (current.project, current.bucket, current.builder),
+ )
+ edit_args = []
+ for k, v in job.properties.iteritems():
+ edit_args.extend(["-p", "%s=%s" % (k, self.m.json.dumps(v))])
+ for k, v in job.dimensions.iteritems():
+ edit_args.extend(["-d", "%s=%s" % (k, v)])
+ led_data = led_data.then("edit", *edit_args)
+ led_data = self.m.led.inject_input_recipes(led_data)
+ final = led_data.then("launch", "-modernize")
+
+ job.task_id = final.launch_result.task_id
+ job.task_server = final.launch_result.swarming_hostname
+
+ presentation.links[job.name] = job.task_url
+ return job
+
+ def collect(self, jobs, presentation):
+ """Collects execution metadata for a list of jobs.
+
+ Args:
+ jobs (list(Job)): The jobs to collect information for.
+ presentation (StepPresentation): The presentation to add logs to.
+
+ Returns:
+ The input jobs with additional details collected from execution.
+ """
+ by_task_id = {job.task_id: job for job in jobs}
+ swarming_results = self.m.swarming.collect(
+ "collect",
+ by_task_id.keys(),
+ output_dir=self.m.path["cleanup"],
+ )
+ for result in swarming_results:
+ job = by_task_id[result.id]
+ job.task_result = result
+
+ # Led launch ensures this file is present in the task root dir.
+ build_proto_path = result.output_dir.join("build.proto.json")
+ build_proto_json = self.m.file.read_text("read build.proto.json",
+ build_proto_path)
+ build_proto = build_pb2.Build()
+ json_format.Parse(build_proto_json, build_proto)
+ job.build_proto = build_proto
+
+ if result.success:
+ job.outcome = "success"
+ elif not result.state:
+ job.outcome = "none"
+ else:
+ # Example result.state: TaskState.COMPLETED
+ job.outcome = str(result.state)[10:].lower()
+
+ presentation.links["%s (%s)" % (job.name, job.outcome)] = job.milo_url
+ return jobs
diff --git a/recipe_modules/job/examples/full.expected/collect.json b/recipe_modules/job/examples/full.expected/collect.json
new file mode 100644
index 0000000..2185d84
--- /dev/null
+++ b/recipe_modules/job/examples/full.expected/collect.json
@@ -0,0 +1,220 @@
+[
+ {
+ "cmd": [],
+ "name": "collect builds",
+ "~followup_annotations": [
+ "@@@STEP_LINK@fake_job0 (success)@https://ci.chromium.org/swarming/task/task_id0?server=None@@@",
+ "@@@STEP_LINK@fake_job1 (success)@https://ci.chromium.org/swarming/task/task_id1?server=None@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect builds.install infra/tools/luci/swarming",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin"
+ ],
+ "infra_step": true,
+ "name": "collect builds.install infra/tools/luci/swarming.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin",
+ "-ensure-file",
+ "infra/tools/luci/swarming/${platform} swarming_module_pin",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "collect builds.install infra/tools/luci/swarming.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-swarming_module_\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/swarming/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin/swarming",
+ "collect",
+ "-server",
+ "https://example.swarmingserver.appspot.com",
+ "-task-summary-json",
+ "/path/to/tmp/json",
+ "-task-output-stdout",
+ "json",
+ "-output-dir",
+ "[CLEANUP]",
+ "task_id1",
+ "task_id0"
+ ],
+ "infra_step": true,
+ "name": "collect builds.collect",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id0\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"exit_code\": 0, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"my_task_0\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"state\": \"COMPLETED\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"task_id0\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id1\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"exit_code\": 0, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"my_task_1\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"state\": \"COMPLETED\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"task_id1\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@task stdout+stderr: my_task_0@hello world!@@@",
+ "@@@STEP_LOG_END@task stdout+stderr: my_task_0@@@",
+ "@@@STEP_LOG_LINE@task stdout+stderr: my_task_1@hello world!@@@",
+ "@@@STEP_LOG_END@task stdout+stderr: my_task_1@@@",
+ "@@@STEP_LINK@task isolated outputs: my_task_0@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@",
+ "@@@STEP_LINK@task isolated outputs: my_task_1@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/task_id0/build.proto.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "collect builds.read build.proto.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@{@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"ci\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:luci-scheduler@appspot.gserviceaccount.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"ref\": \"refs/heads/master\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@}@@@",
+ "@@@STEP_LOG_END@build.proto.json@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/task_id1/build.proto.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "collect builds.read build.proto.json (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@{@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"ci\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:1001\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:luci-scheduler@appspot.gserviceaccount.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"ref\": \"refs/heads/master\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"1001\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@}@@@",
+ "@@@STEP_LOG_END@build.proto.json@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/job/examples/full.expected/collect_failed_states.json b/recipe_modules/job/examples/full.expected/collect_failed_states.json
new file mode 100644
index 0000000..5058057
--- /dev/null
+++ b/recipe_modules/job/examples/full.expected/collect_failed_states.json
@@ -0,0 +1,207 @@
+[
+ {
+ "cmd": [],
+ "name": "collect builds",
+ "~followup_annotations": [
+ "@@@STEP_LINK@fake_job0 (none)@https://ci.chromium.org/swarming/task/task_id0?server=None@@@",
+ "@@@STEP_LINK@fake_job1 (timed_out)@https://ci.chromium.org/swarming/task/task_id1?server=None@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect builds.install infra/tools/luci/swarming",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin"
+ ],
+ "infra_step": true,
+ "name": "collect builds.install infra/tools/luci/swarming.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin",
+ "-ensure-file",
+ "infra/tools/luci/swarming/${platform} swarming_module_pin",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "collect builds.install infra/tools/luci/swarming.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-swarming_module_\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/swarming/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin/swarming",
+ "collect",
+ "-server",
+ "https://example.swarmingserver.appspot.com",
+ "-task-summary-json",
+ "/path/to/tmp/json",
+ "-task-output-stdout",
+ "json",
+ "-output-dir",
+ "[CLEANUP]",
+ "task_id1",
+ "task_id0"
+ ],
+ "infra_step": true,
+ "name": "collect builds.collect",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id0\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"error\": \"Bot could not be contacted\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"task_id0\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id1\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"my_task_1\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"state\": \"TIMED_OUT\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"task_id1\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@task stdout+stderr: None@Bot could not be contacted@@@",
+ "@@@STEP_LOG_END@task stdout+stderr: None@@@",
+ "@@@STEP_LOG_LINE@task stdout+stderr: my_task_1@hello world!@@@",
+ "@@@STEP_LOG_END@task stdout+stderr: my_task_1@@@",
+ "@@@STEP_LINK@task isolated outputs: my_task_1@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/task_id0/build.proto.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "collect builds.read build.proto.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@{@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"ci\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:luci-scheduler@appspot.gserviceaccount.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"ref\": \"refs/heads/master\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@}@@@",
+ "@@@STEP_LOG_END@build.proto.json@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/task_id1/build.proto.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "collect builds.read build.proto.json (2)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@{@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"ci\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:1001\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:luci-scheduler@appspot.gserviceaccount.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"ref\": \"refs/heads/master\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"1001\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@}@@@",
+ "@@@STEP_LOG_END@build.proto.json@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/job/examples/full.expected/current_recipe.json b/recipe_modules/job/examples/full.expected/current_recipe.json
new file mode 100644
index 0000000..b6042b6
--- /dev/null
+++ b/recipe_modules/job/examples/full.expected/current_recipe.json
@@ -0,0 +1,5 @@
+[
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/job/examples/full.expected/launch.json b/recipe_modules/job/examples/full.expected/launch.json
new file mode 100644
index 0000000..42dbac1
--- /dev/null
+++ b/recipe_modules/job/examples/full.expected/launch.json
@@ -0,0 +1,141 @@
+[
+ {
+ "cmd": [],
+ "name": "launch job",
+ "~followup_annotations": [
+ "@@@STEP_LINK@fake_job0@chromium-swarm.appspot.com/task?id=job_task_id@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/led",
+ "-ensure-file",
+ "infra/tools/luci/led/${platform} latest",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "launch job.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/led/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "get-builder",
+ "luci.flutter.try:Linux"
+ ],
+ "name": "launch job.led get-builder",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@proto.output@{@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"buildbucket\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bbagent_args\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"build\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": \"Linux\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"project\": \"flutter\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }, @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"task_id\": \"job_task_id\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@}@@@",
+ "@@@STEP_LOG_END@proto.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "edit",
+ "-p",
+ "foo=[\"a\", \"b\"]",
+ "-p",
+ "recipe=\"fake_recipe\"",
+ "-p",
+ "name=\"fake_job0\"",
+ "-d",
+ "id=fake_bot_id",
+ "-d",
+ "pool=luci.flutter.staging"
+ ],
+ "name": "launch job.led edit",
+ "stdin": "{\n\"buildbucket\": {\n\"bbagent_args\": {\n\"build\": {\n\"builder\": {\n\"bucket\": \"try\", \n\"builder\": \"Linux\", \n\"project\": \"flutter\"\n}, \n\"infra\": {\n\"swarming\": {\n\"task_id\": \"job_task_id\"\n}\n}\n}\n}\n}\n}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@proto.output@{@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"buildbucket\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bbagent_args\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"build\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bucket\": \"try\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": \"Linux\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"project\": \"flutter\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }, @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"task_id\": \"job_task_id\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }, @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"foo\": [@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"a\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"b\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ ], @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"name\": \"fake_job0\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"recipe\": \"fake_recipe\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@}@@@",
+ "@@@STEP_LOG_END@proto.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "launch",
+ "-modernize"
+ ],
+ "name": "launch job.led launch",
+ "stdin": "{\n\"buildbucket\": {\n\"bbagent_args\": {\n\"build\": {\n\"builder\": {\n\"bucket\": \"try\", \n\"builder\": \"Linux\", \n\"project\": \"flutter\"\n}, \n\"infra\": {\n\"swarming\": {\n\"task_id\": \"job_task_id\"\n}\n}, \n\"input\": {\n\"properties\": {\n\"foo\": [\n\"a\", \n\"b\"\n], \n\"name\": \"fake_job0\", \n\"recipe\": \"fake_recipe\"\n}\n}\n}\n}\n}\n}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"host_name\": \"chromium-swarm.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"job_task_id\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@Swarming task@https://chromium-swarm.appspot.com/task?id=job_task_id@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipe_modules/job/examples/full.py b/recipe_modules/job/examples/full.py
new file mode 100644
index 0000000..f54cd54
--- /dev/null
+++ b/recipe_modules/job/examples/full.py
@@ -0,0 +1,74 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from recipe_engine.recipe_api import Property
+
+DEPS = [
+ "flutter/job",
+ "recipe_engine/properties",
+ "recipe_engine/step",
+ "recipe_engine/swarming",
+]
+
+PROPERTIES = {
+ "test_name": Property(kind=str, help="Name of the test"),
+}
+
+
+def RunSteps(api, test_name):
+ job0 = api.job.new("fake_job0")
+ job0.properties.update({
+ "recipe": "fake_recipe",
+ "foo": ["a", "b"],
+ })
+ job0.dimensions.update({
+ "id": "fake_bot_id",
+ "pool": "luci.flutter.staging",
+ })
+
+ job1 = api.job.new("fake_job1")
+
+ if test_name == "launch":
+ with api.step.nest("launch job") as presentation:
+ job0 = api.job.launch(job0, presentation)
+ elif test_name == "collect":
+ job0.task_id = "task_id0"
+ job1.task_id = "task_id1"
+ with api.step.nest("collect builds") as presentation:
+ api.job.collect([job0, job1], presentation)
+ elif test_name == "current_recipe":
+ api.job.current_recipe()
+
+
+def GenTests(api):
+ yield api.test(
+ "launch",
+ api.properties(test_name="launch"),
+ api.job.mock_launch(),
+ )
+ yield api.test(
+ "collect",
+ api.properties(test_name="collect"),
+ api.job.mock_collect(["task_id0", "task_id1"], "collect builds"),
+ )
+ yield api.test(
+ "collect_failed_states",
+ api.properties(test_name="collect"),
+ api.job.mock_collect(
+ ["task_id0", "task_id1"],
+ "collect builds",
+ swarming_results=[
+ api.swarming.task_result(
+ id="task_id0", name="my_task_0", state=None),
+ api.swarming.task_result(
+ id="task_id1",
+ name="my_task_1",
+ state=api.swarming.TaskState.TIMED_OUT),
+ ],
+ ),
+ )
+ yield api.test(
+ "current_recipe",
+ api.properties(test_name="current_recipe"),
+ )
diff --git a/recipe_modules/job/test_api.py b/recipe_modules/job/test_api.py
new file mode 100644
index 0000000..cbb7274
--- /dev/null
+++ b/recipe_modules/job/test_api.py
@@ -0,0 +1,82 @@
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from google.protobuf import json_format
+from PB.go.chromium.org.luci.led.job import job as job_pb2
+from recipe_engine import recipe_test_api
+
+
+class JobTestApi(recipe_test_api.RecipeTestApi):
+
+ def mock_launch(self, buildbucket_build=None, task_id="job_task_id"):
+ """Returns mock data for the launch function.
+
+ Args:
+ buildbucket_build (TestData): Emulates a buildbucket build. It is usually
+ an output from api.buildbucket.x_build().
+ task_id (str): Id of the swarming task.
+ """
+ ret = self.empty_test_data()
+
+ # Attaches current build.
+ if not buildbucket_build:
+ buildbucket_build = self.m.buildbucket.ci_build(
+ project="flutter", bucket="try", builder="Linux")
+ ret += buildbucket_build
+
+ # Attaches task_id of the launched swarming task.
+ # led launch mock will take ....infra.swarming.task_id as this build's
+ # launched swarming ID.
+ jd = job_pb2.Definition()
+ jd.buildbucket.bbagent_args.build.infra.swarming.task_id = task_id
+ ret += self.m.led.mock_get_builder(jd)
+ return ret
+
+ def mock_collect(self,
+ task_ids,
+ presentation_step_name,
+ swarming_results=None,
+ build_protos=None):
+ """Returns mock data for the collect function.
+
+ Args:
+ task_ids (list(str)): List of swarming task ids.
+ presentation_step_name (str): The step name of the presentation.
+ swarming_results (list(dict)): List of the outputs from
+ api.swarming.task_result() in the order of task_ids.
+ build_protos (list(build_pb2.Build)): List of build proto messages in the
+ order of task_ids.
+ """
+ ret = self.empty_test_data()
+
+ # Attaches swarming results.
+ if not swarming_results:
+ swarming_results = [
+ self.m.swarming.task_result(id=task_id, name="my_task_%d" % i)
+ for i, task_id in enumerate(task_ids)
+ ]
+
+ ret += self.step_data(
+ "%s.collect" % presentation_step_name,
+ self.m.swarming.collect(swarming_results),
+ )
+
+ # Attaches build protos.
+ if not build_protos:
+ build_protos = [
+ self.m.buildbucket.ci_build_message(build_id=1000 + i)
+ for i, _ in enumerate(task_ids)
+ ]
+
+ for i, id in enumerate(task_ids):
+ # Mocks read build.proto.json.
+ step_name = "%s.read build.proto.json" % presentation_step_name
+ if i > 0:
+ step_name += " (%d)" % (i + 1)
+ ret += self.step_data(
+ step_name,
+ self.m.file.read_text(json_format.MessageToJson(build_protos[i])),
+ )
+
+ return ret
diff --git a/recipes/devicelab.expected/android_defines_test.json b/recipes/devicelab.expected/android_defines_test.json
index fda27ef..b7b7104 100644
--- a/recipes/devicelab.expected/android_defines_test.json
+++ b/recipes/devicelab.expected/android_defines_test.json
@@ -11,7 +11,7 @@
"--path",
"[START_DIR]/flutter",
"--url",
- "https://abc.com/repo"
+ "https://chromium.googlesource.com/external/github.com/flutter/flutter"
],
"name": "checkout flutter/flutter.git setup",
"~followup_annotations": [
@@ -23,7 +23,7 @@
"git",
"fetch",
"origin",
- "refs/pull/123/head",
+ "master",
"--recurse-submodules",
"--progress",
"--tags"
@@ -129,95 +129,6 @@
"name": "flutter doctor"
},
{
- "cmd": [],
- "name": "read manifest",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@yaml@@@@",
- "@@@STEP_LOG_END@yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read manifest.read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@manifest.yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
- "--yaml_file",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "--json_file",
- "/path/to/tmp/json"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "read manifest.parse",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"android_defines_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Builds an APK with a --dart-define ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"linux/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flavors_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Checks that flavored builds work on Android.\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flutter_gallery_ios__compile\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Collects various performance metrics of ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab_ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
"cmd": [
"pub",
"get"
diff --git a/recipes/devicelab.expected/flavors_test.json b/recipes/devicelab.expected/flavors_test.json
index 27b8e0c..9906e59 100644
--- a/recipes/devicelab.expected/flavors_test.json
+++ b/recipes/devicelab.expected/flavors_test.json
@@ -11,7 +11,7 @@
"--path",
"[START_DIR]/flutter",
"--url",
- "https://abc.com/repo"
+ "https://chromium.googlesource.com/external/github.com/flutter/flutter"
],
"name": "checkout flutter/flutter.git setup",
"~followup_annotations": [
@@ -23,7 +23,7 @@
"git",
"fetch",
"origin",
- "refs/pull/123/head",
+ "master",
"--recurse-submodules",
"--progress",
"--tags"
@@ -129,95 +129,6 @@
"name": "flutter doctor"
},
{
- "cmd": [],
- "name": "read manifest",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@yaml@@@@",
- "@@@STEP_LOG_END@yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read manifest.read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@manifest.yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
- "--yaml_file",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "--json_file",
- "/path/to/tmp/json"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "read manifest.parse",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"android_defines_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Builds an APK with a --dart-define ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"linux/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flavors_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Checks that flavored builds work on Android.\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flutter_gallery_ios__compile\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Collects various performance metrics of ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab_ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
"cmd": [
"pub",
"get"
diff --git a/recipes/devicelab.expected/flutter_gallery_ios__compile.json b/recipes/devicelab.expected/flutter_gallery_ios__compile.json
index a074225..0dd2942 100644
--- a/recipes/devicelab.expected/flutter_gallery_ios__compile.json
+++ b/recipes/devicelab.expected/flutter_gallery_ios__compile.json
@@ -11,7 +11,7 @@
"--path",
"[START_DIR]/flutter",
"--url",
- "https://abc.com/repo"
+ "https://chromium.googlesource.com/external/github.com/flutter/flutter"
],
"name": "checkout flutter/flutter.git setup",
"~followup_annotations": [
@@ -23,7 +23,7 @@
"git",
"fetch",
"origin",
- "refs/pull/123/head",
+ "master",
"--recurse-submodules",
"--progress",
"--tags"
@@ -129,95 +129,6 @@
"name": "flutter doctor"
},
{
- "cmd": [],
- "name": "read manifest",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@yaml@@@@",
- "@@@STEP_LOG_END@yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read manifest.read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@manifest.yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
- "--yaml_file",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "--json_file",
- "/path/to/tmp/json"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "read manifest.parse",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"android_defines_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Builds an APK with a --dart-define ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"linux/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flavors_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Checks that flavored builds work on Android.\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flutter_gallery_ios__compile\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Collects various performance metrics of ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab_ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
"cmd": [
"pub",
"get"
diff --git a/recipes/devicelab.expected/schedule.json b/recipes/devicelab.expected/schedule.json
new file mode 100644
index 0000000..6be6f8e
--- /dev/null
+++ b/recipes/devicelab.expected/schedule.json
@@ -0,0 +1,467 @@
+[
+ {
+ "cmd": [],
+ "name": "checkout flutter/flutter"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+ "--path",
+ "[START_DIR]/flutter",
+ "--url",
+ "https://chromium.googlesource.com/external/github.com/flutter/flutter"
+ ],
+ "name": "checkout flutter/flutter.git setup",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "origin",
+ "master",
+ "--recurse-submodules",
+ "--progress",
+ "--tags"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "env": {
+ "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
+ },
+ "infra_step": true,
+ "name": "checkout flutter/flutter.git fetch",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "FETCH_HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "checkout flutter/flutter.git checkout",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "checkout flutter/flutter.read revision",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@",
+ "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "checkout flutter/flutter.git clean",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "checkout flutter/flutter.submodule sync",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive"
+ ],
+ "cwd": "[START_DIR]/flutter",
+ "infra_step": true,
+ "name": "checkout flutter/flutter.submodule update",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "read manifest",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@yaml@@@@",
+ "@@@STEP_LOG_END@yaml@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
+ "/path/to/tmp/"
+ ],
+ "cwd": "[START_DIR]/flutter/dev/devicelab",
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "PUB_CACHE": "[CACHE]/.pub-cache"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter/bin",
+ "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "infra_step": true,
+ "name": "read manifest.read",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_END@manifest.yaml@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
+ "--yaml_file",
+ "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
+ "--json_file",
+ "/path/to/tmp/json"
+ ],
+ "cwd": "[START_DIR]/flutter/dev/devicelab",
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "PUB_CACHE": "[CACHE]/.pub-cache"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter/bin",
+ "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "name": "read manifest.parse",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"android_defines_test\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"description\": \"Builds an APK with a --dart-define ...\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"on_luci\": true, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"linux/android\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ ], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "launch jobs",
+ "~followup_annotations": [
+ "@@@STEP_LINK@android_defines_test@chromium-swarm.appspot.com/task?id=fake-task-id@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/led",
+ "-ensure-file",
+ "infra/tools/luci/led/${platform} latest",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "launch jobs.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-latest----------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/led/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "get-builder",
+ "luci..:"
+ ],
+ "name": "launch jobs.led get-builder",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@proto.output@{@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"buildbucket\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bbagent_args\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"build\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": {}@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@}@@@",
+ "@@@STEP_LOG_END@proto.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "edit",
+ "-p",
+ "recipe=\"devicelab\"",
+ "-p",
+ "role=\"worker\"",
+ "-p",
+ "name=\"android_defines_test\"",
+ "-p",
+ "first_capability=\"linux/android\"",
+ "-d",
+ "cores=",
+ "-d",
+ "caches=",
+ "-d",
+ "os=",
+ "-d",
+ "cpu=",
+ "-d",
+ "pool=luci.flutter.staging",
+ "-d",
+ "id=flutter-devicelab-linux-8"
+ ],
+ "name": "launch jobs.led edit",
+ "stdin": "{\n\"buildbucket\": {\n\"bbagent_args\": {\n\"build\": {\n\"builder\": {}\n}\n}\n}\n}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@proto.output@{@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"buildbucket\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"bbagent_args\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"build\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"builder\": {}, @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"properties\": {@@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"first_capability\": \"linux/android\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"name\": \"android_defines_test\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"recipe\": \"devicelab\", @@@",
+ "@@@STEP_LOG_LINE@proto.output@ \"role\": \"worker\"@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@ }@@@",
+ "@@@STEP_LOG_LINE@proto.output@}@@@",
+ "@@@STEP_LOG_END@proto.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/led/led",
+ "launch",
+ "-modernize"
+ ],
+ "name": "launch jobs.led launch",
+ "stdin": "{\n\"buildbucket\": {\n\"bbagent_args\": {\n\"build\": {\n\"builder\": {}, \n\"input\": {\n\"properties\": {\n\"first_capability\": \"linux/android\", \n\"name\": \"android_defines_test\", \n\"recipe\": \"devicelab\", \n\"role\": \"worker\"\n}\n}\n}\n}\n}\n}",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"host_name\": \"chromium-swarm.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@Swarming task@https://chromium-swarm.appspot.com/task?id=fake-task-id@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect jobs",
+ "~followup_annotations": [
+ "@@@STEP_LINK@android_defines_test (success)@https://ci.chromium.org/swarming/task/fake-task-id?server=chromium-swarm.appspot.com@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "collect jobs.install infra/tools/luci/swarming",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin"
+ ],
+ "infra_step": true,
+ "name": "collect jobs.install infra/tools/luci/swarming.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin",
+ "-ensure-file",
+ "infra/tools/luci/swarming/${platform} swarming_module_pin",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "collect jobs.install infra/tools/luci/swarming.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@2@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-swarming_module_\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra/tools/luci/swarming/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/infra/tools/luci/swarming/swarming_module_pin/swarming",
+ "collect",
+ "-server",
+ "https://example.swarmingserver.appspot.com",
+ "-task-summary-json",
+ "/path/to/tmp/json",
+ "-task-output-stdout",
+ "json",
+ "-output-dir",
+ "[CLEANUP]",
+ "fake-task-id"
+ ],
+ "infra_step": true,
+ "name": "collect jobs.collect",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"fake-task-id\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"output\": \"hello world!\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs\": [], @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"results\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"bot_id\": \"vm-123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"duration\": 62.35, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"exit_code\": 0, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"name\": \"my_task_0\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"outputs_ref\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolated\": \"abc123\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"isolatedserver\": \"https://isolateserver.appspot.com\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"namespace\": \"default-gzip\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }, @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"state\": \"COMPLETED\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"task_id\": \"fake-task-id\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LOG_LINE@task stdout+stderr: my_task_0@hello world!@@@",
+ "@@@STEP_LOG_END@task stdout+stderr: my_task_0@@@",
+ "@@@STEP_LINK@task isolated outputs: my_task_0@https://isolateserver.appspot.com/browse?namespace=default-gzip&hash=abc123@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "[CLEANUP]/fake-task-id/build.proto.json",
+ "/path/to/tmp/"
+ ],
+ "infra_step": true,
+ "name": "collect jobs.read build.proto.json",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@{@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"builder\": \"builder\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"bucket\": \"ci\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createTime\": \"2018-05-25T23:50:17Z\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"infra\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"swarming\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"priority\": 30@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"resultdb\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"invocation\": \"invocations/build:1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"createdBy\": \"user:luci-scheduler@appspot.gserviceaccount.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"input\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"gitilesCommit\": {@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"project\": \"project\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"host\": \"chromium.googlesource.com\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"ref\": \"refs/heads/master\", @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"2d72510e447ab60a9728aeea2362d8be2cbd7789\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ }, @@@",
+ "@@@STEP_LOG_LINE@build.proto.json@ \"id\": \"1000\"@@@",
+ "@@@STEP_LOG_LINE@build.proto.json@}@@@",
+ "@@@STEP_LOG_END@build.proto.json@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/devicelab.expected/missing_task_name.json b/recipes/devicelab.expected/unknown_role.json
similarity index 63%
rename from recipes/devicelab.expected/missing_task_name.json
rename to recipes/devicelab.expected/unknown_role.json
index ffdbff8..eb5df5b 100644
--- a/recipes/devicelab.expected/missing_task_name.json
+++ b/recipes/devicelab.expected/unknown_role.json
@@ -14,15 +14,15 @@
" File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in invoke_with_properties",
" arg_names, **additional_args)",
" File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in _invoke_with_properties",
- " prop.interpret(all_props.get(prop_name, PROPERTY_SENTINEL), environ))",
- " File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in interpret",
- " self.name, self.__property_type, self.full_decl_name))",
- "ValueError: No default specified and no value provided for 'task_name' from recipe 'flutter::devicelab'"
+ " return callable_obj(*props, **additional_args)",
+ " File \"RECIPE_REPO[flutter]/recipes/devicelab.py\", line 32, in RunSteps",
+ " raise ValueError('Unknown role: %s' % role)",
+ "ValueError: Unknown role: unknown"
]
},
{
"failure": {
- "humanReason": "Uncaught Exception: ValueError(\"No default specified and no value provided for 'task_name' from recipe 'flutter::devicelab'\",)"
+ "humanReason": "Uncaught Exception: ValueError('Unknown role: unknown',)"
},
"name": "$result"
}
diff --git a/recipes/devicelab.expected/unknown_task.json b/recipes/devicelab.expected/unknown_task.json
deleted file mode 100644
index 986ad51..0000000
--- a/recipes/devicelab.expected/unknown_task.json
+++ /dev/null
@@ -1,247 +0,0 @@
-[
- {
- "cmd": [],
- "name": "checkout flutter/flutter"
- },
- {
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
- "--path",
- "[START_DIR]/flutter",
- "--url",
- "https://chromium.googlesource.com/external/github.com/flutter/flutter"
- ],
- "name": "checkout flutter/flutter.git setup",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "fetch",
- "origin",
- "master",
- "--recurse-submodules",
- "--progress",
- "--tags"
- ],
- "cwd": "[START_DIR]/flutter",
- "env": {
- "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
- },
- "infra_step": true,
- "name": "checkout flutter/flutter.git fetch",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "checkout",
- "-f",
- "FETCH_HEAD"
- ],
- "cwd": "[START_DIR]/flutter",
- "infra_step": true,
- "name": "checkout flutter/flutter.git checkout",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "rev-parse",
- "HEAD"
- ],
- "cwd": "[START_DIR]/flutter",
- "infra_step": true,
- "name": "checkout flutter/flutter.read revision",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@",
- "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "clean",
- "-f",
- "-d",
- "-x"
- ],
- "cwd": "[START_DIR]/flutter",
- "infra_step": true,
- "name": "checkout flutter/flutter.git clean",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "submodule",
- "sync"
- ],
- "cwd": "[START_DIR]/flutter",
- "infra_step": true,
- "name": "checkout flutter/flutter.submodule sync",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "git",
- "submodule",
- "update",
- "--init",
- "--recursive"
- ],
- "cwd": "[START_DIR]/flutter",
- "infra_step": true,
- "name": "checkout flutter/flutter.submodule update",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "flutter",
- "doctor"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "flutter doctor"
- },
- {
- "cmd": [],
- "name": "read manifest",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@yaml@@@@",
- "@@@STEP_LOG_END@yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "read manifest.read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@manifest.yaml@@@"
- ]
- },
- {
- "cmd": [
- "vpython",
- "-u",
- "RECIPE_MODULE[flutter::yaml]/resources/parse_yaml.py",
- "--yaml_file",
- "[START_DIR]/flutter/dev/devicelab/manifest.yaml",
- "--json_file",
- "/path/to/tmp/json"
- ],
- "cwd": "[START_DIR]/flutter/dev/devicelab",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "PUB_CACHE": "[CACHE]/.pub-cache"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "read manifest.parse",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_LINE@json.output@{@@@",
- "@@@STEP_LOG_LINE@json.output@ \"tasks\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"android_defines_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Builds an APK with a --dart-define ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"linux/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flavors_test\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Checks that flavored builds work on Android.\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/android\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }, @@@",
- "@@@STEP_LOG_LINE@json.output@ \"flutter_gallery_ios__compile\": {@@@",
- "@@@STEP_LOG_LINE@json.output@ \"description\": \"Collects various performance metrics of ...\", @@@",
- "@@@STEP_LOG_LINE@json.output@ \"required_agent_capabilities\": [@@@",
- "@@@STEP_LOG_LINE@json.output@ \"mac/ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ ], @@@",
- "@@@STEP_LOG_LINE@json.output@ \"stage\": \"devicelab_ios\"@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@ }@@@",
- "@@@STEP_LOG_LINE@json.output@}@@@",
- "@@@STEP_LOG_END@json.output@@@"
- ]
- },
- {
- "cmd": [],
- "name": "RECIPE CRASH (Uncaught exception)",
- "~followup_annotations": [
- "@@@STEP_EXCEPTION@@@",
- "The recipe has crashed at point 'Uncaught exception'!",
- "",
- "Traceback (most recent call last):",
- " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/engine.py\", in run_steps",
- " raw_result = recipe_obj.run_steps(api, engine)",
- " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/recipe_deps.py\", in run_steps",
- " properties_def, api=api)",
- " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in invoke_with_properties",
- " arg_names, **additional_args)",
- " File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in _invoke_with_properties",
- " return callable_obj(*props, **additional_args)",
- " File \"RECIPE_REPO[flutter]/recipes/devicelab.py\", line 52, in RunSteps",
- " raise ValueError('Unknown task: %s' % task_name)",
- "ValueError: Unknown task: unknown_task"
- ]
- },
- {
- "failure": {
- "humanReason": "Uncaught Exception: ValueError('Unknown task: unknown_task',)"
- },
- "name": "$result"
- }
-]
\ No newline at end of file
diff --git a/recipes/devicelab.py b/recipes/devicelab.py
index 9a5ad65..6f6e225 100644
--- a/recipes/devicelab.py
+++ b/recipes/devicelab.py
@@ -6,27 +6,33 @@
DEPS = [
'depot_tools/git',
- 'depot_tools/osx_sdk',
'flutter/android_sdk',
'flutter/repo_util',
'flutter/yaml',
- 'recipe_engine/cipd',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/path',
- 'recipe_engine/platform',
'recipe_engine/properties',
- 'recipe_engine/raw_io',
'recipe_engine/step',
+ "flutter/job",
]
PROPERTIES = {
- 'task_name': Property(kind=str, help='Name of the devicelab task to run'),
+ 'role': Property(kind=str, help='either scheduler or worker'),
}
-def RunSteps(api, task_name):
+def RunSteps(api, role):
+ if role == "scheduler":
+ schedule_all(api)
+ elif role == 'worker':
+ run_task(api)
+ else:
+ raise ValueError('Unknown role: %s' % role)
+
+
+def schedule_all(api):
flutter_path = api.path['start_dir'].join('flutter')
with api.step.nest('checkout flutter/flutter'):
api.repo_util.checkout(
@@ -35,28 +41,78 @@
api.properties.get('git_url'),
api.properties.get('git_ref'),
)
+
env, env_prefixes = api.repo_util.flutter_environment(flutter_path)
devicelab_path = flutter_path.join('dev', 'devicelab')
- with api.context(env=env, env_prefixes=env_prefixes, cwd=devicelab_path):
- api.step('flutter doctor', ['flutter', 'doctor'])
+ sub_jobs = []
+ with api.context(env=env, env_prefixes=env_prefixes, cwd=devicelab_path):
# Reads the manifest.
result = api.yaml.read('read manifest',
devicelab_path.join('manifest.yaml'),
api.json.output())
manifest = result.json.output
+ for task_name, task_body in manifest['tasks'].iteritems():
+ # Example first capability values: linux/android, mac/ios.
+ first_capability = task_body['required_agent_capabilities'][0]
+ if task_body.get('on_luci'):
+ sub_job = api.job.new(task_name)
+ sub_job.properties.update({
+ "recipe": api.job.current_recipe(),
+ "role": "worker",
+ "first_capability": first_capability,
+ })
+ # TODO(wutong): add a devicelab dedicated builder that would save us
+ # from removing extra dimensions like "cores", "os" etc.
+ sub_job.dimensions.update({
+ "id": select_bot(first_capability),
+ "pool": "luci.flutter.staging",
+ "cores": "",
+ "os": "",
+ "cpu": "",
+ "caches": "",
+ })
+ sub_jobs.append(sub_job)
- # Verifies that the manifest contains the task to run.
- task = manifest['tasks'].get(task_name)
- if not task:
- raise ValueError('Unknown task: %s' % task_name)
+ with api.step.nest("launch jobs") as presentation:
+ for sub_job in sub_jobs:
+ api.job.launch(sub_job, presentation)
- # Example first capability values: linux/android, mac/ios.
- first_capability = task['required_agent_capabilities'][0]
- sdk = first_capability.split('/')[1]
+ with api.step.nest("collect jobs") as presentation:
+ api.job.collect(sub_jobs, presentation)
+
+
+def select_bot(first_capability):
+ # TODO(wutong): apply bot selection by dimensions instead of hard-coded ids.
+ mapping = {
+ "linux/android": "flutter-devicelab-linux-8",
+ "mac/android": "flutter-devicelab-mac-22",
+ "mac/ios": "flutter-devicelab-mac-9",
+ }
+ return mapping.get(first_capability)
+
+
+def run_task(api):
+ task_name = api.properties["name"]
+ first_capability = api.properties["first_capability"]
+
+ flutter_path = api.path['start_dir'].join('flutter')
+ with api.step.nest('checkout flutter/flutter'):
+ api.repo_util.checkout(
+ 'flutter',
+ flutter_path,
+ api.properties.get('git_url'),
+ api.properties.get('git_ref'),
+ )
+
+ env, env_prefixes = api.repo_util.flutter_environment(flutter_path)
+ devicelab_path = flutter_path.join('dev', 'devicelab')
+ with api.context(env=env, env_prefixes=env_prefixes, cwd=devicelab_path):
+ api.step('flutter doctor', ['flutter', 'doctor'])
+ api.step('pub get', ['pub', 'get'])
# Runs a task.
- api.step('pub get', ['pub', 'get'])
+ sdk = first_capability.split('/')[1]
if sdk == 'android':
run_android_task(api, task_name)
elif sdk == 'ios':
@@ -83,8 +139,16 @@
def GenTests(api):
+ for t in gen_scheduler_tests(api):
+ yield t
+ for t in gen_worker_tests(api):
+ yield t
+
+
+def gen_scheduler_tests(api):
yield api.test(
- 'missing_task_name',
+ "unknown_role",
+ api.properties(role="unknown"),
api.expect_exception('ValueError'),
)
@@ -94,40 +158,50 @@
"description": "Builds an APK with a --dart-define ...",
"stage": "devicelab",
"required_agent_capabilities": ["linux/android"],
+ "on_luci": True,
+ },
+ },
+ }
+ yield api.test(
+ "schedule", api.properties(role="scheduler"),
+ api.repo_util.flutter_environment_data(),
+ api.step_data('read manifest.parse', api.json.output(sample_manifest)),
+ api.job.mock_collect(["fake-task-id"], "collect jobs"))
+
+
+def gen_worker_tests(api):
+ sample_manifest = {
+ "tasks": {
+ "android_defines_test": {
+ "description": "Builds an APK with a --dart-define ...",
+ "stage": "devicelab",
+ "required_agent_capabilities": ["linux/android"],
+ "on_luci": True,
},
"flavors_test": {
"description": "Checks that flavored builds work on Android.",
"stage": "devicelab",
"required_agent_capabilities": ["mac/android"],
+ "on_luci": True,
},
"flutter_gallery_ios__compile": {
"description": "Collects various performance metrics of ...",
"stage": "devicelab_ios",
"required_agent_capabilities": ["mac/ios"],
+ "on_luci": True,
},
},
}
-
- yield api.test(
- 'unknown_task',
- api.properties(task_name='unknown_task'),
- api.repo_util.flutter_environment_data(),
- api.step_data('read manifest.parse', api.json.output(sample_manifest)),
- api.expect_exception('ValueError'),
- )
-
- for task_name in [
- 'android_defines_test', 'flavors_test', 'flutter_gallery_ios__compile'
- ]:
+ for task_name in sample_manifest["tasks"].keys():
yield api.test(
task_name,
api.properties(
- git_ref='refs/pull/123/head', git_url='https://abc.com/repo'),
- api.properties(
- task_name=task_name,
+ role="worker",
+ name=task_name,
+ first_capability=(sample_manifest["tasks"][task_name]
+ ["required_agent_capabilities"][0]),
android_sdk_license='android_sdk_hash',
android_sdk_preview_license='android_sdk_preview_hash',
),
api.repo_util.flutter_environment_data(),
- api.step_data('read manifest.parse', api.json.output(sample_manifest)),
)
