blob: 2546cd51e8f2934de3655629304ac0e93b9cc909 [file] [log] [blame]
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon
# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
# Required properties:
#
# * description: free form string
# describes what the task is about
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
# run device tests on red Cirrus builds.
# * required_agent_capabilities: a list of strings
# list of capabilities a devicelab agent must have to be able to run this task.
#
# Optional properties:
#
# * flaky: boolean true or false
# whether the task is considered flaky; the result of running a flaky task does not affect
# the overall build status. Tests should be marked flaky when newly added, until they have
# been proved for a few cycles. Tests that are actually flaky but not being actively worked
# on should be hidden because they just cause confusion otherwise.
# * timeout_in_minutes: integer
# a custom task timeout, specified in minutes.
tasks:
# Tests of compiling in a variety of modes
complex_layout_android__compile:
description: >
Collects various performance metrics of compiling the Complex
Layout sample app for Android from Linux.
stage: devicelab
required_agent_capabilities: ["linux/android"]
complex_layout_ios__compile:
description: >
Collects various performance metrics of compiling the Complex
Layout sample app for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_win__compile:
description: >
Collects various performance metrics of compiling the Complex
Layout for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
basic_material_app_android__compile:
description: >
Collects various performance metrics of compiling the default
app for Android from Linux.
stage: devicelab
required_agent_capabilities: ["linux/android"]
basic_material_app_ios__compile:
description: >
Collects various performance metrics of compiling the default
app for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
basic_material_app_win__compile:
description: >
Collects various performance metrics of compiling the default
app for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
codegen_integration_win:
description: >
Runs codegeneration and verifies that it can execute
correctly.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
flaky: true
codegen_integration_mac:
description: >
Runs codegeneration and verifies that it can execute
correctly.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flaky: true
codegen_integration_linux:
description: >
Runs codegeneration and verifies that it can execute
correctly.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flaky: true
flutter_gallery_android__compile:
description: >
Collects various performance metrics of compiling the Flutter
Gallery for Android from Linux.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery_ios__compile:
description: >
Collects various performance metrics of compiling the Flutter
Gallery for iOS from Mac.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_win__compile:
description: >
Collects various performance metrics of compiling the Flutter
Gallery for Android from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
# Android on-device tests
complex_layout_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
tiles_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of the tiles tab in the Complex Layout sample app on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
home_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of scrolling the material page in the
flutter_gallery app on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
cull_opacity_perf__timeline_summary:
description: >
Measures the runtime performance of culling opacity widgets on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
cubic_bezier_perf__timeline_summary:
description: >
Measures the runtime performance of cubic bezier animations on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
flaky: true
flavors_test:
description: >
Checks that flavored builds work on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
channels_integration_test:
description: >
Checks that platform channels work on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
external_ui_integration_test:
description: >
Checks that external UIs work on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
embedded_android_views_integration_test:
description: >
Tests embedded Android views.
stage: devicelab
required_agent_capabilities: ["mac/android"]
android_semantics_integration_test:
description: >
Tests that the Android accessibility bridge produces correct semantics.
stage: devicelab
required_agent_capabilities: ["mac/android"]
run_release_test:
description: >
Checks that `flutter run --release` does not crash.
stage: devicelab
required_agent_capabilities: ["mac/android"]
platform_interaction_test:
description: >
Checks platform interaction on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
platform_channel_sample_test:
description: >
Runs a driver test on the Platform Channel sample app on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
platform_view__start_up:
description: >
Verifies that Platform View can be used from an Android project.
stage: devicelab
required_agent_capabilities: ["mac/android"]
complex_layout__start_up:
description: >
Measures the startup time of the Complex Layout sample app on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
hot_mode_dev_cycle__benchmark:
description: >
Measures the performance of Dart VM hot patching feature.
stage: devicelab
required_agent_capabilities: ["mac/android"]
complex_layout_scroll_perf__memory:
description: >
Measures memory usage of the scroll performance test.
stage: devicelab
required_agent_capabilities: ["mac/android"]
hello_world_android__compile:
description: >
Measures the APK size of Hello World.
stage: devicelab
required_agent_capabilities: ["mac/android"]
hello_world__memory:
description: >
Measures starting memory usage of the simplest Flutter app.
stage: devicelab
required_agent_capabilities: ["mac/android"]
microbenchmarks:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks.
stage: devicelab
required_agent_capabilities: ["mac/android"]
timeout_in_minutes: 30
flutter_view__start_up:
description: >
Verifies that Flutter View can be used from an Android project.
stage: devicelab
required_agent_capabilities: ["mac/android"]
integration_ui:
description: >
Runs end-to-end Flutter tests on Android.
stage: devicelab
required_agent_capabilities: ["mac/android"]
commands_test:
description: >
Runs tests of flutter run commands.
stage: devicelab
required_agent_capabilities: ["mac/android"]
run_machine_concurrent_hot_reload:
description: >
Runs tests of concurrent hot reload commands via flutter run --machine.
stage: devicelab
required_agent_capabilities: ["mac/android"]
service_extensions_test:
description: >
Validates our service protocol extensions.
stage: devicelab
required_agent_capabilities: ["mac/android"]
android_sample_catalog_generator:
description: >
Builds sample catalog markdown pages and Android screenshots
stage: devicelab
required_agent_capabilities: ["mac/android"]
complex_layout_semantics_perf:
description: >
Measures duration of building the initial semantics tree.
stage: devicelab
required_agent_capabilities: ["linux/android"]
routing_test:
description: >
Verifies that `flutter drive --route` still works. No performance numbers.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# TODO(dnfield): Remove these once we're confident it's all working on cirrus
# gradle_plugin_test:
# description: >
# Verifies that the Flutter Gradle plugin supports standard and custom Android build types.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# plugin_test:
# description: >
# Checks that the project template works and supports plugins.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# module_test:
# description: >
# Checks that the module project template works and supports add2app on Android.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
flutter_gallery_instrumentation_test:
description: >
Same as flutter_gallery__transition_perf but uses Android instrumentation
framework, and therefore does not require a host computer to run. This
test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_attach_test:
description: >
Tests the `flutter attach` command.
stage: devicelab
required_agent_capabilities: ["linux/android"]
named_isolates_test:
description: >
Tests naming and attaching to specific isolates.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# TODO(dnfield): Remove these once we're confident it's all working on cirrus
# flutter_create_offline_test_linux:
# description: >
# Tests the `flutter create --offline` command.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# flutter_create_offline_test_windows:
# description: >
# Tests the `flutter create --offline` command.
# stage: devicelab
# required_agent_capabilities: ["windows/android"]
# flutter_create_offline_test_mac:
# description: >
# Tests the `flutter create --offline` command.
# stage: devicelab
# required_agent_capabilities: ["mac/android"]
# iOS on-device tests
flavors_test_ios:
description: >
Checks that flavored builds work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
# TODO(dnfield): Remove these once we're confident it's all working on cirrus
# plugin_test_ios:
# description: >
# Checks that the project template works and supports plugins on iOS.
# stage: devicelab_ios
# required_agent_capabilities: ["mac/ios"]
# module_test_ios:
# description: >
# Checks that the module project template works and supports add2app on iOS.
# stage: devicelab
# required_agent_capabilities: ["mac/ios"]
external_ui_integration_test_ios:
description: >
Checks that external UIs work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
channels_integration_test_ios:
description: >
Checks that platform channels work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_interaction_test_ios:
description: >
Checks platform interaction on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_channel_sample_test_ios:
description: >
Runs a driver test on the Platform Channel sample app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
platform_view_ios__start_up:
description: >
Verifies that Platform View can be used from an iOS project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_ios__start_up:
description: >
Measures the startup time of the Flutter Gallery app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
complex_layout_ios__start_up:
description: >
Measures the startup time of the Complex Layout sample app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
flutter_gallery_ios__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
hello_world_ios__compile:
description: >
Measures the IPA size of Hello World.
stage: devicelab
required_agent_capabilities: ["mac/ios"]
microbenchmarks_ios:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
timeout_in_minutes: 30
flutter_view_ios__start_up:
description: >
Verifies that Flutter View can be used from an iOS project.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
integration_ui_ios:
description: >
Runs end-to-end Flutter tests on iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
timeout_in_minutes: 20
flaky: true
ios_sample_catalog_generator:
description: >
Builds sample catalog markdown pages and iOS screenshots
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
# Tests running on Windows host
flavors_test_win:
description: >
Checks that flavored builds work on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
channels_integration_test_win:
description: >
Checks that platform channels work when app is launched from Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
plugin_test_win:
description: >
Checks that the project template works and supports plugins on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
hot_mode_dev_cycle_win__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on Windows.
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
# Tests running on Linux hosts
hot_mode_dev_cycle_linux__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on a Linux host.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# TODO(dnfield): Remove these once we're confident it's all working on cirrus
# dartdocs:
# description: >
# Tracks how many members are still lacking documentation.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
flutter_test_performance:
description: >
Measures performance of running flutter test.
stage: devicelab
required_agent_capabilities: ["linux/android"]
technical_debt__cost:
description: >
Estimates our technical debt (TODOs, analyzer ignores, etc).
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__start_up:
description: >
Measures the startup time of the Flutter Gallery app on Android.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
Android.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__transition_perf_with_semantics:
description: >
Measures the delta in performance of screen transitions without and
with semantics enabled.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__memory_nav:
description: >
Measures memory usage after repeated navigation in Gallery.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery__back_button_memory:
description: >
Measures memory usage after Android app suspend and resume.
stage: devicelab
required_agent_capabilities: ["linux/android"]
analyzer_benchmark:
description: >
Measures the speed of Dart analyzer.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_gallery_ios32__start_up:
description: >
Measures the startup time of the Flutter Gallery app on 32-bit iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios32"]
flutter_gallery_ios32__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
32-bit iOS.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios32"]