| # Describes the tasks we run in the continuous integration (CI) environment. |
| # |
| # Cocoon[1] uses this file to generate a checklist of tasks to be performed for |
| # every master commit. |
| # |
| # [1] github.com/flutter/cocoon |
| |
| # CI tasks. |
| # |
| # Each key in this dictionary is the unique name of a task, which also |
| # corresponds to a file in the "bin/" directory that the task runner will run. |
| # |
| # Required properties: |
| # |
| # * description: free form string |
| # describes what the task is about |
| # * stage: one of "devicelab", "devicelab_ios", "devicelab_win" |
| # tasks are grouped by stage so they appear next to each on the dashboard and have a distinct |
| # icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not |
| # run device tests on red Travis. |
| # * required_agent_capabilities: a list of strings |
| # list of capabilities a devicelab agent must have to be able to run this task. |
| # |
| # Optional properties: |
| # |
| # * flaky: boolean true or false |
| # whether the task is considered flaky; the result of running a flaky task does not affect |
| # the overall build status. Tests should be marked flaky when newly added, until they have |
| # been proved for a few cycles. Tests that are actually flaky but not being actively worked |
| # on should be hidden because they just cause confusion otherwise. |
| # * timeout_in_minutes: integer |
| # a custom task timeout, specified in minutes. |
| |
| tasks: |
| # Tests of compiling in a variety of modes |
| |
| complex_layout_android__compile: |
| description: > |
| Collects various performance metrics of compiling the Complex |
| Layout sample app for Android from Linux. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| complex_layout_ios__compile: |
| description: > |
| Collects various performance metrics of compiling the Complex |
| Layout sample app for iOS from Mac. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| complex_layout_win__compile: |
| description: > |
| Collects various performance metrics of compiling the Complex |
| Layout for Android from Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| basic_material_app_android__compile: |
| description: > |
| Collects various performance metrics of compiling the default |
| app for Android from Linux. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| basic_material_app_ios__compile: |
| description: > |
| Collects various performance metrics of compiling the default |
| app for iOS from Mac. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| basic_material_app_win__compile: |
| description: > |
| Collects various performance metrics of compiling the default |
| app for Android from Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| flutter_gallery_android__compile: |
| description: > |
| Collects various performance metrics of compiling the Flutter |
| Gallery for Android from Linux. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery_ios__compile: |
| description: > |
| Collects various performance metrics of compiling the Flutter |
| Gallery for iOS from Mac. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| flutter_gallery_win__compile: |
| description: > |
| Collects various performance metrics of compiling the Flutter |
| Gallery for Android from Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| # Android on-device tests |
| |
| complex_layout_scroll_perf__timeline_summary: |
| description: > |
| Measures the runtime performance of the Complex Layout sample app on |
| Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| flavors_test: |
| description: > |
| Checks that flavored builds work on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| channels_integration_test: |
| description: > |
| Checks that platform channels work on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| external_ui_integration_test: |
| description: > |
| Checks that external UIs work on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| platform_interaction_test: |
| description: > |
| Checks platform interaction on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| platform_channel_sample_test: |
| description: > |
| Runs a driver test on the Platform Channel sample app on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| complex_layout__start_up: |
| description: > |
| Measures the startup time of the Complex Layout sample app on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| hot_mode_dev_cycle__benchmark: |
| description: > |
| Measures the performance of Dart VM hot patching feature. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| hot_mode_dev_cycle__benchmark_dart1: |
| description: > |
| Measures the performance of Dart VM hot patching feature under |
| --no-preview-dart-2 option, that disables Dart 2.0 frontend. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| hot_mode_dev_cycle_ios__benchmark_dart1: |
| description: > |
| Measures the performance of Dart VM hot patching feature under |
| --no-preview-dart-2 option, that disables Dart 2.0 frontend. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| flaky: true |
| |
| complex_layout_scroll_perf__memory: |
| description: > |
| Measures memory usage of the scroll performance test. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| hello_world__memory: |
| description: > |
| Measures starting memory usage of the simplest Flutter app. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| microbenchmarks: |
| description: > |
| Runs benchmarks from dev/benchmarks/microbenchmarks. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| timeout_in_minutes: 30 |
| |
| flutter_view__start_up: |
| description: > |
| Verifies that Flutter View can be used from an Android project. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| integration_ui: |
| description: > |
| Runs end-to-end Flutter tests on Android. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| flaky: true |
| |
| commands_test: |
| description: > |
| Runs tests of flutter run commands. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| run_machine_concurrent_hot_reload: |
| description: > |
| Runs tests of concurrent hot reload commands via flutter run --machine. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| service_extensions_test: |
| description: > |
| Validates our service protocol extensions. |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| android_sample_catalog_generator: |
| description: > |
| Builds sample catalog markdown pages and Android screenshots |
| stage: devicelab |
| required_agent_capabilities: ["mac/android"] |
| |
| complex_layout_semantics_perf: |
| description: > |
| Measures duration of building the initial semantics tree. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| routing_test: |
| description: > |
| Verifies that `flutter drive --route` still works. No performance numbers. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| gradle_plugin_test: |
| description: > |
| Verifies that the Flutter Gradle plugin supports standard and custom Android build types. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| plugin_test: |
| description: > |
| Checks that the project template works and supports plugins. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery_instrumentation_test: |
| description: > |
| Same as flutter_gallery__transition_perf but uses Android instrumentation |
| framework, and therefore does not require a host computer to run. This |
| test can run on off-the-shelf infrastructures, such as Firebase Test Lab. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| # iOS on-device tests |
| |
| flavors_test_ios: |
| description: > |
| Checks that flavored builds work on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| plugin_test_ios: |
| description: > |
| Checks that the project template works and supports plugins on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| external_ui_integration_test_ios: |
| description: > |
| Checks that external UIs work on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| channels_integration_test_ios: |
| description: > |
| Checks that platform channels work on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| platform_interaction_test_ios: |
| description: > |
| Checks platform interaction on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| platform_channel_sample_test_ios: |
| description: > |
| Runs a driver test on the Platform Channel sample app on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| complex_layout_scroll_perf_ios__timeline_summary: |
| description: > |
| Measures the runtime performance of the Complex Layout sample app on |
| iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| flutter_gallery_ios__start_up: |
| description: > |
| Measures the startup time of the Flutter Gallery app on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| complex_layout_ios__start_up: |
| description: > |
| Measures the startup time of the Complex Layout sample app on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| flutter_gallery_ios__transition_perf: |
| description: > |
| Measures the performance of screen transitions in Flutter Gallery on |
| iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| flaky: true |
| |
| microbenchmarks_ios: |
| description: > |
| Runs benchmarks from dev/benchmarks/microbenchmarks on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| timeout_in_minutes: 30 |
| |
| flutter_view_ios__start_up: |
| description: > |
| Verifies that Flutter View can be used from an iOS project. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| integration_ui_ios: |
| description: > |
| Runs end-to-end Flutter tests on iOS. |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| timeout_in_minutes: 20 |
| flaky: true |
| |
| ios_sample_catalog_generator: |
| description: > |
| Builds sample catalog markdown pages and iOS screenshots |
| stage: devicelab_ios |
| required_agent_capabilities: ["mac/ios"] |
| |
| # Tests running on Windows host |
| |
| flavors_test_win: |
| description: > |
| Checks that flavored builds work on Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| channels_integration_test_win: |
| description: > |
| Checks that platform channels work when app is launched from Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| plugin_test_win: |
| description: > |
| Checks that the project template works and supports plugins on Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| hot_mode_dev_cycle_win__benchmark: |
| description: > |
| Measures the performance of Dart VM hot patching feature on Windows. |
| stage: devicelab_win |
| required_agent_capabilities: ["windows/android"] |
| |
| # Tests running on Linux hosts |
| |
| hot_mode_dev_cycle_linux__benchmark: |
| description: > |
| Measures the performance of Dart VM hot patching feature on a Linux host. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| dartdocs: |
| description: > |
| Tracks how many members are still lacking documentation. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| technical_debt__cost: |
| description: > |
| Estimates our technical debt (TODOs, analyzer ignores, etc). |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery__start_up: |
| description: > |
| Measures the startup time of the Flutter Gallery app on Android. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery__transition_perf: |
| description: > |
| Measures the performance of screen transitions in Flutter Gallery on |
| Android. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery__transition_perf_with_semantics: |
| description: > |
| Measures the delta in performance of screen transitions without and |
| with semantics enabled. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery__memory_nav: |
| description: > |
| Measures memory usage after repeated navigation in Gallery. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| flutter_gallery__back_button_memory: |
| description: > |
| Measures memory usage after Android app suspend and resume. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
| |
| analyzer_benchmark: |
| description: > |
| Measures the speed of Dart analyzer. |
| stage: devicelab |
| required_agent_capabilities: ["linux/android"] |
