Refactor compilation devicelab tests (#13218)
This fills out the following matrix:
App: `flutter create` default app, Complex Layout, and Gallery.
Platforms: Linux/Android, Mac/iOS, Windows/Android
Build modes: full release, aot blobs, debug
Dart: normal, 2.0 preview
It also renames all the tests that used "build" to mean compilation
build to use "compile" instead, so that it's much clearer on the
dashboard.
Also, mark some flaky tests that are no longer flaky as not flaky.
Also, remove uses of call() since that's probably going to go away at
some point.
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml
index 43542fb..c4badf5 100644
--- a/dev/devicelab/manifest.yaml
+++ b/dev/devicelab/manifest.yaml
@@ -32,23 +32,74 @@
# a custom task timeout, specified in minutes.
tasks:
- # Deviceless tests
+ # Tests of compiling in a variety of modes
- # TODO: make these not require "has-android-device"; it is only there to
- # ensure we have the Android SDK.
-
- complex_layout__build:
+ complex_layout_android__compile:
description: >
- Collects various performance metrics from AOT builds of the Complex
- Layout sample app.
+ Collects various performance metrics of compiling the Complex
+ Layout sample app for Android from Linux.
stage: devicelab
- required_agent_capabilities: ["has-android-device"]
+ required_agent_capabilities: ["linux/android"]
+ flaky: true
- basic_material_app__size:
+ complex_layout_ios__compile:
description: >
- Measures the APK size of a basic material app.
+ Collects various performance metrics of compiling the Complex
+ Layout sample app for iOS from Mac.
+ stage: devicelab_ios
+ required_agent_capabilities: ["has-ios-device"]
+ flaky: true
+
+ complex_layout_win__compile:
+ description: >
+ Collects various performance metrics of compiling the Complex
+ Layout for Android from Windows.
+ stage: devicelab_win
+ required_agent_capabilities: ["windows"]
+
+ basic_material_app_android__compile:
+ description: >
+ Collects various performance metrics of compiling the default
+ app for Android from Linux.
stage: devicelab
- required_agent_capabilities: ["has-android-device"]
+ required_agent_capabilities: ["linux/android"]
+ flaky: true
+
+ basic_material_app_ios__compile:
+ description: >
+ Collects various performance metrics of compiling the default
+ app for iOS from Mac.
+ stage: devicelab_ios
+ required_agent_capabilities: ["has-ios-device"]
+ flaky: true
+
+ basic_material_app_win__compile:
+ description: >
+ Collects various performance metrics of compiling the default
+ app for Android from Windows.
+ stage: devicelab_win
+ required_agent_capabilities: ["windows"]
+
+ flutter_gallery_android__compile:
+ description: >
+ Collects various performance metrics of compiling the Flutter
+ Gallery for Android from Linux.
+ stage: devicelab
+ required_agent_capabilities: ["linux/android"]
+
+ flutter_gallery_ios__compile:
+ description: >
+ Collects various performance metrics of compiling the Flutter
+ Gallery for iOS from Mac.
+ stage: devicelab_ios
+ required_agent_capabilities: ["has-ios-device"]
+
+ flutter_gallery_win__compile:
+ description: >
+ Collects various performance metrics of compiling the Flutter
+ Gallery for Android from Windows.
+ stage: devicelab_win
+ required_agent_capabilities: ["windows"]
# Android on-device tests
@@ -197,7 +248,6 @@
Checks that external UIs work on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
- flaky: true
channels_integration_test_ios:
description: >
@@ -229,14 +279,12 @@
Measures the startup time of the Flutter Gallery app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
- flaky: true
complex_layout_ios__start_up:
description: >
Measures the startup time of the Complex Layout sample app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
- flaky: true
flutter_gallery_ios__transition_perf:
description: >
@@ -245,12 +293,6 @@
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
- basic_material_app_ios__size:
- description: >
- Measures the IPA size of a basic material app.
- stage: devicelab_ios
- required_agent_capabilities: ["has-ios-device"]
-
microbenchmarks_ios:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
@@ -259,12 +301,12 @@
timeout_in_minutes: 30
flaky: true
-# flutter_view_ios__start_up:
-# description: >
-# Verifies that Flutter View can be used from an iOS project.
-# stage: devicelab_ios
-# required_agent_capabilities: ["has-ios-device"]
-# flaky: true
+ flutter_view_ios__start_up:
+ description: >
+ Verifies that Flutter View can be used from an iOS project.
+ stage: devicelab_ios
+ required_agent_capabilities: ["has-ios-device"]
+ flaky: true
integration_ui_ios:
description: >
@@ -292,13 +334,6 @@
stage: devicelab_win
required_agent_capabilities: ["windows"]
- flutter_gallery_win__build:
- description: >
- Collects various performance metrics from AOT builds of the Flutter
- Gallery on Windows.
- stage: devicelab_win
- required_agent_capabilities: ["windows"]
-
hot_mode_dev_cycle_win__benchmark:
description: >
Measures the performance of Dart VM hot patching feature on Windows.
@@ -325,20 +360,6 @@
stage: devicelab
required_agent_capabilities: ["linux/android"]
- flutter_gallery__build:
- description: >
- Collects various performance metrics from AOT builds of the Flutter
- Gallery.
- stage: devicelab
- required_agent_capabilities: ["linux/android"]
-
- flutter_gallery__preview_dart_2_build:
- description: >
- Collects various performance metrics from AOT builds of the Flutter
- Gallery under --preview-dart-2 that enables new Dart 2.0 frontend.
- stage: devicelab
- required_agent_capabilities: ["linux/android"]
-
flutter_gallery__start_up:
description: >
Measures the startup time of the Flutter Gallery app on Android.
