Shard firebase_test_lab_tests (#56594)
diff --git a/.cirrus.yml b/.cirrus.yml
index 7cb0ffb..85c1c5d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -1,6 +1,8 @@
# CIRRUS CONFIGURATION FILE
# https://cirrus-ci.org/guide/writing-tasks/
+# YAML anchors used to share fields between tasks.
+# See https://confluence.atlassian.com/bitbucket/yaml-anchors-960154027.html
web_shard_template: &WEB_SHARD_TEMPLATE
only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
environment:
@@ -32,6 +34,18 @@
- ulimit -S -n 2048 # https://github.com/flutter/flutter/issues/2976
- dart --enable-asserts ./dev/bots/test.dart
+firebase_shard_template: &FIREBASE_SHARD_TEMPLATE
+ environment:
+ # Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of
+ # October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM;
+ # doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it
+ # on the high end of the 20-25 minute range. (This makes sense, as it's just driving the
+ # Firebase test lab remotely.) Less than 4G of RAM made it go OOM.
+ CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+ GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
+ script:
+ - dart --enable-asserts ./dev/bots/test.dart
+
use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true'
environment:
@@ -268,17 +282,15 @@
- git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
- dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
- - name: firebase_test_lab_tests-linux # linux-only
- environment:
- # Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of
- # October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM;
- # doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it
- # on the high end of the 20-25 minute range. (This makes sense, as it's just driving the
- # Firebase test lab remotely.) Less than 4G of RAM made it go OOM.
- CLOUDSDK_CORE_DISABLE_PROMPTS: 1
- GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec]
- script:
- - ./dev/bots/firebase_testlab.sh
+ # firebase_test_lab_tests are linux-only
+ - name: firebase_test_lab_tests-0-linux
+ <<: *FIREBASE_SHARD_TEMPLATE
+
+ - name: firebase_test_lab_tests-1-linux
+ <<: *FIREBASE_SHARD_TEMPLATE
+
+ - name: firebase_test_lab_tests-2-linux
+ <<: *FIREBASE_SHARD_TEMPLATE
- name: web_smoke_test
only_if: "changesInclude('.cirrus.yml', 'examples/hello_world/**' ,'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/internal/**') || $CIRRUS_PR == ''"
diff --git a/dev/bots/firebase_testlab.sh b/dev/bots/firebase_testlab.sh
index e1e3d29..2ee7351 100755
--- a/dev/bots/firebase_testlab.sh
+++ b/dev/bots/firebase_testlab.sh
@@ -8,11 +8,8 @@
# that the app doesn't crash upon startup.
#
# When adding a test, ensure that there's at least a `print()` statement under lib/*.dart.
-tests=(
- "dev/integration_tests/release_smoke_test"
- "dev/integration_tests/abstract_method_smoke_test"
- "dev/integration_tests/android_embedding_v2_smoke_test"
-)
+#
+# The first and only parameter should be the path to an integration test.
# The devices where the tests are run.
#
@@ -79,6 +76,4 @@
popd
}
-for test in ${tests[*]}; do
- test_app_bundle $test
-done
+test_app_bundle "$1"
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index b84566a..5fc9d17 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -118,6 +118,7 @@
'add_to_app_tests': _runAddToAppTests,
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests,
+ 'firebase_test_lab_tests': _runFirebaseTestLabTests,
'framework_coverage': _runFrameworkCoverage,
'framework_tests': _runFrameworkTests,
'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
@@ -634,6 +635,34 @@
});
}
+Future<void> _runFirebaseTestLabTests() async {
+ // Firebase Lab tests take ~20 minutes per integration test,
+ // so only one test is run per shard. Therefore, there are as
+ // many shards available as there are integration tests in this list.
+ // If you add a new test, add a corresponding firebase_test_lab-#-linux
+ // to .cirrus.yml
+ final List<String> integrationTests = <String>[
+ 'release_smoke_test',
+ 'abstract_method_smoke_test',
+ 'android_embedding_v2_smoke_test',
+ ];
+ final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+
+ final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh');
+ final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests');
+
+ for (int index = 0; index < integrationTests.length; index += 1) {
+ final String integrationTestPath = path.join(integrationTestDirectory, integrationTests[index]);
+ subshards['$index'] = () => runCommand(
+ firebaseScript,
+ <String>[ integrationTestPath ],
+ workingDirectory: flutterRoot,
+ );
+ }
+
+ await selectSubshard(subshards);
+}
+
Future<void> _runFrameworkCoverage() async {
final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
if (!coverageFile.existsSync()) {
