blob: a11284411908ecd943bf551e923016c13da15c8b [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io' as io;
import 'package:file/file.dart';
import 'package:platform/platform.dart';
import 'package:uuid/uuid.dart';
import 'common/core.dart';
import 'common/gradle.dart';
import 'common/package_looping_command.dart';
import 'common/process_runner.dart';
import 'common/repository_package.dart';
const int _exitGcloudAuthFailed = 2;
/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
/// Creates an instance of the test runner command.
Directory packagesDir, {
ProcessRunner processRunner = const ProcessRunner(),
Platform platform = const LocalPlatform(),
}) : super(packagesDir, processRunner: processRunner, platform: platform) {
defaultsTo: 'flutter-cirrus',
help: 'The Firebase project name.',
final String? homeDir = io.Platform.environment['HOME'];
defaultsTo: homeDir == null
? null
: path.join(homeDir, 'gcloud-service-key.json'),
help: 'The path to the service key for gcloud authentication.\n'
r'If not provided, \$HOME/gcloud-service-key.json will be '
r'assumed if $HOME is set.');
defaultsTo: const Uuid().v4(),
'Optional string to append to the results path, to avoid conflicts. '
'Randomly chosen on each invocation if none is provided. '
'The default shown here is just an example.');
io.Platform.environment['CIRRUS_BUILD_ID'] ?? 'unknown_build',
'Optional string to append to the results path, to avoid conflicts. '
r'Defaults to $CIRRUS_BUILD_ID if that is set.');
splitCommas: false,
defaultsTo: <String>[
'Device model(s) to test. See for more info');
defaultsTo: 'gs://flutter_cirrus_testlab');
defaultsTo: '',
help: 'Enables the given Dart SDK experiments.',
final String name = 'firebase-test-lab';
final String description = 'Runs the instrumentation tests of the example '
'apps on Firebase Test Lab.\n\n'
'Runs tests in test_instrumentation folder using the '
'instrumentation_test package.';
bool _firebaseProjectConfigured = false;
Future<void> _configureFirebaseProject() async {
if (_firebaseProjectConfigured) {
final String serviceKey = getStringArg('service-key');
if (serviceKey.isEmpty) {
print('No --service-key provided; skipping gcloud authorization');
} else {
final io.ProcessResult result = await
logOnError: true,
if (result.exitCode != 0) {
printError('Unable to activate gcloud account.');
throw ToolExit(_exitGcloudAuthFailed);
final int exitCode = await processRunner.runAndStream('gcloud', <String>[
if (exitCode == 0) {
print('Firebase project configured.');
} else {
'Warning: gcloud config set returned a non-zero exit code. Continuing anyway.');
_firebaseProjectConfigured = true;
Future<PackageResult> runForPackage(RepositoryPackage package) async {
final List<PackageResult> results = <PackageResult>[];
for (final RepositoryPackage example in package.getExamples()) {
results.add(await _runForExample(example, package: package));
// If all results skipped, report skip overall.
if (results
.every((PackageResult result) => result.state == RunState.skipped)) {
return PackageResult.skip('No examples support Android.');
// Otherwise, report failure if there were any failures.
final List<String> allErrors = results
.map((PackageResult result) =>
result.state == RunState.failed ? result.details : <String>[])
.expand((List<String> list) => list)
return allErrors.isEmpty
? PackageResult.success()
/// Runs the test for the given example of [package].
Future<PackageResult> _runForExample(
RepositoryPackage example, {
required RepositoryPackage package,
}) async {
final Directory androidDirectory =
if (!androidDirectory.existsSync()) {
return PackageResult.skip(
'${example.displayName} does not support Android.');
final Directory uiTestDirectory = androidDirectory
if (!uiTestDirectory.existsSync()) {
printError('No androidTest directory found.');
<String>['No tests ran (use --exclude if this is intentional).']);
// Ensure that the Dart integration tests will be run, not just native UI
// tests.
if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) {
printError('No integration_test runner found. '
'See the integration_test package README for setup instructions.');
return<String>['No integration_test runner.']);
// Ensures that gradle wrapper exists
final GradleProject project = GradleProject(example,
processRunner: processRunner, platform: platform);
if (!await _ensureGradleWrapperExists(project)) {
return<String>['Unable to build example apk']);
await _configureFirebaseProject();
if (!await _runGradle(project, 'app:assembleAndroidTest')) {
return<String>['Unable to assemble androidTest']);
final List<String> errors = <String>[];
// Used within the loop to ensure a unique GCS output location for each
// test file's run.
int resultsCounter = 0;
for (final File test in _findIntegrationTestFiles(example)) {
final String testName =
getRelativePosixPath(test, from:;
print('Testing $testName...');
if (!await _runGradle(project, 'app:assembleDebug', testFile: test)) {
printError('Could not build $testName');
errors.add('$testName failed to build');
final String buildId = getStringArg('build-id');
final String testRunId = getStringArg('test-run-id');
final String resultsDir =
// Automatically retry failures; there is significant flake with these
// tests whose cause isn't yet understood, and having to re-run the
// entire shard for a flake in any one test is extremely slow. This should
// be removed once the root cause of the flake is understood.
// See
const int maxRetries = 2;
bool passing = false;
for (int i = 1; i <= maxRetries && !passing; ++i) {
if (i > 1) {
logWarning('$testName failed on attempt ${i - 1}. Retrying...');
passing = await _runFirebaseTest(example, test, resultsDir: resultsDir);
if (!passing) {
printError('Test failure for $testName after $maxRetries attempts');
errors.add('$testName failed tests');
if (errors.isEmpty && resultsCounter == 0) {
printError('No integration tests were run.');
errors.add('No tests ran (use --exclude if this is intentional).');
return errors.isEmpty
? PackageResult.success()
/// Checks that Gradle has been configured for [project], and if not runs a
/// Flutter build to generate it.
/// Returns true if either gradlew was already present, or the build succeeds.
Future<bool> _ensureGradleWrapperExists(GradleProject project) async {
if (!project.isConfigured()) {
print('Running flutter build apk...');
final String experiment = getStringArg(kEnableExperiment);
final int exitCode = await processRunner.runAndStream(
if (experiment.isNotEmpty) '--enable-experiment=$experiment',
workingDir: project.androidDirectory);
if (exitCode != 0) {
return false;
return true;
/// Runs [test] from [example] as a Firebase Test Lab test, returning true if
/// the test passed.
/// [resultsDir] should be a unique-to-the-test-run directory to store the
/// results on the server.
Future<bool> _runFirebaseTest(
RepositoryPackage example,
File test, {
required String resultsDir,
}) async {
final List<String> args = <String>[
for (final String device in getStringListArg('device')) ...<String>[
final int exitCode = await processRunner.runAndStream('gcloud', args,
return exitCode == 0;
/// Builds [target] using Gradle in the given [project]. Assumes Gradle is
/// already configured.
/// [testFile] optionally does the Flutter build with the given test file as
/// the build target.
/// Returns true if the command succeeds.
Future<bool> _runGradle(
GradleProject project,
String target, {
File? testFile,
}) async {
final String experiment = getStringArg(kEnableExperiment);
final String? extraOptions = experiment.isNotEmpty
? Uri.encodeComponent('--enable-experiment=$experiment')
: null;
final int exitCode = await project.runCommand(
arguments: <String>[
if (testFile != null) '-Ptarget=${testFile.path}',
if (extraOptions != null) '-Pextra-front-end-options=$extraOptions',
if (extraOptions != null) '-Pextra-gen-snapshot-options=$extraOptions',
if (exitCode != 0) {
return false;
return true;
/// Finds and returns all integration test files for [example].
Iterable<File> _findIntegrationTestFiles(RepositoryPackage example) sync* {
final Directory integrationTestDir ='integration_test');
if (!integrationTestDir.existsSync()) {
yield* integrationTestDir
.listSync(recursive: true)
.where((FileSystemEntity file) =>
file is File && file.basename.endsWith('_test.dart'))
/// Returns true if any of the test files in [uiTestDirectory] contain the
/// annotation that means that the test will reports the results of running
/// the Dart integration tests.
Future<bool> _testsContainDartIntegrationTestRunner(
Directory uiTestDirectory) async {
return uiTestDirectory
.list(recursive: true, followLinks: false)
.where((FileSystemEntity entity) => entity is File)
.any((File file) {
return file.basename.endsWith('.java') &&