blob: a11284411908ecd943bf551e923016c13da15c8b [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io' as io;
import 'package:file/file.dart';
import 'package:platform/platform.dart';
import 'package:uuid/uuid.dart';
import 'common/core.dart';
import 'common/gradle.dart';
import 'common/package_looping_command.dart';
import 'common/process_runner.dart';
import 'common/repository_package.dart';
const int _exitGcloudAuthFailed = 2;
/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
/// Creates an instance of the test runner command.
FirebaseTestLabCommand(
Directory packagesDir, {
ProcessRunner processRunner = const ProcessRunner(),
Platform platform = const LocalPlatform(),
}) : super(packagesDir, processRunner: processRunner, platform: platform) {
argParser.addOption(
'project',
defaultsTo: 'flutter-cirrus',
help: 'The Firebase project name.',
);
final String? homeDir = io.Platform.environment['HOME'];
argParser.addOption('service-key',
defaultsTo: homeDir == null
? null
: path.join(homeDir, 'gcloud-service-key.json'),
help: 'The path to the service key for gcloud authentication.\n'
r'If not provided, \$HOME/gcloud-service-key.json will be '
r'assumed if $HOME is set.');
argParser.addOption('test-run-id',
defaultsTo: const Uuid().v4(),
help:
'Optional string to append to the results path, to avoid conflicts. '
'Randomly chosen on each invocation if none is provided. '
'The default shown here is just an example.');
argParser.addOption('build-id',
defaultsTo:
io.Platform.environment['CIRRUS_BUILD_ID'] ?? 'unknown_build',
help:
'Optional string to append to the results path, to avoid conflicts. '
r'Defaults to $CIRRUS_BUILD_ID if that is set.');
argParser.addMultiOption('device',
splitCommas: false,
defaultsTo: <String>[
'model=walleye,version=26',
'model=redfin,version=30'
],
help:
'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info');
argParser.addOption('results-bucket',
defaultsTo: 'gs://flutter_cirrus_testlab');
argParser.addOption(
kEnableExperiment,
defaultsTo: '',
help: 'Enables the given Dart SDK experiments.',
);
}
@override
final String name = 'firebase-test-lab';
@override
final String description = 'Runs the instrumentation tests of the example '
'apps on Firebase Test Lab.\n\n'
'Runs tests in test_instrumentation folder using the '
'instrumentation_test package.';
bool _firebaseProjectConfigured = false;
Future<void> _configureFirebaseProject() async {
if (_firebaseProjectConfigured) {
return;
}
final String serviceKey = getStringArg('service-key');
if (serviceKey.isEmpty) {
print('No --service-key provided; skipping gcloud authorization');
} else {
final io.ProcessResult result = await processRunner.run(
'gcloud',
<String>[
'auth',
'activate-service-account',
'--key-file=$serviceKey',
],
logOnError: true,
);
if (result.exitCode != 0) {
printError('Unable to activate gcloud account.');
throw ToolExit(_exitGcloudAuthFailed);
}
final int exitCode = await processRunner.runAndStream('gcloud', <String>[
'config',
'set',
'project',
getStringArg('project'),
]);
print('');
if (exitCode == 0) {
print('Firebase project configured.');
} else {
logWarning(
'Warning: gcloud config set returned a non-zero exit code. Continuing anyway.');
}
}
_firebaseProjectConfigured = true;
}
@override
Future<PackageResult> runForPackage(RepositoryPackage package) async {
final List<PackageResult> results = <PackageResult>[];
for (final RepositoryPackage example in package.getExamples()) {
results.add(await _runForExample(example, package: package));
}
// If all results skipped, report skip overall.
if (results
.every((PackageResult result) => result.state == RunState.skipped)) {
return PackageResult.skip('No examples support Android.');
}
// Otherwise, report failure if there were any failures.
final List<String> allErrors = results
.map((PackageResult result) =>
result.state == RunState.failed ? result.details : <String>[])
.expand((List<String> list) => list)
.toList();
return allErrors.isEmpty
? PackageResult.success()
: PackageResult.fail(allErrors);
}
/// Runs the test for the given example of [package].
Future<PackageResult> _runForExample(
RepositoryPackage example, {
required RepositoryPackage package,
}) async {
final Directory androidDirectory =
example.platformDirectory(FlutterPlatform.android);
if (!androidDirectory.existsSync()) {
return PackageResult.skip(
'${example.displayName} does not support Android.');
}
final Directory uiTestDirectory = androidDirectory
.childDirectory('app')
.childDirectory('src')
.childDirectory('androidTest');
if (!uiTestDirectory.existsSync()) {
printError('No androidTest directory found.');
return PackageResult.fail(
<String>['No tests ran (use --exclude if this is intentional).']);
}
// Ensure that the Dart integration tests will be run, not just native UI
// tests.
if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) {
printError('No integration_test runner found. '
'See the integration_test package README for setup instructions.');
return PackageResult.fail(<String>['No integration_test runner.']);
}
// Ensures that gradle wrapper exists
final GradleProject project = GradleProject(example,
processRunner: processRunner, platform: platform);
if (!await _ensureGradleWrapperExists(project)) {
return PackageResult.fail(<String>['Unable to build example apk']);
}
await _configureFirebaseProject();
if (!await _runGradle(project, 'app:assembleAndroidTest')) {
return PackageResult.fail(<String>['Unable to assemble androidTest']);
}
final List<String> errors = <String>[];
// Used within the loop to ensure a unique GCS output location for each
// test file's run.
int resultsCounter = 0;
for (final File test in _findIntegrationTestFiles(example)) {
final String testName =
getRelativePosixPath(test, from: package.directory);
print('Testing $testName...');
if (!await _runGradle(project, 'app:assembleDebug', testFile: test)) {
printError('Could not build $testName');
errors.add('$testName failed to build');
continue;
}
final String buildId = getStringArg('build-id');
final String testRunId = getStringArg('test-run-id');
final String resultsDir =
'plugins_android_test/${package.displayName}/$buildId/$testRunId/'
'${example.directory.basename}/${resultsCounter++}/';
// Automatically retry failures; there is significant flake with these
// tests whose cause isn't yet understood, and having to re-run the
// entire shard for a flake in any one test is extremely slow. This should
// be removed once the root cause of the flake is understood.
// See https://github.com/flutter/flutter/issues/95063
const int maxRetries = 2;
bool passing = false;
for (int i = 1; i <= maxRetries && !passing; ++i) {
if (i > 1) {
logWarning('$testName failed on attempt ${i - 1}. Retrying...');
}
passing = await _runFirebaseTest(example, test, resultsDir: resultsDir);
}
if (!passing) {
printError('Test failure for $testName after $maxRetries attempts');
errors.add('$testName failed tests');
}
}
if (errors.isEmpty && resultsCounter == 0) {
printError('No integration tests were run.');
errors.add('No tests ran (use --exclude if this is intentional).');
}
return errors.isEmpty
? PackageResult.success()
: PackageResult.fail(errors);
}
/// Checks that Gradle has been configured for [project], and if not runs a
/// Flutter build to generate it.
///
/// Returns true if either gradlew was already present, or the build succeeds.
Future<bool> _ensureGradleWrapperExists(GradleProject project) async {
if (!project.isConfigured()) {
print('Running flutter build apk...');
final String experiment = getStringArg(kEnableExperiment);
final int exitCode = await processRunner.runAndStream(
flutterCommand,
<String>[
'build',
'apk',
if (experiment.isNotEmpty) '--enable-experiment=$experiment',
],
workingDir: project.androidDirectory);
if (exitCode != 0) {
return false;
}
}
return true;
}
/// Runs [test] from [example] as a Firebase Test Lab test, returning true if
/// the test passed.
///
/// [resultsDir] should be a unique-to-the-test-run directory to store the
/// results on the server.
Future<bool> _runFirebaseTest(
RepositoryPackage example,
File test, {
required String resultsDir,
}) async {
final List<String> args = <String>[
'firebase',
'test',
'android',
'run',
'--type',
'instrumentation',
'--app',
'build/app/outputs/apk/debug/app-debug.apk',
'--test',
'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
'--timeout',
'7m',
'--results-bucket=${getStringArg('results-bucket')}',
'--results-dir=$resultsDir',
for (final String device in getStringListArg('device')) ...<String>[
'--device',
device
],
];
final int exitCode = await processRunner.runAndStream('gcloud', args,
workingDir: example.directory);
return exitCode == 0;
}
/// Builds [target] using Gradle in the given [project]. Assumes Gradle is
/// already configured.
///
/// [testFile] optionally does the Flutter build with the given test file as
/// the build target.
///
/// Returns true if the command succeeds.
Future<bool> _runGradle(
GradleProject project,
String target, {
File? testFile,
}) async {
final String experiment = getStringArg(kEnableExperiment);
final String? extraOptions = experiment.isNotEmpty
? Uri.encodeComponent('--enable-experiment=$experiment')
: null;
final int exitCode = await project.runCommand(
target,
arguments: <String>[
'-Pverbose=true',
if (testFile != null) '-Ptarget=${testFile.path}',
if (extraOptions != null) '-Pextra-front-end-options=$extraOptions',
if (extraOptions != null) '-Pextra-gen-snapshot-options=$extraOptions',
],
);
if (exitCode != 0) {
return false;
}
return true;
}
/// Finds and returns all integration test files for [example].
Iterable<File> _findIntegrationTestFiles(RepositoryPackage example) sync* {
final Directory integrationTestDir =
example.directory.childDirectory('integration_test');
if (!integrationTestDir.existsSync()) {
return;
}
yield* integrationTestDir
.listSync(recursive: true)
.where((FileSystemEntity file) =>
file is File && file.basename.endsWith('_test.dart'))
.cast<File>();
}
/// Returns true if any of the test files in [uiTestDirectory] contain the
/// annotation that means that the test will reports the results of running
/// the Dart integration tests.
Future<bool> _testsContainDartIntegrationTestRunner(
Directory uiTestDirectory) async {
return uiTestDirectory
.list(recursive: true, followLinks: false)
.where((FileSystemEntity entity) => entity is File)
.cast<File>()
.any((File file) {
return file.basename.endsWith('.java') &&
file.readAsStringSync().contains('@RunWith(FlutterTestRunner.class)');
});
}
}