blob: 052d888a14df2f832b86056d485b7b86abf698be [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:args/args.dart';
import 'package:meta/meta.dart';
import 'package:package_config/package_config_types.dart';
import '../android/android_device.dart';
import '../application_package.dart';
import '../artifacts.dart';
import '../base/common.dart';
import '../base/file_system.dart';
import '../base/io.dart';
import '../base/logger.dart';
import '../base/platform.dart';
import '../base/signals.dart';
import '../base/utils.dart';
import '../build_info.dart';
import '../dart/package_map.dart';
import '../device.dart';
import '../drive/drive_service.dart';
import '../drive/web_driver_service.dart' show Browser;
import '../globals.dart' as globals;
import '../ios/devices.dart';
import '../resident_runner.dart';
import '../runner/flutter_command.dart' show FlutterCommandCategory, FlutterCommandResult, FlutterOptions;
import '../web/web_device.dart';
import 'run.dart';
/// Runs integration (a.k.a. end-to-end) tests.
///
/// An integration test is a program that runs in a separate process from your
/// Flutter application. It connects to the application and acts like a user,
/// performing taps, scrolls, reading out widget properties and verifying their
/// correctness.
///
/// This command takes a target Flutter application that you would like to test
/// as the `--target` option (defaults to `lib/main.dart`). It then looks for a
/// corresponding test file within the `test_driver` directory. The test file is
/// expected to have the same name but contain the `_test.dart` suffix. The
/// `_test.dart` file would generally be a Dart program that uses
/// `package:flutter_driver` and exercises your application. Most commonly it
/// is a test written using `package:test`, but you are free to use something
/// else.
///
/// The app and the test are launched simultaneously. Once the test completes
/// the application is stopped and the command exits. If all these steps are
/// successful the exit code will be `0`. Otherwise, you will see a non-zero
/// exit code.
class DriveCommand extends RunCommandBase {
DriveCommand({
bool verboseHelp = false,
@visibleForTesting FlutterDriverFactory? flutterDriverFactory,
@visibleForTesting this.signalsToHandle = const <ProcessSignal>{ProcessSignal.sigint, ProcessSignal.sigterm},
required FileSystem fileSystem,
required Logger logger,
required Platform platform,
required this.signals,
}) : _flutterDriverFactory = flutterDriverFactory,
_fileSystem = fileSystem,
_logger = logger,
_fsUtils = FileSystemUtils(fileSystem: fileSystem, platform: platform),
super(verboseHelp: verboseHelp) {
requiresPubspecYaml();
addEnableExperimentation(hide: !verboseHelp);
// By default, the drive app should not publish the VM service port over mDNS
// to prevent a local network permission dialog on iOS 14+,
// which cannot be accepted or dismissed in a CI environment.
addPublishPort(enabledByDefault: false, verboseHelp: verboseHelp);
addMultidexOption();
argParser
..addFlag('keep-app-running',
help: 'Will keep the Flutter application running when done testing.\n'
'By default, "flutter drive" stops the application after tests are finished, '
'and "--keep-app-running" overrides this. On the other hand, if "--use-existing-app" '
'is specified, then "flutter drive" instead defaults to leaving the application '
'running, and "--no-keep-app-running" overrides it.',
)
..addOption('use-existing-app',
help: 'Connect to an already running instance via the given Dart VM Service URL. '
'If this option is given, the application will not be automatically started, '
'and it will only be stopped if "--no-keep-app-running" is explicitly set.',
valueHelp: 'url',
)
..addOption('driver',
help: 'The test file to run on the host (as opposed to the target file to run on '
'the device).\n'
'By default, this file has the same base name as the target file, but in the '
'"test_driver/" directory instead, and with "_test" inserted just before the '
'extension, so e.g. if the target is "lib/main.dart", the driver will be '
'"test_driver/main_test.dart".',
valueHelp: 'path',
)
..addFlag('build',
defaultsTo: true,
help: '(deprecated) Build the app before running. To use an existing app, pass the "--${FlutterOptions.kUseApplicationBinary}" '
'flag with an existing APK.',
)
..addOption('screenshot',
valueHelp: 'path/to/directory',
help: 'Directory location to write screenshots on test failure.',
)
..addOption('driver-port',
defaultsTo: '4444',
help: 'The port where Webdriver server is launched at.',
valueHelp: '4444'
)
..addFlag('headless',
defaultsTo: true,
help: 'Whether the driver browser is going to be launched in headless mode.',
)
..addOption(
'browser-name',
defaultsTo: Browser.chrome.cliName,
help: 'Name of the browser where tests will be executed.',
allowed: Browser.values.map((Browser e) => e.cliName),
allowedHelp: CliEnum.allowedHelp(Browser.values),
)
..addOption('browser-dimension',
defaultsTo: '1600,1024',
help: 'The dimension of the browser when running a Flutter Web test. '
'This will affect screenshot and all offset-related actions.',
valueHelp: 'width,height',
)
..addFlag('android-emulator',
defaultsTo: true,
help: 'Whether to perform Flutter Driver testing using an Android Emulator. '
'Works only if "browser-name" is set to "android-chrome".')
..addOption('chrome-binary',
help: 'Location of the Chrome binary. '
'Works only if "browser-name" is set to "chrome".')
..addOption('write-sksl-on-exit',
help: 'Attempts to write an SkSL file when the drive process is finished '
'to the provided file, overwriting it if necessary.')
..addMultiOption('test-arguments', help: 'Additional arguments to pass to the '
'Dart VM running The test script.')
..addOption('profile-memory', help: 'Launch devtools and profile application memory, writing '
'The output data to the file path provided to this argument as JSON.',
valueHelp: 'profile_memory.json')
..addOption('timeout',
help: 'Timeout the test after the given number of seconds. If the '
'"--screenshot" option is provided, a screenshot will be taken '
'before exiting. Defaults to no timeout.',
valueHelp: '360');
}
final Signals signals;
/// The [ProcessSignal]s that will lead to a screenshot being taken (if the option is provided).
final Set<ProcessSignal> signalsToHandle;
// `pub` must always be run due to the test script running from source,
// even if an application binary is used. Default to true unless the user explicitly
// specified not to.
@override
bool get shouldRunPub {
if (argResults!.wasParsed('pub') && !boolArg('pub')) {
return false;
}
return true;
}
FlutterDriverFactory? _flutterDriverFactory;
final FileSystem _fileSystem;
final Logger _logger;
final FileSystemUtils _fsUtils;
Timer? timeoutTimer;
Map<ProcessSignal, Object>? screenshotTokens;
@override
final String name = 'drive';
@override
final String description = 'Run integration tests for the project on an attached device or emulator.';
@override
String get category => FlutterCommandCategory.project;
@override
final List<String> aliases = <String>['driver'];
String? get userIdentifier => stringArg(FlutterOptions.kDeviceUser);
String? get screenshot => stringArg('screenshot');
@override
bool get startPausedDefault => true;
@override
bool get cachePubGet => false;
String? get applicationBinaryPath => stringArg(FlutterOptions.kUseApplicationBinary);
Future<Device?> get targetedDevice async {
return findTargetDevice(
includeDevicesUnsupportedByProject: applicationBinaryPath == null,
);
}
// Wireless iOS devices need `publish-port` to be enabled because it requires mDNS.
// If the flag wasn't provided as an actual argument and it's a wireless device,
// change it to be enabled.
@override
Future<bool> get disablePortPublication async {
final ArgResults? localArgResults = argResults;
final Device? device = await targetedDevice;
final bool isWirelessIOSDevice = device is IOSDevice && device.isWirelesslyConnected;
if (isWirelessIOSDevice && localArgResults != null && !localArgResults.wasParsed('publish-port')) {
_logger.printTrace('A wireless iOS device is being used. Changing `publish-port` to be enabled.');
return false;
}
return !boolArg('publish-port');
}
@override
Future<void> validateCommand() async {
if (userIdentifier != null) {
final Device? device = await findTargetDevice();
if (device is! AndroidDevice) {
throwToolExit('--${FlutterOptions.kDeviceUser} is only supported for Android');
}
}
return super.validateCommand();
}
@override
Future<FlutterCommandResult> runCommand() async {
final String? testFile = _getTestFile();
if (testFile == null) {
throwToolExit(null);
}
if (await _fileSystem.type(testFile) != FileSystemEntityType.file) {
throwToolExit('Test file not found: $testFile');
}
final Device? device = await targetedDevice;
if (device == null) {
throwToolExit(null);
}
if (screenshot != null && !device.supportsScreenshot) {
_logger.printError('Screenshot not supported for ${device.name}.');
}
final bool web = device is WebServerDevice || device is ChromiumDevice;
_flutterDriverFactory ??= FlutterDriverFactory(
applicationPackageFactory: ApplicationPackageFactory.instance!,
logger: _logger,
processUtils: globals.processUtils,
dartSdkPath: globals.artifacts!.getArtifactPath(Artifact.engineDartBinary),
devtoolsLauncher: DevtoolsLauncher.instance!,
);
final PackageConfig packageConfig = await loadPackageConfigWithLogging(
_fileSystem.file('.packages'),
logger: _logger,
throwOnError: false,
);
final DriverService driverService = _flutterDriverFactory!.createDriverService(web);
final BuildInfo buildInfo = await getBuildInfo();
final DebuggingOptions debuggingOptions = await createDebuggingOptions(web);
final File? applicationBinary = applicationBinaryPath == null
? null
: _fileSystem.file(applicationBinaryPath);
bool screenshotTaken = false;
try {
if (stringArg('use-existing-app') == null) {
await driverService.start(
buildInfo,
device,
debuggingOptions,
ipv6 ?? false,
applicationBinary: applicationBinary,
route: route,
userIdentifier: userIdentifier,
mainPath: targetFile,
platformArgs: <String, Object>{
if (traceStartup)
'trace-startup': traceStartup,
if (web)
'--no-launch-chrome': true,
if (boolArg('multidex'))
'multidex': true,
}
);
} else {
final Uri? uri = Uri.tryParse(stringArg('use-existing-app')!);
if (uri == null) {
throwToolExit('Invalid VM Service URI: ${stringArg('use-existing-app')}');
}
await driverService.reuseApplication(
uri,
device,
debuggingOptions,
ipv6 ?? false,
);
}
final Future<int> testResultFuture = driverService.startTest(
testFile,
stringsArg('test-arguments'),
<String, String>{},
packageConfig,
chromeBinary: stringArg('chrome-binary'),
headless: boolArg('headless'),
webBrowserFlags: stringsArg(FlutterOptions.kWebBrowserFlag),
browserDimension: stringArg('browser-dimension')!.split(','),
browserName: stringArg('browser-name'),
driverPort: stringArg('driver-port') != null
? int.tryParse(stringArg('driver-port')!)
: null,
androidEmulator: boolArg('android-emulator'),
profileMemory: stringArg('profile-memory'),
);
if (screenshot != null) {
// If the test is sent a signal or times out, take a screenshot
_registerScreenshotCallbacks(device, _fileSystem.directory(screenshot));
}
final int testResult = await testResultFuture;
if (timeoutTimer != null) {
timeoutTimer!.cancel();
}
_unregisterScreenshotCallbacks();
if (testResult != 0 && screenshot != null) {
// Take a screenshot while the app is still running.
await _takeScreenshot(device, _fileSystem.directory(screenshot));
screenshotTaken = true;
}
if (boolArg('keep-app-running')) {
_logger.printStatus('Leaving the application running.');
} else {
final File? skslFile = stringArg('write-sksl-on-exit') != null
? _fileSystem.file(stringArg('write-sksl-on-exit'))
: null;
await driverService.stop(userIdentifier: userIdentifier, writeSkslOnExit: skslFile);
}
if (testResult != 0) {
throwToolExit(null);
}
} on Exception catch (_) {
// On exceptions, including ToolExit, take a screenshot on the device
// unless a screenshot was already taken on test failure.
if (!screenshotTaken && screenshot != null) {
await _takeScreenshot(device, _fileSystem.directory(screenshot));
}
rethrow;
}
return FlutterCommandResult.success();
}
int? get _timeoutSeconds {
final String? timeoutString = stringArg('timeout');
if (timeoutString == null) {
return null;
}
final int? timeoutSeconds = int.tryParse(timeoutString);
if (timeoutSeconds == null || timeoutSeconds <= 0) {
throwToolExit(
'Invalid value "$timeoutString" provided to the option --timeout: '
'expected a positive integer representing seconds.',
);
}
return timeoutSeconds;
}
void _registerScreenshotCallbacks(Device device, Directory screenshotDir) {
_logger.printTrace('Registering signal handlers...');
final Map<ProcessSignal, Object> tokens = <ProcessSignal, Object>{};
for (final ProcessSignal signal in signalsToHandle) {
tokens[signal] = signals.addHandler(
signal,
(ProcessSignal signal) {
_unregisterScreenshotCallbacks();
_logger.printError('Caught $signal');
return _takeScreenshot(device, screenshotDir);
},
);
}
screenshotTokens = tokens;
final int? timeoutSeconds = _timeoutSeconds;
if (timeoutSeconds != null) {
timeoutTimer = Timer(
Duration(seconds: timeoutSeconds),
() {
_unregisterScreenshotCallbacks();
_takeScreenshot(device, screenshotDir);
throwToolExit('Timed out after $timeoutSeconds seconds');
}
);
}
}
void _unregisterScreenshotCallbacks() {
if (screenshotTokens != null) {
_logger.printTrace('Unregistering signal handlers...');
for (final MapEntry<ProcessSignal, Object> entry in screenshotTokens!.entries) {
signals.removeHandler(entry.key, entry.value);
}
}
timeoutTimer?.cancel();
}
String? _getTestFile() {
if (argResults!['driver'] != null) {
return stringArg('driver');
}
// If the --driver argument wasn't provided, then derive the value from
// the target file.
String appFile = _fileSystem.path.normalize(targetFile);
// This command extends `flutter run` and therefore CWD == package dir
final String packageDir = _fileSystem.currentDirectory.path;
// Make appFile path relative to package directory because we are looking
// for the corresponding test file relative to it.
if (!_fileSystem.path.isRelative(appFile)) {
if (!_fileSystem.path.isWithin(packageDir, appFile)) {
_logger.printError(
'Application file $appFile is outside the package directory $packageDir'
);
return null;
}
appFile = _fileSystem.path.relative(appFile, from: packageDir);
}
final List<String> parts = _fileSystem.path.split(appFile);
if (parts.length < 2) {
_logger.printError(
'Application file $appFile must reside in one of the sub-directories '
'of the package structure, not in the root directory.'
);
return null;
}
// Look for the test file inside `test_driver/` matching the sub-path, e.g.
// if the application is `lib/foo/bar.dart`, the test file is expected to
// be `test_driver/foo/bar_test.dart`.
final String pathWithNoExtension = _fileSystem.path.withoutExtension(_fileSystem.path.joinAll(
<String>[packageDir, 'test_driver', ...parts.skip(1)]));
return '${pathWithNoExtension}_test${_fileSystem.path.extension(appFile)}';
}
Future<void> _takeScreenshot(Device device, Directory outputDirectory) async {
if (!device.supportsScreenshot) {
return;
}
try {
outputDirectory.createSync(recursive: true);
final File outputFile = _fsUtils.getUniqueFile(
outputDirectory,
'drive',
'png',
);
await device.takeScreenshot(outputFile);
_logger.printStatus('Screenshot written to ${outputFile.path}');
} on Exception catch (error) {
_logger.printError('Error taking screenshot: $error');
}
}
}