blob: e8fb11b5f28924a22540152dc139aff8f6a4c557 [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:file/file.dart';
import 'package:platform/platform.dart';
import 'common/core.dart';
import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/process_runner.dart';
import 'common/repository_package.dart';
const int _exitNoPlatformFlags = 2;
const int _exitNoAvailableDevice = 3;
/// A command to run the example applications for packages via Flutter driver.
class DriveExamplesCommand extends PackageLoopingCommand {
/// Creates an instance of the drive command.
DriveExamplesCommand(
Directory packagesDir, {
ProcessRunner processRunner = const ProcessRunner(),
Platform platform = const LocalPlatform(),
}) : super(packagesDir, processRunner: processRunner, platform: platform) {
argParser.addFlag(platformAndroid,
help: 'Runs the Android implementation of the examples');
argParser.addFlag(platformIOS,
help: 'Runs the iOS implementation of the examples');
argParser.addFlag(platformLinux,
help: 'Runs the Linux implementation of the examples');
argParser.addFlag(platformMacOS,
help: 'Runs the macOS implementation of the examples');
argParser.addFlag(platformWeb,
help: 'Runs the web implementation of the examples');
argParser.addFlag(platformWindows,
help: 'Runs the Windows implementation of the examples');
argParser.addOption(
kEnableExperiment,
defaultsTo: '',
help:
'Runs the driver tests in Dart VM with the given experiments enabled.',
);
}
@override
final String name = 'drive-examples';
@override
final String description = 'Runs driver tests for package example apps.\n\n'
'For each *_test.dart in test_driver/ it drives an application with '
'either the corresponding test in test_driver (for example, '
'test_driver/app_test.dart would match test_driver/app.dart), or the '
'*_test.dart files in integration_test/.\n\n'
'This command requires "flutter" to be in your path.';
Map<String, List<String>> _targetDeviceFlags = const <String, List<String>>{};
@override
Future<void> initializeRun() async {
final List<String> platformSwitches = <String>[
platformAndroid,
platformIOS,
platformLinux,
platformMacOS,
platformWeb,
platformWindows,
];
final int platformCount = platformSwitches
.where((String platform) => getBoolArg(platform))
.length;
// The flutter tool currently doesn't accept multiple device arguments:
// https://github.com/flutter/flutter/issues/35733
// If that is implemented, this check can be relaxed.
if (platformCount != 1) {
printError(
'Exactly one of ${platformSwitches.map((String platform) => '--$platform').join(', ')} '
'must be specified.');
throw ToolExit(_exitNoPlatformFlags);
}
String? androidDevice;
if (getBoolArg(platformAndroid)) {
final List<String> devices = await _getDevicesForPlatform('android');
if (devices.isEmpty) {
printError('No Android devices available');
throw ToolExit(_exitNoAvailableDevice);
}
androidDevice = devices.first;
}
String? iOSDevice;
if (getBoolArg(platformIOS)) {
final List<String> devices = await _getDevicesForPlatform('ios');
if (devices.isEmpty) {
printError('No iOS devices available');
throw ToolExit(_exitNoAvailableDevice);
}
iOSDevice = devices.first;
}
_targetDeviceFlags = <String, List<String>>{
if (getBoolArg(platformAndroid))
platformAndroid: <String>['-d', androidDevice!],
if (getBoolArg(platformIOS)) platformIOS: <String>['-d', iOSDevice!],
if (getBoolArg(platformLinux)) platformLinux: <String>['-d', 'linux'],
if (getBoolArg(platformMacOS)) platformMacOS: <String>['-d', 'macos'],
if (getBoolArg(platformWeb))
platformWeb: <String>[
'-d',
'web-server',
'--web-port=7357',
'--browser-name=chrome',
if (platform.environment.containsKey('CHROME_EXECUTABLE'))
'--chrome-binary=${platform.environment['CHROME_EXECUTABLE']}',
],
if (getBoolArg(platformWindows))
platformWindows: <String>['-d', 'windows'],
};
}
@override
Future<PackageResult> runForPackage(RepositoryPackage package) async {
final bool isPlugin = isFlutterPlugin(package);
if (package.isPlatformInterface && package.getExamples().isEmpty) {
// Platform interface packages generally aren't intended to have
// examples, and don't need integration tests, so skip rather than fail.
return PackageResult.skip(
'Platform interfaces are not expected to have integration tests.');
}
// For plugin packages, skip if the plugin itself doesn't support any
// requested platform(s).
if (isPlugin) {
final Iterable<String> requestedPlatforms = _targetDeviceFlags.keys;
final Iterable<String> unsupportedPlatforms = requestedPlatforms.where(
(String platform) => !pluginSupportsPlatform(platform, package));
for (final String platform in unsupportedPlatforms) {
print('Skipping unsupported platform $platform...');
}
if (unsupportedPlatforms.length == requestedPlatforms.length) {
return PackageResult.skip(
'${package.displayName} does not support any requested platform.');
}
}
int examplesFound = 0;
int supportedExamplesFound = 0;
bool testsRan = false;
final List<String> errors = <String>[];
for (final RepositoryPackage example in package.getExamples()) {
++examplesFound;
final String exampleName =
getRelativePosixPath(example.directory, from: packagesDir);
// Skip examples that don't support any requested platform(s).
final List<String> deviceFlags = _deviceFlagsForExample(example);
if (deviceFlags.isEmpty) {
print(
'Skipping $exampleName; does not support any requested platforms.');
continue;
}
++supportedExamplesFound;
final List<File> drivers = await _getDrivers(example);
if (drivers.isEmpty) {
print('No driver tests found for $exampleName');
continue;
}
for (final File driver in drivers) {
final List<File> testTargets = <File>[];
// Try to find a matching app to drive without the _test.dart
// TODO(stuartmorgan): Migrate all remaining uses of this legacy
// approach (currently only video_player) and remove support for it:
// https://github.com/flutter/flutter/issues/85224.
final File? legacyTestFile = _getLegacyTestFileForTestDriver(driver);
if (legacyTestFile != null) {
testTargets.add(legacyTestFile);
} else {
for (final File testFile in await _getIntegrationTests(example)) {
// Check files for known problematic patterns.
final bool passesValidation = _validateIntegrationTest(testFile);
if (!passesValidation) {
// Report the issue, but continue with the test as the validation
// errors don't prevent running.
errors.add('${testFile.basename} failed validation');
}
testTargets.add(testFile);
}
}
if (testTargets.isEmpty) {
final String driverRelativePath =
getRelativePosixPath(driver, from: package.directory);
printError(
'Found $driverRelativePath, but no integration_test/*_test.dart files.');
errors.add('No test files for $driverRelativePath');
continue;
}
testsRan = true;
final List<File> failingTargets = await _driveTests(
example, driver, testTargets,
deviceFlags: deviceFlags);
for (final File failingTarget in failingTargets) {
errors.add(
getRelativePosixPath(failingTarget, from: package.directory));
}
}
}
if (!testsRan) {
// It is an error for a plugin not to have integration tests, because that
// is the only way to test the method channel communication.
if (isPlugin) {
printError(
'No driver tests were run ($examplesFound example(s) found).');
errors.add('No tests ran (use --exclude if this is intentional).');
} else {
return PackageResult.skip(supportedExamplesFound == 0
? 'No example supports requested platform(s).'
: 'No example is configured for driver tests.');
}
}
return errors.isEmpty
? PackageResult.success()
: PackageResult.fail(errors);
}
/// Returns the device flags for the intersection of the requested platforms
/// and the platforms supported by [example].
List<String> _deviceFlagsForExample(RepositoryPackage example) {
final List<String> deviceFlags = <String>[];
for (final MapEntry<String, List<String>> entry
in _targetDeviceFlags.entries) {
final String platform = entry.key;
if (example.directory.childDirectory(platform).existsSync()) {
deviceFlags.addAll(entry.value);
} else {
final String exampleName =
getRelativePosixPath(example.directory, from: packagesDir);
print('Skipping unsupported platform $platform for $exampleName');
}
}
return deviceFlags;
}
Future<List<String>> _getDevicesForPlatform(String platform) async {
final List<String> deviceIds = <String>[];
final ProcessResult result = await processRunner.run(
flutterCommand, <String>['devices', '--machine'],
stdoutEncoding: utf8);
if (result.exitCode != 0) {
return deviceIds;
}
String output = result.stdout as String;
// --machine doesn't currently prevent the tool from printing banners;
// see https://github.com/flutter/flutter/issues/86055. This workaround
// can be removed once that is fixed.
output = output.substring(output.indexOf('['));
final List<Map<String, dynamic>> devices =
(jsonDecode(output) as List<dynamic>).cast<Map<String, dynamic>>();
for (final Map<String, dynamic> deviceInfo in devices) {
final String targetPlatform =
(deviceInfo['targetPlatform'] as String?) ?? '';
if (targetPlatform.startsWith(platform)) {
final String? deviceId = deviceInfo['id'] as String?;
if (deviceId != null) {
deviceIds.add(deviceId);
}
}
}
return deviceIds;
}
Future<List<File>> _getDrivers(RepositoryPackage example) async {
final List<File> drivers = <File>[];
final Directory driverDir = example.directory.childDirectory('test_driver');
if (driverDir.existsSync()) {
await for (final FileSystemEntity driver in driverDir.list()) {
if (driver is File && driver.basename.endsWith('_test.dart')) {
drivers.add(driver);
}
}
}
return drivers;
}
File? _getLegacyTestFileForTestDriver(File testDriver) {
final String testName = testDriver.basename.replaceAll(
RegExp(r'_test.dart$'),
'.dart',
);
final File testFile = testDriver.parent.childFile(testName);
return testFile.existsSync() ? testFile : null;
}
Future<List<File>> _getIntegrationTests(RepositoryPackage example) async {
final List<File> tests = <File>[];
final Directory integrationTestDir =
example.directory.childDirectory('integration_test');
if (integrationTestDir.existsSync()) {
await for (final FileSystemEntity file in integrationTestDir.list()) {
if (file is File && file.basename.endsWith('_test.dart')) {
tests.add(file);
}
}
}
return tests;
}
/// Checks [testFile] for known bad patterns in integration tests, logging
/// any issues.
///
/// Returns true if the file passes validation without issues.
bool _validateIntegrationTest(File testFile) {
final List<String> lines = testFile.readAsLinesSync();
final RegExp badTestPattern = RegExp(r'\s*test\(');
if (lines.any((String line) => line.startsWith(badTestPattern))) {
final String filename = testFile.basename;
printError(
'$filename uses "test", which will not report failures correctly. '
'Use testWidgets instead.');
return false;
}
return true;
}
/// For each file in [targets], uses
/// `flutter drive --driver [driver] --target <target>`
/// to drive [example], returning a list of any failing test targets.
///
/// [deviceFlags] should contain the flags to run the test on a specific
/// target device (plus any supporting device-specific flags). E.g.:
/// - `['-d', 'macos']` for driving for macOS.
/// - `['-d', 'web-server', '--web-port=<port>', '--browser-name=<browser>]`
/// for web
Future<List<File>> _driveTests(
RepositoryPackage example,
File driver,
List<File> targets, {
required List<String> deviceFlags,
}) async {
final List<File> failures = <File>[];
final String enableExperiment = getStringArg(kEnableExperiment);
for (final File target in targets) {
final int exitCode = await processRunner.runAndStream(
flutterCommand,
<String>[
'drive',
...deviceFlags,
if (enableExperiment.isNotEmpty)
'--enable-experiment=$enableExperiment',
'--driver',
getRelativePosixPath(driver, from: example.directory),
'--target',
getRelativePosixPath(target, from: example.directory),
],
workingDir: example.directory);
if (exitCode != 0) {
failures.add(target);
}
}
return failures;
}
}