[flutter_plugin_tools] Overhaul drive-examples (#4099)
Significantly restructures drive-examples:
- Migrates it to the new package-looping base command
- Enforces that only one platform is passed, since in practice multiple platforms never actually worked. (The logic is structured so that it will be easy to enable multi-platform if `flutter drive` gains multi-platform support.)
- Fixes the issue where `--ios` and `--android` were semi-broken, by doing explicit device targeting for them rather than relying on the default device being the right kind
- Extracts much of the logic to helpers so it's easier to understand the flow
- Removes support for a legacy integration test file structure that is no longer used
- Adds more test coverage; previously no failure cases were actually tested.
Fixes https://github.com/flutter/flutter/issues/85147
Part of https://github.com/flutter/flutter/issues/83413
diff --git a/script/tool/lib/src/build_examples_command.dart b/script/tool/lib/src/build_examples_command.dart
index c8280f4..ee1445f 100644
--- a/script/tool/lib/src/build_examples_command.dart
+++ b/script/tool/lib/src/build_examples_command.dart
@@ -6,7 +6,6 @@
import 'package:file/file.dart';
import 'package:path/path.dart' as p;
-import 'package:platform/platform.dart';
import 'common/core.dart';
import 'common/package_looping_command.dart';
@@ -151,8 +150,6 @@
String flutterBuildType, {
List<String> extraBuildFlags = const <String>[],
}) async {
- final String flutterCommand =
- const LocalPlatform().isWindows ? 'flutter.bat' : 'flutter';
final String enableExperiment = getStringArg(kEnableExperiment);
final int exitCode = await processRunner.runAndStream(
diff --git a/script/tool/lib/src/common/plugin_command.dart b/script/tool/lib/src/common/plugin_command.dart
index 9a96ab1..43d0d0b 100644
--- a/script/tool/lib/src/common/plugin_command.dart
+++ b/script/tool/lib/src/common/plugin_command.dart
@@ -8,6 +8,7 @@
import 'package:file/file.dart';
import 'package:git/git.dart';
import 'package:path/path.dart' as p;
+import 'package:platform/platform.dart';
import 'core.dart';
import 'git_version_finder.dart';
@@ -85,6 +86,10 @@
int? _shardIndex;
int? _shardCount;
+ /// The command to use when running `flutter`.
+ String get flutterCommand =>
+ const LocalPlatform().isWindows ? 'flutter.bat' : 'flutter';
+
/// The shard of the overall command execution that this instance should run.
int get shardIndex {
if (_shardIndex == null) {
diff --git a/script/tool/lib/src/drive_examples_command.dart b/script/tool/lib/src/drive_examples_command.dart
index 8a8cd67..a4aa7c1 100644
--- a/script/tool/lib/src/drive_examples_command.dart
+++ b/script/tool/lib/src/drive_examples_command.dart
@@ -2,17 +2,22 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+import 'dart:convert';
+import 'dart:io';
+
import 'package:file/file.dart';
import 'package:path/path.dart' as p;
-import 'package:platform/platform.dart';
import 'common/core.dart';
-import 'common/plugin_command.dart';
+import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/process_runner.dart';
+const int _exitNoPlatformFlags = 2;
+const int _exitNoAvailableDevice = 3;
+
/// A command to run the example applications for packages via Flutter driver.
-class DriveExamplesCommand extends PluginCommand {
+class DriveExamplesCommand extends PackageLoopingCommand {
/// Creates an instance of the drive command.
DriveExamplesCommand(
Directory packagesDir, {
@@ -43,213 +48,259 @@
@override
final String description = 'Runs driver tests for plugin example apps.\n\n'
- 'For each *_test.dart in test_driver/ it drives an application with a '
- 'corresponding name in the test/ or test_driver/ directories.\n\n'
- 'For example, test_driver/app_test.dart would match test/app.dart.\n\n'
- 'This command requires "flutter" to be in your path.\n\n'
- 'If a file with a corresponding name cannot be found, this driver file'
- 'will be used to drive the tests that match '
- 'integration_test/*_test.dart.';
+ 'For each *_test.dart in test_driver/ it drives an application with '
+ 'either the corresponding test in test_driver (for example, '
+ 'test_driver/app_test.dart would match test_driver/app.dart), or the '
+ '*_test.dart files in integration_test/.\n\n'
+ 'This command requires "flutter" to be in your path.';
+
+ Map<String, List<String>> _targetDeviceFlags = const <String, List<String>>{};
@override
- Future<void> run() async {
- final List<String> failingTests = <String>[];
- final List<String> pluginsWithoutTests = <String>[];
- final bool isLinux = getBoolArg(kPlatformLinux);
- final bool isMacos = getBoolArg(kPlatformMacos);
- final bool isWeb = getBoolArg(kPlatformWeb);
- final bool isWindows = getBoolArg(kPlatformWindows);
- await for (final Directory plugin in getPlugins()) {
- final String pluginName = plugin.basename;
- if (pluginName.endsWith('_platform_interface') &&
- !plugin.childDirectory('example').existsSync()) {
- // Platform interface packages generally aren't intended to have
- // examples, and don't need integration tests, so silently skip them
- // unless for some reason there is an example directory.
- continue;
- }
- print('\n==========\nChecking $pluginName...');
- if (!(await _pluginSupportedOnCurrentPlatform(plugin))) {
- print('Not supported for the target platform; skipping.');
- continue;
- }
- int examplesFound = 0;
- bool testsRan = false;
- final String flutterCommand =
- const LocalPlatform().isWindows ? 'flutter.bat' : 'flutter';
- for (final Directory example in getExamplesForPlugin(plugin)) {
- ++examplesFound;
- final String packageName =
- p.relative(example.path, from: packagesDir.path);
- final Directory driverTests = example.childDirectory('test_driver');
- if (!driverTests.existsSync()) {
- print('No driver tests found for $packageName');
- continue;
- }
- // Look for driver tests ending in _test.dart in test_driver/
- await for (final FileSystemEntity test in driverTests.list()) {
- final String driverTestName =
- p.relative(test.path, from: driverTests.path);
- if (!driverTestName.endsWith('_test.dart')) {
- continue;
- }
- // Try to find a matching app to drive without the _test.dart
- final String deviceTestName = driverTestName.replaceAll(
- RegExp(r'_test.dart$'),
- '.dart',
- );
- String deviceTestPath = p.join('test', deviceTestName);
- if (!example.fileSystem
- .file(p.join(example.path, deviceTestPath))
- .existsSync()) {
- // If the app isn't in test/ folder, look in test_driver/ instead.
- deviceTestPath = p.join('test_driver', deviceTestName);
- }
-
- final List<String> targetPaths = <String>[];
- if (example.fileSystem
- .file(p.join(example.path, deviceTestPath))
- .existsSync()) {
- targetPaths.add(deviceTestPath);
- } else {
- final Directory integrationTests =
- example.childDirectory('integration_test');
-
- if (await integrationTests.exists()) {
- await for (final FileSystemEntity integrationTest
- in integrationTests.list()) {
- if (!integrationTest.basename.endsWith('_test.dart')) {
- continue;
- }
- targetPaths
- .add(p.relative(integrationTest.path, from: example.path));
- }
- }
-
- if (targetPaths.isEmpty) {
- print('''
-Unable to infer a target application for $driverTestName to drive.
-Tried searching for the following:
-1. test/$deviceTestName
-2. test_driver/$deviceTestName
-3. test_driver/*_test.dart
-''');
- failingTests.add(p.relative(test.path, from: example.path));
- continue;
- }
- }
-
- final List<String> driveArgs = <String>['drive'];
-
- final String enableExperiment = getStringArg(kEnableExperiment);
- if (enableExperiment.isNotEmpty) {
- driveArgs.add('--enable-experiment=$enableExperiment');
- }
-
- if (isLinux && isLinuxPlugin(plugin)) {
- driveArgs.addAll(<String>[
- '-d',
- 'linux',
- ]);
- }
- if (isMacos && isMacOsPlugin(plugin)) {
- driveArgs.addAll(<String>[
- '-d',
- 'macos',
- ]);
- }
- if (isWeb && isWebPlugin(plugin)) {
- driveArgs.addAll(<String>[
- '-d',
- 'web-server',
- '--web-port=7357',
- '--browser-name=chrome',
- ]);
- }
- if (isWindows && isWindowsPlugin(plugin)) {
- driveArgs.addAll(<String>[
- '-d',
- 'windows',
- ]);
- }
-
- for (final String targetPath in targetPaths) {
- testsRan = true;
- final int exitCode = await processRunner.runAndStream(
- flutterCommand,
- <String>[
- ...driveArgs,
- '--driver',
- p.join('test_driver', driverTestName),
- '--target',
- targetPath,
- ],
- workingDir: example,
- exitOnError: true);
- if (exitCode != 0) {
- failingTests.add(p.join(packageName, deviceTestPath));
- }
- }
- }
- }
- if (!testsRan) {
- pluginsWithoutTests.add(pluginName);
- print(
- 'No driver tests run for $pluginName ($examplesFound examples found)');
- }
- }
- print('\n\n');
-
- if (failingTests.isNotEmpty) {
- print('The following driver tests are failing (see above for details):');
- for (final String test in failingTests) {
- print(' * $test');
- }
- throw ToolExit(1);
+ Future<void> initializeRun() async {
+ final List<String> platformSwitches = <String>[
+ kPlatformAndroid,
+ kPlatformIos,
+ kPlatformLinux,
+ kPlatformMacos,
+ kPlatformWeb,
+ kPlatformWindows,
+ ];
+ final int platformCount = platformSwitches
+ .where((String platform) => getBoolArg(platform))
+ .length;
+ // The flutter tool currently doesn't accept multiple device arguments:
+ // https://github.com/flutter/flutter/issues/35733
+ // If that is implemented, this check can be relaxed.
+ if (platformCount != 1) {
+ printError(
+ 'Exactly one of ${platformSwitches.map((String platform) => '--$platform').join(', ')} '
+ 'must be specified.');
+ throw ToolExit(_exitNoPlatformFlags);
}
- if (pluginsWithoutTests.isNotEmpty) {
- print('The following plugins did not run any integration tests:');
- for (final String plugin in pluginsWithoutTests) {
- print(' * $plugin');
+ String? androidDevice;
+ if (getBoolArg(kPlatformAndroid)) {
+ final List<String> devices = await _getDevicesForPlatform('android');
+ if (devices.isEmpty) {
+ printError('No Android devices available');
+ throw ToolExit(_exitNoAvailableDevice);
}
- print('If this is intentional, they must be explicitly excluded.');
- throw ToolExit(1);
+ androidDevice = devices.first;
}
- print('All driver tests successful!');
+ String? iosDevice;
+ if (getBoolArg(kPlatformIos)) {
+ final List<String> devices = await _getDevicesForPlatform('ios');
+ if (devices.isEmpty) {
+ printError('No iOS devices available');
+ throw ToolExit(_exitNoAvailableDevice);
+ }
+ iosDevice = devices.first;
+ }
+
+ _targetDeviceFlags = <String, List<String>>{
+ if (getBoolArg(kPlatformAndroid))
+ kPlatformAndroid: <String>['-d', androidDevice!],
+ if (getBoolArg(kPlatformIos)) kPlatformIos: <String>['-d', iosDevice!],
+ if (getBoolArg(kPlatformLinux)) kPlatformLinux: <String>['-d', 'linux'],
+ if (getBoolArg(kPlatformMacos)) kPlatformMacos: <String>['-d', 'macos'],
+ if (getBoolArg(kPlatformWeb))
+ kPlatformWeb: <String>[
+ '-d',
+ 'web-server',
+ '--web-port=7357',
+ '--browser-name=chrome'
+ ],
+ if (getBoolArg(kPlatformWindows))
+ kPlatformWindows: <String>['-d', 'windows'],
+ };
}
- Future<bool> _pluginSupportedOnCurrentPlatform(
- FileSystemEntity plugin) async {
- final bool isAndroid = getBoolArg(kPlatformAndroid);
- final bool isIOS = getBoolArg(kPlatformIos);
- final bool isLinux = getBoolArg(kPlatformLinux);
- final bool isMacos = getBoolArg(kPlatformMacos);
- final bool isWeb = getBoolArg(kPlatformWeb);
- final bool isWindows = getBoolArg(kPlatformWindows);
- if (isAndroid) {
- return isAndroidPlugin(plugin);
+ @override
+ Future<List<String>> runForPackage(Directory package) async {
+ if (package.basename.endsWith('_platform_interface') &&
+ !package.childDirectory('example').existsSync()) {
+ // Platform interface packages generally aren't intended to have
+ // examples, and don't need integration tests, so skip rather than fail.
+ printSkip(
+ 'Platform interfaces are not expected to have integratino tests.');
+ return PackageLoopingCommand.success;
}
- if (isIOS) {
- return isIosPlugin(plugin);
+
+ final List<String> deviceFlags = <String>[];
+ for (final MapEntry<String, List<String>> entry
+ in _targetDeviceFlags.entries) {
+ if (pluginSupportsPlatform(entry.key, package)) {
+ deviceFlags.addAll(entry.value);
+ } else {
+ print('Skipping unsupported platform ${entry.key}...');
+ }
}
- if (isLinux) {
- return isLinuxPlugin(plugin);
+ // If there is no supported target platform, skip the plugin.
+ if (deviceFlags.isEmpty) {
+ printSkip(
+ '${getPackageDescription(package)} does not support any requested platform.');
+ return PackageLoopingCommand.success;
}
- if (isMacos) {
- return isMacOsPlugin(plugin);
+
+ int examplesFound = 0;
+ bool testsRan = false;
+ final List<String> errors = <String>[];
+ for (final Directory example in getExamplesForPlugin(package)) {
+ ++examplesFound;
+ final String exampleName =
+ p.relative(example.path, from: packagesDir.path);
+
+ final List<File> drivers = await _getDrivers(example);
+ if (drivers.isEmpty) {
+ print('No driver tests found for $exampleName');
+ continue;
+ }
+
+ for (final File driver in drivers) {
+ final List<File> testTargets = <File>[];
+
+ // Try to find a matching app to drive without the _test.dart
+ // TODO(stuartmorgan): Migrate all remaining uses of this legacy
+ // approach (currently only video_player) and remove support for it:
+ // https://github.com/flutter/flutter/issues/85224.
+ final File? legacyTestFile = _getLegacyTestFileForTestDriver(driver);
+ if (legacyTestFile != null) {
+ testTargets.add(legacyTestFile);
+ } else {
+ (await _getIntegrationTests(example)).forEach(testTargets.add);
+ }
+
+ if (testTargets.isEmpty) {
+ final String driverRelativePath =
+ p.relative(driver.path, from: package.path);
+ printError(
+ 'Found $driverRelativePath, but no integration_test/*_test.dart files.');
+ errors.add(
+ 'No test files for ${p.relative(driver.path, from: package.path)}');
+ continue;
+ }
+
+ testsRan = true;
+ final List<File> failingTargets = await _driveTests(
+ example, driver, testTargets,
+ deviceFlags: deviceFlags);
+ for (final File failingTarget in failingTargets) {
+ errors.add(p.relative(failingTarget.path, from: package.path));
+ }
+ }
}
- if (isWeb) {
- return isWebPlugin(plugin);
+ if (!testsRan) {
+ printError('No driver tests were run ($examplesFound example(s) found).');
+ errors.add('No tests ran (use --exclude if this is intentional).');
}
- if (isWindows) {
- return isWindowsPlugin(plugin);
+ return errors;
+ }
+
+ Future<List<String>> _getDevicesForPlatform(String platform) async {
+ final List<String> deviceIds = <String>[];
+
+ final ProcessResult result = await processRunner.run(
+ flutterCommand, <String>['devices', '--machine'],
+ stdoutEncoding: utf8, exitOnError: true);
+ if (result.exitCode != 0) {
+ return deviceIds;
}
- // When we are here, no flags are specified. Only return true if the plugin
- // supports Android for legacy command support.
- // TODO(cyanglaz): Make Android flag also required like other platforms
- // (breaking change). https://github.com/flutter/flutter/issues/58285
- return isAndroidPlugin(plugin);
+
+ final List<Map<String, dynamic>> devices =
+ (jsonDecode(result.stdout as String) as List<dynamic>)
+ .cast<Map<String, dynamic>>();
+ for (final Map<String, dynamic> deviceInfo in devices) {
+ final String targetPlatform =
+ (deviceInfo['targetPlatform'] as String?) ?? '';
+ if (targetPlatform.startsWith(platform)) {
+ final String? deviceId = deviceInfo['id'] as String?;
+ if (deviceId != null) {
+ deviceIds.add(deviceId);
+ }
+ }
+ }
+ return deviceIds;
+ }
+
+ Future<List<File>> _getDrivers(Directory example) async {
+ final List<File> drivers = <File>[];
+
+ final Directory driverDir = example.childDirectory('test_driver');
+ if (driverDir.existsSync()) {
+ await for (final FileSystemEntity driver in driverDir.list()) {
+ if (driver is File && driver.basename.endsWith('_test.dart')) {
+ drivers.add(driver);
+ }
+ }
+ }
+ return drivers;
+ }
+
+ File? _getLegacyTestFileForTestDriver(File testDriver) {
+ final String testName = testDriver.basename.replaceAll(
+ RegExp(r'_test.dart$'),
+ '.dart',
+ );
+ final File testFile = testDriver.parent.childFile(testName);
+
+ return testFile.existsSync() ? testFile : null;
+ }
+
+ Future<List<File>> _getIntegrationTests(Directory example) async {
+ final List<File> tests = <File>[];
+ final Directory integrationTestDir =
+ example.childDirectory('integration_test');
+
+ if (integrationTestDir.existsSync()) {
+ await for (final FileSystemEntity file in integrationTestDir.list()) {
+ if (file is File && file.basename.endsWith('_test.dart')) {
+ tests.add(file);
+ }
+ }
+ }
+ return tests;
+ }
+
+ /// For each file in [targets], uses
+ /// `flutter drive --driver [driver] --target <target>`
+ /// to drive [example], returning a list of any failing test targets.
+ ///
+ /// [deviceFlags] should contain the flags to run the test on a specific
+ /// target device (plus any supporting device-specific flags). E.g.:
+ /// - `['-d', 'macos']` for driving for macOS.
+ /// - `['-d', 'web-server', '--web-port=<port>', '--browser-name=<browser>]`
+ /// for web
+ Future<List<File>> _driveTests(
+ Directory example,
+ File driver,
+ List<File> targets, {
+ required List<String> deviceFlags,
+ }) async {
+ final List<File> failures = <File>[];
+
+ final String enableExperiment = getStringArg(kEnableExperiment);
+
+ for (final File target in targets) {
+ final int exitCode = await processRunner.runAndStream(
+ flutterCommand,
+ <String>[
+ 'drive',
+ ...deviceFlags,
+ if (enableExperiment.isNotEmpty)
+ '--enable-experiment=$enableExperiment',
+ '--driver',
+ p.relative(driver.path, from: example.path),
+ '--target',
+ p.relative(target.path, from: example.path),
+ ],
+ workingDir: example,
+ exitOnError: true);
+ if (exitCode != 0) {
+ failures.add(target);
+ }
+ }
+ return failures;
}
}
diff --git a/script/tool/test/drive_examples_command_test.dart b/script/tool/test/drive_examples_command_test.dart
index 3175f71..e441a0f 100644
--- a/script/tool/test/drive_examples_command_test.dart
+++ b/script/tool/test/drive_examples_command_test.dart
@@ -12,8 +12,12 @@
import 'package:platform/platform.dart';
import 'package:test/test.dart';
+import 'mocks.dart';
import 'util.dart';
+const String _fakeIosDevice = '67d5c3d1-8bdf-46ad-8f6b-b00e2a972dda';
+const String _fakeAndroidDevice = 'emulator-1234';
+
void main() {
group('test drive_example_command', () {
late FileSystem fileSystem;
@@ -35,52 +39,92 @@
runner.addCommand(command);
});
- test('driving under folder "test"', () async {
- final Directory pluginDirectory = createFakePlugin(
- 'plugin',
- packagesDir,
- extraFiles: <String>[
- 'example/test_driver/plugin_test.dart',
- 'example/test/plugin.dart',
- ],
- platformSupport: <String, PlatformSupport>{
- kPlatformAndroid: PlatformSupport.inline,
- kPlatformIos: PlatformSupport.inline,
- },
- );
+ void setMockFlutterDevicesOutput({
+ bool hasIosDevice = true,
+ bool hasAndroidDevice = true,
+ }) {
+ final List<String> devices = <String>[
+ if (hasIosDevice) '{"id": "$_fakeIosDevice", "targetPlatform": "ios"}',
+ if (hasAndroidDevice)
+ '{"id": "$_fakeAndroidDevice", "targetPlatform": "android-x86"}',
+ ];
+ final String output = '''[${devices.join(',')}]''';
- final Directory pluginExampleDirectory =
- pluginDirectory.childDirectory('example');
+ final MockProcess mockDevicesProcess = MockProcess();
+ mockDevicesProcess.exitCodeCompleter.complete(0);
+ mockDevicesProcess.stdoutController.close(); // ignore: unawaited_futures
+ processRunner.processToReturn = mockDevicesProcess;
+ processRunner.resultStdout = output;
+ }
- final List<String> output = await runCapturingPrint(runner, <String>[
- 'drive-examples',
- ]);
+ test('fails if no platforms are provided', () async {
+ setMockFlutterDevicesOutput();
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples'], errorHandler: (Error e) {
+ commandError = e;
+ });
+ expect(commandError, isA<ToolExit>());
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Exactly one of'),
]),
);
+ });
- final String deviceTestPath = p.join('test', 'plugin.dart');
- final String driverTestPath = p.join('test_driver', 'plugin_test.dart');
+ test('fails if multiple platforms are provided', () async {
+ setMockFlutterDevicesOutput();
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--ios', '--macos'],
+ errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
expect(
- processRunner.recordedCalls,
- orderedEquals(<ProcessCall>[
- ProcessCall(
- flutterCommand,
- <String>[
- 'drive',
- '--driver',
- driverTestPath,
- '--target',
- deviceTestPath
- ],
- pluginExampleDirectory.path),
- ]));
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Exactly one of'),
+ ]),
+ );
+ });
+
+ test('fails for iOS if no iOS devices are present', () async {
+ setMockFlutterDevicesOutput(hasIosDevice: false);
+
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--ios'], errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('No iOS devices'),
+ ]),
+ );
+ });
+
+ test('fails if Android if no Android devices are present', () async {
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--android'],
+ errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('No Android devices'),
+ ]),
+ );
});
test('driving under folder "test_driver"', () async {
@@ -100,16 +144,15 @@
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
- final List<String> output = await runCapturingPrint(runner, <String>[
- 'drive-examples',
- ]);
+ setMockFlutterDevicesOutput();
+ final List<String> output =
+ await runCapturingPrint(runner, <String>['drive-examples', '--ios']);
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -119,9 +162,13 @@
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ProcessCall(
flutterCommand,
<String>[
'drive',
+ '-d',
+ _fakeIosDevice,
'--driver',
driverTestPath,
'--target',
@@ -133,6 +180,7 @@
test('driving under folder "test_driver" when test files are missing"',
() async {
+ setMockFlutterDevicesOutput();
createFakePlugin(
'plugin',
packagesDir,
@@ -145,13 +193,27 @@
},
);
- await expectLater(
- () => runCapturingPrint(runner, <String>['drive-examples']),
- throwsA(const TypeMatcher<ToolExit>()));
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--android'],
+ errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No driver tests were run (1 example(s) found).'),
+ contains('No test files for example/test_driver/plugin_test.dart'),
+ ]),
+ );
});
test('a plugin without any integration test files is reported as an error',
() async {
+ setMockFlutterDevicesOutput();
createFakePlugin(
'plugin',
packagesDir,
@@ -164,9 +226,22 @@
},
);
- await expectLater(
- () => runCapturingPrint(runner, <String>['drive-examples']),
- throwsA(const TypeMatcher<ToolExit>()));
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--android'],
+ errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No driver tests were run (1 example(s) found).'),
+ contains('No tests ran'),
+ ]),
+ );
});
test(
@@ -190,16 +265,15 @@
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
- final List<String> output = await runCapturingPrint(runner, <String>[
- 'drive-examples',
- ]);
+ setMockFlutterDevicesOutput();
+ final List<String> output =
+ await runCapturingPrint(runner, <String>['drive-examples', '--ios']);
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -209,9 +283,13 @@
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ProcessCall(
flutterCommand,
<String>[
'drive',
+ '-d',
+ _fakeIosDevice,
'--driver',
driverTestPath,
'--target',
@@ -222,6 +300,8 @@
flutterCommand,
<String>[
'drive',
+ '-d',
+ _fakeIosDevice,
'--driver',
driverTestPath,
'--target',
@@ -244,11 +324,10 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- 'Not supported for the target platform; skipping.',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Skipping unsupported platform linux...'),
+ contains('No issues found!'),
]),
);
@@ -280,10 +359,9 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -320,11 +398,10 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- 'Not supported for the target platform; skipping.',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Skipping unsupported platform macos...'),
+ contains('No issues found!'),
]),
);
@@ -332,6 +409,7 @@
// implementation is a no-op.
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
+
test('driving on a macOS plugin', () async {
final Directory pluginDirectory = createFakePlugin(
'plugin',
@@ -356,10 +434,9 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -396,11 +473,9 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- 'Not supported for the target platform; skipping.',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -432,10 +507,9 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -474,11 +548,10 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- 'Not supported for the target platform; skipping.',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Skipping unsupported platform windows...'),
+ contains('No issues found!'),
]),
);
@@ -510,10 +583,9 @@
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
]),
);
@@ -537,7 +609,59 @@
]));
});
- test('driving when plugin does not support mobile is no-op', () async {
+ test('driving on an Android plugin', () async {
+ final Directory pluginDirectory = createFakePlugin(
+ 'plugin',
+ packagesDir,
+ extraFiles: <String>[
+ 'example/test_driver/plugin_test.dart',
+ 'example/test_driver/plugin.dart',
+ ],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformAndroid: PlatformSupport.inline,
+ },
+ );
+
+ final Directory pluginExampleDirectory =
+ pluginDirectory.childDirectory('example');
+
+ setMockFlutterDevicesOutput();
+ final List<String> output = await runCapturingPrint(runner, <String>[
+ 'drive-examples',
+ '--android',
+ ]);
+
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No issues found!'),
+ ]),
+ );
+
+ final String deviceTestPath = p.join('test_driver', 'plugin.dart');
+ final String driverTestPath = p.join('test_driver', 'plugin_test.dart');
+ expect(
+ processRunner.recordedCalls,
+ orderedEquals(<ProcessCall>[
+ ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ProcessCall(
+ flutterCommand,
+ <String>[
+ 'drive',
+ '-d',
+ _fakeAndroidDevice,
+ '--driver',
+ driverTestPath,
+ '--target',
+ deviceTestPath
+ ],
+ pluginExampleDirectory.path),
+ ]));
+ });
+
+ test('driving when plugin does not support Android is no-op', () async {
createFakePlugin(
'plugin',
packagesDir,
@@ -550,43 +674,78 @@
},
);
- final List<String> output = await runCapturingPrint(runner, <String>[
- 'drive-examples',
- ]);
+ setMockFlutterDevicesOutput();
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--android']);
expect(
output,
- orderedEquals(<String>[
- '\n==========\nChecking plugin...',
- 'Not supported for the target platform; skipping.',
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Skipping unsupported platform android...'),
+ contains('No issues found!'),
]),
);
- // Output should be empty since running drive-examples --macos with no macos
- // implementation is a no-op.
- expect(processRunner.recordedCalls, <ProcessCall>[]);
+ // Output should be empty other than the device query.
+ expect(processRunner.recordedCalls, <ProcessCall>[
+ ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ]);
+ });
+
+ test('driving when plugin does not support iOS is no-op', () async {
+ createFakePlugin(
+ 'plugin',
+ packagesDir,
+ extraFiles: <String>[
+ 'example/test_driver/plugin_test.dart',
+ 'example/test_driver/plugin.dart',
+ ],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformMacos: PlatformSupport.inline,
+ },
+ );
+
+ setMockFlutterDevicesOutput();
+ final List<String> output =
+ await runCapturingPrint(runner, <String>['drive-examples', '--ios']);
+
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Skipping unsupported platform ios...'),
+ contains('No issues found!'),
+ ]),
+ );
+
+ // Output should be empty other than the device query.
+ expect(processRunner.recordedCalls, <ProcessCall>[
+ ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ]);
});
test('platform interface plugins are silently skipped', () async {
createFakePlugin('aplugin_platform_interface', packagesDir,
examples: <String>[]);
- final List<String> output = await runCapturingPrint(runner, <String>[
- 'drive-examples',
- ]);
+ setMockFlutterDevicesOutput();
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--macos']);
expect(
output,
- orderedEquals(<String>[
- '\n\n',
- 'All driver tests successful!',
+ containsAllInOrder(<Matcher>[
+ contains('Running for aplugin_platform_interface'),
+ contains(
+ 'SKIPPING: Platform interfaces are not expected to have integratino tests.'),
+ contains('No issues found!'),
]),
);
- // Output should be empty since running drive-examples --macos with no macos
- // implementation is a no-op.
+ // Output should be empty since it's skipped.
expect(processRunner.recordedCalls, <ProcessCall>[]);
});
@@ -596,7 +755,7 @@
packagesDir,
extraFiles: <String>[
'example/test_driver/plugin_test.dart',
- 'example/test/plugin.dart',
+ 'example/test_driver/plugin.dart',
],
platformSupport: <String, PlatformSupport>{
kPlatformAndroid: PlatformSupport.inline,
@@ -607,25 +766,199 @@
final Directory pluginExampleDirectory =
pluginDirectory.childDirectory('example');
+ setMockFlutterDevicesOutput();
await runCapturingPrint(runner, <String>[
'drive-examples',
+ '--ios',
'--enable-experiment=exp1',
]);
- final String deviceTestPath = p.join('test', 'plugin.dart');
+ final String deviceTestPath = p.join('test_driver', 'plugin.dart');
final String driverTestPath = p.join('test_driver', 'plugin_test.dart');
expect(
processRunner.recordedCalls,
orderedEquals(<ProcessCall>[
ProcessCall(
+ flutterCommand, const <String>['devices', '--machine'], null),
+ ProcessCall(
+ flutterCommand,
+ <String>[
+ 'drive',
+ '-d',
+ _fakeIosDevice,
+ '--enable-experiment=exp1',
+ '--driver',
+ driverTestPath,
+ '--target',
+ deviceTestPath
+ ],
+ pluginExampleDirectory.path),
+ ]));
+ });
+
+ test('fails when no example is present', () async {
+ createFakePlugin(
+ 'plugin',
+ packagesDir,
+ examples: <String>[],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformWeb: PlatformSupport.inline,
+ },
+ );
+
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--web'], errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No driver tests were run (0 example(s) found).'),
+ contains('The following packages had errors:'),
+ contains(' plugin:\n'
+ ' No tests ran (use --exclude if this is intentional)'),
+ ]),
+ );
+ });
+
+ test('fails when no driver is present', () async {
+ createFakePlugin(
+ 'plugin',
+ packagesDir,
+ extraFiles: <String>[
+ 'example/integration_test/bar_test.dart',
+ 'example/integration_test/foo_test.dart',
+ ],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformWeb: PlatformSupport.inline,
+ },
+ );
+
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--web'], errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('No driver tests found for plugin/example'),
+ contains('No driver tests were run (1 example(s) found).'),
+ contains('The following packages had errors:'),
+ contains(' plugin:\n'
+ ' No tests ran (use --exclude if this is intentional)'),
+ ]),
+ );
+ });
+
+ test('fails when no integration tests are present', () async {
+ createFakePlugin(
+ 'plugin',
+ packagesDir,
+ extraFiles: <String>[
+ 'example/test_driver/integration_test.dart',
+ ],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformWeb: PlatformSupport.inline,
+ },
+ );
+
+ Error? commandError;
+ final List<String> output = await runCapturingPrint(
+ runner, <String>['drive-examples', '--web'], errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('Found example/test_driver/integration_test.dart, but no '
+ 'integration_test/*_test.dart files.'),
+ contains('No driver tests were run (1 example(s) found).'),
+ contains('The following packages had errors:'),
+ contains(' plugin:\n'
+ ' No test files for example/test_driver/integration_test.dart\n'
+ ' No tests ran (use --exclude if this is intentional)'),
+ ]),
+ );
+ });
+
+ test('reports test failures', () async {
+ final Directory pluginDirectory = createFakePlugin(
+ 'plugin',
+ packagesDir,
+ extraFiles: <String>[
+ 'example/test_driver/integration_test.dart',
+ 'example/integration_test/bar_test.dart',
+ 'example/integration_test/foo_test.dart',
+ ],
+ platformSupport: <String, PlatformSupport>{
+ kPlatformMacos: PlatformSupport.inline,
+ },
+ );
+
+ // Simulate failure from `flutter drive`.
+ final MockProcess mockDriveProcess = MockProcess();
+ mockDriveProcess.exitCodeCompleter.complete(1);
+ processRunner.processToReturn = mockDriveProcess;
+
+ Error? commandError;
+ final List<String> output =
+ await runCapturingPrint(runner, <String>['drive-examples', '--macos'],
+ errorHandler: (Error e) {
+ commandError = e;
+ });
+
+ expect(commandError, isA<ToolExit>());
+ expect(
+ output,
+ containsAllInOrder(<Matcher>[
+ contains('Running for plugin'),
+ contains('The following packages had errors:'),
+ contains(' plugin:\n'
+ ' example/integration_test/bar_test.dart\n'
+ ' example/integration_test/foo_test.dart'),
+ ]),
+ );
+
+ final Directory pluginExampleDirectory =
+ pluginDirectory.childDirectory('example');
+ final String driverTestPath =
+ p.join('test_driver', 'integration_test.dart');
+ expect(
+ processRunner.recordedCalls,
+ orderedEquals(<ProcessCall>[
+ ProcessCall(
flutterCommand,
<String>[
'drive',
- '--enable-experiment=exp1',
+ '-d',
+ 'macos',
'--driver',
driverTestPath,
'--target',
- deviceTestPath
+ p.join('integration_test', 'bar_test.dart'),
+ ],
+ pluginExampleDirectory.path),
+ ProcessCall(
+ flutterCommand,
+ <String>[
+ 'drive',
+ '-d',
+ 'macos',
+ '--driver',
+ driverTestPath,
+ '--target',
+ p.join('integration_test', 'foo_test.dart'),
],
pluginExampleDirectory.path),
]));
