Run flutter tests through mini test engine when run directly (flutter run -t test_file) (#24930)
diff --git a/dev/automated_tests/flutter_run_test/flutter_run_test.dart b/dev/automated_tests/flutter_run_test/flutter_run_test.dart new file mode 100644 index 0000000..eb4f946 --- /dev/null +++ b/dev/automated_tests/flutter_run_test/flutter_run_test.dart
@@ -0,0 +1,19 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_test/flutter_test.dart'; + +void main() { + group('example', () { + test('passed', () { + expect(true, true); + }); + test('failed', () { + expect(true, false); + }); + test('skipped', () { + expect(true, false); + }, skip: true); + }); +}
diff --git a/dev/devicelab/bin/tasks/flutter_run_test.dart b/dev/devicelab/bin/tasks/flutter_run_test.dart new file mode 100644 index 0000000..bc76e62 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_run_test.dart
@@ -0,0 +1,67 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'examples/hello_world')); +final File runTestSource = File(path.join( + flutterDirectory.path, 'dev', 'automated_tests', 'flutter_run_test', 'flutter_run_test.dart', +)); +const Pattern passedMessageMatch = '+0: example passed'; +const Pattern failedMessageMatch = '+1: example failed [E]'; +const Pattern skippedMessageMatch = '+1 -1: example skipped'; +const Pattern finishedMessageMatch = '+1 ~1 -1: Some tests failed.'; + +Future<void> main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createFlutterRunTask); +} + +// verifies that the messages above are printed as a test script is executed. +Future<TaskResult> createFlutterRunTask() async { + bool passedTest = false; + bool failedTest = false; + bool skippedTest = false; + bool finishedMessage = false; + final Device device = await devices.workingDevice; + await device.unlock(); + final List<String> options = <String>[ + '-t', runTestSource.absolute.path, '-d', device.deviceId, + ]; + setLocalEngineOptionIfNecessary(options); + await inDirectory<void>(flutterGalleryDir, () async { + startProcess( + path.join(flutterDirectory.path, 'bin', 'flutter'), + <String>['run']..addAll(options), + environment: null + ); + final Completer<void> finished = Completer<void>(); + final StreamSubscription<void> subscription = device.logcat.listen((String line) { + // tests execute in order. + if (line.contains(passedMessageMatch)) { + passedTest = true; + } else if (line.contains(failedMessageMatch)) { + failedTest = true; + } else if (line.contains(skippedMessageMatch)) { + skippedTest = true; + } else if (line.contains(finishedMessageMatch)) { + finishedMessage = true; + finished.complete(); + } + }); + await finished.future.timeout(const Duration(minutes: 1)); + subscription.cancel(); + }); + return passedTest && failedTest && skippedTest && finishedMessage + ? TaskResult.success(<String, dynamic>{}) + : TaskResult.failure('Test did not execute as expected.'); +} +
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 1756792..d4a3a56 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml
@@ -283,6 +283,13 @@ stage: devicelab required_agent_capabilities: ["linux/android"] + flutter_run_test: + description: > + Tests the `flutter run -t` command with a testfile. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true + named_isolates_test: description: > Tests naming and attaching to specific isolates.
diff --git a/packages/flutter_test/lib/flutter_test.dart b/packages/flutter_test/lib/flutter_test.dart index 12dc015..61e439a 100644 --- a/packages/flutter_test/lib/flutter_test.dart +++ b/packages/flutter_test/lib/flutter_test.dart
@@ -56,6 +56,7 @@ export 'src/nonconst.dart'; export 'src/stack_manipulation.dart'; export 'src/test_async_utils.dart'; +export 'src/test_compat.dart'; export 'src/test_exception_reporter.dart'; export 'src/test_pointer.dart'; export 'src/test_text_input.dart';
diff --git a/packages/flutter_test/lib/src/test_compat.dart b/packages/flutter_test/lib/src/test_compat.dart new file mode 100644 index 0000000..7334274 --- /dev/null +++ b/packages/flutter_test/lib/src/test_compat.dart
@@ -0,0 +1,514 @@ +// Copyright 2018 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test_api/src/backend/declarer.dart'; // ignore: implementation_imports +import 'package:test_api/src/frontend/timeout.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/group_entry.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/message.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports +import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports + +import 'package:test_api/test_api.dart'; + +Declarer _localDeclarer; +Declarer get _declarer { + final Declarer declarer = Zone.current[#test.declarer]; + if (declarer != null) { + return declarer; + } + // If no declarer is defined, this test is being run via `flutter run -t test_file.dart`. + if (_localDeclarer == null) { + _localDeclarer = Declarer(); + Future<void>(() { + Invoker.guard<Future<void>>(() async { + final _Reporter reporter = _Reporter(color: false); // disable color when run directly. + final Group group = _declarer.build(); + final Suite suite = Suite(group, SuitePlatform(Runtime.vm)); + await _runGroup(suite, group, <Group>[], reporter); + reporter._onDone(); + }); + }); + } + return _localDeclarer; +} + +Future<void> _runGroup(Suite suiteConfig, Group group, List<Group> parents, _Reporter reporter) async { + parents.add(group); + try { + final bool skipGroup = group.metadata.skip; + bool setUpAllSucceeded = true; + if (!skipGroup && group.setUpAll != null) { + final LiveTest liveTest = group.setUpAll.load(suiteConfig, groups: parents); + await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false); + setUpAllSucceeded = liveTest.state.result.isPassing; + } + if (setUpAllSucceeded) { + for (GroupEntry entry in group.entries) { + if (entry is Group) { + await _runGroup(suiteConfig, entry, parents, reporter); + } else if (entry.metadata.skip) { + await _runSkippedTest(suiteConfig, entry, parents, reporter); + } else { + final Test test = entry; + await _runLiveTest(suiteConfig, test.load(suiteConfig, groups: parents), reporter); + } + } + } + // Even if we're closed or setUpAll failed, we want to run all the + // teardowns to ensure that any state is properly cleaned up. + if (!skipGroup && group.tearDownAll != null) { + final LiveTest liveTest = group.tearDownAll.load(suiteConfig, groups: parents); + await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false); + } + } finally { + parents.remove(group); + } +} + +Future<void> _runLiveTest(Suite suiteConfig, LiveTest liveTest, _Reporter reporter, {bool countSuccess = true}) async { + reporter._onTestStarted(liveTest); + // Schedule a microtask to ensure that [onTestStarted] fires before the + // first [LiveTest.onStateChange] event. + await Future<void>.microtask(liveTest.run); + // Once the test finishes, use await null to do a coarse-grained event + // loop pump to avoid starving non-microtask events. + await null; + final bool isSuccess = liveTest.state.result.isPassing; + if (isSuccess) { + reporter.passed.add(liveTest); + } else { + reporter.failed.add(liveTest); + } +} + +Future<void> _runSkippedTest(Suite suiteConfig, Test test, List<Group> parents, _Reporter reporter) async { + final LocalTest skipped = LocalTest(test.name, test.metadata, () {}, trace: test.trace); + if (skipped.metadata.skipReason != null) { + print('Skip: ${skipped.metadata.skipReason}'); + } + final LiveTest liveTest = skipped.load(suiteConfig); + reporter._onTestStarted(liveTest); + reporter.skipped.add(skipped); +} + +// TODO(nweiz): This and other top-level functions should throw exceptions if +// they're called after the declarer has finished declaring. +/// Creates a new test case with the given description (converted to a string) +/// and body. +/// +/// The description will be added to the descriptions of any surrounding +/// [group]s. If [testOn] is passed, it's parsed as a [platform selector][]; the +/// test will only be run on matching platforms. +/// +/// [platform selector]: https://github.com/dart-lang/test/tree/master/pkgs/test#platform-selectors +/// +/// If [timeout] is passed, it's used to modify or replace the default timeout +/// of 30 seconds. Timeout modifications take precedence in suite-group-test +/// order, so [timeout] will also modify any timeouts set on the group or suite. +/// +/// If [skip] is a String or `true`, the test is skipped. If it's a String, it +/// should explain why the test is skipped; this reason will be printed instead +/// of running the test. +/// +/// If [tags] is passed, it declares user-defined tags that are applied to the +/// test. These tags can be used to select or skip the test on the command line, +/// or to do bulk test configuration. All tags should be declared in the +/// [package configuration file][configuring tags]. The parameter can be an +/// [Iterable] of tag names, or a [String] representing a single tag. +/// +/// If [retry] is passed, the test will be retried the provided number of times +/// before being marked as a failure. +/// +/// [configuring tags]: https://github.com/dart-lang/test/blob/master/doc/package_config.md#configuring-tags +/// +/// [onPlatform] allows tests to be configured on a platform-by-platform +/// basis. It's a map from strings that are parsed as [PlatformSelector]s to +/// annotation classes: [Timeout], [Skip], or lists of those. These +/// annotations apply only on the given platforms. For example: +/// +/// test('potentially slow test', () { +/// // ... +/// }, onPlatform: { +/// // This test is especially slow on Windows. +/// 'windows': new Timeout.factor(2), +/// 'browser': [ +/// new Skip('TODO: add browser support'), +/// // This will be slow on browsers once it works on them. +/// new Timeout.factor(2) +/// ] +/// }); +/// +/// If multiple platforms match, the annotations apply in order as through +/// they were in nested groups. +/// +/// If the `solo` flag is `true`, only tests and groups marked as +/// 'solo' will be be run. This only restricts tests *within this test +/// suite*—tests in other suites will run as normal. We recommend that users +/// avoid this flag if possible and instead use the test runner flag `-n` to +/// filter tests by name. +void test(Object description, Function body, { + String testOn, + Timeout timeout, + dynamic skip, + dynamic tags, + Map<String, dynamic> onPlatform, + int retry, + }) { + _declarer.test( + description.toString(), body, + testOn: testOn, + timeout: timeout, + skip: skip, + onPlatform: onPlatform, + tags: tags, + retry: retry, + ); +} + +/// Creates a group of tests. +/// +/// A group's description (converted to a string) is included in the descriptions +/// of any tests or sub-groups it contains. [setUp] and [tearDown] are also scoped +/// to the containing group. +/// +/// If [testOn] is passed, it's parsed as a [platform selector][]; the test will +/// only be run on matching platforms. +/// +/// [platform selector]: https://github.com/dart-lang/test/tree/master/pkgs/test#platform-selectors +/// +/// If [timeout] is passed, it's used to modify or replace the default timeout +/// of 30 seconds. Timeout modifications take precedence in suite-group-test +/// order, so [timeout] will also modify any timeouts set on the suite, and will +/// be modified by any timeouts set on individual tests. +/// +/// If [skip] is a String or `true`, the group is skipped. If it's a String, it +/// should explain why the group is skipped; this reason will be printed instead +/// of running the group's tests. +/// +/// If [tags] is passed, it declares user-defined tags that are applied to the +/// test. These tags can be used to select or skip the test on the command line, +/// or to do bulk test configuration. All tags should be declared in the +/// [package configuration file][configuring tags]. The parameter can be an +/// [Iterable] of tag names, or a [String] representing a single tag. +/// +/// [configuring tags]: https://github.com/dart-lang/test/blob/master/doc/package_config.md#configuring-tags +/// +/// [onPlatform] allows groups to be configured on a platform-by-platform +/// basis. It's a map from strings that are parsed as [PlatformSelector]s to +/// annotation classes: [Timeout], [Skip], or lists of those. These +/// annotations apply only on the given platforms. For example: +/// +/// group('potentially slow tests', () { +/// // ... +/// }, onPlatform: { +/// // These tests are especially slow on Windows. +/// 'windows': new Timeout.factor(2), +/// 'browser': [ +/// new Skip('TODO: add browser support'), +/// // They'll be slow on browsers once it works on them. +/// new Timeout.factor(2) +/// ] +/// }); +/// +/// If multiple platforms match, the annotations apply in order as through +/// they were in nested groups. +/// +/// If the `solo` flag is `true`, only tests and groups marked as +/// 'solo' will be be run. This only restricts tests *within this test +/// suite*—tests in other suites will run as normal. We recommend that users +/// avoid this flag if possible, and instead use the test runner flag `-n` to +/// filter tests by name. +void group(Object description, Function body) { + _declarer.group(description.toString(), body); +} + +/// Registers a function to be run before tests. +/// +/// This function will be called before each test is run. [callback] may be +/// asynchronous; if so, it must return a [Future]. +/// +/// If this is called within a test group, it applies only to tests in that +/// group. [callback] will be run after any set-up callbacks in parent groups or +/// at the top level. +/// +/// Each callback at the top level or in a given group will be run in the order +/// they were declared. +void setUp(Function body) { + _declarer.setUp(body); +} + +/// Registers a function to be run after tests. +/// +/// This function will be called after each test is run. [callback] may be +/// asynchronous; if so, it must return a [Future]. +/// +/// If this is called within a test group, it applies only to tests in that +/// group. [callback] will be run before any tear-down callbacks in parent +/// groups or at the top level. +/// +/// Each callback at the top level or in a given group will be run in the +/// reverse of the order they were declared. +/// +/// See also [addTearDown], which adds tear-downs to a running test. +void tearDown(Function body) { + _declarer.tearDown(body); +} + +/// Registers a function to be run once before all tests. +/// +/// [callback] may be asynchronous; if so, it must return a [Future]. +/// +/// If this is called within a test group, [callback] will run before all tests +/// in that group. It will be run after any [setUpAll] callbacks in parent +/// groups or at the top level. It won't be run if none of the tests in the +/// group are run. +/// +/// **Note**: This function makes it very easy to accidentally introduce hidden +/// dependencies between tests that should be isolated. In general, you should +/// prefer [setUp], and only use [setUpAll] if the callback is prohibitively +/// slow. +void setUpAll(Function body) { + _declarer.setUpAll(body); +} + +/// Registers a function to be run once after all tests. +/// +/// If this is called within a test group, [callback] will run after all tests +/// in that group. It will be run before any [tearDownAll] callbacks in parent +/// groups or at the top level. It won't be run if none of the tests in the +/// group are run. +/// +/// **Note**: This function makes it very easy to accidentally introduce hidden +/// dependencies between tests that should be isolated. In general, you should +/// prefer [tearDown], and only use [tearDownAll] if the callback is +/// prohibitively slow. +void tearDownAll(Function body) { + _declarer.tearDownAll(body); +} + + +/// A reporter that prints each test on its own line. +/// +/// This is currently used in place of [CompactReporter] by `lib/test.dart`, +/// which can't transitively import `dart:io` but still needs access to a runner +/// so that test files can be run directly. This means that until issue 6943 is +/// fixed, this must not import `dart:io`. +class _Reporter { + _Reporter({bool color = true, bool printPath = true}) + : _printPath = printPath, + _green = color ? '\u001b[32m' : '', + _red = color ? '\u001b[31m' : '', + _yellow = color ? '\u001b[33m' : '', + _bold = color ? '\u001b[1m' : '', + _noColor = color ? '\u001b[0m' : ''; + final List<LiveTest> passed = <LiveTest>[]; + final List<LiveTest> failed = <LiveTest>[]; + final List<Test> skipped = <Test>[]; + + /// The terminal escape for green text, or the empty string if this is Windows + /// or not outputting to a terminal. + final String _green; + + /// The terminal escape for red text, or the empty string if this is Windows + /// or not outputting to a terminal. + final String _red; + + /// The terminal escape for yellow text, or the empty string if this is + /// Windows or not outputting to a terminal. + final String _yellow; + + /// The terminal escape for bold text, or the empty string if this is + /// Windows or not outputting to a terminal. + final String _bold; + + /// The terminal escape for removing test coloring, or the empty string if + /// this is Windows or not outputting to a terminal. + final String _noColor; + + /// Whether the path to each test's suite should be printed. + final bool _printPath; + + /// A stopwatch that tracks the duration of the full run. + final Stopwatch _stopwatch = Stopwatch(); + + /// The size of `_engine.passed` last time a progress notification was + /// printed. + int _lastProgressPassed; + + /// The size of `_engine.skipped` last time a progress notification was + /// printed. + int _lastProgressSkipped; + + /// The size of `_engine.failed` last time a progress notification was + /// printed. + int _lastProgressFailed; + + /// The message printed for the last progress notification. + String _lastProgressMessage; + + /// The suffix added to the last progress notification. + String _lastProgressSuffix; + + /// The set of all subscriptions to various streams. + final Set<StreamSubscription<void>> _subscriptions = Set<StreamSubscription<void>>(); + + /// A callback called when the engine begins running [liveTest]. + void _onTestStarted(LiveTest liveTest) { + if (!_stopwatch.isRunning) { + _stopwatch.start(); + } + _progressLine(_description(liveTest)); + _subscriptions.add(liveTest.onStateChange.listen((State state) => _onStateChange(liveTest, state))); + _subscriptions.add(liveTest.onError.listen((AsyncError error) => _onError(liveTest, error.error, error.stackTrace))); + _subscriptions.add(liveTest.onMessage.listen((Message message) { + _progressLine(_description(liveTest)); + String text = message.text; + if (message.type == MessageType.skip) { + text = ' $_yellow$text$_noColor'; + } + print(text); + })); + } + + /// A callback called when [liveTest]'s state becomes [state]. + void _onStateChange(LiveTest liveTest, State state) { + if (state.status != Status.complete) { + return; + } + } + + /// A callback called when [liveTest] throws [error]. + void _onError(LiveTest liveTest, Object error, StackTrace stackTrace) { + if (liveTest.state.status != Status.complete) { + return; + } + _progressLine(_description(liveTest), suffix: ' $_bold$_red[E]$_noColor'); + print(_indent(error.toString())); + print(_indent('$stackTrace')); + } + + /// A callback called when the engine is finished running tests. + /// + /// [success] will be `true` if all tests passed, `false` if some tests + /// failed, and `null` if the engine was closed prematurely. + void _onDone() { + final bool success = failed.isEmpty; + if (success == null) { + return; + } + if (!success) { + _progressLine('Some tests failed.', color: _red); + } else if (passed.isEmpty) { + _progressLine('All tests skipped.'); + } else { + _progressLine('All tests passed!'); + } + } + + /// Prints a line representing the current state of the tests. + /// + /// [message] goes after the progress report. If [color] is passed, it's used + /// as the color for [message]. If [suffix] is passed, it's added to the end + /// of [message]. + void _progressLine(String message, {String color, String suffix}) { + // Print nothing if nothing has changed since the last progress line. + if (passed.length == _lastProgressPassed && + skipped.length == _lastProgressSkipped && + failed.length == _lastProgressFailed && + message == _lastProgressMessage && + // Don't re-print just because a suffix was removed. + (suffix == null || suffix == _lastProgressSuffix)) { + return; + } + _lastProgressPassed = passed.length; + _lastProgressSkipped = skipped.length; + _lastProgressFailed = failed.length; + _lastProgressMessage = message; + _lastProgressSuffix = suffix; + + if (suffix != null) { + message += suffix; + } + color ??= ''; + final Duration duration = _stopwatch.elapsed; + final StringBuffer buffer = StringBuffer(); + + // \r moves back to the beginning of the current line. + buffer.write('${_timeString(duration)} '); + buffer.write(_green); + buffer.write('+'); + buffer.write(passed.length); + buffer.write(_noColor); + + if (skipped.isNotEmpty) { + buffer.write(_yellow); + buffer.write(' ~'); + buffer.write(skipped.length); + buffer.write(_noColor); + } + + if (failed.isNotEmpty) { + buffer.write(_red); + buffer.write(' -'); + buffer.write(failed.length); + buffer.write(_noColor); + } + + buffer.write(': '); + buffer.write(color); + buffer.write(message); + buffer.write(_noColor); + + print(buffer.toString()); + } + + /// Returns a representation of [duration] as `MM:SS`. + String _timeString(Duration duration) { + final String minutes = duration.inMinutes.toString().padLeft(2, '0'); + final String seconds = (duration.inSeconds % 60).toString().padLeft(2, '0'); + return '$minutes:$seconds'; + } + + /// Returns a description of [liveTest]. + /// + /// This differs from the test's own description in that it may also include + /// the suite's name. + String _description(LiveTest liveTest) { + String name = liveTest.test.name; + if (_printPath && liveTest.suite.path != null) { + name = '${liveTest.suite.path}: $name'; + } + return name; + } +} + +String _indent(String string, {int size, String first}) { + size ??= first == null ? 2 : first.length; + return _prefixLines(string, ' ' * size, first: first); +} + +String _prefixLines(String text, String prefix, {String first, String last, String single}) { + first ??= prefix; + last ??= prefix; + single ??= first ?? last ?? prefix; + final List<String> lines = text.split('\n'); + if (lines.length == 1) { + return '$single$text'; + } + final StringBuffer buffer = StringBuffer('$first${lines.first}\n'); + // Write out all but the first and last lines with [prefix]. + for (String line in lines.skip(1).take(lines.length - 2)) { + buffer.writeln('$prefix$line'); + } + buffer.write('$last${lines.last}'); + return buffer.toString(); +}
diff --git a/packages/flutter_test/lib/src/widget_tester.dart b/packages/flutter_test/lib/src/widget_tester.dart index ef584aa..b50ab03 100644 --- a/packages/flutter_test/lib/src/widget_tester.dart +++ b/packages/flutter_test/lib/src/widget_tester.dart
@@ -19,12 +19,22 @@ import 'finders.dart'; import 'matchers.dart'; import 'test_async_utils.dart'; +import 'test_compat.dart'; import 'test_text_input.dart'; /// Keep users from needing multiple imports to test semantics. export 'package:flutter/rendering.dart' show SemanticsHandle; +/// Hide these imports so that they do not conflict with our own implementations in +/// test_compat.dart. This handles setting up a declarer when one is not defined, which +/// can happen when a test is executed via flutter_run. export 'package:test_api/test_api.dart' hide + test, + group, + setUpAll, + tearDownAll, + setUp, + tearDown, expect, // we have our own wrapper below TypeMatcher, // matcher's TypeMatcher conflicts with the one in the Flutter framework isInstanceOf; // we have our own wrapper in matchers.dart @@ -63,7 +73,7 @@ final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized(); final WidgetTester tester = WidgetTester._(binding); timeout ??= binding.defaultTestTimeout; - test_package.test( + test( description, () { tester._recordNumberOfSemanticsHandles();