[e2e] Fix incorrect test results when one test passes then another fails (#2866)
* [e2e] Fix incorrect test results when one test passes then another fails
For example, the following test will result in an error reported for the first test case.
```
void main() {
testWidgets('a test that passes', (tester) async {
expect(true, true);
});
testWidgets('a test that fails', (tester) async {
expect(true, false);
});
}
```
We need to reset `reportTestException` back to the previous value after completion of `runTest`, or repeated failures will cause the exception handler for a previous test to be invoked, as they "stack". Instead of reseting it, however, do this once in the constructor because the test description is already provided by the function signature.
* Add a mechanism for testing test resultsdiff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md
index 2df897d..bc844ce 100644
--- a/packages/e2e/CHANGELOG.md
+++ b/packages/e2e/CHANGELOG.md
@@ -1,6 +1,10 @@
+## 0.6.2+1
+
+* Fix incorrect test results when one test passes then another fails
+
## 0.6.2
-* Fix `setSurfaceSize` for e2e tests.
+* Fix `setSurfaceSize` for e2e tests
## 0.6.1
diff --git a/packages/e2e/lib/e2e.dart b/packages/e2e/lib/e2e.dart
index d773b9b..93ce1f2 100644
--- a/packages/e2e/lib/e2e.dart
+++ b/packages/e2e/lib/e2e.dart
@@ -32,13 +32,26 @@
}
await _channel.invokeMethod<void>(
'allTestsFinished',
- <String, dynamic>{'results': _results},
+ <String, dynamic>{'results': results},
);
} on MissingPluginException {
print('Warning: E2E test plugin was not detected.');
}
if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(true);
});
+
+ // TODO(jackson): Report the results individually instead of all at once
+ // See https://github.com/flutter/flutter/issues/38985
+ final TestExceptionReporter oldTestExceptionReporter = reportTestException;
+ reportTestException =
+ (FlutterErrorDetails details, String testDescription) {
+ results[testDescription] = 'failed';
+ _failureMethodsDetails.add(Failure(testDescription, details.toString()));
+ if (!_allTestsPassed.isCompleted) {
+ _allTestsPassed.complete(false);
+ }
+ oldTestExceptionReporter(details, testDescription);
+ };
}
// TODO(dnfield): Remove the ignore once we bump the minimum Flutter version
@@ -100,7 +113,12 @@
static const MethodChannel _channel = MethodChannel('plugins.flutter.io/e2e');
- static Map<String, String> _results = <String, String>{};
+ /// Test results that will be populated after the tests have completed.
+ ///
+ /// Keys are the test descriptions, and values are either `success` or
+ /// `failed`.
+ @visibleForTesting
+ Map<String, String> results = <String, String>{};
/// The extra data for the reported result.
///
@@ -158,24 +176,12 @@
String description = '',
Duration timeout,
}) async {
- // TODO(jackson): Report the results individually instead of all at once
- // See https://github.com/flutter/flutter/issues/38985
- final TestExceptionReporter oldTestExceptionReporter = reportTestException;
- reportTestException =
- (FlutterErrorDetails details, String testDescription) {
- _results[description] = 'failed';
- _failureMethodsDetails.add(Failure(testDescription, details.toString()));
- if (!_allTestsPassed.isCompleted) {
- _allTestsPassed.complete(false);
- }
- oldTestExceptionReporter(details, testDescription);
- };
await super.runTest(
testBody,
invariantTester,
description: description,
timeout: timeout,
);
- _results[description] ??= 'success';
+ results[description] ??= 'success';
}
}
diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml
index 956a23f..e3f39c0 100644
--- a/packages/e2e/pubspec.yaml
+++ b/packages/e2e/pubspec.yaml
@@ -1,6 +1,6 @@
name: e2e
description: Runs tests that use the flutter_test API as integration tests.
-version: 0.6.2
+version: 0.6.2+1
homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
environment:
diff --git a/packages/e2e/test/binding_fail_test.dart b/packages/e2e/test/binding_fail_test.dart
new file mode 100644
index 0000000..0b00e11
--- /dev/null
+++ b/packages/e2e/test/binding_fail_test.dart
@@ -0,0 +1,81 @@
+import 'dart:async';
+import 'dart:io';
+import 'dart:convert';
+
+import 'package:flutter_test/flutter_test.dart';
+
+// Assumes that the flutter command is in `$PATH`.
+const String _flutterBin = 'flutter';
+const String _e2eResultsPrefix = 'E2EWidgetsFlutterBinding test results:';
+
+void main() async {
+ group('E2E binding result', () {
+ test('when multiple tests pass', () async {
+ final Map<String, dynamic> results =
+ await _runTest('test/data/pass_test_script.dart');
+
+ expect(
+ results,
+ equals({
+ 'passing test 1': 'success',
+ 'passing test 2': 'success',
+ }));
+ });
+
+ test('when multiple tests fail', () async {
+ final Map<String, dynamic> results =
+ await _runTest('test/data/fail_test_script.dart');
+
+ expect(
+ results,
+ equals({
+ 'failing test 1': 'failed',
+ 'failing test 2': 'failed',
+ }));
+ });
+
+ test('when one test passes, then another fails', () async {
+ final Map<String, dynamic> results =
+ await _runTest('test/data/pass_then_fail_test_script.dart');
+
+ expect(
+ results,
+ equals({
+ 'passing test': 'success',
+ 'failing test': 'failed',
+ }));
+ });
+ });
+}
+
+/// Runs a test script and returns the [E2EWidgetsFlutterBinding.result].
+///
+/// [scriptPath] is relative to the package root.
+Future<Map<String, dynamic>> _runTest(String scriptPath) async {
+ final Process process =
+ await Process.start(_flutterBin, ['test', '--machine', scriptPath]);
+
+ // In the test [tearDownAll] block, the test results are encoded into JSON and
+ // are printed with the [_e2eResultsPrefix] prefix.
+ //
+ // See the following for the test event spec which we parse the printed lines
+ // out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md
+ final String testResults = (await process.stdout
+ .transform(utf8.decoder)
+ .expand((String text) => text.split('\n'))
+ .map((String line) {
+ try {
+ return jsonDecode(line);
+ } on FormatException {
+ // Only interested in test events which are JSON.
+ }
+ })
+ .where((dynamic testEvent) =>
+ testEvent != null && testEvent['type'] == 'print')
+ .map((dynamic printEvent) => printEvent['message'] as String)
+ .firstWhere(
+ (String message) => message.startsWith(_e2eResultsPrefix)))
+ .replaceAll(_e2eResultsPrefix, '');
+
+ return jsonDecode(testResults);
+}
diff --git a/packages/e2e/test/data/README.md b/packages/e2e/test/data/README.md
new file mode 100644
index 0000000..e52aca1
--- /dev/null
+++ b/packages/e2e/test/data/README.md
@@ -0,0 +1,4 @@
+Files in this directory are not invoked directly by the test command.
+
+They are used as inputs for the other test files outside of this directory, so
+that failures can be tested.
\ No newline at end of file
diff --git a/packages/e2e/test/data/fail_test_script.dart b/packages/e2e/test/data/fail_test_script.dart
new file mode 100644
index 0000000..cbca590
--- /dev/null
+++ b/packages/e2e/test/data/fail_test_script.dart
@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+ final E2EWidgetsFlutterBinding binding =
+ E2EWidgetsFlutterBinding.ensureInitialized();
+
+ testWidgets('failing test 1', (WidgetTester tester) async {
+ expect(true, false);
+ });
+
+ testWidgets('failing test 2', (WidgetTester tester) async {
+ expect(true, false);
+ });
+
+ tearDownAll(() {
+ print(
+ 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+ });
+}
diff --git a/packages/e2e/test/data/pass_test_script.dart b/packages/e2e/test/data/pass_test_script.dart
new file mode 100644
index 0000000..194f71c
--- /dev/null
+++ b/packages/e2e/test/data/pass_test_script.dart
@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+ final E2EWidgetsFlutterBinding binding =
+ E2EWidgetsFlutterBinding.ensureInitialized();
+
+ testWidgets('passing test 1', (WidgetTester tester) async {
+ expect(true, true);
+ });
+
+ testWidgets('passing test 2', (WidgetTester tester) async {
+ expect(true, true);
+ });
+
+ tearDownAll(() {
+ print(
+ 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+ });
+}
diff --git a/packages/e2e/test/data/pass_then_fail_test_script.dart b/packages/e2e/test/data/pass_then_fail_test_script.dart
new file mode 100644
index 0000000..ffb7cac
--- /dev/null
+++ b/packages/e2e/test/data/pass_then_fail_test_script.dart
@@ -0,0 +1,22 @@
+import 'dart:convert';
+
+import 'package:e2e/e2e.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+void main() async {
+ final E2EWidgetsFlutterBinding binding =
+ E2EWidgetsFlutterBinding.ensureInitialized();
+
+ testWidgets('passing test', (WidgetTester tester) async {
+ expect(true, true);
+ });
+
+ testWidgets('failing test', (WidgetTester tester) async {
+ expect(true, false);
+ });
+
+ tearDownAll(() {
+ print(
+ 'E2EWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+ });
+}
