[e2e] Adding failure details for driver tests (#2593)
* Adding failure details for driver tests
* formate fix
* addressing reviewer comments
* addressing the reviewer comments. part 2
* fix error in response_data
* addressing reviewer comments
* fixing formatting
* addressin reviewer comments. Carrying the repeated driver test code to e2e package
* chaning the failure's details type from map to list
* format changes
* add documentation to public members
* formatting
* changing namespace name
diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md
index 72528d6..7996912 100644
--- a/packages/e2e/CHANGELOG.md
+++ b/packages/e2e/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.4.0
+
+* **Breaking change** Driver request_data call's response has changed to
+ encapsulate the failure details.
+* Details for failure cases are added: failed method name, stack trace.
+
## 0.3.0+1
* Replace deprecated `getFlutterEngine` call on Android.
diff --git a/packages/e2e/README.md b/packages/e2e/README.md
index 7d571a5..823b1b8 100644
--- a/packages/e2e/README.md
+++ b/packages/e2e/README.md
@@ -43,16 +43,11 @@
```dart
import 'dart:async';
-import 'dart:io';
-import 'package:flutter_driver/flutter_driver.dart';
-Future<void> main() async {
- final FlutterDriver driver = await FlutterDriver.connect();
- final String result =
- await driver.requestData(null, timeout: const Duration(minutes: 1));
- await driver.close();
- exit(result == 'pass' ? 0 : 1);
-}
+import 'package:e2e/e2e_driver.dart' as e2e;
+
+Future<void> main() async => e2e.main();
+
```
To run a example app test with Flutter driver:
@@ -69,7 +64,7 @@
flutter drive --driver=test_driver/<package_name>_test.dart test/<package_name>_e2e.dart
```
-You can run tests on web on release mode.
+You can run tests on web in release or profile mode.
First you need to make sure you have downloaded the driver for the browser.
diff --git a/packages/e2e/example/test_driver/example_e2e_test.dart b/packages/e2e/example/test_driver/example_e2e_test.dart
index 6147d44..983c386 100644
--- a/packages/e2e/example/test_driver/example_e2e_test.dart
+++ b/packages/e2e/example/test_driver/example_e2e_test.dart
@@ -1,12 +1,5 @@
import 'dart:async';
-import 'dart:io';
-import 'package:flutter_driver/flutter_driver.dart';
+import 'package:e2e/e2e_driver.dart' as e2e;
-Future<void> main() async {
- final FlutterDriver driver = await FlutterDriver.connect();
- final String result =
- await driver.requestData(null, timeout: const Duration(minutes: 1));
- await driver.close();
- exit(result == 'pass' ? 0 : 1);
-}
+Future<void> main() async => e2e.main();
diff --git a/packages/e2e/example/test_driver/failure_test.dart b/packages/e2e/example/test_driver/failure_test.dart
index 9177309..a828df6 100644
--- a/packages/e2e/example/test_driver/failure_test.dart
+++ b/packages/e2e/example/test_driver/failure_test.dart
@@ -1,17 +1,19 @@
import 'dart:async';
+import 'package:e2e/common.dart' as common;
import 'package:flutter_driver/flutter_driver.dart';
import 'package:test/test.dart';
Future<void> main() async {
test('fails gracefully', () async {
final FlutterDriver driver = await FlutterDriver.connect();
- final String result =
+ final String jsonResult =
await driver.requestData(null, timeout: const Duration(minutes: 1));
+ common.Response response = common.Response.fromJson(jsonResult);
await driver.close();
expect(
- result,
- 'fail',
+ response.allTestsPassed,
+ false,
);
});
}
diff --git a/packages/e2e/lib/common.dart b/packages/e2e/lib/common.dart
new file mode 100644
index 0000000..39efcc8
--- /dev/null
+++ b/packages/e2e/lib/common.dart
@@ -0,0 +1,116 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+
+/// An object sent from e2e back to the Flutter Driver in response to
+/// `request_data` command.
+class Response {
+ final List<Failure> _failureDetails;
+
+ final bool _allTestsPassed;
+
+ /// Constructor to use for positive response.
+ Response.allTestsPassed()
+ : this._allTestsPassed = true,
+ this._failureDetails = null;
+
+ /// Constructor for failure response.
+ Response.someTestsFailed(this._failureDetails) : this._allTestsPassed = false;
+
+ /// Whether the test ran successfully or not.
+ bool get allTestsPassed => _allTestsPassed;
+
+ /// If the result are failures get the formatted details.
+ String get formattedFailureDetails =>
+ _allTestsPassed ? '' : formatFailures(_failureDetails);
+
+ /// Failure details as a list.
+ List<Failure> get failureDetails => _failureDetails;
+
+ /// Serializes this message to a JSON map.
+ String toJson() => json.encode(<String, dynamic>{
+ 'result': allTestsPassed.toString(),
+ 'failureDetails': _failureDetailsAsString(),
+ });
+
+ /// Deserializes the result from JSON.
+ static Response fromJson(String source) {
+ Map<String, dynamic> responseJson = json.decode(source);
+ if (responseJson['result'] == 'true') {
+ return Response.allTestsPassed();
+ } else {
+ return Response.someTestsFailed(
+ _failureDetailsFromJson(responseJson['failureDetails']));
+ }
+ }
+
+ /// Method for formating the test failures' details.
+ String formatFailures(List<Failure> failureDetails) {
+ if (failureDetails.isEmpty) {
+ return '';
+ }
+
+ StringBuffer sb = StringBuffer();
+ int failureCount = 1;
+ failureDetails.forEach((Failure f) {
+ sb.writeln('Failure in method: ${f.methodName}');
+ sb.writeln('${f.details}');
+ sb.writeln('end of failure ${failureCount.toString()}\n\n');
+ failureCount++;
+ });
+ return sb.toString();
+ }
+
+ /// Create a list of Strings from [_failureDetails].
+ List<String> _failureDetailsAsString() {
+ final List<String> list = List<String>();
+ if (_failureDetails == null || _failureDetails.isEmpty) {
+ return list;
+ }
+
+ _failureDetails.forEach((Failure f) {
+ list.add(f.toString());
+ });
+
+ return list;
+ }
+
+ /// Creates a [Failure] list using a json response.
+ static List<Failure> _failureDetailsFromJson(List<dynamic> list) {
+ final List<Failure> failureList = List<Failure>();
+ list.forEach((s) {
+ final String failure = s as String;
+ failureList.add(Failure.fromJsonString(failure));
+ });
+ return failureList;
+ }
+}
+
+/// Representing a failure includes the method name and the failure details.
+class Failure {
+ /// The name of the test method which failed.
+ final String methodName;
+
+ /// The details of the failure such as stack trace.
+ final String details;
+
+ /// Constructor requiring all fields during initialization.
+ Failure(this.methodName, this.details);
+
+ /// Serializes the object to JSON.
+ @override
+ String toString() {
+ return json.encode(<String, String>{
+ 'methodName': methodName,
+ 'details': details,
+ });
+ }
+
+ /// Decode a JSON string to create a Failure object.
+ static Failure fromJsonString(String jsonString) {
+ Map<String, dynamic> failure = json.decode(jsonString);
+ return Failure(failure['methodName'], failure['details']);
+ }
+}
diff --git a/packages/e2e/lib/e2e.dart b/packages/e2e/lib/e2e.dart
index eef594a..799b77e 100644
--- a/packages/e2e/lib/e2e.dart
+++ b/packages/e2e/lib/e2e.dart
@@ -3,11 +3,13 @@
// found in the LICENSE file.
import 'dart:async';
+
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter/foundation.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
+import 'common.dart';
import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
@@ -36,6 +38,11 @@
final Completer<bool> _allTestsPassed = Completer<bool>();
+ /// Stores failure details.
+ ///
+ /// Failed test method's names used as key.
+ final List<Failure> _failureMethodsDetails = List<Failure>();
+
/// Similar to [WidgetsFlutterBinding.ensureInitialized].
///
/// Returns an instance of the [E2EWidgetsFlutterBinding], creating and
@@ -63,7 +70,9 @@
case 'request_data':
final bool allTestsPassed = await _allTestsPassed.future;
response = <String, String>{
- 'message': allTestsPassed ? 'pass' : 'fail',
+ 'message': allTestsPassed
+ ? Response.allTestsPassed().toJson()
+ : Response.someTestsFailed(_failureMethodsDetails).toJson(),
};
break;
case 'get_health':
@@ -94,6 +103,7 @@
reportTestException =
(FlutterErrorDetails details, String testDescription) {
_results[description] = 'failed';
+ _failureMethodsDetails.add(Failure(testDescription, details.toString()));
if (!_allTestsPassed.isCompleted) _allTestsPassed.complete(false);
valueBeforeTest(details, testDescription);
};
diff --git a/packages/e2e/lib/e2e_driver.dart b/packages/e2e/lib/e2e_driver.dart
new file mode 100644
index 0000000..2e43c5a
--- /dev/null
+++ b/packages/e2e/lib/e2e_driver.dart
@@ -0,0 +1,21 @@
+import 'dart:async';
+import 'dart:io';
+
+import 'package:e2e/common.dart' as e2e;
+import 'package:flutter_driver/flutter_driver.dart';
+
+Future<void> main() async {
+ final FlutterDriver driver = await FlutterDriver.connect();
+ final String jsonResult =
+ await driver.requestData(null, timeout: const Duration(minutes: 1));
+ final e2e.Response response = e2e.Response.fromJson(jsonResult);
+ await driver.close();
+
+ if (response.allTestsPassed) {
+ print('All tests passed.');
+ exit(0);
+ } else {
+ print('Failure Details:\n${response.formattedFailureDetails}');
+ exit(1);
+ }
+}
diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml
index a087bce..1af3ab9 100644
--- a/packages/e2e/pubspec.yaml
+++ b/packages/e2e/pubspec.yaml
@@ -1,6 +1,6 @@
name: e2e
description: Runs tests that use the flutter_test API as integration tests.
-version: 0.3.0+1
+version: 0.4.0
homepage: https://github.com/flutter/plugins/tree/master/packages/e2e
environment:
@@ -10,6 +10,8 @@
dependencies:
flutter:
sdk: flutter
+ flutter_driver:
+ sdk: flutter
flutter_test:
sdk: flutter
