Make gallery tests more robust (#15957)
diff --git a/examples/flutter_gallery/lib/demo/material/backdrop_demo.dart b/examples/flutter_gallery/lib/demo/material/backdrop_demo.dart
index 553ee96..e086377 100644
--- a/examples/flutter_gallery/lib/demo/material/backdrop_demo.dart
+++ b/examples/flutter_gallery/lib/demo/material/backdrop_demo.dart
@@ -14,6 +14,8 @@
const Category({ this.title, this.assets });
final String title;
final List<String> assets;
+ @override
+ String toString() => '$runtimeType("$title")';
}
const List<Category> allCategories = const <Category>[
@@ -178,7 +180,10 @@
alignment: AlignmentDirectional.centerStart,
child: new DefaultTextStyle(
style: theme.textTheme.subhead,
- child: title,
+ child: new Tooltip(
+ message: 'Tap to dismiss',
+ child: title,
+ ),
),
),
),
diff --git a/examples/flutter_gallery/lib/gallery/item.dart b/examples/flutter_gallery/lib/gallery/item.dart
index e4247a8..a9707da 100644
--- a/examples/flutter_gallery/lib/gallery/item.dart
+++ b/examples/flutter_gallery/lib/gallery/item.dart
@@ -118,7 +118,7 @@
),
new GalleryItem(
title: 'Data tables',
- subtitle: 'Data tables',
+ subtitle: 'Rows and columns',
category: 'Material Components',
routeName: DataTableDemo.routeName,
buildRoute: (BuildContext context) => new DataTableDemo(),
diff --git a/examples/flutter_gallery/test/live_smoketest.dart b/examples/flutter_gallery/test/live_smoketest.dart
index dda898e..5f21972 100644
--- a/examples/flutter_gallery/test/live_smoketest.dart
+++ b/examples/flutter_gallery/test/live_smoketest.dart
@@ -11,30 +11,55 @@
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter_gallery/gallery/app.dart';
+import 'package:flutter_gallery/gallery/item.dart';
-/// Reports success or failure to the native code.
+// Reports success or failure to the native code.
const MethodChannel _kTestChannel = const MethodChannel('io.flutter.demo.gallery/TestLifecycleListener');
+// The titles for all of the Gallery demos.
+final List<String> _kAllDemos = kAllGalleryItems.map((GalleryItem item) => item.title).toList();
+
+// We don't want to wait for animations to complete before tapping the
+// back button in the demos with these titles.
+const List<String> _kUnsynchronizedDemos = const <String>[
+ 'Progress indicators',
+ 'Activity Indicator',
+ 'Video',
+];
+
+// These demos can't be backed out of by tapping a button whose
+// tooltip is 'Back'.
+const List<String> _kSkippedDemos = const <String>[
+ 'Backdrop',
+ 'Pull to refresh',
+];
+
Future<Null> main() async {
try {
+ // Verify that _kUnsynchronizedDemos and _kSkippedDemos identify
+ // demos that actually exist.
+ if (!new Set<String>.from(_kAllDemos).containsAll(_kUnsynchronizedDemos))
+ fail('Unrecognized demo names in _kUnsynchronizedDemos: $_kUnsynchronizedDemos');
+ if (!new Set<String>.from(_kAllDemos).containsAll(_kSkippedDemos))
+ fail('Unrecognized demo names in _kSkippedDemos: $_kSkippedDemos');
+
runApp(const GalleryApp());
-
- const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
final _LiveWidgetController controller = new _LiveWidgetController();
-
- for (Demo demo in demos) {
- print('Testing "${demo.title}" demo');
- final Finder menuItem = find.text(demo.title);
+ for (String demo in _kAllDemos) {
+ print('Testing "$demo" demo');
+ final Finder menuItem = find.text(demo);
await controller.scrollIntoView(menuItem, alignment: 0.5);
- await new Future<Null>.delayed(kWaitBetweenActions);
+
+ if (_kSkippedDemos.contains(demo)) {
+ print('> skipped $demo');
+ continue;
+ }
for (int i = 0; i < 2; i += 1) {
await controller.tap(menuItem); // Launch the demo
- await new Future<Null>.delayed(kWaitBetweenActions);
- controller.frameSync = demo.synchronized;
+ controller.frameSync = !_kUnsynchronizedDemos.contains(demo);
await controller.tap(find.byTooltip('Back'));
controller.frameSync = true;
- await new Future<Null>.delayed(kWaitBetweenActions);
}
print('Success');
}
@@ -45,70 +70,6 @@
}
}
-class Demo {
- const Demo(this.title, {this.synchronized = true});
-
- /// The title of the demo.
- final String title;
-
- /// True if frameSync should be enabled for this test.
- final bool synchronized;
-}
-
-// Warning: this list must be kept in sync with the value of
-// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
-const List<Demo> demos = const <Demo>[
- // Demos
- const Demo('Shrine'),
- const Demo('Contact profile'),
- const Demo('Animation'),
-
- // Material Components
- const Demo('Bottom navigation'),
- const Demo('Buttons'),
- const Demo('Cards'),
- const Demo('Chips'),
- const Demo('Date and time pickers'),
- const Demo('Dialog'),
- const Demo('Drawer'),
- const Demo('Expand/collapse list control'),
- const Demo('Expansion panels'),
- const Demo('Floating action button'),
- const Demo('Grid'),
- const Demo('Icons'),
- const Demo('Leave-behind list items'),
- const Demo('List'),
- const Demo('Menus'),
- const Demo('Modal bottom sheet'),
- const Demo('Page selector'),
- const Demo('Persistent bottom sheet'),
- const Demo('Progress indicators', synchronized: false),
- const Demo('Pull to refresh'),
- const Demo('Scrollable tabs'),
- const Demo('Selection controls'),
- const Demo('Sliders'),
- const Demo('Snackbar'),
- const Demo('Tabs'),
- const Demo('Text fields'),
- const Demo('Tooltips'),
-
- // Cupertino Components
- const Demo('Activity Indicator', synchronized: false),
- const Demo('Buttons'),
- const Demo('Dialogs'),
- const Demo('Navigation'),
- const Demo('Sliders'),
- const Demo('Switches'),
-
- // Media
- const Demo('Animated images'),
-
- // Style
- const Demo('Colors'),
- const Demo('Typography'),
-];
-
-
class _LiveWidgetController {
final WidgetController _controller = new WidgetController(WidgetsBinding.instance);
diff --git a/examples/flutter_gallery/test/smoke_test.dart b/examples/flutter_gallery/test/smoke_test.dart
index c550cac..1bb9666 100644
--- a/examples/flutter_gallery/test/smoke_test.dart
+++ b/examples/flutter_gallery/test/smoke_test.dart
@@ -102,11 +102,19 @@
await tester.pump();
await tester.pump(const Duration(milliseconds: 400));
+ // This demo's back button isn't initially visible.
+ if (routeName == '/material/backdrop') {
+ await tester.tap(find.byTooltip('Tap to dismiss'));
+ await tester.pumpAndSettle();
+ }
+
// Go back
await tester.pageBack();
+ await tester.pumpAndSettle();
await tester.pump(); // Start the pop "back" operation.
await tester.pump(); // Complete the willPop() Future.
await tester.pump(const Duration(milliseconds: 400)); // Wait until it has finished.
+
return null;
}
@@ -126,8 +134,6 @@
final Finder finder = findGalleryItemByRouteName(tester, routeName);
Scrollable.ensureVisible(tester.element(finder), alignment: 0.5);
await tester.pumpAndSettle();
- if (routeName == '/material/backdrop')
- continue;
await smokeDemo(tester, routeName);
tester.binding.debugAssertNoTransientCallbacks('A transient callback was still active after leaving route $routeName');
}
diff --git a/examples/flutter_gallery/test_driver/memory_nav_test.dart b/examples/flutter_gallery/test_driver/memory_nav_test.dart
index c61a104..6925cee 100644
--- a/examples/flutter_gallery/test_driver/memory_nav_test.dart
+++ b/examples/flutter_gallery/test_driver/memory_nav_test.dart
@@ -23,18 +23,14 @@
final SerializableFinder menuItem = find.text('Text fields');
driver.waitFor(menuItem).then<Null>((Null value) async {
scroll = false;
- await new Future<Null>.delayed(kWaitBetweenActions);
for (int i = 0; i < 15; i++) {
await driver.tap(menuItem);
- await new Future<Null>.delayed(kWaitBetweenActions);
await driver.tap(find.byTooltip('Back'));
- await new Future<Null>.delayed(kWaitBetweenActions);
}
completer.complete();
});
while (scroll) {
await driver.scroll(find.text('Flutter Gallery'), 0.0, -500.0, const Duration(milliseconds: 80));
- await new Future<Null>.delayed(kWaitBetweenActions);
}
await completer.future;
}, timeout: const Timeout(const Duration(minutes: 1)));
diff --git a/examples/flutter_gallery/test_driver/transitions_perf.dart b/examples/flutter_gallery/test_driver/transitions_perf.dart
index c0010d0..dcb401c 100644
--- a/examples/flutter_gallery/test_driver/transitions_perf.dart
+++ b/examples/flutter_gallery/test_driver/transitions_perf.dart
@@ -2,10 +2,21 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
+import 'dart:async';
+import 'dart:convert' show JsonEncoder;
+
import 'package:flutter_driver/driver_extension.dart';
+import 'package:flutter_gallery/gallery/item.dart';
import 'package:flutter_gallery/main.dart' as app;
+Future<String> _handleMessages(String message) async {
+ assert(message == 'demoNames');
+ return const JsonEncoder.withIndent(' ').convert(
+ kAllGalleryItems.map((GalleryItem item) => item.title).toList(),
+ );
+}
+
void main() {
- enableFlutterDriverExtension();
+ enableFlutterDriverExtension(handler: _handleMessages);
app.main();
}
diff --git a/examples/flutter_gallery/test_driver/transitions_perf_test.dart b/examples/flutter_gallery/test_driver/transitions_perf_test.dart
index 0605380..1128934 100644
--- a/examples/flutter_gallery/test_driver/transitions_perf_test.dart
+++ b/examples/flutter_gallery/test_driver/transitions_perf_test.dart
@@ -3,7 +3,7 @@
// found in the LICENSE file.
import 'dart:async';
-import 'dart:convert' show JsonEncoder;
+import 'dart:convert' show JsonEncoder, JsonDecoder;
import 'package:file/file.dart';
import 'package:file/local.dart';
@@ -11,81 +11,46 @@
import 'package:path/path.dart' as path;
import 'package:test/test.dart';
-class Demo {
- const Demo(this.title, {this.synchronized = true, this.profiled = false});
-
- /// The title of the demo.
- final String title;
-
- /// True if frameSync should be enabled for this test.
- final bool synchronized;
-
- // True if timeline data should be collected for this test.
- //
- // Warning: The number of tests executed with timeline collection enabled
- // significantly impacts heap size of the running app. When run with
- // --trace-startup, as we do in this test, the VM stores trace events in an
- // endless buffer instead of a ring buffer.
- final bool profiled;
-}
-
-// Warning: this list must be kept in sync with the value of
-// kAllGalleryItems.map((GalleryItem item) => item.title).toList();
-const List<Demo> demos = const <Demo>[
- // Demos
- const Demo('Shrine', profiled: true),
- const Demo('Contact profile', profiled: true),
- const Demo('Animation', profiled: true),
-
- // Material Components
- const Demo('Bottom navigation', profiled: true),
- const Demo('Buttons', profiled: true),
- const Demo('Cards', profiled: true),
- const Demo('Chips', profiled: true),
- const Demo('Date and time pickers', profiled: true),
- const Demo('Dialog', profiled: true),
- const Demo('Drawer'),
- const Demo('Expand/collapse list control'),
- const Demo('Expansion panels'),
- const Demo('Floating action button'),
- const Demo('Grid'),
- const Demo('Icons'),
- const Demo('Leave-behind list items'),
- const Demo('List'),
- const Demo('Menus'),
- const Demo('Modal bottom sheet'),
- const Demo('Page selector'),
- const Demo('Persistent bottom sheet'),
- const Demo('Progress indicators', synchronized: false),
- const Demo('Pull to refresh'),
- const Demo('Scrollable tabs'),
- const Demo('Selection controls'),
- const Demo('Sliders'),
- const Demo('Snackbar'),
- const Demo('Tabs'),
- const Demo('Text fields'),
- const Demo('Tooltips'),
-
- // Cupertino Components
- const Demo('Activity Indicator', synchronized: false),
- const Demo('Buttons'),
- const Demo('Dialogs'),
- const Demo('Navigation'),
- const Demo('Pickers'),
- const Demo('Sliders'),
- const Demo('Switches'),
-
- // Media
- const Demo('Animated images'),
-
- // Style
- const Demo('Colors'),
- const Demo('Typography'),
-];
-
const FileSystem _fs = const LocalFileSystem();
-const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
+// Demos for which timeline data will be collected using
+// FlutterDriver.traceAction().
+//
+// Warning: The number of tests executed with timeline collection enabled
+// significantly impacts heap size of the running app. When run with
+// --trace-startup, as we do in this test, the VM stores trace events in an
+// endless buffer instead of a ring buffer.
+//
+// These names must match GalleryItem titles from kAllGalleryItems
+// in examples/flutter_gallery/lib/gallery.item.dart
+const List<String> kProfiledDemos = const <String>[
+ 'Shrine',
+ 'Contact profile',
+ 'Animation',
+ 'Bottom navigation',
+ 'Buttons',
+ 'Cards',
+ 'Chips',
+ 'Date and time pickers',
+ 'Dialog',
+];
+
+// Demos that will be backed out of within FlutterDriver.runUnsynchronized();
+//
+// These names must match GalleryItem titles from kAllGalleryItems
+// in examples/flutter_gallery/lib/gallery.item.dart
+const List<String> kUnsynchronizedDemos = const <String>[
+ 'Progress indicators',
+ 'Activity Indicator',
+ 'Video',
+];
+
+// All of the gallery demo titles in the order they appear on the
+// gallery home page.
+//
+// These names are reported by the test app, see _handleMessages()
+// in transitions_perf.dart.
+List<String> _allDemos = <String>[];
/// Extracts event data from [events] recorded by timeline, validates it, turns
/// it into a histogram, and saves to a JSON file.
@@ -155,25 +120,29 @@
/// Scrolls each demo menu item into view, launches it, then returns to the
/// home screen twice.
-Future<Null> runDemos(Iterable<Demo> demos, FlutterDriver driver) async {
- for (Demo demo in demos) {
- print('Testing "${demo.title}" demo');
- final SerializableFinder menuItem = find.text(demo.title);
- await driver.scrollIntoView(menuItem, alignment: 0.5);
- await new Future<Null>.delayed(kWaitBetweenActions);
+Future<Null> runDemos(List<String> demos, FlutterDriver driver) async {
+ for (String demo in demos) {
+ print('Testing "$demo" demo');
+ final SerializableFinder menuItem = find.text(demo);
+ await driver.scrollUntilVisible(find.byType('CustomScrollView'), menuItem,
+ dyScroll: -48.0,
+ alignment: 0.5,
+ );
for (int i = 0; i < 2; i += 1) {
await driver.tap(menuItem); // Launch the demo
- await new Future<Null>.delayed(kWaitBetweenActions);
- if (demo.synchronized) {
- await driver.tap(find.byTooltip('Back'));
- } else {
+
+ // This demo's back button isn't initially visible.
+ if (demo == 'Backdrop')
+ await driver.tap(find.byTooltip('Tap to dismiss'));
+
+ if (kUnsynchronizedDemos.contains(demo)) {
await driver.runUnsynchronized<Future<Null>>(() async {
- await new Future<Null>.delayed(kWaitBetweenActions);
await driver.tap(find.byTooltip('Back'));
});
+ } else {
+ await driver.tap(find.byTooltip('Back'));
}
- await new Future<Null>.delayed(kWaitBetweenActions);
}
print('Success');
}
@@ -184,10 +153,16 @@
FlutterDriver driver;
setUpAll(() async {
driver = await FlutterDriver.connect();
+
if (args.contains('--with_semantics')) {
print('Enabeling semantics...');
await driver.setSemantics(true);
}
+
+ // See _handleMessages() in transitions_perf.dart.
+ _allDemos = const JsonDecoder().convert(await driver.requestData('demoNames'));
+ if (_allDemos.isEmpty)
+ throw 'no demo names found';
});
tearDownAll(() async {
@@ -197,14 +172,15 @@
test('all demos', () async {
// Collect timeline data for just a limited set of demos to avoid OOMs.
- final Timeline timeline = await driver.traceAction(() async {
- final Iterable<Demo> profiledDemos = demos.where((Demo demo) => demo.profiled);
- await runDemos(profiledDemos, driver);
- },
- streams: const <TimelineStream>[
- TimelineStream.dart,
- TimelineStream.embedder,
- ]);
+ final Timeline timeline = await driver.traceAction(
+ () async {
+ await runDemos(kProfiledDemos, driver);
+ },
+ streams: const <TimelineStream>[
+ TimelineStream.dart,
+ TimelineStream.embedder,
+ ],
+ );
// Save the duration (in microseconds) of the first timeline Frame event
// that follows a 'Start Transition' event. The Gallery app adds a
@@ -214,9 +190,15 @@
final String histogramPath = path.join(testOutputsDirectory, 'transition_durations.timeline.json');
await saveDurationsHistogram(timeline.json['traceEvents'], histogramPath);
+ // Scroll back to the top
+ await driver.scrollUntilVisible(find.byType('CustomScrollView'), find.text(_allDemos[0]),
+ dyScroll: 200.0,
+ alignment: 0.0
+ );
+
// Execute the remaining tests.
- final Iterable<Demo> unprofiledDemos = demos.where((Demo demo) => !demo.profiled);
- await runDemos(unprofiledDemos, driver);
+ final Set<String> unprofiledDemos = new Set<String>.from(_allDemos)..removeAll(kProfiledDemos);
+ await runDemos(unprofiledDemos.toList(), driver);
}, timeout: const Timeout(const Duration(minutes: 5)));
});
diff --git a/packages/flutter_driver/lib/src/driver/driver.dart b/packages/flutter_driver/lib/src/driver/driver.dart
index c269187..3380183 100644
--- a/packages/flutter_driver/lib/src/driver/driver.dart
+++ b/packages/flutter_driver/lib/src/driver/driver.dart
@@ -409,10 +409,71 @@
/// Scrolls the Scrollable ancestor of the widget located by [finder]
/// until the widget is completely visible.
+ ///
+ /// If the widget located by [finder] is contained by a scrolling widget
+ /// that lazily creates its children, like [ListView] or [CustomScrollView],
+ /// then this method may fail because [finder] doesn't actually exist.
+ /// The [scrollUntilVisible] method can be used in this case.
Future<Null> scrollIntoView(SerializableFinder finder, { double alignment: 0.0, Duration timeout }) async {
return await _sendCommand(new ScrollIntoView(finder, alignment: alignment, timeout: timeout)).then((Map<String, dynamic> _) => null);
}
+ /// Repeatedly [scroll] the widget located by [scrollable] by [dxScroll] and
+ /// [dyScroll] until [item] is visible, and then use [scrollIntoView] to
+ /// ensure the item's final position matches [alignment].
+ ///
+ /// The [scrollable] must locate the scrolling widget that contains [item].
+ /// Typically `find.byType('ListView') or `find.byType('CustomScrollView')`.
+ ///
+ /// Atleast one of [dxScroll] and [dyScroll] must be non-zero.
+ ///
+ /// If [item] is below the currently visible items, then specify a negative
+ /// value for [dyScroll] that's a small enough increment to expose [item]
+ /// without potentially scrolling it up and completely out of view. Similarly
+ /// if [item] is above, then specify a positve value for [dyScroll].
+ ///
+ /// If [item] is to the right of the the currently visible items, then
+ /// specify a negative value for [dxScroll] that's a small enough increment to
+ /// expose [item] without potentially scrolling it up and completely out of
+ /// view. Similarly if [item] is to the left, then specify a positve value
+ /// for [dyScroll].
+ ///
+ /// The [timeout] value should be long enough to accommodate as many scrolls
+ /// as needed to bring an item into view. The default is 10 seconds.
+ Future<Null> scrollUntilVisible(SerializableFinder scrollable, SerializableFinder item, {
+ double alignment: 0.0,
+ double dxScroll: 0.0,
+ double dyScroll: 0.0,
+ Duration timeout: const Duration(seconds: 10),
+ }) async {
+ assert(scrollable != null);
+ assert(item != null);
+ assert(alignment != null);
+ assert(dxScroll != null);
+ assert(dyScroll != null);
+ assert(dxScroll != 0.0 || dyScroll != 0.0);
+ assert(timeout != null);
+
+ // If the item is already visible then we're done.
+ bool isVisible = false;
+ try {
+ await waitFor(item, timeout: const Duration(milliseconds: 100));
+ isVisible = true;
+ } on DriverError {
+ // Assume that that waitFor timed out because the item isn't visible.
+ }
+
+ if (!isVisible) {
+ waitFor(item, timeout: timeout).then((Null _) { isVisible = true; });
+ while (!isVisible) {
+ await scroll(scrollable, dxScroll, dyScroll, const Duration(milliseconds: 100));
+ await new Future<Null>.delayed(const Duration(milliseconds: 500));
+ }
+ }
+
+ return scrollIntoView(item, alignment: alignment);
+ }
+
/// Returns the text in the `Text` widget located by [finder].
Future<String> getText(SerializableFinder finder, { Duration timeout }) async {
return GetTextResult.fromJson(await _sendCommand(new GetText(finder, timeout: timeout))).text;
