add web_long_running_tests shard containing long-running web tests (#67324)
diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart
index bc04450..5a5aaeb 100644
--- a/dev/bots/run_command.dart
+++ b/dev/bots/run_command.dart
@@ -128,10 +128,8 @@
.transform(const Utf8Encoder());
switch (outputMode) {
case OutputMode.print:
- await Future.wait<void>(<Future<void>>[
- io.stdout.addStream(stdoutSource),
- io.stderr.addStream(process.stderr),
- ]);
+ stdoutSource.listen(io.stdout.add);
+ process.stderr.listen(io.stderr.add);
break;
case OutputMode.capture:
savedStdout = stdoutSource.toList();
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index e05c6fc..5d9ae8e 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -75,6 +75,11 @@
? int.parse(Platform.environment['WEB_SHARD_COUNT'])
: 8;
+/// The number of shards the long-running Web tests are split into.
+///
+/// WARNING: this number must match the shard count in LUCI configs.
+const int kWebLongRunningTestShardCount = 3;
+
/// Tests that we don't run on Web for various reasons.
//
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
@@ -122,6 +127,7 @@
'tool_tests': _runToolTests,
'web_tests': _runWebUnitTests,
'web_integration_tests': _runWebIntegrationTests,
+ 'web_long_running_tests': _runWebLongRunningTests,
});
} on ExitException catch (error) {
error.apply();
@@ -813,6 +819,125 @@
await selectSubshard(subshards);
}
+/// Coarse-grained integration tests running on the Web.
+///
+/// These tests are sharded into [kWebLongRunningTestShardCount] shards.
+Future<void> _runWebLongRunningTests() async {
+ final List<ShardRunner> tests = <ShardRunner>[
+ () => _runGalleryE2eWebTest('debug'),
+ () => _runGalleryE2eWebTest('debug', canvasKit: true),
+ () => _runGalleryE2eWebTest('profile'),
+ () => _runGalleryE2eWebTest('profile', canvasKit: true),
+ () => _runGalleryE2eWebTest('release'),
+ () => _runGalleryE2eWebTest('release', canvasKit: true),
+ ].map(_withChromeDriver).toList();
+ await _selectIndexedSubshard(tests, kWebLongRunningTestShardCount);
+}
+
+// The `chromedriver` process created by this test.
+//
+// If an existing chromedriver is already available on port 4444, the existing
+// process is reused and this variable remains null.
+Command _chromeDriver;
+
+/// Creates a shard runner that runs the given [originalRunner] with ChromeDriver
+/// enabled.
+ShardRunner _withChromeDriver(ShardRunner originalRunner) {
+ return () async {
+ try {
+ await _ensureChromeDriverIsRunning();
+ await originalRunner();
+ } finally {
+ await _stopChromeDriver();
+ }
+ };
+}
+
+Future<bool> _isChromeDriverRunning() async {
+ try {
+ (await Socket.connect('localhost', 4444)).destroy();
+ return true;
+ } on SocketException {
+ return false;
+ }
+}
+
+Future<void> _ensureChromeDriverIsRunning() async {
+ // If we cannot connect to ChromeDriver, assume it is not running. Launch it.
+ if (!await _isChromeDriverRunning()) {
+ print('Starting chromedriver');
+ // Assume chromedriver is in the PATH.
+ _chromeDriver = await startCommand(
+ 'chromedriver',
+ <String>['--port=4444'],
+ );
+ while (!await _isChromeDriverRunning()) {
+ await Future<void>.delayed(const Duration(milliseconds: 100));
+ print('Waiting for chromedriver to start up.');
+ }
+ }
+
+ final HttpClient client = HttpClient();
+ final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
+ final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
+ final HttpClientResponse response = await request.close();
+ final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join('')) as Map<String, dynamic>;
+ client.close();
+ final bool webDriverReady = webDriverStatus['value']['ready'] as bool;
+ if (!webDriverReady) {
+ throw Exception('WebDriver not available.');
+ }
+}
+
+Future<void> _stopChromeDriver() async {
+ if (_chromeDriver == null) {
+ return;
+ }
+ _chromeDriver.process.kill();
+ while (await _isChromeDriverRunning()) {
+ await Future<void>.delayed(const Duration(milliseconds: 100));
+ print('Waiting for chromedriver to stop.');
+ }
+}
+
+/// Exercises the old gallery in a browser for a long period of time, looking
+/// for memory leaks and dangling pointers.
+///
+/// This is not a performance test.
+///
+/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
+///
+/// The test is written using `package:integration_test` (despite the "e2e" in
+/// the name, which is there for historic reasons).
+Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
+ print('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
+ final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
+ await runCommand(
+ flutter,
+ <String>[ 'clean' ],
+ workingDirectory: testAppDirectory,
+ );
+ await runCommand(
+ flutter,
+ <String>[
+ 'drive',
+ if (canvasKit)
+ '--dart-define=FLUTTER_WEB_USE_SKIA=true',
+ '--driver=test_driver/transitions_perf_e2e_test.dart',
+ '--target=test_driver/transitions_perf_e2e.dart',
+ '--browser-name=chrome',
+ '-d',
+ 'web-server',
+ '--$buildMode',
+ ],
+ workingDirectory: testAppDirectory,
+ environment: <String, String>{
+ 'FLUTTER_WEB': 'true',
+ },
+ );
+ print('${green}Integration test passed.$reset');
+}
+
Future<void> _runWebIntegrationTests() async {
await _runWebStackTraceTest('profile', 'lib/stack_trace.dart');
await _runWebStackTraceTest('release', 'lib/stack_trace.dart');
diff --git a/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart b/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart
index 09c29fe..2df9fab 100644
--- a/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart
+++ b/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart
@@ -5,12 +5,21 @@
import 'dart:ui';
import 'package:flutter/cupertino.dart';
+import 'package:flutter/foundation.dart';
import 'package:flutter/widgets.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:flutter_gallery/demo_lists.dart';
-const List<String> kSkippedDemos = <String>[];
+/// The demos we don't run as part of the integraiton test.
+///
+/// Demo names are formatted as 'DEMO_NAME@DEMO_CATEGORY' (see
+/// `demo_lists.dart` for more examples).
+final List<String> kSkippedDemos = <String>[
+ // The CI uses Chromium, which lacks the video codecs to run this demo.
+ if (kIsWeb)
+ 'Video@Media',
+];
/// Scrolls each demo menu item into view, launches it, then returns to the
/// home screen twice.
diff --git a/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart b/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart
index 69e9be5..d1f6c4f 100644
--- a/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart
+++ b/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart
@@ -14,8 +14,6 @@
import 'run_demos.dart';
-const List<String> kSkippedDemos = <String>[];
-
// All of the gallery demos, identified as "title@category".
//
// These names are reported by the test app, see _handleMessages()
diff --git a/packages/flutter_driver/lib/src/driver/web_driver.dart b/packages/flutter_driver/lib/src/driver/web_driver.dart
index 2126b75..7c5facd 100644
--- a/packages/flutter_driver/lib/src/driver/web_driver.dart
+++ b/packages/flutter_driver/lib/src/driver/web_driver.dart
@@ -187,7 +187,11 @@
class FlutterWebConnection {
/// Creates a FlutterWebConnection with WebDriver
/// and whether the WebDriver supports timeline action.
- FlutterWebConnection(this._driver, this.supportsTimelineAction);
+ FlutterWebConnection(this._driver, this.supportsTimelineAction) {
+ _driver.logs.get(async_io.LogType.browser).listen((async_io.LogEntry entry) {
+ print('[${entry.level}]: ${entry.message}');
+ });
+ }
final async_io.WebDriver _driver;
diff --git a/packages/flutter_tools/lib/src/drive/web_driver_service.dart b/packages/flutter_tools/lib/src/drive/web_driver_service.dart
index 21a6a9b..481d071 100644
--- a/packages/flutter_tools/lib/src/drive/web_driver_service.dart
+++ b/packages/flutter_tools/lib/src/drive/web_driver_service.dart
@@ -186,7 +186,10 @@
return <String, dynamic>{
'acceptInsecureCerts': true,
'browserName': 'chrome',
- 'goog:loggingPrefs': <String, String>{ async_io.LogType.performance: 'ALL'},
+ 'goog:loggingPrefs': <String, String>{
+ async_io.LogType.browser: 'INFO',
+ async_io.LogType.performance: 'ALL',
+ },
'chromeOptions': <String, dynamic>{
if (chromeBinary != null)
'binary': chromeBinary,
diff --git a/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart b/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart
index 9b85ab1..91f2c77 100644
--- a/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart
+++ b/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart
@@ -12,7 +12,10 @@
final Map<String, dynamic> expected = <String, dynamic>{
'acceptInsecureCerts': true,
'browserName': 'chrome',
- 'goog:loggingPrefs': <String, String>{ sync_io.LogType.performance: 'ALL'},
+ 'goog:loggingPrefs': <String, String>{
+ sync_io.LogType.browser: 'INFO',
+ sync_io.LogType.performance: 'ALL',
+ },
'chromeOptions': <String, dynamic>{
'w3c': false,
'args': <String>[
@@ -44,7 +47,10 @@
final Map<String, dynamic> expected = <String, dynamic>{
'acceptInsecureCerts': true,
'browserName': 'chrome',
- 'goog:loggingPrefs': <String, String>{ sync_io.LogType.performance: 'ALL'},
+ 'goog:loggingPrefs': <String, String>{
+ sync_io.LogType.browser: 'INFO',
+ sync_io.LogType.performance: 'ALL',
+ },
'chromeOptions': <String, dynamic>{
'binary': chromeBinary,
'w3c': false,
