Move web integration tool tests to web.shard (#70226)
* Move web integration tool tests to web.shard
Web integration tool tests depend on DDC changes in SDK. This change
moves them to a separate shard and subshard so CI bot configurations
can run them separately.
In particular, with will allow running those tests on dart CI flutter
HHH web bot instead of a non-web one, allowing early detection and easy
classification of issues caused by SDK changes as VM- or Web related.
* Enabled verbose mode for flaky web_tool_tests
* Split out the test changes to be commited first
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index 89f59d0..c8a659f 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -125,6 +125,7 @@
'framework_tests': _runFrameworkTests,
'tool_coverage': _runToolCoverage,
'tool_tests': _runToolTests,
+ 'web_tool_tests': _runWebToolTests,
'web_tests': _runWebUnitTests,
'web_integration_tests': _runWebIntegrationTests,
'web_long_running_tests': _runWebLongRunningTests,
@@ -313,6 +314,7 @@
Future<void> _runToolTests() async {
const String kDotShard = '.shard';
+ const String kWeb = 'web';
const String kTest = 'test';
final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
@@ -321,6 +323,7 @@
.listSync()
.map<String>((FileSystemEntity entry) => entry.path)
.where((String name) => name.endsWith(kDotShard))
+ .where((String name) => path.basenameWithoutExtension(name) != kWeb)
.map<String>((String name) => path.basenameWithoutExtension(name)),
// The `dynamic` on the next line is because Map.fromIterable isn't generic.
value: (dynamic subshard) => () async {
@@ -337,10 +340,27 @@
);
},
);
- // Prevent web tests from running if not explicitly requested.
- if (Platform.environment[CIRRUS_TASK_NAME] == null) {
- subshards.remove('web');
- }
+
+ await selectSubshard(subshards);
+}
+
+Future<void> _runWebToolTests() async {
+ const String kDotShard = '.shard';
+ const String kWeb = 'web';
+ const String kTest = 'test';
+ final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
+
+ final Map<String, ShardRunner> subshards = <String, ShardRunner>{
+ kWeb:
+ () async {
+ await _pubRunTest(
+ toolsPath,
+ forceSingleCore: true,
+ testPaths: <String>[path.join(kTest, '$kWeb$kDotShard', '')],
+ enableFlutterToolAsserts: true,
+ );
+ }
+ };
await selectSubshard(subshards);
}
diff --git a/packages/flutter_tools/test/web.shard/README.md b/packages/flutter_tools/test/web.shard/README.md
new file mode 100644
index 0000000..39130a1
--- /dev/null
+++ b/packages/flutter_tools/test/web.shard/README.md
@@ -0,0 +1,17 @@
+# Web integration tests
+
+These tests are not hermetic, and use the actual Flutter SDK. While
+they don't require actual devices, they run `flutter_tester` to test
+Dart web debug services (dwds) and Flutter integration.
+
+Use this command to run (from the `flutter_tools` directory):
+
+```shell
+../../bin/cache/dart-sdk/bin/pub run test test/web.shard
+```
+
+These tests are expensive to run and do not give meaningful coverage
+information for the flutter tool (since they are black-box tests that
+run the tool as a subprocess, rather than being unit tests). For this
+reason, they are in a separate shard when running on continuous
+integration and are not run when calculating coverage.
diff --git a/packages/flutter_tools/test/integration.shard/debugger_stepping_web_test.dart b/packages/flutter_tools/test/web.shard/debugger_stepping_web_test.dart
similarity index 84%
rename from packages/flutter_tools/test/integration.shard/debugger_stepping_web_test.dart
rename to packages/flutter_tools/test/web.shard/debugger_stepping_web_test.dart
index 6ab2ec0..aa0c048 100644
--- a/packages/flutter_tools/test/integration.shard/debugger_stepping_web_test.dart
+++ b/packages/flutter_tools/test/web.shard/debugger_stepping_web_test.dart
@@ -6,10 +6,10 @@
import 'package:file/file.dart';
+import '../integration.shard/test_data/stepping_project.dart';
+import '../integration.shard/test_driver.dart';
+import '../integration.shard/test_utils.dart';
import '../src/common.dart';
-import 'test_data/stepping_project.dart';
-import 'test_driver.dart';
-import 'test_utils.dart';
void main() {
Directory tempDirectory;
@@ -25,7 +25,9 @@
flutter = FlutterRunTestDriver(tempDirectory);
- await flutter.run(withDebugger: true, startPaused: true, chrome: true);
+ await flutter.run(
+ withDebugger: true, startPaused: true, chrome: true,
+ additionalCommandArgs: <String>['--verbose']);
await flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
await flutter.resume();
await flutter.waitForPause(); // Now we should be on the breakpoint.
diff --git a/packages/flutter_tools/test/web.shard/expression_evaluation_web_test.dart b/packages/flutter_tools/test/web.shard/expression_evaluation_web_test.dart
index ad26488..0f00b48 100644
--- a/packages/flutter_tools/test/web.shard/expression_evaluation_web_test.dart
+++ b/packages/flutter_tools/test/web.shard/expression_evaluation_web_test.dart
@@ -33,7 +33,8 @@
// No need to start paused as all breakpoint would be eventually reached.
return _flutter.run(
withDebugger: true, chrome: true,
- expressionEvaluation: expressionEvaluation);
+ expressionEvaluation: expressionEvaluation,
+ additionalCommandArgs: <String>['--verbose']);
}
Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
@@ -139,7 +140,8 @@
return _flutter.run(
withDebugger: true, chrome: true,
expressionEvaluation: expressionEvaluation,
- startPaused: true, script: _project.testFilePath);
+ startPaused: true, script: _project.testFilePath,
+ additionalCommandArgs: <String>['--verbose']);
}
testWithoutContext('flutter test expression evaluation - error if expression evaluation disabled', () async {
diff --git a/packages/flutter_tools/test/web.shard/hot_reload_web_test.dart b/packages/flutter_tools/test/web.shard/hot_reload_web_test.dart
index a1af4ef..c319c74 100644
--- a/packages/flutter_tools/test/web.shard/hot_reload_web_test.dart
+++ b/packages/flutter_tools/test/web.shard/hot_reload_web_test.dart
@@ -24,11 +24,12 @@
tearDown(() async {
await flutter?.stop();
+ await flutter?.done;
tryToDelete(tempDir);
});
testWithoutContext('hot restart works without error', () async {
- await flutter.run(chrome: true);
+ await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
await flutter.hotRestart();
});
@@ -40,7 +41,7 @@
completer.complete();
}
});
- await flutter.run(chrome: true);
+ await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
project.uncommentHotReloadPrint();
try {
await flutter.hotRestart();
@@ -58,7 +59,8 @@
completer.complete();
}
});
- await flutter.run(chrome: true, additionalCommandArgs: <String>['--dart-define=FLUTTER_WEB_USE_SKIA=true']);
+ await flutter.run(chrome: true,
+ additionalCommandArgs: <String>['--dart-define=FLUTTER_WEB_USE_SKIA=true', '--verbose']);
project.uncommentHotReloadPrint();
try {
await flutter.hotRestart();
diff --git a/packages/flutter_tools/test/integration.shard/web_run_test.dart b/packages/flutter_tools/test/web.shard/web_run_test.dart
similarity index 76%
rename from packages/flutter_tools/test/integration.shard/web_run_test.dart
rename to packages/flutter_tools/test/web.shard/web_run_test.dart
index ae0a6b4..bfd7e01 100644
--- a/packages/flutter_tools/test/integration.shard/web_run_test.dart
+++ b/packages/flutter_tools/test/web.shard/web_run_test.dart
@@ -4,10 +4,11 @@
import 'package:file/file.dart';
import 'package:flutter_tools/src/base/file_system.dart';
+
+import '../integration.shard/test_data/basic_project.dart';
+import '../integration.shard/test_driver.dart';
+import '../integration.shard/test_utils.dart';
import '../src/common.dart';
-import 'test_data/basic_project.dart';
-import 'test_driver.dart';
-import 'test_utils.dart';
void main() {
Directory tempDir;
@@ -26,6 +27,6 @@
});
testWithoutContext('flutter run works on web devices with a unary main function', () async {
- await flutter.run(chrome: true);
+ await flutter.run(chrome: true, additionalCommandArgs: <String>['--verbose']);
}, skip: platform.isMacOS);
}
