Pipe through test-randomize-ordering-seed (#47243)
diff --git a/packages/flutter_tools/bin/fuchsia_tester.dart b/packages/flutter_tools/bin/fuchsia_tester.dart index a272e2d..04bf548 100644 --- a/packages/flutter_tools/bin/fuchsia_tester.dart +++ b/packages/flutter_tools/bin/fuchsia_tester.dart
@@ -22,6 +22,7 @@ import 'package:flutter_tools/src/reporting/reporting.dart'; import 'package:flutter_tools/src/test/coverage_collector.dart'; import 'package:flutter_tools/src/test/runner.dart'; +import 'package:flutter_tools/src/test/test_wrapper.dart'; // This was largely inspired by lib/src/commands/test.dart. @@ -142,6 +143,7 @@ } exitCode = await runTests( + const TestWrapper(), tests.keys.toList(), workDir: testDirectory, watcher: collector,
diff --git a/packages/flutter_tools/lib/src/commands/test.dart b/packages/flutter_tools/lib/src/commands/test.dart index 9c59093..bedf99a 100644 --- a/packages/flutter_tools/lib/src/commands/test.dart +++ b/packages/flutter_tools/lib/src/commands/test.dart
@@ -21,10 +21,14 @@ import '../test/coverage_collector.dart'; import '../test/event_printer.dart'; import '../test/runner.dart'; +import '../test/test_wrapper.dart'; import '../test/watcher.dart'; class TestCommand extends FastFlutterCommand { - TestCommand({ bool verboseHelp = false }) { + TestCommand({ + bool verboseHelp = false, + this.testWrapper = const TestWrapper(), + }) : assert(testWrapper != null) { requiresPubspecYaml(); usesPubOption(); argParser @@ -100,10 +104,20 @@ allowed: const <String>['tester', 'chrome'], defaultsTo: 'tester', help: 'The platform to run the unit tests on. Defaults to "tester".', + ) + ..addOption('test-randomize-ordering-seed', + defaultsTo: '0', + help: 'If positive, use this as a seed to randomize the execution of ' + 'test cases (must be a 32bit unsigned integer).\n' + 'If "random", pick a random seed to use.\n' + 'If 0 or not set, do not randomize test case execution order.', ); usesTrackWidgetCreation(verboseHelp: verboseHelp); } + /// The interface for starting and configuring the tester. + final TestWrapper testWrapper; + @override Future<Set<DevelopmentArtifact>> get requiredArtifacts async { final Set<DevelopmentArtifact> results = <DevelopmentArtifact>{}; @@ -223,6 +237,7 @@ boolArg('disable-service-auth-codes'); final int result = await runTests( + testWrapper, files, workDir: workDir, names: names, @@ -240,6 +255,7 @@ buildTestAssets: buildTestAssets, flutterProject: flutterProject, web: stringArg('platform') == 'chrome', + randomSeed: stringArg('test-randomize-ordering-seed'), ); if (collector != null) {
diff --git a/packages/flutter_tools/lib/src/test/flutter_platform.dart b/packages/flutter_tools/lib/src/test/flutter_platform.dart index 3952fcc..be1e878 100644 --- a/packages/flutter_tools/lib/src/test/flutter_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_platform.dart
@@ -7,10 +7,7 @@ import 'package:meta/meta.dart'; import 'package:stream_channel/stream_channel.dart'; -import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports @@ -27,6 +24,7 @@ import '../dart/package_map.dart'; import '../globals.dart'; import '../project.dart'; +import '../test/test_wrapper.dart'; import '../vmservice.dart'; import 'test_compiler.dart'; import 'test_config.dart'; @@ -71,6 +69,7 @@ /// (that is, one Dart file with a `*_test.dart` file name and a single `void /// main()`), you can set an observatory port explicitly. FlutterPlatform installHook({ + TestWrapper testWrapper = const TestWrapper(), @required String shellPath, TestWatcher watcher, bool enableObservatory = false, @@ -91,11 +90,12 @@ String icudtlPath, PlatformPluginRegistration platformPluginRegistration, }) { + assert(testWrapper != null); assert(enableObservatory || (!startPaused && observatoryPort == null)); // registerPlatformPlugin can be injected for testing since it's not very mock-friendly. platformPluginRegistration ??= (FlutterPlatform platform) { - hack.registerPlatformPlugin( + testWrapper.registerPlatformPlugin( <Runtime>[Runtime.vm], () { return platform;
diff --git a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart index 210164e..2323e0a 100644 --- a/packages/flutter_tools/lib/src/test/flutter_web_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_web_platform.dart
@@ -17,7 +17,6 @@ import 'package:shelf_static/shelf_static.dart'; import 'package:shelf_web_socket/shelf_web_socket.dart'; import 'package:stream_channel/stream_channel.dart'; -import 'package:test_api/backend.dart'; // ignore: deprecated_member_use import 'package:test_api/src/backend/runtime.dart'; import 'package:test_api/src/backend/suite_platform.dart'; import 'package:test_api/src/util/stack_trace_mapper.dart';
diff --git a/packages/flutter_tools/lib/src/test/runner.dart b/packages/flutter_tools/lib/src/test/runner.dart index 58c4e52..527584d 100644 --- a/packages/flutter_tools/lib/src/test/runner.dart +++ b/packages/flutter_tools/lib/src/test/runner.dart
@@ -5,9 +5,6 @@ import 'dart:async'; import 'package:meta/meta.dart'; -import 'package:test_api/backend.dart'; // ignore: deprecated_member_use -import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports -import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports import '../artifacts.dart'; import '../base/common.dart'; @@ -22,10 +19,12 @@ import '../web/compile.dart'; import 'flutter_platform.dart' as loader; import 'flutter_web_platform.dart'; +import 'test_wrapper.dart'; import 'watcher.dart'; /// Runs tests using package:test and the Flutter engine. Future<int> runTests( + TestWrapper testWrapper, List<String> testFiles, { Directory workDir, List<String> names = const <String>[], @@ -47,6 +46,7 @@ String icudtlPath, Directory coverageDirectory, bool web = false, + String randomSeed = '0', }) async { // Configure package:test to use the Flutter engine for child processes. final String shellPath = artifacts.getArtifactPath(Artifact.flutterTester); @@ -67,6 +67,7 @@ ...<String>['--name', name], for (String plainName in plainNames) ...<String>['--plain-name', plainName], + '--test-randomize-ordering-seed=$randomSeed', ]; if (web) { final String tempBuildDir = fs.systemTempDirectory @@ -89,7 +90,7 @@ ..add('--precompiled=$tempBuildDir') ..add('--') ..addAll(testFiles); - hack.registerPlatformPlugin( + testWrapper.registerPlatformPlugin( <Runtime>[Runtime.chrome], () { return FlutterWebPlatform.start( @@ -100,7 +101,7 @@ ); }, ); - await test.main(testArgs); + await testWrapper.main(testArgs); return exitCode; } @@ -112,6 +113,7 @@ ipv6 ? InternetAddressType.IPv6 : InternetAddressType.IPv4; final loader.FlutterPlatform platform = loader.installHook( + testWrapper: testWrapper, shellPath: shellPath, watcher: watcher, enableObservatory: enableObservatory, @@ -144,7 +146,7 @@ } printTrace('running test package with arguments: $testArgs'); - await test.main(testArgs); + await testWrapper.main(testArgs); // test.main() sets dart:io's exitCode global. printTrace('test package returned with exit code $exitCode');
diff --git a/packages/flutter_tools/lib/src/test/test_wrapper.dart b/packages/flutter_tools/lib/src/test/test_wrapper.dart new file mode 100644 index 0000000..fd180beb --- /dev/null +++ b/packages/flutter_tools/lib/src/test/test_wrapper.dart
@@ -0,0 +1,34 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test_api/backend.dart'; // ignore: deprecated_member_use +import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports +import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports +import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports + +export 'package:test_api/backend.dart' show Runtime; // ignore: deprecated_member_use +export 'package:test_core/src/runner/platform.dart' show PlatformPlugin; // ignore: implementation_imports + +abstract class TestWrapper { + const factory TestWrapper() = _DefaultTestWrapper; + + Future<void> main(List<String> args); + void registerPlatformPlugin(Iterable<Runtime> runtimes, FutureOr<PlatformPlugin> Function() platforms); +} + +class _DefaultTestWrapper implements TestWrapper { + const _DefaultTestWrapper(); + + @override + Future<void> main(List<String> args) async { + await test.main(args); + } + + @override + void registerPlatformPlugin(Iterable<Runtime> runtimes, FutureOr<PlatformPlugin> Function() platforms) { + hack.registerPlatformPlugin(runtimes, platforms); + } +}
diff --git a/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart new file mode 100644 index 0000000..30ccfff --- /dev/null +++ b/packages/flutter_tools/test/commands.shard/hermetic/test_test.dart
@@ -0,0 +1,66 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:args/command_runner.dart'; +import 'package:file/memory.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/cache.dart'; +import 'package:flutter_tools/src/commands/test.dart'; +import 'package:flutter_tools/src/test/test_wrapper.dart'; +import 'package:process/process.dart'; + +import '../../src/common.dart'; +import '../../src/context.dart'; +import '../../src/testbed.dart'; + +void main() { + Cache.disableLocking(); + MemoryFileSystem fs; + + setUp(() { + fs = MemoryFileSystem(); + fs.file('pubspec.yaml').createSync(); + fs.directory('test').childFile('some_test.dart').createSync(recursive: true); + }); + + testUsingContext('Pipes test-randomize-ordering-seed to package:test', + () async { + final FakePackageTest fakePackageTest = FakePackageTest(); + + final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); + final CommandRunner<void> commandRunner = + createTestCommandRunner(testCommand); + + await commandRunner.run(const <String>[ + 'test', + '--test-randomize-ordering-seed=random', + '--no-pub', + ]); + expect( + fakePackageTest.lastArgs, + contains('--test-randomize-ordering-seed=random'), + ); + }, overrides: <Type, Generator>{ + FileSystem: () => fs, + ProcessManager: () => FakeProcessManager.any(), + Cache: () => FakeCache(), + }); +} + +class FakePackageTest implements TestWrapper { + List<String> lastArgs; + + @override + Future<void> main(List<String> args) async { + lastArgs = args; + } + + @override + void registerPlatformPlugin( + Iterable<Runtime> runtimes, + FutureOr<PlatformPlugin> Function() platforms, + ) {} +}
diff --git a/packages/flutter_tools/test/general.shard/forbidden_imports_test.dart b/packages/flutter_tools/test/general.shard/forbidden_imports_test.dart index 047f0ec..bd74954 100644 --- a/packages/flutter_tools/test/general.shard/forbidden_imports_test.dart +++ b/packages/flutter_tools/test/general.shard/forbidden_imports_test.dart
@@ -66,7 +66,7 @@ fs.path.join(flutterTools, 'lib', 'src', 'build_runner', 'build_script.dart'), fs.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_platform.dart'), fs.path.join(flutterTools, 'lib', 'src', 'test', 'flutter_web_platform.dart'), - fs.path.join(flutterTools, 'lib', 'src', 'test', 'runner.dart'), + fs.path.join(flutterTools, 'lib', 'src', 'test', 'test_wrapper.dart'), ]; bool _isNotWhitelisted(FileSystemEntity entity) => whitelistedPaths.every((String path) => path != entity.path);
diff --git a/packages/flutter_tools/tool/tool_coverage.dart b/packages/flutter_tools/tool/tool_coverage.dart index caf67a0..0c368f6 100644 --- a/packages/flutter_tools/tool/tool_coverage.dart +++ b/packages/flutter_tools/tool/tool_coverage.dart
@@ -11,15 +11,12 @@ import 'package:coverage/coverage.dart'; import 'package:flutter_tools/src/base/common.dart'; import 'package:flutter_tools/src/context_runner.dart'; +import 'package:flutter_tools/src/test/test_wrapper.dart'; import 'package:path/path.dart' as path; import 'package:stream_channel/isolate_channel.dart'; import 'package:stream_channel/stream_channel.dart'; -import 'package:test_core/src/runner/hack_register_platform.dart' as hack; // ignore: implementation_imports -import 'package:test_core/src/executable.dart' as test; // ignore: implementation_imports import 'package:vm_service_client/vm_service_client.dart'; // ignore: deprecated_member_use -import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports -import 'package:test_core/src/runner/platform.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/runner_suite.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/suite.dart'; // ignore: implementation_imports import 'package:test_core/src/runner/plugin/platform_helpers.dart'; // ignore: implementation_imports @@ -35,7 +32,8 @@ Future<void> main(List<String> arguments) async { return runInContext(() async { final VMPlatform vmPlatform = VMPlatform(); - hack.registerPlatformPlugin( + const TestWrapper test = TestWrapper(); + test.registerPlatformPlugin( <Runtime>[Runtime.vm], () => vmPlatform, );