Revert "Clean up test infrastructure (#41880)" (#42982)
This reverts commit 1781d5c9bbb4a1b408e40bd40e433c3541bb68fc.
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index fea54fb..9914dd2 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -4,11 +4,11 @@
import 'dart:async';
import 'dart:io';
-import 'dart:math' as math;
import 'package:googleapis/bigquery/v2.dart' as bq;
import 'package:googleapis_auth/auth_io.dart' as auth;
import 'package:http/http.dart' as http;
+import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import 'flutter_compact_formatter.dart';
@@ -30,54 +30,21 @@
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
final String pubCache = path.join(flutterRoot, '.pub-cache');
final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
-
-/// The arguments to pass to `flutter test` (typically the local engine
-/// configuration) -- prefilled with the arguments passed to test.dart.
final List<String> flutterTestArgs = <String>[];
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
-/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
-///
-/// WARNING: if you change this number, also change .cirrus.yml
-/// and make sure it runs _all_ shards.
-const int kDeviceLabShardCount = 6;
-
-/// The number of Cirrus jobs that run Web tests in parallel.
-///
-/// WARNING: if you change this number, also change .cirrus.yml
-/// and make sure it runs _all_ shards.
-///
-/// The last shard also runs the Web plugin tests.
-const int kWebShardCount = 6;
-
-/// Maximum number of Web tests to run in a single `flutter test`. We found that
-/// large batches can get flaky, possibly because we reuse a single instance of
-/// the browser, and after many tests the browser's state gets corrupted.
-const int kWebBatchSize = 20;
-
-/// Tests that we don't run on Web for various reasons.
-//
-// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
-const List<String> kWebTestDirectoryBlacklist = <String>[
- 'cupertino',
- 'examples',
- 'material',
-];
-const List<String> kWebTestFileBlacklist = <String>[
- 'test/widgets/heroes_test.dart',
- 'test/widgets/text_test.dart',
- 'test/widgets/selectable_text_test.dart',
- 'test/widgets/color_filter_test.dart',
- 'test/widgets/editable_text_cursor_test.dart',
- 'test/widgets/shadow_test.dart',
- 'test/widgets/raw_keyboard_listener_test.dart',
- 'test/widgets/editable_text_test.dart',
- 'test/widgets/widget_inspector_test.dart',
- 'test/widgets/draggable_test.dart',
- 'test/widgets/shortcuts_test.dart',
-];
+const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
+ 'tests': _runTests,
+ 'web_tests': _runWebTests,
+ 'tool_tests': _runToolTests,
+ 'tool_coverage': _runToolCoverage,
+ 'build_tests': _runBuildTests,
+ 'coverage': _runCoverage,
+ 'integration_tests': _runIntegrationTests,
+ 'add2app_test': _runAdd2AppTest,
+};
/// When you call this, you can pass additional arguments to pass custom
/// arguments to flutter test. For example, you might want to call this
@@ -86,30 +53,31 @@
///
/// To run the tool_tests part, run it with SHARD=tool_tests
///
-/// Examples:
+/// For example:
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
Future<void> main(List<String> args) async {
flutterTestArgs.addAll(args);
- if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
- print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
- print('═' * 80);
- await _runSmokeTests();
- print('═' * 80);
- await selectShard(const <String, ShardRunner>{
- 'add_to_app_tests': _runAddToAppTests,
- 'build_tests': _runBuildTests,
- 'framework_coverage': _runFrameworkCoverage,
- 'framework_tests': _runFrameworkTests,
- 'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
- 'tool_coverage': _runToolCoverage,
- 'tool_tests': _runToolTests,
- 'web_tests': _runWebTests,
- });
+
+ final String shard = Platform.environment['SHARD'];
+ if (shard != null) {
+ if (!_kShards.containsKey(shard)) {
+ print('Invalid shard: $shard');
+ print('The available shards are: ${_kShards.keys.join(", ")}');
+ exit(1);
+ }
+ print('${bold}SHARD=$shard$reset');
+ await _kShards[shard]();
+ } else {
+ for (String currentShard in _kShards.keys) {
+ print('${bold}SHARD=$currentShard$reset');
+ await _kShards[currentShard]();
+ print('');
+ }
+ }
}
Future<void> _runSmokeTests() async {
- print('${green}Running smoketests...$reset');
// Verify that the tests actually return failure on failure and success on
// success.
final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
@@ -140,11 +108,10 @@
script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
expectFailure: true,
printOutput: false,
- outputChecker: (CapturedOutput output) {
- return output.stdout.contains('failingPendingTimerTest')
- ? null
- : 'Failed to find the stack trace for the pending Timer.';
- }
+ outputChecker: (CapturedOutput output) =>
+ output.stdout.contains('failingPendingTimerTest')
+ ? null
+ : 'Failed to find the stack trace for the pending Timer.',
);
// We run the remaining smoketests in parallel, because they each take some
// time to run (e.g. compiling), so we don't want to run them in series,
@@ -186,11 +153,8 @@
);
// Verify that we correctly generated the version file.
- final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
- if (versionError != null) {
- print(redLine);
- print(versionError);
- print(redLine);
+ final bool validVersion = await verifyVersion(path.join(flutterRoot, 'version'));
+ if (!validVersion) {
exit(1);
}
}
@@ -215,7 +179,7 @@
final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes);
return bq.BigqueryApi(client);
} catch (e) {
- print('${red}Failed to get BigQuery API client.$reset');
+ print('Failed to get BigQuery API client.');
print(e);
return null;
}
@@ -239,6 +203,7 @@
Future<void> _runToolTests() async {
final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
+ await _runSmokeTests();
const String kDotShard = '.shard';
const String kTest = 'test';
@@ -267,10 +232,11 @@
await selectSubshard(subshards);
}
-/// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps
-/// without crashing. It does not actually launch the apps. That happens later
-/// in the devicelab. This is just a smoke-test. In particular, this will verify
-/// we can build when there are spaces in the path name for the Flutter SDK and
+/// Verifies that AOT, APK, and IPA (if on macOS) builds the
+/// examples apps without crashing. It does not actually
+/// launch the apps. That happens later in the devicelab. This is
+/// just a smoke-test. In particular, this will verify we can build
+/// when there are spaces in the path name for the Flutter SDK and
/// target app.
Future<void> _runBuildTests() async {
final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list();
@@ -279,11 +245,10 @@
continue;
}
final String examplePath = fileEntity.path;
+
await _flutterBuildAot(examplePath);
await _flutterBuildApk(examplePath);
- if (Platform.isMacOS) {
- await _flutterBuildIpa(examplePath);
- }
+ await _flutterBuildIpa(examplePath);
}
// Web compilation tests.
await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart'));
@@ -291,44 +256,12 @@
await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'),
path.join('lib', 'dart_io_import.dart'),
);
-}
-Future<void> _flutterBuildAot(String relativePathToApplication) async {
- print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...');
- await runCommand(flutter,
- <String>['build', 'aot', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- );
-}
-
-Future<void> _flutterBuildApk(String relativePathToApplication) async {
- print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...');
- await runCommand(flutter,
- <String>['build', 'apk', '--debug', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- );
-}
-
-Future<void> _flutterBuildIpa(String relativePathToApplication) async {
- assert(Platform.isMacOS);
- print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
- // Install Cocoapods. We don't have these checked in for the examples,
- // and build ios doesn't take care of it automatically.
- final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
- if (podfile.existsSync()) {
- await runCommand('pod',
- <String>['install'],
- workingDirectory: podfile.parent.path,
- );
- }
- await runCommand(flutter,
- <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- );
+ print('${bold}DONE: All build tests successful.$reset');
}
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
- print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
+ print('Running Dart2JS build tests...');
await runCommand(flutter,
<String>['build', 'web', '-v', '--target=$target'],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
@@ -337,101 +270,211 @@
'FLUTTER_WEB': 'true',
},
);
+ print('Done.');
}
-Future<void> _runAddToAppTests() async {
- if (Platform.isMacOS) {
- print('${green}Running add-to-app iOS integration tests$reset...');
- final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
- await runCommand('./build_and_test.sh',
- <String>[],
- workingDirectory: addToAppDir,
+Future<void> _flutterBuildAot(String relativePathToApplication) async {
+ print('Running AOT build tests...');
+ await runCommand(flutter,
+ <String>['build', 'aot', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ expectNonZeroExit: false,
+ );
+ print('Done.');
+}
+
+Future<void> _flutterBuildApk(String relativePathToApplication) async {
+ if (
+ (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) &&
+ (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) {
+ return;
+ }
+ print('Running APK build tests...');
+ await runCommand(flutter,
+ <String>['build', 'apk', '--debug', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ expectNonZeroExit: false,
+ );
+ print('Done.');
+}
+
+Future<void> _flutterBuildIpa(String relativePathToApplication) async {
+ if (!Platform.isMacOS) {
+ return;
+ }
+ print('Running IPA build tests...');
+ // Install Cocoapods. We don't have these checked in for the examples,
+ // and build ios doesn't take care of it automatically.
+ final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
+ if (podfile.existsSync()) {
+ await runCommand('pod',
+ <String>['install'],
+ workingDirectory: podfile.parent.path,
+ expectNonZeroExit: false,
);
}
+ await runCommand(flutter,
+ <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ expectNonZeroExit: false,
+ );
+ print('Done.');
}
-Future<void> _runFrameworkTests() async {
+Future<void> _runAdd2AppTest() async {
+ if (!Platform.isMacOS) {
+ return;
+ }
+ print('Running Add2App iOS integration tests...');
+ final String add2AppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
+ await runCommand('./build_and_test.sh',
+ <String>[],
+ workingDirectory: add2AppDir,
+ expectNonZeroExit: false,
+ );
+ print('Done.');
+}
+
+Future<void> _runTests() async {
final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
+ await _runSmokeTests();
+ final String subShard = Platform.environment['SUBSHARD'];
Future<void> runWidgets() async {
- print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
+ await _runFlutterTest(
+ path.join(flutterRoot, 'packages', 'flutter'),
+ tableData: bigqueryApi?.tabledata,
+ tests: <String>[
+ path.join('test', 'widgets') + path.separator,
+ ],
+ );
+ // Only packages/flutter/test/widgets/widget_inspector_test.dart really
+ // needs to be run with --track-widget-creation but it is nice to run
+ // all of the tests in package:flutter with the flag to ensure that
+ // the Dart kernel transformer triggered by the flag does not break anything.
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--track-widget-creation'],
tableData: bigqueryApi?.tabledata,
- tests: <String>[ path.join('test', 'widgets') + path.separator ],
+ tests: <String>[
+ path.join('test', 'widgets') + path.separator,
+ ],
);
- await _runFlutterTest(
- path.join(flutterRoot, 'packages', 'flutter'),
- options: <String>['--no-track-widget-creation'],
- tableData: bigqueryApi?.tabledata,
- tests: <String>[ path.join('test', 'widgets') + path.separator ],
- );
- // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata);
}
- Future<void> runLibraries() async {
+ Future<void> runFrameworkOthers() async {
final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
.listSync(followLinks: false, recursive: false)
.whereType<Directory>()
.where((Directory dir) => dir.path.endsWith('widgets') == false)
- .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
+ .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
.toList();
- print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
+
+ print('Running tests for: ${tests.join(';')}');
+
+ await _runFlutterTest(
+ path.join(flutterRoot, 'packages', 'flutter'),
+ tableData: bigqueryApi?.tabledata,
+ tests: tests,
+ );
+ // Only packages/flutter/test/widgets/widget_inspector_test.dart really
+ // needs to be run with --track-widget-creation but it is nice to run
+ // all of the tests in package:flutter with the flag to ensure that
+ // the Dart kernel transformer triggered by the flag does not break anything.
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--track-widget-creation'],
tableData: bigqueryApi?.tabledata,
tests: tests,
);
- await _runFlutterTest(
- path.join(flutterRoot, 'packages', 'flutter'),
- options: <String>['--no-track-widget-creation'],
- tableData: bigqueryApi?.tabledata,
- tests: tests,
- );
}
- Future<void> runMisc() async {
- print('${green}Running package tests$reset for directories other than packages/flutter');
+ Future<void> runExtras() async {
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata);
await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata);
await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(
- path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'),
- tableData: bigqueryApi?.tabledata,
- environment: <String, String>{
- 'FLUTTER_EXPERIMENTAL_BUILD': 'true',
- },
- );
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata);
+ // Regression test to ensure that code outside of package:flutter can run
+ // with --track-widget-creation.
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
+ // Smoke test for code generation.
+ await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: <String, String>{
+ 'FLUTTER_EXPERIMENTAL_BUILD': 'true',
+ });
+ }
+ switch (subShard) {
+ case 'widgets':
+ await runWidgets();
+ break;
+ case 'framework_other':
+ await runFrameworkOthers();
+ break;
+ case 'extras':
+ runExtras();
+ break;
+ default:
+ print('Unknown sub-shard $subShard, running all tests!');
+ await runWidgets();
+ await runFrameworkOthers();
+ await runExtras();
+
}
- await selectSubshard(<String, ShardRunner>{
- 'widgets': runWidgets,
- 'libraries': runLibraries,
- 'misc': runMisc,
- });
+ print('${bold}DONE: All tests successful.$reset');
}
-Future<void> _runFrameworkCoverage() async {
+// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
+const List<String> kWebTestDirectoryBlacklist = <String>[
+ 'test/cupertino',
+ 'test/examples',
+ 'test/material',
+];
+const List<String> kWebTestFileBlacklist = <String>[
+ 'test/widgets/heroes_test.dart',
+ 'test/widgets/text_test.dart',
+ 'test/widgets/selectable_text_test.dart',
+ 'test/widgets/color_filter_test.dart',
+ 'test/widgets/editable_text_cursor_test.dart',
+ 'test/widgets/shadow_test.dart',
+ 'test/widgets/raw_keyboard_listener_test.dart',
+ 'test/widgets/editable_text_test.dart',
+ 'test/widgets/widget_inspector_test.dart',
+ 'test/widgets/draggable_test.dart',
+ 'test/widgets/shortcuts_test.dart',
+];
+
+Future<void> _runWebTests() async {
+ final Directory flutterPackageDir = Directory(path.join(flutterRoot, 'packages', 'flutter'));
+ final Directory testDir = Directory(path.join(flutterPackageDir.path, 'test'));
+
+ final List<String> directories = testDir
+ .listSync()
+ .whereType<Directory>()
+ .map<String>((Directory dir) => path.relative(dir.path, from: flutterPackageDir.path))
+ .where((String relativePath) => !kWebTestDirectoryBlacklist.contains(relativePath))
+ .toList();
+
+ await _runFlutterWebTest(flutterPackageDir.path, tests: directories);
+ await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter_web_plugins'), tests: <String>['test']);
+}
+
+Future<void> _runCoverage() async {
final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
if (!coverageFile.existsSync()) {
print('${red}Coverage file not found.$reset');
- print('Expected to find: $cyan${coverageFile.absolute}$reset');
- print('This file is normally obtained by running `${green}flutter update-packages$reset`.');
+ print('Expected to find: ${coverageFile.absolute}');
+ print('This file is normally obtained by running `flutter update-packages`.');
exit(1);
}
coverageFile.deleteSync();
@@ -440,98 +483,11 @@
);
if (!coverageFile.existsSync()) {
print('${red}Coverage file not found.$reset');
- print('Expected to find: $cyan${coverageFile.absolute}$reset');
- print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.');
+ print('Expected to find: ${coverageFile.absolute}');
+ print('This file should have been generated by the `flutter test --coverage` script, but was not.');
exit(1);
}
-}
-
-Future<void> _runWebTests() async {
- final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
-
- final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
- final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
-
- final List<String> allTests = flutterPackageTestDirectory
- .listSync()
- .whereType<Directory>()
- .where((Directory directory) => !kWebTestDirectoryBlacklist.contains(path.basename(directory.path)))
- .expand((Directory directory) => directory
- .listSync(recursive: true)
- .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
- )
- .whereType<File>()
- .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
- .where((String filePath) => !kWebTestFileBlacklist.contains(filePath))
- .toList()
- // Finally we shuffle the list because we want the average cost per file to be uniformly
- // distributed. If the list is not sorted then different shards and batches may have
- // very different characteristics.
- // We use a constant seed for repeatability.
- ..shuffle(math.Random(0));
-
- assert(kWebShardCount >= 1);
- final int testsPerShard = (allTests.length / kWebShardCount).ceil();
- assert(testsPerShard * kWebShardCount >= allTests.length);
-
- // This for loop computes all but the last shard.
- for (int index = 0; index < kWebShardCount - 1; index += 1) {
- subshards['$index'] = () => _runFlutterWebTest(
- flutterPackageDirectory.path,
- allTests.sublist(
- index * testsPerShard,
- (index + 1) * testsPerShard,
- ),
- );
- }
-
- // The last shard also runs the flutter_web_plugins tests.
- //
- // We make sure the last shard ends in _last so it's easier to catch mismatches
- // between `.cirrus.yml` and `test.dart`.
- subshards['${kWebShardCount - 1}_last'] = () async {
- await _runFlutterWebTest(
- flutterPackageDirectory.path,
- allTests.sublist(
- (kWebShardCount - 1) * testsPerShard,
- allTests.length,
- ),
- );
- await _runFlutterWebTest(
- path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
- <String>['test'],
- );
- };
-
- await selectSubshard(subshards);
-}
-
-Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async {
- final List<String> batch = <String>[];
- for (int i = 0; i < tests.length; i += 1) {
- final String testFilePath = tests[i];
- batch.add(testFilePath);
- if (batch.length == kWebBatchSize || i == tests.length - 1) {
- await runCommand(
- flutter,
- <String>[
- 'test',
- if (ciProvider == CiProviders.cirrus)
- '--concurrency=1', // do not parallelize on Cirrus, to reduce flakiness
- '-v',
- '--platform=chrome',
- ...?flutterTestArgs,
- ...batch,
- ],
- workingDirectory: workingDirectory,
- environment: <String, String>{
- 'FLUTTER_WEB': 'true',
- 'FLUTTER_LOW_RESOURCE_MODE': 'true',
- },
- );
- batch.clear();
- }
- }
+ print('${bold}DONE: Coverage collection successful.$reset');
}
Future<void> _pubRunTest(String workingDirectory, {
@@ -540,33 +496,14 @@
bool useBuildRunner = false,
bq.TabledataResourceApi tableData,
}) async {
- final List<String> args = <String>['run'];
+ final List<String> args = <String>['run', '--verbose'];
if (useBuildRunner) {
- final String posixTestPath = path.posix.joinAll(path.split(testPath));
- args.addAll(<String>[
- 'build_runner',
- 'test',
- '--build-filter=$posixTestPath/*.dill',
- '--build-filter=$posixTestPath/**/*.dill',
- '--',
- ]);
+ args.addAll(<String>['build_runner', 'test', '--']);
} else {
args.add('test');
}
args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
- int cpus;
- final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
- if (cpuVariable != null) {
- cpus = int.tryParse(cpuVariable, radix: 10);
- if (cpus == null) {
- print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset');
- print('Actual value: "$cpuVariable"');
- exit(1);
- }
- } else {
- cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
- }
- args.add('-j$cpus');
+ args.add('-j1'); // TODO(ianh): Scale based on CPUs.
if (!hasColor)
args.add('--no-color');
if (testPath != null)
@@ -610,6 +547,238 @@
}
}
+void deleteFile(String path) {
+ // There's a race condition here but in theory we're not racing anyone
+ // while this script runs, so should be ok.
+ final File file = File(path);
+ if (file.existsSync())
+ file.deleteSync();
+}
+
+enum CiProviders {
+ cirrus,
+ luci,
+}
+
+CiProviders _getCiProvider() {
+ if (Platform.environment['CIRRUS_CI'] == 'true') {
+ return CiProviders.cirrus;
+ }
+ if (Platform.environment['LUCI_CONTEXT'] != null) {
+ return CiProviders.luci;
+ }
+ return null;
+}
+
+String _getCiProviderName() {
+ switch(_getCiProvider()) {
+ case CiProviders.cirrus:
+ return 'cirrusci';
+ case CiProviders.luci:
+ return 'luci';
+ }
+ return 'unknown';
+}
+
+int _getPrNumber() {
+ switch(_getCiProvider()) {
+ case CiProviders.cirrus:
+ return Platform.environment['CIRRUS_PR'] == null
+ ? -1
+ : int.tryParse(Platform.environment['CIRRUS_PR']);
+ case CiProviders.luci:
+ return -1; // LUCI doesn't know about this.
+ }
+ return -1;
+}
+
+Future<String> _getAuthors() async {
+ final String exe = Platform.isWindows ? '.exe' : '';
+ final String author = await runAndGetStdout(
+ 'git$exe', <String>['-c', 'log.showSignature=false', 'log', _getGitHash(), '--pretty="%an <%ae>"'],
+ workingDirectory: flutterRoot,
+ ).first;
+ return author;
+}
+
+String _getCiUrl() {
+ switch(_getCiProvider()) {
+ case CiProviders.cirrus:
+ return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
+ case CiProviders.luci:
+ return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
+ }
+ return '';
+}
+
+String _getGitHash() {
+ switch(_getCiProvider()) {
+ case CiProviders.cirrus:
+ return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
+ case CiProviders.luci:
+ return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
+ }
+ return '';
+}
+
+Future<void> _processTestOutput(
+ FlutterCompactFormatter formatter,
+ Stream<String> testOutput,
+ bq.TabledataResourceApi tableData,
+) async {
+ final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
+ print('Processing...');
+ });
+
+ await testOutput.forEach(formatter.processRawOutput);
+ heartbeat.cancel();
+ formatter.finish();
+ if (tableData == null || formatter.tests.isEmpty) {
+ return;
+ }
+ final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
+ final String authors = await _getAuthors();
+ request.rows = List<bq.TableDataInsertAllRequestRows>.from(
+ formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
+ bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
+ 'json': <String, dynamic>{
+ 'source': <String, dynamic>{
+ 'provider': _getCiProviderName(),
+ 'url': _getCiUrl(),
+ 'platform': <String, dynamic>{
+ 'os': Platform.operatingSystem,
+ 'version': Platform.operatingSystemVersion,
+ },
+ },
+ 'test': <String, dynamic>{
+ 'name': result.name,
+ 'result': result.status.toString(),
+ 'file': result.path,
+ 'line': result.line,
+ 'column': result.column,
+ 'time': result.totalTime,
+ },
+ 'git': <String, dynamic>{
+ 'author': authors,
+ 'pull_request': _getPrNumber(),
+ 'commit': _getGitHash(),
+ 'organization': 'flutter',
+ 'repository': 'flutter',
+ },
+ 'error': result.status != TestStatus.failed ? null : <String, dynamic>{
+ 'message': result.errorMessage,
+ 'stack_trace': result.stackTrace,
+ },
+ 'information': result.messages,
+ },
+ }),
+ ),
+ growable: false,
+ );
+ final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
+ if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
+ print('${red}BigQuery insert errors:');
+ print(response.toJson());
+ print(reset);
+ }
+}
+
+class EvalResult {
+ EvalResult({
+ this.stdout,
+ this.stderr,
+ this.exitCode = 0,
+ });
+
+ final String stdout;
+ final String stderr;
+ final int exitCode;
+}
+
+/// The number of Cirrus jobs that run web tests in parallel.
+///
+/// WARNING: if you change this number, also change .cirrus.yml
+/// and make sure it runs _all_ shards.
+const int _kWebShardCount = 6;
+
+Future<void> _runFlutterWebTest(String workingDirectory, {
+ List<String> tests,
+}) async {
+ List<String> allTests = <String>[];
+ for (String testDirPath in tests) {
+ final Directory testDir = Directory(path.join(workingDirectory, testDirPath));
+ allTests.addAll(
+ testDir.listSync(recursive: true)
+ .whereType<File>()
+ .where((File file) => file.path.endsWith('_test.dart'))
+ .map<String>((File file) => path.relative(file.path, from: workingDirectory))
+ .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)),
+ );
+ }
+
+ // If a shard is specified only run tests in that shard.
+ final int webShard = int.tryParse(Platform.environment['WEB_SHARD'] ?? 'n/a');
+ if (webShard != null) {
+ if (webShard >= _kWebShardCount) {
+ throw 'WEB_SHARD must be <= _kWebShardCount, but was $webShard';
+ }
+ final List<String> shard = <String>[];
+ for (int i = webShard; i < allTests.length; i += _kWebShardCount) {
+ shard.add(allTests[i]);
+ }
+ allTests = shard;
+ }
+
+ print(allTests.join('\n'));
+ print('${allTests.length} tests total');
+
+ // Maximum number of tests to run in a single `flutter test`. We found that
+ // large batches can get flaky, possibly because we reuse a single instance
+ // of the browser, and after many tests the browser's state gets corrupted.
+ const int kBatchSize = 20;
+ List<String> batch = <String>[];
+ for (int i = 0; i < allTests.length; i += 1) {
+ final String testFilePath = allTests[i];
+ batch.add(testFilePath);
+ if (batch.length == kBatchSize || i == allTests.length - 1) {
+ await _runFlutterWebTestBatch(workingDirectory, batch: batch);
+ batch = <String>[];
+ }
+ }
+}
+
+Future<void> _runFlutterWebTestBatch(String workingDirectory, {
+ List<String> batch,
+}) async {
+ final List<String> args = <String>[
+ 'test',
+ if (_getCiProvider() == CiProviders.cirrus)
+ '--concurrency=1', // do not parallelize on Cirrus to reduce flakiness
+ '-v',
+ '--platform=chrome',
+ ...?flutterTestArgs,
+ ...batch,
+ ];
+
+ // TODO(jonahwilliams): fix relative path issues to make this unecessary.
+ final Directory oldCurrent = Directory.current;
+ Directory.current = Directory(path.join(flutterRoot, 'packages', 'flutter'));
+ try {
+ await runCommand(
+ flutter,
+ args,
+ workingDirectory: workingDirectory,
+ expectFlaky: false,
+ environment: <String, String>{
+ 'FLUTTER_WEB': 'true',
+ 'FLUTTER_LOW_RESOURCE_MODE': 'true',
+ },
+ );
+ } finally {
+ Directory.current = oldCurrent;
+ }
+}
+
Future<void> _runFlutterTest(String workingDirectory, {
String script,
bool expectFailure = false,
@@ -621,7 +790,8 @@
Map<String, String> environment,
List<String> tests = const <String>[],
}) async {
- assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
+ assert(!printOutput || outputChecker == null,
+ 'Output either can be printed or checked but not both');
final List<String> args = <String>[
'test',
@@ -630,15 +800,16 @@
];
final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
- if (shouldProcessOutput)
+ if (shouldProcessOutput) {
args.add('--machine');
+ }
if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) {
- print('${red}Could not find test$reset: $green$fullScriptPath$reset');
- print('Working directory: $cyan$workingDirectory$reset');
- print('Script: $green$script$reset');
+ print('Could not find test: $fullScriptPath');
+ print('Working directory: $workingDirectory');
+ print('Script: $script');
if (!printOutput)
print('This is one of the tests that does not normally print output.');
if (skip)
@@ -705,290 +876,110 @@
}
}
-Map<String, String> _initGradleEnvironment() {
- final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
- ? Platform.environment['ANDROID_SDK_ROOT']
- : Platform.environment['ANDROID_HOME'];
- if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
- print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset');
- exit(1);
+// the optional `file` argument is an override for testing
+@visibleForTesting
+Future<bool> verifyVersion(String filename, [File file]) async {
+ final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
+ file ??= File(filename);
+ final String version = await file.readAsString();
+ if (!file.existsSync()) {
+ print('$redLine');
+ print('The version logic failed to create the Flutter version file.');
+ print('$redLine');
+ return false;
}
- return <String, String>{
- 'ANDROID_HOME': androidSdkRoot,
- 'ANDROID_SDK_ROOT': androidSdkRoot,
- };
+ if (version == '0.0.0-unknown') {
+ print('$redLine');
+ print('The version logic failed to determine the Flutter version.');
+ print('$redLine');
+ return false;
+ }
+ if (!version.contains(pattern)) {
+ print('$redLine');
+ print('The version logic generated an invalid version string: "$version".');
+ print('$redLine');
+ return false;
+ }
+ return true;
}
-final Map<String, String> gradleEnvironment = _initGradleEnvironment();
+Future<void> _runIntegrationTests() async {
+ final String subShard = Platform.environment['SUBSHARD'];
-Future<void> _runHostOnlyDeviceLabTests() async {
- if (Platform.isWindows) {
- // TODO(ianh): remove when https://github.com/flutter/flutter/issues/36311 fixed by https://github.com/flutter/flutter/pull/42709
- return;
+ switch (subShard) {
+ case 'gradle1':
+ case 'gradle2':
+ // This runs some gradle integration tests if the subshard is Android.
+ await _androidGradleTests(subShard);
+ break;
+ default:
+ await _runDevicelabTest('dartdocs');
+
+ if (Platform.isLinux) {
+ await _runDevicelabTest('flutter_create_offline_test_linux');
+ } else if (Platform.isWindows) {
+ await _runDevicelabTest('flutter_create_offline_test_windows');
+ } else if (Platform.isMacOS) {
+ await _runDevicelabTest('flutter_create_offline_test_mac');
+ await _runDevicelabTest('plugin_lint_mac');
+// TODO(jmagman): Re-enable once flakiness is resolved.
+// await _runDevicelabTest('module_test_ios');
+ }
}
-
- // Please don't add more tests here. We should not be using the devicelab
- // logic to run tests outside devicelab, that's just confusing.
- // Instead, create tests that are not devicelab tests, and run those.
-
- // TODO(ianh): Move the tests that are not running on devicelab any more out
- // of the device lab directory.
-
- // List the tests to run.
- // We split these into subshards. The tests are randomly distributed into
- // those subshards so as to get a uniform distribution of costs, but the
- // seed is fixed so that issues are reproducible.
- final List<ShardRunner> tests = <ShardRunner>[
- // Keep this in alphabetical order.
- () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: false),
- if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'),
- if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'),
- if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'),
- // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494
- if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: false),
- if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true),
- () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
- // TODO(jmagman): Re-enable once flakiness is resolved, https://github.com/flutter/flutter/issues/37525
- // if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'),
- if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'),
- () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
- () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
- ]..shuffle(math.Random(0));
-
- final int testsPerShard = tests.length ~/ kDeviceLabShardCount;
- final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
-
- for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) {
- String last = '';
- List<ShardRunner> sublist;
- if (subshard < kDeviceLabShardCount - 1) {
- sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
- } else {
- sublist = tests.sublist(subshard * testsPerShard, tests.length);
- // We make sure the last shard ends in _last so it's easier to catch mismatches
- // between `.cirrus.yml` and `test.dart`.
- last = '_last';
- }
- subshards['$subshard$last'] = () async {
- for (ShardRunner test in sublist)
- await test();
- };
- }
-
- await selectSubshard(subshards);
}
-Future<void> _runDevicelabTest(String testName, {
- Map<String, String> environment,
- bool testEmbeddingV2 = false,
-}) async {
+Future<void> _runDevicelabTest(String testName, {Map<String, String> env}) async {
await runCommand(
dart,
<String>['bin/run.dart', '-t', testName],
workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
- environment: <String, String>{
- ...?environment,
- if (testEmbeddingV2)
- 'ENABLE_ANDROID_EMBEDDING_V2': 'true',
- },
+ environment: env,
);
}
-void deleteFile(String path) {
- // This is technically a race condition but nobody else should be running
- // while this script runs, so we should be ok. (Sadly recursive:true does not
- // obviate the need for existsSync, at least on Windows.)
- final File file = File(path);
- if (file.existsSync())
- file.deleteSync();
+String get androidSdkRoot {
+ final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
+ ? Platform.environment['ANDROID_SDK_ROOT']
+ : Platform.environment['ANDROID_HOME'];
+ if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
+ return null;
+ }
+ return androidSdkRoot;
}
-enum CiProviders {
- cirrus,
- luci,
-}
-
-Future<void> _processTestOutput(
- FlutterCompactFormatter formatter,
- Stream<String> testOutput,
- bq.TabledataResourceApi tableData,
-) async {
- final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
- print('Processing...');
- });
-
- await testOutput.forEach(formatter.processRawOutput);
- heartbeat.cancel();
- formatter.finish();
- if (tableData == null || formatter.tests.isEmpty) {
+Future<void> _androidGradleTests(String subShard) async {
+ // TODO(dnfield): gradlew is crashing on the cirrus image and it's not clear why.
+ if (androidSdkRoot == null || Platform.isWindows) {
+ print('No Android SDK detected or on Windows, skipping Android gradle test.');
return;
}
- final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
- final String authors = await _getAuthors();
- request.rows = List<bq.TableDataInsertAllRequestRows>.from(
- formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
- bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
- 'json': <String, dynamic>{
- 'source': <String, dynamic>{
- 'provider': ciProviderName,
- 'url': ciUrl,
- 'platform': <String, dynamic>{
- 'os': Platform.operatingSystem,
- 'version': Platform.operatingSystemVersion,
- },
- },
- 'test': <String, dynamic>{
- 'name': result.name,
- 'result': result.status.toString(),
- 'file': result.path,
- 'line': result.line,
- 'column': result.column,
- 'time': result.totalTime,
- },
- 'git': <String, dynamic>{
- 'author': authors,
- 'pull_request': prNumber,
- 'commit': gitHash,
- 'organization': 'flutter',
- 'repository': 'flutter',
- },
- 'error': result.status != TestStatus.failed ? null : <String, dynamic>{
- 'message': result.errorMessage,
- 'stack_trace': result.stackTrace,
- },
- 'information': result.messages,
- },
- }),
- ),
- growable: false,
- );
- final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
- if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
- print('${red}BigQuery insert errors:');
- print(response.toJson());
- print(reset);
+ final Map<String, String> defaultEnv = <String, String>{
+ 'ANDROID_HOME': androidSdkRoot,
+ 'ANDROID_SDK_ROOT': androidSdkRoot,
+ 'ENABLE_ANDROID_EMBEDDING_V2': Platform.environment['ENABLE_ANDROID_EMBEDDING_V2'] ?? '',
+ };
+ if (subShard == 'gradle1') {
+ await _runDevicelabTest('gradle_plugin_light_apk_test', env: defaultEnv);
+ await _runDevicelabTest('gradle_plugin_fat_apk_test', env: defaultEnv);
+ await _runDevicelabTest('gradle_r8_test', env: defaultEnv);
+ await _runDevicelabTest('gradle_non_android_plugin_test', env: defaultEnv);
+ await _runDevicelabTest('gradle_jetifier_test', env: defaultEnv);
+ }
+ if (subShard == 'gradle2') {
+ await _runDevicelabTest('gradle_plugin_bundle_test', env: defaultEnv);
+ await _runDevicelabTest('module_test', env: defaultEnv);
+ await _runDevicelabTest('module_host_with_custom_build_test', env: defaultEnv);
+ await _runDevicelabTest('build_aar_module_test', env: defaultEnv);
+ await _runDevicelabTest('plugin_test', env: defaultEnv);
}
}
-CiProviders get ciProvider {
- if (Platform.environment['CIRRUS_CI'] == 'true') {
- return CiProviders.cirrus;
- }
- if (Platform.environment['LUCI_CONTEXT'] != null) {
- return CiProviders.luci;
- }
- return null;
-}
+Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard');
+Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard');
-String get ciProviderName {
- switch (ciProvider) {
- case CiProviders.cirrus:
- return 'cirrusci';
- case CiProviders.luci:
- return 'luci';
- }
- return 'unknown';
-}
-
-int get prNumber {
- switch (ciProvider) {
- case CiProviders.cirrus:
- return Platform.environment['CIRRUS_PR'] == null
- ? -1
- : int.tryParse(Platform.environment['CIRRUS_PR']);
- case CiProviders.luci:
- return -1; // LUCI doesn't know about this.
- }
- return -1;
-}
-
-Future<String> _getAuthors() async {
- final String exe = Platform.isWindows ? '.exe' : '';
- final String author = await runAndGetStdout(
- 'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'],
- workingDirectory: flutterRoot,
- ).first;
- return author;
-}
-
-String get ciUrl {
- switch (ciProvider) {
- case CiProviders.cirrus:
- return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
- case CiProviders.luci:
- return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
- }
- return '';
-}
-
-String get gitHash {
- switch(ciProvider) {
- case CiProviders.cirrus:
- return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
- case CiProviders.luci:
- return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
- }
- return '';
-}
-
-/// Checks the given file's contents to determine if they match the allowed
-/// pattern for version strings.
-///
-/// Returns null if the contents are good. Returns a string if they are bad.
-/// The string is an error message.
-Future<String> verifyVersion(File file) async {
- final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
- final String version = await file.readAsString();
- if (!file.existsSync())
- return 'The version logic failed to create the Flutter version file.';
- if (version == '0.0.0-unknown')
- return 'The version logic failed to determine the Flutter version.';
- if (!version.contains(pattern))
- return 'The version logic generated an invalid version string: "$version".';
- return null;
-}
-
-/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
-/// the shard and subshard (parsing it in the form shard-subshard-platform, ignoring
-/// the platform).
-///
-/// However, for local testing you can just set the SHARD and SUBSHARD
-/// environment variables. For example, to run all the framework tests you can
-/// just set SHARD=framework_tests. To run specifically the third subshard of
-/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
-Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0);
-Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1);
-
-const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
-
-Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
- String item = Platform.environment[key];
- if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
- final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-');
- assert(positionInTaskName < parts.length);
- item = parts[positionInTaskName];
- }
- if (item == null) {
- for (String currentItem in items.keys) {
- print('$bold$key=$currentItem$reset');
- await items[currentItem]();
- print('');
- }
- } else {
+Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async {
+ final String item = Platform.environment[key];
+ if (item != null) {
if (!items.containsKey(item)) {
print('${red}Invalid $name: $item$reset');
print('The available ${name}s are: ${items.keys.join(", ")}');
@@ -996,5 +987,11 @@
}
print('$bold$key=$item$reset');
await items[item]();
+ } else {
+ for (String currentItem in items.keys) {
+ print('$bold$key=$currentItem$reset');
+ await items[currentItem]();
+ print('');
+ }
}
}
