Clean up test infrastructure (#43030)
See #41880 for history.
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index 9914dd2..fea54fb 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -4,11 +4,11 @@
import 'dart:async';
import 'dart:io';
+import 'dart:math' as math;
import 'package:googleapis/bigquery/v2.dart' as bq;
import 'package:googleapis_auth/auth_io.dart' as auth;
import 'package:http/http.dart' as http;
-import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import 'flutter_compact_formatter.dart';
@@ -30,21 +30,54 @@
final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
final String pubCache = path.join(flutterRoot, '.pub-cache');
final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools');
+
+/// The arguments to pass to `flutter test` (typically the local engine
+/// configuration) -- prefilled with the arguments passed to test.dart.
final List<String> flutterTestArgs = <String>[];
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true';
-const Map<String, ShardRunner> _kShards = <String, ShardRunner>{
- 'tests': _runTests,
- 'web_tests': _runWebTests,
- 'tool_tests': _runToolTests,
- 'tool_coverage': _runToolCoverage,
- 'build_tests': _runBuildTests,
- 'coverage': _runCoverage,
- 'integration_tests': _runIntegrationTests,
- 'add2app_test': _runAdd2AppTest,
-};
+/// The number of Cirrus jobs that run host-only devicelab tests in parallel.
+///
+/// WARNING: if you change this number, also change .cirrus.yml
+/// and make sure it runs _all_ shards.
+const int kDeviceLabShardCount = 6;
+
+/// The number of Cirrus jobs that run Web tests in parallel.
+///
+/// WARNING: if you change this number, also change .cirrus.yml
+/// and make sure it runs _all_ shards.
+///
+/// The last shard also runs the Web plugin tests.
+const int kWebShardCount = 6;
+
+/// Maximum number of Web tests to run in a single `flutter test`. We found that
+/// large batches can get flaky, possibly because we reuse a single instance of
+/// the browser, and after many tests the browser's state gets corrupted.
+const int kWebBatchSize = 20;
+
+/// Tests that we don't run on Web for various reasons.
+//
+// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
+const List<String> kWebTestDirectoryBlacklist = <String>[
+ 'cupertino',
+ 'examples',
+ 'material',
+];
+const List<String> kWebTestFileBlacklist = <String>[
+ 'test/widgets/heroes_test.dart',
+ 'test/widgets/text_test.dart',
+ 'test/widgets/selectable_text_test.dart',
+ 'test/widgets/color_filter_test.dart',
+ 'test/widgets/editable_text_cursor_test.dart',
+ 'test/widgets/shadow_test.dart',
+ 'test/widgets/raw_keyboard_listener_test.dart',
+ 'test/widgets/editable_text_test.dart',
+ 'test/widgets/widget_inspector_test.dart',
+ 'test/widgets/draggable_test.dart',
+ 'test/widgets/shortcuts_test.dart',
+];
/// When you call this, you can pass additional arguments to pass custom
/// arguments to flutter test. For example, you might want to call this
@@ -53,31 +86,30 @@
///
/// To run the tool_tests part, run it with SHARD=tool_tests
///
-/// For example:
+/// Examples:
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
Future<void> main(List<String> args) async {
flutterTestArgs.addAll(args);
-
- final String shard = Platform.environment['SHARD'];
- if (shard != null) {
- if (!_kShards.containsKey(shard)) {
- print('Invalid shard: $shard');
- print('The available shards are: ${_kShards.keys.join(", ")}');
- exit(1);
- }
- print('${bold}SHARD=$shard$reset');
- await _kShards[shard]();
- } else {
- for (String currentShard in _kShards.keys) {
- print('${bold}SHARD=$currentShard$reset');
- await _kShards[currentShard]();
- print('');
- }
- }
+ if (Platform.environment.containsKey(CIRRUS_TASK_NAME))
+ print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
+ print('═' * 80);
+ await _runSmokeTests();
+ print('═' * 80);
+ await selectShard(const <String, ShardRunner>{
+ 'add_to_app_tests': _runAddToAppTests,
+ 'build_tests': _runBuildTests,
+ 'framework_coverage': _runFrameworkCoverage,
+ 'framework_tests': _runFrameworkTests,
+ 'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests,
+ 'tool_coverage': _runToolCoverage,
+ 'tool_tests': _runToolTests,
+ 'web_tests': _runWebTests,
+ });
}
Future<void> _runSmokeTests() async {
+ print('${green}Running smoketests...$reset');
// Verify that the tests actually return failure on failure and success on
// success.
final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests');
@@ -108,10 +140,11 @@
script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
expectFailure: true,
printOutput: false,
- outputChecker: (CapturedOutput output) =>
- output.stdout.contains('failingPendingTimerTest')
- ? null
- : 'Failed to find the stack trace for the pending Timer.',
+ outputChecker: (CapturedOutput output) {
+ return output.stdout.contains('failingPendingTimerTest')
+ ? null
+ : 'Failed to find the stack trace for the pending Timer.';
+ }
);
// We run the remaining smoketests in parallel, because they each take some
// time to run (e.g. compiling), so we don't want to run them in series,
@@ -153,8 +186,11 @@
);
// Verify that we correctly generated the version file.
- final bool validVersion = await verifyVersion(path.join(flutterRoot, 'version'));
- if (!validVersion) {
+ final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
+ if (versionError != null) {
+ print(redLine);
+ print(versionError);
+ print(redLine);
exit(1);
}
}
@@ -179,7 +215,7 @@
final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes);
return bq.BigqueryApi(client);
} catch (e) {
- print('Failed to get BigQuery API client.');
+ print('${red}Failed to get BigQuery API client.$reset');
print(e);
return null;
}
@@ -203,7 +239,6 @@
Future<void> _runToolTests() async {
final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
- await _runSmokeTests();
const String kDotShard = '.shard';
const String kTest = 'test';
@@ -232,11 +267,10 @@
await selectSubshard(subshards);
}
-/// Verifies that AOT, APK, and IPA (if on macOS) builds the
-/// examples apps without crashing. It does not actually
-/// launch the apps. That happens later in the devicelab. This is
-/// just a smoke-test. In particular, this will verify we can build
-/// when there are spaces in the path name for the Flutter SDK and
+/// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps
+/// without crashing. It does not actually launch the apps. That happens later
+/// in the devicelab. This is just a smoke-test. In particular, this will verify
+/// we can build when there are spaces in the path name for the Flutter SDK and
/// target app.
Future<void> _runBuildTests() async {
final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list();
@@ -245,10 +279,11 @@
continue;
}
final String examplePath = fileEntity.path;
-
await _flutterBuildAot(examplePath);
await _flutterBuildApk(examplePath);
- await _flutterBuildIpa(examplePath);
+ if (Platform.isMacOS) {
+ await _flutterBuildIpa(examplePath);
+ }
}
// Web compilation tests.
await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart'));
@@ -256,12 +291,44 @@
await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'),
path.join('lib', 'dart_io_import.dart'),
);
+}
- print('${bold}DONE: All build tests successful.$reset');
+Future<void> _flutterBuildAot(String relativePathToApplication) async {
+ print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...');
+ await runCommand(flutter,
+ <String>['build', 'aot', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ );
+}
+
+Future<void> _flutterBuildApk(String relativePathToApplication) async {
+ print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...');
+ await runCommand(flutter,
+ <String>['build', 'apk', '--debug', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ );
+}
+
+Future<void> _flutterBuildIpa(String relativePathToApplication) async {
+ assert(Platform.isMacOS);
+ print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
+ // Install Cocoapods. We don't have these checked in for the examples,
+ // and build ios doesn't take care of it automatically.
+ final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
+ if (podfile.existsSync()) {
+ await runCommand('pod',
+ <String>['install'],
+ workingDirectory: podfile.parent.path,
+ );
+ }
+ await runCommand(flutter,
+ <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
+ workingDirectory: path.join(flutterRoot, relativePathToApplication),
+ );
}
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
- print('Running Dart2JS build tests...');
+ print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter,
<String>['build', 'web', '-v', '--target=$target'],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
@@ -270,211 +337,101 @@
'FLUTTER_WEB': 'true',
},
);
- print('Done.');
}
-Future<void> _flutterBuildAot(String relativePathToApplication) async {
- print('Running AOT build tests...');
- await runCommand(flutter,
- <String>['build', 'aot', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- expectNonZeroExit: false,
- );
- print('Done.');
-}
-
-Future<void> _flutterBuildApk(String relativePathToApplication) async {
- if (
- (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) &&
- (Platform.environment['ANDROID_SDK_ROOT']?.isEmpty ?? true)) {
- return;
- }
- print('Running APK build tests...');
- await runCommand(flutter,
- <String>['build', 'apk', '--debug', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- expectNonZeroExit: false,
- );
- print('Done.');
-}
-
-Future<void> _flutterBuildIpa(String relativePathToApplication) async {
- if (!Platform.isMacOS) {
- return;
- }
- print('Running IPA build tests...');
- // Install Cocoapods. We don't have these checked in for the examples,
- // and build ios doesn't take care of it automatically.
- final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile'));
- if (podfile.existsSync()) {
- await runCommand('pod',
- <String>['install'],
- workingDirectory: podfile.parent.path,
- expectNonZeroExit: false,
+Future<void> _runAddToAppTests() async {
+ if (Platform.isMacOS) {
+ print('${green}Running add-to-app iOS integration tests$reset...');
+ final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
+ await runCommand('./build_and_test.sh',
+ <String>[],
+ workingDirectory: addToAppDir,
);
}
- await runCommand(flutter,
- <String>['build', 'ios', '--no-codesign', '--debug', '-v'],
- workingDirectory: path.join(flutterRoot, relativePathToApplication),
- expectNonZeroExit: false,
- );
- print('Done.');
}
-Future<void> _runAdd2AppTest() async {
- if (!Platform.isMacOS) {
- return;
- }
- print('Running Add2App iOS integration tests...');
- final String add2AppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app');
- await runCommand('./build_and_test.sh',
- <String>[],
- workingDirectory: add2AppDir,
- expectNonZeroExit: false,
- );
- print('Done.');
-}
-
-Future<void> _runTests() async {
+Future<void> _runFrameworkTests() async {
final bq.BigqueryApi bigqueryApi = await _getBigqueryApi();
- await _runSmokeTests();
- final String subShard = Platform.environment['SUBSHARD'];
Future<void> runWidgets() async {
- await _runFlutterTest(
- path.join(flutterRoot, 'packages', 'flutter'),
- tableData: bigqueryApi?.tabledata,
- tests: <String>[
- path.join('test', 'widgets') + path.separator,
- ],
- );
- // Only packages/flutter/test/widgets/widget_inspector_test.dart really
- // needs to be run with --track-widget-creation but it is nice to run
- // all of the tests in package:flutter with the flag to ensure that
- // the Dart kernel transformer triggered by the flag does not break anything.
+ print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--track-widget-creation'],
tableData: bigqueryApi?.tabledata,
- tests: <String>[
- path.join('test', 'widgets') + path.separator,
- ],
+ tests: <String>[ path.join('test', 'widgets') + path.separator ],
);
+ await _runFlutterTest(
+ path.join(flutterRoot, 'packages', 'flutter'),
+ options: <String>['--no-track-widget-creation'],
+ tableData: bigqueryApi?.tabledata,
+ tests: <String>[ path.join('test', 'widgets') + path.separator ],
+ );
+ // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata);
}
- Future<void> runFrameworkOthers() async {
+ Future<void> runLibraries() async {
final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test'))
.listSync(followLinks: false, recursive: false)
.whereType<Directory>()
.where((Directory dir) => dir.path.endsWith('widgets') == false)
- .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
+ .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
.toList();
-
- print('Running tests for: ${tests.join(';')}');
-
- await _runFlutterTest(
- path.join(flutterRoot, 'packages', 'flutter'),
- tableData: bigqueryApi?.tabledata,
- tests: tests,
- );
- // Only packages/flutter/test/widgets/widget_inspector_test.dart really
- // needs to be run with --track-widget-creation but it is nice to run
- // all of the tests in package:flutter with the flag to ensure that
- // the Dart kernel transformer triggered by the flag does not break anything.
+ print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--track-widget-creation'],
tableData: bigqueryApi?.tabledata,
tests: tests,
);
+ await _runFlutterTest(
+ path.join(flutterRoot, 'packages', 'flutter'),
+ options: <String>['--no-track-widget-creation'],
+ tableData: bigqueryApi?.tabledata,
+ tests: tests,
+ );
}
- Future<void> runExtras() async {
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
+ Future<void> runMisc() async {
+ print('${green}Running package tests$reset for directories other than packages/flutter');
await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata);
await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata);
await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata);
await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata);
- // Regression test to ensure that code outside of package:flutter can run
- // with --track-widget-creation.
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata);
- await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata);
- // Smoke test for code generation.
- await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: <String, String>{
- 'FLUTTER_EXPERIMENTAL_BUILD': 'true',
- });
- }
- switch (subShard) {
- case 'widgets':
- await runWidgets();
- break;
- case 'framework_other':
- await runFrameworkOthers();
- break;
- case 'extras':
- runExtras();
- break;
- default:
- print('Unknown sub-shard $subShard, running all tests!');
- await runWidgets();
- await runFrameworkOthers();
- await runExtras();
-
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata);
+ await _runFlutterTest(
+ path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'),
+ tableData: bigqueryApi?.tabledata,
+ environment: <String, String>{
+ 'FLUTTER_EXPERIMENTAL_BUILD': 'true',
+ },
+ );
}
- print('${bold}DONE: All tests successful.$reset');
+ await selectSubshard(<String, ShardRunner>{
+ 'widgets': runWidgets,
+ 'libraries': runLibraries,
+ 'misc': runMisc,
+ });
}
-// TODO(yjbanov): we're getting rid of these blacklists as part of https://github.com/flutter/flutter/projects/60
-const List<String> kWebTestDirectoryBlacklist = <String>[
- 'test/cupertino',
- 'test/examples',
- 'test/material',
-];
-const List<String> kWebTestFileBlacklist = <String>[
- 'test/widgets/heroes_test.dart',
- 'test/widgets/text_test.dart',
- 'test/widgets/selectable_text_test.dart',
- 'test/widgets/color_filter_test.dart',
- 'test/widgets/editable_text_cursor_test.dart',
- 'test/widgets/shadow_test.dart',
- 'test/widgets/raw_keyboard_listener_test.dart',
- 'test/widgets/editable_text_test.dart',
- 'test/widgets/widget_inspector_test.dart',
- 'test/widgets/draggable_test.dart',
- 'test/widgets/shortcuts_test.dart',
-];
-
-Future<void> _runWebTests() async {
- final Directory flutterPackageDir = Directory(path.join(flutterRoot, 'packages', 'flutter'));
- final Directory testDir = Directory(path.join(flutterPackageDir.path, 'test'));
-
- final List<String> directories = testDir
- .listSync()
- .whereType<Directory>()
- .map<String>((Directory dir) => path.relative(dir.path, from: flutterPackageDir.path))
- .where((String relativePath) => !kWebTestDirectoryBlacklist.contains(relativePath))
- .toList();
-
- await _runFlutterWebTest(flutterPackageDir.path, tests: directories);
- await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter_web_plugins'), tests: <String>['test']);
-}
-
-Future<void> _runCoverage() async {
+Future<void> _runFrameworkCoverage() async {
final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info'));
if (!coverageFile.existsSync()) {
print('${red}Coverage file not found.$reset');
- print('Expected to find: ${coverageFile.absolute}');
- print('This file is normally obtained by running `flutter update-packages`.');
+ print('Expected to find: $cyan${coverageFile.absolute}$reset');
+ print('This file is normally obtained by running `${green}flutter update-packages$reset`.');
exit(1);
}
coverageFile.deleteSync();
@@ -483,11 +440,98 @@
);
if (!coverageFile.existsSync()) {
print('${red}Coverage file not found.$reset');
- print('Expected to find: ${coverageFile.absolute}');
- print('This file should have been generated by the `flutter test --coverage` script, but was not.');
+ print('Expected to find: $cyan${coverageFile.absolute}$reset');
+ print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.');
exit(1);
}
- print('${bold}DONE: Coverage collection successful.$reset');
+}
+
+Future<void> _runWebTests() async {
+ final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+
+ final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
+ final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
+
+ final List<String> allTests = flutterPackageTestDirectory
+ .listSync()
+ .whereType<Directory>()
+ .where((Directory directory) => !kWebTestDirectoryBlacklist.contains(path.basename(directory.path)))
+ .expand((Directory directory) => directory
+ .listSync(recursive: true)
+ .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
+ )
+ .whereType<File>()
+ .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
+ .where((String filePath) => !kWebTestFileBlacklist.contains(filePath))
+ .toList()
+ // Finally we shuffle the list because we want the average cost per file to be uniformly
+ // distributed. If the list is not sorted then different shards and batches may have
+ // very different characteristics.
+ // We use a constant seed for repeatability.
+ ..shuffle(math.Random(0));
+
+ assert(kWebShardCount >= 1);
+ final int testsPerShard = (allTests.length / kWebShardCount).ceil();
+ assert(testsPerShard * kWebShardCount >= allTests.length);
+
+ // This for loop computes all but the last shard.
+ for (int index = 0; index < kWebShardCount - 1; index += 1) {
+ subshards['$index'] = () => _runFlutterWebTest(
+ flutterPackageDirectory.path,
+ allTests.sublist(
+ index * testsPerShard,
+ (index + 1) * testsPerShard,
+ ),
+ );
+ }
+
+ // The last shard also runs the flutter_web_plugins tests.
+ //
+ // We make sure the last shard ends in _last so it's easier to catch mismatches
+ // between `.cirrus.yml` and `test.dart`.
+ subshards['${kWebShardCount - 1}_last'] = () async {
+ await _runFlutterWebTest(
+ flutterPackageDirectory.path,
+ allTests.sublist(
+ (kWebShardCount - 1) * testsPerShard,
+ allTests.length,
+ ),
+ );
+ await _runFlutterWebTest(
+ path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
+ <String>['test'],
+ );
+ };
+
+ await selectSubshard(subshards);
+}
+
+Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async {
+ final List<String> batch = <String>[];
+ for (int i = 0; i < tests.length; i += 1) {
+ final String testFilePath = tests[i];
+ batch.add(testFilePath);
+ if (batch.length == kWebBatchSize || i == tests.length - 1) {
+ await runCommand(
+ flutter,
+ <String>[
+ 'test',
+ if (ciProvider == CiProviders.cirrus)
+ '--concurrency=1', // do not parallelize on Cirrus, to reduce flakiness
+ '-v',
+ '--platform=chrome',
+ ...?flutterTestArgs,
+ ...batch,
+ ],
+ workingDirectory: workingDirectory,
+ environment: <String, String>{
+ 'FLUTTER_WEB': 'true',
+ 'FLUTTER_LOW_RESOURCE_MODE': 'true',
+ },
+ );
+ batch.clear();
+ }
+ }
}
Future<void> _pubRunTest(String workingDirectory, {
@@ -496,14 +540,33 @@
bool useBuildRunner = false,
bq.TabledataResourceApi tableData,
}) async {
- final List<String> args = <String>['run', '--verbose'];
+ final List<String> args = <String>['run'];
if (useBuildRunner) {
- args.addAll(<String>['build_runner', 'test', '--']);
+ final String posixTestPath = path.posix.joinAll(path.split(testPath));
+ args.addAll(<String>[
+ 'build_runner',
+ 'test',
+ '--build-filter=$posixTestPath/*.dill',
+ '--build-filter=$posixTestPath/**/*.dill',
+ '--',
+ ]);
} else {
args.add('test');
}
args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact');
- args.add('-j1'); // TODO(ianh): Scale based on CPUs.
+ int cpus;
+ final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
+ if (cpuVariable != null) {
+ cpus = int.tryParse(cpuVariable, radix: 10);
+ if (cpus == null) {
+ print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset');
+ print('Actual value: "$cpuVariable"');
+ exit(1);
+ }
+ } else {
+ cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
+ }
+ args.add('-j$cpus');
if (!hasColor)
args.add('--no-color');
if (testPath != null)
@@ -547,238 +610,6 @@
}
}
-void deleteFile(String path) {
- // There's a race condition here but in theory we're not racing anyone
- // while this script runs, so should be ok.
- final File file = File(path);
- if (file.existsSync())
- file.deleteSync();
-}
-
-enum CiProviders {
- cirrus,
- luci,
-}
-
-CiProviders _getCiProvider() {
- if (Platform.environment['CIRRUS_CI'] == 'true') {
- return CiProviders.cirrus;
- }
- if (Platform.environment['LUCI_CONTEXT'] != null) {
- return CiProviders.luci;
- }
- return null;
-}
-
-String _getCiProviderName() {
- switch(_getCiProvider()) {
- case CiProviders.cirrus:
- return 'cirrusci';
- case CiProviders.luci:
- return 'luci';
- }
- return 'unknown';
-}
-
-int _getPrNumber() {
- switch(_getCiProvider()) {
- case CiProviders.cirrus:
- return Platform.environment['CIRRUS_PR'] == null
- ? -1
- : int.tryParse(Platform.environment['CIRRUS_PR']);
- case CiProviders.luci:
- return -1; // LUCI doesn't know about this.
- }
- return -1;
-}
-
-Future<String> _getAuthors() async {
- final String exe = Platform.isWindows ? '.exe' : '';
- final String author = await runAndGetStdout(
- 'git$exe', <String>['-c', 'log.showSignature=false', 'log', _getGitHash(), '--pretty="%an <%ae>"'],
- workingDirectory: flutterRoot,
- ).first;
- return author;
-}
-
-String _getCiUrl() {
- switch(_getCiProvider()) {
- case CiProviders.cirrus:
- return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
- case CiProviders.luci:
- return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
- }
- return '';
-}
-
-String _getGitHash() {
- switch(_getCiProvider()) {
- case CiProviders.cirrus:
- return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
- case CiProviders.luci:
- return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
- }
- return '';
-}
-
-Future<void> _processTestOutput(
- FlutterCompactFormatter formatter,
- Stream<String> testOutput,
- bq.TabledataResourceApi tableData,
-) async {
- final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
- print('Processing...');
- });
-
- await testOutput.forEach(formatter.processRawOutput);
- heartbeat.cancel();
- formatter.finish();
- if (tableData == null || formatter.tests.isEmpty) {
- return;
- }
- final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
- final String authors = await _getAuthors();
- request.rows = List<bq.TableDataInsertAllRequestRows>.from(
- formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
- bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
- 'json': <String, dynamic>{
- 'source': <String, dynamic>{
- 'provider': _getCiProviderName(),
- 'url': _getCiUrl(),
- 'platform': <String, dynamic>{
- 'os': Platform.operatingSystem,
- 'version': Platform.operatingSystemVersion,
- },
- },
- 'test': <String, dynamic>{
- 'name': result.name,
- 'result': result.status.toString(),
- 'file': result.path,
- 'line': result.line,
- 'column': result.column,
- 'time': result.totalTime,
- },
- 'git': <String, dynamic>{
- 'author': authors,
- 'pull_request': _getPrNumber(),
- 'commit': _getGitHash(),
- 'organization': 'flutter',
- 'repository': 'flutter',
- },
- 'error': result.status != TestStatus.failed ? null : <String, dynamic>{
- 'message': result.errorMessage,
- 'stack_trace': result.stackTrace,
- },
- 'information': result.messages,
- },
- }),
- ),
- growable: false,
- );
- final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
- if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
- print('${red}BigQuery insert errors:');
- print(response.toJson());
- print(reset);
- }
-}
-
-class EvalResult {
- EvalResult({
- this.stdout,
- this.stderr,
- this.exitCode = 0,
- });
-
- final String stdout;
- final String stderr;
- final int exitCode;
-}
-
-/// The number of Cirrus jobs that run web tests in parallel.
-///
-/// WARNING: if you change this number, also change .cirrus.yml
-/// and make sure it runs _all_ shards.
-const int _kWebShardCount = 6;
-
-Future<void> _runFlutterWebTest(String workingDirectory, {
- List<String> tests,
-}) async {
- List<String> allTests = <String>[];
- for (String testDirPath in tests) {
- final Directory testDir = Directory(path.join(workingDirectory, testDirPath));
- allTests.addAll(
- testDir.listSync(recursive: true)
- .whereType<File>()
- .where((File file) => file.path.endsWith('_test.dart'))
- .map<String>((File file) => path.relative(file.path, from: workingDirectory))
- .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)),
- );
- }
-
- // If a shard is specified only run tests in that shard.
- final int webShard = int.tryParse(Platform.environment['WEB_SHARD'] ?? 'n/a');
- if (webShard != null) {
- if (webShard >= _kWebShardCount) {
- throw 'WEB_SHARD must be <= _kWebShardCount, but was $webShard';
- }
- final List<String> shard = <String>[];
- for (int i = webShard; i < allTests.length; i += _kWebShardCount) {
- shard.add(allTests[i]);
- }
- allTests = shard;
- }
-
- print(allTests.join('\n'));
- print('${allTests.length} tests total');
-
- // Maximum number of tests to run in a single `flutter test`. We found that
- // large batches can get flaky, possibly because we reuse a single instance
- // of the browser, and after many tests the browser's state gets corrupted.
- const int kBatchSize = 20;
- List<String> batch = <String>[];
- for (int i = 0; i < allTests.length; i += 1) {
- final String testFilePath = allTests[i];
- batch.add(testFilePath);
- if (batch.length == kBatchSize || i == allTests.length - 1) {
- await _runFlutterWebTestBatch(workingDirectory, batch: batch);
- batch = <String>[];
- }
- }
-}
-
-Future<void> _runFlutterWebTestBatch(String workingDirectory, {
- List<String> batch,
-}) async {
- final List<String> args = <String>[
- 'test',
- if (_getCiProvider() == CiProviders.cirrus)
- '--concurrency=1', // do not parallelize on Cirrus to reduce flakiness
- '-v',
- '--platform=chrome',
- ...?flutterTestArgs,
- ...batch,
- ];
-
- // TODO(jonahwilliams): fix relative path issues to make this unecessary.
- final Directory oldCurrent = Directory.current;
- Directory.current = Directory(path.join(flutterRoot, 'packages', 'flutter'));
- try {
- await runCommand(
- flutter,
- args,
- workingDirectory: workingDirectory,
- expectFlaky: false,
- environment: <String, String>{
- 'FLUTTER_WEB': 'true',
- 'FLUTTER_LOW_RESOURCE_MODE': 'true',
- },
- );
- } finally {
- Directory.current = oldCurrent;
- }
-}
-
Future<void> _runFlutterTest(String workingDirectory, {
String script,
bool expectFailure = false,
@@ -790,8 +621,7 @@
Map<String, String> environment,
List<String> tests = const <String>[],
}) async {
- assert(!printOutput || outputChecker == null,
- 'Output either can be printed or checked but not both');
+ assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
final List<String> args = <String>[
'test',
@@ -800,16 +630,15 @@
];
final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
- if (shouldProcessOutput) {
+ if (shouldProcessOutput)
args.add('--machine');
- }
if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) {
- print('Could not find test: $fullScriptPath');
- print('Working directory: $workingDirectory');
- print('Script: $script');
+ print('${red}Could not find test$reset: $green$fullScriptPath$reset');
+ print('Working directory: $cyan$workingDirectory$reset');
+ print('Script: $green$script$reset');
if (!printOutput)
print('This is one of the tests that does not normally print output.');
if (skip)
@@ -876,110 +705,290 @@
}
}
-// the optional `file` argument is an override for testing
-@visibleForTesting
-Future<bool> verifyVersion(String filename, [File file]) async {
- final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
- file ??= File(filename);
- final String version = await file.readAsString();
- if (!file.existsSync()) {
- print('$redLine');
- print('The version logic failed to create the Flutter version file.');
- print('$redLine');
- return false;
- }
- if (version == '0.0.0-unknown') {
- print('$redLine');
- print('The version logic failed to determine the Flutter version.');
- print('$redLine');
- return false;
- }
- if (!version.contains(pattern)) {
- print('$redLine');
- print('The version logic generated an invalid version string: "$version".');
- print('$redLine');
- return false;
- }
- return true;
-}
-
-Future<void> _runIntegrationTests() async {
- final String subShard = Platform.environment['SUBSHARD'];
-
- switch (subShard) {
- case 'gradle1':
- case 'gradle2':
- // This runs some gradle integration tests if the subshard is Android.
- await _androidGradleTests(subShard);
- break;
- default:
- await _runDevicelabTest('dartdocs');
-
- if (Platform.isLinux) {
- await _runDevicelabTest('flutter_create_offline_test_linux');
- } else if (Platform.isWindows) {
- await _runDevicelabTest('flutter_create_offline_test_windows');
- } else if (Platform.isMacOS) {
- await _runDevicelabTest('flutter_create_offline_test_mac');
- await _runDevicelabTest('plugin_lint_mac');
-// TODO(jmagman): Re-enable once flakiness is resolved.
-// await _runDevicelabTest('module_test_ios');
- }
- }
-}
-
-Future<void> _runDevicelabTest(String testName, {Map<String, String> env}) async {
- await runCommand(
- dart,
- <String>['bin/run.dart', '-t', testName],
- workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
- environment: env,
- );
-}
-
-String get androidSdkRoot {
+Map<String, String> _initGradleEnvironment() {
final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true)
? Platform.environment['ANDROID_SDK_ROOT']
: Platform.environment['ANDROID_HOME'];
if (androidSdkRoot == null || androidSdkRoot.isEmpty) {
- return null;
+ print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset');
+ exit(1);
}
- return androidSdkRoot;
-}
-
-Future<void> _androidGradleTests(String subShard) async {
- // TODO(dnfield): gradlew is crashing on the cirrus image and it's not clear why.
- if (androidSdkRoot == null || Platform.isWindows) {
- print('No Android SDK detected or on Windows, skipping Android gradle test.');
- return;
- }
- final Map<String, String> defaultEnv = <String, String>{
+ return <String, String>{
'ANDROID_HOME': androidSdkRoot,
'ANDROID_SDK_ROOT': androidSdkRoot,
- 'ENABLE_ANDROID_EMBEDDING_V2': Platform.environment['ENABLE_ANDROID_EMBEDDING_V2'] ?? '',
};
- if (subShard == 'gradle1') {
- await _runDevicelabTest('gradle_plugin_light_apk_test', env: defaultEnv);
- await _runDevicelabTest('gradle_plugin_fat_apk_test', env: defaultEnv);
- await _runDevicelabTest('gradle_r8_test', env: defaultEnv);
- await _runDevicelabTest('gradle_non_android_plugin_test', env: defaultEnv);
- await _runDevicelabTest('gradle_jetifier_test', env: defaultEnv);
+}
+
+final Map<String, String> gradleEnvironment = _initGradleEnvironment();
+
+Future<void> _runHostOnlyDeviceLabTests() async {
+ if (Platform.isWindows) {
+ // TODO(ianh): remove when https://github.com/flutter/flutter/issues/36311 fixed by https://github.com/flutter/flutter/pull/42709
+ return;
}
- if (subShard == 'gradle2') {
- await _runDevicelabTest('gradle_plugin_bundle_test', env: defaultEnv);
- await _runDevicelabTest('module_test', env: defaultEnv);
- await _runDevicelabTest('module_host_with_custom_build_test', env: defaultEnv);
- await _runDevicelabTest('build_aar_module_test', env: defaultEnv);
- await _runDevicelabTest('plugin_test', env: defaultEnv);
+
+ // Please don't add more tests here. We should not be using the devicelab
+ // logic to run tests outside devicelab, that's just confusing.
+ // Instead, create tests that are not devicelab tests, and run those.
+
+ // TODO(ianh): Move the tests that are not running on devicelab any more out
+ // of the device lab directory.
+
+ // List the tests to run.
+ // We split these into subshards. The tests are randomly distributed into
+ // those subshards so as to get a uniform distribution of costs, but the
+ // seed is fixed so that issues are reproducible.
+ final List<ShardRunner> tests = <ShardRunner>[
+ // Keep this in alphabetical order.
+ () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'),
+ if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'),
+ if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'),
+ // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494
+ if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ // TODO(jmagman): Re-enable once flakiness is resolved, https://github.com/flutter/flutter/issues/37525
+ // if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'),
+ if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'),
+ () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: false),
+ () => _runDevicelabTest('plugin_test', environment: gradleEnvironment, testEmbeddingV2: true),
+ ]..shuffle(math.Random(0));
+
+ final int testsPerShard = tests.length ~/ kDeviceLabShardCount;
+ final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
+
+ for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) {
+ String last = '';
+ List<ShardRunner> sublist;
+ if (subshard < kDeviceLabShardCount - 1) {
+ sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard);
+ } else {
+ sublist = tests.sublist(subshard * testsPerShard, tests.length);
+ // We make sure the last shard ends in _last so it's easier to catch mismatches
+ // between `.cirrus.yml` and `test.dart`.
+ last = '_last';
+ }
+ subshards['$subshard$last'] = () async {
+ for (ShardRunner test in sublist)
+ await test();
+ };
+ }
+
+ await selectSubshard(subshards);
+}
+
+Future<void> _runDevicelabTest(String testName, {
+ Map<String, String> environment,
+ bool testEmbeddingV2 = false,
+}) async {
+ await runCommand(
+ dart,
+ <String>['bin/run.dart', '-t', testName],
+ workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'),
+ environment: <String, String>{
+ ...?environment,
+ if (testEmbeddingV2)
+ 'ENABLE_ANDROID_EMBEDDING_V2': 'true',
+ },
+ );
+}
+
+void deleteFile(String path) {
+ // This is technically a race condition but nobody else should be running
+ // while this script runs, so we should be ok. (Sadly recursive:true does not
+ // obviate the need for existsSync, at least on Windows.)
+ final File file = File(path);
+ if (file.existsSync())
+ file.deleteSync();
+}
+
+enum CiProviders {
+ cirrus,
+ luci,
+}
+
+Future<void> _processTestOutput(
+ FlutterCompactFormatter formatter,
+ Stream<String> testOutput,
+ bq.TabledataResourceApi tableData,
+) async {
+ final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
+ print('Processing...');
+ });
+
+ await testOutput.forEach(formatter.processRawOutput);
+ heartbeat.cancel();
+ formatter.finish();
+ if (tableData == null || formatter.tests.isEmpty) {
+ return;
+ }
+ final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest();
+ final String authors = await _getAuthors();
+ request.rows = List<bq.TableDataInsertAllRequestRows>.from(
+ formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) =>
+ bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> {
+ 'json': <String, dynamic>{
+ 'source': <String, dynamic>{
+ 'provider': ciProviderName,
+ 'url': ciUrl,
+ 'platform': <String, dynamic>{
+ 'os': Platform.operatingSystem,
+ 'version': Platform.operatingSystemVersion,
+ },
+ },
+ 'test': <String, dynamic>{
+ 'name': result.name,
+ 'result': result.status.toString(),
+ 'file': result.path,
+ 'line': result.line,
+ 'column': result.column,
+ 'time': result.totalTime,
+ },
+ 'git': <String, dynamic>{
+ 'author': authors,
+ 'pull_request': prNumber,
+ 'commit': gitHash,
+ 'organization': 'flutter',
+ 'repository': 'flutter',
+ },
+ 'error': result.status != TestStatus.failed ? null : <String, dynamic>{
+ 'message': result.errorMessage,
+ 'stack_trace': result.stackTrace,
+ },
+ 'information': result.messages,
+ },
+ }),
+ ),
+ growable: false,
+ );
+ final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci');
+ if (response.insertErrors != null && response.insertErrors.isNotEmpty) {
+ print('${red}BigQuery insert errors:');
+ print(response.toJson());
+ print(reset);
}
}
-Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard');
-Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard');
+CiProviders get ciProvider {
+ if (Platform.environment['CIRRUS_CI'] == 'true') {
+ return CiProviders.cirrus;
+ }
+ if (Platform.environment['LUCI_CONTEXT'] != null) {
+ return CiProviders.luci;
+ }
+ return null;
+}
-Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name) async {
- final String item = Platform.environment[key];
- if (item != null) {
+String get ciProviderName {
+ switch (ciProvider) {
+ case CiProviders.cirrus:
+ return 'cirrusci';
+ case CiProviders.luci:
+ return 'luci';
+ }
+ return 'unknown';
+}
+
+int get prNumber {
+ switch (ciProvider) {
+ case CiProviders.cirrus:
+ return Platform.environment['CIRRUS_PR'] == null
+ ? -1
+ : int.tryParse(Platform.environment['CIRRUS_PR']);
+ case CiProviders.luci:
+ return -1; // LUCI doesn't know about this.
+ }
+ return -1;
+}
+
+Future<String> _getAuthors() async {
+ final String exe = Platform.isWindows ? '.exe' : '';
+ final String author = await runAndGetStdout(
+ 'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'],
+ workingDirectory: flutterRoot,
+ ).first;
+ return author;
+}
+
+String get ciUrl {
+ switch (ciProvider) {
+ case CiProviders.cirrus:
+ return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}';
+ case CiProviders.luci:
+ return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build?
+ }
+ return '';
+}
+
+String get gitHash {
+ switch(ciProvider) {
+ case CiProviders.cirrus:
+ return Platform.environment['CIRRUS_CHANGE_IN_REPO'];
+ case CiProviders.luci:
+ return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI.
+ }
+ return '';
+}
+
+/// Checks the given file's contents to determine if they match the allowed
+/// pattern for version strings.
+///
+/// Returns null if the contents are good. Returns a string if they are bad.
+/// The string is an error message.
+Future<String> verifyVersion(File file) async {
+ final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$');
+ final String version = await file.readAsString();
+ if (!file.existsSync())
+ return 'The version logic failed to create the Flutter version file.';
+ if (version == '0.0.0-unknown')
+ return 'The version logic failed to determine the Flutter version.';
+ if (!version.contains(pattern))
+ return 'The version logic generated an invalid version string: "$version".';
+ return null;
+}
+
+/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
+/// the shard and subshard (parsing it in the form shard-subshard-platform, ignoring
+/// the platform).
+///
+/// However, for local testing you can just set the SHARD and SUBSHARD
+/// environment variables. For example, to run all the framework tests you can
+/// just set SHARD=framework_tests. To run specifically the third subshard of
+/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
+Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0);
+Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1);
+
+const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
+
+Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
+ String item = Platform.environment[key];
+ if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
+ final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-');
+ assert(positionInTaskName < parts.length);
+ item = parts[positionInTaskName];
+ }
+ if (item == null) {
+ for (String currentItem in items.keys) {
+ print('$bold$key=$currentItem$reset');
+ await items[currentItem]();
+ print('');
+ }
+ } else {
if (!items.containsKey(item)) {
print('${red}Invalid $name: $item$reset');
print('The available ${name}s are: ${items.keys.join(", ")}');
@@ -987,11 +996,5 @@
}
print('$bold$key=$item$reset');
await items[item]();
- } else {
- for (String currentItem in items.keys) {
- print('$bold$key=$currentItem$reset');
- await items[currentItem]();
- print('');
- }
}
}
