| // Copyright 2017 The Chromium Authors. All rights reserved. | |
| // Use of this source code is governed by a BSD-style license that can be | |
| // found in the LICENSE file. | |
| import 'dart:async'; | |
| import 'dart:io'; | |
| import 'dart:math' as math; | |
| import 'package:googleapis/bigquery/v2.dart' as bq; | |
| import 'package:googleapis_auth/auth_io.dart' as auth; | |
| import 'package:http/http.dart' as http; | |
| import 'package:path/path.dart' as path; | |
| import 'flutter_compact_formatter.dart'; | |
| import 'run_command.dart'; | |
| typedef ShardRunner = Future<void> Function(); | |
| /// A function used to validate the output of a test. | |
| /// | |
| /// If the output matches expectations, the function shall return null. | |
| /// | |
| /// If the output does not match expectations, the function shall return an | |
| /// appropriate error message. | |
| typedef OutputChecker = String Function(CapturedOutput); | |
| final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script)))); | |
| final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter'); | |
| final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart'); | |
| final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub'); | |
| final String pubCache = path.join(flutterRoot, '.pub-cache'); | |
| final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools'); | |
| /// The arguments to pass to `flutter test` (typically the local engine | |
| /// configuration) -- prefilled with the arguments passed to test.dart. | |
| final List<String> flutterTestArgs = <String>[]; | |
| final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true'; | |
| final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true'; | |
| /// The number of Cirrus jobs that run host-only devicelab tests in parallel. | |
| /// | |
| /// WARNING: if you change this number, also change .cirrus.yml | |
| /// and make sure it runs _all_ shards. | |
| const int kDeviceLabShardCount = 4; | |
| /// The number of Cirrus jobs that run Web tests in parallel. | |
| /// | |
| /// WARNING: if you change this number, also change .cirrus.yml | |
| /// and make sure it runs _all_ shards. | |
| /// | |
| /// The last shard also runs the Web plugin tests. | |
| const int kWebShardCount = 8; | |
| /// Maximum number of Web tests to run in a single `flutter test`. We found that | |
| /// large batches can get flaky, possibly because we reuse a single instance of | |
| /// the browser, and after many tests the browser's state gets corrupted. | |
| const int kWebBatchSize = 20; | |
| /// Tests that we don't run on Web for various reasons. | |
| // | |
| // TODO(yjbanov): we're getting rid of this blacklist as part of https://github.com/flutter/flutter/projects/60 | |
| const List<String> kWebTestFileBlacklist = <String>[ | |
| // This test doesn't compile because it depends on code outside the flutter package. | |
| 'test/examples/sector_layout_test.dart', | |
| 'test/widgets/selectable_text_test.dart', | |
| 'test/widgets/color_filter_test.dart', | |
| 'test/widgets/editable_text_cursor_test.dart', | |
| 'test/widgets/shadow_test.dart', | |
| 'test/widgets/raw_keyboard_listener_test.dart', | |
| 'test/widgets/editable_text_test.dart', | |
| 'test/widgets/widget_inspector_test.dart', | |
| 'test/widgets/shortcuts_test.dart', | |
| 'test/material/text_form_field_test.dart', | |
| 'test/material/data_table_test.dart', | |
| 'test/cupertino/dialog_test.dart', | |
| 'test/cupertino/nav_bar_test.dart', | |
| 'test/cupertino/nav_bar_transition_test.dart', | |
| 'test/cupertino/refresh_test.dart', | |
| 'test/cupertino/switch_test.dart', | |
| 'test/cupertino/text_field_test.dart', | |
| 'test/cupertino/date_picker_test.dart', | |
| 'test/cupertino/slider_test.dart', | |
| 'test/cupertino/text_field_test.dart', | |
| 'test/cupertino/segmented_control_test.dart', | |
| 'test/cupertino/route_test.dart', | |
| 'test/cupertino/activity_indicator_test.dart', | |
| ]; | |
| /// When you call this, you can pass additional arguments to pass custom | |
| /// arguments to flutter test. For example, you might want to call this | |
| /// script with the parameter --local-engine=host_debug_unopt to | |
| /// use your own build of the engine. | |
| /// | |
| /// To run the tool_tests part, run it with SHARD=tool_tests | |
| /// | |
| /// Examples: | |
| /// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart | |
| /// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt | |
| Future<void> main(List<String> args) async { | |
| flutterTestArgs.addAll(args); | |
| if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) | |
| print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}'); | |
| print('═' * 80); | |
| await _runSmokeTests(); | |
| print('═' * 80); | |
| await selectShard(const <String, ShardRunner>{ | |
| 'add_to_app_tests': _runAddToAppTests, | |
| 'build_tests': _runBuildTests, | |
| 'framework_coverage': _runFrameworkCoverage, | |
| 'framework_tests': _runFrameworkTests, | |
| 'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests, | |
| 'tool_coverage': _runToolCoverage, | |
| 'tool_tests': _runToolTests, | |
| 'web_tests': _runWebTests, | |
| }); | |
| } | |
| Future<void> _runSmokeTests() async { | |
| print('${green}Running smoketests...$reset'); | |
| // Verify that the tests actually return failure on failure and success on | |
| // success. | |
| final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); | |
| // We run the "pass" and "fail" smoke tests first, and alone, because those | |
| // are particularly critical and sensitive. If one of these fails, there's no | |
| // point even trying the others. | |
| await _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'pass_test.dart'), | |
| printOutput: false, | |
| ); | |
| await _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'fail_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ); | |
| // We run the timeout tests individually because they are timing-sensitive. | |
| await _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'timeout_pass_test.dart'), | |
| expectFailure: false, | |
| printOutput: false, | |
| ); | |
| await _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'timeout_fail_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ); | |
| await _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| outputChecker: (CapturedOutput output) { | |
| return output.stdout.contains('failingPendingTimerTest') | |
| ? null | |
| : 'Failed to find the stack trace for the pending Timer.'; | |
| } | |
| ); | |
| // We run the remaining smoketests in parallel, because they each take some | |
| // time to run (e.g. compiling), so we don't want to run them in series, | |
| // especially on 20-core machines... | |
| await Future.wait<void>( | |
| <Future<void>>[ | |
| _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'crash1_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ), | |
| _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'crash2_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ), | |
| _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ), | |
| _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'missing_import_test.broken_dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ), | |
| _runFlutterTest(automatedTests, | |
| script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'), | |
| expectFailure: true, | |
| printOutput: false, | |
| ), | |
| runCommand(flutter, | |
| <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')], | |
| workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'), | |
| expectNonZeroExit: true, | |
| outputMode: OutputMode.discard, | |
| ), | |
| ], | |
| ); | |
| // Verify that we correctly generated the version file. | |
| final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version'))); | |
| if (versionError != null) { | |
| print(redLine); | |
| print(versionError); | |
| print(redLine); | |
| exit(1); | |
| } | |
| } | |
| Future<bq.BigqueryApi> _getBigqueryApi() async { | |
| if (!useFlutterTestFormatter) { | |
| return null; | |
| } | |
| // TODO(dnfield): How will we do this on LUCI? | |
| final String privateKey = Platform.environment['GCLOUD_SERVICE_ACCOUNT_KEY']; | |
| // If we're on Cirrus and a non-collaborator is doing this, we can't get the key. | |
| if (privateKey == null || privateKey.isEmpty || privateKey.startsWith('ENCRYPTED[')) { | |
| return null; | |
| } | |
| try { | |
| final auth.ServiceAccountCredentials accountCredentials = auth.ServiceAccountCredentials( | |
| 'flutter-ci-test-reporter@flutter-infra.iam.gserviceaccount.com', | |
| auth.ClientId.serviceAccount('114390419920880060881.apps.googleusercontent.com'), | |
| '-----BEGIN PRIVATE KEY-----\n$privateKey\n-----END PRIVATE KEY-----\n', | |
| ); | |
| final List<String> scopes = <String>[bq.BigqueryApi.BigqueryInsertdataScope]; | |
| final http.Client client = await auth.clientViaServiceAccount(accountCredentials, scopes); | |
| return bq.BigqueryApi(client); | |
| } catch (e) { | |
| print('${red}Failed to get BigQuery API client.$reset'); | |
| print(e); | |
| return null; | |
| } | |
| } | |
| Future<void> _runToolCoverage() async { | |
| await runCommand( // Precompile tests to speed up subsequent runs. | |
| pub, | |
| <String>['run', 'build_runner', 'build'], | |
| workingDirectory: toolRoot, | |
| ); | |
| await runCommand( | |
| dart, | |
| <String>[path.join('tool', 'tool_coverage.dart')], | |
| workingDirectory: toolRoot, | |
| environment: <String, String>{ | |
| 'FLUTTER_ROOT': flutterRoot, | |
| } | |
| ); | |
| } | |
| Future<void> _runToolTests() async { | |
| final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); | |
| const String kDotShard = '.shard'; | |
| const String kTest = 'test'; | |
| final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools'); | |
| final Map<String, ShardRunner> subshards = Map<String, ShardRunner>.fromIterable( | |
| Directory(path.join(toolsPath, kTest)) | |
| .listSync() | |
| .map<String>((FileSystemEntity entry) => entry.path) | |
| .where((String name) => name.endsWith(kDotShard)) | |
| .map<String>((String name) => path.basenameWithoutExtension(name)), | |
| // The `dynamic` on the next line is because Map.fromIterable isn't generic. | |
| value: (dynamic subshard) => () async { | |
| await _pubRunTest( | |
| toolsPath, | |
| testPath: path.join(kTest, '$subshard$kDotShard'), | |
| useBuildRunner: canUseBuildRunner, | |
| tableData: bigqueryApi?.tabledata, | |
| enableFlutterToolAsserts: true, | |
| ); | |
| }, | |
| ); | |
| await selectSubshard(subshards); | |
| } | |
| /// Verifies that AOT, APK, and IPA (if on macOS) builds the examples apps | |
| /// without crashing. It does not actually launch the apps. That happens later | |
| /// in the devicelab. This is just a smoke-test. In particular, this will verify | |
| /// we can build when there are spaces in the path name for the Flutter SDK and | |
| /// target app. | |
| Future<void> _runBuildTests() async { | |
| final Stream<FileSystemEntity> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).list(); | |
| await for (FileSystemEntity fileEntity in exampleDirectories) { | |
| if (fileEntity is! Directory) { | |
| continue; | |
| } | |
| final String examplePath = fileEntity.path; | |
| await _flutterBuildAot(examplePath); | |
| await _flutterBuildApk(examplePath); | |
| if (Platform.isMacOS) { | |
| await _flutterBuildIpa(examplePath); | |
| } | |
| } | |
| // Web compilation tests. | |
| await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web'), path.join('lib', 'main.dart')); | |
| // Should not fail to compile with dart:io. | |
| await _flutterBuildDart2js(path.join('dev', 'integration_tests', 'web_compile_tests'), | |
| path.join('lib', 'dart_io_import.dart'), | |
| ); | |
| } | |
| Future<void> _flutterBuildAot(String relativePathToApplication) async { | |
| print('${green}Testing AOT build$reset for $cyan$relativePathToApplication$reset...'); | |
| await runCommand(flutter, | |
| <String>['build', 'aot', '-v'], | |
| workingDirectory: path.join(flutterRoot, relativePathToApplication), | |
| ); | |
| } | |
| Future<void> _flutterBuildApk(String relativePathToApplication) async { | |
| print('${green}Testing APK --debug build$reset for $cyan$relativePathToApplication$reset...'); | |
| await runCommand(flutter, | |
| <String>['build', 'apk', '--debug', '-v'], | |
| workingDirectory: path.join(flutterRoot, relativePathToApplication), | |
| ); | |
| } | |
| Future<void> _flutterBuildIpa(String relativePathToApplication) async { | |
| assert(Platform.isMacOS); | |
| print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...'); | |
| // Install Cocoapods. We don't have these checked in for the examples, | |
| // and build ios doesn't take care of it automatically. | |
| final File podfile = File(path.join(flutterRoot, relativePathToApplication, 'ios', 'Podfile')); | |
| if (podfile.existsSync()) { | |
| await runCommand('pod', | |
| <String>['install'], | |
| workingDirectory: podfile.parent.path, | |
| ); | |
| } | |
| await runCommand(flutter, | |
| <String>['build', 'ios', '--no-codesign', '--debug', '-v'], | |
| workingDirectory: path.join(flutterRoot, relativePathToApplication), | |
| ); | |
| } | |
| Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async { | |
| print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...'); | |
| await runCommand(flutter, | |
| <String>['build', 'web', '-v', '--target=$target'], | |
| workingDirectory: path.join(flutterRoot, relativePathToApplication), | |
| expectNonZeroExit: expectNonZeroExit, | |
| environment: <String, String>{ | |
| 'FLUTTER_WEB': 'true', | |
| }, | |
| ); | |
| } | |
| Future<void> _runAddToAppTests() async { | |
| if (Platform.isMacOS) { | |
| print('${green}Running add-to-app iOS integration tests$reset...'); | |
| final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app'); | |
| await runCommand('./build_and_test.sh', | |
| <String>[], | |
| workingDirectory: addToAppDir, | |
| ); | |
| } | |
| } | |
| Future<void> _runFrameworkTests() async { | |
| final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); | |
| Future<void> runWidgets() async { | |
| print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset'); | |
| await _runFlutterTest( | |
| path.join(flutterRoot, 'packages', 'flutter'), | |
| options: <String>['--track-widget-creation'], | |
| tableData: bigqueryApi?.tabledata, | |
| tests: <String>[ path.join('test', 'widgets') + path.separator ], | |
| ); | |
| await _runFlutterTest( | |
| path.join(flutterRoot, 'packages', 'flutter'), | |
| options: <String>['--no-track-widget-creation'], | |
| tableData: bigqueryApi?.tabledata, | |
| tests: <String>[ path.join('test', 'widgets') + path.separator ], | |
| ); | |
| // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--track-widget-creation'], tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: <String>['--no-track-widget-creation'], tableData: bigqueryApi?.tabledata); | |
| } | |
| Future<void> runLibraries() async { | |
| final List<String> tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test')) | |
| .listSync(followLinks: false, recursive: false) | |
| .whereType<Directory>() | |
| .where((Directory dir) => dir.path.endsWith('widgets') == false) | |
| .map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) | |
| .toList(); | |
| print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset'); | |
| await _runFlutterTest( | |
| path.join(flutterRoot, 'packages', 'flutter'), | |
| options: <String>['--track-widget-creation'], | |
| tableData: bigqueryApi?.tabledata, | |
| tests: tests, | |
| ); | |
| await _runFlutterTest( | |
| path.join(flutterRoot, 'packages', 'flutter'), | |
| options: <String>['--no-track-widget-creation'], | |
| tableData: bigqueryApi?.tabledata, | |
| tests: tests, | |
| ); | |
| } | |
| Future<void> runMisc() async { | |
| print('${green}Running package tests$reset for directories other than packages/flutter'); | |
| await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata); | |
| await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata); | |
| await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata); | |
| await _runFlutterTest( | |
| path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), | |
| tableData: bigqueryApi?.tabledata, | |
| environment: <String, String>{ | |
| 'FLUTTER_EXPERIMENTAL_BUILD': 'true', | |
| }, | |
| ); | |
| } | |
| await selectSubshard(<String, ShardRunner>{ | |
| 'widgets': runWidgets, | |
| 'libraries': runLibraries, | |
| 'misc': runMisc, | |
| }); | |
| } | |
| Future<void> _runFrameworkCoverage() async { | |
| final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info')); | |
| if (!coverageFile.existsSync()) { | |
| print('${red}Coverage file not found.$reset'); | |
| print('Expected to find: $cyan${coverageFile.absolute}$reset'); | |
| print('This file is normally obtained by running `${green}flutter update-packages$reset`.'); | |
| exit(1); | |
| } | |
| coverageFile.deleteSync(); | |
| await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), | |
| options: const <String>['--coverage'], | |
| ); | |
| if (!coverageFile.existsSync()) { | |
| print('${red}Coverage file not found.$reset'); | |
| print('Expected to find: $cyan${coverageFile.absolute}$reset'); | |
| print('This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.'); | |
| exit(1); | |
| } | |
| } | |
| Future<void> _runWebTests() async { | |
| final Map<String, ShardRunner> subshards = <String, ShardRunner>{}; | |
| final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter')); | |
| final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test')); | |
| final List<String> allTests = flutterPackageTestDirectory | |
| .listSync() | |
| .whereType<Directory>() | |
| .expand((Directory directory) => directory | |
| .listSync(recursive: true) | |
| .where((FileSystemEntity entity) => entity.path.endsWith('_test.dart')) | |
| ) | |
| .whereType<File>() | |
| .map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path)) | |
| .where((String filePath) => !kWebTestFileBlacklist.contains(filePath)) | |
| .toList() | |
| // Finally we shuffle the list because we want the average cost per file to be uniformly | |
| // distributed. If the list is not sorted then different shards and batches may have | |
| // very different characteristics. | |
| // We use a constant seed for repeatability. | |
| ..shuffle(math.Random(0)); | |
| assert(kWebShardCount >= 1); | |
| final int testsPerShard = (allTests.length / kWebShardCount).ceil(); | |
| assert(testsPerShard * kWebShardCount >= allTests.length); | |
| // This for loop computes all but the last shard. | |
| for (int index = 0; index < kWebShardCount - 1; index += 1) { | |
| subshards['$index'] = () => _runFlutterWebTest( | |
| flutterPackageDirectory.path, | |
| allTests.sublist( | |
| index * testsPerShard, | |
| (index + 1) * testsPerShard, | |
| ), | |
| ); | |
| } | |
| // The last shard also runs the flutter_web_plugins tests. | |
| // | |
| // We make sure the last shard ends in _last so it's easier to catch mismatches | |
| // between `.cirrus.yml` and `test.dart`. | |
| subshards['${kWebShardCount - 1}_last'] = () async { | |
| await _runFlutterWebTest( | |
| flutterPackageDirectory.path, | |
| allTests.sublist( | |
| (kWebShardCount - 1) * testsPerShard, | |
| allTests.length, | |
| ), | |
| ); | |
| await _runFlutterWebTest( | |
| path.join(flutterRoot, 'packages', 'flutter_web_plugins'), | |
| <String>['test'], | |
| ); | |
| }; | |
| await selectSubshard(subshards); | |
| } | |
| Future<void> _runFlutterWebTest(String workingDirectory, List<String> tests) async { | |
| final List<String> batch = <String>[]; | |
| for (int i = 0; i < tests.length; i += 1) { | |
| final String testFilePath = tests[i]; | |
| batch.add(testFilePath); | |
| if (batch.length == kWebBatchSize || i == tests.length - 1) { | |
| await runCommand( | |
| flutter, | |
| <String>[ | |
| 'test', | |
| if (ciProvider == CiProviders.cirrus) | |
| '--concurrency=1', // do not parallelize on Cirrus, to reduce flakiness | |
| '-v', | |
| '--platform=chrome', | |
| ...?flutterTestArgs, | |
| ...batch, | |
| ], | |
| workingDirectory: workingDirectory, | |
| environment: <String, String>{ | |
| 'FLUTTER_WEB': 'true', | |
| 'FLUTTER_LOW_RESOURCE_MODE': 'true', | |
| }, | |
| ); | |
| batch.clear(); | |
| } | |
| } | |
| } | |
| Future<void> _pubRunTest(String workingDirectory, { | |
| String testPath, | |
| bool enableFlutterToolAsserts = true, | |
| bool useBuildRunner = false, | |
| bq.TabledataResourceApi tableData, | |
| }) async { | |
| final List<String> args = <String>['run']; | |
| if (useBuildRunner) { | |
| final String posixTestPath = path.posix.joinAll(path.split(testPath)); | |
| args.addAll(<String>[ | |
| 'build_runner', | |
| 'test', | |
| '--build-filter=$posixTestPath/*.dill', | |
| '--build-filter=$posixTestPath/**/*.dill', | |
| '--', | |
| ]); | |
| } else { | |
| args.add('test'); | |
| } | |
| args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact'); | |
| int cpus; | |
| final String cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml | |
| if (cpuVariable != null) { | |
| cpus = int.tryParse(cpuVariable, radix: 10); | |
| if (cpus == null) { | |
| print('${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset'); | |
| print('Actual value: "$cpuVariable"'); | |
| exit(1); | |
| } | |
| } else { | |
| cpus = 2; // Don't default to 1, otherwise we won't catch race conditions. | |
| } | |
| args.add('-j$cpus'); | |
| if (!hasColor) | |
| args.add('--no-color'); | |
| if (testPath != null) | |
| args.add(testPath); | |
| final Map<String, String> pubEnvironment = <String, String>{ | |
| 'FLUTTER_ROOT': flutterRoot, | |
| }; | |
| if (Directory(pubCache).existsSync()) { | |
| pubEnvironment['PUB_CACHE'] = pubCache; | |
| } | |
| if (enableFlutterToolAsserts) { | |
| // If an existing env variable exists append to it, but only if | |
| // it doesn't appear to already include enable-asserts. | |
| String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? ''; | |
| if (!toolsArgs.contains('--enable-asserts')) | |
| toolsArgs += ' --enable-asserts'; | |
| pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim(); | |
| // The flutter_tool will originally have been snapshotted without asserts. | |
| // We need to force it to be regenerated with them enabled. | |
| deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot')); | |
| deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp')); | |
| } | |
| if (useFlutterTestFormatter) { | |
| final FlutterCompactFormatter formatter = FlutterCompactFormatter(); | |
| final Stream<String> testOutput = runAndGetStdout( | |
| pub, | |
| args, | |
| workingDirectory: workingDirectory, | |
| environment: pubEnvironment, | |
| beforeExit: formatter.finish, | |
| ); | |
| await _processTestOutput(formatter, testOutput, tableData); | |
| } else { | |
| await runCommand( | |
| pub, | |
| args, | |
| workingDirectory: workingDirectory, | |
| environment: pubEnvironment, | |
| removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, | |
| ); | |
| } | |
| } | |
| Future<void> _runFlutterTest(String workingDirectory, { | |
| String script, | |
| bool expectFailure = false, | |
| bool printOutput = true, | |
| OutputChecker outputChecker, | |
| List<String> options = const <String>[], | |
| bool skip = false, | |
| bq.TabledataResourceApi tableData, | |
| Map<String, String> environment, | |
| List<String> tests = const <String>[], | |
| }) async { | |
| assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); | |
| final List<String> args = <String>[ | |
| 'test', | |
| ...options, | |
| ...?flutterTestArgs, | |
| ]; | |
| final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage'); | |
| if (shouldProcessOutput) | |
| args.add('--machine'); | |
| if (script != null) { | |
| final String fullScriptPath = path.join(workingDirectory, script); | |
| if (!FileSystemEntity.isFileSync(fullScriptPath)) { | |
| print('${red}Could not find test$reset: $green$fullScriptPath$reset'); | |
| print('Working directory: $cyan$workingDirectory$reset'); | |
| print('Script: $green$script$reset'); | |
| if (!printOutput) | |
| print('This is one of the tests that does not normally print output.'); | |
| if (skip) | |
| print('This is one of the tests that is normally skipped in this configuration.'); | |
| exit(1); | |
| } | |
| args.add(script); | |
| } | |
| args.addAll(tests); | |
| if (!shouldProcessOutput) { | |
| OutputMode outputMode = OutputMode.discard; | |
| CapturedOutput output; | |
| if (outputChecker != null) { | |
| outputMode = OutputMode.capture; | |
| output = CapturedOutput(); | |
| } else if (printOutput) { | |
| outputMode = OutputMode.print; | |
| } | |
| await runCommand( | |
| flutter, | |
| args, | |
| workingDirectory: workingDirectory, | |
| expectNonZeroExit: expectFailure, | |
| outputMode: outputMode, | |
| output: output, | |
| skip: skip, | |
| environment: environment, | |
| ); | |
| if (outputChecker != null) { | |
| final String message = outputChecker(output); | |
| if (message != null) { | |
| print('$redLine'); | |
| print(message); | |
| print('$redLine'); | |
| exit(1); | |
| } | |
| } | |
| return; | |
| } | |
| if (useFlutterTestFormatter) { | |
| final FlutterCompactFormatter formatter = FlutterCompactFormatter(); | |
| final Stream<String> testOutput = runAndGetStdout( | |
| flutter, | |
| args, | |
| workingDirectory: workingDirectory, | |
| expectNonZeroExit: expectFailure, | |
| beforeExit: formatter.finish, | |
| environment: environment, | |
| ); | |
| await _processTestOutput(formatter, testOutput, tableData); | |
| } else { | |
| await runCommand( | |
| flutter, | |
| args, | |
| workingDirectory: workingDirectory, | |
| expectNonZeroExit: expectFailure, | |
| ); | |
| } | |
| } | |
| Map<String, String> _initGradleEnvironment() { | |
| final String androidSdkRoot = (Platform.environment['ANDROID_HOME']?.isEmpty ?? true) | |
| ? Platform.environment['ANDROID_SDK_ROOT'] | |
| : Platform.environment['ANDROID_HOME']; | |
| if (androidSdkRoot == null || androidSdkRoot.isEmpty) { | |
| print('${red}Could not find Android SDK; set ANDROID_SDK_ROOT (or ANDROID_HOME).$reset'); | |
| exit(1); | |
| } | |
| return <String, String>{ | |
| 'ANDROID_HOME': androidSdkRoot, | |
| 'ANDROID_SDK_ROOT': androidSdkRoot, | |
| }; | |
| } | |
| final Map<String, String> gradleEnvironment = _initGradleEnvironment(); | |
| Future<void> _runHostOnlyDeviceLabTests() async { | |
| // Please don't add more tests here. We should not be using the devicelab | |
| // logic to run tests outside devicelab, that's just confusing. | |
| // Instead, create tests that are not devicelab tests, and run those. | |
| // TODO(ianh): Move the tests that are not running on devicelab any more out | |
| // of the device lab directory. | |
| // List the tests to run. | |
| // We split these into subshards. The tests are randomly distributed into | |
| // those subshards so as to get a uniform distribution of costs, but the | |
| // seed is fixed so that issues are reproducible. | |
| final List<ShardRunner> tests = <ShardRunner>[ | |
| // Keep this in alphabetical order. | |
| () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment), | |
| if (Platform.isMacOS) () => _runDevicelabTest('flutter_create_offline_test_mac'), | |
| if (Platform.isLinux) () => _runDevicelabTest('flutter_create_offline_test_linux'), | |
| if (Platform.isWindows) () => _runDevicelabTest('flutter_create_offline_test_windows'), | |
| // TODO(ianh): Fails on macOS looking for "dexdump", https://github.com/flutter/flutter/issues/42494 | |
| if (!Platform.isMacOS) () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('gradle_non_android_plugin_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('gradle_plugin_bundle_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('gradle_plugin_fat_apk_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('gradle_r8_test', environment: gradleEnvironment), | |
| () => _runDevicelabTest('module_host_with_custom_build_test', environment: gradleEnvironment, testEmbeddingV2: true), | |
| () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true), | |
| // TODO(jmagman): Re-enable once flakiness is resolved, https://github.com/flutter/flutter/issues/37525 | |
| // if (Platform.isMacOS) () => _runDevicelabTest('module_test_ios'), | |
| if (Platform.isMacOS) () => _runDevicelabTest('build_ios_framework_module_test'), | |
| if (Platform.isMacOS) () => _runDevicelabTest('plugin_lint_mac'), | |
| () => _runDevicelabTest('plugin_test', environment: gradleEnvironment), | |
| ]..shuffle(math.Random(0)); | |
| final int testsPerShard = tests.length ~/ kDeviceLabShardCount; | |
| final Map<String, ShardRunner> subshards = <String, ShardRunner>{}; | |
| for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) { | |
| String last = ''; | |
| List<ShardRunner> sublist; | |
| if (subshard < kDeviceLabShardCount - 1) { | |
| sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard); | |
| } else { | |
| sublist = tests.sublist(subshard * testsPerShard, tests.length); | |
| // We make sure the last shard ends in _last so it's easier to catch mismatches | |
| // between `.cirrus.yml` and `test.dart`. | |
| last = '_last'; | |
| } | |
| subshards['$subshard$last'] = () async { | |
| for (ShardRunner test in sublist) | |
| await test(); | |
| }; | |
| } | |
| await selectSubshard(subshards); | |
| } | |
| Future<void> _runDevicelabTest(String testName, { | |
| Map<String, String> environment, | |
| // testEmbeddingV2 is only supported by certain specific devicelab tests. | |
| // Don't use it unless you're sure the test actually supports it. | |
| // You can check by looking to see if the test examines the environment | |
| // for the ENABLE_ANDROID_EMBEDDING_V2 variable. | |
| bool testEmbeddingV2 = false, | |
| }) async { | |
| await runCommand( | |
| dart, | |
| <String>['bin/run.dart', '-t', testName], | |
| workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'), | |
| environment: <String, String>{ | |
| ...?environment, | |
| if (testEmbeddingV2) | |
| 'ENABLE_ANDROID_EMBEDDING_V2': 'true', | |
| }, | |
| ); | |
| } | |
| void deleteFile(String path) { | |
| // This is technically a race condition but nobody else should be running | |
| // while this script runs, so we should be ok. (Sadly recursive:true does not | |
| // obviate the need for existsSync, at least on Windows.) | |
| final File file = File(path); | |
| if (file.existsSync()) | |
| file.deleteSync(); | |
| } | |
| enum CiProviders { | |
| cirrus, | |
| luci, | |
| } | |
| Future<void> _processTestOutput( | |
| FlutterCompactFormatter formatter, | |
| Stream<String> testOutput, | |
| bq.TabledataResourceApi tableData, | |
| ) async { | |
| final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) { | |
| print('Processing...'); | |
| }); | |
| await testOutput.forEach(formatter.processRawOutput); | |
| heartbeat.cancel(); | |
| formatter.finish(); | |
| if (tableData == null || formatter.tests.isEmpty) { | |
| return; | |
| } | |
| final bq.TableDataInsertAllRequest request = bq.TableDataInsertAllRequest(); | |
| final String authors = await _getAuthors(); | |
| request.rows = List<bq.TableDataInsertAllRequestRows>.from( | |
| formatter.tests.map<bq.TableDataInsertAllRequestRows>((TestResult result) => | |
| bq.TableDataInsertAllRequestRows.fromJson(<String, dynamic> { | |
| 'json': <String, dynamic>{ | |
| 'source': <String, dynamic>{ | |
| 'provider': ciProviderName, | |
| 'url': ciUrl, | |
| 'platform': <String, dynamic>{ | |
| 'os': Platform.operatingSystem, | |
| 'version': Platform.operatingSystemVersion, | |
| }, | |
| }, | |
| 'test': <String, dynamic>{ | |
| 'name': result.name, | |
| 'result': result.status.toString(), | |
| 'file': result.path, | |
| 'line': result.line, | |
| 'column': result.column, | |
| 'time': result.totalTime, | |
| }, | |
| 'git': <String, dynamic>{ | |
| 'author': authors, | |
| 'pull_request': prNumber, | |
| 'commit': gitHash, | |
| 'organization': 'flutter', | |
| 'repository': 'flutter', | |
| }, | |
| 'error': result.status != TestStatus.failed ? null : <String, dynamic>{ | |
| 'message': result.errorMessage, | |
| 'stack_trace': result.stackTrace, | |
| }, | |
| 'information': result.messages, | |
| }, | |
| }), | |
| ), | |
| growable: false, | |
| ); | |
| final bq.TableDataInsertAllResponse response = await tableData.insertAll(request, 'flutter-infra', 'tests', 'ci'); | |
| if (response.insertErrors != null && response.insertErrors.isNotEmpty) { | |
| print('${red}BigQuery insert errors:'); | |
| print(response.toJson()); | |
| print(reset); | |
| } | |
| } | |
| CiProviders get ciProvider { | |
| if (Platform.environment['CIRRUS_CI'] == 'true') { | |
| return CiProviders.cirrus; | |
| } | |
| if (Platform.environment['LUCI_CONTEXT'] != null) { | |
| return CiProviders.luci; | |
| } | |
| return null; | |
| } | |
| String get ciProviderName { | |
| switch (ciProvider) { | |
| case CiProviders.cirrus: | |
| return 'cirrusci'; | |
| case CiProviders.luci: | |
| return 'luci'; | |
| } | |
| return 'unknown'; | |
| } | |
| int get prNumber { | |
| switch (ciProvider) { | |
| case CiProviders.cirrus: | |
| return Platform.environment['CIRRUS_PR'] == null | |
| ? -1 | |
| : int.tryParse(Platform.environment['CIRRUS_PR']); | |
| case CiProviders.luci: | |
| return -1; // LUCI doesn't know about this. | |
| } | |
| return -1; | |
| } | |
| Future<String> _getAuthors() async { | |
| final String exe = Platform.isWindows ? '.exe' : ''; | |
| final String author = await runAndGetStdout( | |
| 'git$exe', <String>['-c', 'log.showSignature=false', 'log', gitHash, '--pretty="%an <%ae>"'], | |
| workingDirectory: flutterRoot, | |
| ).first; | |
| return author; | |
| } | |
| String get ciUrl { | |
| switch (ciProvider) { | |
| case CiProviders.cirrus: | |
| return 'https://cirrus-ci.com/task/${Platform.environment['CIRRUS_TASK_ID']}'; | |
| case CiProviders.luci: | |
| return 'https://ci.chromium.org/p/flutter/g/framework/console'; // TODO(dnfield): can we get a direct link to the actual build? | |
| } | |
| return ''; | |
| } | |
| String get gitHash { | |
| switch(ciProvider) { | |
| case CiProviders.cirrus: | |
| return Platform.environment['CIRRUS_CHANGE_IN_REPO']; | |
| case CiProviders.luci: | |
| return 'HEAD'; // TODO(dnfield): Set this in the env for LUCI. | |
| } | |
| return ''; | |
| } | |
| /// Checks the given file's contents to determine if they match the allowed | |
| /// pattern for version strings. | |
| /// | |
| /// Returns null if the contents are good. Returns a string if they are bad. | |
| /// The string is an error message. | |
| Future<String> verifyVersion(File file) async { | |
| final RegExp pattern = RegExp(r'^\d+\.\d+\.\d+(\+hotfix\.\d+)?(-pre\.\d+)?$'); | |
| final String version = await file.readAsString(); | |
| if (!file.existsSync()) | |
| return 'The version logic failed to create the Flutter version file.'; | |
| if (version == '0.0.0-unknown') | |
| return 'The version logic failed to determine the Flutter version.'; | |
| if (!version.contains(pattern)) | |
| return 'The version logic generated an invalid version string: "$version".'; | |
| return null; | |
| } | |
| /// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine | |
| /// the shard and subshard (parsing it in the form shard-subshard-platform, ignoring | |
| /// the platform). | |
| /// | |
| /// However, for local testing you can just set the SHARD and SUBSHARD | |
| /// environment variables. For example, to run all the framework tests you can | |
| /// just set SHARD=framework_tests. To run specifically the third subshard of | |
| /// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based). | |
| Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, 'SHARD', 'shard', 0); | |
| Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard', 1); | |
| const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; | |
| Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async { | |
| String item = Platform.environment[key]; | |
| if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) { | |
| final List<String> parts = Platform.environment[CIRRUS_TASK_NAME].split('-'); | |
| assert(positionInTaskName < parts.length); | |
| item = parts[positionInTaskName]; | |
| } | |
| if (item == null) { | |
| for (String currentItem in items.keys) { | |
| print('$bold$key=$currentItem$reset'); | |
| await items[currentItem](); | |
| print(''); | |
| } | |
| } else { | |
| if (!items.containsKey(item)) { | |
| print('${red}Invalid $name: $item$reset'); | |
| print('The available ${name}s are: ${items.keys.join(", ")}'); | |
| exit(1); | |
| } | |
| print('$bold$key=$item$reset'); | |
| await items[item](); | |
| } | |
| } |