blob: 3b8a17c0a787482f70972d4052659a055bd44493 [file] [log] [blame] [edit]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
// --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
// --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
// --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.
import 'dart:convert';
import 'dart:core' as system show print;
import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit;
import 'dart:math' as math;
import 'package:path/path.dart' as path;
import 'run_command.dart';
import 'suite_runners/run_add_to_app_life_cycle_tests.dart';
import 'suite_runners/run_analyze_tests.dart';
import 'suite_runners/run_android_java11_integration_tool_tests.dart';
import 'suite_runners/run_android_preview_integration_tool_tests.dart';
import 'suite_runners/run_customer_testing_tests.dart';
import 'suite_runners/run_docs_tests.dart';
import 'suite_runners/run_flutter_driver_android_tests.dart';
import 'suite_runners/run_flutter_packages_tests.dart';
import 'suite_runners/run_framework_coverage_tests.dart';
import 'suite_runners/run_framework_tests.dart';
import 'suite_runners/run_fuchsia_precache.dart';
import 'suite_runners/run_realm_checker_tests.dart';
import 'suite_runners/run_skp_generator_tests.dart';
import 'suite_runners/run_test_harness_tests.dart';
import 'suite_runners/run_verify_binaries_codesigned_tests.dart';
import 'suite_runners/run_web_tests.dart';
import 'utils.dart';
typedef ShardRunner = Future<void> Function();
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};
const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
/// When you call this, you can pass additional arguments to pass custom
/// arguments to flutter test. For example, you might want to call this
/// script with the parameter --local-engine=host_debug_unopt to
/// use your own build of the engine.
///
/// To run the tool_tests part, run it with SHARD=tool_tests
///
/// Examples:
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt --local-engine-host=host_debug_unopt
Future<void> main(List<String> args) async {
try {
printProgress('STARTING ANALYSIS');
for (final String arg in args) {
if (arg.startsWith('--local-engine=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--local-engine-host=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE_HOST'] = arg.substring('--local-engine-host='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--local-engine-src-path=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--test-randomize-ordering-seed=')) {
shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
} else if (arg.startsWith('--verbose')) {
print = (Object? message) {
system.print(message);
};
} else if (arg.startsWith('--abort-on-error')) {
onError = () {
system.exit(1);
};
} else {
flutterTestArgs.add(arg);
}
}
if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
}
final WebTestsSuite webTestsSuite = WebTestsSuite(flutterTestArgs);
await selectShard(<String, ShardRunner>{
'add_to_app_life_cycle_tests': addToAppLifeCycleRunner,
'build_tests': _runBuildTests,
'framework_coverage': frameworkCoverageRunner,
'framework_tests': frameworkTestsRunner,
'tool_tests': _runToolTests,
'web_tool_tests': _runWebToolTests,
'tool_integration_tests': _runIntegrationToolTests,
'android_preview_tool_integration_tests': androidPreviewIntegrationToolTestsRunner,
'android_java11_tool_integration_tests': androidJava11IntegrationToolTestsRunner,
'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
'web_tests': webTestsSuite.runWebHtmlUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
'web_canvaskit_tests': webTestsSuite.runWebCanvasKitUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --wasm --web-renderer=skwasm`
'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests,
// All web integration tests
'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner,
'flutter_driver_android': runFlutterDriverAndroidTests,
'flutter_plugins': flutterPackagesRunner,
'skp_generator': skpGeneratorTestsRunner,
'realm_checker': realmCheckerTestRunner,
'customer_testing': customerTestingRunner,
'analyze': analyzeRunner,
'fuchsia_precache': fuchsiaPrecacheRunner,
'snippets': _runSnippetsTests,
'docs': docsRunner,
'verify_binaries_codesigned': verifyCodesignedTestRunner,
kTestHarnessShardName: testHarnessTestsRunner, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
});
} catch (error, stackTrace) {
foundError(<String>[
'UNEXPECTED ERROR!',
error.toString(),
...stackTrace.toString().split('\n'),
'The test.dart script should be corrected to catch this error and call foundError().',
'${yellow}Some tests are likely to have been skipped.$reset',
]);
system.exit(255);
}
if (hasError) {
reportErrorsAndExit('${bold}Test failed.$reset');
}
reportSuccessAndExit('${bold}Test successful.$reset');
}
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
Future<void> _runGeneralToolTests() async {
await runDartTest(
_toolsPath,
testPaths: <String>[path.join('test', 'general.shard')],
enableFlutterToolAsserts: false,
// Detect unit test time regressions (poor time delay handling, etc).
// This overrides the 15 minute default for tools tests.
// See the README.md and dart_test.yaml files in the flutter_tools package.
perTestTimeout: const Duration(seconds: 2),
);
}
Future<void> _runCommandsToolTests() async {
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: <String>[path.join('test', 'commands.shard')],
);
}
Future<void> _runWebToolTests() async {
final List<File> allFiles = Directory(path.join(_toolsPath, 'test', 'web.shard'))
.listSync(recursive: true).whereType<File>().toList();
final List<String> allTests = <String>[];
for (final File file in allFiles) {
if (file.path.endsWith('_test.dart')) {
allTests.add(file.path);
}
}
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: selectIndexOfTotalSubshard<String>(allTests),
includeLocalEngineEnv: true,
);
}
Future<void> _runToolHostCrossArchTests() {
return runDartTest(
_toolsPath,
// These are integration tests
forceSingleCore: true,
testPaths: <String>[path.join('test', 'host_cross_arch.shard')],
);
}
Future<void> _runIntegrationToolTests() async {
final List<String> allTests = Directory(path.join(_toolsPath, 'test', 'integration.shard'))
.listSync(recursive: true).whereType<File>()
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: selectIndexOfTotalSubshard<String>(allTests),
collectMetrics: true,
);
}
Future<void> _runToolTests() async {
await selectSubshard(<String, ShardRunner>{
'general': _runGeneralToolTests,
'commands': _runCommandsToolTests,
});
}
Future<void> _runSnippetsTests() async {
final String snippetsPath = path.join(flutterRoot, 'dev', 'snippets');
final List<String> allTests = Directory(path.join(snippetsPath, 'test'))
.listSync(recursive: true).whereType<File>()
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await runDartTest(
snippetsPath,
forceSingleCore: true,
testPaths: selectIndexOfTotalSubshard<String>(allTests),
collectMetrics: true,
);
}
Future<void> runForbiddenFromReleaseTests() async {
// Build a release APK to get the snapshot json.
final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_forbidden_imports.');
final List<String> command = <String>[
'build',
'apk',
'--target-platform',
'android-arm64',
'--release',
'--analyze-size',
'--code-size-directory',
tempDirectory.path,
'-v',
];
await runCommand(
flutter,
command,
workingDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
);
// First, a smoke test.
final List<String> smokeTestArgs = <String>[
path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
'--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
'--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
'--forbidden-type', 'package:flutter/src/widgets/framework.dart::Widget',
];
await runCommand(
dart,
smokeTestArgs,
workingDirectory: flutterRoot,
expectNonZeroExit: true,
);
// Actual test.
final List<String> args = <String>[
path.join(flutterRoot, 'dev', 'forbidden_from_release_tests', 'bin', 'main.dart'),
'--snapshot', path.join(tempDirectory.path, 'snapshot.arm64-v8a.json'),
'--package-config', path.join(flutterRoot, 'examples', 'hello_world', '.dart_tool', 'package_config.json'),
'--forbidden-type', 'package:flutter/src/widgets/widget_inspector.dart::WidgetInspectorService',
'--forbidden-type', 'package:flutter/src/widgets/framework.dart::DebugCreator',
'--forbidden-type', 'package:flutter/src/foundation/print.dart::debugPrint',
];
await runCommand(
dart,
args,
workingDirectory: flutterRoot,
);
}
/// Verifies that APK, and IPA (if on macOS), and native desktop builds the
/// examples apps without crashing. It does not actually launch the apps. That
/// happens later in the devicelab. This is just a smoke-test. In particular,
/// this will verify we can build when there are spaces in the path name for the
/// Flutter SDK and target app.
///
/// Also does some checking about types included in hello_world.
Future<void> _runBuildTests() async {
final List<Directory> exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync()
// API example builds will be tested in a separate shard.
.where((FileSystemEntity entity) => entity is Directory && path.basename(entity.path) != 'api').cast<Directory>().toList()
..add(Directory(path.join(flutterRoot, 'packages', 'integration_test', 'example')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'android_views')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'channels')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'hybrid_android_views')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_platform_view_tests')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ios_app_with_extensions')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'platform_interaction')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'spell_check')))
..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')));
// The tests are randomly distributed into subshards so as to get a uniform
// distribution of costs, but the seed is fixed so that issues are reproducible.
final List<ShardRunner> tests = <ShardRunner>[
for (final Directory exampleDirectory in exampleDirectories)
() => _runExampleProjectBuildTests(exampleDirectory),
...<ShardRunner>[
// Web compilation tests.
() => _flutterBuildDart2js(
path.join('dev', 'integration_tests', 'web'),
path.join('lib', 'main.dart'),
),
// Should not fail to compile with dart:io.
() => _flutterBuildDart2js(
path.join('dev', 'integration_tests', 'web_compile_tests'),
path.join('lib', 'dart_io_import.dart'),
),
// Should be able to compile with a call to:
// BackgroundIsolateBinaryMessenger.ensureInitialized.
() => _flutterBuildDart2js(
path.join('dev', 'integration_tests', 'web_compile_tests'),
path.join('lib', 'background_isolate_binary_messenger.dart'),
),
],
runForbiddenFromReleaseTests,
]..shuffle(math.Random(0));
await runShardRunnerIndexOfTotalSubshard(tests);
}
Future<void> _runExampleProjectBuildTests(Directory exampleDirectory, [File? mainFile]) async {
// Only verify caching with flutter gallery.
final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery');
final String examplePath = path.relative(exampleDirectory.path, from: Directory.current.path);
final List<String> additionalArgs = <String>[
if (mainFile != null) path.relative(mainFile.path, from: exampleDirectory.absolute.path),
];
if (Directory(path.join(examplePath, 'android')).existsSync()) {
await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
} else {
print('Example project ${path.basename(examplePath)} has no android directory, skipping apk');
}
if (Platform.isMacOS) {
if (Directory(path.join(examplePath, 'ios')).existsSync()) {
await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
} else {
print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa');
}
}
if (Platform.isLinux) {
if (Directory(path.join(examplePath, 'linux')).existsSync()) {
await _flutterBuildLinux(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
await _flutterBuildLinux(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
} else {
print('Example project ${path.basename(examplePath)} has no linux directory, skipping Linux');
}
}
if (Platform.isMacOS) {
if (Directory(path.join(examplePath, 'macos')).existsSync()) {
await _flutterBuildMacOS(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
await _flutterBuildMacOS(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
} else {
print('Example project ${path.basename(examplePath)} has no macos directory, skipping macOS');
}
}
if (Platform.isWindows) {
if (Directory(path.join(examplePath, 'windows')).existsSync()) {
await _flutterBuildWin32(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
await _flutterBuildWin32(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching);
} else {
print('Example project ${path.basename(examplePath)} has no windows directory, skipping Win32');
}
}
}
Future<void> _flutterBuildApk(String relativePathToApplication, {
required bool release,
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'APK', 'apk',
release: release,
verifyCaching: verifyCaching,
additionalArgs: additionalArgs
);
}
Future<void> _flutterBuildIpa(String relativePathToApplication, {
required bool release,
List<String> additionalArgs = const <String>[],
bool verifyCaching = false,
}) async {
assert(Platform.isMacOS);
printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
release: release,
verifyCaching: verifyCaching,
additionalArgs: <String>[...additionalArgs, '--no-codesign'],
);
}
Future<void> _flutterBuildLinux(String relativePathToApplication, {
required bool release,
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
assert(Platform.isLinux);
await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
release: release,
verifyCaching: verifyCaching,
additionalArgs: additionalArgs
);
}
Future<void> _flutterBuildMacOS(String relativePathToApplication, {
required bool release,
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
assert(Platform.isMacOS);
await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
release: release,
verifyCaching: verifyCaching,
additionalArgs: additionalArgs
);
}
Future<void> _flutterBuildWin32(String relativePathToApplication, {
required bool release,
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
assert(Platform.isWindows);
printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
release: release,
verifyCaching: verifyCaching,
additionalArgs: additionalArgs
);
}
Future<void> _flutterBuild(
String relativePathToApplication,
String platformLabel,
String platformBuildName, {
required bool release,
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
await runCommand(flutter,
<String>[
'build',
platformBuildName,
...additionalArgs,
if (release)
'--release'
else
'--debug',
'-v',
],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
);
if (verifyCaching) {
printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter,
<String>[
'build',
platformBuildName,
'--performance-measurement-file=perf.json',
...additionalArgs,
if (release)
'--release'
else
'--debug',
'-v',
],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
);
final File file = File(path.join(flutterRoot, relativePathToApplication, 'perf.json'));
if (!_allTargetsCached(file)) {
foundError(<String>[
'${red}Not all build targets cached after second run.$reset',
'The target performance data was: ${file.readAsStringSync().replaceAll('},', '},\n')}',
]);
}
}
}
bool _allTargetsCached(File performanceFile) {
final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
as Map<String, Object?>;
final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)
.cast<Map<String, Object?>>();
return targets.every((Map<String, Object?> element) => element['skipped'] == true);
}
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter,
<String>['build', 'web', '-v', '--target=$target'],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
expectNonZeroExit: expectNonZeroExit,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
}