blob: c8aef8f09df4a96d29d9a0e9ae409f8bd382d1a3 [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// @dart = 2.8
// TODO(gspencergoog): Remove this tag once this test's state leaks/test
// dependencies have been fixed.
// https://github.com/flutter/flutter/issues/85160
// Fails with "flutter test --test-randomize-ordering-seed=1000"
@Tags(<String>['no-shuffle'])
import 'dart:async';
import 'dart:convert';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/base/io.dart';
import '../src/common.dart';
import 'test_utils.dart';
// This test depends on some files in ///dev/automated_tests/flutter_test/*
final String automatedTestsDirectory = fileSystem.path.join('..', '..', 'dev', 'automated_tests');
final String missingDependencyDirectory = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
final String flutterTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'flutter_test');
final String integrationTestDirectory = fileSystem.path.join(automatedTestsDirectory, 'integration_test');
final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', platform.isWindows ? 'flutter.bat' : 'flutter');
// Running Integration Tests in the Flutter Tester will still exercise the same
// flows specific to Integration Tests.
final List<String> integrationTestExtraArgs = <String>['-d', 'flutter-tester'];
void main() {
setUpAll(() async {
await processManager.run(
<String>[
flutterBin,
'pub',
'get'
],
workingDirectory: flutterTestDirectory
);
await processManager.run(
<String>[
flutterBin,
'pub',
'get'
],
workingDirectory: missingDependencyDirectory
);
});
testWithoutContext('flutter test should not have extraneous error messages', () async {
return _testFile('trivial_widget', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
});
testWithoutContext('integration test should not have extraneous error messages', () async {
return _testFile('trivial_widget', automatedTestsDirectory, integrationTestDirectory, exitCode: isZero, extraArguments: integrationTestExtraArgs);
});
testWithoutContext('flutter test set the working directory correctly', () async {
return _testFile('working_directory', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
});
testWithoutContext('flutter test should report nice errors for exceptions thrown within testWidgets()', () async {
return _testFile('exception_handling', automatedTestsDirectory, flutterTestDirectory);
});
testWithoutContext('integration test should report nice errors for exceptions thrown within testWidgets()', () async {
return _testFile('exception_handling', automatedTestsDirectory, integrationTestDirectory, extraArguments: integrationTestExtraArgs);
});
testWithoutContext('flutter test should report a nice error when a guarded function was called without await', () async {
return _testFile('test_async_utils_guarded', automatedTestsDirectory, flutterTestDirectory);
});
testWithoutContext('flutter test should report a nice error when an async function was called without await', () async {
return _testFile('test_async_utils_unguarded', automatedTestsDirectory, flutterTestDirectory);
});
testWithoutContext('flutter test should report a nice error when a Ticker is left running', () async {
return _testFile('ticker', automatedTestsDirectory, flutterTestDirectory);
});
testWithoutContext('flutter test should report a nice error when a pubspec.yaml is missing a flutter_test dependency', () async {
final String missingDependencyTests = fileSystem.path.join('..', '..', 'dev', 'missing_dependency_tests');
return _testFile('trivial', missingDependencyTests, missingDependencyTests);
});
testWithoutContext('flutter test should report which user-created widget caused the error', () async {
return _testFile('print_user_created_ancestor', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--track-widget-creation']);
});
testWithoutContext('flutter test should report which user-created widget caused the error - no flag', () async {
return _testFile('print_user_created_ancestor_no_flag', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--no-track-widget-creation']);
});
testWithoutContext('flutter test should report the correct user-created widget that caused the error', () async {
return _testFile('print_correct_local_widget', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--track-widget-creation']);
});
testWithoutContext('flutter test should can load assets within its own package', () async {
return _testFile('package_assets', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
});
testWithoutContext('flutter test should support dart defines', () async {
return _testFile('dart_defines', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero,
extraArguments: <String>['--dart-define=flutter.test.foo=bar']);
});
testWithoutContext('flutter test should run a test when its name matches a regexp', () async {
final ProcessResult result = await _runFlutterTest('filtering', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--name', 'inc.*de']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should run a test when its name contains a string', () async {
final ProcessResult result = await _runFlutterTest('filtering', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--plain-name', 'include']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should run a test with a given tag', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--tags', 'include-tag']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should not run a test with excluded tag', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should run all tests when tags are unspecified', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag', automatedTestsDirectory, flutterTestDirectory);
expect(result.stdout, contains(RegExp(r'\+\d+ -1: Some tests failed\.')));
expect(result.exitCode, 1);
});
testWithoutContext('flutter test should run a widgetTest with a given tag', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--tags', 'include-tag']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should not run a widgetTest with excluded tag', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--exclude-tags', 'exclude-tag']);
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed!')));
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should run all widgetTest when tags are unspecified', () async {
final ProcessResult result = await _runFlutterTest('filtering_tag_widget', automatedTestsDirectory, flutterTestDirectory);
expect(result.stdout, contains(RegExp(r'\+\d+ -1: Some tests failed\.')));
expect(result.exitCode, 1);
});
testWithoutContext('flutter test should test runs to completion', () async {
final ProcessResult result = await _runFlutterTest('trivial', automatedTestsDirectory, flutterTestDirectory,
extraArguments: const <String>['--verbose']);
final String stdout = (result.stdout as String).replaceAll('\r', '\n');
expect(stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
expect(stdout, contains('test 0: Starting flutter_tester process with command'));
expect(stdout, contains('test 0: deleting temporary directory'));
expect(stdout, contains('test 0: finished'));
expect(stdout, contains('test package returned with exit code 0'));
if ((result.stderr as String).isNotEmpty) {
fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
}
expect(result.exitCode, 0);
});
testWithoutContext('flutter test should run all tests inside of a directory with no trailing slash', () async {
final ProcessResult result = await _runFlutterTest(null, automatedTestsDirectory, '$flutterTestDirectory/child_directory',
extraArguments: const <String>['--verbose']);
final String stdout = (result.stdout as String).replaceAll('\r', '\n');
expect(result.stdout, contains(RegExp(r'\+\d+: All tests passed\!')));
expect(stdout, contains('test 0: Starting flutter_tester process with command'));
expect(stdout, contains('test 0: deleting temporary directory'));
expect(stdout, contains('test 0: finished'));
expect(stdout, contains('test package returned with exit code 0'));
if ((result.stderr as String).isNotEmpty) {
fail('unexpected error output from test:\n\n${result.stderr}\n-- end stderr --\n\n');
}
expect(result.exitCode, 0);
});
testWithoutContext('flutter gold skips tests where the expectations are missing', () async {
return _testFile('flutter_gold', automatedTestsDirectory, flutterTestDirectory, exitCode: isZero);
});
}
Future<void> _testFile(
String testName,
String workingDirectory,
String testDirectory, {
Matcher exitCode,
List<String> extraArguments = const <String>[],
}) async {
exitCode ??= isNonZero;
final String fullTestExpectation = fileSystem.path.join(testDirectory, '${testName}_expectation.txt');
final File expectationFile = fileSystem.file(fullTestExpectation);
if (!expectationFile.existsSync()) {
fail('missing expectation file: $expectationFile');
}
final ProcessResult exec = await _runFlutterTest(
testName,
workingDirectory,
testDirectory,
extraArguments: extraArguments,
);
expect(exec.exitCode, exitCode);
final List<String> output = (exec.stdout as String).split('\n');
if (output.first.startsWith('Waiting for another flutter command to release the startup lock...')) {
output.removeAt(0);
}
if (output.first.startsWith('Running "flutter pub get" in')) {
output.removeAt(0);
}
// Whether cached artifacts need to be downloaded is dependent on what
// previous tests have run. Disregard these messages.
output.removeWhere(RegExp(r'Downloading .*\.\.\.').hasMatch);
output.add('<<stderr>>');
output.addAll((exec.stderr as String).split('\n'));
final List<String> expectations = fileSystem.file(fullTestExpectation).readAsLinesSync();
bool allowSkip = false;
int expectationLineNumber = 0;
int outputLineNumber = 0;
bool haveSeenStdErrMarker = false;
while (expectationLineNumber < expectations.length) {
expect(
output,
hasLength(greaterThan(outputLineNumber)),
reason: 'Failure in $testName to compare to $fullTestExpectation',
);
final String expectationLine = expectations[expectationLineNumber];
String outputLine = output[outputLineNumber];
if (expectationLine == '<<skip until matching line>>') {
allowSkip = true;
expectationLineNumber += 1;
continue;
}
if (allowSkip) {
if (!RegExp(expectationLine).hasMatch(outputLine)) {
outputLineNumber += 1;
continue;
}
allowSkip = false;
}
if (expectationLine == '<<stderr>>') {
expect(haveSeenStdErrMarker, isFalse);
haveSeenStdErrMarker = true;
}
if (!RegExp(expectationLine).hasMatch(outputLine) && outputLineNumber + 1 < output.length) {
// Check if the RegExp can match the next two lines in the output so
// that it is possible to write expectations that still hold even if a
// line is wrapped slightly differently due to for example a file name
// being longer on one platform than another.
final String mergedLines = '$outputLine\n${output[outputLineNumber+1]}';
if (RegExp(expectationLine).hasMatch(mergedLines)) {
outputLineNumber += 1;
outputLine = mergedLines;
}
}
expect(outputLine, matches(expectationLine), reason: 'Full output:\n- - - -----8<----- - - -\n${output.join("\n")}\n- - - -----8<----- - - -');
expectationLineNumber += 1;
outputLineNumber += 1;
}
expect(allowSkip, isFalse);
if (!haveSeenStdErrMarker) {
expect(exec.stderr, '');
}
}
Future<ProcessResult> _runFlutterTest(
String testName,
String workingDirectory,
String testDirectory, {
List<String> extraArguments = const <String>[],
}) async {
String testPath;
if (testName == null) {
// Test everything in the directory.
testPath = testDirectory;
final Directory directoryToTest = fileSystem.directory(testPath);
if (!directoryToTest.existsSync()) {
fail('missing test directory: $directoryToTest');
}
} else {
// Test just a specific test file.
testPath = fileSystem.path.join(testDirectory, '${testName}_test.dart');
final File testFile = fileSystem.file(testPath);
if (!testFile.existsSync()) {
fail('missing test file: $testFile');
}
}
final List<String> args = <String>[
'test',
'--no-color',
'--no-version-check',
'--no-pub',
...extraArguments,
testPath,
];
return Process.run(
flutterBin, // Uses the precompiled flutter tool for faster tests,
args,
workingDirectory: workingDirectory,
stdoutEncoding: utf8,
stderrEncoding: utf8,
);
}