Give `_runFlutterTest` the ability to validate command output
In another change (#37646), I want to test that a test fails and
prints expected output. I didn't see an existing way to do that, so
I modified `_runFlutterTest` and `runCommand` to allow capturing the
output. Currently capturing and printing output are mutually
exclusive since we don't need both.
Some awkward bits:
* There already exists a `runAndGetStdout` function that is very
similar to `runCommand`, and this change makes the conceptual
distinction more confusing.
* `runFlutterTest` has multiple code paths for different
configurations. I don't understand what the different paths are
for, and I added output checking only along one of them.
diff --git a/dev/bots/analyze.dart b/dev/bots/analyze.dart
index 138da9b..31a321d 100644
--- a/dev/bots/analyze.dart
+++ b/dev/bots/analyze.dart
@@ -585,7 +585,7 @@
}
await runCommand(flutter, <String>['inject-plugins'],
workingDirectory: package,
- printOutput: false,
+ outputMode: OutputMode.discard,
);
for (File registrant in fileToContent.keys) {
if (registrant.readAsStringSync() != fileToContent[registrant]) {
diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart
index 507ce59..d96abb1 100644
--- a/dev/bots/run_command.dart
+++ b/dev/bots/run_command.dart
@@ -83,12 +83,15 @@
bool expectNonZeroExit = false,
int expectedExitCode,
String failureMessage,
- bool printOutput = true,
+ OutputMode outputMode = OutputMode.print,
+ CapturedOutput output,
bool skip = false,
bool expectFlaky = false,
Duration timeout = _kLongTimeout,
bool Function(String) removeLine,
}) async {
+ assert((outputMode == OutputMode.capture) == (output != null));
+
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory);
if (skip) {
@@ -110,7 +113,7 @@
.where((String line) => removeLine == null || !removeLine(line))
.map((String line) => '$line\n')
.transform(const Utf8Encoder());
- if (printOutput) {
+ if (outputMode == OutputMode.print) {
await Future.wait<void>(<Future<void>>[
stdout.addStream(stdoutSource),
stderr.addStream(process.stderr),
@@ -125,6 +128,12 @@
return (expectNonZeroExit || expectFlaky) ? 0 : 1;
});
print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: ');
+
+ if (output != null) {
+ output.stdout = flattenToString(await savedStdout);
+ output.stderr = flattenToString(await savedStderr);
+ }
+
// If the test is flaky we don't care about the actual exit.
if (expectFlaky) {
return;
@@ -133,9 +142,12 @@
if (failureMessage != null) {
print(failureMessage);
}
- if (!printOutput) {
- stdout.writeln(utf8.decode((await savedStdout).expand<int>((List<int> ints) => ints).toList()));
- stderr.writeln(utf8.decode((await savedStderr).expand<int>((List<int> ints) => ints).toList()));
+
+ // Print the output when we get unexpected results (unless output was
+ // printed already).
+ if (outputMode != OutputMode.print) {
+ stdout.writeln(flattenToString(await savedStdout));
+ stderr.writeln(flattenToString(await savedStderr));
}
print(
'$redLine\n'
@@ -147,3 +159,18 @@
exit(1);
}
}
+
+T identity<T>(T x) => x;
+
+/// Flattens a nested list of UTF-8 code units into a single string.
+String flattenToString(List<List<int>> chunks) =>
+ utf8.decode(chunks.expand<int>(identity).toList(growable: false));
+
+/// Specifies what to do with command output from [runCommand].
+enum OutputMode { print, capture, discard }
+
+/// Stores command output from [runCommand] when used with [OutputMode.capture].
+class CapturedOutput {
+ String stdout;
+ String stderr;
+}
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index c5234e1..6a49654 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -15,6 +15,14 @@
typedef ShardRunner = Future<void> Function();
+/// A function used to validate the output of a test.
+///
+/// If the output matches expectations, the function shall return null.
+///
+/// If the output does not match expectations, the function shall return an
+/// appropriate error message.
+typedef OutputChecker = String Function(CapturedOutput);
+
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
@@ -142,7 +150,7 @@
<String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
expectNonZeroExit: true,
- printOutput: false,
+ outputMode: OutputMode.discard,
timeout: _kShortTimeout,
),
],
@@ -765,8 +773,6 @@
}
Future<void> _runFlutterWebTest(String workingDirectory, {
- bool printOutput = true,
- bool skip = false,
Duration timeout = _kLongTimeout,
List<String> tests,
}) async {
@@ -802,6 +808,7 @@
String script,
bool expectFailure = false,
bool printOutput = true,
+ OutputChecker outputChecker,
List<String> options = const <String>[],
bool skip = false,
Duration timeout = _kLongTimeout,
@@ -809,6 +816,9 @@
Map<String, String> environment,
List<String> tests = const <String>[],
}) async {
+ // Support printing output or capturing it for matching, but not both.
+ assert(_implies(printOutput, outputChecker == null));
+
final List<String> args = <String>[
'test',
...options,
@@ -838,14 +848,38 @@
args.addAll(tests);
if (!shouldProcessOutput) {
- return runCommand(flutter, args,
+ OutputMode outputMode = OutputMode.discard;
+ CapturedOutput output;
+
+ if (outputChecker != null) {
+ outputMode = OutputMode.capture;
+ output = CapturedOutput();
+ } else if (printOutput) {
+ outputMode = OutputMode.print;
+ }
+
+ await runCommand(
+ flutter,
+ args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
- printOutput: printOutput,
+ outputMode: outputMode,
+ output: output,
skip: skip,
timeout: timeout,
environment: environment,
);
+
+ if (outputChecker != null) {
+ final String message = outputChecker(output);
+ if (message != null) {
+ print('$redLine');
+ print(message);
+ print('$redLine');
+ exit(1);
+ }
+ }
+ return;
}
if (useFlutterTestFormatter) {
@@ -975,3 +1009,8 @@
await _runDevicelabTest('module_host_with_custom_build_test', env: env);
}
}
+
+/// Returns true if `p` logically implies `q`, false otherwise.
+///
+/// If `p` is true, `q` must be true.
+bool _implies(bool p, bool q) => !p || q;
