Test performance of "flutter test" (#18810)
diff --git a/dev/devicelab/bin/tasks/flutter_test_performance.dart b/dev/devicelab/bin/tasks/flutter_test_performance.dart new file mode 100644 index 0000000..a2f96d2 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_test_performance.dart
@@ -0,0 +1,126 @@ +// Copyright (c) 2017 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// This test runs `flutter test` on the `trivial_widget_test.dart` four times. +// +// The first time, the result is ignored, on the basis that it's warming the +// cache. +// +// The second time tests how long a regular test takes to run. +// +// Before the third time, a change is made to the implementation of one of the +// files that the test depends on (indirectly). +// +// Before the fourth time, a change is made to the interface in that same file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +// Matches the output of package:test, e.g.: "00:01 +1 loading foo" +final RegExp testOutputPattern = new RegExp(r'^[0-9][0-9]:[0-9][0-9] \+[0-9]+: (.+?) *$'); + +enum TestStep { + starting, + buildingFlutterTool, + runningPubGet, + testWritesFirstCarriageReturn, + testLoading, + testRunning, + testPassed, +} + +Future<int> runTest() async { + final Stopwatch clock = new Stopwatch()..start(); + final Process analysis = await startProcess( + path.join(flutterDirectory.path, 'bin', 'flutter'), + <String>['test', path.join('flutter_test', 'trivial_widget_test.dart')], + workingDirectory: path.join(flutterDirectory.path, 'dev', 'automated_tests'), + ); + int badLines = 0; + TestStep step = TestStep.starting; + await for (String entry in analysis.stdout.transform(utf8.decoder).transform(const LineSplitter())) { + print('test stdout ($step): $entry'); + if (step == TestStep.starting && entry == 'Building flutter tool...') { + // ignore this line + step = TestStep.buildingFlutterTool; + } else if (step.index < TestStep.runningPubGet.index && entry == 'Running "flutter packages get" in automated_tests...') { + // ignore this line + step = TestStep.runningPubGet; + } else if (step.index < TestStep.testWritesFirstCarriageReturn.index && entry == '') { + // we have a blank line at the start + step = TestStep.testWritesFirstCarriageReturn; + } else { + final Match match = testOutputPattern.matchAsPrefix(entry); + if (match == null) { + badLines += 1; + } else { + if (step.index >= TestStep.testWritesFirstCarriageReturn.index && step.index <= TestStep.testLoading.index && match.group(1).startsWith('loading ')) { + // first the test loads + step = TestStep.testLoading; + } else if (step.index <= TestStep.testRunning.index && match.group(1) == 'A trivial widget test') { + // then the test runs + step = TestStep.testRunning; + } else if (step.index < TestStep.testPassed.index && match.group(1) == 'All tests passed!') { + // then the test finishes + step = TestStep.testPassed; + } else { + badLines += 1; + } + } + } + } + await for (String entry in analysis.stderr.transform(utf8.decoder).transform(const LineSplitter())) { + print('test stderr: $entry'); + badLines += 1; + } + final int result = await analysis.exitCode; + clock.stop(); + if (result != 0) + throw new Exception('flutter test failed with exit code $result'); + if (badLines > 0) + throw new Exception('flutter test renderered unexpected output ($badLines bad lines)'); + if (step != TestStep.testPassed) + throw new Exception('flutter test did not finish (only reached step $step)'); + print('elapsed time: ${clock.elapsedMilliseconds}ms'); + return clock.elapsedMilliseconds; +} + +void main() { + task(() async { + final File nodeSourceFile = new File(path.join( + flutterDirectory.path, 'packages', 'flutter', 'lib', 'src', 'foundation', 'node.dart', + )); + final String originalSource = await nodeSourceFile.readAsString(); + try { + await runTest(); // first number is meaningless; could have had to build the tool, run pub get, have a cache, etc + final int withoutChange = await runTest(); // run test again with no change + await nodeSourceFile.writeAsString( // only change implementation + originalSource + .replaceAll('_owner', '_xyzzy') + ); + final int implementationChange = await runTest(); // run test again with implementation changed + await nodeSourceFile.writeAsString( // change interface as well + originalSource + .replaceAll('_owner', '_xyzzy') + .replaceAll('owner', '_owner') + .replaceAll('_xyzzy', 'owner') + ); + final int interfaceChange = await runTest(); // run test again with interface changed + final Map<String, dynamic> data = <String, dynamic>{ + 'without_change_elapsed_time_ms': withoutChange, + 'implementation_change_elapsed_time_ms': implementationChange, + 'interface_change_elapsed_time_ms': interfaceChange, + }; + return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); + } finally { + await nodeSourceFile.writeAsString(originalSource); + } + }); +}
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 67814ed..d780647 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml
@@ -425,6 +425,13 @@ stage: devicelab required_agent_capabilities: ["linux/android"] + flutter_test_performance: + description: > + Measures performance of running flutter test. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true + technical_debt__cost: description: > Estimates our technical debt (TODOs, analyzer ignores, etc).
diff --git a/packages/flutter/lib/src/foundation/node.dart b/packages/flutter/lib/src/foundation/node.dart index 0d3e265..847554b 100644 --- a/packages/flutter/lib/src/foundation/node.dart +++ b/packages/flutter/lib/src/foundation/node.dart
@@ -4,6 +4,10 @@ import 'package:meta/meta.dart'; +// This file gets mutated by //dev/devicelab/bin/tasks/flutter_test_performance.dart +// during device lab performance tests. When editing this file, check to make sure +// that it didn't break that test. + /// An abstract node in a tree. /// /// AbstractNode has as notion of depth, attachment, and parent, but does not @@ -142,5 +146,4 @@ if (attached) child.detach(); } - }