Collect metrics - load balancing tests (#108752)
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index c353e4a..6802968 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -63,6 +63,7 @@
import 'browser.dart';
import 'run_command.dart';
import 'service_worker_test.dart';
+import 'tool_subsharding.dart';
import 'utils.dart';
typedef ShardRunner = Future<void> Function();
@@ -464,6 +465,7 @@
_toolsPath,
forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
+ collectMetrics: true,
);
}
@@ -1736,6 +1738,7 @@
bool includeLocalEngineEnv = false,
bool ensurePrecompiledTool = true,
bool shuffleTests = true,
+ bool collectMetrics = false,
}) async {
int? cpus;
final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
@@ -1757,6 +1760,8 @@
cpus = 1;
}
+ const LocalFileSystem fileSystem = LocalFileSystem();
+ final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
final List<String> args = <String>[
'run',
'test',
@@ -1771,6 +1776,8 @@
if (testPaths != null)
for (final String testPath in testPaths)
testPath,
+ if (collectMetrics)
+ '--file-reporter=json:${metricFile.path}',
];
final Map<String, String> environment = <String, String>{
'FLUTTER_ROOT': flutterRoot,
@@ -1795,6 +1802,23 @@
environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
);
+
+ if (collectMetrics) {
+ try {
+ final List<String> testList = <String>[];
+ final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
+ for (final TestSpecs testSpecs in allTestSpecs.values) {
+ testList.add(testSpecs.toJson());
+ }
+ if (testList.isNotEmpty) {
+ final String testJson = json.encode(testList);
+ final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
+ testResults.writeAsStringSync(testJson);
+ }
+ } on fs.FileSystemException catch (e){
+ print('Failed to generate metrics: $e');
+ }
+ }
}
Future<void> _runFlutterTest(String workingDirectory, {
diff --git a/dev/bots/test/tool_subsharding_test.dart b/dev/bots/test/tool_subsharding_test.dart
new file mode 100644
index 0000000..19e9f8e
--- /dev/null
+++ b/dev/bots/test/tool_subsharding_test.dart
@@ -0,0 +1,69 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/memory.dart';
+
+import '../tool_subsharding.dart';
+import 'common.dart';
+
+void main() {
+ group('generateMetrics', () {
+ late MemoryFileSystem fileSystem;
+
+ setUp(() {
+ fileSystem = MemoryFileSystem.test();
+ });
+
+ test('empty metrics', () async {
+ final File file = fileSystem.file('success_file');
+ const String output = '''
+ {"missing": "entry"}
+ {"other": true}''';
+ file.writeAsStringSync(output);
+ final Map<int, TestSpecs> result = generateMetrics(file);
+ expect(result, isEmpty);
+ });
+
+ test('have metrics', () async {
+ final File file = fileSystem.file('success_file');
+ const String output = '''
+ {"protocolVersion":"0.1.1","runnerVersion":"1.21.6","pid":93376,"type":"start","time":0}
+ {"suite":{"id":0,"platform":"vm","path":"test/general.shard/project_validator_result_test.dart"},"type":"suite","time":0}
+ {"count":1,"time":12,"type":"allSuites"}
+ {"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":4798}
+ {"test":{"id":4,"name":"ProjectValidatorResult success status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":50,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4803}
+ {"testID":4,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4837}
+ {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
+ {"test":{"id":5,"name":"ProjectValidatorResult success status with warning","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":60,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4837}
+ {"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4839}
+ {"test":{"id":6,"name":"ProjectValidatorResult error status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":71,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4839}
+ {"testID":6,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4841}
+ {"group":{"id":7,"suiteID":0,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":4841}
+ {"test":{"id":8,"name":"ProjectValidatorTask error status","suiteID":0,"groupIDs":[2,7],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":89,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4842}
+ {"testID":8,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4860}
+ {"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
+ {"success":true,"type":"done","time":4870}''';
+ file.writeAsStringSync(output);
+ final Map<int, TestSpecs> result = generateMetrics(file);
+ expect(result, contains(0));
+ expect(result, contains(1));
+ expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
+ expect(result[0]!.milliseconds, 4841);
+ expect(result[1]!.path, 'other_path');
+ expect(result[1]!.milliseconds, 4000);
+ });
+
+ test('missing success entry', () async {
+ final File file = fileSystem.file('success_file');
+ const String output = '''
+ {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
+ {"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
+ file.writeAsStringSync(output);
+ final Map<int, TestSpecs> result = generateMetrics(file);
+ expect(result, isEmpty);
+ });
+ });
+}
diff --git a/dev/bots/tool_subsharding.dart b/dev/bots/tool_subsharding.dart
new file mode 100644
index 0000000..7d7d676
--- /dev/null
+++ b/dev/bots/tool_subsharding.dart
@@ -0,0 +1,79 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+class TestSpecs {
+
+ TestSpecs({
+ required this.path,
+ required this.startTime,
+ });
+
+ final String path;
+ int startTime;
+ int? _endTime;
+
+ int get milliseconds {
+ return endTime - startTime;
+ }
+
+ set endTime(int value) {
+ _endTime = value;
+ }
+
+ int get endTime {
+ if (_endTime == null) {
+ return 0;
+ }
+ return _endTime!;
+ }
+
+ String toJson() {
+ return json.encode(
+ <String, String>{'path': path, 'runtime': milliseconds.toString()}
+ );
+ }
+}
+
+/// Intended to parse the output file of `dart test --file-reporter json:file_name
+Map<int, TestSpecs> generateMetrics(File metrics) {
+ final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
+ if (!metrics.existsSync()) {
+ return allTestSpecs;
+ }
+
+ bool success = false;
+ for(final String metric in metrics.readAsLinesSync()) {
+ final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
+ if (entry.containsKey('suite')) {
+ final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
+ allTestSpecs[suite['id'] as int] = TestSpecs(
+ path: suite['path'] as String,
+ startTime: entry['time'] as int,
+ );
+ } else if (_isMetricDone(entry, allTestSpecs)) {
+ final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
+ final int suiteID = group['suiteID'] as int;
+ final TestSpecs testSpec = allTestSpecs[suiteID]!;
+ testSpec.endTime = entry['time'] as int;
+ } else if (entry.containsKey('success') && entry['success'] == true) {
+ success = true;
+ }
+ }
+
+ if (!success) { // means that not all tests succeeded therefore no metrics are stored
+ return <int, TestSpecs>{};
+ }
+ return allTestSpecs;
+}
+
+bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
+ if (entry.containsKey('group') && entry['type'] as String == 'group') {
+ final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
+ return allTestSpecs.containsKey(group['suiteID'] as int);
+ }
+ return false;
+}
