Collect metrics - load balancing tests (#108752)
diff --git a/dev/bots/test.dart b/dev/bots/test.dart index c353e4a..6802968 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart
@@ -63,6 +63,7 @@ import 'browser.dart'; import 'run_command.dart'; import 'service_worker_test.dart'; +import 'tool_subsharding.dart'; import 'utils.dart'; typedef ShardRunner = Future<void> Function(); @@ -464,6 +465,7 @@ _toolsPath, forceSingleCore: true, testPaths: _selectIndexOfTotalSubshard<String>(allTests), + collectMetrics: true, ); } @@ -1736,6 +1738,7 @@ bool includeLocalEngineEnv = false, bool ensurePrecompiledTool = true, bool shuffleTests = true, + bool collectMetrics = false, }) async { int? cpus; final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml @@ -1757,6 +1760,8 @@ cpus = 1; } + const LocalFileSystem fileSystem = LocalFileSystem(); + final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json')); final List<String> args = <String>[ 'run', 'test', @@ -1771,6 +1776,8 @@ if (testPaths != null) for (final String testPath in testPaths) testPath, + if (collectMetrics) + '--file-reporter=json:${metricFile.path}', ]; final Map<String, String> environment = <String, String>{ 'FLUTTER_ROOT': flutterRoot, @@ -1795,6 +1802,23 @@ environment: environment, removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, ); + + if (collectMetrics) { + try { + final List<String> testList = <String>[]; + final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile); + for (final TestSpecs testSpecs in allTestSpecs.values) { + testList.add(testSpecs.toJson()); + } + if (testList.isNotEmpty) { + final String testJson = json.encode(testList); + final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json')); + testResults.writeAsStringSync(testJson); + } + } on fs.FileSystemException catch (e){ + print('Failed to generate metrics: $e'); + } + } } Future<void> _runFlutterTest(String workingDirectory, {
diff --git a/dev/bots/test/tool_subsharding_test.dart b/dev/bots/test/tool_subsharding_test.dart new file mode 100644 index 0000000..19e9f8e --- /dev/null +++ b/dev/bots/test/tool_subsharding_test.dart
@@ -0,0 +1,69 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:file/memory.dart'; + +import '../tool_subsharding.dart'; +import 'common.dart'; + +void main() { + group('generateMetrics', () { + late MemoryFileSystem fileSystem; + + setUp(() { + fileSystem = MemoryFileSystem.test(); + }); + + test('empty metrics', () async { + final File file = fileSystem.file('success_file'); + const String output = ''' + {"missing": "entry"} + {"other": true}'''; + file.writeAsStringSync(output); + final Map<int, TestSpecs> result = generateMetrics(file); + expect(result, isEmpty); + }); + + test('have metrics', () async { + final File file = fileSystem.file('success_file'); + const String output = ''' + {"protocolVersion":"0.1.1","runnerVersion":"1.21.6","pid":93376,"type":"start","time":0} + {"suite":{"id":0,"platform":"vm","path":"test/general.shard/project_validator_result_test.dart"},"type":"suite","time":0} + {"count":1,"time":12,"type":"allSuites"} + {"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":4798} + {"test":{"id":4,"name":"ProjectValidatorResult success status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":50,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4803} + {"testID":4,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4837} + {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000} + {"test":{"id":5,"name":"ProjectValidatorResult success status with warning","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":60,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4837} + {"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4839} + {"test":{"id":6,"name":"ProjectValidatorResult error status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":71,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4839} + {"testID":6,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4841} + {"group":{"id":7,"suiteID":0,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":4841} + {"test":{"id":8,"name":"ProjectValidatorTask error status","suiteID":0,"groupIDs":[2,7],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":89,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4842} + {"testID":8,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4860} + {"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000} + {"success":true,"type":"done","time":4870}'''; + file.writeAsStringSync(output); + final Map<int, TestSpecs> result = generateMetrics(file); + expect(result, contains(0)); + expect(result, contains(1)); + expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart'); + expect(result[0]!.milliseconds, 4841); + expect(result[1]!.path, 'other_path'); + expect(result[1]!.milliseconds, 4000); + }); + + test('missing success entry', () async { + final File file = fileSystem.file('success_file'); + const String output = ''' + {"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000} + {"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}'''; + file.writeAsStringSync(output); + final Map<int, TestSpecs> result = generateMetrics(file); + expect(result, isEmpty); + }); + }); +}
diff --git a/dev/bots/tool_subsharding.dart b/dev/bots/tool_subsharding.dart new file mode 100644 index 0000000..7d7d676 --- /dev/null +++ b/dev/bots/tool_subsharding.dart
@@ -0,0 +1,79 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +class TestSpecs { + + TestSpecs({ + required this.path, + required this.startTime, + }); + + final String path; + int startTime; + int? _endTime; + + int get milliseconds { + return endTime - startTime; + } + + set endTime(int value) { + _endTime = value; + } + + int get endTime { + if (_endTime == null) { + return 0; + } + return _endTime!; + } + + String toJson() { + return json.encode( + <String, String>{'path': path, 'runtime': milliseconds.toString()} + ); + } +} + +/// Intended to parse the output file of `dart test --file-reporter json:file_name +Map<int, TestSpecs> generateMetrics(File metrics) { + final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{}; + if (!metrics.existsSync()) { + return allTestSpecs; + } + + bool success = false; + for(final String metric in metrics.readAsLinesSync()) { + final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>; + if (entry.containsKey('suite')) { + final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>; + allTestSpecs[suite['id'] as int] = TestSpecs( + path: suite['path'] as String, + startTime: entry['time'] as int, + ); + } else if (_isMetricDone(entry, allTestSpecs)) { + final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>; + final int suiteID = group['suiteID'] as int; + final TestSpecs testSpec = allTestSpecs[suiteID]!; + testSpec.endTime = entry['time'] as int; + } else if (entry.containsKey('success') && entry['success'] == true) { + success = true; + } + } + + if (!success) { // means that not all tests succeeded therefore no metrics are stored + return <int, TestSpecs>{}; + } + return allTestSpecs; +} + +bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) { + if (entry.containsKey('group') && entry['type'] as String == 'group') { + final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>; + return allTestSpecs.containsKey(group['suiteID'] as int); + } + return false; +}