[devicelab] Add results path flag to test runner (#72765)
diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart
index 8582b96..c3f46be 100644
--- a/dev/devicelab/bin/run.dart
+++ b/dev/devicelab/bin/run.dart
@@ -42,6 +42,9 @@
/// Whether to exit on first test failure.
bool exitOnFirstTestFailure;
+/// Path to write test results to.
+String resultsPath;
+
/// File containing a service account token.
///
/// If passed, the test run results will be uploaded to Flutter infrastructure.
@@ -65,6 +68,16 @@
return;
}
+ deviceId = args['device-id'] as String;
+ exitOnFirstTestFailure = args['exit'] as bool;
+ gitBranch = args['git-branch'] as String;
+ localEngine = args['local-engine'] as String;
+ localEngineSrcPath = args['local-engine-src-path'] as String;
+ luciBuilder = args['luci-builder'] as String;
+ resultsPath = args['results-file'] as String;
+ serviceAccountTokenFile = args['service-account-token-file'] as String;
+ silent = args['silent'] as bool;
+
if (!args.wasParsed('task')) {
if (args.wasParsed('stage') || args.wasParsed('all')) {
addTasks(
@@ -89,15 +102,6 @@
return;
}
- deviceId = args['device-id'] as String;
- exitOnFirstTestFailure = args['exit'] as bool;
- gitBranch = args['git-branch'] as String;
- localEngine = args['local-engine'] as String;
- localEngineSrcPath = args['local-engine-src-path'] as String;
- luciBuilder = args['luci-builder'] as String;
- serviceAccountTokenFile = args['service-account-token-file'] as String;
- silent = args['silent'] as bool;
-
if (args.wasParsed('ab')) {
await _runABTest();
} else {
@@ -120,8 +124,17 @@
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
- if (serviceAccountTokenFile != null) {
+ if (resultsPath != null) {
+ final Cocoon cocoon = Cocoon();
+ await cocoon.writeTaskResultToFile(
+ builderName: luciBuilder,
+ gitBranch: gitBranch,
+ result: result,
+ resultsPath: resultsPath,
+ );
+ } else if (serviceAccountTokenFile != null) {
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
+
/// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
}
@@ -224,7 +237,7 @@
File file = File(parts[0] + parts[1]);
int i = 1;
while (file.existsSync()) {
- file = File(parts[0]+i.toString()+parts[1]);
+ file = File(parts[0] + i.toString() + parts[1]);
i++;
}
return file;
@@ -355,10 +368,7 @@
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
)
- ..addOption(
- 'luci-builder',
- help: '[Flutter infrastructure] Name of the LUCI builder being run on.'
- )
+ ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
..addFlag(
'match-host-platform',
defaultsTo: true,
@@ -368,6 +378,11 @@
'`required_agent_capabilities`\nin the `manifest.yaml` file.',
)
..addOption(
+ 'results-file',
+ help: '[Flutter infrastructure] File path for test results. If passed with\n'
+ 'task, will write test results to the file.'
+ )
+ ..addOption(
'service-account-token-file',
help: '[Flutter infrastructure] Authentication for uploading results.',
)
diff --git a/dev/devicelab/bin/test_runner.dart b/dev/devicelab/bin/test_runner.dart
new file mode 100644
index 0000000..fa8dc01
--- /dev/null
+++ b/dev/devicelab/bin/test_runner.dart
@@ -0,0 +1,21 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/command_runner.dart';
+import 'package:flutter_devicelab/command/upload_metrics.dart';
+
+final CommandRunner<void> runner =
+ CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
+ ..addCommand(UploadMetricsCommand());
+
+Future<void> main(List<String> rawArgs) async {
+ runner.run(rawArgs).catchError((dynamic error) {
+ stderr.writeln('$error\n');
+ stderr.writeln('Usage:\n');
+ stderr.writeln(runner.usage);
+ exit(64); // Exit code 64 indicates a usage error.
+ });
+}
diff --git a/dev/devicelab/lib/command/upload_metrics.dart b/dev/devicelab/lib/command/upload_metrics.dart
new file mode 100644
index 0000000..f4935f5
--- /dev/null
+++ b/dev/devicelab/lib/command/upload_metrics.dart
@@ -0,0 +1,32 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:args/command_runner.dart';
+
+import '../framework/cocoon.dart';
+
+class UploadMetricsCommand extends Command<void> {
+ UploadMetricsCommand() {
+ argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
+ argParser.addOption(
+ 'service-account-token-file',
+ help: 'Authentication token for uploading results.',
+ );
+ }
+
+ @override
+ String get name => 'upload-metrics';
+
+ @override
+ String get description => '[Flutter infrastructure] Upload metrics data to Cocoon';
+
+ @override
+ Future<void> run() async {
+ final String resultsPath = argResults['results-file'] as String;
+ final String serviceAccountTokenFile = argResults['service-account-token-file'] as String;
+
+ final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
+ return cocoon.sendResultsPath(resultsPath);
+ }
+}
diff --git a/dev/devicelab/lib/framework/cocoon.dart b/dev/devicelab/lib/framework/cocoon.dart
index fe62fbb..7f5c8c6 100644
--- a/dev/devicelab/lib/framework/cocoon.dart
+++ b/dev/devicelab/lib/framework/cocoon.dart
@@ -34,9 +34,9 @@
Cocoon({
String serviceAccountTokenPath,
@visibleForTesting Client httpClient,
- @visibleForTesting FileSystem filesystem,
+ @visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
- }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: filesystem);
+ }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: fs);
/// Client to make http requests to Cocoon.
final AuthenticatedCocoonClient _httpClient;
@@ -46,6 +46,9 @@
/// Url used to send results to.
static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
+ /// Underlying [FileSystem] to use.
+ final FileSystem fs;
+
static final Logger logger = Logger('CocoonClient');
String get commitSha => _commitSha ?? _readCommitSha();
@@ -61,8 +64,25 @@
return _commitSha = result.stdout as String;
}
+ /// Upload the JSON results in [resultsPath] to Cocoon.
+ ///
+ /// Flutter infrastructure's workflow is:
+ /// 1. Run DeviceLab test, writing results to a known path
+ /// 2. Request service account token from luci auth (valid for at least 3 minutes)
+ /// 3. Upload results from (1) to Cocooon
+ Future<void> sendResultsPath(String resultsPath) async {
+ final File resultFile = fs.file(resultsPath);
+ final Map<String, dynamic> resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
+ await _sendUpdateTaskRequest(resultsJson);
+ }
+
/// Send [TaskResult] to Cocoon.
- Future<void> sendTaskResult({@required String builderName, @required TaskResult result, @required String gitBranch}) async {
+ // TODO(chillers): Remove when sendResultsPath is used in prod. https://github.com/flutter/flutter/issues/72457
+ Future<void> sendTaskResult({
+ @required String builderName,
+ @required TaskResult result,
+ @required String gitBranch,
+ }) async {
assert(builderName != null);
assert(gitBranch != null);
assert(result != null);
@@ -73,7 +93,45 @@
print('${rec.level.name}: ${rec.time}: ${rec.message}');
});
- final Map<String, dynamic> status = <String, dynamic>{
+ final Map<String, dynamic> updateRequest = _constructUpdateRequest(
+ gitBranch: gitBranch,
+ builderName: builderName,
+ result: result,
+ );
+ await _sendUpdateTaskRequest(updateRequest);
+ }
+
+ /// Write the given parameters into an update task request and store the JSON in [resultsPath].
+ Future<void> writeTaskResultToFile({
+ @required String builderName,
+ @required String gitBranch,
+ @required TaskResult result,
+ @required String resultsPath,
+ }) async {
+ assert(builderName != null);
+ assert(gitBranch != null);
+ assert(result != null);
+ assert(resultsPath != null);
+
+ final Map<String, dynamic> updateRequest = _constructUpdateRequest(
+ gitBranch: gitBranch,
+ builderName: builderName,
+ result: result,
+ );
+ final File resultFile = fs.file(resultsPath);
+ if (resultFile.existsSync()) {
+ resultFile.deleteSync();
+ }
+ resultFile.createSync();
+ resultFile.writeAsStringSync(json.encode(updateRequest));
+ }
+
+ Map<String, dynamic> _constructUpdateRequest({
+ @required String builderName,
+ @required TaskResult result,
+ @required String gitBranch,
+ }) {
+ final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
@@ -81,7 +139,7 @@
};
// Make a copy of result data because we may alter it for validation below.
- status['ResultData'] = result.data;
+ updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
@@ -95,9 +153,13 @@
}
}
}
- status['BenchmarkScoreKeys'] = validScoreKeys;
+ updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
- final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', status);
+ return updateRequest;
+ }
+
+ Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
+ final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
if (response['Name'] != null) {
logger.info('Updated Cocoon with results from this task');
} else {
diff --git a/dev/devicelab/test/cocoon_test.dart b/dev/devicelab/test/cocoon_test.dart
index 8485142..a1f9ccd 100644
--- a/dev/devicelab/test/cocoon_test.dart
+++ b/dev/devicelab/test/cocoon_test.dart
@@ -48,7 +48,7 @@
_processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
- filesystem: fs,
+ fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
@@ -60,7 +60,7 @@
_processResult = ProcessResult(1, 1, '', '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
- filesystem: fs,
+ fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
@@ -68,12 +68,69 @@
expect(() => cocoon.commitSha, throwsA(isA<CocoonException>()));
});
+ test('writes expected update task json', () async {
+ _processResult = ProcessResult(1, 0, commitSha, '');
+ final TaskResult result = TaskResult.fromJson(<String, dynamic>{
+ 'success': true,
+ 'data': <String, dynamic>{
+ 'i': 0,
+ 'j': 0,
+ 'not_a_metric': 'something',
+ },
+ 'benchmarkScoreKeys': <String>['i', 'j'],
+ });
+
+ cocoon = Cocoon(
+ fs: fs,
+ processRunSync: runSyncStub,
+ );
+
+ const String resultsPath = 'results.json';
+ await cocoon.writeTaskResultToFile(
+ builderName: 'builderAbc',
+ gitBranch: 'master',
+ result: result,
+ resultsPath: resultsPath,
+ );
+
+ final String resultJson = fs.file(resultsPath).readAsStringSync();
+ const String expectedJson = '{'
+ '"CommitBranch":"master",'
+ '"CommitSha":"$commitSha",'
+ '"BuilderName":"builderAbc",'
+ '"NewStatus":"Succeeded",'
+ '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
+ '"BenchmarkScoreKeys":["i","j"]}';
+ expect(resultJson, expectedJson);
+ });
+
+ test('uploads expected update task payload from results file', () async {
+ _processResult = ProcessResult(1, 0, commitSha, '');
+ cocoon = Cocoon(
+ fs: fs,
+ httpClient: mockClient,
+ processRunSync: runSyncStub,
+ serviceAccountTokenPath: serviceAccountTokenPath,
+ );
+
+ const String resultsPath = 'results.json';
+ const String updateTaskJson = '{'
+ '"CommitBranch":"master",'
+ '"CommitSha":"$commitSha",'
+ '"BuilderName":"builderAbc",'
+ '"NewStatus":"Succeeded",'
+ '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
+ '"BenchmarkScoreKeys":["i","j"]}';
+ fs.file(resultsPath).writeAsStringSync(updateTaskJson);
+ await cocoon.sendResultsPath(resultsPath);
+ });
+
test('sends expected request from successful task', () async {
mockClient = MockClient((Request request) async => Response('{}', 200));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
- filesystem: fs,
+ fs: fs,
httpClient: mockClient,
);
@@ -87,12 +144,13 @@
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
- filesystem: fs,
+ fs: fs,
httpClient: mockClient,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
- expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result), throwsA(isA<ClientException>()));
+ expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result),
+ throwsA(isA<ClientException>()));
});
test('null git branch throws error', () async {
@@ -100,12 +158,13 @@
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
- filesystem: fs,
+ fs: fs,
httpClient: mockClient,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
- expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result), throwsA(isA<AssertionError>()));
+ expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result),
+ throwsA(isA<AssertionError>()));
});
});
