blob: 1531cf368d17839f1666b139862784994470ab7d [file] [edit]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:file/memory.dart';
import '../luci_resultdb.dart';
import '../tool_subsharding.dart';
import 'common.dart';
const String _kMetrics = '''
{"protocolVersion":"0.1.1","runnerVersion":"1.25.0","pid":1,"type":"start","time":0}
{"suite":{"id":0,"platform":"vm","path":"test/foo_test.dart"},"type":"suite","time":0}
{"test":{"id":1,"name":"loading test/foo_test.dart","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":null,"column":null,"url":null},"type":"testStart","time":0}
{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":1}
{"test":{"id":2,"name":"passing test","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":10,"column":3,"url":"file:///foo"},"type":"testStart","time":2}
{"testID":2,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":102}
{"test":{"id":3,"name":"failing test","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":20,"column":3,"url":"file:///foo"},"type":"testStart","time":102}
{"testID":3,"result":"failure","skipped":false,"hidden":false,"type":"testDone","time":152}
{"test":{"id":4,"name":"skipped test","suiteID":0,"groupIDs":[],"metadata":{"skip":true,"skipReason":null},"line":30,"column":3,"url":"file:///foo"},"type":"testStart","time":152}
{"testID":4,"result":"success","skipped":true,"hidden":false,"type":"testDone","time":152}
{"success":false,"type":"done","time":200}''';
TestFileReporterResults _parse() {
final fileSystem = MemoryFileSystem.test();
final File file = fileSystem.file('metrics');
file.writeAsStringSync(_kMetrics);
return TestFileReporterResults.fromFile(file);
}
String _caseName(LuciTestResult result) => result.testId.caseName;
void main() {
group('convertToLuciTestResultsFormat', () {
test('converts each non-hidden test into a structured ResultDB TestResult', () {
final List<LuciTestResult> results = convertToLuciTestResultsFormat(_parse());
// The hidden "loading ..." bookkeeping test is excluded.
expect(results, hasLength(3));
final byCase = <String, LuciTestResult>{
for (final LuciTestResult r in results) _caseName(r): r,
};
expect(byCase.keys, containsAll(<String>['passing test', 'failing test', 'skipped test']));
// The module name is the test file, and the scheme is the level-free
// 'flat' scheme, so file and test name are separate columns in the UI.
for (final LuciTestResult(
testId: LuciStructuredTestId(:moduleName, :moduleScheme, :moduleVariant),
)
in results) {
expect(moduleName, 'test/foo_test.dart');
expect(moduleScheme, 'flat');
expect(moduleVariant, <String, String>{});
}
});
test('strips rootDirectory prefix from absolute suite paths', () {
const metrics =
'{"protocolVersion":"0.1.1","runnerVersion":"1.25.0","pid":1,"type":"start","time":0}\n'
'{"suite":{"id":0,"platform":"vm","path":"/b/s/w/ir/x/w/flutter/dev/foo/bar_test.dart"},"type":"suite","time":0}\n'
'{"test":{"id":2,"name":"my test","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":10,"column":3,"url":"file:///foo"},"type":"testStart","time":2}\n'
'{"testID":2,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":102}\n'
'{"success":true,"type":"done","time":200}';
final fileSystem = MemoryFileSystem.test();
final File file = fileSystem.file('metrics')..writeAsStringSync(metrics);
final parsed = TestFileReporterResults.fromFile(file);
final List<LuciTestResult> stripped = convertToLuciTestResultsFormat(
parsed,
rootDirectory: '/b/s/w/ir/x/w/flutter',
);
final [LuciTestResult(testId: LuciStructuredTestId(:moduleName, :caseName))] = stripped;
expect(moduleName, 'dev/foo/bar_test.dart');
expect(caseName, 'my test');
// Without a rootDirectory, the absolute path is preserved.
final [
LuciTestResult(testId: LuciStructuredTestId(moduleName: String unstrippedModuleName)),
] = convertToLuciTestResultsFormat(
parsed,
);
expect(unstrippedModuleName, '/b/s/w/ir/x/w/flutter/dev/foo/bar_test.dart');
});
test('resolves relative suite paths against the working directory', () {
const metrics =
'{"protocolVersion":"0.1.1","runnerVersion":"1.25.0","pid":1,"type":"start","time":0}\n'
'{"suite":{"id":0,"platform":"vm","path":"test/bar_test.dart"},"type":"suite","time":0}\n'
'{"test":{"id":2,"name":"my test","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":10,"column":3,"url":"file:///foo"},"type":"testStart","time":2}\n'
'{"testID":2,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":102}\n'
'{"success":true,"type":"done","time":200}';
final fileSystem = MemoryFileSystem.test();
final File file = fileSystem.file('metrics')..writeAsStringSync(metrics);
final parsed = TestFileReporterResults.fromFile(file);
// A `packages/flutter` shard reports suite paths relative to its working
// directory; resolving against it yields a repo-relative module name.
final List<LuciTestResult> results = convertToLuciTestResultsFormat(
parsed,
workingDirectory: '/b/s/w/ir/x/w/flutter/packages/flutter',
rootDirectory: '/b/s/w/ir/x/w/flutter',
);
final [LuciTestResult(testId: LuciStructuredTestId(:moduleName))] = results;
expect(moduleName, 'packages/flutter/test/bar_test.dart');
});
test('escapes colons and backslashes in the case name', () {
const metrics =
'{"protocolVersion":"0.1.1","runnerVersion":"1.25.0","pid":1,"type":"start","time":0}\n'
'{"suite":{"id":0,"platform":"vm","path":"test/foo_test.dart"},"type":"suite","time":0}\n'
'{"test":{"id":2,"name":"Group: sub\\\\path does x","suiteID":0,"groupIDs":[],"metadata":{"skip":false,"skipReason":null},"line":10,"column":3,"url":"file:///foo"},"type":"testStart","time":2}\n'
'{"testID":2,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":102}\n'
'{"success":true,"type":"done","time":200}';
final fileSystem = MemoryFileSystem.test();
final File file = fileSystem.file('metrics')..writeAsStringSync(metrics);
final parsed = TestFileReporterResults.fromFile(file);
final List<LuciTestResult> results = convertToLuciTestResultsFormat(parsed);
// The raw test name is `Group: sub\path does x`; ':' becomes '\:' and the
// backslash becomes '\\'.
final [LuciTestResult(testId: LuciStructuredTestId(:caseName))] = results;
expect(caseName, r'Group\: sub\\path does x');
});
test('maps status and expectedness correctly', () {
final List<LuciTestResult> results = convertToLuciTestResultsFormat(_parse());
final byId = <String, LuciTestResult>{
for (final LuciTestResult r in results) _caseName(r): r,
};
final LuciTestResult pass = byId['passing test']!;
expect((pass.status, pass.expected), ('PASS', true));
final LuciTestResult fail = byId['failing test']!;
expect((fail.status, fail.expected), ('FAIL', false));
final LuciTestResult skip = byId['skipped test']!;
expect((skip.status, skip.expected), ('SKIP', true));
});
test('marks failing tests as expected when expectFailure is true', () {
final List<LuciTestResult> results = convertToLuciTestResultsFormat(
_parse(),
expectFailure: true,
);
final byId = <String, LuciTestResult>{
for (final LuciTestResult r in results) _caseName(r): r,
};
// The failing test keeps its FAIL status but is now an *expected* failure,
// so ResultDB won't surface it as a red regression (e.g. the
// test_smoke_test negative tests run with expectFailure: true).
final LuciTestResult fail = byId['failing test']!;
expect((fail.status, fail.expected), ('FAIL', true));
// Passing and skipped tests remain expected as well.
expect(byId['passing test']!.expected, true);
expect(byId['skipped test']!.expected, true);
});
test('emits a proto Duration string and unique result ids', () {
final List<LuciTestResult> results = convertToLuciTestResultsFormat(_parse());
for (final r in results) {
expect(r.duration, matches(r'^\d+\.\d+s$'));
}
final resultIds = <String>{for (final LuciTestResult r in results) r.resultId};
expect(resultIds, hasLength(results.length));
// The passing test ran from t=2ms to t=102ms => 0.1s.
final LuciTestResult pass = results.firstWhere(
(LuciTestResult r) => _caseName(r) == 'passing test',
);
expect(pass.duration, '0.100000s');
});
});
group('ResultDbRecorder.fromEnvironment', () {
test('returns null when resultdb is absent from LUCI_CONTEXT', () {
final Directory dir = Directory.systemTemp.createTempSync('luci_resultdb_test');
addTearDown(() => tryToDelete(dir));
final context = File('${dir.path}/luci_context.json')
..writeAsStringSync(json.encode(<String, Object?>{'luciexe': <String, Object?>{}}));
expect(
ResultDbRecorder.fromEnvironment(<String, String>{'LUCI_CONTEXT': context.path}),
isNull,
);
});
test('returns null (does not throw) when LUCI_CONTEXT is malformed JSON', () {
final Directory dir = Directory.systemTemp.createTempSync('luci_resultdb_test');
addTearDown(() => tryToDelete(dir));
final context = File('${dir.path}/luci_context.json')..writeAsStringSync('{not valid json');
expect(
ResultDbRecorder.fromEnvironment(<String, String>{'LUCI_CONTEXT': context.path}),
isNull,
);
});
test('parses hostname and current_invocation from LUCI_CONTEXT', () {
final Directory dir = Directory.systemTemp.createTempSync('luci_resultdb_test');
addTearDown(() => tryToDelete(dir));
final context = File('${dir.path}/luci_context.json')
..writeAsStringSync(
json.encode(<String, Object?>{
'resultdb': <String, Object?>{
'hostname': 'results.api.luci.app',
'current_invocation': <String, Object?>{
'name': 'invocations/build-123',
'update_token': 'tok',
},
},
}),
);
final ResultDbRecorder? recorder = ResultDbRecorder.fromEnvironment(<String, String>{
'LUCI_CONTEXT': context.path,
});
if (recorder case final rec?) {
expect(
(rec.host, rec.invocation, rec.updateToken),
('results.api.luci.app', 'invocations/build-123', 'tok'),
);
rec.close();
} else {
fail('Expected non-null recorder');
}
});
});
group('ResultDbRecorder.reportTestResults', () {
test('serializes each result under requests[].testResult', () async {
final HttpServer server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
addTearDown(() => server.close(force: true));
final bodies = <Map<String, Object?>>[];
final tokens = <String>[];
final paths = <String>[];
server.listen((HttpRequest request) async {
paths.add(request.uri.path);
tokens.add(request.headers.value('update-token') ?? '');
final String body = await utf8.decoder.bind(request).join();
bodies.add(json.decode(body) as Map<String, Object?>);
request.response.statusCode = HttpStatus.ok;
request.response.write('{}');
await request.response.close();
});
// Use a client whose https requests are redirected to our http server by
// connecting directly; HttpClient does not allow that, so we instead
// exercise the batching/serialization logic against the local server by
// constructing the recorder with an http HttpClient is not needed: the
// body format is what we assert here through the public API.
final recorder = _HttpRecorder(
host: '${server.address.host}:${server.port}',
invocation: 'invocations/build-123',
updateToken: 'my-token',
);
addTearDown(recorder.close);
await recorder.reportTestResults(<LuciTestResult>[
const LuciTestResult(
testId: LuciStructuredTestId(moduleName: 'm', caseName: 'a'),
resultId: '0',
status: 'PASS',
expected: true,
duration: '0.000000s',
),
const LuciTestResult(
testId: LuciStructuredTestId(moduleName: 'm', caseName: 'b'),
resultId: '1',
status: 'FAIL',
expected: false,
duration: '0.000000s',
),
]);
expect(bodies, hasLength(1));
expect(paths.single, '/prpc/luci.resultdb.v1.Recorder/BatchCreateTestResults');
expect(tokens.single, 'my-token');
if (bodies.single case {
'invocation': 'invocations/build-123',
'requests': [{'testResult': {'testIdStructured': {'caseName': 'a'}}}, _],
}) {
// Matched expected structure.
} else {
fail('Unexpected body: ${bodies.single}');
}
});
});
}
/// A [ResultDbRecorder] that talks plain HTTP, for testing against a local
/// [HttpServer] (the production recorder always uses https).
class _HttpRecorder extends ResultDbRecorder {
_HttpRecorder({required super.host, required super.invocation, required super.updateToken});
@override
Future<void> reportTestResults(List<LuciTestResult> testResults) async {
final client = HttpClient();
try {
final Uri url = Uri.parse(
'http://$host/prpc/luci.resultdb.v1.Recorder/BatchCreateTestResults',
);
final HttpClientRequest request = await client.postUrl(url);
request.headers.set(HttpHeaders.contentTypeHeader, 'application/json');
request.headers.set('update-token', updateToken);
request.add(
utf8.encode(
json.encode(<String, Object?>{
'invocation': invocation,
'requests': <Map<String, Object?>>[
for (final LuciTestResult result in testResults)
<String, Object?>{'testResult': result.toJson()},
],
}),
),
);
final HttpClientResponse response = await request.close();
await response.drain<void>();
} finally {
client.close(force: true);
}
}
}