Add test_with_coverage.dart (#375)
* Add test_with_coverage.dart
* Fix analysis
* Revert unnecessary change to collect.dart
* Remove timeout
* Improve logging for functions missing line numbers
* Trying to fix run_and_collect timeout
* Readme
* Fix signal handler
* Another attemped timeout fix
* Yet another attempted timeout fix
* Update README.md
Co-authored-by: Nate Bosch <nbosch@google.com>
* Address comments
Co-authored-by: Nate Bosch <nbosch@google.com>
diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml
index c906867..5c89e93 100644
--- a/.github/workflows/test-package.yml
+++ b/.github/workflows/test-package.yml
@@ -72,9 +72,9 @@
name: Install dependencies
run: dart pub get
- name: Collect and report coverage
- run: ./tool/test_and_collect.sh
+ run: dart run bin/test_with_coverage.dart --port=9292
- name: Upload coverage
uses: coverallsapp/github-action@v1.1.2
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
- path-to-lcov: var/lcov.info
+ path-to-lcov: coverage/lcov.info
diff --git a/.gitignore b/.gitignore
index e540fed..bc117f9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@
# Temp files
*~
+coverage/
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4568c09..bc3aa24 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,9 @@
the .package file. Deprecate the `--packages` flag.
* Deprecate the packagesPath parameter and add packagePath instead, in
`HitMap.parseJson`, `HitMap.parseFiles`, `createHitmap`, and `parseCoverage`.
+* Add a new executable to the package, `test_with_coverage`. This simplifies the
+ most common use case of coverage, running all the tests for a package, and
+ generating an lcov.info file.
## 1.2.0
diff --git a/README.md b/README.md
index 5269e64..a1a9578 100644
--- a/README.md
+++ b/README.md
@@ -20,6 +20,30 @@
See [Running a script from your PATH](https://dart.dev/tools/pub/cmd/pub-global#running-a-script-from-your-path)
for more details.
+
+#### Running tests with coverage
+
+For the common use case where you just want to run all your tests, and generate
+an lcov.info file, you can use the test_with_coverage script:
+
+```
+dart pub global run coverage:test_with_coverage
+```
+
+By default, this script assumes it's being run from the root directory of a
+package, and outputs a coverage.json and lcov.info file to ./coverage/
+
+This script is essentially the same as running:
+
+```
+dart run --pause-isolates-on-exit --disable-service-auth-codes --enable-vm-service=8181 test &
+dart pub global run coverage:collect_coverage --wait-paused --uri=http://127.0.0.1:8181/ -o coverage/coverage.json --resume-isolates --scope-output=foo
+dart pub global run coverage:format_coverage --packages=.dart_tool/package_config.json --lcov -i coverage/coverage.json -o coverage/lcov.info
+```
+
+For more complicated use cases, where you want to control each of these stages,
+see the sections below.
+
#### Collecting coverage from the VM
```
@@ -89,3 +113,9 @@
dart --pause-isolates-on-exit --disable-service-auth-codes --enable-vm-service=NNNN --branch-coverage script.dart
dart pub global run coverage:collect_coverage --uri=http://... -o coverage.json --resume-isolates --function-coverage --branch-coverage
```
+
+These flags can also be passed to test_with_coverage:
+
+```
+pub global run coverage:test_with_coverage --branch-coverage --function-coverage
+```
diff --git a/bin/test_with_coverage.dart b/bin/test_with_coverage.dart
new file mode 100644
index 0000000..38aed5e
--- /dev/null
+++ b/bin/test_with_coverage.dart
@@ -0,0 +1,204 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert' show utf8, LineSplitter;
+import 'dart:io';
+
+import 'package:args/args.dart';
+import 'package:coverage/src/util.dart' show extractVMServiceUri;
+import 'package:package_config/package_config.dart';
+import 'package:path/path.dart' as path;
+
+final allProcesses = <Process>[];
+
+Future<void> dartRun(List<String> args,
+ {void Function(String)? onStdout, String? workingDir}) async {
+ final process = await Process.start(
+ Platform.executable,
+ args,
+ workingDirectory: workingDir,
+ );
+ allProcesses.add(process);
+ final broadStdout = process.stdout.asBroadcastStream();
+ broadStdout.listen(stdout.add);
+ if (onStdout != null) {
+ broadStdout
+ .transform(utf8.decoder)
+ .transform(const LineSplitter())
+ .listen(onStdout);
+ }
+ process.stderr.listen(stderr.add);
+ final result = await process.exitCode;
+ if (result != 0) {
+ throw ProcessException(Platform.executable, args, '', result);
+ }
+}
+
+Future<String?> packageNameFromConfig(String packageDir) async {
+ final config = await findPackageConfig(Directory(packageDir));
+ return config?.packageOf(Uri.directory(packageDir))?.name;
+}
+
+void watchExitSignal(ProcessSignal signal) {
+ signal.watch().listen((sig) {
+ for (final process in allProcesses) {
+ process.kill(sig);
+ }
+ exit(1);
+ });
+}
+
+ArgParser createArgParser() {
+ final parser = ArgParser();
+ parser.addOption(
+ 'package',
+ help: 'Root directory of the package to test.',
+ defaultsTo: '.',
+ );
+ parser.addOption(
+ 'package-name',
+ help: 'Name of the package to test. '
+ 'Deduced from --package if not provided.',
+ );
+ parser.addOption('port', help: 'VM service port.', defaultsTo: '8181');
+ parser.addOption('out',
+ abbr: 'o', help: 'Output directory. Defaults to <package-dir>/coverage.');
+ parser.addOption('test', help: 'Test script to run.', defaultsTo: 'test');
+ parser.addFlag(
+ 'function-coverage',
+ abbr: 'f',
+ defaultsTo: false,
+ help: 'Collect function coverage info.',
+ );
+ parser.addFlag(
+ 'branch-coverage',
+ abbr: 'b',
+ defaultsTo: false,
+ help: 'Collect branch coverage info.',
+ );
+ parser.addFlag('help', abbr: 'h', negatable: false, help: 'Show this help.');
+ return parser;
+}
+
+class Flags {
+ Flags(this.packageDir, this.packageName, this.outDir, this.port,
+ this.testScript, this.functionCoverage, this.branchCoverage);
+
+ final String packageDir;
+ final String packageName;
+ final String outDir;
+ final String port;
+ final String testScript;
+ final bool functionCoverage;
+ final bool branchCoverage;
+}
+
+Future<Flags> parseArgs(List<String> arguments) async {
+ final parser = createArgParser();
+ final args = parser.parse(arguments);
+
+ void printUsage() {
+ print('Runs tests and collects coverage for a package. By default this '
+ "script assumes it's being run from the root directory of a package, and "
+ 'outputs a coverage.json and lcov.info to ./coverage/');
+ print('Usage: dart test_with_coverage.dart [OPTIONS...]\n');
+ print(parser.usage);
+ }
+
+ Never fail(String msg) {
+ print('\n$msg\n');
+ printUsage();
+ exit(1);
+ }
+
+ if (args['help'] as bool) {
+ printUsage();
+ exit(0);
+ }
+
+ final packageDir = path.canonicalize(args['package'] as String);
+ if (!FileSystemEntity.isDirectorySync(packageDir)) {
+ fail('--package is not a valid directory.');
+ }
+
+ final packageName = (args['package-name'] as String?) ??
+ await packageNameFromConfig(packageDir);
+ if (packageName == null) {
+ fail(
+ "Couldn't figure out package name from --package. Make sure this is a "
+ 'package directory, or try passing --package-name explicitly.',
+ );
+ }
+
+ return Flags(
+ packageDir,
+ packageName,
+ (args['out'] as String?) ?? path.join(packageDir, 'coverage'),
+ args['port'] as String,
+ args['test'] as String,
+ args['function-coverage'] as bool,
+ args['branch-coverage'] as bool,
+ );
+}
+
+Future<void> main(List<String> arguments) async {
+ final flags = await parseArgs(arguments);
+ final thisDir = path.dirname(Platform.script.path);
+ final outJson = path.join(flags.outDir, 'coverage.json');
+ final outLcov = path.join(flags.outDir, 'lcov.info');
+
+ if (!FileSystemEntity.isDirectorySync(flags.outDir)) {
+ await Directory(flags.outDir).create(recursive: true);
+ }
+
+ watchExitSignal(ProcessSignal.sighup);
+ watchExitSignal(ProcessSignal.sigint);
+ watchExitSignal(ProcessSignal.sigterm);
+
+ final serviceUriCompleter = Completer<Uri>();
+ final testProcess = dartRun([
+ if (flags.branchCoverage) '--branch-coverage',
+ 'run',
+ '--pause-isolates-on-exit',
+ '--disable-service-auth-codes',
+ '--enable-vm-service=${flags.port}',
+ flags.testScript,
+ ], onStdout: (line) {
+ if (!serviceUriCompleter.isCompleted) {
+ final uri = extractVMServiceUri(line);
+ if (uri != null) {
+ serviceUriCompleter.complete(uri);
+ }
+ }
+ });
+ final serviceUri = await serviceUriCompleter.future;
+
+ await dartRun([
+ 'run',
+ 'collect_coverage.dart',
+ '--wait-paused',
+ '--resume-isolates',
+ '--uri=$serviceUri',
+ '--scope-output=${flags.packageName}',
+ if (flags.branchCoverage) '--branch-coverage',
+ if (flags.functionCoverage) '--function-coverage',
+ '-o',
+ outJson,
+ ], workingDir: thisDir);
+ await testProcess;
+
+ await dartRun([
+ 'run',
+ 'format_coverage.dart',
+ '--lcov',
+ '--check-ignore',
+ '--package=${flags.packageDir}',
+ '-i',
+ outJson,
+ '-o',
+ outLcov,
+ ], workingDir: thisDir);
+ exit(0);
+}
diff --git a/lib/src/collect.dart b/lib/src/collect.dart
index 0a5fc24..2f2cafc 100644
--- a/lib/src/collect.dart
+++ b/lib/src/collect.dart
@@ -114,7 +114,7 @@
final branchCoverageSupported = _versionCheck(version, 3, 56);
if (branchCoverage && !branchCoverageSupported) {
branchCoverage = false;
- stderr.write('Branch coverage was requested, but is not supported'
+ stderr.writeln('Branch coverage was requested, but is not supported'
' by the VM version. Try updating to a newer version of Dart');
}
final sourceReportKinds = [
@@ -251,8 +251,9 @@
final line = _getLineFromTokenPos(script, tokenPos);
if (line == null) {
- stderr.write(
- 'tokenPos $tokenPos has no line mapping for script ${script.uri!}');
+ stderr.writeln(
+ 'tokenPos $tokenPos in function ${funcRef.name} has no line mapping '
+ 'for script ${script.uri!}');
return;
}
hits.funcNames![line] = funcName;
diff --git a/lib/src/run_and_collect.dart b/lib/src/run_and_collect.dart
index b279f46..a448e8d 100644
--- a/lib/src/run_and_collect.dart
+++ b/lib/src/run_and_collect.dart
@@ -29,7 +29,7 @@
dartArgs.addAll(scriptArgs);
}
- final process = await Process.start('dart', dartArgs);
+ final process = await Process.start(Platform.executable, dartArgs);
final serviceUriCompleter = Completer<Uri>();
process.stdout
.transform(utf8.decoder)
diff --git a/pubspec.yaml b/pubspec.yaml
index 0abb08e..fc00d00 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -23,3 +23,4 @@
executables:
collect_coverage:
format_coverage:
+ test_with_coverage:
diff --git a/test/test_all.dart b/test/test_all.dart
deleted file mode 100644
index 0b9e880..0000000
--- a/test/test_all.dart
+++ /dev/null
@@ -1,25 +0,0 @@
-// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-import 'package:test/test.dart';
-
-import 'chrome_test.dart' as chrome;
-import 'collect_coverage_api_test.dart' as collect_coverage_api;
-import 'collect_coverage_test.dart' as collect_coverage;
-import 'format_coverage_test.dart' as format_coverage;
-import 'lcov_test.dart' as lcov;
-import 'resolver_test.dart' as resolver;
-import 'run_and_collect_test.dart' as run_and_collect;
-import 'util_test.dart' as util;
-
-void main() {
- group('collect_coverage_api', collect_coverage_api.main);
- group('collect_coverage', collect_coverage.main);
- group('format_coverage', format_coverage.main);
- group('lcov', lcov.main);
- group('resolver', resolver.main);
- group('run_and_collect', run_and_collect.main);
- group('util', util.main);
- group('chrome', chrome.main);
-}
diff --git a/tool/test_and_collect.sh b/tool/test_and_collect.sh
deleted file mode 100755
index ff0d396..0000000
--- a/tool/test_and_collect.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/bin/bash
-
-# Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
-# for details. All rights reserved. Use of this source code is governed by a
-# BSD-style license that can be found in the LICENSE file.
-
-# Fast fail the script on failures.
-set -e
-
-# Gather coverage
-if [[ $(dart --version 2>&1 ) =~ '(dev)' ]]; then
- OBS_PORT=9292
- echo "Collecting coverage on port $OBS_PORT..."
-
- # Start tests in one VM.
- dart --disable-service-auth-codes \
- --enable-vm-service=$OBS_PORT \
- --pause-isolates-on-exit \
- test/test_all.dart &
-
- # Run the coverage collector to generate the JSON coverage report.
- dart bin/collect_coverage.dart \
- --port=$OBS_PORT \
- --out=var/coverage.json \
- --wait-paused \
- --resume-isolates
-
- echo "Generating LCOV report..."
- dart bin/format_coverage.dart \
- --lcov \
- --in=var/coverage.json \
- --out=var/lcov.info \
- --report-on=lib \
- --check-ignore
-fi
