Add a benchmark (#419)
* Add a benchmark
* Format and fix analysis
* Fix bugs
* Fix analysis
* Install dart
* Run dart pub get
* Reduce the number of benchmarks
* Always comment
* Reduce alert threshold
* Report results in multiples of no-coverage baseline
* Show few decimal places
* Remove function coverage benchmark
* Add timeout/retry logic
* Force exit once the benchmark is done
* Await file write
* Force exit after file write
* Increase the alert threshold
diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml
index c8cb613..4e2de79 100644
--- a/.github/workflows/test-package.yml
+++ b/.github/workflows/test-package.yml
@@ -77,3 +77,31 @@
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
path-to-lcov: coverage/lcov.info
+
+ benchmark:
+ needs: test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: dart-lang/setup-dart@v1.0
+ with:
+ sdk: dev
+ - name: Install dependencies
+ run: dart pub get
+ - name: Run benchmark
+ run: dart run benchmark/run_benchmarks.dart
+ - name: Download previous benchmark data
+ uses: actions/cache@v1
+ with:
+ path: benchmark/data/cache
+ key: ${{ runner.os }}-benchmark
+ - name: Check benchmark result
+ uses: benchmark-action/github-action-benchmark@v1
+ with:
+ tool: 'customSmallerIsBetter'
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ output-file-path: benchmark/data/benchmark_result.json
+ external-data-json-path: benchmark/data/cache/benchmark_result.json
+ fail-on-alert: true
+ comment-always: true
+ alert-threshold: 150%
diff --git a/benchmark/.gitignore b/benchmark/.gitignore
new file mode 100644
index 0000000..1269488
--- /dev/null
+++ b/benchmark/.gitignore
@@ -0,0 +1 @@
+data
diff --git a/benchmark/many_isolates.dart b/benchmark/many_isolates.dart
new file mode 100644
index 0000000..f0b3c04
--- /dev/null
+++ b/benchmark/many_isolates.dart
@@ -0,0 +1,27 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:isolate';
+import 'dart:math';
+
+Future<void> main(List<String> args, dynamic message) async {
+ if (message == null) {
+ // If there is no message, it means this instance was created by
+ // run_benchmarks.dart. In that case, this is the parent instance that
+ // spawns all the others.
+ int sum = 0;
+ for (int i = 0; i < 10; ++i) {
+ final port = ReceivePort();
+ final isolate =
+ Isolate.spawnUri(Uri.file('many_isolates.dart'), [], port.sendPort);
+ sum += await port.first as int;
+ await isolate;
+ }
+ print('sum = $sum');
+ } else {
+ // If there is a message, it means this instance is one of the child
+ // instances. The message is the port that this instance replies on.
+ (message as SendPort).send(Random().nextInt(1000));
+ }
+}
diff --git a/benchmark/run_benchmarks.dart b/benchmark/run_benchmarks.dart
new file mode 100644
index 0000000..4f5fd1b
--- /dev/null
+++ b/benchmark/run_benchmarks.dart
@@ -0,0 +1,143 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+
+import '../bin/collect_coverage.dart' as collect_coverage;
+import '../bin/format_coverage.dart' as format_coverage;
+
+// Runs a test script with various different coverage configurations.
+class CoverageBenchmark extends AsyncBenchmarkBase {
+ CoverageBenchmark(
+ ScoreEmitter emitter,
+ String name,
+ this.script, {
+ this.gatherCoverage = false,
+ this.functionCoverage = false,
+ this.branchCoverage = false,
+ }) : super(name, emitter: emitter);
+
+ final String script;
+ final bool gatherCoverage;
+ final bool functionCoverage;
+ final bool branchCoverage;
+ int iteration = 0;
+
+ @override
+ Future<void> run() async {
+ print('Running $name...');
+ final covFile = 'data/$name $iteration coverage.json';
+ final lcovFile = 'data/$name $iteration lcov.info';
+ ++iteration;
+
+ await Process.start(
+ Platform.executable,
+ [
+ if (branchCoverage) '--branch-coverage',
+ 'run',
+ if (gatherCoverage) ...[
+ '--pause-isolates-on-exit',
+ '--disable-service-auth-codes',
+ '--enable-vm-service=1234',
+ ],
+ script,
+ ],
+ mode: ProcessStartMode.detached,
+ );
+ if (gatherCoverage) {
+ await collect_coverage.main([
+ '--wait-paused',
+ '--resume-isolates',
+ '--uri=http://127.0.0.1:1234/',
+ if (branchCoverage) '--branch-coverage',
+ if (functionCoverage) '--function-coverage',
+ '-o',
+ covFile,
+ ]);
+
+ await format_coverage.main([
+ '--lcov',
+ '--check-ignore',
+ '-i',
+ covFile,
+ '-o',
+ lcovFile,
+ ]);
+ }
+ }
+}
+
+// Emitter that just captures the value.
+class CaptureEmitter implements ScoreEmitter {
+ late double capturedValue;
+
+ @override
+ void emit(String testName, double value) {
+ capturedValue = value;
+ }
+}
+
+// Prints a JSON representation of the benchmark results, in a format compatible
+// with the github benchmark action.
+class JsonEmitter implements ScoreEmitter {
+ JsonEmitter(this._baseline);
+
+ final double _baseline;
+ final _results = <String, double>{};
+
+ @override
+ void emit(String testName, double value) {
+ _results[testName] = value;
+ }
+
+ String write() => '[${_results.entries.map((entry) => """{
+ "name": "${entry.key}",
+ "unit": "times slower",
+ "value": ${(entry.value / _baseline).toStringAsFixed(2)}
+}""").join(',\n')}]';
+}
+
+Future<void> runBenchmark(CoverageBenchmark benchmark) async {
+ for (int i = 0; i < 3; ++i) {
+ try {
+ await benchmark.report().timeout(Duration(minutes: 2));
+ return;
+ } on TimeoutException {
+ print('Timed out');
+ }
+ }
+ print('Timed out too many times. Giving up.');
+ exit(127);
+}
+
+Future<String> runBenchmarkSet(String name, String script) async {
+ final captureEmitter = CaptureEmitter();
+ await runBenchmark(
+ CoverageBenchmark(captureEmitter, '$name - no coverage', script));
+ final benchmarkBaseline = captureEmitter.capturedValue;
+
+ final emitter = JsonEmitter(benchmarkBaseline);
+ await runBenchmark(CoverageBenchmark(
+ emitter, '$name - basic coverage', script,
+ gatherCoverage: true));
+ await runBenchmark(CoverageBenchmark(
+ emitter, '$name - function coverage', script,
+ gatherCoverage: true, functionCoverage: true));
+ await runBenchmark(CoverageBenchmark(
+ emitter, '$name - branch coverage', script,
+ gatherCoverage: true, branchCoverage: true));
+ return emitter.write();
+}
+
+Future<void> main() async {
+ // Assume this script was started from the root coverage directory. Change to
+ // the benchmark directory.
+ Directory.current = 'benchmark';
+ final result = await runBenchmarkSet('Many isolates', 'many_isolates.dart');
+ await File('data/benchmark_result.json').writeAsString(result);
+ exit(0);
+}
diff --git a/pubspec.yaml b/pubspec.yaml
index 429085f..16d2cdb 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -16,6 +16,7 @@
vm_service: ^9.2.0
dev_dependencies:
+ benchmark_harness: ^2.2.0
build_runner: ^2.1.10
lints: ^1.0.0
mockito: ^5.1.0
