blob: 6676dc8f6ba9f437b8ff20aa96338fbc34f289ac [file] [log] [blame]
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show json;
import 'dart:html' as html;
import 'dart:math' as math;
import 'src/common.dart';
import 'src/recorder.dart';
export 'src/recorder.dart';
/// Signature for a function that creates a [Recorder].
typedef RecorderFactory = Recorder Function();
/// List of all benchmarks that run in the devicelab.
///
/// When adding a new benchmark, add it to this map. Make sure that the name
/// of your benchmark is unique.
late Map<String, RecorderFactory> _benchmarks;
final LocalBenchmarkServerClient _client = LocalBenchmarkServerClient();
/// Starts a local benchmark client to run [benchmarks].
///
/// Usually used in combination with a benchmark server, which orders the
/// client to run each benchmark in order.
///
/// When used without a server, prompts the user to select a benchmark to
/// run next.
Future<void> runBenchmarks(Map<String, RecorderFactory> benchmarks) async {
assert(benchmarks != null);
// Set local benchmarks.
_benchmarks = benchmarks;
// Check if the benchmark server wants us to run a specific benchmark.
final String nextBenchmark = await _client.requestNextBenchmark();
if (nextBenchmark == LocalBenchmarkServerClient.kManualFallback) {
_fallbackToManual(
'The server did not tell us which benchmark to run next.');
return;
}
await _runBenchmark(nextBenchmark);
html.window.location.reload();
}
Future<void> _runBenchmark(String? benchmarkName) async {
final RecorderFactory? recorderFactory = _benchmarks[benchmarkName];
if (recorderFactory == null) {
_fallbackToManual('Benchmark $benchmarkName not found.');
return;
}
await runZoned<Future<void>>(
() async {
final Recorder recorder = recorderFactory();
final Runner runner = recorder.isTracingEnabled && !_client.isInManualMode
? Runner(
recorder: recorder,
setUpAllDidRun: () =>
_client.startPerformanceTracing(benchmarkName),
tearDownAllWillRun: _client.stopPerformanceTracing,
)
: Runner(recorder: recorder);
final Profile profile = await runner.run();
if (!_client.isInManualMode) {
await _client.sendProfileData(profile);
} else {
_printResultsToScreen(profile);
print(profile); // ignore: avoid_print
}
},
zoneSpecification: ZoneSpecification(
print: (Zone self, ZoneDelegate parent, Zone zone, String line) async {
if (_client.isInManualMode) {
parent.print(zone, '[$benchmarkName] $line');
} else {
await _client.printToConsole(line);
}
},
handleUncaughtError: (
Zone self,
ZoneDelegate parent,
Zone zone,
Object error,
StackTrace stackTrace,
) async {
if (_client.isInManualMode) {
parent.print(zone, '[$benchmarkName] $error, $stackTrace');
parent.handleUncaughtError(zone, error, stackTrace);
} else {
await _client.reportError(error, stackTrace);
}
},
),
);
}
void _fallbackToManual(String error) {
html.document.body!.appendHtml('''
<div id="manual-panel">
<h3>$error</h3>
<p>Choose one of the following benchmarks:</p>
<!-- Absolutely position it so it receives the clicks and not the glasspane -->
<ul style="position: absolute">
${_benchmarks.keys.map((String name) => '<li><button id="$name">$name</button></li>').join('\n')}
</ul>
</div>
''',
validator: html.NodeValidatorBuilder()
..allowHtml5()
..allowInlineStyles());
for (final String benchmarkName in _benchmarks.keys) {
// Find the button elements added above.
final html.Element button = html.document.querySelector('#$benchmarkName')!;
button.addEventListener('click', (_) {
final html.Element? manualPanel =
html.document.querySelector('#manual-panel');
manualPanel?.remove();
_runBenchmark(benchmarkName);
});
}
}
/// Visualizes results on the Web page for manual inspection.
void _printResultsToScreen(Profile profile) {
final html.BodyElement body = html.document.body!;
body.innerHtml = '<h2>${profile.name}</h2>';
profile.scoreData.forEach((String scoreKey, Timeseries timeseries) {
body.appendHtml('<h2>$scoreKey</h2>');
body.appendHtml('<pre>${timeseries.computeStats()}</pre>');
body.append(TimeseriesVisualization(timeseries).render());
});
}
/// Draws timeseries data and statistics on a canvas.
class TimeseriesVisualization {
/// Creates a visualization for a [Timeseries].
TimeseriesVisualization(this._timeseries) {
_stats = _timeseries.computeStats();
_canvas = html.CanvasElement();
_screenWidth = html.window.screen!.width!;
_canvas.width = _screenWidth;
_canvas.height = (_kCanvasHeight * html.window.devicePixelRatio).round();
_canvas.style
..width = '100%'
..height = '${_kCanvasHeight}px'
..outline = '1px solid green';
_ctx = _canvas.context2D;
// The amount of vertical space available on the chart. Because some
// outliers can be huge they can dwarf all the useful values. So we
// limit it to 1.5 x the biggest non-outlier.
_maxValueChartRange = 1.5 *
_stats.samples
.where((AnnotatedSample sample) => !sample.isOutlier)
.map<double>((AnnotatedSample sample) => sample.magnitude)
.fold<double>(0, math.max);
}
static const double _kCanvasHeight = 200;
final Timeseries _timeseries;
late TimeseriesStats _stats;
late html.CanvasElement _canvas;
late html.CanvasRenderingContext2D _ctx;
late int _screenWidth;
// Used to normalize benchmark values to chart height.
late double _maxValueChartRange;
/// Converts a sample value to vertical canvas coordinates.
///
/// This does not work for horizontal coordinates.
double _normalized(double value) {
return _kCanvasHeight * value / _maxValueChartRange;
}
/// A utility for drawing lines.
void drawLine(num x1, num y1, num x2, num y2) {
_ctx.beginPath();
_ctx.moveTo(x1, y1);
_ctx.lineTo(x2, y2);
_ctx.stroke();
}
/// Renders the timeseries into a `<canvas>` and returns the canvas element.
html.CanvasElement render() {
_ctx.translate(0, _kCanvasHeight * html.window.devicePixelRatio);
_ctx.scale(1, -html.window.devicePixelRatio);
final double barWidth = _screenWidth / _stats.samples.length;
double xOffset = 0;
for (int i = 0; i < _stats.samples.length; i++) {
final AnnotatedSample sample = _stats.samples[i];
if (sample.isWarmUpValue) {
// Put gray background behind warm-up samples.
_ctx.fillStyle = 'rgba(200,200,200,1)';
_ctx.fillRect(xOffset, 0, barWidth, _normalized(_maxValueChartRange));
}
if (sample.magnitude > _maxValueChartRange) {
// The sample value is so big it doesn't fit on the chart. Paint it purple.
_ctx.fillStyle = 'rgba(100,50,100,0.8)';
} else if (sample.isOutlier) {
// The sample is an outlier, color it light red.
_ctx.fillStyle = 'rgba(255,50,50,0.6)';
} else {
// A non-outlier sample, color it light blue.
_ctx.fillStyle = 'rgba(50,50,255,0.6)';
}
_ctx.fillRect(xOffset, 0, barWidth - 1, _normalized(sample.magnitude));
xOffset += barWidth;
}
// Draw a horizontal solid line corresponding to the average.
_ctx.lineWidth = 1;
drawLine(0, _normalized(_stats.average), _screenWidth,
_normalized(_stats.average));
// Draw a horizontal dashed line corresponding to the outlier cut off.
_ctx.setLineDash(<num>[5, 5]);
drawLine(0, _normalized(_stats.outlierCutOff), _screenWidth,
_normalized(_stats.outlierCutOff));
// Draw a light red band that shows the noise (1 stddev in each direction).
_ctx.fillStyle = 'rgba(255,50,50,0.3)';
_ctx.fillRect(
0,
_normalized(_stats.average * (1 - _stats.noise)),
_screenWidth,
_normalized(2 * _stats.average * _stats.noise),
);
return _canvas;
}
}
/// Implements the client REST API for the local benchmark server.
///
/// The local server is optional. If it is not available the benchmark UI must
/// implement a manual fallback. This allows debugging benchmarks using plain
/// `flutter run`.
class LocalBenchmarkServerClient {
/// This value is returned by [requestNextBenchmark].
static const String kManualFallback = '__manual_fallback__';
/// Whether we fell back to manual mode.
///
/// This happens when you run benchmarks using plain `flutter run` rather than
/// devicelab test harness. The test harness spins up a special server that
/// provides API for automatically picking the next benchmark to run.
late bool isInManualMode;
/// Asks the local server for the name of the next benchmark to run.
///
/// Returns [kManualFallback] if local server is not available (uses 404 as a
/// signal).
Future<String> requestNextBenchmark() async {
final html.HttpRequest request = await _requestXhr(
'/next-benchmark',
method: 'POST',
mimeType: 'application/json',
sendData: json.encode(_benchmarks.keys.toList()),
);
// `kEndOfBenchmarks` is expected when the benchmark server is telling us there are no more benchmarks to run.
// 404 is expected when the benchmark is run using plain `flutter run`, which does not provide "next-benchmark" handler.
if (request.responseText == kEndOfBenchmarks || request.status == 404) {
isInManualMode = true;
return kManualFallback;
}
isInManualMode = false;
return request.responseText ?? kManualFallback;
}
void _checkNotManualMode() {
if (isInManualMode) {
throw StateError('Operation not supported in manual fallback mode.');
}
}
/// Asks the local server to begin tracing performance.
///
/// This uses the chrome://tracing tracer, which is not available from within
/// the page itself, and therefore must be controlled from outside using the
/// DevTools Protocol.
Future<void> startPerformanceTracing(String? benchmarkName) async {
_checkNotManualMode();
await html.HttpRequest.request(
'/start-performance-tracing?label=$benchmarkName',
method: 'POST',
mimeType: 'application/json',
);
}
/// Stops the performance tracing session started by [startPerformanceTracing].
Future<void> stopPerformanceTracing() async {
_checkNotManualMode();
await html.HttpRequest.request(
'/stop-performance-tracing',
method: 'POST',
mimeType: 'application/json',
);
}
/// Sends the profile data collected by the benchmark to the local benchmark
/// server.
Future<void> sendProfileData(Profile profile) async {
_checkNotManualMode();
final html.HttpRequest request = await html.HttpRequest.request(
'/profile-data',
method: 'POST',
mimeType: 'application/json',
sendData: json.encode(profile.toJson()),
);
if (request.status != 200) {
throw Exception('Failed to report profile data to benchmark server. '
'The server responded with status code ${request.status}.');
}
}
/// Reports an error to the benchmark server.
///
/// The server will halt the devicelab task and log the error.
Future<void> reportError(dynamic error, StackTrace stackTrace) async {
_checkNotManualMode();
await html.HttpRequest.request(
'/on-error',
method: 'POST',
mimeType: 'application/json',
sendData: json.encode(<String, dynamic>{
'error': '$error',
'stackTrace': '$stackTrace',
}),
);
}
/// Reports a message about the demo to the benchmark server.
Future<void> printToConsole(String report) async {
_checkNotManualMode();
await html.HttpRequest.request(
'/print-to-console',
method: 'POST',
mimeType: 'text/plain',
sendData: report,
);
}
/// This is the same as calling [html.HttpRequest.request] but it doesn't
/// crash on 404, which we use to detect `flutter run`.
Future<html.HttpRequest> _requestXhr(
String url, {
required String method,
required String mimeType,
required dynamic sendData,
}) {
final Completer<html.HttpRequest> completer = Completer<html.HttpRequest>();
final html.HttpRequest xhr = html.HttpRequest();
xhr.open(method, url, async: true);
xhr.overrideMimeType(mimeType);
xhr.onLoad.listen((html.ProgressEvent e) {
completer.complete(xhr);
});
xhr.onError.listen(completer.completeError);
xhr.send(sendData);
return completer.future;
}
}