blob: ae583dc8ba80e64ac45086528488d55d91930158 [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:js_interop';
// The analyzer currently thinks `js_interop_unsafe` is unused, but it is used
// for `JSObject.[]=`.
// ignore: unused_import
import 'dart:js_interop_unsafe';
import 'dart:math' as math;
import 'dart:ui';
import 'package:flutter/foundation.dart';
import 'package:flutter/gestures.dart';
import 'package:flutter/rendering.dart';
import 'package:flutter/scheduler.dart';
import 'package:flutter/services.dart';
import 'package:flutter/widgets.dart';
import 'package:meta/meta.dart';
import 'package:web/web.dart' as web;
/// The default number of samples from warm-up iterations.
///
/// This value is used when [Profile.useCustomWarmUp] is set to false.
///
/// The benchmark is warmed up prior to measuring to allow JIT and caches to settle.
const int _kDefaultWarmUpSampleCount = 200;
/// The default number of samples collected to compute benchmark statistics.
///
/// This value is used when [Profile.useCustomWarmUp] is set to false.
const int _kDefaultMeasuredSampleCount = 100;
/// The default total number of samples collected by a benchmark.
///
/// This value is used when [Profile.useCustomWarmUp] is set to false.
const int kDefaultTotalSampleCount = _kDefaultWarmUpSampleCount + _kDefaultMeasuredSampleCount;
/// A benchmark metric that includes frame-related computations prior to
/// submitting layer and picture operations to the underlying renderer, such as
/// HTML and CanvasKit. During this phase we compute transforms, clips, and
/// other information needed for rendering.
const String kProfilePrerollFrame = 'preroll_frame';
/// A benchmark metric that includes submitting layer and picture information
/// to the renderer.
const String kProfileApplyFrame = 'apply_frame';
/// Measures the amount of time [action] takes.
///
/// See also:
///
/// * [timeAsyncAction], which measures the time of asynchronous work.
Duration timeAction(VoidCallback action) {
final Stopwatch stopwatch = Stopwatch()..start();
action();
stopwatch.stop();
return stopwatch.elapsed;
}
/// Measures the amount of time the future returned by [action] takes to complete.
///
/// See also:
///
/// * [timeAction], which measures the time of synchronous work.
Future<Duration> timeAsyncAction(AsyncCallback action) async {
final Stopwatch stopwatch = Stopwatch()..start();
await action();
stopwatch.stop();
return stopwatch.elapsed;
}
/// A function that performs asynchronous work.
typedef AsyncVoidCallback = Future<void> Function();
/// An [AsyncVoidCallback] that doesn't do anything.
///
/// This is used just so we don't have to deal with null all over the place.
Future<void> _dummyAsyncVoidCallback() async {}
/// Runs the benchmark using the given [recorder].
///
/// Notifies about "set up" and "tear down" events via the [setUpAllDidRun]
/// and [tearDownAllWillRun] callbacks.
@sealed
class Runner {
/// Creates a runner for the [recorder].
///
/// All arguments must not be null.
Runner({
required this.recorder,
this.setUpAllDidRun = _dummyAsyncVoidCallback,
this.tearDownAllWillRun = _dummyAsyncVoidCallback,
});
/// The recorder that will run and record the benchmark.
final Recorder recorder;
/// Called immediately after [Recorder.setUpAll] future is resolved.
///
/// This is useful, for example, to kick off a profiler or a tracer such that
/// the "set up" computations are not included in the metrics.
final AsyncVoidCallback setUpAllDidRun;
/// Called just before calling [Recorder.tearDownAll].
///
/// This is useful, for example, to stop a profiler or a tracer such that
/// the "tear down" computations are not included in the metrics.
final AsyncVoidCallback tearDownAllWillRun;
/// Runs the benchmark and reports the results.
Future<Profile> run() async {
await recorder.setUpAll();
await setUpAllDidRun();
final Profile profile = await recorder.run();
await tearDownAllWillRun();
await recorder.tearDownAll();
return profile;
}
}
/// Base class for benchmark recorders.
///
/// Each benchmark recorder has a [name] and a [run] method at a minimum.
abstract class Recorder {
Recorder._(this.name, this.isTracingEnabled);
/// Whether this recorder requires tracing using Chrome's DevTools Protocol's
/// "Tracing" API.
final bool isTracingEnabled;
/// The name of the benchmark.
///
/// The results displayed in the Flutter Dashboard will use this name as a
/// prefix.
final String name;
/// Returns the recorded profile.
///
/// This value is only available while the benchmark is running.
Profile? get profile;
/// Whether the benchmark should continue running.
///
/// Returns `false` if the benchmark collected enough data and it's time to
/// stop.
bool shouldContinue() => profile?.shouldContinue() ?? true;
/// Called once before all runs of this benchmark recorder.
///
/// This is useful for doing one-time setup work that's needed for the
/// benchmark.
Future<void> setUpAll() async {}
/// The implementation of the benchmark that will produce a [Profile].
Future<Profile> run();
/// Called once after all runs of this benchmark recorder.
///
/// This is useful for doing one-time clean up work after the benchmark is
/// complete.
Future<void> tearDownAll() async {}
}
/// A recorder for benchmarking raw execution of Dart code.
///
/// This is useful for benchmarks that don't need frames or widgets.
///
/// Example:
///
/// ```
/// class BenchForLoop extends RawRecorder {
/// BenchForLoop() : super(name: benchmarkName);
///
/// static const String benchmarkName = 'for_loop';
///
/// @override
/// void body(Profile profile) {
/// profile.record('loop', () {
/// double x = 0;
/// for (int i = 0; i < 10000000; i++) {
/// x *= 1.5;
/// }
/// });
/// }
/// }
/// ```
abstract class RawRecorder extends Recorder {
RawRecorder({required String name, bool useCustomWarmUp = false})
: _useCustomWarmUp = useCustomWarmUp, super._(name, false);
/// Whether to delimit warm-up frames in a custom way.
final bool _useCustomWarmUp;
/// The body of the benchmark.
///
/// This is the part that records measurements of the benchmark.
FutureOr<void> body(Profile profile);
@override
Profile? get profile => _profile;
Profile? _profile;
@override
@nonVirtual
Future<Profile> run() async {
_profile = Profile(name: name, useCustomWarmUp: _useCustomWarmUp);
do {
await Future<void>.delayed(Duration.zero);
final FutureOr<void> result = body(_profile!);
if (result is Future) {
await result;
}
} while (shouldContinue());
return _profile!;
}
}
/// A recorder for benchmarking interactions with the engine without the
/// framework by directly exercising [SceneBuilder].
///
/// To implement a benchmark, extend this class and implement [onDrawFrame].
///
/// Example:
///
/// ```
/// class BenchDrawCircle extends SceneBuilderRecorder {
/// BenchDrawCircle() : super(name: benchmarkName);
///
/// static const String benchmarkName = 'draw_circle';
///
/// @override
/// void onDrawFrame(SceneBuilder sceneBuilder) {
/// final PictureRecorder pictureRecorder = PictureRecorder();
/// final Canvas canvas = Canvas(pictureRecorder);
/// final Paint paint = Paint()..color = const Color.fromARGB(255, 255, 0, 0);
/// final Size windowSize = window.physicalSize;
/// canvas.drawCircle(windowSize.center(Offset.zero), 50.0, paint);
/// final Picture picture = pictureRecorder.endRecording();
/// sceneBuilder.addPicture(picture);
/// }
/// }
/// ```
abstract class SceneBuilderRecorder extends Recorder {
SceneBuilderRecorder({required String name}) : super._(name, true);
@override
Profile? get profile => _profile;
Profile? _profile;
/// Called from [dart:ui.PlatformDispatcher.onBeginFrame].
@mustCallSuper
void onBeginFrame() {}
/// Called on every frame.
///
/// An implementation should exercise the [sceneBuilder] to build a frame.
/// However, it must not call [SceneBuilder.build] or
/// [dart:ui.FlutterView.render]. Instead the benchmark harness will call them
/// and time them appropriately.
void onDrawFrame(SceneBuilder sceneBuilder);
@override
Future<Profile> run() {
final Completer<Profile> profileCompleter = Completer<Profile>();
_profile = Profile(name: name);
PlatformDispatcher.instance.onBeginFrame = (_) {
try {
startMeasureFrame(profile!);
onBeginFrame();
} catch (error, stackTrace) {
profileCompleter.completeError(error, stackTrace);
rethrow;
}
};
PlatformDispatcher.instance.onDrawFrame = () {
try {
_profile!.record('drawFrameDuration', () {
final SceneBuilder sceneBuilder = SceneBuilder();
onDrawFrame(sceneBuilder);
_profile!.record('sceneBuildDuration', () {
final Scene scene = sceneBuilder.build();
_profile!.record('windowRenderDuration', () {
view.render(scene);
}, reported: false);
}, reported: false);
}, reported: true);
endMeasureFrame();
if (shouldContinue()) {
PlatformDispatcher.instance.scheduleFrame();
} else {
profileCompleter.complete(_profile);
}
} catch (error, stackTrace) {
profileCompleter.completeError(error, stackTrace);
rethrow;
}
};
PlatformDispatcher.instance.scheduleFrame();
return profileCompleter.future;
}
FlutterView get view {
assert(PlatformDispatcher.instance.implicitView != null, 'This benchmark requires the embedder to provide an implicit view.');
return PlatformDispatcher.instance.implicitView!;
}
}
/// A recorder for benchmarking interactions with the framework by creating
/// widgets.
///
/// To implement a benchmark, extend this class and implement [createWidget].
///
/// Example:
///
/// ```
/// class BenchListView extends WidgetRecorder {
/// BenchListView() : super(name: benchmarkName);
///
/// static const String benchmarkName = 'bench_list_view';
///
/// @override
/// Widget createWidget() {
/// return Directionality(
/// textDirection: TextDirection.ltr,
/// child: _TestListViewWidget(),
/// );
/// }
/// }
///
/// class _TestListViewWidget extends StatefulWidget {
/// @override
/// State<StatefulWidget> createState() {
/// return _TestListViewWidgetState();
/// }
/// }
///
/// class _TestListViewWidgetState extends State<_TestListViewWidget> {
/// ScrollController scrollController;
///
/// @override
/// void initState() {
/// super.initState();
/// scrollController = ScrollController();
/// Timer.run(() async {
/// bool forward = true;
/// while (true) {
/// await scrollController.animateTo(
/// forward ? 300 : 0,
/// curve: Curves.linear,
/// duration: const Duration(seconds: 1),
/// );
/// forward = !forward;
/// }
/// });
/// }
///
/// @override
/// Widget build(BuildContext context) {
/// return ListView.builder(
/// controller: scrollController,
/// itemCount: 10000,
/// itemBuilder: (BuildContext context, int index) {
/// return Text('Item #$index');
/// },
/// );
/// }
/// }
/// ```
abstract class WidgetRecorder extends Recorder implements FrameRecorder {
WidgetRecorder({
required String name,
this.useCustomWarmUp = false,
}) : super._(name, true);
/// Creates a widget to be benchmarked.
///
/// The widget must create its own animation to drive the benchmark. The
/// animation should continue indefinitely. The benchmark harness will stop
/// pumping frames automatically.
Widget createWidget();
final List<VoidCallback> _didStopCallbacks = <VoidCallback>[];
@override
void registerDidStop(VoidCallback cb) {
_didStopCallbacks.add(cb);
}
@override
Profile? profile;
Completer<void>? _runCompleter;
/// Whether to delimit warm-up frames in a custom way.
final bool useCustomWarmUp;
late Stopwatch _drawFrameStopwatch;
@override
@mustCallSuper
void frameWillDraw() {
startMeasureFrame(profile!);
_drawFrameStopwatch = Stopwatch()..start();
}
@override
@mustCallSuper
void frameDidDraw() {
endMeasureFrame();
profile!.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
if (shouldContinue()) {
PlatformDispatcher.instance.scheduleFrame();
} else {
for (final VoidCallback fn in _didStopCallbacks) {
fn();
}
_runCompleter!.complete();
}
}
@override
void _onError(Object error, StackTrace? stackTrace) {
_runCompleter!.completeError(error, stackTrace);
}
late final _RecordingWidgetsBinding _binding;
@override
@mustCallSuper
Future<void> setUpAll() async {
_binding = _RecordingWidgetsBinding.ensureInitialized();
}
@override
Future<Profile> run() async {
_runCompleter = Completer<void>();
final Profile localProfile = profile = Profile(name: name, useCustomWarmUp: useCustomWarmUp);
final Widget widget = createWidget();
registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) {
localProfile.addDataPoint(
kProfilePrerollFrame,
Duration(microseconds: value.toInt()),
reported: false,
);
});
registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) {
localProfile.addDataPoint(
kProfileApplyFrame,
Duration(microseconds: value.toInt()),
reported: false,
);
});
_binding._beginRecording(this, widget);
try {
await _runCompleter!.future;
return localProfile;
} finally {
stopListeningToEngineBenchmarkValues(kProfilePrerollFrame);
stopListeningToEngineBenchmarkValues(kProfileApplyFrame);
_runCompleter = null;
profile = null;
}
}
}
/// A recorder for measuring the performance of building a widget from scratch
/// starting from an empty frame.
///
/// The recorder will call [createWidget] and render it, then it will pump
/// another frame that clears the screen. It repeats this process, measuring the
/// performance of frames that render the widget and ignoring the frames that
/// clear the screen.
abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder {
WidgetBuildRecorder({required String name}) : super._(name, true);
/// Creates a widget to be benchmarked.
///
/// The widget is not expected to animate as we only care about construction
/// of the widget. If you are interested in benchmarking an animation,
/// consider using [WidgetRecorder].
Widget createWidget();
final List<VoidCallback> _didStopCallbacks = <VoidCallback>[];
@override
void registerDidStop(VoidCallback cb) {
_didStopCallbacks.add(cb);
}
@override
Profile? profile;
Completer<void>? _runCompleter;
late Stopwatch _drawFrameStopwatch;
/// Whether in this frame we should call [createWidget] and render it.
///
/// If false, then this frame will clear the screen.
bool showWidget = true;
/// The state that hosts the widget under test.
late _WidgetBuildRecorderHostState _hostState;
Widget? _getWidgetForFrame() {
if (showWidget) {
return createWidget();
} else {
return null;
}
}
late final _RecordingWidgetsBinding _binding;
@override
@mustCallSuper
Future<void> setUpAll() async {
_binding = _RecordingWidgetsBinding.ensureInitialized();
}
@override
@mustCallSuper
void frameWillDraw() {
if (showWidget) {
startMeasureFrame(profile!);
_drawFrameStopwatch = Stopwatch()..start();
}
}
@override
@mustCallSuper
void frameDidDraw() {
// Only record frames that show the widget.
if (showWidget) {
endMeasureFrame();
profile!.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, reported: true);
}
if (shouldContinue()) {
showWidget = !showWidget;
_hostState._setStateTrampoline();
} else {
for (final VoidCallback fn in _didStopCallbacks) {
fn();
}
_runCompleter!.complete();
}
}
@override
void _onError(Object error, StackTrace? stackTrace) {
_runCompleter!.completeError(error, stackTrace);
}
@override
Future<Profile> run() async {
_runCompleter = Completer<void>();
final Profile localProfile = profile = Profile(name: name);
_binding._beginRecording(this, _WidgetBuildRecorderHost(this));
try {
await _runCompleter!.future;
return localProfile;
} finally {
_runCompleter = null;
profile = null;
}
}
}
/// Hosts widgets created by [WidgetBuildRecorder].
class _WidgetBuildRecorderHost extends StatefulWidget {
const _WidgetBuildRecorderHost(this.recorder);
final WidgetBuildRecorder recorder;
@override
State<StatefulWidget> createState() => _WidgetBuildRecorderHostState();
}
class _WidgetBuildRecorderHostState extends State<_WidgetBuildRecorderHost> {
@override
void initState() {
super.initState();
widget.recorder._hostState = this;
}
// This is just to bypass the @protected on setState.
void _setStateTrampoline() {
setState(() {});
}
@override
Widget build(BuildContext context) {
return SizedBox.expand(
child: widget.recorder._getWidgetForFrame(),
);
}
}
/// Series of time recordings indexed in time order.
///
/// A timeseries is expected to contain at least one warm-up frame added by
/// calling [add] with `isWarmUpValue` set to true, followed by at least one
/// measured value added by calling [add] with `isWarmUpValue` set to false.
class Timeseries {
/// Creates an empty timeseries.
///
/// The [name] is a unique name of this timeseries. If [isReported] is true
/// this timeseries is reported to the benchmark dashboard.
Timeseries(this.name, this.isReported);
/// The label of this timeseries used for debugging and result inspection.
final String name;
/// Whether this timeseries is reported to the benchmark dashboard.
///
/// If `true` a new benchmark card is created for the timeseries and is
/// visible on the dashboard.
///
/// If `false` the data is stored but it does not show up on the dashboard.
/// Use unreported metrics for metrics that are useful for manual inspection
/// but that are too fine-grained to be useful for tracking on the dashboard.
final bool isReported;
/// The number of samples ignored as warm-up frames.
int _warmUpSampleCount = 0;
/// List of all the values that have been recorded.
///
/// This list has no limit.
final List<double> _allValues = <double>[];
/// The total amount of data collected, including ones that were dropped
/// because of the sample size limit.
int get count => _allValues.length;
/// Extracts useful statistics out of this timeseries.
///
/// See [TimeseriesStats] for more details.
TimeseriesStats computeStats() {
// Assertions do not use the `assert` keyword because benchmarks run in
// profile mode, where asserts are tree-shaken out.
if (_warmUpSampleCount == 0) {
throw StateError(
'The benchmark did not warm-up. Use at least one sample to warm-up '
'the benchmark to reduce noise.');
}
if (_warmUpSampleCount >= count) {
throw StateError(
'The benchmark did not report any measured samples. Add at least one '
'sample after warm-up is done. There were $_warmUpSampleCount warm-up '
'samples, and no measured samples in this timeseries.'
);
}
// The first few values we simply discard and never look at. They're from the warm-up phase.
final List<double> warmUpValues = _allValues.sublist(0, _warmUpSampleCount);
// Values we analyze.
final List<double> candidateValues = _allValues.sublist(_warmUpSampleCount);
// The average that includes outliers.
final double dirtyAverage = _computeAverage(name, candidateValues);
// The standard deviation that includes outliers.
final double dirtyStandardDeviation = _computeStandardDeviationForPopulation(name, candidateValues);
// Any value that's higher than this is considered an outlier.
// Two standard deviations captures 95% of a normal distribution.
final double outlierCutOff = dirtyAverage + dirtyStandardDeviation * 2;
// Candidates with outliers removed.
final Iterable<double> cleanValues = candidateValues.where((double value) => value <= outlierCutOff);
// Outlier candidates.
final Iterable<double> outliers = candidateValues.where((double value) => value > outlierCutOff);
// Final statistics.
final double cleanAverage = _computeAverage(name, cleanValues);
final double standardDeviation = _computeStandardDeviationForPopulation(name, cleanValues);
final double noise = cleanAverage > 0.0 ? standardDeviation / cleanAverage : 0.0;
// Compute outlier average. If there are no outliers the outlier average is
// the same as clean value average. In other words, in a perfect benchmark
// with no noise the difference between average and outlier average is zero,
// which the best possible outcome. Noise produces a positive difference
// between the two.
final double outlierAverage = outliers.isNotEmpty ? _computeAverage(name, outliers) : cleanAverage;
final List<AnnotatedSample> annotatedValues = <AnnotatedSample>[
for (final double warmUpValue in warmUpValues)
AnnotatedSample(
magnitude: warmUpValue,
isOutlier: warmUpValue > outlierCutOff,
isWarmUpValue: true,
),
for (final double candidate in candidateValues)
AnnotatedSample(
magnitude: candidate,
isOutlier: candidate > outlierCutOff,
isWarmUpValue: false,
),
];
return TimeseriesStats(
name: name,
average: cleanAverage,
outlierCutOff: outlierCutOff,
outlierAverage: outlierAverage,
standardDeviation: standardDeviation,
noise: noise,
cleanSampleCount: cleanValues.length,
outlierSampleCount: outliers.length,
samples: annotatedValues,
);
}
// Whether the timeseries is in the warm-up phase.
bool _isWarmingUp = true;
/// Adds a value to this timeseries.
void add(double value, {required bool isWarmUpValue}) {
if (value < 0.0) {
throw StateError(
'Timeseries $name: negative metric values are not supported. Got: $value',
);
}
if (isWarmUpValue) {
if (!_isWarmingUp) {
throw StateError(
'A warm-up value was added to the timeseries after the warm-up phase finished.'
);
}
_warmUpSampleCount += 1;
} else if (_isWarmingUp) {
_isWarmingUp = false;
}
_allValues.add(value);
}
}
/// Various statistics about a [Timeseries].
///
/// See the docs on the individual fields for more details.
@sealed
class TimeseriesStats {
const TimeseriesStats({
required this.name,
required this.average,
required this.outlierCutOff,
required this.outlierAverage,
required this.standardDeviation,
required this.noise,
required this.cleanSampleCount,
required this.outlierSampleCount,
required this.samples,
});
/// The label used to refer to the corresponding timeseries.
final String name;
/// The average value of the measured samples without outliers.
final double average;
/// The standard deviation in the measured samples without outliers.
final double standardDeviation;
/// The noise as a multiple of the [average] value takes from clean samples.
///
/// This value can be multiplied by 100.0 to get noise as a percentage of
/// the average.
///
/// If [average] is zero, treats the result as perfect score, returns zero.
final double noise;
/// The maximum value a sample can have without being considered an outlier.
///
/// See [Timeseries.computeStats] for details on how this value is computed.
final double outlierCutOff;
/// The average of outlier samples.
///
/// This value can be used to judge how badly we jank, when we jank.
///
/// Another useful metrics is the difference between [outlierAverage] and
/// [average]. The smaller the value the more predictable is the performance
/// of the corresponding benchmark.
final double outlierAverage;
/// The number of measured samples after outlier are removed.
final int cleanSampleCount;
/// The number of outliers.
final int outlierSampleCount;
/// All collected samples, annotated with statistical information.
///
/// See [AnnotatedSample] for more details.
final List<AnnotatedSample> samples;
/// Outlier average divided by clean average.
///
/// This is a measure of performance consistency. The higher this number the
/// worse is jank when it happens. Smaller is better, with 1.0 being the
/// perfect score. If [average] is zero, this value defaults to 1.0.
double get outlierRatio => average > 0.0
? outlierAverage / average
: 1.0; // this can only happen in perfect benchmark that reports only zeros
@override
String toString() {
final StringBuffer buffer = StringBuffer();
buffer.writeln(
'$name: (samples: $cleanSampleCount clean/$outlierSampleCount '
'outliers/${cleanSampleCount + outlierSampleCount} '
'measured/${samples.length} total)');
buffer.writeln(' | average: $average μs');
buffer.writeln(' | outlier average: $outlierAverage μs');
buffer.writeln(' | outlier/clean ratio: ${outlierRatio}x');
buffer.writeln(' | noise: ${_ratioToPercent(noise)}');
return buffer.toString();
}
}
/// Annotates a single measurement with statistical information.
@sealed
class AnnotatedSample {
const AnnotatedSample({
required this.magnitude,
required this.isOutlier,
required this.isWarmUpValue,
});
/// The non-negative raw result of the measurement.
final double magnitude;
/// Whether this sample was considered an outlier.
final bool isOutlier;
/// Whether this sample was taken during the warm-up phase.
///
/// If this value is `true`, this sample does not participate in
/// statistical computations. However, the sample would still be
/// shown in the visualization of results so that the benchmark
/// can be inspected manually to make sure there's a predictable
/// warm-up regression slope.
final bool isWarmUpValue;
}
/// Base class for a profile collected from running a benchmark.
class Profile {
/// Creates an empty profile that can be populated with benchmark samples
/// using [record], [recordAsync], and [addDataPoint] methods.
///
/// The [name] is the unique name of this profile that distinguishes is from
/// other profiles. Typically, the name will describe the benchmark.
///
/// If [useCustomWarmUp] is true the benchmark will continue running until
/// [stopBenchmark] is called. Otherwise, the benchmark collects the
/// [kDefaultTotalSampleCount] samples and stops automatically.
Profile({required this.name, this.useCustomWarmUp = false});
/// The name of the benchmark that produced this profile.
final String name;
/// Whether to delimit warm-up frames in a custom way.
final bool useCustomWarmUp;
/// True if the benchmark is currently measuring warm-up frames.
bool get isWarmingUp => _isWarmingUp;
bool _isWarmingUp = true;
/// True if the benchmark is currently running.
bool get isRunning => _isRunning;
bool _isRunning = true;
/// Stops the warm-up phase.
///
/// After calling this method, subsequent calls to [record], [recordAsync],
/// and [addDataPoint] will record measured data samples.
///
/// Call this method only once for each profile and only when [isWarmingUp]
/// is true.
void stopWarmingUp() {
if (!_isWarmingUp) {
throw StateError('Warm-up already stopped.');
} else {
_isWarmingUp = false;
}
}
/// Stops the benchmark.
///
/// Call this method only once for each profile and only when [isWarmingUp]
/// is false (i.e. after calling [stopWarmingUp]).
void stopBenchmark() {
if (_isWarmingUp) {
throw StateError(
'Warm-up has not finished yet. Benchmark should only be stopped after '
'it recorded at least one sample after the warm-up.'
);
} else if (scoreData.isEmpty) {
throw StateError(
'The benchmark did not collect any data.'
);
} else {
_isRunning = false;
}
}
/// This data will be used to display cards in the Flutter Dashboard.
final Map<String, Timeseries> scoreData = <String, Timeseries>{};
/// This data isn't displayed anywhere. It's stored for completeness purposes.
final Map<String, dynamic> extraData = <String, dynamic>{};
/// Invokes [callback] and records the duration of its execution under [key].
///
/// See also:
///
/// * [recordAsync], which records asynchronous work.
Duration record(String key, VoidCallback callback, { required bool reported }) {
final Duration duration = timeAction(callback);
addDataPoint(key, duration, reported: reported);
return duration;
}
/// Invokes [callback] and records the amount of time the returned future takes.
///
/// See also:
///
/// * [record], which records synchronous work.
Future<Duration> recordAsync(String key, AsyncCallback callback, { required bool reported }) async {
final Duration duration = await timeAsyncAction(callback);
addDataPoint(key, duration, reported: reported);
return duration;
}
/// Adds a timed sample to the timeseries corresponding to [key].
///
/// Set [reported] to `true` to report the timeseries to the dashboard UI.
///
/// Set [reported] to `false` to store the data, but not show it on the
/// dashboard UI.
void addDataPoint(String key, Duration duration, { required bool reported }) {
scoreData.putIfAbsent(
key,
() => Timeseries(key, reported),
).add(duration.inMicroseconds.toDouble(), isWarmUpValue: isWarmingUp);
if (!useCustomWarmUp) {
// The stopWarmingUp and stopBenchmark will not be called. Use the
// auto-stopping logic.
_autoUpdateBenchmarkPhase();
}
}
/// A convenience wrapper over [addDataPoint] for adding [AggregatedTimedBlock]
/// to the profile.
///
/// Uses [AggregatedTimedBlock.name] as the name of the data point, and
/// [AggregatedTimedBlock.duration] as the duration.
void addTimedBlock(AggregatedTimedBlock timedBlock, { required bool reported }) {
addDataPoint(timedBlock.name, Duration(microseconds: timedBlock.duration.toInt()), reported: reported);
}
/// Checks the samples collected so far and sets the appropriate benchmark phase.
///
/// If enough warm-up samples have been collected, stops the warm-up phase and
/// begins the measuring phase.
///
/// If enough total samples have been collected, stops the benchmark.
void _autoUpdateBenchmarkPhase() {
if (useCustomWarmUp) {
StateError(
'Must not call _autoUpdateBenchmarkPhase if custom warm-up is used. '
'Call `stopWarmingUp` and `stopBenchmark` instead.'
);
}
if (_isWarmingUp) {
final bool doesHaveEnoughWarmUpSamples = scoreData.keys
.every((String key) => scoreData[key]!.count >= _kDefaultWarmUpSampleCount);
if (doesHaveEnoughWarmUpSamples) {
stopWarmingUp();
}
} else if (_isRunning) {
final bool doesHaveEnoughTotalSamples = scoreData.keys
.every((String key) => scoreData[key]!.count >= kDefaultTotalSampleCount);
if (doesHaveEnoughTotalSamples) {
stopBenchmark();
}
}
}
/// Decides whether the data collected so far is sufficient to stop, or
/// whether the benchmark should continue collecting more data.
///
/// The signals used are sample size, noise, and duration.
///
/// If any of the timeseries doesn't satisfy the noise requirements, this
/// method will return true (asking the benchmark to continue collecting
/// data).
bool shouldContinue() {
// If there are no `Timeseries` in the `scoreData`, then we haven't
// recorded anything yet. Don't stop.
if (scoreData.isEmpty) {
return true;
}
return isRunning;
}
/// Returns a JSON representation of the profile that will be sent to the
/// server.
Map<String, dynamic> toJson() {
final List<String> scoreKeys = <String>[];
final Map<String, dynamic> json = <String, dynamic>{
'name': name,
'scoreKeys': scoreKeys,
};
for (final String key in scoreData.keys) {
final Timeseries timeseries = scoreData[key]!;
if (timeseries.isReported) {
scoreKeys.add('$key.average');
// Report `outlierRatio` rather than `outlierAverage`, because
// the absolute value of outliers is less interesting than the
// ratio.
scoreKeys.add('$key.outlierRatio');
}
final TimeseriesStats stats = timeseries.computeStats();
json['$key.average'] = stats.average;
json['$key.outlierAverage'] = stats.outlierAverage;
json['$key.outlierRatio'] = stats.outlierRatio;
json['$key.noise'] = stats.noise;
}
json.addAll(extraData);
return json;
}
@override
String toString() {
final StringBuffer buffer = StringBuffer();
buffer.writeln('name: $name');
for (final String key in scoreData.keys) {
final Timeseries timeseries = scoreData[key]!;
final TimeseriesStats stats = timeseries.computeStats();
buffer.writeln(stats.toString());
}
for (final String key in extraData.keys) {
final dynamic value = extraData[key];
if (value is List) {
buffer.writeln('$key:');
for (final dynamic item in value) {
buffer.writeln(' - $item');
}
} else {
buffer.writeln('$key: $value');
}
}
return buffer.toString();
}
}
/// Computes the arithmetic mean (or average) of given [values].
double _computeAverage(String label, Iterable<double> values) {
if (values.isEmpty) {
throw StateError('$label: attempted to compute an average of an empty value list.');
}
final double sum = values.reduce((double a, double b) => a + b);
return sum / values.length;
}
/// Computes population standard deviation.
///
/// Unlike sample standard deviation, which divides by N - 1, this divides by N.
///
/// See also:
///
/// * <https://en.wikipedia.org/wiki/Standard_deviation>
double _computeStandardDeviationForPopulation(String label, Iterable<double> population) {
if (population.isEmpty) {
throw StateError('$label: attempted to compute the standard deviation of empty population.');
}
final double mean = _computeAverage(label, population);
final double sumOfSquaredDeltas = population.fold<double>(
0.0,
(double previous, double value) => previous += math.pow(value - mean, 2),
);
return math.sqrt(sumOfSquaredDeltas / population.length);
}
String _ratioToPercent(double value) {
return '${(value * 100).toStringAsFixed(2)}%';
}
/// Implemented by recorders that use [_RecordingWidgetsBinding] to receive
/// frame life-cycle calls.
abstract class FrameRecorder {
/// Add a callback that will be called by the recorder when it stops recording.
void registerDidStop(VoidCallback cb);
/// Called just before calling [SchedulerBinding.handleDrawFrame].
void frameWillDraw();
/// Called immediately after calling [SchedulerBinding.handleDrawFrame].
void frameDidDraw();
/// Reports an error.
///
/// The implementation is expected to halt benchmark execution as soon as possible.
void _onError(Object error, StackTrace? stackTrace);
}
/// A variant of [WidgetsBinding] that collaborates with a [Recorder] to decide
/// when to stop pumping frames.
///
/// A normal [WidgetsBinding] typically always pumps frames whenever a widget
/// instructs it to do so by calling [scheduleFrame] (transitively via
/// `setState`). This binding will stop pumping new frames as soon as benchmark
/// parameters are satisfactory (e.g. when the metric noise levels become low
/// enough).
class _RecordingWidgetsBinding extends BindingBase
with
GestureBinding,
SchedulerBinding,
ServicesBinding,
PaintingBinding,
SemanticsBinding,
RendererBinding,
WidgetsBinding {
@override
void initInstances() {
super.initInstances();
_instance = this;
}
/// The singleton instance of this object.
///
/// Provides access to the features exposed by this class. The binding must
/// be initialized before using this getter; this is typically done by calling
/// [_RecordingWidgetsBinding.ensureInitialized].
static _RecordingWidgetsBinding get instance => BindingBase.checkInstance(_instance);
static _RecordingWidgetsBinding? _instance;
/// Returns an instance of the [_RecordingWidgetsBinding], creating and
/// initializing it if necessary.
///
/// See also:
///
/// * [WidgetsFlutterBinding.ensureInitialized], the equivalent in the widgets framework.
static _RecordingWidgetsBinding ensureInitialized() {
if (_instance == null) {
_RecordingWidgetsBinding();
}
return instance;
}
FrameRecorder? _recorder;
bool _hasErrored = false;
/// To short-circuit all frame lifecycle methods when the benchmark has
/// stopped collecting data.
bool _benchmarkStopped = false;
void _beginRecording(FrameRecorder recorder, Widget widget) {
if (_recorder != null) {
throw Exception(
'Cannot call _RecordingWidgetsBinding._beginRecording more than once',
);
}
final FlutterExceptionHandler? originalOnError = FlutterError.onError;
recorder.registerDidStop(() {
_benchmarkStopped = true;
});
// Fail hard and fast on errors. Benchmarks should not have any errors.
FlutterError.onError = (FlutterErrorDetails details) {
_haltBenchmarkWithError(details.exception, details.stack);
originalOnError?.call(details);
};
_recorder = recorder;
runApp(widget);
}
void _haltBenchmarkWithError(Object error, StackTrace? stackTrace) {
if (_hasErrored) {
return;
}
_recorder?._onError(error, stackTrace);
_hasErrored = true;
}
@override
void handleBeginFrame(Duration? rawTimeStamp) {
// Don't keep on truckin' if there's an error or the benchmark has stopped.
if (_hasErrored || _benchmarkStopped) {
return;
}
try {
super.handleBeginFrame(rawTimeStamp);
} catch (error, stackTrace) {
_haltBenchmarkWithError(error, stackTrace);
rethrow;
}
}
@override
void scheduleFrame() {
// Don't keep on truckin' if there's an error or the benchmark has stopped.
if (_hasErrored || _benchmarkStopped) {
return;
}
super.scheduleFrame();
}
@override
void handleDrawFrame() {
// Don't keep on truckin' if there's an error or the benchmark has stopped.
if (_hasErrored || _benchmarkStopped) {
return;
}
try {
_recorder?.frameWillDraw();
super.handleDrawFrame();
_recorder?.frameDidDraw();
} catch (error, stackTrace) {
_haltBenchmarkWithError(error, stackTrace);
rethrow;
}
}
}
int _currentFrameNumber = 1;
/// If [_calledStartMeasureFrame] is true, we have called [startMeasureFrame]
/// but have not its pairing [endMeasureFrame] yet.
///
/// This flag ensures that [startMeasureFrame] and [endMeasureFrame] are always
/// called in pairs, with [startMeasureFrame] followed by [endMeasureFrame].
bool _calledStartMeasureFrame = false;
/// Whether we are recording a measured frame.
///
/// This flag ensures that we always stop measuring a frame if we
/// have started one. Because we want to skip warm-up frames, this flag
/// is necessary.
bool _isMeasuringFrame = false;
/// Adds a marker indication the beginning of frame rendering.
///
/// This adds an event to the performance trace used to find measured frames in
/// Chrome tracing data. The tracing data contains all frames, but some
/// benchmarks are only interested in a subset of frames. For example,
/// [WidgetBuildRecorder] only measures frames that build widgets, and ignores
/// frames that clear the screen.
///
/// Warm-up frames are not measured. If [profile.isWarmingUp] is true,
/// this function does nothing.
void startMeasureFrame(Profile profile) {
if (_calledStartMeasureFrame) {
throw Exception('`startMeasureFrame` called twice in a row.');
}
_calledStartMeasureFrame = true;
if (!profile.isWarmingUp) {
// Tell the browser to mark the beginning of the frame.
web.window.performance.mark('measured_frame_start#$_currentFrameNumber');
_isMeasuringFrame = true;
}
}
/// Signals the end of a measured frame.
///
/// See [startMeasureFrame] for details on what this instrumentation is used
/// for.
///
/// Warm-up frames are not measured. If [profile.isWarmingUp] was true
/// when the corresponding [startMeasureFrame] was called,
/// this function does nothing.
void endMeasureFrame() {
if (!_calledStartMeasureFrame) {
throw Exception('`startMeasureFrame` has not been called before calling `endMeasureFrame`');
}
_calledStartMeasureFrame = false;
if (_isMeasuringFrame) {
// Tell the browser to mark the end of the frame, and measure the duration.
web.window.performance.mark('measured_frame_end#$_currentFrameNumber');
web.window.performance.measure(
'measured_frame',
'measured_frame_start#$_currentFrameNumber'.toJS,
'measured_frame_end#$_currentFrameNumber',
);
// Increment the current frame number.
_currentFrameNumber += 1;
_isMeasuringFrame = false;
}
}
/// A function that receives a benchmark value from the framework.
typedef EngineBenchmarkValueListener = void Function(num value);
// Maps from a value label name to a listener.
final Map<String, EngineBenchmarkValueListener> _engineBenchmarkListeners = <String, EngineBenchmarkValueListener>{};
/// Registers a [listener] for engine benchmark values labeled by [name].
///
/// If another listener is already registered, overrides it.
void registerEngineBenchmarkValueListener(String name, EngineBenchmarkValueListener listener) {
if (_engineBenchmarkListeners.containsKey(name)) {
throw StateError(
'A listener for "$name" is already registered.\n'
'Call `stopListeningToEngineBenchmarkValues` to unregister the previous '
'listener before registering a new one.'
);
}
if (_engineBenchmarkListeners.isEmpty) {
// The first listener is being registered. Register the global listener.
web.window['_flutter_internal_on_benchmark'.toJS] =
// Upcast to [Object] to export.
// ignore: unnecessary_cast
(_dispatchEngineBenchmarkValue as Object).toJS;
}
_engineBenchmarkListeners[name] = listener;
}
/// Stops listening to engine benchmark values under labeled by [name].
void stopListeningToEngineBenchmarkValues(String name) {
_engineBenchmarkListeners.remove(name);
if (_engineBenchmarkListeners.isEmpty) {
// The last listener unregistered. Remove the global listener.
web.window['_flutter_internal_on_benchmark'.toJS] = null;
}
}
// Dispatches a benchmark value reported by the engine to the relevant listener.
//
// If there are no listeners registered for [name], ignores the value.
void _dispatchEngineBenchmarkValue(String name, double value) {
final EngineBenchmarkValueListener? listener = _engineBenchmarkListeners[name];
if (listener != null) {
listener(value);
}
}