| // Copyright 2013 The Flutter Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:html' as html; |
| import 'dart:js'; |
| import 'dart:js_util' as js_util; |
| import 'dart:math' as math; |
| import 'dart:ui'; |
| |
| import 'package:flutter/foundation.dart'; |
| import 'package:flutter/gestures.dart'; |
| import 'package:flutter/rendering.dart'; |
| import 'package:flutter/scheduler.dart'; |
| import 'package:flutter/services.dart'; |
| import 'package:flutter/widgets.dart'; |
| import 'package:meta/meta.dart'; |
| |
| import 'common.dart'; |
| |
| /// The number of samples from warm-up iterations. |
| /// |
| /// We warm-up the benchmark prior to measuring to allow JIT and caches to settle. |
| const int _kWarmUpSampleCount = 200; |
| |
| /// The total number of samples collected by a benchmark. |
| const int kTotalSampleCount = _kWarmUpSampleCount + kMeasuredSampleCount; |
| |
| /// A benchmark metric that includes frame-related computations prior to |
| /// submitting layer and picture operations to the underlying renderer, such as |
| /// HTML and CanvasKit. During this phase we compute transforms, clips, and |
| /// other information needed for rendering. |
| const String kProfilePrerollFrame = 'preroll_frame'; |
| |
| /// A benchmark metric that includes submitting layer and picture information |
| /// to the renderer. |
| const String kProfileApplyFrame = 'apply_frame'; |
| |
| /// Measures the amount of time [action] takes. |
| Duration timeAction(VoidCallback action) { |
| final Stopwatch stopwatch = Stopwatch()..start(); |
| action(); |
| stopwatch.stop(); |
| return stopwatch.elapsed; |
| } |
| |
| /// A function that performs asynchronous work. |
| typedef AsyncVoidCallback = Future<void> Function(); |
| |
| /// An [AsyncVoidCallback] that doesn't do anything. |
| /// |
| /// This is used just so we don't have to deal with null all over the place. |
| Future<void> _dummyAsyncVoidCallback() async {} |
| |
| /// Runs the benchmark using the given [recorder]. |
| /// |
| /// Notifies about "set up" and "tear down" events via the [setUpAllDidRun] |
| /// and [tearDownAllWillRun] callbacks. |
| @sealed |
| class Runner { |
| /// Creates a runner for the [recorder]. |
| /// |
| /// All arguments must not be null. |
| Runner({ |
| required this.recorder, |
| this.setUpAllDidRun = _dummyAsyncVoidCallback, |
| this.tearDownAllWillRun = _dummyAsyncVoidCallback, |
| }); |
| |
| /// The recorder that will run and record the benchmark. |
| final Recorder recorder; |
| |
| /// Called immediately after [Recorder.setUpAll] future is resolved. |
| /// |
| /// This is useful, for example, to kick off a profiler or a tracer such that |
| /// the "set up" computations are not included in the metrics. |
| final AsyncVoidCallback setUpAllDidRun; |
| |
| /// Called just before calling [Recorder.tearDownAll]. |
| /// |
| /// This is useful, for example, to stop a profiler or a tracer such that |
| /// the "tear down" computations are not included in the metrics. |
| final AsyncVoidCallback tearDownAllWillRun; |
| |
| /// Runs the benchmark and reports the results. |
| Future<Profile> run() async { |
| await recorder.setUpAll(); |
| await setUpAllDidRun(); |
| final Profile profile = await recorder.run(); |
| await tearDownAllWillRun(); |
| await recorder.tearDownAll(); |
| return profile; |
| } |
| } |
| |
| /// Base class for benchmark recorders. |
| /// |
| /// Each benchmark recorder has a [name] and a [run] method at a minimum. |
| abstract class Recorder { |
| Recorder._(this.name, this.isTracingEnabled); |
| |
| /// Whether this recorder requires tracing using Chrome's DevTools Protocol's |
| /// "Tracing" API. |
| final bool isTracingEnabled; |
| |
| /// The name of the benchmark. |
| /// |
| /// The results displayed in the Flutter Dashboard will use this name as a |
| /// prefix. |
| final String name; |
| |
| /// Returns the recorded profile. |
| /// |
| /// This value is only available while the benchmark is running. |
| Profile? get profile; |
| |
| /// Whether the benchmark should continue running. |
| /// |
| /// Returns `false` if the benchmark collected enough data and it's time to |
| /// stop. |
| bool shouldContinue() => profile!.shouldContinue(); |
| |
| /// Called once before all runs of this benchmark recorder. |
| /// |
| /// This is useful for doing one-time setup work that's needed for the |
| /// benchmark. |
| Future<void> setUpAll() async {} |
| |
| /// The implementation of the benchmark that will produce a [Profile]. |
| Future<Profile> run(); |
| |
| /// Called once after all runs of this benchmark recorder. |
| /// |
| /// This is useful for doing one-time clean up work after the benchmark is |
| /// complete. |
| Future<void> tearDownAll() async {} |
| } |
| |
| /// A recorder for benchmarking raw execution of Dart code. |
| /// |
| /// This is useful for benchmarks that don't need frames or widgets. |
| /// |
| /// Example: |
| /// |
| /// ``` |
| /// class BenchForLoop extends RawRecorder { |
| /// BenchForLoop() : super(name: benchmarkName); |
| /// |
| /// static const String benchmarkName = 'for_loop'; |
| /// |
| /// @override |
| /// void body(Profile profile) { |
| /// profile.record('loop', () { |
| /// double x = 0; |
| /// for (int i = 0; i < 10000000; i++) { |
| /// x *= 1.5; |
| /// } |
| /// }); |
| /// } |
| /// } |
| /// ``` |
| abstract class RawRecorder extends Recorder { |
| /// Creates a raw benchmark recorder with a name. |
| /// |
| /// [name] must not be null. |
| RawRecorder({required String name}) : super._(name, false); |
| |
| /// The body of the benchmark. |
| /// |
| /// This is the part that records measurements of the benchmark. |
| void body(Profile profile); |
| |
| @override |
| Profile get profile => _profile; |
| late Profile _profile; |
| |
| @override |
| @nonVirtual |
| Future<Profile> run() async { |
| _profile = Profile(name: name); |
| do { |
| await Future<void>.delayed(Duration.zero); |
| body(_profile); |
| } while (shouldContinue()); |
| return _profile; |
| } |
| } |
| |
| /// A recorder for benchmarking interactions with the engine without the |
| /// framework by directly exercising [SceneBuilder]. |
| /// |
| /// To implement a benchmark, extend this class and implement [onDrawFrame]. |
| /// |
| /// Example: |
| /// |
| /// ``` |
| /// class BenchDrawCircle extends SceneBuilderRecorder { |
| /// BenchDrawCircle() : super(name: benchmarkName); |
| /// |
| /// static const String benchmarkName = 'draw_circle'; |
| /// |
| /// @override |
| /// void onDrawFrame(SceneBuilder sceneBuilder) { |
| /// final PictureRecorder pictureRecorder = PictureRecorder(); |
| /// final Canvas canvas = Canvas(pictureRecorder); |
| /// final Paint paint = Paint()..color = const Color.fromARGB(255, 255, 0, 0); |
| /// final Size windowSize = window.physicalSize; |
| /// canvas.drawCircle(windowSize.center(Offset.zero), 50.0, paint); |
| /// final Picture picture = pictureRecorder.endRecording(); |
| /// sceneBuilder.addPicture(picture); |
| /// } |
| /// } |
| /// ``` |
| abstract class SceneBuilderRecorder extends Recorder { |
| /// Creates a [SceneBuilder] benchmark recorder. |
| /// |
| /// [name] must not be null. |
| SceneBuilderRecorder({required String name}) : super._(name, true); |
| |
| @override |
| Profile get profile => _profile; |
| late Profile _profile; |
| |
| /// Called from [Window.onBeginFrame]. |
| @mustCallSuper |
| void onBeginFrame() {} |
| |
| /// Called on every frame. |
| /// |
| /// An implementation should exercise the [sceneBuilder] to build a frame. |
| /// However, it must not call [SceneBuilder.build] or [Window.render]. |
| /// Instead the benchmark harness will call them and time them appropriately. |
| void onDrawFrame(SceneBuilder sceneBuilder); |
| |
| @override |
| Future<Profile> run() { |
| final Completer<Profile> profileCompleter = Completer<Profile>(); |
| _profile = Profile(name: name); |
| |
| window.onBeginFrame = (_) { |
| try { |
| startMeasureFrame(profile); |
| onBeginFrame(); |
| } catch (error, stackTrace) { |
| profileCompleter.completeError(error, stackTrace); |
| rethrow; |
| } |
| }; |
| window.onDrawFrame = () { |
| try { |
| _profile.record('drawFrameDuration', () { |
| final SceneBuilder sceneBuilder = SceneBuilder(); |
| onDrawFrame(sceneBuilder); |
| _profile.record('sceneBuildDuration', () { |
| final Scene scene = sceneBuilder.build(); |
| _profile.record('windowRenderDuration', () { |
| window.render(scene); |
| }, reported: false); |
| }, reported: false); |
| }, reported: true); |
| endMeasureFrame(); |
| |
| if (shouldContinue()) { |
| window.scheduleFrame(); |
| } else { |
| profileCompleter.complete(_profile); |
| } |
| } catch (error, stackTrace) { |
| profileCompleter.completeError(error, stackTrace); |
| rethrow; |
| } |
| }; |
| window.scheduleFrame(); |
| return profileCompleter.future; |
| } |
| } |
| |
| /// A recorder for benchmarking interactions with the framework by creating |
| /// widgets. |
| /// |
| /// To implement a benchmark, extend this class and implement [createWidget]. |
| /// |
| /// Example: |
| /// |
| /// ``` |
| /// class BenchListView extends WidgetRecorder { |
| /// BenchListView() : super(name: benchmarkName); |
| /// |
| /// static const String benchmarkName = 'bench_list_view'; |
| /// |
| /// @override |
| /// Widget createWidget() { |
| /// return Directionality( |
| /// textDirection: TextDirection.ltr, |
| /// child: _TestListViewWidget(), |
| /// ); |
| /// } |
| /// } |
| /// |
| /// class _TestListViewWidget extends StatefulWidget { |
| /// @override |
| /// State<StatefulWidget> createState() { |
| /// return _TestListViewWidgetState(); |
| /// } |
| /// } |
| /// |
| /// class _TestListViewWidgetState extends State<_TestListViewWidget> { |
| /// ScrollController scrollController; |
| /// |
| /// @override |
| /// void initState() { |
| /// super.initState(); |
| /// scrollController = ScrollController(); |
| /// Timer.run(() async { |
| /// bool forward = true; |
| /// while (true) { |
| /// await scrollController.animateTo( |
| /// forward ? 300 : 0, |
| /// curve: Curves.linear, |
| /// duration: const Duration(seconds: 1), |
| /// ); |
| /// forward = !forward; |
| /// } |
| /// }); |
| /// } |
| /// |
| /// @override |
| /// Widget build(BuildContext context) { |
| /// return ListView.builder( |
| /// controller: scrollController, |
| /// itemCount: 10000, |
| /// itemBuilder: (BuildContext context, int index) { |
| /// return Text('Item #$index'); |
| /// }, |
| /// ); |
| /// } |
| /// } |
| /// ``` |
| abstract class WidgetRecorder extends Recorder implements FrameRecorder { |
| /// Creates a widget benchmark recorder. |
| /// |
| /// [name] must not be null. |
| /// |
| /// If [useCustomWarmUp] is true, delegates the benchmark warm-up to the |
| /// benchmark implementation instead of using a built-in strategy. The |
| /// benchmark is expected to call [Profile.stopWarmingUp] to signal that |
| /// the warm-up phase is finished. |
| WidgetRecorder({ |
| required String name, |
| this.useCustomWarmUp = false, |
| }) : super._(name, true); |
| |
| /// Creates a widget to be benchmarked. |
| /// |
| /// The widget must create its own animation to drive the benchmark. The |
| /// animation should continue indefinitely. The benchmark harness will stop |
| /// pumping frames automatically. |
| Widget createWidget(); |
| |
| final List<VoidCallback> _didStopCallbacks = <VoidCallback>[]; |
| |
| @override |
| void registerDidStop(VoidCallback fn) { |
| _didStopCallbacks.add(fn); |
| } |
| |
| @override |
| late Profile profile; |
| |
| // This will be initialized in [run]. |
| late Completer<void> _runCompleter; |
| |
| /// Whether to delimit warm-up frames in a custom way. |
| final bool useCustomWarmUp; |
| |
| late Stopwatch _drawFrameStopwatch; |
| |
| @override |
| @mustCallSuper |
| void frameWillDraw() { |
| startMeasureFrame(profile); |
| _drawFrameStopwatch = Stopwatch()..start(); |
| } |
| |
| @override |
| @mustCallSuper |
| void frameDidDraw() { |
| endMeasureFrame(); |
| profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, |
| reported: true); |
| |
| if (shouldContinue()) { |
| window.scheduleFrame(); |
| } else { |
| for (final VoidCallback fn in _didStopCallbacks) { |
| fn(); |
| } |
| _runCompleter.complete(); |
| } |
| } |
| |
| @override |
| void _onError(dynamic error, StackTrace? stackTrace) { |
| _runCompleter.completeError(error as Object, stackTrace); |
| } |
| |
| @override |
| Future<Profile> run() async { |
| _runCompleter = Completer<void>(); |
| final Profile localProfile = |
| profile = Profile(name: name, useCustomWarmUp: useCustomWarmUp); |
| final _RecordingWidgetsBinding binding = |
| _RecordingWidgetsBinding.ensureInitialized(); |
| final Widget widget = createWidget(); |
| |
| registerEngineBenchmarkValueListener(kProfilePrerollFrame, (num value) { |
| localProfile.addDataPoint( |
| kProfilePrerollFrame, |
| Duration(microseconds: value.toInt()), |
| reported: false, |
| ); |
| }); |
| registerEngineBenchmarkValueListener(kProfileApplyFrame, (num value) { |
| localProfile.addDataPoint( |
| kProfileApplyFrame, |
| Duration(microseconds: value.toInt()), |
| reported: false, |
| ); |
| }); |
| |
| binding._beginRecording(this, widget); |
| |
| try { |
| await _runCompleter.future; |
| return localProfile; |
| } finally { |
| stopListeningToEngineBenchmarkValues(kProfilePrerollFrame); |
| stopListeningToEngineBenchmarkValues(kProfileApplyFrame); |
| } |
| } |
| } |
| |
| /// A recorder for measuring the performance of building a widget from scratch |
| /// starting from an empty frame. |
| /// |
| /// The recorder will call [createWidget] and render it, then it will pump |
| /// another frame that clears the screen. It repeats this process, measuring the |
| /// performance of frames that render the widget and ignoring the frames that |
| /// clear the screen. |
| abstract class WidgetBuildRecorder extends Recorder implements FrameRecorder { |
| /// Creates a widget build benchmark recorder. |
| /// |
| /// [name] must not be null. |
| WidgetBuildRecorder({required String name}) : super._(name, true); |
| |
| /// Creates a widget to be benchmarked. |
| /// |
| /// The widget is not expected to animate as we only care about construction |
| /// of the widget. If you are interested in benchmarking an animation, |
| /// consider using [WidgetRecorder]. |
| Widget createWidget(); |
| |
| final List<VoidCallback> _didStopCallbacks = <VoidCallback>[]; |
| |
| @override |
| void registerDidStop(VoidCallback fn) { |
| _didStopCallbacks.add(fn); |
| } |
| |
| @override |
| late Profile profile; |
| Completer<void>? _runCompleter; |
| |
| late Stopwatch _drawFrameStopwatch; |
| |
| /// Whether in this frame we should call [createWidget] and render it. |
| /// |
| /// If false, then this frame will clear the screen. |
| bool showWidget = true; |
| |
| /// The state that hosts the widget under test. |
| late _WidgetBuildRecorderHostState _hostState; |
| |
| Widget? _getWidgetForFrame() { |
| if (showWidget) { |
| return createWidget(); |
| } else { |
| return null; |
| } |
| } |
| |
| @override |
| @mustCallSuper |
| void frameWillDraw() { |
| if (showWidget) { |
| startMeasureFrame(profile); |
| _drawFrameStopwatch = Stopwatch()..start(); |
| } |
| } |
| |
| @override |
| @mustCallSuper |
| void frameDidDraw() { |
| // Only record frames that show the widget. |
| if (showWidget) { |
| endMeasureFrame(); |
| profile.addDataPoint('drawFrameDuration', _drawFrameStopwatch.elapsed, |
| reported: true); |
| } |
| |
| if (shouldContinue()) { |
| showWidget = !showWidget; |
| _hostState._setStateTrampoline(); |
| } else { |
| for (final VoidCallback fn in _didStopCallbacks) { |
| fn(); |
| } |
| _runCompleter!.complete(); |
| } |
| } |
| |
| @override |
| void _onError(dynamic error, StackTrace? stackTrace) { |
| _runCompleter!.completeError(error as Object, stackTrace); |
| } |
| |
| @override |
| Future<Profile> run() async { |
| _runCompleter = Completer<void>(); |
| final Profile localProfile = profile = Profile(name: name); |
| final _RecordingWidgetsBinding binding = |
| _RecordingWidgetsBinding.ensureInitialized(); |
| binding._beginRecording(this, _WidgetBuildRecorderHost(this)); |
| |
| try { |
| await _runCompleter!.future; |
| return localProfile; |
| } finally { |
| _runCompleter = null; |
| } |
| } |
| } |
| |
| /// Hosts widgets created by [WidgetBuildRecorder]. |
| class _WidgetBuildRecorderHost extends StatefulWidget { |
| const _WidgetBuildRecorderHost(this.recorder); |
| |
| final WidgetBuildRecorder recorder; |
| |
| @override |
| State<StatefulWidget> createState() => _WidgetBuildRecorderHostState(); |
| } |
| |
| class _WidgetBuildRecorderHostState extends State<_WidgetBuildRecorderHost> { |
| @override |
| void initState() { |
| super.initState(); |
| widget.recorder._hostState = this; |
| } |
| |
| // This is just to bypass the @protected on setState. |
| void _setStateTrampoline() { |
| setState(() {}); |
| } |
| |
| @override |
| Widget build(BuildContext context) { |
| return SizedBox.expand( |
| child: widget.recorder._getWidgetForFrame(), |
| ); |
| } |
| } |
| |
| /// Series of time recordings indexed in time order. |
| /// |
| /// It can calculate [average], [standardDeviation] and [noise]. If the amount |
| /// of data collected is higher than [_kMeasuredSampleCount], then these |
| /// calculations will only apply to the latest [_kMeasuredSampleCount] data |
| /// points. |
| class Timeseries { |
| /// Creates an empty timeseries. |
| /// |
| /// [name], [isReported], and [useCustomWarmUp] must not be null. |
| Timeseries(this.name, this.isReported, {this.useCustomWarmUp = false}) |
| : _warmUpFrameCount = useCustomWarmUp ? 0 : null; |
| |
| /// The label of this timeseries used for debugging and result inspection. |
| final String name; |
| |
| /// Whether this timeseries is reported to the benchmark dashboard. |
| /// |
| /// If `true` a new benchmark card is created for the timeseries and is |
| /// visible on the dashboard. |
| /// |
| /// If `false` the data is stored but it does not show up on the dashboard. |
| /// Use unreported metrics for metrics that are useful for manual inspection |
| /// but that are too fine-grained to be useful for tracking on the dashboard. |
| final bool isReported; |
| |
| /// Whether to delimit warm-up frames in a custom way. |
| final bool useCustomWarmUp; |
| |
| /// The number of frames ignored as warm-up frames, used only |
| /// when [useCustomWarmUp] is true. |
| int? _warmUpFrameCount; |
| |
| /// The number of frames ignored as warm-up frames. |
| int get warmUpFrameCount => |
| useCustomWarmUp ? _warmUpFrameCount! : count - kMeasuredSampleCount; |
| |
| /// List of all the values that have been recorded. |
| /// |
| /// This list has no limit. |
| final List<double> _allValues = <double>[]; |
| |
| /// The total amount of data collected, including ones that were dropped |
| /// because of the sample size limit. |
| int get count => _allValues.length; |
| |
| /// Extracts useful statistics out of this timeseries. |
| /// |
| /// See [TimeseriesStats] for more details. |
| TimeseriesStats computeStats() { |
| final int finalWarmUpFrameCount = warmUpFrameCount; |
| |
| assert(finalWarmUpFrameCount >= 0 && finalWarmUpFrameCount < count); |
| |
| // The first few values we simply discard and never look at. They're from the warm-up phase. |
| final List<double> warmUpValues = |
| _allValues.sublist(0, finalWarmUpFrameCount); |
| |
| // Values we analyze. |
| final List<double> candidateValues = |
| _allValues.sublist(finalWarmUpFrameCount); |
| |
| // The average that includes outliers. |
| final double dirtyAverage = _computeAverage(name, candidateValues); |
| |
| // The standard deviation that includes outliers. |
| final double dirtyStandardDeviation = |
| _computeStandardDeviationForPopulation(name, candidateValues); |
| |
| // Any value that's higher than this is considered an outlier. |
| final double outlierCutOff = dirtyAverage + dirtyStandardDeviation; |
| |
| // Candidates with outliers removed. |
| final Iterable<double> cleanValues = |
| candidateValues.where((double value) => value <= outlierCutOff); |
| |
| // Outlier candidates. |
| final Iterable<double> outliers = |
| candidateValues.where((double value) => value > outlierCutOff); |
| |
| // Final statistics. |
| final double cleanAverage = _computeAverage(name, cleanValues); |
| final double standardDeviation = |
| _computeStandardDeviationForPopulation(name, cleanValues); |
| final double noise = |
| cleanAverage > 0.0 ? standardDeviation / cleanAverage : 0.0; |
| |
| // Compute outlier average. If there are no outliers the outlier average is |
| // the same as clean value average. In other words, in a perfect benchmark |
| // with no noise the difference between average and outlier average is zero, |
| // which the best possible outcome. Noise produces a positive difference |
| // between the two. |
| final double outlierAverage = |
| outliers.isNotEmpty ? _computeAverage(name, outliers) : cleanAverage; |
| |
| final List<AnnotatedSample> annotatedValues = <AnnotatedSample>[ |
| for (final double warmUpValue in warmUpValues) |
| AnnotatedSample( |
| magnitude: warmUpValue, |
| isOutlier: warmUpValue > outlierCutOff, |
| isWarmUpValue: true, |
| ), |
| for (final double candidate in candidateValues) |
| AnnotatedSample( |
| magnitude: candidate, |
| isOutlier: candidate > outlierCutOff, |
| isWarmUpValue: false, |
| ), |
| ]; |
| |
| return TimeseriesStats( |
| name: name, |
| average: cleanAverage, |
| outlierCutOff: outlierCutOff, |
| outlierAverage: outlierAverage, |
| standardDeviation: standardDeviation, |
| noise: noise, |
| cleanSampleCount: cleanValues.length, |
| outlierSampleCount: outliers.length, |
| samples: annotatedValues, |
| ); |
| } |
| |
| /// Adds a value to this timeseries. |
| void add(double value, {required bool isWarmUpValue}) { |
| if (value < 0.0) { |
| throw StateError( |
| 'Timeseries $name: negative metric values are not supported. Got: $value', |
| ); |
| } |
| _allValues.add(value); |
| if (useCustomWarmUp && isWarmUpValue) { |
| _warmUpFrameCount = warmUpFrameCount + 1; |
| } |
| } |
| } |
| |
| /// Various statistics about a [Timeseries]. |
| /// |
| /// See the docs on the individual fields for more details. |
| @sealed |
| class TimeseriesStats { |
| /// Creates statistics for a time series. |
| const TimeseriesStats({ |
| required this.name, |
| required this.average, |
| required this.outlierCutOff, |
| required this.outlierAverage, |
| required this.standardDeviation, |
| required this.noise, |
| required this.cleanSampleCount, |
| required this.outlierSampleCount, |
| required this.samples, |
| }); |
| |
| /// The label used to refer to the corresponding timeseries. |
| final String name; |
| |
| /// The average value of the measured samples without outliers. |
| final double average; |
| |
| /// The standard deviation in the measured samples without outliers. |
| final double standardDeviation; |
| |
| /// The noise as a multiple of the [average] value takes from clean samples. |
| /// |
| /// This value can be multiplied by 100.0 to get noise as a percentage of |
| /// the average. |
| /// |
| /// If [average] is zero, treats the result as perfect score, returns zero. |
| final double noise; |
| |
| /// The maximum value a sample can have without being considered an outlier. |
| /// |
| /// See [Timeseries.computeStats] for details on how this value is computed. |
| final double outlierCutOff; |
| |
| /// The average of outlier samples. |
| /// |
| /// This value can be used to judge how badly we jank, when we jank. |
| /// |
| /// Another useful metrics is the difference between [outlierAverage] and |
| /// [average]. The smaller the value the more predictable is the performance |
| /// of the corresponding benchmark. |
| final double outlierAverage; |
| |
| /// The number of measured samples after outlier are removed. |
| final int cleanSampleCount; |
| |
| /// The number of outliers. |
| final int outlierSampleCount; |
| |
| /// All collected samples, annotated with statistical information. |
| /// |
| /// See [AnnotatedSample] for more details. |
| final List<AnnotatedSample> samples; |
| |
| /// Outlier average divided by clean average. |
| /// |
| /// This is a measure of performance consistency. The higher this number the |
| /// worse is jank when it happens. Smaller is better, with 1.0 being the |
| /// perfect score. If [average] is zero, this value defaults to 1.0. |
| double get outlierRatio => average > 0.0 |
| ? outlierAverage / average |
| : 1.0; // this can only happen in perfect benchmark that reports only zeros |
| |
| @override |
| String toString() { |
| final StringBuffer buffer = StringBuffer(); |
| buffer.writeln( |
| '$name: (samples: $cleanSampleCount clean/$outlierSampleCount ' |
| 'outliers/${cleanSampleCount + outlierSampleCount} ' |
| 'measured/${samples.length} total)', |
| ); |
| buffer.writeln(' | average: $average μs'); |
| buffer.writeln(' | outlier average: $outlierAverage μs'); |
| buffer.writeln(' | outlier/clean ratio: ${outlierRatio}x'); |
| buffer.writeln(' | noise: ${_ratioToPercent(noise)}'); |
| return buffer.toString(); |
| } |
| } |
| |
| /// Annotates a single measurement with statistical information. |
| @sealed |
| class AnnotatedSample { |
| /// Creates an annotated measurement sample. |
| const AnnotatedSample({ |
| required this.magnitude, |
| required this.isOutlier, |
| required this.isWarmUpValue, |
| }); |
| |
| /// The non-negative raw result of the measurement. |
| final double magnitude; |
| |
| /// Whether this sample was considered an outlier. |
| final bool isOutlier; |
| |
| /// Whether this sample was taken during the warm-up phase. |
| /// |
| /// If this value is `true`, this sample does not participate in |
| /// statistical computations. However, the sample would still be |
| /// shown in the visualization of results so that the benchmark |
| /// can be inspected manually to make sure there's a predictable |
| /// warm-up regression slope. |
| final bool isWarmUpValue; |
| } |
| |
| /// Base class for a profile collected from running a benchmark. |
| class Profile { |
| /// Creates an empty profile. |
| /// |
| /// [name] and [useCustomWarmUp] must not be null. |
| Profile({required this.name, this.useCustomWarmUp = false}) |
| : assert(name != null), |
| _isWarmingUp = useCustomWarmUp; |
| |
| /// The name of the benchmark that produced this profile. |
| final String name; |
| |
| /// Whether to delimit warm-up frames in a custom way. |
| final bool useCustomWarmUp; |
| |
| /// Whether we are measuring warm-up frames currently. |
| bool get isWarmingUp => _isWarmingUp; |
| |
| bool _isWarmingUp; |
| |
| /// Stop the warm-up phase. |
| /// |
| /// Call this method only when [useCustomWarmUp] and [isWarmingUp] are both |
| /// true. |
| /// Call this method only once for each profile. |
| void stopWarmingUp() { |
| if (!useCustomWarmUp) { |
| throw Exception( |
| '`stopWarmingUp` should be used only when `useCustomWarmUp` is true.'); |
| } else if (!_isWarmingUp) { |
| throw Exception('Warm-up already stopped.'); |
| } else { |
| _isWarmingUp = false; |
| } |
| } |
| |
| /// This data will be used to display cards in the Flutter Dashboard. |
| final Map<String, Timeseries> scoreData = <String, Timeseries>{}; |
| |
| /// This data isn't displayed anywhere. It's stored for completeness purposes. |
| final Map<String, dynamic> extraData = <String, dynamic>{}; |
| |
| /// Invokes [callback] and records the duration of its execution under [key]. |
| Duration record(String key, VoidCallback callback, {required bool reported}) { |
| final Duration duration = timeAction(callback); |
| addDataPoint(key, duration, reported: reported); |
| return duration; |
| } |
| |
| /// Adds a timed sample to the timeseries corresponding to [key]. |
| /// |
| /// Set [reported] to `true` to report the timeseries to the dashboard UI. |
| /// |
| /// Set [reported] to `false` to store the data, but not show it on the |
| /// dashboard UI. |
| void addDataPoint(String key, Duration duration, {required bool reported}) { |
| scoreData |
| .putIfAbsent( |
| key, |
| () => Timeseries(key, reported, useCustomWarmUp: useCustomWarmUp), |
| ) |
| .add(duration.inMicroseconds.toDouble(), isWarmUpValue: isWarmingUp); |
| } |
| |
| /// Decides whether the data collected so far is sufficient to stop, or |
| /// whether the benchmark should continue collecting more data. |
| /// |
| /// The signals used are sample size, noise, and duration. |
| /// |
| /// If any of the timeseries doesn't satisfy the noise requirements, this |
| /// method will return true (asking the benchmark to continue collecting |
| /// data). |
| bool shouldContinue() { |
| // If there are no `Timeseries` in the `scoreData`, then we haven't |
| // recorded anything yet. Don't stop. |
| if (scoreData.isEmpty) { |
| return true; |
| } |
| |
| // We have recorded something, but do we have enough samples? If every |
| // timeseries has collected enough samples, stop the benchmark. |
| return !scoreData.keys |
| .every((String key) => scoreData[key]!.count >= kTotalSampleCount); |
| } |
| |
| /// Returns a JSON representation of the profile that will be sent to the |
| /// server. |
| Map<String, dynamic> toJson() { |
| final List<String> scoreKeys = <String>[]; |
| final Map<String, dynamic> json = <String, dynamic>{ |
| 'name': name, |
| 'scoreKeys': scoreKeys, |
| }; |
| |
| for (final String key in scoreData.keys) { |
| final Timeseries timeseries = scoreData[key]!; |
| |
| if (timeseries.isReported) { |
| scoreKeys.add('$key.average'); |
| // Report `outlierRatio` rather than `outlierAverage`, because |
| // the absolute value of outliers is less interesting than the |
| // ratio. |
| scoreKeys.add('$key.outlierRatio'); |
| } |
| |
| final TimeseriesStats stats = timeseries.computeStats(); |
| json['$key.average'] = stats.average; |
| json['$key.outlierAverage'] = stats.outlierAverage; |
| json['$key.outlierRatio'] = stats.outlierRatio; |
| json['$key.noise'] = stats.noise; |
| } |
| |
| json.addAll(extraData); |
| |
| return json; |
| } |
| |
| @override |
| String toString() { |
| final StringBuffer buffer = StringBuffer(); |
| buffer.writeln('name: $name'); |
| for (final String key in scoreData.keys) { |
| final Timeseries timeseries = scoreData[key]!; |
| final TimeseriesStats stats = timeseries.computeStats(); |
| buffer.writeln(stats.toString()); |
| } |
| for (final String key in extraData.keys) { |
| final dynamic value = extraData[key]; |
| if (value is List) { |
| buffer.writeln('$key:'); |
| for (final dynamic item in value) { |
| buffer.writeln(' - $item'); |
| } |
| } else { |
| buffer.writeln('$key: $value'); |
| } |
| } |
| return buffer.toString(); |
| } |
| } |
| |
| /// Computes the arithmetic mean (or average) of given [values]. |
| double _computeAverage(String label, Iterable<double> values) { |
| if (values.isEmpty) { |
| throw StateError( |
| '$label: attempted to compute an average of an empty value list.'); |
| } |
| |
| final double sum = values.reduce((double a, double b) => a + b); |
| return sum / values.length; |
| } |
| |
| /// Computes population standard deviation. |
| /// |
| /// Unlike sample standard deviation, which divides by N - 1, this divides by N. |
| /// |
| /// See also: |
| /// |
| /// * https://en.wikipedia.org/wiki/Standard_deviation |
| double _computeStandardDeviationForPopulation( |
| String label, Iterable<double> population) { |
| if (population.isEmpty) { |
| throw StateError( |
| '$label: attempted to compute the standard deviation of empty population.'); |
| } |
| final double mean = _computeAverage(label, population); |
| final double sumOfSquaredDeltas = population.fold<double>( |
| 0.0, |
| (double previous, double value) => previous += math.pow(value - mean, 2), |
| ); |
| return math.sqrt(sumOfSquaredDeltas / population.length); |
| } |
| |
| String _ratioToPercent(double value) { |
| return '${(value * 100).toStringAsFixed(2)}%'; |
| } |
| |
| /// Implemented by recorders that use [_RecordingWidgetsBinding] to receive |
| /// frame life-cycle calls. |
| abstract class FrameRecorder { |
| /// Add a callback that will be called by the recorder when it stops recording. |
| void registerDidStop(VoidCallback cb); |
| |
| /// Called just before calling [SchedulerBinding.handleDrawFrame]. |
| void frameWillDraw(); |
| |
| /// Called immediately after calling [SchedulerBinding.handleDrawFrame]. |
| void frameDidDraw(); |
| |
| /// Reports an error. |
| /// |
| /// The implementation is expected to halt benchmark execution as soon as possible. |
| void _onError(dynamic error, StackTrace? stackTrace); |
| } |
| |
| /// A variant of [WidgetsBinding] that collaborates with a [Recorder] to decide |
| /// when to stop pumping frames. |
| /// |
| /// A normal [WidgetsBinding] typically always pumps frames whenever a widget |
| /// instructs it to do so by calling [scheduleFrame] (transitively via |
| /// `setState`). This binding will stop pumping new frames as soon as benchmark |
| /// parameters are satisfactory (e.g. when the metric noise levels become low |
| /// enough). |
| class _RecordingWidgetsBinding extends BindingBase |
| with |
| GestureBinding, |
| SchedulerBinding, |
| ServicesBinding, |
| PaintingBinding, |
| SemanticsBinding, |
| RendererBinding, |
| WidgetsBinding { |
| @override |
| void initInstances() { |
| super.initInstances(); |
| _instance = this; |
| } |
| |
| static _RecordingWidgetsBinding get instance => |
| BindingBase.checkInstance(_instance); |
| static _RecordingWidgetsBinding? _instance; |
| |
| /// Makes an instance of [_RecordingWidgetsBinding] the current binding. |
| static _RecordingWidgetsBinding ensureInitialized() { |
| if (_instance == null) { |
| _RecordingWidgetsBinding(); |
| } |
| return _RecordingWidgetsBinding.instance; |
| } |
| |
| // This will be not null when the benchmark is running. |
| FrameRecorder? _recorder; |
| bool _hasErrored = false; |
| |
| /// To short-circuit all frame lifecycle methods when the benchmark has |
| /// stopped collecting data. |
| bool _benchmarkStopped = false; |
| |
| void _beginRecording(FrameRecorder recorder, Widget widget) { |
| if (_recorder != null) { |
| throw Exception( |
| 'Cannot call _RecordingWidgetsBinding._beginRecording more than once', |
| ); |
| } |
| final FlutterExceptionHandler? originalOnError = FlutterError.onError; |
| |
| recorder.registerDidStop(() { |
| _benchmarkStopped = true; |
| }); |
| |
| // Fail hard and fast on errors. Benchmarks should not have any errors. |
| FlutterError.onError = (FlutterErrorDetails details) { |
| _haltBenchmarkWithError(details.exception, details.stack); |
| originalOnError!(details); |
| }; |
| _recorder = recorder; |
| runApp(widget); |
| } |
| |
| void _haltBenchmarkWithError(dynamic error, StackTrace? stackTrace) { |
| if (_hasErrored) { |
| return; |
| } |
| _recorder!._onError(error, stackTrace); |
| _hasErrored = true; |
| } |
| |
| @override |
| void handleBeginFrame(Duration? rawTimeStamp) { |
| // Don't keep on truckin' if there's an error or the benchmark has stopped. |
| if (_hasErrored || _benchmarkStopped) { |
| return; |
| } |
| try { |
| super.handleBeginFrame(rawTimeStamp); |
| } catch (error, stackTrace) { |
| _haltBenchmarkWithError(error, stackTrace); |
| rethrow; |
| } |
| } |
| |
| @override |
| void scheduleFrame() { |
| // Don't keep on truckin' if there's an error or the benchmark has stopped. |
| if (_hasErrored || _benchmarkStopped) { |
| return; |
| } |
| super.scheduleFrame(); |
| } |
| |
| @override |
| void handleDrawFrame() { |
| // Don't keep on truckin' if there's an error or the benchmark has stopped. |
| if (_hasErrored || _benchmarkStopped) { |
| return; |
| } |
| try { |
| _recorder!.frameWillDraw(); |
| super.handleDrawFrame(); |
| _recorder!.frameDidDraw(); |
| } catch (error, stackTrace) { |
| _haltBenchmarkWithError(error, stackTrace); |
| rethrow; |
| } |
| } |
| } |
| |
| int _currentFrameNumber = 1; |
| |
| /// If [_calledStartMeasureFrame] is true, we have called [startMeasureFrame] |
| /// but have not its pairing [endMeasureFrame] yet. |
| /// |
| /// This flag ensures that [startMeasureFrame] and [endMeasureFrame] are always |
| /// called in pairs, with [startMeasureFrame] followed by [endMeasureFrame]. |
| bool _calledStartMeasureFrame = false; |
| |
| /// Whether we are recording a measured frame. |
| /// |
| /// This flag ensures that we always stop measuring a frame if we |
| /// have started one. Because we want to skip warm-up frames, this flag |
| /// is necessary. |
| bool _isMeasuringFrame = false; |
| |
| /// Adds a marker indication the beginning of frame rendering. |
| /// |
| /// This adds an event to the performance trace used to find measured frames in |
| /// Chrome tracing data. The tracing data contains all frames, but some |
| /// benchmarks are only interested in a subset of frames. For example, |
| /// [WidgetBuildRecorder] only measures frames that build widgets, and ignores |
| /// frames that clear the screen. |
| /// |
| /// Warm-up frames are not measured. If [profile.isWarmingUp] is true, |
| /// this function does nothing. |
| void startMeasureFrame(Profile profile) { |
| if (_calledStartMeasureFrame) { |
| throw Exception('`startMeasureFrame` called twice in a row.'); |
| } |
| |
| _calledStartMeasureFrame = true; |
| |
| if (!profile.isWarmingUp) { |
| // Tell the browser to mark the beginning of the frame. |
| html.window.performance.mark('measured_frame_start#$_currentFrameNumber'); |
| |
| _isMeasuringFrame = true; |
| } |
| } |
| |
| /// Signals the end of a measured frame. |
| /// |
| /// See [startMeasureFrame] for details on what this instrumentation is used |
| /// for. |
| /// |
| /// Warm-up frames are not measured. If [profile.isWarmingUp] was true |
| /// when the corresponding [startMeasureFrame] was called, |
| /// this function does nothing. |
| void endMeasureFrame() { |
| if (!_calledStartMeasureFrame) { |
| throw Exception( |
| '`startMeasureFrame` has not been called before calling `endMeasureFrame`'); |
| } |
| |
| _calledStartMeasureFrame = false; |
| |
| if (_isMeasuringFrame) { |
| // Tell the browser to mark the end of the frame, and measure the duration. |
| html.window.performance.mark('measured_frame_end#$_currentFrameNumber'); |
| html.window.performance.measure( |
| 'measured_frame', |
| 'measured_frame_start#$_currentFrameNumber', |
| 'measured_frame_end#$_currentFrameNumber', |
| ); |
| |
| // Increment the current frame number. |
| _currentFrameNumber += 1; |
| |
| _isMeasuringFrame = false; |
| } |
| } |
| |
| /// A function that receives a benchmark value from the framework. |
| typedef EngineBenchmarkValueListener = void Function(num value); |
| |
| // Maps from a value label name to a listener. |
| final Map<String, EngineBenchmarkValueListener> _engineBenchmarkListeners = |
| <String, EngineBenchmarkValueListener>{}; |
| |
| /// Registers a [listener] for engine benchmark values labeled by [name]. |
| /// |
| /// If another listener is already registered, overrides it. |
| void registerEngineBenchmarkValueListener( |
| String name, EngineBenchmarkValueListener listener) { |
| if (listener == null) { |
| throw ArgumentError( |
| 'Listener must not be null. To stop listening to engine benchmark values ' |
| 'under label "$name", call stopListeningToEngineBenchmarkValues(\'$name\').', |
| ); |
| } |
| |
| if (_engineBenchmarkListeners.containsKey(name)) { |
| throw StateError('A listener for "$name" is already registered.\n' |
| 'Call `stopListeningToEngineBenchmarkValues` to unregister the previous ' |
| 'listener before registering a new one.'); |
| } |
| |
| if (_engineBenchmarkListeners.isEmpty) { |
| // The first listener is being registered. Register the global listener. |
| js_util.setProperty(html.window, '_flutter_internal_on_benchmark', |
| allowInterop(_dispatchEngineBenchmarkValue)); |
| } |
| |
| _engineBenchmarkListeners[name] = listener; |
| } |
| |
| /// Stops listening to engine benchmark values under labeled by [name]. |
| void stopListeningToEngineBenchmarkValues(String name) { |
| _engineBenchmarkListeners.remove(name); |
| if (_engineBenchmarkListeners.isEmpty) { |
| // The last listener unregistered. Remove the global listener. |
| js_util.setProperty(html.window, '_flutter_internal_on_benchmark', null); |
| } |
| } |
| |
| // Dispatches a benchmark value reported by the engine to the relevant listener. |
| // |
| // If there are no listeners registered for [name], ignores the value. |
| void _dispatchEngineBenchmarkValue(String name, double value) { |
| final EngineBenchmarkValueListener? listener = |
| _engineBenchmarkListeners[name]; |
| if (listener != null) { |
| listener(value); |
| } |
| } |