blob: 03de1085e75b9a4ab5b5623dd89810aeffa1a3d4 [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JsonEncoder, LineSplitter, json, utf8;
import 'dart:io' as io;
import 'dart:math' as math;
import 'package:path/path.dart' as path;
import 'package:webkit_inspection_protocol/webkit_inspection_protocol.dart';
/// The number of samples used to extract metrics, such as noise, means,
/// max/min values.
///
/// Keep this constant in sync with the same constant defined in `dev/benchmarks/macrobenchmarks/lib/src/web/recorder.dart`.
const int _kMeasuredSampleCount = 10;
/// Options passed to Chrome when launching it.
class ChromeOptions {
ChromeOptions({
this.userDataDirectory,
this.url,
this.windowWidth = 1024,
this.windowHeight = 1024,
this.headless,
this.debugPort,
});
/// If not null passed as `--user-data-dir`.
final String? userDataDirectory;
/// If not null launches a Chrome tab at this URL.
final String? url;
/// The width of the Chrome window.
///
/// This is important for screenshots and benchmarks.
final int windowWidth;
/// The height of the Chrome window.
///
/// This is important for screenshots and benchmarks.
final int windowHeight;
/// Launches code in "headless" mode, which allows running Chrome in
/// environments without a display, such as LUCI and Cirrus.
final bool? headless;
/// The port Chrome will use for its debugging protocol.
///
/// If null, Chrome is launched without debugging. When running in headless
/// mode without a debug port, Chrome quits immediately. For most tests it is
/// typical to set [headless] to true and set a non-null debug port.
final int? debugPort;
}
/// A function called when the Chrome process encounters an error.
typedef ChromeErrorCallback = void Function(String);
/// Manages a single Chrome process.
class Chrome {
Chrome._(this._chromeProcess, this._onError, this._debugConnection) {
// If the Chrome process quits before it was asked to quit, notify the
// error listener.
_chromeProcess.exitCode.then((int exitCode) {
if (!_isStopped) {
_onError('Chrome process exited prematurely with exit code $exitCode');
}
});
}
/// Launches Chrome with the give [options].
///
/// The [onError] callback is called with an error message when the Chrome
/// process encounters an error. In particular, [onError] is called when the
/// Chrome process exits prematurely, i.e. before [stop] is called.
static Future<Chrome> launch(ChromeOptions options, { String? workingDirectory, required ChromeErrorCallback onError }) async {
if (!io.Platform.isWindows) {
final io.ProcessResult versionResult = io.Process.runSync(_findSystemChromeExecutable(), const <String>['--version']);
print('Launching ${versionResult.stdout}');
} else {
print('Launching Chrome...');
}
final bool withDebugging = options.debugPort != null;
final List<String> args = <String>[
if (options.userDataDirectory != null)
'--user-data-dir=${options.userDataDirectory}',
if (options.url != null)
options.url!,
if (io.Platform.environment['CHROME_NO_SANDBOX'] == 'true')
'--no-sandbox',
if (options.headless ?? false)
'--headless',
if (withDebugging)
'--remote-debugging-port=${options.debugPort}',
'--window-size=${options.windowWidth},${options.windowHeight}',
'--disable-extensions',
'--disable-popup-blocking',
// Indicates that the browser is in "browse without sign-in" (Guest session) mode.
'--bwsi',
'--no-first-run',
'--no-default-browser-check',
'--disable-default-apps',
'--disable-translate',
];
final io.Process chromeProcess = await _spawnChromiumProcess(
_findSystemChromeExecutable(),
args,
workingDirectory: workingDirectory,
);
WipConnection? debugConnection;
if (withDebugging) {
debugConnection = await _connectToChromeDebugPort(chromeProcess, options.debugPort!);
}
return Chrome._(chromeProcess, onError, debugConnection);
}
final io.Process _chromeProcess;
final ChromeErrorCallback _onError;
final WipConnection? _debugConnection;
bool _isStopped = false;
Completer<void> ?_tracingCompleter;
StreamSubscription<WipEvent>? _tracingSubscription;
List<Map<String, dynamic>>? _tracingData;
/// Starts recording a performance trace.
///
/// If there is already a tracing session in progress, throws an error. Call
/// [endRecordingPerformance] before starting a new tracing session.
///
/// The [label] is for debugging convenience.
Future<void> beginRecordingPerformance(String label) async {
if (_tracingCompleter != null) {
throw StateError(
'Cannot start a new performance trace. A tracing session labeled '
'"$label" is already in progress.'
);
}
_tracingCompleter = Completer<void>();
_tracingData = <Map<String, dynamic>>[];
// Subscribe to tracing events prior to calling "Tracing.start". Otherwise,
// we'll miss tracing data.
_tracingSubscription = _debugConnection?.onNotification.listen((WipEvent event) {
// We receive data as a sequence of "Tracing.dataCollected" followed by
// "Tracing.tracingComplete" at the end. Until "Tracing.tracingComplete"
// is received, the data may be incomplete.
if (event.method == 'Tracing.tracingComplete') {
_tracingCompleter!.complete();
_tracingSubscription!.cancel();
_tracingSubscription = null;
} else if (event.method == 'Tracing.dataCollected') {
final dynamic value = event.params?['value'];
if (value is! List) {
throw FormatException('"Tracing.dataCollected" returned malformed data. '
'Expected a List but got: ${value.runtimeType}');
}
_tracingData?.addAll((event.params?['value'] as List<dynamic>).cast<Map<String, dynamic>>());
}
});
await _debugConnection?.sendCommand('Tracing.start', <String, dynamic>{
// The choice of categories is as follows:
//
// blink:
// provides everything on the UI thread, including scripting,
// style recalculations, layout, painting, and some compositor
// work.
// blink.user_timing:
// provides marks recorded using window.performance. We use marks
// to find frames that the benchmark cares to measure.
// gpu:
// provides tracing data from the GPU data
// disabled due to https://bugs.chromium.org/p/chromium/issues/detail?id=1068259
// TODO(yjbanov): extract useful GPU data
'categories': 'blink,blink.user_timing',
'transferMode': 'SendAsStream',
});
}
/// Stops a performance tracing session started by [beginRecordingPerformance].
///
/// Returns all the collected tracing data unfiltered.
Future<List<Map<String, dynamic>>?> endRecordingPerformance() async {
await _debugConnection!.sendCommand('Tracing.end');
await _tracingCompleter!.future;
final List<Map<String, dynamic>>? data = _tracingData;
_tracingCompleter = null;
_tracingData = null;
return data;
}
Future<void> reloadPage({bool ignoreCache = false}) async {
await _debugConnection?.page.reload(ignoreCache: ignoreCache);
}
/// Stops the Chrome process.
void stop() {
_isStopped = true;
_tracingSubscription?.cancel();
_chromeProcess.kill();
}
}
String _findSystemChromeExecutable() {
// On some environments, such as the Dart HHH tester, Chrome resides in a
// non-standard location and is provided via the following environment
// variable.
final String? envExecutable = io.Platform.environment['CHROME_EXECUTABLE'];
if (envExecutable != null) {
return envExecutable;
}
if (io.Platform.isLinux) {
final io.ProcessResult which =
io.Process.runSync('which', <String>['google-chrome']);
if (which.exitCode != 0) {
throw Exception('Failed to locate system Chrome installation.');
}
return (which.stdout as String).trim();
} else if (io.Platform.isMacOS) {
return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
} else if (io.Platform.isWindows) {
const String kWindowsExecutable = r'Google\Chrome\Application\chrome.exe';
final List<String> kWindowsPrefixes = <String?>[
io.Platform.environment['LOCALAPPDATA'],
io.Platform.environment['PROGRAMFILES'],
io.Platform.environment['PROGRAMFILES(X86)'],
].whereType<String>().toList();
final String windowsPrefix = kWindowsPrefixes.firstWhere((String prefix) {
final String expectedPath = path.join(prefix, kWindowsExecutable);
return io.File(expectedPath).existsSync();
}, orElse: () => '.');
return path.join(windowsPrefix, kWindowsExecutable);
} else {
throw Exception('Web benchmarks cannot run on ${io.Platform.operatingSystem}.');
}
}
/// Waits for Chrome to print DevTools URI and connects to it.
Future<WipConnection> _connectToChromeDebugPort(io.Process chromeProcess, int port) async {
final Uri devtoolsUri = await _getRemoteDebuggerUrl(Uri.parse('http://localhost:$port'));
print('Connecting to DevTools: $devtoolsUri');
final ChromeConnection chromeConnection = ChromeConnection('localhost', port);
final Iterable<ChromeTab> tabs = (await chromeConnection.getTabs()).where((ChromeTab tab) {
return tab.url.startsWith('http://localhost');
});
final ChromeTab tab = tabs.single;
final WipConnection debugConnection = await tab.connect();
print('Connected to Chrome tab: ${tab.title} (${tab.url})');
return debugConnection;
}
/// Gets the Chrome debugger URL for the web page being benchmarked.
Future<Uri> _getRemoteDebuggerUrl(Uri base) async {
final io.HttpClient client = io.HttpClient();
final io.HttpClientRequest request = await client.getUrl(base.resolve('/json/list'));
final io.HttpClientResponse response = await request.close();
final List<dynamic>? jsonObject = await json.fuse(utf8).decoder.bind(response).single as List<dynamic>?;
if (jsonObject == null || jsonObject.isEmpty) {
return base;
}
return base.resolve((jsonObject.first as Map<String, dynamic>)['webSocketDebuggerUrl'] as String);
}
/// Summarizes a Blink trace down to a few interesting values.
class BlinkTraceSummary {
BlinkTraceSummary._({
required this.averageBeginFrameTime,
required this.averageUpdateLifecyclePhasesTime,
}) : averageTotalUIFrameTime = averageBeginFrameTime + averageUpdateLifecyclePhasesTime;
static BlinkTraceSummary? fromJson(List<Map<String, dynamic>> traceJson) {
try {
// Convert raw JSON data to BlinkTraceEvent objects sorted by timestamp.
List<BlinkTraceEvent> events = traceJson
.map<BlinkTraceEvent>(BlinkTraceEvent.fromJson)
.toList()
..sort((BlinkTraceEvent a, BlinkTraceEvent b) => a.ts! - b.ts!);
Exception noMeasuredFramesFound() => Exception(
'No measured frames found in benchmark tracing data. This likely '
'indicates a bug in the benchmark. For example, the benchmark failed '
"to pump enough frames. It may also indicate a change in Chrome's "
'tracing data format. Check if Chrome version changed recently and '
'adjust the parsing code accordingly.',
);
// Use the pid from the first "measured_frame" event since the event is
// emitted by the script running on the process we're interested in.
//
// We previously tried using the "CrRendererMain" event. However, for
// reasons unknown, Chrome in the devicelab refuses to emit this event
// sometimes, causing to flakes.
final BlinkTraceEvent firstMeasuredFrameEvent = events.firstWhere(
(BlinkTraceEvent event) => event.isBeginMeasuredFrame,
orElse: () => throw noMeasuredFramesFound(),
);
if (firstMeasuredFrameEvent == null) {
// This happens in benchmarks that do not measure frames, such as some
// of the text layout benchmarks.
return null;
}
final int tabPid = firstMeasuredFrameEvent.pid!;
// Filter out data from unrelated processes
events = events.where((BlinkTraceEvent element) => element.pid == tabPid).toList();
// Extract frame data.
final List<BlinkFrame> frames = <BlinkFrame>[];
int skipCount = 0;
BlinkFrame frame = BlinkFrame();
for (final BlinkTraceEvent event in events) {
if (event.isBeginFrame) {
frame.beginFrame = event;
} else if (event.isUpdateAllLifecyclePhases) {
frame.updateAllLifecyclePhases = event;
if (frame.endMeasuredFrame != null) {
frames.add(frame);
} else {
skipCount += 1;
}
frame = BlinkFrame();
} else if (event.isBeginMeasuredFrame) {
frame.beginMeasuredFrame = event;
} else if (event.isEndMeasuredFrame) {
frame.endMeasuredFrame = event;
}
}
print('Extracted ${frames.length} measured frames.');
print('Skipped $skipCount non-measured frames.');
if (frames.isEmpty) {
throw noMeasuredFramesFound();
}
// Compute averages and summarize.
return BlinkTraceSummary._(
averageBeginFrameTime: _computeAverageDuration(frames.map((BlinkFrame frame) => frame.beginFrame).whereType<BlinkTraceEvent>().toList()),
averageUpdateLifecyclePhasesTime: _computeAverageDuration(frames.map((BlinkFrame frame) => frame.updateAllLifecyclePhases).whereType<BlinkTraceEvent>().toList()),
);
} catch (_) {
final io.File traceFile = io.File('./chrome-trace.json');
io.stderr.writeln('Failed to interpret the Chrome trace contents. The trace was saved in ${traceFile.path}');
traceFile.writeAsStringSync(const JsonEncoder.withIndent(' ').convert(traceJson));
rethrow;
}
}
/// The average duration of "WebViewImpl::beginFrame" events.
///
/// This event contains all of scripting time of an animation frame, plus an
/// unknown small amount of work browser does before and after scripting.
final Duration averageBeginFrameTime;
/// The average duration of "WebViewImpl::updateAllLifecyclePhases" events.
///
/// This event contains style, layout, painting, and compositor computations,
/// which are not included in the scripting time. This event does not
/// include GPU time, which happens on a separate thread.
final Duration averageUpdateLifecyclePhasesTime;
/// The average sum of [averageBeginFrameTime] and
/// [averageUpdateLifecyclePhasesTime].
///
/// This value contains the vast majority of work the UI thread performs in
/// any given animation frame.
final Duration averageTotalUIFrameTime;
@override
String toString() => '$BlinkTraceSummary('
'averageBeginFrameTime: ${averageBeginFrameTime.inMicroseconds / 1000}ms, '
'averageUpdateLifecyclePhasesTime: ${averageUpdateLifecyclePhasesTime.inMicroseconds / 1000}ms)';
}
/// Contains events pertaining to a single frame in the Blink trace data.
class BlinkFrame {
/// Corresponds to 'WebViewImpl::beginFrame' event.
BlinkTraceEvent? beginFrame;
/// Corresponds to 'WebViewImpl::updateAllLifecyclePhases' event.
BlinkTraceEvent? updateAllLifecyclePhases;
/// Corresponds to 'measured_frame' begin event.
BlinkTraceEvent? beginMeasuredFrame;
/// Corresponds to 'measured_frame' end event.
BlinkTraceEvent? endMeasuredFrame;
}
/// Takes a list of events that have non-null [BlinkTraceEvent.tdur] computes
/// their average as a [Duration] value.
Duration _computeAverageDuration(List<BlinkTraceEvent> events) {
// Compute the sum of "tdur" fields of the last _kMeasuredSampleCount events.
final double sum = events
.skip(math.max(events.length - _kMeasuredSampleCount, 0))
.fold(0.0, (double previousValue, BlinkTraceEvent event) {
if (event.tdur == null) {
throw FormatException('Trace event lacks "tdur" field: $event');
}
return previousValue + event.tdur!;
});
final int sampleCount = math.min(events.length, _kMeasuredSampleCount);
return Duration(microseconds: sum ~/ sampleCount);
}
/// An event collected by the Blink tracer (in Chrome accessible using chrome://tracing).
///
/// See also:
/// * https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
class BlinkTraceEvent {
BlinkTraceEvent._({
required this.args,
required this.cat,
required this.name,
required this.ph,
this.pid,
this.tid,
this.ts,
this.tts,
this.tdur,
});
/// Parses an event from its JSON representation.
///
/// Sample event encoded as JSON (the data is bogus, this just shows the format):
///
/// ```
/// {
/// "name": "myName",
/// "cat": "category,list",
/// "ph": "B",
/// "ts": 12345,
/// "pid": 123,
/// "tid": 456,
/// "args": {
/// "someArg": 1,
/// "anotherArg": {
/// "value": "my value"
/// }
/// }
/// }
/// ```
///
/// For detailed documentation of the format see:
///
/// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview
static BlinkTraceEvent fromJson(Map<String, dynamic> json) {
return BlinkTraceEvent._(
args: json['args'] as Map<String, dynamic>,
cat: json['cat'] as String,
name: json['name'] as String,
ph: json['ph'] as String,
pid: _readInt(json, 'pid'),
tid: _readInt(json, 'tid'),
ts: _readInt(json, 'ts'),
tts: _readInt(json, 'tts'),
tdur: _readInt(json, 'tdur'),
);
}
/// Event-specific data.
final Map<String, dynamic> args;
/// Event category.
final String cat;
/// Event name.
final String name;
/// Event "phase".
final String ph;
/// Process ID of the process that emitted the event.
final int? pid;
/// Thread ID of the thread that emitted the event.
final int? tid;
/// Timestamp in microseconds using tracer clock.
final int? ts;
/// Timestamp in microseconds using thread clock.
final int? tts;
/// Event duration in microseconds.
final int? tdur;
/// A "begin frame" event contains all of the scripting time of an animation
/// frame (JavaScript, WebAssembly), plus a negligible amount of internal
/// browser overhead.
///
/// This event does not include non-UI thread scripting, such as web workers,
/// service workers, and CSS Paint paintlets.
///
/// WebViewImpl::beginFrame was used in earlier versions of Chrome, kept
/// for compatibility.
///
/// This event is a duration event that has its `tdur` populated.
bool get isBeginFrame {
return ph == 'X' && (
name == 'WebViewImpl::beginFrame' ||
name == 'WebFrameWidgetBase::BeginMainFrame' ||
name == 'WebFrameWidgetImpl::BeginMainFrame'
);
}
/// An "update all lifecycle phases" event contains UI thread computations
/// related to an animation frame that's outside the scripting phase.
///
/// This event includes style recalculation, layer tree update, layout,
/// painting, and parts of compositing work.
///
/// WebViewImpl::updateAllLifecyclePhases was used in earlier versions of
/// Chrome, kept for compatibility.
///
/// This event is a duration event that has its `tdur` populated.
bool get isUpdateAllLifecyclePhases {
return ph == 'X' && (
name == 'WebViewImpl::updateAllLifecyclePhases' ||
name == 'WebFrameWidgetImpl::UpdateLifecycle'
);
}
/// Whether this is the beginning of a "measured_frame" event.
///
/// This event is a custom event emitted by our benchmark test harness.
///
/// See also:
/// * `recorder.dart`, which emits this event.
bool get isBeginMeasuredFrame => ph == 'b' && name == 'measured_frame';
/// Whether this is the end of a "measured_frame" event.
///
/// This event is a custom event emitted by our benchmark test harness.
///
/// See also:
/// * `recorder.dart`, which emits this event.
bool get isEndMeasuredFrame => ph == 'e' && name == 'measured_frame';
@override
String toString() => '$BlinkTraceEvent('
'args: ${json.encode(args)}, '
'cat: $cat, '
'name: $name, '
'ph: $ph, '
'pid: $pid, '
'tid: $tid, '
'ts: $ts, '
'tts: $tts, '
'tdur: $tdur)';
}
/// Read an integer out of [json] stored under [key].
///
/// Since JSON does not distinguish between `int` and `double`, extra
/// validation and conversion is needed.
///
/// Returns null if the value is null.
int? _readInt(Map<String, dynamic> json, String key) {
final num? jsonValue = json[key] as num?;
if (jsonValue == null) {
return null;
}
return jsonValue.toInt();
}
/// Used by [Chrome.launch] to detect a glibc bug and retry launching the
/// browser.
///
/// Once every few thousands of launches we hit this glibc bug:
///
/// https://sourceware.org/bugzilla/show_bug.cgi?id=19329.
///
/// When this happens Chrome spits out something like the following then exits with code 127:
///
/// Inconsistency detected by ld.so: ../elf/dl-tls.c: 493: _dl_allocate_tls_init: Assertion `listp->slotinfo[cnt].gen <= GL(dl_tls_generation)' failed!
const String _kGlibcError = 'Inconsistency detected by ld.so';
Future<io.Process> _spawnChromiumProcess(String executable, List<String> args, { String? workingDirectory }) async {
// Keep attempting to launch the browser until one of:
// - Chrome launched successfully, in which case we just return from the loop.
// - The tool detected an unretryable Chrome error, in which case we throw ToolExit.
while (true) {
final io.Process process = await io.Process.start(executable, args, workingDirectory: workingDirectory);
process.stdout
.transform(utf8.decoder)
.transform(const LineSplitter())
.listen((String line) {
print('[CHROME STDOUT]: $line');
});
// Wait until the DevTools are listening before trying to connect. This is
// only required for flutter_test --platform=chrome and not flutter run.
bool hitGlibcBug = false;
await process.stderr
.transform(utf8.decoder)
.transform(const LineSplitter())
.map((String line) {
print('[CHROME STDERR]:$line');
if (line.contains(_kGlibcError)) {
hitGlibcBug = true;
}
return line;
})
.firstWhere((String line) => line.startsWith('DevTools listening'), orElse: () {
if (hitGlibcBug) {
print(
'Encountered glibc bug https://sourceware.org/bugzilla/show_bug.cgi?id=19329. '
'Will try launching browser again.',
);
return '';
}
print('Failed to launch browser. Command used to launch it: ${args.join(' ')}');
throw Exception(
'Failed to launch browser. Make sure you are using an up-to-date '
'Chrome or Edge. Otherwise, consider using -d web-server instead '
'and filing an issue at https://github.com/flutter/flutter/issues.',
);
});
if (!hitGlibcBug) {
return process;
}
// A precaution that avoids accumulating browser processes, in case the
// glibc bug doesn't cause the browser to quit and we keep looping and
// launching more processes.
unawaited(process.exitCode.timeout(const Duration(seconds: 1), onTimeout: () {
process.kill();
return 0;
}));
}
}