blob: 48bf20dd0edbf5904308c25819873e54d540226d [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show LineSplitter, json, utf8;
import 'dart:io';
import 'dart:math' as math;
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import 'package:xml/xml.dart';
import '../framework/devices.dart';
import '../framework/framework.dart';
import '../framework/host_agent.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
/// Must match flutter_driver/lib/src/common.dart.
///
/// Redefined here to avoid taking a dependency on flutter_driver.
String _testOutputDirectory(String testDirectory) {
return Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '$testDirectory/build';
}
TaskFunction createComplexLayoutScrollPerfTest({
bool measureCpuGpu = true,
bool badScroll = false,
bool? enableImpeller,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
badScroll
? 'test_driver/scroll_perf_bad.dart'
: 'test_driver/scroll_perf.dart',
'complex_layout_scroll_perf',
measureCpuGpu: measureCpuGpu,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createTilesScrollPerfTest({bool? enableImpeller}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
'test_driver/scroll_perf.dart',
'tiles_scroll_perf',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createUiKitViewScrollPerfTest({bool? enableImpeller}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
'test_driver/uikit_view_scroll_perf.dart',
'platform_views_scroll_perf',
testDriver: 'test_driver/scroll_perf_test.dart',
needsFullTimeline: false,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createUiKitViewScrollPerfNonIntersectingTest({bool? enableImpeller}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
'test_driver/uikit_view_scroll_perf_non_intersecting.dart',
'platform_views_scroll_perf_non_intersecting',
testDriver: 'test_driver/scroll_perf_non_intersecting_test.dart',
needsFullTimeline: false,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createAndroidTextureScrollPerfTest({bool? enableImpeller}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/platform_views_layout',
'test_driver/android_view_scroll_perf.dart',
'platform_views_scroll_perf',
testDriver: 'test_driver/scroll_perf_test.dart',
needsFullTimeline: false,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createAndroidViewScrollPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/platform_views_layout_hybrid_composition',
'test_driver/android_view_scroll_perf.dart',
'platform_views_scroll_perf_hybrid_composition',
testDriver: 'test_driver/scroll_perf_test.dart',
).run;
}
TaskFunction createHomeScrollPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
'test_driver/scroll_perf.dart',
'home_scroll_perf',
).run;
}
TaskFunction createCullOpacityPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'cull_opacity_perf',
testDriver: 'test_driver/cull_opacity_perf_test.dart',
).run;
}
TaskFunction createCullOpacityPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/cull_opacity_perf_e2e.dart',
).run;
}
TaskFunction createCubicBezierPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'cubic_bezier_perf',
testDriver: 'test_driver/cubic_bezier_perf_test.dart',
).run;
}
TaskFunction createCubicBezierPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/cubic_bezier_perf_e2e.dart',
).run;
}
TaskFunction createBackdropFilterPerfTest({
bool measureCpuGpu = true,
bool? enableImpeller,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'backdrop_filter_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/backdrop_filter_perf_test.dart',
saveTraceFile: true,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createAnimationWithMicrotasksPerfTest({bool measureCpuGpu = true}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'animation_with_microtasks_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/animation_with_microtasks_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createBackdropFilterPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/backdrop_filter_perf_e2e.dart',
).run;
}
TaskFunction createPostBackdropFilterPerfTest({bool measureCpuGpu = true}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'post_backdrop_filter_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/post_backdrop_filter_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createSimpleAnimationPerfTest({
bool measureCpuGpu = true,
bool? enableImpeller,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'simple_animation_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/simple_animation_perf_test.dart',
saveTraceFile: true,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createAnimatedPlaceholderPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/animated_placeholder_perf_e2e.dart',
).run;
}
TaskFunction createPictureCachePerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'picture_cache_perf',
testDriver: 'test_driver/picture_cache_perf_test.dart',
).run;
}
TaskFunction createPictureCachePerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/picture_cache_perf_e2e.dart',
).run;
}
TaskFunction createPictureCacheComplexityScoringPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'picture_cache_complexity_scoring_perf',
testDriver: 'test_driver/picture_cache_complexity_scoring_perf_test.dart',
).run;
}
TaskFunction createOpenPayScrollPerfTest({bool measureCpuGpu = true}) {
return PerfTest(
openpayDirectory.path,
'test_driver/scroll_perf.dart',
'openpay_scroll_perf',
measureCpuGpu: measureCpuGpu,
testDriver: 'test_driver/scroll_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createFlutterGalleryStartupTest({String target = 'lib/main.dart', Map<String, String>? runEnvironment}) {
return StartupTest(
'${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
target: target,
runEnvironment: runEnvironment,
).run;
}
TaskFunction createComplexLayoutStartupTest() {
return StartupTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
).run;
}
TaskFunction createFlutterGalleryCompileTest() {
return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
}
TaskFunction createHelloWorldCompileTest() {
return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
}
TaskFunction createWebCompileTest() {
return const WebCompileTest().run;
}
TaskFunction createFlutterViewStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/flutter_view',
reportMetrics: false,
).run;
}
TaskFunction createPlatformViewStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/platform_view',
reportMetrics: false,
).run;
}
TaskFunction createBasicMaterialCompileTest() {
return () async {
const String sampleAppName = 'sample_flutter_app';
final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
rmTree(sampleDir);
await inDirectory<void>(Directory.systemTemp, () async {
await flutter('create', options: <String>['--template=app', sampleAppName]);
});
if (!sampleDir.existsSync()) {
throw 'Failed to create default Flutter app in ${sampleDir.path}';
}
return CompileTest(sampleDir.path).run();
};
}
TaskFunction createTextfieldPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'textfield_perf',
testDriver: 'test_driver/textfield_perf_test.dart',
).run;
}
TaskFunction createTextfieldPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/textfield_perf_e2e.dart',
).run;
}
TaskFunction createVeryLongPictureScrollingPerfE2ETest({required bool enableImpeller}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/very_long_picture_scrolling_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createSlidersPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'sliders_perf',
testDriver: 'test_driver/sliders_perf_test.dart',
).run;
}
TaskFunction createStackSizeTest() {
final String testDirectory =
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
const String testTarget = 'test_driver/run_app.dart';
const String testDriver = 'test_driver/stack_size_perf_test.dart';
return () {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
await flutter('packages', options: <String>['get']);
await flutter('drive', options: <String>[
'--no-android-gradle-daemon',
'-v',
'--verbose-system-logs',
'--profile',
'-t', testTarget,
'--driver', testDriver,
'-d',
deviceId,
]);
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/stack_size.json').readAsStringSync(),
) as Map<String, dynamic>;
final Map<String, dynamic> result = <String, dynamic>{
'stack_size_per_nesting_level': data['stack_size'],
};
return TaskResult.success(
result,
benchmarkScoreKeys: result.keys.toList(),
);
});
};
}
TaskFunction createFullscreenTextfieldPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'fullscreen_textfield_perf',
testDriver: 'test_driver/fullscreen_textfield_perf_test.dart',
).run;
}
TaskFunction createFullscreenTextfieldPerfE2ETest({
bool? enableImpeller,
}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/fullscreen_textfield_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createClipperCachePerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/clipper_cache_perf_e2e.dart',
).run;
}
TaskFunction createColorFilterAndFadePerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'color_filter_and_fade_perf',
testDriver: 'test_driver/color_filter_and_fade_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createColorFilterAndFadePerfE2ETest({bool? enableImpeller}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/color_filter_and_fade_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createColorFilterCachePerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/color_filter_cache_perf_e2e.dart',
).run;
}
TaskFunction createColorFilterWithUnstableChildPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/color_filter_with_unstable_child_perf_e2e.dart',
).run;
}
TaskFunction createRasterCacheUseMemoryPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/raster_cache_use_memory_perf_e2e.dart',
).run;
}
TaskFunction createShaderMaskCachePerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/shader_mask_cache_perf_e2e.dart',
).run;
}
TaskFunction createFadingChildAnimationPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'fading_child_animation_perf',
testDriver: 'test_driver/fading_child_animation_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createImageFilteredTransformAnimationPerfTest({
bool? enableImpeller,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'imagefiltered_transform_animation_perf',
testDriver: 'test_driver/imagefiltered_transform_animation_perf_test.dart',
saveTraceFile: true,
enableImpeller: enableImpeller,
).run;
}
TaskFunction createsMultiWidgetConstructPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/multi_widget_construction_perf_e2e.dart',
).run;
}
TaskFunction createListTextLayoutPerfE2ETest({bool? enableImpeller}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/list_text_layout_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createsScrollSmoothnessPerfTest() {
final String testDirectory =
'${flutterDirectory.path}/dev/benchmarks/complex_layout';
const String testTarget = 'test/measure_scroll_smoothness.dart';
return () {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
await flutter('packages', options: <String>['get']);
await flutter('drive', options: <String>[
'--no-android-gradle-daemon',
'-v',
'--verbose-system-logs',
'--profile',
'-t', testTarget,
'-d',
deviceId,
]);
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/scroll_smoothness_test.json').readAsStringSync(),
) as Map<String, dynamic>;
final Map<String, dynamic> result = <String, dynamic>{};
void addResult(dynamic data, String suffix) {
assert(data is Map<String, dynamic>);
if (data is Map<String, dynamic>) {
const List<String> metricKeys = <String>[
'janky_count',
'average_abs_jerk',
'dropped_frame_count',
];
for (final String key in metricKeys) {
result[key + suffix] = data[key];
}
}
}
addResult(data['resample on with 90Hz input'], '_with_resampler_90Hz');
addResult(data['resample on with 59Hz input'], '_with_resampler_59Hz');
addResult(data['resample off with 90Hz input'], '_without_resampler_90Hz');
addResult(data['resample off with 59Hz input'], '_without_resampler_59Hz');
return TaskResult.success(
result,
benchmarkScoreKeys: result.keys.toList(),
);
});
};
}
TaskFunction createFramePolicyIntegrationTest() {
final String testDirectory =
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
const String testTarget = 'test/frame_policy.dart';
return () {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
await flutter('packages', options: <String>['get']);
await flutter('drive', options: <String>[
'--no-android-gradle-daemon',
'-v',
'--verbose-system-logs',
'--profile',
'-t', testTarget,
'-d',
deviceId,
]);
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/frame_policy_event_delay.json').readAsStringSync(),
) as Map<String, dynamic>;
final Map<String, dynamic> fullLiveData = data['fullyLive'] as Map<String, dynamic>;
final Map<String, dynamic> benchmarkLiveData = data['benchmarkLive'] as Map<String, dynamic>;
final Map<String, dynamic> dataFormatted = <String, dynamic>{
'average_delay_fullyLive_millis':
fullLiveData['average_delay_millis'],
'average_delay_benchmarkLive_millis':
benchmarkLiveData['average_delay_millis'],
'90th_percentile_delay_fullyLive_millis':
fullLiveData['90th_percentile_delay_millis'],
'90th_percentile_delay_benchmarkLive_millis':
benchmarkLiveData['90th_percentile_delay_millis'],
};
return TaskResult.success(
dataFormatted,
benchmarkScoreKeys: dataFormatted.keys.toList(),
);
});
};
}
TaskFunction createOpacityPeepholeOneRectPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_one_rect_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeColOfRowsPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_col_of_rows_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeOpacityOfGridPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_opacity_of_grid_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeGridOfOpacityPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_grid_of_opacity_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeFadeTransitionTextPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_fade_transition_text_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeGridOfAlphaSaveLayersPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_grid_of_alpha_savelayers_perf_e2e.dart',
).run;
}
TaskFunction createOpacityPeepholeColOfAlphaSaveLayerRowsPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/opacity_peephole_col_of_alpha_savelayer_rows_perf_e2e.dart',
).run;
}
TaskFunction createGradientDynamicPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/gradient_dynamic_perf_e2e.dart',
).run;
}
TaskFunction createGradientConsistentPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/gradient_consistent_perf_e2e.dart',
).run;
}
TaskFunction createGradientStaticPerfE2ETest() {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/gradient_static_perf_e2e.dart',
).run;
}
TaskFunction createAnimatedAdvancedBlendPerfTest({
bool? enableImpeller,
bool? forceOpenGLES,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'animated_advanced_blend_perf',
enableImpeller: enableImpeller,
forceOpenGLES: forceOpenGLES,
testDriver: 'test_driver/animated_advanced_blend_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createAnimatedBlurBackropFilterPerfTest({
bool? enableImpeller,
bool? forceOpenGLES,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'animated_blur_backdrop_filter_perf',
enableImpeller: enableImpeller,
forceOpenGLES: forceOpenGLES,
testDriver: 'test_driver/animated_blur_backdrop_filter_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createDrawPointsPerfTest({
bool? enableImpeller,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'draw_points_perf',
enableImpeller: enableImpeller,
testDriver: 'test_driver/draw_points_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createDrawAtlasPerfTest({
bool? forceOpenGLES,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'draw_atlas_perf',
enableImpeller: true,
testDriver: 'test_driver/draw_atlas_perf_test.dart',
saveTraceFile: true,
forceOpenGLES: forceOpenGLES,
).run;
}
TaskFunction createDrawVerticesPerfTest({
bool? forceOpenGLES,
}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'draw_vertices_perf',
enableImpeller: true,
testDriver: 'test_driver/draw_vertices_perf_test.dart',
saveTraceFile: true,
forceOpenGLES: forceOpenGLES,
).run;
}
TaskFunction createPathTessellationStaticPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'tessellation_perf_static',
enableImpeller: true,
testDriver: 'test_driver/path_tessellation_static_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createPathTessellationDynamicPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/run_app.dart',
'tessellation_perf_dynamic',
enableImpeller: true,
testDriver: 'test_driver/path_tessellation_dynamic_perf_test.dart',
saveTraceFile: true,
).run;
}
TaskFunction createAnimatedComplexOpacityPerfE2ETest({
bool? enableImpeller,
}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/animated_complex_opacity_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
TaskFunction createAnimatedComplexImageFilteredPerfE2ETest({
bool? enableImpeller,
}) {
return PerfTest.e2e(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test/animated_complex_image_filtered_perf_e2e.dart',
enableImpeller: enableImpeller,
).run;
}
Map<String, dynamic> _average(List<Map<String, dynamic>> results, int iterations) {
final Map<String, dynamic> tally = <String, dynamic>{};
for (final Map<String, dynamic> item in results) {
item.forEach((String key, dynamic value) {
if (tally.containsKey(key)) {
tally[key] = (tally[key] as int) + (value as int);
} else {
tally[key] = value;
}
});
}
tally.forEach((String key, dynamic value) {
tally[key] = (value as int) ~/ iterations;
});
return tally;
}
/// Opens the file at testDirectory + 'android/app/src/main/AndroidManifest.xml'
/// and adds the following entry to the application.
/// <meta-data
/// android:name="io.flutter.embedding.android.ImpellerBackend"
/// android:value="opengles" />
void _addOpenGLESToManifest(String testDirectory) {
final String manifestPath = path.join(
testDirectory, 'android', 'app', 'src', 'main', 'AndroidManifest.xml');
final File file = File(manifestPath);
if (!file.existsSync()) {
throw Exception('AndroidManifest.xml not found at $manifestPath');
}
final String xmlStr = file.readAsStringSync();
final XmlDocument xmlDoc = XmlDocument.parse(xmlStr);
const String key = 'io.flutter.embedding.android.ImpellerBackend';
const String value = 'opengles';
final XmlElement applicationNode =
xmlDoc.findAllElements('application').first;
// Check if the meta-data node already exists.
final Iterable<XmlElement> existingMetaData = applicationNode
.findAllElements('meta-data')
.where((XmlElement node) => node.getAttribute('android:name') == key);
if (existingMetaData.isNotEmpty) {
final XmlElement existingEntry = existingMetaData.first;
existingEntry.setAttribute('android:value', value);
} else {
final XmlElement metaData = XmlElement(
XmlName('meta-data'),
<XmlAttribute>[
XmlAttribute(XmlName('android:name'), key),
XmlAttribute(XmlName('android:value'), value)
],
);
applicationNode.children.add(metaData);
}
file.writeAsStringSync(xmlDoc.toXmlString(pretty: true, indent: ' '));
}
Future<void> _resetManifest(String testDirectory) async {
final String manifestPath = path.join(
testDirectory, 'android', 'app', 'src', 'main', 'AndroidManifest.xml');
final File file = File(manifestPath);
if (!file.existsSync()) {
throw Exception('AndroidManifest.xml not found at $manifestPath');
}
await exec('git', <String>['checkout', file.path]);
}
/// Measure application startup performance.
class StartupTest {
const StartupTest(
this.testDirectory, {
this.reportMetrics = true,
this.target = 'lib/main.dart',
this.runEnvironment,
});
final String testDirectory;
final bool reportMetrics;
final String target;
final Map<String, String>? runEnvironment;
Future<TaskResult> run() async {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
const int iterations = 5;
final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
section('Building application');
String? applicationBinaryPath;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm,android-arm64',
'--target=$target',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.androidArm:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm',
'--target=$target',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.androidArm64:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm64',
'--target=$target',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.fake:
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.linux:
break;
case DeviceOperatingSystem.ios:
case DeviceOperatingSystem.macos:
await flutter('build', options: <String>[
if (deviceOperatingSystem == DeviceOperatingSystem.ios) 'ios' else 'macos',
'-v',
'--profile',
'--target=$target',
]);
final String buildRoot = path.join(testDirectory, 'build');
applicationBinaryPath = _findDarwinAppInBuildDirectory(buildRoot);
case DeviceOperatingSystem.windows:
await flutter('build', options: <String>[
'windows',
'-v',
'--profile',
'--target=$target',
]);
final String basename = path.basename(testDirectory);
applicationBinaryPath = path.join(
testDirectory,
'build',
'windows',
'x64',
'runner',
'Profile',
'$basename.exe'
);
}
const int maxFailures = 3;
int currentFailures = 0;
for (int i = 0; i < iterations; i += 1) {
// Startup should not take more than a few minutes. After 10 minutes,
// take a screenshot to help debug.
final Timer timer = Timer(const Duration(minutes: 10), () async {
print('Startup not completed within 10 minutes. Taking a screenshot...');
await _flutterScreenshot(
device.deviceId,
'screenshot_startup_${DateTime.now().toLocal().toIso8601String()}.png',
);
});
final int result = await flutter(
'run',
options: <String>[
'--no-android-gradle-daemon',
'--no-publish-port',
'--verbose',
'--profile',
'--trace-startup',
// TODO(vashworth): Remove once done debugging https://github.com/flutter/flutter/issues/129836
if (device is IosDevice)
'--verbose-system-logs',
'--target=$target',
'-d',
device.deviceId,
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath',
],
environment: runEnvironment,
canFail: true,
);
timer.cancel();
if (result == 0) {
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/start_up_info.json').readAsStringSync(),
) as Map<String, dynamic>;
results.add(data);
} else {
currentFailures += 1;
await _flutterScreenshot(
device.deviceId,
'screenshot_startup_failure_$currentFailures.png',
);
i -= 1;
if (currentFailures == maxFailures) {
return TaskResult.failure('Application failed to start $maxFailures times');
}
}
await device.uninstallApp();
}
final Map<String, dynamic> averageResults = _average(results, iterations);
if (!reportMetrics) {
return TaskResult.success(averageResults);
}
return TaskResult.success(averageResults, benchmarkScoreKeys: <String>[
'timeToFirstFrameMicros',
'timeToFirstFrameRasterizedMicros',
]);
});
}
Future<void> _flutterScreenshot(String deviceId, String screenshotName) async {
if (hostAgent.dumpDirectory != null) {
await flutter(
'screenshot',
options: <String>[
'-d',
deviceId,
'--out',
hostAgent.dumpDirectory!
.childFile(screenshotName)
.path,
],
canFail: true,
);
}
}
}
/// A one-off test to verify that devtools starts in profile mode.
class DevtoolsStartupTest {
const DevtoolsStartupTest(this.testDirectory);
final String testDirectory;
Future<TaskResult> run() async {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
section('Building application');
String? applicationBinaryPath;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm,android-arm64',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.androidArm:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.androidArm64:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm64',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
case DeviceOperatingSystem.ios:
await flutter('build', options: <String>[
'ios',
'-v',
'--profile',
]);
applicationBinaryPath = _findDarwinAppInBuildDirectory('$testDirectory/build/ios/iphoneos');
case DeviceOperatingSystem.fake:
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.linux:
case DeviceOperatingSystem.macos:
case DeviceOperatingSystem.windows:
break;
}
final Process process = await startFlutter(
'run',
options: <String>[
'--no-android-gradle-daemon',
'--no-publish-port',
'--verbose',
'--profile',
'-d',
device.deviceId,
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath',
],
);
final Completer<void> completer = Completer<void>();
bool sawLine = false;
process.stdout
.transform(utf8.decoder)
.transform(const LineSplitter())
.listen((String line) {
print('[STDOUT]: $line');
// Wait for devtools output.
if (line.contains('The Flutter DevTools debugger and profiler')) {
sawLine = true;
completer.complete();
}
});
bool didExit = false;
unawaited(process.exitCode.whenComplete(() {
didExit = true;
}));
await Future.any(<Future<void>>[completer.future, Future<void>.delayed(const Duration(minutes: 5)), process.exitCode]);
if (!didExit) {
process.stdin.writeln('q');
await process.exitCode;
}
await device.uninstallApp();
if (sawLine) {
return TaskResult.success(null, benchmarkScoreKeys: <String>[]);
}
return TaskResult.failure('Did not see line "The Flutter DevTools debugger and profiler" in output');
});
}
}
/// A callback function to be used to mock the flutter drive command in PerfTests.
///
/// The `options` contains all the arguments in the `flutter drive` command in PerfTests.
typedef FlutterDriveCallback = void Function(List<String> options);
/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
const PerfTest(
this.testDirectory,
this.testTarget,
this.timelineFileName, {
this.measureCpuGpu = true,
this.measureMemory = true,
this.measureTotalGCTime = true,
this.saveTraceFile = false,
this.testDriver,
this.needsFullTimeline = true,
this.benchmarkScoreKeys,
this.dartDefine = '',
String? resultFilename,
this.device,
this.flutterDriveCallback,
this.timeoutSeconds,
this.enableImpeller,
this.forceOpenGLES,
}): _resultFilename = resultFilename;
const PerfTest.e2e(
this.testDirectory,
this.testTarget, {
this.measureCpuGpu = false,
this.measureMemory = false,
this.measureTotalGCTime = false,
this.testDriver = 'test_driver/e2e_test.dart',
this.needsFullTimeline = false,
this.benchmarkScoreKeys = _kCommonScoreKeys,
this.dartDefine = '',
String resultFilename = 'e2e_perf_summary',
this.device,
this.flutterDriveCallback,
this.timeoutSeconds,
this.enableImpeller,
this.forceOpenGLES,
}) : saveTraceFile = false, timelineFileName = null, _resultFilename = resultFilename;
/// The directory where the app under test is defined.
final String testDirectory;
/// The main entry-point file of the application, as run on the device.
final String testTarget;
// The prefix name of the filename such as `<timelineFileName>.timeline_summary.json`.
final String? timelineFileName;
String get traceFilename => '$timelineFileName.timeline';
String get resultFilename => _resultFilename ?? '$timelineFileName.timeline_summary';
final String? _resultFilename;
/// The test file to run on the host.
final String? testDriver;
/// Whether to collect CPU and GPU metrics.
final bool measureCpuGpu;
/// Whether to collect memory metrics.
final bool measureMemory;
/// Whether to summarize total GC time on the UI thread from the timeline.
final bool measureTotalGCTime;
/// Whether to collect full timeline, meaning if `--trace-startup` flag is needed.
final bool needsFullTimeline;
/// Whether to save the trace timeline file `*.timeline.json`.
final bool saveTraceFile;
/// The device to test on.
///
/// If null, the device is selected depending on the current environment.
final Device? device;
/// The function called instead of the actually `flutter drive`.
///
/// If it is not `null`, `flutter drive` will not happen in the PerfTests.
final FlutterDriveCallback? flutterDriveCallback;
/// Whether the perf test should enable Impeller.
final bool? enableImpeller;
/// Whether the perf test force Impeller's OpenGLES backend.
final bool? forceOpenGLES;
/// Number of seconds to time out the test after, allowing debug callbacks to run.
final int? timeoutSeconds;
/// The keys of the values that need to be reported.
///
/// If it's `null`, then report:
/// ```Dart
/// <String>[
/// 'average_frame_build_time_millis',
/// 'worst_frame_build_time_millis',
/// '90th_percentile_frame_build_time_millis',
/// '99th_percentile_frame_build_time_millis',
/// 'average_frame_rasterizer_time_millis',
/// 'worst_frame_rasterizer_time_millis',
/// '90th_percentile_frame_rasterizer_time_millis',
/// '99th_percentile_frame_rasterizer_time_millis',
/// 'average_vsync_transitions_missed',
/// '90th_percentile_vsync_transitions_missed',
/// '99th_percentile_vsync_transitions_missed',
/// if (measureCpuGpu) 'average_cpu_usage',
/// if (measureCpuGpu) 'average_gpu_usage',
/// ]
/// ```
final List<String>? benchmarkScoreKeys;
/// Additional flags for `--dart-define` to control the test
final String dartDefine;
Future<TaskResult> run() {
return internalRun();
}
@protected
Future<TaskResult> internalRun({
String? existingApp,
}) {
return inDirectory<TaskResult>(testDirectory, () async {
late Device selectedDevice;
if (device != null) {
selectedDevice = device!;
} else {
selectedDevice = await devices.workingDevice;
}
await selectedDevice.unlock();
final String deviceId = selectedDevice.deviceId;
final String? localEngine = localEngineFromEnv;
final String? localEngineHost = localEngineHostFromEnv;
final String? localEngineSrcPath = localEngineSrcPathFromEnv;
Future<void> Function()? manifestReset;
if (forceOpenGLES ?? false) {
assert(enableImpeller!);
_addOpenGLESToManifest(testDirectory);
manifestReset = () => _resetManifest(testDirectory);
}
try {
final List<String> options = <String>[
if (localEngine != null) ...<String>['--local-engine', localEngine],
if (localEngineHost != null) ...<String>[
'--local-engine-host',
localEngineHost
],
if (localEngineSrcPath != null) ...<String>[
'--local-engine-src-path',
localEngineSrcPath
],
'--no-dds',
'--no-android-gradle-daemon',
'-v',
'--verbose-system-logs',
'--profile',
if (timeoutSeconds != null) ...<String>[
'--timeout',
timeoutSeconds.toString(),
],
if (needsFullTimeline)
'--trace-startup', // Enables "endless" timeline event buffering.
'-t', testTarget,
if (testDriver != null) ...<String>['--driver', testDriver!],
if (existingApp != null) ...<String>[
'--use-existing-app',
existingApp
],
if (dartDefine.isNotEmpty) ...<String>['--dart-define', dartDefine],
if (enableImpeller != null && enableImpeller!) '--enable-impeller',
if (enableImpeller != null && !enableImpeller!)
'--no-enable-impeller',
'-d',
deviceId,
];
if (flutterDriveCallback != null) {
flutterDriveCallback!(options);
} else {
await flutter('drive', options: options);
}
} finally {
if (manifestReset != null) {
await manifestReset();
}
}
final Map<String, dynamic> data = json.decode(
file('${_testOutputDirectory(testDirectory)}/$resultFilename.json').readAsStringSync(),
) as Map<String, dynamic>;
if (data['frame_count'] as int < 5) {
return TaskResult.failure(
'Timeline contains too few frames: ${data['frame_count']}. Possibly '
'trace events are not being captured.',
);
}
final bool recordGPU;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
recordGPU = true;
case DeviceOperatingSystem.android:
case DeviceOperatingSystem.androidArm:
case DeviceOperatingSystem.androidArm64:
recordGPU = enableImpeller ?? false;
case DeviceOperatingSystem.fake:
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.linux:
case DeviceOperatingSystem.macos:
case DeviceOperatingSystem.windows:
recordGPU = false;
}
// TODO(liyuqian): Remove isAndroid restriction once
// https://github.com/flutter/flutter/issues/61567 is fixed.
final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
return TaskResult.success(
data,
detailFiles: <String>[
if (saveTraceFile)
'${_testOutputDirectory(testDirectory)}/$traceFilename.json',
],
benchmarkScoreKeys: benchmarkScoreKeys ?? <String>[
..._kCommonScoreKeys,
'average_vsync_transitions_missed',
'90th_percentile_vsync_transitions_missed',
'99th_percentile_vsync_transitions_missed',
'average_frame_request_pending_latency',
'90th_percentile_frame_request_pending_latency',
'99th_percentile_frame_request_pending_latency',
if (measureCpuGpu && !isAndroid) ...<String>[
// See https://github.com/flutter/flutter/issues/68888
if (data['average_cpu_usage'] != null) 'average_cpu_usage',
if (data['average_gpu_usage'] != null) 'average_gpu_usage',
],
if (measureMemory && !isAndroid) ...<String>[
// See https://github.com/flutter/flutter/issues/68888
if (data['average_memory_usage'] != null) 'average_memory_usage',
if (data['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
if (data['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
],
if (measureTotalGCTime) 'total_ui_gc_time',
if (data['30hz_frame_percentage'] != null) '30hz_frame_percentage',
if (data['60hz_frame_percentage'] != null) '60hz_frame_percentage',
if (data['80hz_frame_percentage'] != null) '80hz_frame_percentage',
if (data['90hz_frame_percentage'] != null) '90hz_frame_percentage',
if (data['120hz_frame_percentage'] != null) '120hz_frame_percentage',
if (data['illegal_refresh_rate_frame_count'] != null) 'illegal_refresh_rate_frame_count',
if (recordGPU) ...<String>[
if (data['average_gpu_frame_time'] != null) 'average_gpu_frame_time',
if (data['90th_percentile_gpu_frame_time'] != null) '90th_percentile_gpu_frame_time',
if (data['99th_percentile_gpu_frame_time'] != null) '99th_percentile_gpu_frame_time',
if (data['worst_gpu_frame_time'] != null) 'worst_gpu_frame_time',
]
],
);
});
}
}
const List<String> _kCommonScoreKeys = <String>[
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
'90th_percentile_frame_build_time_millis',
'99th_percentile_frame_build_time_millis',
'average_frame_rasterizer_time_millis',
'worst_frame_rasterizer_time_millis',
'90th_percentile_frame_rasterizer_time_millis',
'99th_percentile_frame_rasterizer_time_millis',
'average_layer_cache_count',
'90th_percentile_layer_cache_count',
'99th_percentile_layer_cache_count',
'worst_layer_cache_count',
'average_layer_cache_memory',
'90th_percentile_layer_cache_memory',
'99th_percentile_layer_cache_memory',
'worst_layer_cache_memory',
'average_picture_cache_count',
'90th_percentile_picture_cache_count',
'99th_percentile_picture_cache_count',
'worst_picture_cache_count',
'average_picture_cache_memory',
'90th_percentile_picture_cache_memory',
'99th_percentile_picture_cache_memory',
'worst_picture_cache_memory',
'old_gen_gc_count',
];
/// Measures how long it takes to compile a Flutter app to JavaScript and how
/// big the compiled code is.
class WebCompileTest {
const WebCompileTest();
Future<TaskResult> run() async {
final Map<String, Object> metrics = <String, Object>{};
metrics.addAll(await runSingleBuildTest(
directory: '${flutterDirectory.path}/examples/hello_world',
metric: 'hello_world',
));
metrics.addAll(await runSingleBuildTest(
directory: '${flutterDirectory.path}/dev/integration_tests/flutter_gallery',
metric: 'flutter_gallery',
));
const String sampleAppName = 'sample_flutter_app';
final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
rmTree(sampleDir);
await inDirectory<void>(Directory.systemTemp, () async {
await flutter('create', options: <String>['--template=app', sampleAppName]);
});
metrics.addAll(await runSingleBuildTest(
directory: sampleDir.path,
metric: 'basic_material_app',
));
return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
}
/// Run a single web compile test and return its metrics.
///
/// Run a single web compile test for the app under [directory], and store
/// its metrics with prefix [metric].
static Future<Map<String, int>> runSingleBuildTest({
required String directory,
required String metric,
bool measureBuildTime = false,
}) {
return inDirectory<Map<String, int>>(directory, () async {
final Map<String, int> metrics = <String, int>{};
await flutter('clean');
await flutter('packages', options: <String>['get']);
final Stopwatch? watch = measureBuildTime ? Stopwatch() : null;
watch?.start();
await evalFlutter('build', options: <String>[
'web',
'-v',
'--release',
'--no-pub',
'--no-web-resources-cdn',
]);
watch?.stop();
final String buildDir = path.join(directory, 'build', 'web');
metrics.addAll(await getSize(
directories: <String, String>{'web_build_dir': buildDir},
files: <String, String>{
'dart2js': path.join(buildDir, 'main.dart.js'),
'canvaskit_wasm': path.join(buildDir, 'canvaskit', 'canvaskit.wasm'),
'canvaskit_js': path.join(buildDir, 'canvaskit', 'canvaskit.js'),
'flutter_js': path.join(buildDir, 'flutter.js'),
},
metric: metric,
));
if (measureBuildTime) {
metrics['${metric}_dart2js_millis'] = watch!.elapsedMilliseconds;
}
return metrics;
});
}
/// Obtains the size and gzipped size of both [dartBundleFile] and [buildDir].
static Future<Map<String, int>> getSize({
/// Mapping of metric key name to file system path for directories to measure
Map<String, String> directories = const <String, String>{},
/// Mapping of metric key name to file system path for files to measure
Map<String, String> files = const <String, String>{},
required String metric,
}) async {
const String kGzipCompressionLevel = '-9';
final Map<String, int> sizeMetrics = <String, int>{};
final Directory tempDir = Directory.systemTemp.createTempSync('perf_tests_gzips');
try {
for (final MapEntry<String, String> entry in files.entries) {
final String key = entry.key;
final String filePath = entry.value;
sizeMetrics['${metric}_${key}_uncompressed_bytes'] = File(filePath).lengthSync();
await Process.run('gzip',<String>['--keep', kGzipCompressionLevel, filePath]);
// gzip does not provide a CLI option to specify an output file, so
// instead just move the output file to the temp dir
final File compressedFile = File('$filePath.gz')
.renameSync(path.join(tempDir.absolute.path, '$key.gz'));
sizeMetrics['${metric}_${key}_compressed_bytes'] = compressedFile.lengthSync();
}
for (final MapEntry<String, String> entry in directories.entries) {
final String key = entry.key;
final String dirPath = entry.value;
final String tarball = path.join(tempDir.absolute.path, '$key.tar');
await Process.run('tar', <String>[
'--create',
'--verbose',
'--file=$tarball',
dirPath,
]);
sizeMetrics['${metric}_${key}_uncompressed_bytes'] = File(tarball).lengthSync();
// get size of compressed build directory
await Process.run('gzip',<String>['--keep', kGzipCompressionLevel, tarball]);
sizeMetrics['${metric}_${key}_compressed_bytes'] = File('$tarball.gz').lengthSync();
}
} finally {
try {
tempDir.deleteSync(recursive: true);
} on FileSystemException {
print('Failed to delete ${tempDir.path}.');
}
}
return sizeMetrics;
}
}
/// Measures how long it takes to compile a Flutter app and how big the compiled
/// code is.
class CompileTest {
const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
final String testDirectory;
final bool reportPackageContentSizes;
Future<TaskResult> run() async {
return inDirectory<TaskResult>(testDirectory, () async {
await flutter('packages', options: <String>['get']);
// "initial" compile required downloading and creating the `android/.gradle` directory while "full"
// compiles only run `flutter clean` between runs.
final Map<String, dynamic> compileInitialRelease = await _compileApp(deleteGradleCache: true);
final Map<String, dynamic> compileFullRelease = await _compileApp(deleteGradleCache: false);
final Map<String, dynamic> compileInitialDebug = await _compileDebug(
clean: true,
deleteGradleCache: true,
metricKey: 'debug_initial_compile_millis',
);
final Map<String, dynamic> compileFullDebug = await _compileDebug(
clean: true,
deleteGradleCache: false,
metricKey: 'debug_full_compile_millis',
);
// Build again without cleaning, should be faster.
final Map<String, dynamic> compileSecondDebug = await _compileDebug(
clean: false,
deleteGradleCache: false,
metricKey: 'debug_second_compile_millis',
);
final Map<String, dynamic> metrics = <String, dynamic>{
...compileInitialRelease,
...compileFullRelease,
...compileInitialDebug,
...compileFullDebug,
...compileSecondDebug,
};
final File mainDart = File('$testDirectory/lib/main.dart');
if (mainDart.existsSync()) {
final List<int> bytes = mainDart.readAsBytesSync();
// "Touch" the file
mainDart.writeAsStringSync(' ', mode: FileMode.append, flush: true);
// Build after "edit" without clean should be faster than first build
final Map<String, dynamic> compileAfterEditDebug = await _compileDebug(
clean: false,
deleteGradleCache: false,
metricKey: 'debug_compile_after_edit_millis',
);
metrics.addAll(compileAfterEditDebug);
// Revert the changes
mainDart.writeAsBytesSync(bytes, flush: true);
}
return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
});
}
Future<Map<String, dynamic>> _compileApp({required bool deleteGradleCache}) async {
await flutter('clean');
if (deleteGradleCache) {
final Directory gradleCacheDir = Directory('$testDirectory/android/.gradle');
rmTree(gradleCacheDir);
}
final Stopwatch watch = Stopwatch();
int releaseSizeInBytes;
final List<String> options = <String>['--release'];
final Map<String, dynamic> metrics = <String, dynamic>{};
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
case DeviceOperatingSystem.macos:
unawaited(stderr.flush());
late final String deviceId;
if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
deviceId = 'ios';
} else if (deviceOperatingSystem == DeviceOperatingSystem.macos) {
deviceId = 'macos';
} else {
throw Exception('Attempted to run darwin compile workflow with $deviceOperatingSystem');
}
options.insert(0, deviceId);
options.add('--tree-shake-icons');
options.add('--split-debug-info=infos/');
watch.start();
await flutter('build', options: options);
watch.stop();
final Directory buildDirectory = dir(path.join(
cwd,
'build',
));
final String? appPath =
_findDarwinAppInBuildDirectory(buildDirectory.path);
if (appPath == null) {
throw 'Failed to find app bundle in ${buildDirectory.path}';
}
// Validate changes in Dart snapshot format and data layout do not
// change compression size. This also simulates the size of an IPA on iOS.
await exec('tar', <String>['-zcf', 'build/app.tar.gz', appPath]);
releaseSizeInBytes = await file('$cwd/build/app.tar.gz').length();
if (reportPackageContentSizes) {
final Map<String, Object> sizeMetrics = await getSizesFromDarwinApp(
appPath: appPath,
operatingSystem: deviceOperatingSystem,
);
metrics.addAll(sizeMetrics);
}
case DeviceOperatingSystem.android:
case DeviceOperatingSystem.androidArm:
options.insert(0, 'apk');
options.add('--target-platform=android-arm');
options.add('--tree-shake-icons');
options.add('--split-debug-info=infos/');
watch.start();
await flutter('build', options: options);
watch.stop();
final String apkPath = '$cwd/build/app/outputs/flutter-apk/app-release.apk';
final File apk = file(apkPath);
releaseSizeInBytes = apk.lengthSync();
if (reportPackageContentSizes) {
metrics.addAll(await getSizesFromApk(apkPath));
}
case DeviceOperatingSystem.androidArm64:
options.insert(0, 'apk');
options.add('--target-platform=android-arm64');
options.add('--tree-shake-icons');
options.add('--split-debug-info=infos/');
watch.start();
await flutter('build', options: options);
watch.stop();
final String apkPath = '$cwd/build/app/outputs/flutter-apk/app-release.apk';
final File apk = file(apkPath);
releaseSizeInBytes = apk.lengthSync();
if (reportPackageContentSizes) {
metrics.addAll(await getSizesFromApk(apkPath));
}
case DeviceOperatingSystem.fake:
throw Exception('Unsupported option for fake devices');
case DeviceOperatingSystem.fuchsia:
throw Exception('Unsupported option for Fuchsia devices');
case DeviceOperatingSystem.linux:
throw Exception('Unsupported option for Linux devices');
case DeviceOperatingSystem.windows:
unawaited(stderr.flush());
options.insert(0, 'windows');
options.add('--tree-shake-icons');
options.add('--split-debug-info=infos/');
watch.start();
await flutter('build', options: options);
watch.stop();
final String basename = path.basename(cwd);
final String exePath = path.join(
cwd,
'build',
'windows',
'x64',
'runner',
'release',
'$basename.exe');
final File exe = file(exePath);
// On Windows, we do not produce a single installation package file,
// rather a directory containing an .exe and .dll files.
// The release size is set to the size of the produced .exe file
releaseSizeInBytes = exe.lengthSync();
}
metrics.addAll(<String, dynamic>{
'release_${deleteGradleCache ? 'initial' : 'full'}_compile_millis': watch.elapsedMilliseconds,
'release_size_bytes': releaseSizeInBytes,
});
return metrics;
}
Future<Map<String, dynamic>> _compileDebug({
required bool deleteGradleCache,
required bool clean,
required String metricKey,
}) async {
if (clean) {
await flutter('clean');
}
if (deleteGradleCache) {
final Directory gradleCacheDir = Directory('$testDirectory/android/.gradle');
rmTree(gradleCacheDir);
}
final Stopwatch watch = Stopwatch();
final List<String> options = <String>['--debug'];
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
options.insert(0, 'ios');
case DeviceOperatingSystem.android:
case DeviceOperatingSystem.androidArm:
options.insert(0, 'apk');
options.add('--target-platform=android-arm');
case DeviceOperatingSystem.androidArm64:
options.insert(0, 'apk');
options.add('--target-platform=android-arm64');
case DeviceOperatingSystem.fake:
throw Exception('Unsupported option for fake devices');
case DeviceOperatingSystem.fuchsia:
throw Exception('Unsupported option for Fuchsia devices');
case DeviceOperatingSystem.linux:
throw Exception('Unsupported option for Linux devices');
case DeviceOperatingSystem.macos:
unawaited(stderr.flush());
options.insert(0, 'macos');
case DeviceOperatingSystem.windows:
unawaited(stderr.flush());
options.insert(0, 'windows');
}
watch.start();
await flutter('build', options: options);
watch.stop();
return <String, dynamic>{
metricKey: watch.elapsedMilliseconds,
};
}
static Future<Map<String, Object>> getSizesFromDarwinApp({
required String appPath,
required DeviceOperatingSystem operatingSystem,
}) async {
late final File flutterFramework;
late final String frameworkDirectory;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
frameworkDirectory = path.join(
appPath,
'Frameworks',
);
flutterFramework = File(path.join(
frameworkDirectory,
'Flutter.framework',
'Flutter',
));
case DeviceOperatingSystem.macos:
frameworkDirectory = path.join(
appPath,
'Contents',
'Frameworks',
);
flutterFramework = File(path.join(
frameworkDirectory,
'FlutterMacOS.framework',
'FlutterMacOS',
)); // https://github.com/flutter/flutter/issues/70413
case DeviceOperatingSystem.android:
case DeviceOperatingSystem.androidArm:
case DeviceOperatingSystem.androidArm64:
case DeviceOperatingSystem.fake:
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.linux:
case DeviceOperatingSystem.windows:
throw Exception('Called ${CompileTest.getSizesFromDarwinApp} with $operatingSystem.');
}
final File appFramework = File(path.join(
frameworkDirectory,
'App.framework',
'App',
));
return <String, Object>{
'app_framework_uncompressed_bytes': await appFramework.length(),
'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
};
}
static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
final String output = await eval('unzip', <String>['-v', apkPath]);
final List<String> lines = output.split('\n');
final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};
// First three lines are header, last two lines are footer.
for (int i = 3; i < lines.length - 2; i++) {
final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
fileToMetadata[entry.path] = entry;
}
final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so']!;
final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so']!;
final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES.Z']!;
return <String, dynamic>{
'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
'libflutter_compressed_bytes': libflutter.compressedSize,
'libapp_uncompressed_bytes': libapp.uncompressedSize,
'libapp_compressed_bytes': libapp.compressedSize,
'license_uncompressed_bytes': license.uncompressedSize,
'license_compressed_bytes': license.compressedSize,
};
}
}
/// Measure application memory usage.
class MemoryTest {
MemoryTest(this.project, this.test, this.package);
final String project;
final String test;
final String package;
/// Completes when the log line specified in the last call to
/// [prepareForNextMessage] is seen by `adb logcat`.
Future<void>? get receivedNextMessage => _receivedNextMessage?.future;
Completer<void>? _receivedNextMessage;
String? _nextMessage;
/// Prepares the [receivedNextMessage] future such that it will complete
/// when `adb logcat` sees a log line with the given `message`.
void prepareForNextMessage(String message) {
_nextMessage = message;
_receivedNextMessage = Completer<void>();
}
int get iterationCount => 10;
Device? get device => _device;
Device? _device;
Future<TaskResult> run() {
return inDirectory<TaskResult>(project, () async {
// This test currently only works on Android, because device.logcat,
// device.getMemoryStats, etc, aren't implemented for iOS.
_device = await devices.workingDevice;
await device!.unlock();
await flutter('packages', options: <String>['get']);
final StreamSubscription<String> adb = device!.logcat.listen(
(String data) {
if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ====')) {
_receivedNextMessage?.complete();
}
},
);
for (int iteration = 0; iteration < iterationCount; iteration += 1) {
print('running memory test iteration $iteration...');
_startMemoryUsage = null;
await useMemory();
assert(_startMemoryUsage != null);
assert(_startMemory.length == iteration + 1);
assert(_endMemory.length == iteration + 1);
assert(_diffMemory.length == iteration + 1);
print('terminating...');
await device!.stop(package);
await Future<void>.delayed(const Duration(milliseconds: 10));
}
await adb.cancel();
await device!.uninstallApp();
final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
final Map<String, dynamic> memoryUsage = <String, dynamic>{
...startMemoryStatistics.asMap('start'),
...endMemoryStatistics.asMap('end'),
...diffMemoryStatistics.asMap('diff'),
};
_device = null;
_startMemory.clear();
_endMemory.clear();
_diffMemory.clear();
return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
});
}
/// Starts the app specified by [test] on the [device].
///
/// The [run] method will terminate it by its package name ([package]).
Future<void> launchApp() async {
prepareForNextMessage('READY');
print('launching $project$test on device...');
await flutter('run', options: <String>[
'--verbose',
'--release',
'--no-resident',
'-d', device!.deviceId,
test,
]);
print('awaiting "ready" message...');
await receivedNextMessage;
}
/// To change the behavior of the test, override this.
///
/// Make sure to call recordStart() and recordEnd() once each in that order.
///
/// By default it just launches the app, records memory usage, taps the device,
/// awaits a DONE notification, and records memory usage again.
Future<void> useMemory() async {
await launchApp();
await recordStart();
prepareForNextMessage('DONE');
print('tapping device...');
await device!.tap(100, 100);
print('awaiting "done" message...');
await receivedNextMessage;
await recordEnd();
}
final List<int> _startMemory = <int>[];
final List<int> _endMemory = <int>[];
final List<int> _diffMemory = <int>[];
Map<String, dynamic>? _startMemoryUsage;
@protected
Future<void> recordStart() async {
assert(_startMemoryUsage == null);
print('snapshotting memory usage...');
_startMemoryUsage = await device!.getMemoryStats(package);
}
@protected
Future<void> recordEnd() async {
assert(_startMemoryUsage != null);
print('snapshotting memory usage...');
final Map<String, dynamic> endMemoryUsage = await device!.getMemoryStats(package);
_startMemory.add(_startMemoryUsage!['total_kb'] as int);
_endMemory.add(endMemoryUsage['total_kb'] as int);
_diffMemory.add((endMemoryUsage['total_kb'] as int) - (_startMemoryUsage!['total_kb'] as int));
}
}
class DevToolsMemoryTest {
DevToolsMemoryTest(this.project, this.driverTest);
final String project;
final String driverTest;
Future<TaskResult> run() {
return inDirectory<TaskResult>(project, () async {
_device = await devices.workingDevice;
await _device.unlock();
await flutter(
'drive',
options: <String>[
'-d', _device.deviceId,
'--profile',
'--profile-memory', _kJsonFileName,
'--no-publish-port',
'-v',
driverTest,
],
);
final Map<String, dynamic> data = json.decode(
file('$project/$_kJsonFileName').readAsStringSync(),
) as Map<String, dynamic>;
final List<dynamic> samples = (data['samples'] as Map<String, dynamic>)['data'] as List<dynamic>;
int maxRss = 0;
int maxAdbTotal = 0;
for (final Map<String, dynamic> sample in samples.cast<Map<String, dynamic>>()) {
if (sample['rss'] != null) {
maxRss = math.max(maxRss, sample['rss'] as int);
}
if (sample['adb_memoryInfo'] != null) {
maxAdbTotal = math.max(maxAdbTotal, (sample['adb_memoryInfo'] as Map<String, dynamic>)['Total'] as int);
}
}
return TaskResult.success(
<String, dynamic>{'maxRss': maxRss, 'maxAdbTotal': maxAdbTotal},
benchmarkScoreKeys: <String>['maxRss', 'maxAdbTotal'],
);
});
}
late Device _device;
static const String _kJsonFileName = 'devtools_memory.json';
}
enum ReportedDurationTestFlavor {
debug, profile, release
}
String _reportedDurationTestToString(ReportedDurationTestFlavor flavor) {
switch (flavor) {
case ReportedDurationTestFlavor.debug:
return 'debug';
case ReportedDurationTestFlavor.profile:
return 'profile';
case ReportedDurationTestFlavor.release:
return 'release';
}
}
class ReportedDurationTest {
ReportedDurationTest(this.flavor, this.project, this.test, this.package, this.durationPattern);
final ReportedDurationTestFlavor flavor;
final String project;
final String test;
final String package;
final RegExp durationPattern;
final Completer<int> durationCompleter = Completer<int>();
int get iterationCount => 10;
Device? get device => _device;
Device? _device;
Future<TaskResult> run() {
return inDirectory<TaskResult>(project, () async {
// This test currently only works on Android, because device.logcat,
// device.getMemoryStats, etc, aren't implemented for iOS.
_device = await devices.workingDevice;
await device!.unlock();
await flutter('packages', options: <String>['get']);
final StreamSubscription<String> adb = device!.logcat.listen(
(String data) {
if (durationPattern.hasMatch(data)) {
durationCompleter.complete(int.parse(durationPattern.firstMatch(data)!.group(1)!));
}
},
);
print('launching $project$test on device...');
await flutter('run', options: <String>[
'--verbose',
'--no-publish-port',
'--no-fast-start',
'--${_reportedDurationTestToString(flavor)}',
'--no-resident',
'-d', device!.deviceId,
test,
]);
final int duration = await durationCompleter.future;
print('terminating...');
await device!.stop(package);
await adb.cancel();
_device = null;
final Map<String, dynamic> reportedDuration = <String, dynamic>{
'duration': duration,
};
_device = null;
return TaskResult.success(reportedDuration, benchmarkScoreKeys: reportedDuration.keys.toList());
});
}
}
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
factory ListStatistics(Iterable<int> data) {
assert(data.isNotEmpty);
assert(data.length.isOdd);
final List<int> sortedData = data.toList()..sort();
return ListStatistics._(
sortedData.first,
sortedData.last,
sortedData[(sortedData.length - 1) ~/ 2],
);
}
const ListStatistics._(this.min, this.max, this.median);
final int min;
final int max;
final int median;
Map<String, int> asMap(String prefix) {
return <String, int>{
'$prefix-min': min,
'$prefix-max': max,
'$prefix-median': median,
};
}
}
class _UnzipListEntry {
factory _UnzipListEntry.fromLine(String line) {
final List<String> data = line.trim().split(RegExp(r'\s+'));
assert(data.length == 8);
return _UnzipListEntry._(
uncompressedSize: int.parse(data[0]),
compressedSize: int.parse(data[2]),
path: data[7],
);
}
_UnzipListEntry._({
required this.uncompressedSize,
required this.compressedSize,
required this.path,
}) : assert(compressedSize <= uncompressedSize);
final int uncompressedSize;
final int compressedSize;
final String path;
}
/// Wait for up to 1 hour for the file to appear.
Future<File> waitForFile(String path) async {
for (int i = 0; i < 180; i += 1) {
final File file = File(path);
print('looking for ${file.path}');
if (file.existsSync()) {
return file;
}
await Future<void>.delayed(const Duration(seconds: 20));
}
throw StateError('Did not find vmservice out file after 1 hour');
}
String? _findDarwinAppInBuildDirectory(String searchDirectory) {
for (final FileSystemEntity entity in Directory(searchDirectory)
.listSync(recursive: true)) {
if (entity.path.endsWith('.app')) {
return entity.path;
}
}
return null;
}