dev/devicelab/lib/tasks/perf_tests.dart - mirrors/flutter - Git at Google

 // Copyright (c) 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

 import 'dart:async';
 import 'dart:convert' show json;
 import 'dart:io';

 import 'package:meta/meta.dart';
 import 'package:path/path.dart' as p;

 import '../framework/adb.dart';
 import '../framework/framework.dart';
 import '../framework/ios.dart';
 import '../framework/utils.dart';

 TaskFunction createComplexLayoutScrollPerfTest() {
   return new PerfTest(
     '${flutterDirectory.path}/dev/benchmarks/complex_layout',
     'test_driver/scroll_perf.dart',
     'complex_layout_scroll_perf',
   ).run;
 }

 TaskFunction createTilesScrollPerfTest() {
   return new PerfTest(
     '${flutterDirectory.path}/dev/benchmarks/complex_layout',
     'test_driver/scroll_perf.dart',
     'tiles_scroll_perf',
   ).run;
 }

 TaskFunction createComplexLayoutScrollMemoryTest() {
   return new MemoryTest(
     '${flutterDirectory.path}/dev/benchmarks/complex_layout',
     'com.yourcompany.complexLayout',
     testTarget: 'test_driver/scroll_perf.dart',
   ).run;
 }

 TaskFunction createFlutterGalleryStartupTest() {
   return new StartupTest(
     '${flutterDirectory.path}/examples/flutter_gallery',
   ).run;
 }

 TaskFunction createComplexLayoutStartupTest() {
   return new StartupTest(
     '${flutterDirectory.path}/dev/benchmarks/complex_layout',
   ).run;
 }

 TaskFunction createFlutterGalleryCompileTest() {
   return new CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
 }

 TaskFunction createHelloWorldCompileTest() {
   return new CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
 }

 TaskFunction createComplexLayoutCompileTest() {
   return new CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
 }

 TaskFunction createHelloWorldMemoryTest() {
   return new MemoryTest(
     '${flutterDirectory.path}/examples/hello_world',
     'io.flutter.examples.hello_world',
   ).run;
 }

 TaskFunction createGalleryNavigationMemoryTest() {
   return new MemoryTest(
     '${flutterDirectory.path}/examples/flutter_gallery',
     'io.flutter.demo.gallery',
     testTarget: 'test_driver/memory_nav.dart',
   ).run;
 }

 TaskFunction createGalleryBackButtonMemoryTest() {
   return new AndroidBackButtonMemoryTest(
     '${flutterDirectory.path}/examples/flutter_gallery',
     'io.flutter.demo.gallery',
     'io.flutter.demo.gallery.MainActivity',
   ).run;
 }

 TaskFunction createFlutterViewStartupTest() {
   return new StartupTest(
       '${flutterDirectory.path}/examples/flutter_view',
       reportMetrics: false,
   ).run;
 }

 TaskFunction createPlatformViewStartupTest() {
   return new StartupTest(
     '${flutterDirectory.path}/examples/platform_view',
     reportMetrics: false,
   ).run;
 }

 TaskFunction createBasicMaterialCompileTest() {
   return () async {
     const String sampleAppName = 'sample_flutter_app';
     final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

     if (await sampleDir.exists())
       rmTree(sampleDir);

     await inDirectory(Directory.systemTemp, () async {
       await flutter('create', options: <String>[sampleAppName]);
     });

     if (!(await sampleDir.exists()))
       throw 'Failed to create default Flutter app in ${sampleDir.path}';

     return new CompileTest(sampleDir.path).run();
   };
 }


 /// Measure application startup performance.
 class StartupTest {
   static const Duration _startupTimeout = const Duration(minutes: 5);

   const StartupTest(this.testDirectory, { this.reportMetrics = true });

   final String testDirectory;
   final bool reportMetrics;

   Future<TaskResult> run() async {
     return await inDirectory(testDirectory, () async {
       final String deviceId = (await devices.workingDevice).deviceId;
       await flutter('packages', options: <String>['get']);

       if (deviceOperatingSystem == DeviceOperatingSystem.ios)
         await prepareProvisioningCertificates(testDirectory);

       await flutter('run', options: <String>[
         '--verbose',
         '--profile',
         '--trace-startup',
         '-d',
         deviceId,
       ]).timeout(_startupTimeout);
       final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());

       if (!reportMetrics)
         return new TaskResult.success(data);

       return new TaskResult.success(data, benchmarkScoreKeys: <String>[
         'timeToFirstFrameMicros',
       ]);
     });
   }
 }

 /// Measures application runtime performance, specifically per-frame
 /// performance.
 class PerfTest {
   const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);

   final String testDirectory;
   final String testTarget;
   final String timelineFileName;

   Future<TaskResult> run() {
     return inDirectory(testDirectory, () async {
       final Device device = await devices.workingDevice;
       await device.unlock();
       final String deviceId = device.deviceId;
       await flutter('packages', options: <String>['get']);

       if (deviceOperatingSystem == DeviceOperatingSystem.ios)
         await prepareProvisioningCertificates(testDirectory);

       await flutter('drive', options: <String>[
         '-v',
         '--profile',
         '--trace-startup', // Enables "endless" timeline event buffering.
         '-t',
         testTarget,
         '-d',
         deviceId,
       ]);
       final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());

       if (data['frame_count'] < 5) {
         return new TaskResult.failure(
           'Timeline contains too few frames: ${data['frame_count']}. Possibly '
           'trace events are not being captured.',
         );
       }

       return new TaskResult.success(data, benchmarkScoreKeys: <String>[
         'average_frame_build_time_millis',
         'worst_frame_build_time_millis',
         'missed_frame_build_budget_count',
         'average_frame_rasterizer_time_millis',
         'worst_frame_rasterizer_time_millis',
         '90th_percentile_frame_rasterizer_time_millis',
         '99th_percentile_frame_rasterizer_time_millis',
       ]);
     });
   }
 }

 /// Measures how long it takes to compile a Flutter app and how big the compiled
 /// code is.
 class CompileTest {
   const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });

   final String testDirectory;
   final bool reportPackageContentSizes;

   Future<TaskResult> run() async {
     return await inDirectory(testDirectory, () async {
       final Device device = await devices.workingDevice;
       await device.unlock();
       await flutter('packages', options: <String>['get']);

       final Map<String, dynamic> metrics = <String, dynamic>{}
         ..addAll(await _compileAot())
         ..addAll(await _compileApp(reportPackageContentSizes: reportPackageContentSizes))
         ..addAll(await _compileDebug());

       return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
     });
   }

   static Future<Map<String, dynamic>> _compileAot({ bool previewDart2 = true }) async {
     // Generate blobs instead of assembly.
     await flutter('clean');
     final Stopwatch watch = new Stopwatch()..start();
     final List<String> options = <String>[
       'aot',
       '-v',
       '--extra-gen-snapshot-options=--print_snapshot_sizes',
       '--release',
       '--no-pub',
       '--target-platform',
     ];
     switch (deviceOperatingSystem) {
       case DeviceOperatingSystem.ios:
         options.add('ios');
         break;
       case DeviceOperatingSystem.android:
         options.add('android-arm');
         break;
     }
     if (previewDart2)
       options.add('--preview-dart-2');
     else
       options.add('--no-preview-dart-2');
     setLocalEngineOptionIfNecessary(options);
     final String compileLog = await evalFlutter('build', options: options);
     watch.stop();

     final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
     final Map<String, dynamic> metrics = <String, dynamic>{};
     for (Match m in metricExpression.allMatches(compileLog)) {
       metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
     }
     if (metrics.length != _kSdkNameToMetricNameMapping.length) {
       throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
     }
     metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;

     return metrics;
   }

   static Future<Map<String, dynamic>> _compileApp({ bool previewDart2 = true, bool reportPackageContentSizes = false }) async {
     await flutter('clean');
     final Stopwatch watch = new Stopwatch();
     int releaseSizeInBytes;
     final List<String> options = <String>['--release'];
     if (previewDart2)
       options.add('--preview-dart-2');
     else
       options.add('--no-preview-dart-2');
     setLocalEngineOptionIfNecessary(options);
     final Map<String, dynamic> metrics = <String, dynamic>{};

     switch (deviceOperatingSystem) {
       case DeviceOperatingSystem.ios:
         options.insert(0, 'ios');
         await prepareProvisioningCertificates(cwd);
         watch.start();
         await flutter('build', options: options);
         watch.stop();
         final String appPath =  '$cwd/build/ios/Release-iphoneos/Runner.app/';
         // IPAs are created manually, https://flutter.io/ios-release/
         await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
         releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
         if (reportPackageContentSizes)
           metrics.addAll(await getSizesFromIosApp(appPath));
         break;
       case DeviceOperatingSystem.android:
         options.insert(0, 'apk');
         watch.start();
         await flutter('build', options: options);
         watch.stop();
         String apkPath = '$cwd/build/app/outputs/apk/app.apk';
         File apk = file(apkPath);
         if (!apk.existsSync()) {
           // Pre Android SDK 26 path
           apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
           apk = file(apkPath);
         }
         releaseSizeInBytes = apk.lengthSync();
         if (reportPackageContentSizes)
           metrics.addAll(await getSizesFromApk(apkPath));
         break;
     }

     metrics.addAll(<String, dynamic>{
       'release_full_compile_millis': watch.elapsedMilliseconds,
       'release_size_bytes': releaseSizeInBytes,
     });

     return metrics;
   }

   static Future<Map<String, dynamic>> _compileDebug({ bool previewDart2 = true }) async {
     await flutter('clean');
     final Stopwatch watch = new Stopwatch();
     final List<String> options = <String>['--debug'];
     if (previewDart2)
       options.add('--preview-dart-2');
     else
       options.add('--no-preview-dart-2');
     setLocalEngineOptionIfNecessary(options);
     switch (deviceOperatingSystem) {
       case DeviceOperatingSystem.ios:
         options.insert(0, 'ios');
         await prepareProvisioningCertificates(cwd);
         break;
       case DeviceOperatingSystem.android:
         options.insert(0, 'apk');
         break;
     }
     watch.start();
     await flutter('build', options: options);
     watch.stop();

     return <String, dynamic>{
       'debug_full_compile_millis': watch.elapsedMilliseconds,
     };
   }

   static const Map<String, String> _kSdkNameToMetricNameMapping = const <String, String> {
     'VMIsolate': 'aot_snapshot_size_vmisolate',
     'Isolate': 'aot_snapshot_size_isolate',
     'ReadOnlyData': 'aot_snapshot_size_rodata',
     'Instructions': 'aot_snapshot_size_instructions',
     'Total': 'aot_snapshot_size_total',
   };

   static String _sdkNameToMetricName(String sdkName) {

     if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
       throw 'Unrecognized SDK snapshot metric name: $sdkName';

     return _kSdkNameToMetricNameMapping[sdkName];
   }

   static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
     // Thin the binary to only contain one architecture.
     final String xcodeBackend = p.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
     await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
       'ARCHS': 'arm64',
       'WRAPPER_NAME': p.basename(appPath),
       'TARGET_BUILD_DIR': p.dirname(appPath),
     });

     final File appFramework = new File(p.join(appPath, 'Frameworks', 'App.framework', 'App'));
     final File flutterFramework = new File(p.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));

     return <String, dynamic>{
       'app_framework_uncompressed_bytes': await appFramework.length(),
       'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
     };
   }


   static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
     final  String output = await eval('unzip', <String>['-v', apkPath]);
     final List<String> lines = output.split('\n');
     final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

     // First three lines are header, last two lines are footer.
     for (int i = 3; i < lines.length - 2; i++) {
       final _UnzipListEntry entry = new _UnzipListEntry.fromLine(lines[i]);
       fileToMetadata[entry.path] = entry;
     }

     final _UnzipListEntry icudtl = fileToMetadata['assets/flutter_shared/icudtl.dat'];
     final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
     final _UnzipListEntry isolateSnapshotData = fileToMetadata['assets/isolate_snapshot_data'];
     final _UnzipListEntry isolateSnapshotInstr = fileToMetadata['assets/isolate_snapshot_instr'];
     final _UnzipListEntry vmSnapshotData = fileToMetadata['assets/vm_snapshot_data'];
     final _UnzipListEntry vmSnapshotInstr = fileToMetadata['assets/vm_snapshot_instr'];

     return <String, dynamic>{
       'icudtl_uncompressed_bytes': icudtl.uncompressedSize,
       'icudtl_compressed_bytes': icudtl.compressedSize,
       'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
       'libflutter_compressed_bytes': libflutter.compressedSize,
       'snapshot_uncompressed_bytes': isolateSnapshotData.uncompressedSize +
           isolateSnapshotInstr.uncompressedSize +
           vmSnapshotData.uncompressedSize +
           vmSnapshotInstr.uncompressedSize,
       'snapshot_compressed_bytes': isolateSnapshotData.compressedSize +
           isolateSnapshotInstr.compressedSize +
           vmSnapshotData.compressedSize +
           vmSnapshotInstr.compressedSize,
     };
   }
 }

 /// Measure application memory usage.
 class MemoryTest {
   const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });

   final String testDirectory;
   final String packageName;

   /// Path to a flutter driver script that will run after starting the app.
   ///
   /// If not specified, then the test will start the app, gather statistics, and then exit.
   final String testTarget;

   Future<TaskResult> run() {
     return inDirectory(testDirectory, () async {
       final Device device = await devices.workingDevice;
       await device.unlock();
       final String deviceId = device.deviceId;
       await flutter('packages', options: <String>['get']);

       if (deviceOperatingSystem == DeviceOperatingSystem.ios)
         await prepareProvisioningCertificates(testDirectory);

       final List<String> runOptions = <String>[
         '-v',
         '--profile',
         '--trace-startup', // wait for the first frame to render
         '-d',
         deviceId,
         '--observatory-port',
         '0',
       ];
       if (testTarget != null)
         runOptions.addAll(<String>['-t', testTarget]);
       final String output = await evalFlutter('run', options: runOptions);
       final int observatoryPort = parseServicePort(output, prefix: 'Successfully connected to service protocol: ', multiLine: true);
       if (observatoryPort == null)
         throw new Exception('Could not find observatory port in "flutter run" output.');

       final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

       final Map<String, dynamic> data = <String, dynamic>{
          'start_total_kb': startData['total_kb'],
       };

       if (testTarget != null) {
         await flutter('drive', options: <String>[
           '-v',
           '-t',
           testTarget,
           '-d',
           deviceId,
           '--use-existing-app=http://localhost:$observatoryPort',
         ]);

         final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
         data['end_total_kb'] = endData['total_kb'];
         data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
       }

       await device.stop(packageName);

       return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
     });
   }
 }

 /// Measure application memory usage after pausing and resuming the app
 /// with the Android back button.
 class AndroidBackButtonMemoryTest {
   const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);

   final String testDirectory;
   final String packageName;
   final String activityName;

   Future<TaskResult> run() {
     return inDirectory(testDirectory, () async {
       if (deviceOperatingSystem != DeviceOperatingSystem.android) {
         throw 'This test is only supported on Android';
       }

       final AndroidDevice device = await devices.workingDevice;
       await device.unlock();
       final String deviceId = device.deviceId;
       await flutter('packages', options: <String>['get']);

       await flutter('run', options: <String>[
         '-v',
         '--profile',
         '--trace-startup', // wait for the first frame to render
         '-d',
         deviceId,
       ]);

       final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

       final Map<String, dynamic> data = <String, dynamic>{
          'start_total_kb': startData['total_kb'],
       };

       // Perform a series of back button suspend and resume cycles.
       for (int i = 0; i < 10; i++) {
         await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
         await new Future<Null>.delayed(const Duration(milliseconds: 1000));
         final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
         print(output);
         if (output.contains('Error'))
           return new TaskResult.failure('unable to launch activity');
         await new Future<Null>.delayed(const Duration(milliseconds: 1000));
       }

       final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
       data['end_total_kb'] = endData['total_kb'];
       data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];

       await device.stop(packageName);

       return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
     });
   }
 }

 class _UnzipListEntry {
   factory _UnzipListEntry.fromLine(String line) {
     final List<String> data = line.trim().split(new RegExp('\\s+'));
     assert(data.length == 8);
     return new _UnzipListEntry._(
       uncompressedSize:  int.parse(data[0]),
       compressedSize: int.parse(data[2]),
       path: data[7],
     );
   }

   _UnzipListEntry._({
     @required this.uncompressedSize,
     @required this.compressedSize,
     @required this.path,
   }) : assert(uncompressedSize != null),
        assert(compressedSize != null),
        assert(compressedSize <= uncompressedSize),
        assert(path != null);

   final int uncompressedSize;
   final int compressedSize;
   final String path;
 }
	// Copyright (c) 2016 The Chromium Authors. All rights reserved.
	// Use of this source code is governed by a BSD-style license that can be
	// found in the LICENSE file.

	import 'dart:async';
	import 'dart:convert' show json;
	import 'dart:io';

	import 'package:meta/meta.dart';
	import 'package:path/path.dart' as p;

	import '../framework/adb.dart';
	import '../framework/framework.dart';
	import '../framework/ios.dart';
	import '../framework/utils.dart';

	TaskFunction createComplexLayoutScrollPerfTest() {
	return new PerfTest(
	'${flutterDirectory.path}/dev/benchmarks/complex_layout',
	'test_driver/scroll_perf.dart',
	'complex_layout_scroll_perf',
	).run;
	}

	TaskFunction createTilesScrollPerfTest() {
	return new PerfTest(
	'${flutterDirectory.path}/dev/benchmarks/complex_layout',
	'test_driver/scroll_perf.dart',
	'tiles_scroll_perf',
	).run;
	}

	TaskFunction createComplexLayoutScrollMemoryTest() {
	return new MemoryTest(
	'${flutterDirectory.path}/dev/benchmarks/complex_layout',
	'com.yourcompany.complexLayout',
	testTarget: 'test_driver/scroll_perf.dart',
	).run;
	}

	TaskFunction createFlutterGalleryStartupTest() {
	return new StartupTest(
	'${flutterDirectory.path}/examples/flutter_gallery',
	).run;
	}

	TaskFunction createComplexLayoutStartupTest() {
	return new StartupTest(
	'${flutterDirectory.path}/dev/benchmarks/complex_layout',
	).run;
	}

	TaskFunction createFlutterGalleryCompileTest() {
	return new CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
	}

	TaskFunction createHelloWorldCompileTest() {
	return new CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
	}

	TaskFunction createComplexLayoutCompileTest() {
	return new CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
	}

	TaskFunction createHelloWorldMemoryTest() {
	return new MemoryTest(
	'${flutterDirectory.path}/examples/hello_world',
	'io.flutter.examples.hello_world',
	).run;
	}

	TaskFunction createGalleryNavigationMemoryTest() {
	return new MemoryTest(
	'${flutterDirectory.path}/examples/flutter_gallery',
	'io.flutter.demo.gallery',
	testTarget: 'test_driver/memory_nav.dart',
	).run;
	}

	TaskFunction createGalleryBackButtonMemoryTest() {
	return new AndroidBackButtonMemoryTest(
	'${flutterDirectory.path}/examples/flutter_gallery',
	'io.flutter.demo.gallery',
	'io.flutter.demo.gallery.MainActivity',
	).run;
	}

	TaskFunction createFlutterViewStartupTest() {
	return new StartupTest(
	'${flutterDirectory.path}/examples/flutter_view',
	reportMetrics: false,
	).run;
	}

	TaskFunction createPlatformViewStartupTest() {
	return new StartupTest(
	'${flutterDirectory.path}/examples/platform_view',
	reportMetrics: false,
	).run;
	}

	TaskFunction createBasicMaterialCompileTest() {
	return () async {
	const String sampleAppName = 'sample_flutter_app';
	final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');

	if (await sampleDir.exists())
	rmTree(sampleDir);

	await inDirectory(Directory.systemTemp, () async {
	await flutter('create', options: <String>[sampleAppName]);
	});

	if (!(await sampleDir.exists()))
	throw 'Failed to create default Flutter app in ${sampleDir.path}';

	return new CompileTest(sampleDir.path).run();
	};
	}


	/// Measure application startup performance.
	class StartupTest {
	static const Duration _startupTimeout = const Duration(minutes: 5);

	const StartupTest(this.testDirectory, { this.reportMetrics = true });

	final String testDirectory;
	final bool reportMetrics;

	Future<TaskResult> run() async {
	return await inDirectory(testDirectory, () async {
	final String deviceId = (await devices.workingDevice).deviceId;
	await flutter('packages', options: <String>['get']);

	if (deviceOperatingSystem == DeviceOperatingSystem.ios)
	await prepareProvisioningCertificates(testDirectory);

	await flutter('run', options: <String>[
	'--verbose',
	'--profile',
	'--trace-startup',
	'-d',
	deviceId,
	]).timeout(_startupTimeout);
	final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());

	if (!reportMetrics)
	return new TaskResult.success(data);

	return new TaskResult.success(data, benchmarkScoreKeys: <String>[
	'timeToFirstFrameMicros',
	]);
	});
	}
	}

	/// Measures application runtime performance, specifically per-frame
	/// performance.
	class PerfTest {
	const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName);

	final String testDirectory;
	final String testTarget;
	final String timelineFileName;

	Future<TaskResult> run() {
	return inDirectory(testDirectory, () async {
	final Device device = await devices.workingDevice;
	await device.unlock();
	final String deviceId = device.deviceId;
	await flutter('packages', options: <String>['get']);

	if (deviceOperatingSystem == DeviceOperatingSystem.ios)
	await prepareProvisioningCertificates(testDirectory);

	await flutter('drive', options: <String>[
	'-v',
	'--profile',
	'--trace-startup', // Enables "endless" timeline event buffering.
	'-t',
	testTarget,
	'-d',
	deviceId,
	]);
	final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());

	if (data['frame_count'] < 5) {
	return new TaskResult.failure(
	'Timeline contains too few frames: ${data['frame_count']}. Possibly '
	'trace events are not being captured.',
	);
	}

	return new TaskResult.success(data, benchmarkScoreKeys: <String>[
	'average_frame_build_time_millis',
	'worst_frame_build_time_millis',
	'missed_frame_build_budget_count',
	'average_frame_rasterizer_time_millis',
	'worst_frame_rasterizer_time_millis',
	'90th_percentile_frame_rasterizer_time_millis',
	'99th_percentile_frame_rasterizer_time_millis',
	]);
	});
	}
	}

	/// Measures how long it takes to compile a Flutter app and how big the compiled
	/// code is.
	class CompileTest {
	const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });

	final String testDirectory;
	final bool reportPackageContentSizes;

	Future<TaskResult> run() async {
	return await inDirectory(testDirectory, () async {
	final Device device = await devices.workingDevice;
	await device.unlock();
	await flutter('packages', options: <String>['get']);

	final Map<String, dynamic> metrics = <String, dynamic>{}
	..addAll(await _compileAot())
	..addAll(await _compileApp(reportPackageContentSizes: reportPackageContentSizes))
	..addAll(await _compileDebug());

	return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
	});
	}

	static Future<Map<String, dynamic>> _compileAot({ bool previewDart2 = true }) async {
	// Generate blobs instead of assembly.
	await flutter('clean');
	final Stopwatch watch = new Stopwatch()..start();
	final List<String> options = <String>[
	'aot',
	'-v',
	'--extra-gen-snapshot-options=--print_snapshot_sizes',
	'--release',
	'--no-pub',
	'--target-platform',
	];
	switch (deviceOperatingSystem) {
	case DeviceOperatingSystem.ios:
	options.add('ios');
	break;
	case DeviceOperatingSystem.android:
	options.add('android-arm');
	break;
	}
	if (previewDart2)
	options.add('--preview-dart-2');
	else
	options.add('--no-preview-dart-2');
	setLocalEngineOptionIfNecessary(options);
	final String compileLog = await evalFlutter('build', options: options);
	watch.stop();

	final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
	final Map<String, dynamic> metrics = <String, dynamic>{};
	for (Match m in metricExpression.allMatches(compileLog)) {
	metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
	}
	if (metrics.length != _kSdkNameToMetricNameMapping.length) {
	throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
	}
	metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;

	return metrics;
	}

	static Future<Map<String, dynamic>> _compileApp({ bool previewDart2 = true, bool reportPackageContentSizes = false }) async {
	await flutter('clean');
	final Stopwatch watch = new Stopwatch();
	int releaseSizeInBytes;
	final List<String> options = <String>['--release'];
	if (previewDart2)
	options.add('--preview-dart-2');
	else
	options.add('--no-preview-dart-2');
	setLocalEngineOptionIfNecessary(options);
	final Map<String, dynamic> metrics = <String, dynamic>{};

	switch (deviceOperatingSystem) {
	case DeviceOperatingSystem.ios:
	options.insert(0, 'ios');
	await prepareProvisioningCertificates(cwd);
	watch.start();
	await flutter('build', options: options);
	watch.stop();
	final String appPath = '$cwd/build/ios/Release-iphoneos/Runner.app/';
	// IPAs are created manually, https://flutter.io/ios-release/
	await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
	releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
	if (reportPackageContentSizes)
	metrics.addAll(await getSizesFromIosApp(appPath));
	break;
	case DeviceOperatingSystem.android:
	options.insert(0, 'apk');
	watch.start();
	await flutter('build', options: options);
	watch.stop();
	String apkPath = '$cwd/build/app/outputs/apk/app.apk';
	File apk = file(apkPath);
	if (!apk.existsSync()) {
	// Pre Android SDK 26 path
	apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
	apk = file(apkPath);
	}
	releaseSizeInBytes = apk.lengthSync();
	if (reportPackageContentSizes)
	metrics.addAll(await getSizesFromApk(apkPath));
	break;
	}

	metrics.addAll(<String, dynamic>{
	'release_full_compile_millis': watch.elapsedMilliseconds,
	'release_size_bytes': releaseSizeInBytes,
	});

	return metrics;
	}

	static Future<Map<String, dynamic>> _compileDebug({ bool previewDart2 = true }) async {
	await flutter('clean');
	final Stopwatch watch = new Stopwatch();
	final List<String> options = <String>['--debug'];
	if (previewDart2)
	options.add('--preview-dart-2');
	else
	options.add('--no-preview-dart-2');
	setLocalEngineOptionIfNecessary(options);
	switch (deviceOperatingSystem) {
	case DeviceOperatingSystem.ios:
	options.insert(0, 'ios');
	await prepareProvisioningCertificates(cwd);
	break;
	case DeviceOperatingSystem.android:
	options.insert(0, 'apk');
	break;
	}
	watch.start();
	await flutter('build', options: options);
	watch.stop();

	return <String, dynamic>{
	'debug_full_compile_millis': watch.elapsedMilliseconds,
	};
	}

	static const Map<String, String> _kSdkNameToMetricNameMapping = const <String, String> {
	'VMIsolate': 'aot_snapshot_size_vmisolate',
	'Isolate': 'aot_snapshot_size_isolate',
	'ReadOnlyData': 'aot_snapshot_size_rodata',
	'Instructions': 'aot_snapshot_size_instructions',
	'Total': 'aot_snapshot_size_total',
	};

	static String _sdkNameToMetricName(String sdkName) {

	if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
	throw 'Unrecognized SDK snapshot metric name: $sdkName';

	return _kSdkNameToMetricNameMapping[sdkName];
	}

	static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
	// Thin the binary to only contain one architecture.
	final String xcodeBackend = p.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
	await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
	'ARCHS': 'arm64',
	'WRAPPER_NAME': p.basename(appPath),
	'TARGET_BUILD_DIR': p.dirname(appPath),
	});

	final File appFramework = new File(p.join(appPath, 'Frameworks', 'App.framework', 'App'));
	final File flutterFramework = new File(p.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));

	return <String, dynamic>{
	'app_framework_uncompressed_bytes': await appFramework.length(),
	'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
	};
	}


	static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
	final String output = await eval('unzip', <String>['-v', apkPath]);
	final List<String> lines = output.split('\n');
	final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};

	// First three lines are header, last two lines are footer.
	for (int i = 3; i < lines.length - 2; i++) {
	final _UnzipListEntry entry = new _UnzipListEntry.fromLine(lines[i]);
	fileToMetadata[entry.path] = entry;
	}

	final _UnzipListEntry icudtl = fileToMetadata['assets/flutter_shared/icudtl.dat'];
	final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
	final _UnzipListEntry isolateSnapshotData = fileToMetadata['assets/isolate_snapshot_data'];
	final _UnzipListEntry isolateSnapshotInstr = fileToMetadata['assets/isolate_snapshot_instr'];
	final _UnzipListEntry vmSnapshotData = fileToMetadata['assets/vm_snapshot_data'];
	final _UnzipListEntry vmSnapshotInstr = fileToMetadata['assets/vm_snapshot_instr'];

	return <String, dynamic>{
	'icudtl_uncompressed_bytes': icudtl.uncompressedSize,
	'icudtl_compressed_bytes': icudtl.compressedSize,
	'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
	'libflutter_compressed_bytes': libflutter.compressedSize,
	'snapshot_uncompressed_bytes': isolateSnapshotData.uncompressedSize +
	isolateSnapshotInstr.uncompressedSize +
	vmSnapshotData.uncompressedSize +
	vmSnapshotInstr.uncompressedSize,
	'snapshot_compressed_bytes': isolateSnapshotData.compressedSize +
	isolateSnapshotInstr.compressedSize +
	vmSnapshotData.compressedSize +
	vmSnapshotInstr.compressedSize,
	};
	}
	}

	/// Measure application memory usage.
	class MemoryTest {
	const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });

	final String testDirectory;
	final String packageName;

	/// Path to a flutter driver script that will run after starting the app.
	///
	/// If not specified, then the test will start the app, gather statistics, and then exit.
	final String testTarget;

	Future<TaskResult> run() {
	return inDirectory(testDirectory, () async {
	final Device device = await devices.workingDevice;
	await device.unlock();
	final String deviceId = device.deviceId;
	await flutter('packages', options: <String>['get']);

	if (deviceOperatingSystem == DeviceOperatingSystem.ios)
	await prepareProvisioningCertificates(testDirectory);

	final List<String> runOptions = <String>[
	'-v',
	'--profile',
	'--trace-startup', // wait for the first frame to render
	'-d',
	deviceId,
	'--observatory-port',
	'0',
	];
	if (testTarget != null)
	runOptions.addAll(<String>['-t', testTarget]);
	final String output = await evalFlutter('run', options: runOptions);
	final int observatoryPort = parseServicePort(output, prefix: 'Successfully connected to service protocol: ', multiLine: true);
	if (observatoryPort == null)
	throw new Exception('Could not find observatory port in "flutter run" output.');

	final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

	final Map<String, dynamic> data = <String, dynamic>{
	'start_total_kb': startData['total_kb'],
	};

	if (testTarget != null) {
	await flutter('drive', options: <String>[
	'-v',
	'-t',
	testTarget,
	'-d',
	deviceId,
	'--use-existing-app=http://localhost:$observatoryPort',
	]);

	final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
	data['end_total_kb'] = endData['total_kb'];
	data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
	}

	await device.stop(packageName);

	return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
	});
	}
	}

	/// Measure application memory usage after pausing and resuming the app
	/// with the Android back button.
	class AndroidBackButtonMemoryTest {
	const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);

	final String testDirectory;
	final String packageName;
	final String activityName;

	Future<TaskResult> run() {
	return inDirectory(testDirectory, () async {
	if (deviceOperatingSystem != DeviceOperatingSystem.android) {
	throw 'This test is only supported on Android';
	}

	final AndroidDevice device = await devices.workingDevice;
	await device.unlock();
	final String deviceId = device.deviceId;
	await flutter('packages', options: <String>['get']);

	await flutter('run', options: <String>[
	'-v',
	'--profile',
	'--trace-startup', // wait for the first frame to render
	'-d',
	deviceId,
	]);

	final Map<String, dynamic> startData = await device.getMemoryStats(packageName);

	final Map<String, dynamic> data = <String, dynamic>{
	'start_total_kb': startData['total_kb'],
	};

	// Perform a series of back button suspend and resume cycles.
	for (int i = 0; i < 10; i++) {
	await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
	await new Future<Null>.delayed(const Duration(milliseconds: 1000));
	final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
	print(output);
	if (output.contains('Error'))
	return new TaskResult.failure('unable to launch activity');
	await new Future<Null>.delayed(const Duration(milliseconds: 1000));
	}

	final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
	data['end_total_kb'] = endData['total_kb'];
	data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];

	await device.stop(packageName);

	return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
	});
	}
	}

	class _UnzipListEntry {
	factory _UnzipListEntry.fromLine(String line) {
	final List<String> data = line.trim().split(new RegExp('\\s+'));
	assert(data.length == 8);
	return new _UnzipListEntry._(
	uncompressedSize: int.parse(data[0]),
	compressedSize: int.parse(data[2]),
	path: data[7],
	);
	}

	_UnzipListEntry._({
	@required this.uncompressedSize,
	@required this.compressedSize,
	@required this.path,
	}) : assert(uncompressedSize != null),
	assert(compressedSize != null),
	assert(compressedSize <= uncompressedSize),
	assert(path != null);

	final int uncompressedSize;
	final int compressedSize;
	final String path;
	}