Implement size analyzer to unzip & parse APK and AOT size snapshot to generate analysis json (#62495)
* Implement size analyzer to unzip & parse APK and AOT size snapshot to generate analysis json
diff --git a/packages/flutter_tools/lib/src/base/analyze_size.dart b/packages/flutter_tools/lib/src/base/analyze_size.dart
new file mode 100644
index 0000000..aa60c53
--- /dev/null
+++ b/packages/flutter_tools/lib/src/base/analyze_size.dart
@@ -0,0 +1,370 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:meta/meta.dart';
+import 'package:vm_snapshot_analysis/treemap.dart';
+
+import '../base/file_system.dart';
+import '../convert.dart';
+import 'logger.dart';
+import 'process.dart';
+import 'terminal.dart';
+
+/// A class to analyze APK and AOT snapshot and generate a breakdown of the data.
+class SizeAnalyzer {
+ SizeAnalyzer({
+ @required this.fileSystem,
+ @required this.logger,
+ @required this.processUtils,
+ });
+
+ final FileSystem fileSystem;
+ final Logger logger;
+ final ProcessUtils processUtils;
+
+ static const String aotSnapshotFileName = 'aot-snapshot.json';
+
+ static const int tableWidth = 80;
+
+ /// Analyzes [apk] and [aotSnapshot] to output a [Map] object that includes
+ /// the breakdown of the both files, where the breakdown of [aotSnapshot] is placed
+ /// under 'lib/arm64-v8a/libapp.so'.
+ ///
+ /// The [aotSnapshot] can be either instruction sizes snapshot or v8 snapshot.
+ Future<Map<String, dynamic>> analyzeApkSizeAndAotSnapshot({
+ @required File apk,
+ @required File aotSnapshot,
+ }) async {
+ logger.printStatus('▒' * tableWidth);
+ _printEntitySize(
+ '${apk.basename} (total compressed)',
+ byteSize: apk.lengthSync(),
+ level: 0,
+ showColor: false,
+ );
+ logger.printStatus('━' * tableWidth);
+ final Directory tempApkContent = fileSystem.systemTempDirectory.createTempSync('flutter_tools.');
+ // TODO(peterdjlee): Implement a way to unzip the APK for Windows. See issue #62603.
+ String unzipOut;
+ try {
+ // TODO(peterdjlee): Use zipinfo instead of unzip.
+ unzipOut = (await processUtils.run(<String>[
+ 'unzip',
+ '-o',
+ '-v',
+ apk.path,
+ '-d',
+ tempApkContent.path
+ ])).stdout;
+ } on Exception catch (e) {
+ logger.printError(e.toString());
+ } finally {
+ // We just want the the stdout printout. We don't need the files.
+ tempApkContent.deleteSync(recursive: true);
+ }
+
+ final _SymbolNode apkAnalysisRoot = _parseUnzipFile(unzipOut);
+
+ // Convert an AOT snapshot file into a map.
+ final Map<String, dynamic> processedAotSnapshotJson = treemapFromJson(
+ json.decode(aotSnapshot.readAsStringSync()),
+ );
+ final _SymbolNode aotSnapshotJsonRoot = _parseAotSnapshot(processedAotSnapshotJson);
+
+ for (final _SymbolNode firstLevelPath in apkAnalysisRoot.children) {
+ _printEntitySize(
+ firstLevelPath.name,
+ byteSize: firstLevelPath.byteSize,
+ level: 1,
+ );
+ // Print the expansion of lib directory to show more info for libapp.so.
+ if (firstLevelPath.name == 'lib') {
+ _printLibChildrenPaths(firstLevelPath, '', aotSnapshotJsonRoot);
+ }
+ }
+
+ logger.printStatus('▒' * tableWidth);
+
+ Map<String, dynamic> apkAnalysisJson = apkAnalysisRoot.toJson();
+
+ apkAnalysisJson['type'] = 'apk';
+
+ // TODO(peterdjlee): Add aot snapshot for all platforms.
+ apkAnalysisJson = _addAotSnapshotDataToApkAnalysis(
+ apkAnalysisJson: apkAnalysisJson,
+ path: 'lib/arm64-v8a/libapp.so (Dart AOT)'.split('/'), // Pass in a list of paths by splitting with '/'.
+ aotSnapshotJson: processedAotSnapshotJson,
+ );
+
+ return apkAnalysisJson;
+ }
+
+
+ // Expression to match 'Size' column to group 1 and 'Name' column to group 2.
+ final RegExp _parseUnzipOutput = RegExp(r'^\s*\d+\s+[\w|:]+\s+(\d+)\s+.* (.+)$');
+
+ // Parse the output of unzip -v which shows the zip's contents' compressed sizes.
+ // Example output of unzip -v:
+ // Length Method Size Cmpr Date Time CRC-32 Name
+ // -------- ------ ------- ---- ---------- ----- -------- ----
+ // 11708 Defl:N 2592 78% 00-00-1980 00:00 07733eef AndroidManifest.xml
+ // 1399 Defl:N 1092 22% 00-00-1980 00:00 f53d952a META-INF/CERT.RSA
+ // 46298 Defl:N 14530 69% 00-00-1980 00:00 17df02b8 META-INF/CERT.SF
+ _SymbolNode _parseUnzipFile(String unzipOut) {
+ final Map<List<String>, int> pathsToSize = <List<String>, int>{};
+
+ // Parse each path into pathsToSize so that the key is a list of
+ // path parts and the value is the size.
+ // For example:
+ // 'path/to/file' where file = 1500 => pathsToSize[['path', 'to', 'file']] = 1500
+ for (final String line in const LineSplitter().convert(unzipOut)) {
+ final RegExpMatch match = _parseUnzipOutput.firstMatch(line);
+ if (match == null) {
+ continue;
+ }
+ const int sizeGroupIndex = 1;
+ const int nameGroupIndex = 2;
+ pathsToSize[match.group(nameGroupIndex).split('/')] = int.parse(match.group(sizeGroupIndex));
+ }
+
+ final _SymbolNode rootNode = _SymbolNode('Root');
+
+ _SymbolNode currentNode = rootNode;
+ for (final List<String> paths in pathsToSize.keys) {
+ for (final String path in paths) {
+ _SymbolNode childWithPathAsName = currentNode.childByName(path);
+
+ if (childWithPathAsName == null) {
+ childWithPathAsName = _SymbolNode(path);
+ if (path.endsWith('libapp.so')) {
+ childWithPathAsName.name += ' (Dart AOT)';
+ } else if (path.endsWith('libflutter.so')) {
+ childWithPathAsName.name += ' (Flutter Engine)';
+ }
+ currentNode.addChild(childWithPathAsName);
+ }
+ childWithPathAsName.addSize(pathsToSize[paths]);
+ currentNode = childWithPathAsName;
+ }
+ currentNode = rootNode;
+ }
+
+ return rootNode;
+ }
+
+ /// Prints all children paths for the lib/ directory in an APK.
+ ///
+ /// A brief summary of aot snapshot is printed under 'lib/arm64-v8a/libapp.so'.
+ void _printLibChildrenPaths(
+ _SymbolNode currentNode,
+ String totalPath,
+ _SymbolNode aotSnapshotJsonRoot,
+ ) {
+ totalPath += currentNode.name;
+
+ if (currentNode.children.isNotEmpty && !currentNode.name.contains('libapp.so')) {
+ for (final _SymbolNode child in currentNode.children) {
+ _printLibChildrenPaths(child, '$totalPath/', aotSnapshotJsonRoot);
+ }
+ } else {
+ // Print total path and size if currentNode does not have any chilren.
+ _printEntitySize(totalPath, byteSize: currentNode.byteSize, level: 2);
+
+ // We picked this file because arm64-v8a is likely the most popular
+ // architecture. ther architecture sizes should be similar.
+ const String libappPath = 'lib/arm64-v8a/libapp.so';
+ // TODO(peterdjlee): Analyze aot size for all platforms.
+ if (totalPath.contains(libappPath)) {
+ _printAotSnapshotSummary(aotSnapshotJsonRoot);
+ }
+ }
+ }
+
+ /// Go through the AOT gen snapshot size JSON and print out a collapsed summary
+ /// for the first package level.
+ void _printAotSnapshotSummary(_SymbolNode aotSnapshotRoot, {int maxDirectoriesShown = 10}) {
+ _printEntitySize(
+ 'Dart AOT symbols accounted decompressed size',
+ byteSize: aotSnapshotRoot.byteSize,
+ level: 3,
+ );
+
+ final List<_SymbolNode> sortedSymbols = aotSnapshotRoot.children.toList()
+ ..sort((_SymbolNode a, _SymbolNode b) => b.byteSize.compareTo(a.byteSize));
+ for (final _SymbolNode node in sortedSymbols.take(maxDirectoriesShown)) {
+ _printEntitySize(node.name, byteSize: node.byteSize, level: 4);
+ }
+ }
+
+ /// Adds breakdown of aot snapshot data as the children of the node at the given path.
+ Map<String, dynamic> _addAotSnapshotDataToApkAnalysis({
+ @required Map<String, dynamic> apkAnalysisJson,
+ @required List<String> path,
+ @required Map<String, dynamic> aotSnapshotJson,
+ }) {
+ Map<String, dynamic> currentLevel = apkAnalysisJson;
+ while (path.isNotEmpty) {
+ final List<Map<String, dynamic>> children = currentLevel['children'] as List<Map<String, dynamic>>;
+ final Map<String, dynamic> childWithPathAsName = children.firstWhere(
+ (Map<String, dynamic> child) => child['n'] as String == path.first,
+ );
+ path.removeAt(0);
+ currentLevel = childWithPathAsName;
+ }
+ currentLevel['children'] = aotSnapshotJson['children'];
+ return apkAnalysisJson;
+ }
+
+ /// Print an entity's name with its size on the same line.
+ void _printEntitySize(
+ String entityName, {
+ @required int byteSize,
+ @required int level,
+ bool showColor = true,
+ }) {
+ final bool emphasis = level <= 1;
+ final String formattedSize = _prettyPrintBytes(byteSize);
+
+ TerminalColor color = TerminalColor.green;
+ if (formattedSize.endsWith('MB')) {
+ color = TerminalColor.cyan;
+ } else if (formattedSize.endsWith('KB')) {
+ color = TerminalColor.yellow;
+ }
+
+ final int spaceInBetween = tableWidth - level * 2 - entityName.length - formattedSize.length;
+ logger.printStatus(
+ entityName + ' ' * spaceInBetween,
+ newline: false,
+ emphasis: emphasis,
+ indent: level * 2,
+ );
+ logger.printStatus(formattedSize, color: showColor ? color : null);
+ }
+
+ String _prettyPrintBytes(int numBytes) {
+ const int kB = 1024;
+ const int mB = kB * 1024;
+ if (numBytes < kB) {
+ return '$numBytes B';
+ } else if (numBytes < mB) {
+ return '${(numBytes / kB).round()} KB';
+ } else {
+ return '${(numBytes / mB).round()} MB';
+ }
+ }
+
+ _SymbolNode _parseAotSnapshot(Map<String, dynamic> aotSnapshotJson) {
+ final bool isLeafNode = aotSnapshotJson['children'] == null;
+ if (!isLeafNode) {
+ return _buildNodeWithChildren(aotSnapshotJson);
+ } else {
+ // TODO(peterdjlee): Investigate why there are leaf nodes with size of null.
+ final int byteSize = aotSnapshotJson['value'] as int;
+ if (byteSize == null) {
+ return null;
+ }
+ return _buildNode(aotSnapshotJson, byteSize);
+ }
+ }
+
+ _SymbolNode _buildNode(
+ Map<String, dynamic> aotSnapshotJson,
+ int byteSize, {
+ List<_SymbolNode> children = const <_SymbolNode>[],
+ }) {
+ final String name = aotSnapshotJson['n'] as String;
+ final Map<String, _SymbolNode> childrenMap = <String, _SymbolNode>{};
+
+ for (final _SymbolNode child in children) {
+ childrenMap[child.name] = child;
+ }
+
+ return _SymbolNode(
+ name,
+ byteSize: byteSize,
+ )..addAllChildren(children);
+ }
+
+ /// Builds a node by recursively building all of its children first
+ /// in order to calculate the sum of its children's sizes.
+ _SymbolNode _buildNodeWithChildren(Map<String, dynamic> aotSnapshotJson) {
+ final List<dynamic> rawChildren = aotSnapshotJson['children'] as List<dynamic>;
+ final List<_SymbolNode> symbolNodeChildren = <_SymbolNode>[];
+ int totalByteSize = 0;
+
+ // Given a child, build its subtree.
+ for (final dynamic child in rawChildren) {
+ final _SymbolNode childTreemapNode = _parseAotSnapshot(child as Map<String, dynamic>);
+ symbolNodeChildren.add(childTreemapNode);
+ totalByteSize += childTreemapNode.byteSize;
+ }
+
+ // If none of the children matched the diff tree type
+ if (totalByteSize == 0) {
+ return null;
+ } else {
+ return _buildNode(
+ aotSnapshotJson,
+ totalByteSize,
+ children: symbolNodeChildren,
+ );
+ }
+ }
+}
+
+/// A node class that represents a single symbol for AOT size snapshots.
+class _SymbolNode {
+ _SymbolNode(
+ this.name, {
+ this.byteSize = 0,
+ }) : assert(name != null),
+ assert(byteSize != null),
+ _children = <String, _SymbolNode>{};
+
+ /// The human friendly identifier for this node.
+ String name;
+
+ int byteSize;
+ void addSize(int sizeToBeAdded) {
+ byteSize += sizeToBeAdded;
+ }
+
+ _SymbolNode get parent => _parent;
+ _SymbolNode _parent;
+
+ Iterable<_SymbolNode> get children => _children.values;
+ final Map<String, _SymbolNode> _children;
+
+ _SymbolNode childByName(String name) => _children[name];
+
+ _SymbolNode addChild(_SymbolNode child) {
+ assert(child.parent == null);
+ assert(!_children.containsKey(child.name),
+ 'Cannot add duplicate child key ${child.name}');
+
+ child._parent = this;
+ _children[child.name] = child;
+ return child;
+ }
+
+ void addAllChildren(List<_SymbolNode> children) {
+ children.forEach(addChild);
+ }
+
+ Map<String, dynamic> toJson() {
+ final Map<String, dynamic> json = <String, dynamic>{
+ 'n': name,
+ 'value': byteSize
+ };
+ final List<Map<String, dynamic>> childrenAsJson = <Map<String, dynamic>>[];
+ for (final _SymbolNode child in children) {
+ childrenAsJson.add(child.toJson());
+ }
+ if (childrenAsJson.isNotEmpty) {
+ json['children'] = childrenAsJson;
+ }
+ return json;
+ }
+}
diff --git a/packages/flutter_tools/test/general.shard/base/analyze_size_test.dart b/packages/flutter_tools/test/general.shard/base/analyze_size_test.dart
new file mode 100644
index 0000000..df554e9
--- /dev/null
+++ b/packages/flutter_tools/test/general.shard/base/analyze_size_test.dart
@@ -0,0 +1,167 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file/memory.dart';
+import 'package:flutter_tools/src/base/analyze_size.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/base/logger.dart';
+import 'package:flutter_tools/src/base/process.dart';
+
+import '../../src/common.dart';
+import '../../src/context.dart';
+
+const FakeCommand unzipCommmand = FakeCommand(
+ command: <String>[
+ 'unzip',
+ '-o',
+ '-v',
+ 'test.apk',
+ '-d',
+ '/.tmp_rand0/flutter_tools.rand0'
+ ],
+ stdout: '''
+Length Method Size Cmpr Date Time CRC-32 Name
+-------- ------ ------- ---- ---------- ----- -------- ----
+11708 Defl:N 2592 78% 00-00-1980 00:00 07733eef AndroidManifest.xml
+1399 Defl:N 1092 22% 00-00-1980 00:00 f53d952a META-INF/CERT.RSA
+46298 Defl:N 14530 69% 00-00-1980 00:00 17df02b8 META-INF/CERT.SF
+46298 Defl:N 14530 69% 00-00-1980 00:00 17df02b8 lib/arm64-v8a/libapp.so
+46298 Defl:N 14530 69% 00-00-1980 00:00 17df02b8 lib/arm64-v8a/libflutter.so
+''',
+);
+
+const String aotSizeOutput = '''[
+ {
+ "l": "dart:_internal",
+ "c": "SubListIterable",
+ "n": "[Optimized] skip",
+ "s": 2400
+ },
+ {
+ "l": "dart:_internal",
+ "c": "SubListIterable",
+ "n": "[Optimized] new SubListIterable.",
+ "s": 3560
+ },
+ {
+ "l": "dart:core",
+ "c": "RangeError",
+ "n": "[Optimized] new RangeError.range",
+ "s": 3920
+ },
+ {
+ "l": "dart:core",
+ "c": "ArgumentError",
+ "n": "[Stub] Allocate ArgumentError",
+ "s": 4650
+ }
+]
+''';
+
+void main() {
+ MemoryFileSystem fileSystem;
+ BufferLogger logger;
+ FakeProcessManager processManager;
+
+ setUp(() {
+ fileSystem = MemoryFileSystem.test();
+ logger = BufferLogger.test();
+ processManager = FakeProcessManager.list(<FakeCommand>[unzipCommmand]);
+ });
+
+ test('builds APK analysis correctly', () async {
+ final SizeAnalyzer sizeAnalyzer = SizeAnalyzer(
+ fileSystem: fileSystem,
+ logger: logger,
+ processUtils: ProcessUtils(
+ processManager: processManager,
+ logger: logger,
+ ),
+ );
+
+ final File apk = fileSystem.file('test.apk')..createSync();
+ final File aotSizeJson = fileSystem.file('test.json')
+ ..createSync()
+ ..writeAsStringSync(aotSizeOutput);
+ final Map<String, dynamic> result = await sizeAnalyzer.analyzeApkSizeAndAotSnapshot(apk: apk, aotSnapshot: aotSizeJson);
+
+ expect(result['type'], contains('apk'));
+
+ final Map<String, dynamic> androidManifestMap = result['children'][0] as Map<String, dynamic>;
+ expect(androidManifestMap['n'], equals('AndroidManifest.xml'));
+ expect(androidManifestMap['value'], equals(2592));
+
+ final Map<String, dynamic> metaInfMap = result['children'][1] as Map<String, dynamic>;
+ expect(metaInfMap['n'], equals('META-INF'));
+ expect(metaInfMap['value'], equals(15622));
+ final Map<String, dynamic> certRsaMap = metaInfMap['children'][0] as Map<String, dynamic>;
+ expect(certRsaMap['n'], equals('CERT.RSA'));
+ expect(certRsaMap['value'], equals(1092));
+ final Map<String, dynamic> certSfMap = metaInfMap['children'][1] as Map<String, dynamic>;
+ expect(certSfMap['n'], equals('CERT.SF'));
+ expect(certSfMap['value'], equals(14530));
+
+ final Map<String, dynamic> libMap = result['children'][2] as Map<String, dynamic>;
+ expect(libMap['n'], equals('lib'));
+ expect(libMap['value'], equals(29060));
+ final Map<String, dynamic> arm64Map = libMap['children'][0] as Map<String, dynamic>;
+ expect(arm64Map['n'], equals('arm64-v8a'));
+ expect(arm64Map['value'], equals(29060));
+ final Map<String, dynamic> libAppMap = arm64Map['children'][0] as Map<String, dynamic>;
+ expect(libAppMap['n'], equals('libapp.so (Dart AOT)'));
+ expect(libAppMap['value'], equals(14530));
+ expect(libAppMap['children'].length, equals(3));
+ final Map<String, dynamic> internalMap = libAppMap['children'][0] as Map<String, dynamic>;
+ final Map<String, dynamic> skipMap = internalMap['children'][0] as Map<String, dynamic>;
+ expect(skipMap['n'], 'skip');
+ expect(skipMap['value'], 2400);
+ final Map<String, dynamic> subListIterableMap = internalMap['children'][1] as Map<String, dynamic>;
+ expect(subListIterableMap['n'], 'new SubListIterable.');
+ expect(subListIterableMap['value'], 3560);
+ final Map<String, dynamic> coreMap = libAppMap['children'][1] as Map<String, dynamic>;
+ final Map<String, dynamic> rangeErrorMap = coreMap['children'][0] as Map<String, dynamic>;
+ expect(rangeErrorMap['n'], 'new RangeError.range');
+ expect(rangeErrorMap['value'], 3920);
+ final Map<String, dynamic> stubsMap = libAppMap['children'][2] as Map<String, dynamic>;
+ final Map<String, dynamic> allocateMap = stubsMap['children'][0] as Map<String, dynamic>;
+ expect(allocateMap['n'], 'Allocate ArgumentError');
+ expect(allocateMap['value'], 4650);
+ final Map<String, dynamic> libFlutterMap = arm64Map['children'][1] as Map<String, dynamic>;
+ expect(libFlutterMap['n'], equals('libflutter.so (Flutter Engine)'));
+ expect(libFlutterMap['value'], equals(14530));
+ });
+
+ test('outputs summary to command line correctly', () async {
+ final SizeAnalyzer sizeAnalyzer = SizeAnalyzer(
+ fileSystem: fileSystem,
+ logger: logger,
+ processUtils: ProcessUtils(
+ processManager: processManager,
+ logger: logger,
+ ),
+ );
+
+ final File apk = fileSystem.file('test.apk')..createSync();
+ final File aotSizeJson = fileSystem.file('test.json')
+ ..createSync()
+ ..writeAsStringSync(aotSizeOutput);
+ await sizeAnalyzer.analyzeApkSizeAndAotSnapshot(apk: apk, aotSnapshot: aotSizeJson);
+
+ final List<String> stdout = logger.statusText.split('\n');
+ expect(
+ stdout,
+ containsAll(<String>[
+ ' AndroidManifest.xml 3 KB',
+ ' META-INF 15 KB',
+ ' lib 28 KB',
+ ' lib/arm64-v8a/libapp.so (Dart AOT) 14 KB',
+ ' Dart AOT symbols accounted decompressed size 14 KB',
+ ' dart:_internal/SubListIterable 6 KB',
+ ' @stubs/allocation-stubs/dart:core/ArgumentError 5 KB',
+ ' dart:core/RangeError 4 KB',
+ ' lib/arm64-v8a/libflutter.so (Flutter Engine) 14 KB',
+ ]),
+ );
+ });
+}
