dev/devicelab/lib/tasks/gallery.dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;

import '../framework/devices.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
import 'build_test_task.dart';

final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');

/// Temp function during gallery tests transition to build+test model.
///
/// https://github.com/flutter/flutter/issues/103542
TaskFunction createGalleryTransitionBuildTest(List<String> args, {bool semanticsEnabled = false}) {
  return GalleryTransitionBuildTest(args, semanticsEnabled: semanticsEnabled).call;
}

TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
  return GalleryTransitionTest(semanticsEnabled: semanticsEnabled).call;
}

TaskFunction createGalleryTransitionE2EBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
  bool? enableImpeller,
}) {
  return GalleryTransitionBuildTest(
    args,
    testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e',
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
    timelineTraceFile: null,
    driverFile: 'transitions_perf_e2e_test',
    enableImpeller: enableImpeller,
  ).call;
}

TaskFunction createGalleryTransitionE2ETest({
  bool semanticsEnabled = false,
  bool? enableImpeller,
}) {
  return GalleryTransitionTest(
    testFile: semanticsEnabled
        ? 'transitions_perf_e2e_with_semantics'
        : 'transitions_perf_e2e',
    needFullTimeline: false,
    timelineSummaryFile: 'e2e_perf_summary',
    transitionDurationFile: null,
    timelineTraceFile: null,
    driverFile: 'transitions_perf_e2e_test',
    enableImpeller: enableImpeller,
  ).call;
}

TaskFunction createGalleryTransitionHybridBuildTest(
  List<String> args, {
  bool semanticsEnabled = false,
}) {
  return GalleryTransitionBuildTest(
    args,
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test',
  ).call;
}

TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
  return GalleryTransitionTest(
    semanticsEnabled: semanticsEnabled,
    driverFile: semanticsEnabled
        ? 'transitions_perf_hybrid_with_semantics_test'
        : 'transitions_perf_hybrid_test',
  ).call;
}

class GalleryTransitionTest {

  GalleryTransitionTest({
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
    this.timelineTraceFile = 'transitions.timeline',
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
    this.measureCpuGpu = true,
    this.measureMemory = true,
    this.enableImpeller,
  });

  final bool semanticsEnabled;
  final bool needFullTimeline;
  final bool measureCpuGpu;
  final bool measureMemory;
  final bool? enableImpeller;
  final String testFile;
  final String timelineSummaryFile;
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;

  Future<TaskResult> call() async {
    final Device device = await devices.workingDevice;
    await device.unlock();
    final String deviceId = device.deviceId;
    final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
    await inDirectory<void>(galleryDirectory, () async {
      String? applicationBinaryPath;
      if (deviceOperatingSystem == DeviceOperatingSystem.android) {
        section('BUILDING APPLICATION');
        await flutter(
          'build',
          options: <String>[
            'apk',
            '--no-android-gradle-daemon',
            '--profile',
            '-t',
            'test_driver/$testFile.dart',
            '--target-platform',
            'android-arm,android-arm64',
          ],
        );
        applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
      }

      final String testDriver = driverFile ?? (semanticsEnabled
          ? '${testFile}_with_semantics_test'
          : '${testFile}_test');
      section('DRIVE START');
      await flutter('drive', options: <String>[
        '--no-dds',
        '--profile',
        if (enableImpeller != null && enableImpeller!) '--enable-impeller',
        if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
        if (needFullTimeline)
          '--trace-startup',
        if (applicationBinaryPath != null)
          '--use-application-binary=$applicationBinaryPath'
        else
          ...<String>[
            '-t',
            'test_driver/$testFile.dart',
          ],
        '--driver',
        'test_driver/$testDriver.dart',
        '-d',
        deviceId,
        '-v',
        '--verbose-system-logs'
      ]);
    });

    final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
    final Map<String, dynamic> summary = json.decode(
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
    ) as Map<String, dynamic>;

    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }

    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
    return TaskResult.success(summary,
      detailFiles: <String>[
        if (transitionDurationFile != null)
          '$testOutputDirectory/$transitionDurationFile.json',
        if (timelineTraceFile != null)
          '$testOutputDirectory/$timelineTraceFile.json',
      ],
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null)
          'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
        'stddev_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
      ],
    );
  }
}

class GalleryTransitionBuildTest extends BuildTestTask {
  GalleryTransitionBuildTest(
    super.args, {
    this.semanticsEnabled = false,
    this.testFile = 'transitions_perf',
    this.needFullTimeline = true,
    this.timelineSummaryFile = 'transitions.timeline_summary',
    this.timelineTraceFile = 'transitions.timeline',
    this.transitionDurationFile = 'transition_durations.timeline',
    this.driverFile,
    this.measureCpuGpu = true,
    this.measureMemory = true,
    this.enableImpeller,
  }) : super(workingDirectory: galleryDirectory);

  final bool semanticsEnabled;
  final bool needFullTimeline;
  final bool measureCpuGpu;
  final bool measureMemory;
  final bool? enableImpeller;
  final String testFile;
  final String timelineSummaryFile;
  final String? timelineTraceFile;
  final String? transitionDurationFile;
  final String? driverFile;

  final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';

  @override
  void copyArtifacts() {
    if (applicationBinaryPath == null) {
      return;
    }
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
      copy(
        file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'),
        Directory(applicationBinaryPath!),
      );
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      recursiveCopy(
        Directory('${galleryDirectory.path}/build/ios/iphoneos'),
        Directory(applicationBinaryPath!),
      );
    }
  }

  @override
  List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
      return <String>[
        'apk',
        '--no-android-gradle-daemon',
        '--profile',
        '-t',
        'test_driver/$testFile.dart',
        '--target-platform',
        'android-arm,android-arm64',
      ];
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      return <String>[
        'ios',
        '--codesign',
        '--profile',
        '-t',
        'test_driver/$testFile.dart',
      ];
    }
    throw Exception('$deviceOperatingSystem has no build configuration');
  }

  @override
  List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
    final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test');
    return <String>[
      '--no-dds',
      '--profile',
      if (enableImpeller != null && enableImpeller!) '--enable-impeller',
      if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
      if (needFullTimeline) '--trace-startup',
      '-t',
      'test_driver/$testFile.dart',
      if (applicationBinaryPath != null) '--use-application-binary=${getApplicationBinaryPath()}',
      '--driver',
      'test_driver/$testDriver.dart',
      '-d',
      deviceId,
    ];
  }

  @override
  Future<TaskResult> parseTaskResult() async {
    final Map<String, dynamic> summary = json.decode(
      file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
    ) as Map<String, dynamic>;

    if (transitionDurationFile != null) {
      final Map<String, dynamic> original = json.decode(
        file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
      ) as Map<String, dynamic>;
      final Map<String, List<int>> transitions = <String, List<int>>{};
      for (final String key in original.keys) {
        transitions[key] = List<int>.from(original[key] as List<dynamic>);
      }
      summary['transitions'] = transitions;
      summary['missed_transition_count'] = _countMissedTransitions(transitions);
    }

    final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
    return TaskResult.success(
      summary,
      detailFiles: <String>[
        if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json',
        if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json',
      ],
      benchmarkScoreKeys: <String>[
        if (transitionDurationFile != null) 'missed_transition_count',
        'average_frame_build_time_millis',
        'worst_frame_build_time_millis',
        '90th_percentile_frame_build_time_millis',
        '99th_percentile_frame_build_time_millis',
        'average_frame_rasterizer_time_millis',
        'worst_frame_rasterizer_time_millis',
        '90th_percentile_frame_rasterizer_time_millis',
        '99th_percentile_frame_rasterizer_time_millis',
        'average_layer_cache_count',
        '90th_percentile_layer_cache_count',
        '99th_percentile_layer_cache_count',
        'worst_layer_cache_count',
        'average_layer_cache_memory',
        '90th_percentile_layer_cache_memory',
        '99th_percentile_layer_cache_memory',
        'worst_layer_cache_memory',
        'average_picture_cache_count',
        '90th_percentile_picture_cache_count',
        '99th_percentile_picture_cache_count',
        'worst_picture_cache_count',
        'average_picture_cache_memory',
        '90th_percentile_picture_cache_memory',
        '99th_percentile_picture_cache_memory',
        'worst_picture_cache_memory',
        if (measureCpuGpu && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
          if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
        ],
        if (measureMemory && !isAndroid) ...<String>[
          // See https://github.com/flutter/flutter/issues/68888
          if (summary['average_memory_usage'] != null) 'average_memory_usage',
          if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
          if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
        ],
      ],
    );
  }

  @override
  String getApplicationBinaryPath() {
    if (deviceOperatingSystem == DeviceOperatingSystem.android) {
      return '$applicationBinaryPath/app-profile.apk';
    } else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
      return '$applicationBinaryPath/Flutter Gallery.app';
    } else {
      return applicationBinaryPath!;
    }
  }
}

int _countMissedTransitions(Map<String, List<int>> transitions) {
  const int kTransitionBudget = 100000; // µs
  int count = 0;
  transitions.forEach((String demoName, List<int> durations) {
    final int longestDuration = durations.reduce(math.max);
    if (longestDuration > kTransitionBudget) {
      print('$demoName missed transition time budget ($longestDuration µs > $kTransitionBudget µs)');
      count++;
    }
  });
  return count;
}