script/tool/lib/src/firebase_test_lab_command.dart

// Copyright 2013 The Flutter Authors
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:io' as io;

import 'package:file/file.dart';
import 'package:uuid/uuid.dart';

import 'common/core.dart';
import 'common/file_filters.dart';
import 'common/flutter_command_utils.dart';
import 'common/gradle.dart';
import 'common/output_utils.dart';
import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/repository_package.dart';

const int _exitGcloudAuthFailed = 3;

/// A command to run tests via Firebase test lab.
class FirebaseTestLabCommand extends PackageLoopingCommand {
  /// Creates an instance of the test runner command.
  FirebaseTestLabCommand(super.packagesDir, {super.processRunner, super.platform, super.gitDir}) {
    argParser.addOption(_gCloudProjectArg, help: 'The Firebase project name.');
    argParser.addOption(
      _gCloudServiceKeyArg,
      help:
          'The path to the service key for gcloud authentication.\n'
          'If not provided, setup will be skipped, so testing will fail '
          'unless gcloud is already configured.',
    );
    argParser.addOption(
      'test-run-id',
      defaultsTo: const Uuid().v4(),
      help:
          'Optional string to append to the results path, to avoid conflicts. '
          'Randomly chosen on each invocation if none is provided. '
          'The default shown here is just an example.',
    );
    argParser.addOption(
      'build-id',
      defaultsTo: io.Platform.environment['CIRRUS_BUILD_ID'] ?? 'unknown_build',
      help:
          'Optional string to append to the results path, to avoid conflicts. '
          r'Defaults to $CIRRUS_BUILD_ID if that is set.',
    );
    argParser.addMultiOption(
      'device',
      splitCommas: false,
      defaultsTo: <String>['model=walleye,version=26', 'model=redfin,version=30'],
      help:
          'Device model(s) to test. See https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run for more info',
    );
    argParser.addOption(_gCloudResultsBucketArg, mandatory: true);
    argParser.addOption(
      kEnableExperiment,
      defaultsTo: '',
      help: 'Enables the given Dart SDK experiments.',
    );
  }

  static const String _gCloudServiceKeyArg = 'service-key';
  static const String _gCloudProjectArg = 'project';
  static const String _gCloudResultsBucketArg = 'results-bucket';

  @override
  final String name = 'firebase-test-lab';

  @override
  final String description =
      'Runs the instrumentation tests of the example '
      'apps on Firebase Test Lab.\n\n'
      'Runs tests in test_instrumentation folder using the '
      'instrumentation_test package.';

  bool _firebaseProjectConfigured = false;

  Future<void> _configureFirebaseProject() async {
    if (_firebaseProjectConfigured) {
      return;
    }

    final String serviceKey = getStringArg(_gCloudServiceKeyArg);
    if (serviceKey.isEmpty) {
      print('No --$_gCloudServiceKeyArg provided; skipping gcloud authorization');
    } else {
      final io.ProcessResult result = await processRunner.run('gcloud', <String>[
        'auth',
        'activate-service-account',
        '--key-file=$serviceKey',
      ], logOnError: true);
      if (result.exitCode != 0) {
        printError('Unable to activate gcloud account.');
        throw ToolExit(_exitGcloudAuthFailed);
      }
    }
    final String project = getStringArg(_gCloudProjectArg);
    if (project.isEmpty) {
      print('No --$_gCloudProjectArg provided; skipping gcloud config');
    } else {
      final int exitCode = await processRunner.runAndStream('gcloud', <String>[
        'config',
        'set',
        'project',
        project,
      ]);
      print('');
      if (exitCode == 0) {
        print('Firebase project configured.');
      } else {
        logWarning('Warning: gcloud config set returned a non-zero exit code. Continuing anyway.');
      }
    }
    _firebaseProjectConfigured = true;
  }

  @override
  bool shouldIgnoreFile(String path) {
    return isRepoLevelNonCodeImpactingFile(path) || isPackageSupportFile(path);
  }

  @override
  Future<PackageResult> runForPackage(RepositoryPackage package) async {
    final results = <PackageResult>[];
    for (final RepositoryPackage example in package.getExamples()) {
      results.add(await _runForExample(example, package: package));
    }

    // If all results skipped, report skip overall.
    if (results.every((PackageResult result) => result.state == RunState.skipped)) {
      return PackageResult.skip('No examples support Android.');
    }
    // Otherwise, report failure if there were any failures.
    final List<String> allErrors = results
        .map(
          (PackageResult result) => result.state == RunState.failed ? result.details : <String>[],
        )
        .expand((List<String> list) => list)
        .toList();
    return allErrors.isEmpty ? PackageResult.success() : PackageResult.fail(allErrors);
  }

  /// Runs the test for the given example of [package].
  Future<PackageResult> _runForExample(
    RepositoryPackage example, {
    required RepositoryPackage package,
  }) async {
    final Directory androidDirectory = example.platformDirectory(FlutterPlatform.android);
    if (!androidDirectory.existsSync()) {
      return PackageResult.skip('${example.displayName} does not support Android.');
    }

    final Directory uiTestDirectory = androidDirectory
        .childDirectory('app')
        .childDirectory('src')
        .childDirectory('androidTest');
    if (!uiTestDirectory.existsSync()) {
      printError('No androidTest directory found.');
      if (isFlutterPlugin(package)) {
        return PackageResult.fail(<String>['No tests ran (use --exclude if this is intentional).']);
      } else {
        return PackageResult.skip('${example.displayName} has no native Android tests.');
      }
    }

    // Ensure that the Dart integration tests will be run, not just native UI
    // tests.
    if (!await _testsContainDartIntegrationTestRunner(uiTestDirectory)) {
      printError(
        'No integration_test runner found. '
        'See the integration_test package README for setup instructions.',
      );
      return PackageResult.fail(<String>['No integration_test runner.']);
    }

    // Ensures that gradle wrapper exists
    final project = GradleProject(example, processRunner: processRunner, platform: platform);
    if (!await _ensureGradleWrapperExists(example, project)) {
      return PackageResult.fail(<String>['Unable to build example apk']);
    }

    await _configureFirebaseProject();

    if (!await _runGradle(project, 'app:assembleAndroidTest')) {
      return PackageResult.fail(<String>['Unable to assemble androidTest']);
    }

    final errors = <String>[];

    // Used within the loop to ensure a unique GCS output location for each
    // test file's run.
    var resultsCounter = 0;
    for (final File test in _findIntegrationTestFiles(example)) {
      final String testName = getRelativePosixPath(test, from: package.directory);
      print('Testing $testName...');
      if (!await _runGradle(project, 'app:assembleDebug', testFile: test)) {
        printError('Could not build $testName');
        errors.add('$testName failed to build');
        continue;
      }
      final String buildId = getStringArg('build-id');
      final String testRunId = getStringArg('test-run-id');
      final resultsDir =
          'plugins_android_test/${package.displayName}/$buildId/$testRunId/'
          '${example.directory.basename}/${resultsCounter++}/';

      // Automatically retry failures; there is significant flake with these
      // tests whose cause isn't yet understood, and having to re-run the
      // entire shard for a flake in any one test is extremely slow. This should
      // be removed once the root cause of the flake is understood.
      // See https://github.com/flutter/flutter/issues/95063
      const maxRetries = 2;
      var passing = false;
      for (var i = 1; i <= maxRetries && !passing; ++i) {
        if (i > 1) {
          logWarning('$testName failed on attempt ${i - 1}. Retrying...');
        }
        passing = await _runFirebaseTest(example, test, resultsDir: resultsDir);
      }
      if (!passing) {
        printError('Test failure for $testName after $maxRetries attempts');
        errors.add('$testName failed tests');
      }
    }

    if (errors.isEmpty && resultsCounter == 0) {
      printError('No integration tests were run.');
      errors.add('No tests ran (use --exclude if this is intentional).');
    }

    return errors.isEmpty ? PackageResult.success() : PackageResult.fail(errors);
  }

  /// Checks that Gradle has been configured for [project], and if not runs a
  /// Flutter build to generate it.
  ///
  /// Returns true if either gradlew was already present, or the build succeeds.
  Future<bool> _ensureGradleWrapperExists(RepositoryPackage package, GradleProject project) async {
    // Unconditionally re-run build with --debug --config-only, to ensure that
    // the project is in a debug state even if it was previously configured.
    print('Running flutter build apk...');
    final String experiment = getStringArg(kEnableExperiment);
    return runConfigOnlyBuild(
      package,
      processRunner,
      platform,
      FlutterPlatform.android,
      buildDebug: true,
      extraArgs: <String>[if (experiment.isNotEmpty) '--enable-experiment=$experiment'],
    );
  }

  /// Runs [test] from [example] as a Firebase Test Lab test, returning true if
  /// the test passed.
  ///
  /// [resultsDir] should be a unique-to-the-test-run directory to store the
  /// results on the server.
  Future<bool> _runFirebaseTest(
    RepositoryPackage example,
    File test, {
    required String resultsDir,
  }) async {
    final args = <String>[
      'firebase',
      'test',
      'android',
      'run',
      '--type',
      'instrumentation',
      '--app',
      'build/app/outputs/apk/debug/app-debug.apk',
      '--test',
      'build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk',
      '--timeout',
      '7m',
      '--results-bucket=gs://${getStringArg(_gCloudResultsBucketArg)}',
      '--results-dir=$resultsDir',
      for (final String device in getStringListArg('device')) ...<String>['--device', device],
    ];
    final int exitCode = await processRunner.runAndStream(
      'gcloud',
      args,
      workingDir: example.directory,
    );

    return exitCode == 0;
  }

  /// Builds [target] using Gradle in the given [project]. Assumes Gradle is
  /// already configured.
  ///
  /// [testFile] optionally does the Flutter build with the given test file as
  /// the build target.
  ///
  /// Returns true if the command succeeds.
  Future<bool> _runGradle(GradleProject project, String target, {File? testFile}) async {
    final String experiment = getStringArg(kEnableExperiment);
    final String? extraOptions = experiment.isNotEmpty
        ? Uri.encodeComponent('--enable-experiment=$experiment')
        : null;

    final int exitCode = await project.runCommand(
      target,
      arguments: <String>[
        '-Pverbose=true',
        if (testFile != null) '-Ptarget=${testFile.path}',
        if (extraOptions != null) '-Pextra-front-end-options=$extraOptions',
        if (extraOptions != null) '-Pextra-gen-snapshot-options=$extraOptions',
      ],
    );

    if (exitCode != 0) {
      return false;
    }
    return true;
  }

  /// Finds and returns all integration test files for [example].
  Iterable<File> _findIntegrationTestFiles(RepositoryPackage example) sync* {
    final Directory integrationTestDir = example.directory.childDirectory('integration_test');

    if (!integrationTestDir.existsSync()) {
      return;
    }

    yield* integrationTestDir
        .listSync(recursive: true)
        .where((FileSystemEntity file) => file is File && file.basename.endsWith('_test.dart'))
        .cast<File>();
  }

  /// Returns true if any of the test files in [uiTestDirectory] contain the
  /// annotation that means that the test will reports the results of running
  /// the Dart integration tests.
  Future<bool> _testsContainDartIntegrationTestRunner(Directory uiTestDirectory) async {
    return uiTestDirectory
        .list(recursive: true, followLinks: false)
        .where((FileSystemEntity entity) => entity is File)
        .cast<File>()
        .any((File file) {
          if (file.basename.endsWith('.java')) {
            return file.readAsStringSync().contains('@RunWith(FlutterTestRunner.class)');
          } else if (file.basename.endsWith('.kt')) {
            return file.readAsStringSync().contains('@RunWith(FlutterTestRunner::class)');
          }
          return false;
        });
  }
}