script/tool/lib/src/native_test_command.dart

// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:file/file.dart';
import 'package:platform/platform.dart';

import 'common/cmake.dart';
import 'common/core.dart';
import 'common/gradle.dart';
import 'common/package_looping_command.dart';
import 'common/plugin_utils.dart';
import 'common/process_runner.dart';
import 'common/repository_package.dart';
import 'common/xcode.dart';

const String _unitTestFlag = 'unit';
const String _integrationTestFlag = 'integration';

const String _iOSDestinationFlag = 'ios-destination';

const int _exitNoIOSSimulators = 3;

/// The command to run native tests for plugins:
/// - iOS and macOS: XCTests (XCUnitTest and XCUITest)
/// - Android: JUnit tests
/// - Windows and Linux: GoogleTest tests
class NativeTestCommand extends PackageLoopingCommand {
  /// Creates an instance of the test command.
  NativeTestCommand(
    Directory packagesDir, {
    ProcessRunner processRunner = const ProcessRunner(),
    Platform platform = const LocalPlatform(),
  })  : _xcode = Xcode(processRunner: processRunner, log: true),
        super(packagesDir, processRunner: processRunner, platform: platform) {
    argParser.addOption(
      _iOSDestinationFlag,
      help: 'Specify the destination when running iOS tests.\n'
          'This is passed to the `-destination` argument in the xcodebuild command.\n'
          'See https://developer.apple.com/library/archive/technotes/tn2339/_index.html#//apple_ref/doc/uid/DTS40014588-CH1-UNIT '
          'for details on how to specify the destination.',
    );
    argParser.addFlag(platformAndroid, help: 'Runs Android tests');
    argParser.addFlag(platformIOS, help: 'Runs iOS tests');
    argParser.addFlag(platformLinux, help: 'Runs Linux tests');
    argParser.addFlag(platformMacOS, help: 'Runs macOS tests');
    argParser.addFlag(platformWindows, help: 'Runs Windows tests');

    // By default, both unit tests and integration tests are run, but provide
    // flags to disable one or the other.
    argParser.addFlag(_unitTestFlag,
        help: 'Runs native unit tests', defaultsTo: true);
    argParser.addFlag(_integrationTestFlag,
        help: 'Runs native integration (UI) tests', defaultsTo: true);
  }

  // The device destination flags for iOS tests.
  List<String> _iOSDestinationFlags = <String>[];

  final Xcode _xcode;

  @override
  final String name = 'native-test';

  @override
  final String description = '''
Runs native unit tests and native integration tests.

Currently supported platforms:
- Android
- iOS: requires 'xcrun' to be in your path.
- Linux (unit tests only)
- macOS: requires 'xcrun' to be in your path.
- Windows (unit tests only)

The example app(s) must be built for all targeted platforms before running
this command.
''';

  Map<String, _PlatformDetails> _platforms = <String, _PlatformDetails>{};

  List<String> _requestedPlatforms = <String>[];

  @override
  Future<void> initializeRun() async {
    _platforms = <String, _PlatformDetails>{
      platformAndroid: _PlatformDetails('Android', _testAndroid),
      platformIOS: _PlatformDetails('iOS', _testIOS),
      platformLinux: _PlatformDetails('Linux', _testLinux),
      platformMacOS: _PlatformDetails('macOS', _testMacOS),
      platformWindows: _PlatformDetails('Windows', _testWindows),
    };
    _requestedPlatforms = _platforms.keys
        .where((String platform) => getBoolArg(platform))
        .toList();
    _requestedPlatforms.sort();

    if (_requestedPlatforms.isEmpty) {
      printError('At least one platform flag must be provided.');
      throw ToolExit(exitInvalidArguments);
    }

    if (!(getBoolArg(_unitTestFlag) || getBoolArg(_integrationTestFlag))) {
      printError('At least one test type must be enabled.');
      throw ToolExit(exitInvalidArguments);
    }

    if (getBoolArg(platformWindows) && getBoolArg(_integrationTestFlag)) {
      logWarning('This command currently only supports unit tests for Windows. '
          'See https://github.com/flutter/flutter/issues/70233.');
    }

    if (getBoolArg(platformLinux) && getBoolArg(_integrationTestFlag)) {
      logWarning('This command currently only supports unit tests for Linux. '
          'See https://github.com/flutter/flutter/issues/70235.');
    }

    // iOS-specific run-level state.
    if (_requestedPlatforms.contains('ios')) {
      String destination = getStringArg(_iOSDestinationFlag);
      if (destination.isEmpty) {
        final String? simulatorId =
            await _xcode.findBestAvailableIphoneSimulator();
        if (simulatorId == null) {
          printError('Cannot find any available iOS simulators.');
          throw ToolExit(_exitNoIOSSimulators);
        }
        destination = 'id=$simulatorId';
      }
      _iOSDestinationFlags = <String>[
        '-destination',
        destination,
      ];
    }
  }

  @override
  Future<PackageResult> runForPackage(RepositoryPackage package) async {
    final List<String> testPlatforms = <String>[];
    for (final String platform in _requestedPlatforms) {
      if (!pluginSupportsPlatform(platform, package,
          requiredMode: PlatformSupport.inline)) {
        print('No implementation for ${_platforms[platform]!.label}.');
        continue;
      }
      if (!pluginHasNativeCodeForPlatform(platform, package)) {
        print('No native code for ${_platforms[platform]!.label}.');
        continue;
      }
      testPlatforms.add(platform);
    }

    if (testPlatforms.isEmpty) {
      return PackageResult.skip('Nothing to test for target platform(s).');
    }

    final _TestMode mode = _TestMode(
      unit: getBoolArg(_unitTestFlag),
      integration: getBoolArg(_integrationTestFlag),
    );

    bool ranTests = false;
    bool failed = false;
    final List<String> failureMessages = <String>[];
    for (final String platform in testPlatforms) {
      final _PlatformDetails platformInfo = _platforms[platform]!;
      print('Running tests for ${platformInfo.label}...');
      print('----------------------------------------');
      final _PlatformResult result =
          await platformInfo.testFunction(package, mode);
      ranTests |= result.state != RunState.skipped;
      if (result.state == RunState.failed) {
        failed = true;

        final String? error = result.error;
        // Only provide the failing platforms in the failure details if testing
        // multiple platforms, otherwise it's just noise.
        if (_requestedPlatforms.length > 1) {
          failureMessages.add(error != null
              ? '${platformInfo.label}: $error'
              : platformInfo.label);
        } else if (error != null) {
          // If there's only one platform, only provide error details in the
          // summary if the platform returned a message.
          failureMessages.add(error);
        }
      }
    }

    if (!ranTests) {
      return PackageResult.skip('No tests found.');
    }
    return failed
        ? PackageResult.fail(failureMessages)
        : PackageResult.success();
  }

  Future<_PlatformResult> _testAndroid(
      RepositoryPackage plugin, _TestMode mode) async {
    bool exampleHasUnitTests(RepositoryPackage example) {
      return example
              .platformDirectory(FlutterPlatform.android)
              .childDirectory('app')
              .childDirectory('src')
              .childDirectory('test')
              .existsSync() ||
          plugin
              .platformDirectory(FlutterPlatform.android)
              .childDirectory('src')
              .childDirectory('test')
              .existsSync();
    }

    bool exampleHasNativeIntegrationTests(RepositoryPackage example) {
      final Directory integrationTestDirectory = example
          .platformDirectory(FlutterPlatform.android)
          .childDirectory('app')
          .childDirectory('src')
          .childDirectory('androidTest');
      // There are two types of integration tests that can be in the androidTest
      // directory:
      // - FlutterTestRunner.class tests, which bridge to Dart integration tests
      // - Purely native tests
      // Only the latter is supported by this command; the former will hang if
      // run here because they will wait for a Dart call that will never come.
      //
      // This repository uses a convention of putting the former in a
      // *ActivityTest.java file, so ignore that file when checking for tests.
      // Also ignore DartIntegrationTest.java, which defines the annotation used
      // below for filtering the former out when running tests.
      //
      // If those are the only files, then there are no tests to run here.
      return integrationTestDirectory.existsSync() &&
          integrationTestDirectory
              .listSync(recursive: true)
              .whereType<File>()
              .any((File file) {
            final String basename = file.basename;
            return !basename.endsWith('ActivityTest.java') &&
                basename != 'DartIntegrationTest.java';
          });
    }

    final Iterable<RepositoryPackage> examples = plugin.getExamples();

    bool ranUnitTests = false;
    bool ranAnyTests = false;
    bool failed = false;
    bool hasMissingBuild = false;
    for (final RepositoryPackage example in examples) {
      final bool hasUnitTests = exampleHasUnitTests(example);
      final bool hasIntegrationTests =
          exampleHasNativeIntegrationTests(example);

      if (mode.unit && !hasUnitTests) {
        _printNoExampleTestsMessage(example, 'Android unit');
      }
      if (mode.integration && !hasIntegrationTests) {
        _printNoExampleTestsMessage(example, 'Android integration');
      }

      final bool runUnitTests = mode.unit && hasUnitTests;
      final bool runIntegrationTests = mode.integration && hasIntegrationTests;
      if (!runUnitTests && !runIntegrationTests) {
        continue;
      }

      final String exampleName = example.displayName;
      _printRunningExampleTestsMessage(example, 'Android');

      final GradleProject project = GradleProject(
        example,
        processRunner: processRunner,
        platform: platform,
      );
      if (!project.isConfigured()) {
        printError('ERROR: Run "flutter build apk" on $exampleName, or run '
            'this tool\'s "build-examples --apk" command, '
            'before executing tests.');
        failed = true;
        hasMissingBuild = true;
        continue;
      }

      if (runUnitTests) {
        print('Running unit tests...');
        final int exitCode = await project.runCommand('testDebugUnitTest');
        if (exitCode != 0) {
          printError('$exampleName unit tests failed.');
          failed = true;
        }
        ranUnitTests = true;
        ranAnyTests = true;
      }

      if (runIntegrationTests) {
        // FlutterTestRunner-based tests will hang forever if run in a normal
        // app build, since they wait for a Dart call from integration_test that
        // will never come. Those tests have an extra annotation to allow
        // filtering them out.
        const String filter =
            'notAnnotation=io.flutter.plugins.DartIntegrationTest';

        print('Running integration tests...');
        final int exitCode = await project.runCommand(
          'app:connectedAndroidTest',
          arguments: <String>[
            '-Pandroid.testInstrumentationRunnerArguments.$filter',
          ],
        );
        if (exitCode != 0) {
          printError('$exampleName integration tests failed.');
          failed = true;
        }
        ranAnyTests = true;
      }
    }

    if (failed) {
      return _PlatformResult(RunState.failed,
          error: hasMissingBuild
              ? 'Examples must be built before testing.'
              : null);
    }
    if (!mode.integrationOnly && !ranUnitTests) {
      printError('No unit tests ran. Plugins are required to have unit tests.');
      return _PlatformResult(RunState.failed,
          error: 'No unit tests ran (use --exclude if this is intentional).');
    }
    if (!ranAnyTests) {
      return _PlatformResult(RunState.skipped);
    }
    return _PlatformResult(RunState.succeeded);
  }

  Future<_PlatformResult> _testIOS(RepositoryPackage plugin, _TestMode mode) {
    return _runXcodeTests(plugin, 'iOS', mode,
        extraFlags: _iOSDestinationFlags);
  }

  Future<_PlatformResult> _testMacOS(RepositoryPackage plugin, _TestMode mode) {
    return _runXcodeTests(plugin, 'macOS', mode);
  }

  /// Runs all applicable tests for [plugin], printing status and returning
  /// the test result.
  ///
  /// The tests targets must be added to the Xcode project of the example app,
  /// usually at "example/{ios,macos}/Runner.xcworkspace".
  Future<_PlatformResult> _runXcodeTests(
    RepositoryPackage plugin,
    String platform,
    _TestMode mode, {
    List<String> extraFlags = const <String>[],
  }) async {
    String? testTarget;
    const String unitTestTarget = 'RunnerTests';
    if (mode.unitOnly) {
      testTarget = unitTestTarget;
    } else if (mode.integrationOnly) {
      testTarget = 'RunnerUITests';
    }

    bool ranUnitTests = false;
    // Assume skipped until at least one test has run.
    RunState overallResult = RunState.skipped;
    for (final RepositoryPackage example in plugin.getExamples()) {
      final String exampleName = example.displayName;

      // If running a specific target, check that. Otherwise, check if there
      // are unit tests, since having no unit tests for a plugin is fatal
      // (by repo policy) even if there are integration tests.
      bool exampleHasUnitTests = false;
      final String? targetToCheck =
          testTarget ?? (mode.unit ? unitTestTarget : null);
      final Directory xcodeProject = example.directory
          .childDirectory(platform.toLowerCase())
          .childDirectory('Runner.xcodeproj');
      if (targetToCheck != null) {
        final bool? hasTarget =
            await _xcode.projectHasTarget(xcodeProject, targetToCheck);
        if (hasTarget == null) {
          printError('Unable to check targets for $exampleName.');
          overallResult = RunState.failed;
          continue;
        } else if (!hasTarget) {
          print('No "$targetToCheck" target in $exampleName; skipping.');
          continue;
        } else if (targetToCheck == unitTestTarget) {
          exampleHasUnitTests = true;
        }
      }

      _printRunningExampleTestsMessage(example, platform);
      final int exitCode = await _xcode.runXcodeBuild(
        example.directory,
        actions: <String>['test'],
        workspace: '${platform.toLowerCase()}/Runner.xcworkspace',
        scheme: 'Runner',
        configuration: 'Debug',
        extraFlags: <String>[
          if (testTarget != null) '-only-testing:$testTarget',
          ...extraFlags,
          'GCC_TREAT_WARNINGS_AS_ERRORS=YES',
        ],
      );

      // The exit code from 'xcodebuild test' when there are no tests.
      const int xcodebuildNoTestExitCode = 66;
      switch (exitCode) {
        case xcodebuildNoTestExitCode:
          _printNoExampleTestsMessage(example, platform);
          break;
        case 0:
          printSuccess('Successfully ran $platform xctest for $exampleName');
          // If this is the first test, assume success until something fails.
          if (overallResult == RunState.skipped) {
            overallResult = RunState.succeeded;
          }
          if (exampleHasUnitTests) {
            ranUnitTests = true;
          }
          break;
        default:
          // Any failure means a failure overall.
          overallResult = RunState.failed;
          // If unit tests ran, note that even if they failed.
          if (exampleHasUnitTests) {
            ranUnitTests = true;
          }
          break;
      }
    }

    if (!mode.integrationOnly && !ranUnitTests) {
      printError('No unit tests ran. Plugins are required to have unit tests.');
      // Only return a specific summary error message about the missing unit
      // tests if there weren't also failures, to avoid having a misleadingly
      // specific message.
      if (overallResult != RunState.failed) {
        return _PlatformResult(RunState.failed,
            error: 'No unit tests ran (use --exclude if this is intentional).');
      }
    }

    return _PlatformResult(overallResult);
  }

  Future<_PlatformResult> _testWindows(
      RepositoryPackage plugin, _TestMode mode) async {
    if (mode.integrationOnly) {
      return _PlatformResult(RunState.skipped);
    }

    bool isTestBinary(File file) {
      return file.basename.endsWith('_test.exe') ||
          file.basename.endsWith('_tests.exe');
    }

    return _runGoogleTestTests(plugin, 'Windows', 'Debug',
        isTestBinary: isTestBinary);
  }

  Future<_PlatformResult> _testLinux(
      RepositoryPackage plugin, _TestMode mode) async {
    if (mode.integrationOnly) {
      return _PlatformResult(RunState.skipped);
    }

    bool isTestBinary(File file) {
      return file.basename.endsWith('_test') ||
          file.basename.endsWith('_tests');
    }

    // Since Linux uses a single-config generator, building-examples only
    // generates the build files for release, so the tests have to be run in
    // release mode as well.
    //
    // TODO(stuartmorgan): Consider adding a command to `flutter` that would
    // generate build files without doing a build, and using that instead of
    // relying on running build-examples. See
    // https://github.com/flutter/flutter/issues/93407.
    return _runGoogleTestTests(plugin, 'Linux', 'Release',
        isTestBinary: isTestBinary);
  }

  /// Finds every file in the [buildDirectoryName] subdirectory of [plugin]'s
  /// build directory for which [isTestBinary] is true, and runs all of them,
  /// returning the overall result.
  ///
  /// The binaries are assumed to be Google Test test binaries, thus returning
  /// zero for success and non-zero for failure.
  Future<_PlatformResult> _runGoogleTestTests(
    RepositoryPackage plugin,
    String platformName,
    String buildMode, {
    required bool Function(File) isTestBinary,
  }) async {
    final List<File> testBinaries = <File>[];
    bool hasMissingBuild = false;
    bool buildFailed = false;
    for (final RepositoryPackage example in plugin.getExamples()) {
      final CMakeProject project = CMakeProject(example.directory,
          buildMode: buildMode,
          processRunner: processRunner,
          platform: platform);
      if (!project.isConfigured()) {
        printError('ERROR: Run "flutter build" on ${example.displayName}, '
            'or run this tool\'s "build-examples" command, for the target '
            'platform before executing tests.');
        hasMissingBuild = true;
        continue;
      }

      // By repository convention, example projects create an aggregate target
      // called 'unit_tests' that builds all unit tests (usually just an alias
      // for a specific test target).
      final int exitCode = await project.runBuild('unit_tests');
      if (exitCode != 0) {
        printError('${example.displayName} unit tests failed to build.');
        buildFailed = true;
      }

      testBinaries.addAll(project.buildDirectory
          .listSync(recursive: true)
          .whereType<File>()
          .where(isTestBinary)
          .where((File file) {
        // Only run the `buildMode` build of the unit tests, to avoid running
        // the same tests multiple times.
        final List<String> components = path.split(file.path);
        return components.contains(buildMode) ||
            components.contains(buildMode.toLowerCase());
      }));
    }

    if (hasMissingBuild) {
      return _PlatformResult(RunState.failed,
          error: 'Examples must be built before testing.');
    }

    if (buildFailed) {
      return _PlatformResult(RunState.failed,
          error: 'Failed to build $platformName unit tests.');
    }

    if (testBinaries.isEmpty) {
      final String binaryExtension = platform.isWindows ? '.exe' : '';
      printError(
          'No test binaries found. At least one *_test(s)$binaryExtension '
          'binary should be built by the example(s)');
      return _PlatformResult(RunState.failed,
          error: 'No $platformName unit tests found');
    }

    bool passing = true;
    for (final File test in testBinaries) {
      print('Running ${test.basename}...');
      final int exitCode =
          await processRunner.runAndStream(test.path, <String>[]);
      passing &= exitCode == 0;
    }
    return _PlatformResult(passing ? RunState.succeeded : RunState.failed);
  }

  /// Prints a standard format message indicating that [platform] tests for
  /// [plugin]'s [example] are about to be run.
  void _printRunningExampleTestsMessage(
      RepositoryPackage example, String platform) {
    print('Running $platform tests for ${example.displayName}...');
  }

  /// Prints a standard format message indicating that no tests were found for
  /// [plugin]'s [example] for [platform].
  void _printNoExampleTestsMessage(RepositoryPackage example, String platform) {
    print('No $platform tests found for ${example.displayName}');
  }
}

// The type for a function that takes a plugin directory and runs its native
// tests for a specific platform.
typedef _TestFunction = Future<_PlatformResult> Function(
    RepositoryPackage, _TestMode);

/// A collection of information related to a specific platform.
class _PlatformDetails {
  const _PlatformDetails(
    this.label,
    this.testFunction,
  );

  /// The name to use in output.
  final String label;

  /// The function to call to run tests.
  final _TestFunction testFunction;
}

/// Enabled state for different test types.
class _TestMode {
  const _TestMode({required this.unit, required this.integration});

  final bool unit;
  final bool integration;

  bool get integrationOnly => integration && !unit;
  bool get unitOnly => unit && !integration;
}

/// The result of running a single platform's tests.
class _PlatformResult {
  _PlatformResult(this.state, {this.error});

  /// The overall state of the platform's tests. This should be:
  /// - failed if any tests failed.
  /// - succeeded if at least one test ran, and all tests passed.
  /// - skipped if no tests ran.
  final RunState state;

  /// An optional error string to include in the summary for this platform.
  ///
  /// Ignored unless [state] is `failed`.
  final String? error;
}