#!/usr/bin/env python3
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
A top level harness to run all unit-tests in a specific engine build.
"""

from pathlib import Path

import argparse
import errno
from functools import reduce
import glob
import logging
import logging.handlers
import multiprocessing
import os
import re
import shutil
import subprocess
# Explicitly import the parts of sys that are needed. This is to avoid using
# sys.stdout and sys.stderr directly. Instead, only the logger defined below
# should be used for output.
from sys import exit as sys_exit, platform as sys_platform
import tempfile
import time
import typing
import xvfb

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
BUILDROOT_DIR = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..'))
OUT_DIR = os.path.join(BUILDROOT_DIR, 'out')
GOLDEN_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'resources')
FONTS_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
ROBOTO_FONT_PATH = os.path.join(FONTS_DIR, 'Roboto-Regular.ttf')
FONT_SUBSET_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'font_subset')

ENCODING = 'UTF-8'

logger = logging.getLogger(__name__)
logger_handler = logging.StreamHandler()


# Override print so that it uses the logger instead of stdout directly.
def print(*args, **kwargs):  # pylint: disable=redefined-builtin
  logger.info(*args, **kwargs)


def print_divider(char='='):
  logger.info('\n')
  for _ in range(4):
    logger.info(''.join([char for _ in range(80)]))
  logger.info('\n')


def is_asan(build_dir):
  with open(os.path.join(build_dir, 'args.gn')) as args:
    if 'is_asan = true' in args.read():
      return True

  return False


def run_cmd(
    cmd: typing.List[str],
    forbidden_output: typing.List[str] = None,
    expect_failure: bool = False,
    env: typing.Dict[str, str] = None,
    allowed_failure_output: typing.List[str] = None,
    **kwargs
) -> None:
  if forbidden_output is None:
    forbidden_output = []
  if allowed_failure_output is None:
    allowed_failure_output = []

  command_string = ' '.join(cmd)

  print_divider('>')
  logger.info('Running command "%s"', command_string)

  start_time = time.time()

  process = subprocess.Popen(
      cmd,
      stdout=subprocess.PIPE,
      stderr=subprocess.STDOUT,
      env=env,
      universal_newlines=True,
      **kwargs
  )
  output = ''

  for line in iter(process.stdout.readline, ''):
    output += line
    logger.info(line.rstrip())

  process.wait()
  end_time = time.time()

  if process.returncode != 0 and not expect_failure:
    print_divider('!')

    logger.error(
        'Failed Command:\n\n%s\n\nExit Code: %s\n\nOutput:\n%s', command_string, process.returncode,
        output
    )

    print_divider('!')

    allowed_failure = False
    for allowed_string in allowed_failure_output:
      if allowed_string in output:
        allowed_failure = True

    if not allowed_failure:
      raise RuntimeError('Command "%s" exited with code %s.' % (command_string, process.returncode))

  for forbidden_string in forbidden_output:
    if forbidden_string in output:
      raise RuntimeError(
          'command "%s" contained forbidden string "%s"' % (command_string, forbidden_string)
      )

  print_divider('<')
  logger.info('Command run successfully in %.2f seconds: %s', end_time - start_time, command_string)


def is_mac():
  return sys_platform == 'darwin'


def is_aarm64():
  assert is_mac()
  output = subprocess.check_output(['sysctl', 'machdep.cpu'])
  text = output.decode('utf-8')
  aarm64 = text.find('Apple') >= 0
  if not aarm64:
    assert text.find('GenuineIntel') >= 0
  return aarm64


def is_linux():
  return sys_platform.startswith('linux')


def is_windows():
  return sys_platform.startswith(('cygwin', 'win'))


def executable_suffix():
  return '.exe' if is_windows() else ''


def find_executable_path(path):
  if os.path.exists(path):
    return path

  if is_windows():
    exe_path = path + '.exe'
    if os.path.exists(exe_path):
      return exe_path

    bat_path = path + '.bat'
    if os.path.exists(bat_path):
      return bat_path

  raise Exception('Executable %s does not exist!' % path)


def build_engine_executable_command(
    build_dir, executable_name, flags=None, coverage=False, gtest=False
):
  if flags is None:
    flags = []

  unstripped_exe = os.path.join(build_dir, 'exe.unstripped', executable_name)
  # We cannot run the unstripped binaries directly when coverage is enabled.
  if is_linux() and os.path.exists(unstripped_exe) and not coverage:
    # Use unstripped executables in order to get better symbolized crash
    # stack traces on Linux.
    executable = unstripped_exe
  else:
    executable = find_executable_path(os.path.join(build_dir, executable_name))

  coverage_script = os.path.join(BUILDROOT_DIR, 'flutter', 'build', 'generate_coverage.py')

  if coverage:
    coverage_flags = [
        '-t', executable, '-o',
        os.path.join(build_dir, 'coverage', executable_name), '-f', 'html'
    ]
    updated_flags = ['--args=%s' % ' '.join(flags)]
    test_command = [coverage_script] + coverage_flags + updated_flags
  else:
    test_command = [executable] + flags
    if gtest:
      gtest_parallel = os.path.join(
          BUILDROOT_DIR, 'flutter', 'third_party', 'gtest-parallel', 'gtest-parallel'
      )
      test_command = ['python3', gtest_parallel] + test_command

  return test_command


def run_engine_executable( # pylint: disable=too-many-arguments
    build_dir,
    executable_name,
    executable_filter,
    flags=None,
    cwd=BUILDROOT_DIR,
    forbidden_output=None,
    allowed_failure_output=None,
    expect_failure=False,
    coverage=False,
    extra_env=None,
    gtest=False,
):
  if executable_filter is not None and executable_name not in executable_filter:
    logger.info('Skipping %s due to filter.', executable_name)
    return

  if flags is None:
    flags = []
  if forbidden_output is None:
    forbidden_output = []
  if allowed_failure_output is None:
    allowed_failure_output = []
  if extra_env is None:
    extra_env = {}

  unstripped_exe = os.path.join(build_dir, 'exe.unstripped', executable_name)
  env = os.environ.copy()
  if is_linux():
    env['LD_LIBRARY_PATH'] = build_dir
    env['VK_DRIVER_FILES'] = os.path.join(build_dir, 'vk_swiftshader_icd.json')
    if os.path.exists(unstripped_exe):
      unstripped_vulkan = os.path.join(build_dir, 'lib.unstripped', 'libvulkan.so.1')
      if os.path.exists(unstripped_vulkan):
        vulkan_path = unstripped_vulkan
      else:
        vulkan_path = os.path.join(build_dir, 'libvulkan.so.1')
      try:
        os.symlink(vulkan_path, os.path.join(build_dir, 'exe.unstripped', 'libvulkan.so.1'))
      except OSError as err:
        if err.errno == errno.EEXIST:
          pass
        else:
          raise
  elif is_mac():
    env['DYLD_LIBRARY_PATH'] = build_dir
  else:
    env['PATH'] = build_dir + ':' + env['PATH']

  logger.info('Running %s in %s', executable_name, cwd)

  test_command = build_engine_executable_command(
      build_dir,
      executable_name,
      flags=flags,
      coverage=coverage,
      gtest=gtest,
  )

  env['FLUTTER_BUILD_DIRECTORY'] = build_dir
  for key, value in extra_env.items():
    env[key] = value

  try:
    run_cmd(
        test_command,
        cwd=cwd,
        forbidden_output=forbidden_output,
        expect_failure=expect_failure,
        env=env,
        allowed_failure_output=allowed_failure_output,
    )
  except:
    # The LUCI environment may provide a variable containing a directory path
    # for additional output files that will be uploaded to cloud storage.
    # If the command generated a core dump, then run a script to analyze
    # the dump and output a report that will be uploaded.
    luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
    core_path = os.path.join(cwd, 'core')
    if luci_test_outputs_path and os.path.exists(core_path) and os.path.exists(unstripped_exe):
      dump_path = os.path.join(
          luci_test_outputs_path, '%s_%s.txt' % (executable_name, sys_platform)
      )
      logger.error('Writing core dump analysis to %s', dump_path)
      subprocess.call([
          os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'analyze_core_dump.sh'),
          BUILDROOT_DIR,
          unstripped_exe,
          core_path,
          dump_path,
      ])
      os.unlink(core_path)
    raise


class EngineExecutableTask():  # pylint: disable=too-many-instance-attributes

  def __init__( # pylint: disable=too-many-arguments
      self,
      build_dir,
      executable_name,
      executable_filter,
      flags=None,
      cwd=BUILDROOT_DIR,
      forbidden_output=None,
      allowed_failure_output=None,
      expect_failure=False,
      coverage=False,
      extra_env=None,
  ):
    self.build_dir = build_dir
    self.executable_name = executable_name
    self.executable_filter = executable_filter
    self.flags = flags
    self.cwd = cwd
    self.forbidden_output = forbidden_output
    self.allowed_failure_output = allowed_failure_output
    self.expect_failure = expect_failure
    self.coverage = coverage
    self.extra_env = extra_env

  def __call__(self, *args):
    run_engine_executable(
        self.build_dir,
        self.executable_name,
        self.executable_filter,
        flags=self.flags,
        cwd=self.cwd,
        forbidden_output=self.forbidden_output,
        allowed_failure_output=self.allowed_failure_output,
        expect_failure=self.expect_failure,
        coverage=self.coverage,
        extra_env=self.extra_env,
    )

  def __str__(self):
    command = build_engine_executable_command(
        self.build_dir, self.executable_name, flags=self.flags, coverage=self.coverage
    )
    return ' '.join(command)


shuffle_flags = [
    '--gtest_repeat=2',
    '--gtest_shuffle',
]


def run_cc_tests(build_dir, executable_filter, coverage, capture_core_dump):
  logger.info('Running Engine Unit-tests.')

  if capture_core_dump and is_linux():
    import resource  # pylint: disable=import-outside-toplevel
    resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))

  repeat_flags = [
      '--repeat=2',
  ]

  def make_test(name, flags=None, extra_env=None):
    if flags is None:
      flags = repeat_flags
    if extra_env is None:
      extra_env = {}
    return (name, flags, extra_env)

  unittests = [
      make_test('client_wrapper_glfw_unittests'),
      make_test('client_wrapper_unittests'),
      make_test('common_cpp_core_unittests'),
      make_test('common_cpp_unittests'),
      make_test('dart_plugin_registrant_unittests'),
      make_test('display_list_rendertests'),
      make_test('display_list_unittests'),
      make_test('embedder_a11y_unittests'),
      make_test('embedder_proctable_unittests'),
      make_test('embedder_unittests'),
      make_test('fml_unittests'),
      make_test('fml_arc_unittests'),
      make_test('no_dart_plugin_registrant_unittests'),
      make_test('runtime_unittests'),
      make_test('testing_unittests'),
      make_test('tonic_unittests'),
      # The image release unit test can take a while on slow machines.
      make_test('ui_unittests', flags=repeat_flags + ['--timeout=90']),
  ]

  if not is_windows():
    unittests += [
        # https://github.com/google/googletest/issues/2490
        make_test('android_external_view_embedder_unittests'),
        make_test('jni_unittests'),
        make_test('platform_view_android_delegate_unittests'),
        # https://github.com/flutter/flutter/issues/36295
        make_test('shell_unittests'),
    ]

  if is_windows():
    unittests += [
        # The accessibility library only supports Mac and Windows.
        make_test('accessibility_unittests'),
        make_test('client_wrapper_windows_unittests'),
        make_test('flutter_windows_unittests'),
    ]

  # These unit-tests are Objective-C and can only run on Darwin.
  if is_mac():
    unittests += [
        # The accessibility library only supports Mac and Windows.
        make_test('accessibility_unittests'),
        make_test('availability_version_check_unittests'),
        make_test('framework_common_unittests'),
        make_test('spring_animation_unittests'),
        make_test('gpu_surface_metal_unittests'),
    ]

  if is_linux():
    flow_flags = [
        '--golden-dir=%s' % GOLDEN_DIR,
        '--font-file=%s' % ROBOTO_FONT_PATH,
    ]
    icu_flags = ['--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')]
    unittests += [
        make_test('flow_unittests', flags=repeat_flags + ['--'] + flow_flags),
        make_test('flutter_glfw_unittests'),
        make_test('flutter_linux_unittests', extra_env={'G_DEBUG': 'fatal-criticals'}),
        # https://github.com/flutter/flutter/issues/36296
        make_test('txt_unittests', flags=repeat_flags + ['--'] + icu_flags),
    ]
  else:
    flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
    unittests += [
        make_test('flow_unittests', flags=repeat_flags + flow_flags),
    ]

  build_name = os.path.basename(build_dir)
  try:
    if is_linux():
      xvfb.start_virtual_x(build_name, build_dir)
    for test, flags, extra_env in unittests:
      run_engine_executable(
          build_dir,
          test,
          executable_filter,
          flags,
          coverage=coverage,
          extra_env=extra_env,
          gtest=True
      )
  finally:
    if is_linux():
      xvfb.stop_virtual_x(build_name)

  if is_mac():
    # flutter_desktop_darwin_unittests uses global state that isn't handled
    # correctly by gtest-parallel.
    # https://github.com/flutter/flutter/issues/104789
    if not os.path.basename(build_dir).startswith('host_debug'):
      # Test is disabled for flaking in debug runs:
      # https://github.com/flutter/flutter/issues/127441
      run_engine_executable(
          build_dir,
          'flutter_desktop_darwin_unittests',
          executable_filter,
          shuffle_flags,
          coverage=coverage
      )
    extra_env = {
        # pylint: disable=line-too-long
        # See https://developer.apple.com/documentation/metal/diagnosing_metal_programming_issues_early?language=objc
        'MTL_SHADER_VALIDATION': '1',  # Enables all shader validation tests.
        'MTL_SHADER_VALIDATION_GLOBAL_MEMORY':
            '1',  # Validates accesses to device and constant memory.
        'MTL_SHADER_VALIDATION_THREADGROUP_MEMORY':
            '1',  # Validates accesses to threadgroup memory.
        'MTL_SHADER_VALIDATION_TEXTURE_USAGE':
            '1',  # Validates that texture references are not nil.
        # Note: built from //third_party/swiftshader
        'VK_ICD_FILENAMES': os.path.join(build_dir, 'vk_swiftshader_icd.json'),
        # Note: built from //third_party/vulkan_validation_layers:vulkan_gen_json_files
        # and //third_party/vulkan_validation_layers.
        'VK_LAYER_PATH': os.path.join(build_dir, 'vulkan-data'),
        'VK_INSTANCE_LAYERS': 'VK_LAYER_KHRONOS_validation',
    }
    if is_aarm64():
      extra_env.update({
          'METAL_DEBUG_ERROR_MODE': '0',  # Enables metal validation.
          'METAL_DEVICE_WRAPPER_TYPE': '1',  # Enables metal validation.
      })
    mac_impeller_unittests_flags = shuffle_flags + [
        '--enable_vulkan_validation',
        '--gtest_filter=-*OpenGLES'  # These are covered in the golden tests.
    ]
    # Impeller tests are only supported on macOS for now.
    run_engine_executable(
        build_dir,
        'impeller_unittests',
        executable_filter,
        mac_impeller_unittests_flags,
        coverage=coverage,
        extra_env=extra_env,
        # TODO(https://github.com/flutter/flutter/issues/123733): Remove this allowlist.
        # See also https://github.com/flutter/flutter/issues/114872.
        allowed_failure_output=[
            '[MTLCompiler createVertexStageAndLinkPipelineWithFragment:',
            '[MTLCompiler pipelineStateWithVariant:',
        ]
    )

    # Run one interactive Vulkan test with validation enabled.
    #
    # TODO(matanlurey): https://github.com/flutter/flutter/issues/134852; enable
    # more of the suite, and ideally we'd like to use Skia gold and take screen
    # shots as well.
    run_engine_executable(
        build_dir,
        'impeller_unittests',
        executable_filter,
        shuffle_flags + [
            '--enable_vulkan_validation',
            '--enable_playground',
            '--playground_timeout_ms=4000',
            '--gtest_filter="*ColorWheel/Vulkan"',
        ],
        coverage=coverage,
        extra_env=extra_env,
    )

    # Run the Flutter GPU test suite.
    run_engine_executable(
        build_dir,
        'impeller_dart_unittests',
        executable_filter,
        shuffle_flags + [
            '--enable_vulkan_validation',
            # TODO(https://github.com/flutter/flutter/issues/142642): Remove this.
            '--gtest_filter=-*OpenGLES',
        ],
        coverage=coverage,
        extra_env=extra_env,
    )


def run_engine_benchmarks(build_dir, executable_filter):
  logger.info('Running Engine Benchmarks.')

  icu_flags = ['--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')]

  run_engine_executable(build_dir, 'shell_benchmarks', executable_filter, icu_flags)

  run_engine_executable(build_dir, 'fml_benchmarks', executable_filter, icu_flags)

  run_engine_executable(build_dir, 'ui_benchmarks', executable_filter, icu_flags)

  run_engine_executable(build_dir, 'display_list_builder_benchmarks', executable_filter, icu_flags)

  run_engine_executable(build_dir, 'geometry_benchmarks', executable_filter, icu_flags)

  run_engine_executable(build_dir, 'canvas_benchmarks', executable_filter, icu_flags)

  if is_linux():
    run_engine_executable(build_dir, 'txt_benchmarks', executable_filter, icu_flags)


class FlutterTesterOptions():

  def __init__(
      self,
      multithreaded=False,
      enable_impeller=False,
      enable_observatory=False,
      expect_failure=False
  ):
    self.multithreaded = multithreaded
    self.enable_impeller = enable_impeller
    self.enable_observatory = enable_observatory
    self.expect_failure = expect_failure

  def apply_args(self, command_args):
    if not self.enable_observatory:
      command_args.append('--disable-observatory')

    if self.enable_impeller:
      command_args += ['--enable-impeller']
    else:
      command_args += ['--no-enable-impeller']

    if self.multithreaded:
      command_args.insert(0, '--force-multithreading')

  def threading_description(self):
    if self.multithreaded:
      return 'multithreaded'
    return 'single-threaded'

  def impeller_enabled(self):
    if self.enable_impeller:
      return 'impeller swiftshader'
    return 'skia software'


def gather_dart_test(build_dir, dart_file, options):
  kernel_file_name = os.path.basename(dart_file) + '.dill'
  kernel_file_output = os.path.join(build_dir, 'gen', kernel_file_name)
  error_message = "%s doesn't exist. Please run the build that populates %s" % (
      kernel_file_output, build_dir
  )
  assert os.path.isfile(kernel_file_output), error_message

  command_args = []

  options.apply_args(command_args)

  dart_file_contents = open(dart_file, 'r')
  custom_options = re.findall('// FlutterTesterOptions=(.*)', dart_file_contents.read())
  dart_file_contents.close()
  command_args.extend(custom_options)

  command_args += [
      '--use-test-fonts',
      '--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat'),
      '--flutter-assets-dir=%s' % os.path.join(build_dir, 'gen', 'flutter', 'lib', 'ui', 'assets'),
      '--disable-asset-fonts',
      kernel_file_output,
  ]

  tester_name = 'flutter_tester'
  logger.info(
      "Running test '%s' using '%s' (%s, %s)", kernel_file_name, tester_name,
      options.threading_description(), options.impeller_enabled()
  )
  forbidden_output = [] if 'unopt' in build_dir or options.expect_failure else ['[ERROR']
  return EngineExecutableTask(
      build_dir,
      tester_name,
      None,
      command_args,
      forbidden_output=forbidden_output,
      expect_failure=options.expect_failure,
  )


def ensure_ios_tests_are_built(ios_out_dir):
  """Builds the engine variant and the test dylib containing the XCTests"""
  tmp_out_dir = os.path.join(OUT_DIR, ios_out_dir)
  ios_test_lib = os.path.join(tmp_out_dir, 'libios_test_flutter.dylib')
  message = []
  message.append('gn --ios --unoptimized --runtime-mode=debug --no-lto --simulator')
  message.append('ninja -C %s ios_test_flutter' % ios_out_dir)
  final_message = "%s or %s doesn't exist. Please run the following commands: \n%s" % (
      ios_out_dir, ios_test_lib, '\n'.join(message)
  )
  assert os.path.exists(tmp_out_dir) and os.path.exists(ios_test_lib), final_message


def assert_expected_xcode_version():
  """Checks that the user has a version of Xcode installed"""
  version_output = subprocess.check_output(['xcodebuild', '-version'])
  # TODO ricardoamador: remove this check when python 2 is deprecated.
  version_output = version_output if isinstance(version_output,
                                                str) else version_output.decode(ENCODING)
  version_output = version_output.strip()
  match = re.match(r'Xcode (\d+)', version_output)
  message = 'Xcode must be installed to run the iOS embedding unit tests'
  assert match, message


def java_home():
  script_path = os.path.dirname(os.path.realpath(__file__))
  if is_mac():
    return os.path.join(
        script_path, '..', '..', 'third_party', 'java', 'openjdk', 'Contents', 'Home'
    )
  return os.path.join(script_path, '..', '..', 'third_party', 'java', 'openjdk')


def java_bin():
  return os.path.join(java_home(), 'bin', 'java.exe' if is_windows() else 'java')


def run_java_tests(executable_filter, android_variant='android_debug_unopt'):
  """Runs the Java JUnit unit tests for the Android embedding"""
  test_runner_dir = os.path.join(
      BUILDROOT_DIR, 'flutter', 'shell', 'platform', 'android', 'test_runner'
  )
  gradle_bin = os.path.join(
      BUILDROOT_DIR, 'third_party', 'gradle', 'bin', 'gradle.bat' if is_windows() else 'gradle'
  )
  flutter_jar = os.path.join(OUT_DIR, android_variant, 'flutter.jar')
  android_home = os.path.join(BUILDROOT_DIR, 'third_party', 'android_tools', 'sdk')
  build_dir = os.path.join(OUT_DIR, android_variant, 'robolectric_tests', 'build')
  gradle_cache_dir = os.path.join(OUT_DIR, android_variant, 'robolectric_tests', '.gradle')

  test_class = executable_filter if executable_filter else '*'
  command = [
      gradle_bin,
      '-Pflutter_jar=%s' % flutter_jar,
      '-Pbuild_dir=%s' % build_dir,
      'testDebugUnitTest',
      '--tests=%s' % test_class,
      '--rerun-tasks',
      '--no-daemon',
      '--project-cache-dir=%s' % gradle_cache_dir,
      '--gradle-user-home=%s' % gradle_cache_dir,
  ]

  env = dict(os.environ, ANDROID_HOME=android_home, JAVA_HOME=java_home())
  run_cmd(command, cwd=test_runner_dir, env=env)


def run_android_tests(android_variant='android_debug_unopt', adb_path=None):
  test_runner_name = 'flutter_shell_native_unittests'
  tests_path = os.path.join(OUT_DIR, android_variant, test_runner_name)
  remote_path = '/data/local/tmp'
  remote_tests_path = os.path.join(remote_path, test_runner_name)
  if adb_path is None:
    adb_path = 'adb'
  run_cmd([adb_path, 'push', tests_path, remote_path], cwd=BUILDROOT_DIR)
  run_cmd([adb_path, 'shell', remote_tests_path])

  systrace_test = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'android_systrace_test.py')
  scenario_apk = os.path.join(OUT_DIR, android_variant, 'firebase_apks', 'scenario_app.apk')
  run_cmd([
      systrace_test, '--adb-path', adb_path, '--apk-path', scenario_apk, '--package-name',
      'dev.flutter.scenarios', '--activity-name', '.PlatformViewsActivity'
  ])


def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None):
  """Runs Objective-C XCTest unit tests for the iOS embedding"""
  assert_expected_xcode_version()
  ios_out_dir = os.path.join(OUT_DIR, ios_variant)
  ensure_ios_tests_are_built(ios_out_dir)

  new_simulator_name = 'IosUnitTestsSimulator'

  # Delete simulators with this name in case any were leaked
  # from another test run.
  delete_simulator(new_simulator_name)

  create_simulator = [
      'xcrun '
      'simctl '
      'create '
      '%s com.apple.CoreSimulator.SimDeviceType.iPhone-11' % new_simulator_name
  ]
  run_cmd(create_simulator, shell=True)

  try:
    ios_unit_test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests')

    with tempfile.TemporaryDirectory(suffix='ios_embedding_xcresult') as result_bundle_temp:
      result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding')

      # Avoid using xcpretty unless the following can be addressed:
      # - Make sure all relevant failure output is printed on a failure.
      # - Make sure that a failing exit code is set for CI.
      # See https://github.com/flutter/flutter/issues/63742
      test_command = [
          'xcodebuild '
          '-sdk iphonesimulator '
          '-scheme IosUnitTests '
          '-resultBundlePath ' + result_bundle_path + ' '
          '-destination name=' + new_simulator_name + ' '
          'test '
          'FLUTTER_ENGINE=' + ios_variant
      ]
      if test_filter is not None:
        test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter
      try:
        run_cmd(test_command, cwd=ios_unit_test_dir, shell=True)

      except:
        # The LUCI environment may provide a variable containing a directory path
        # for additional output files that will be uploaded to cloud storage.
        # Upload the xcresult when the tests fail.
        luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
        xcresult_bundle = os.path.join(result_bundle_temp, 'ios_embedding.xcresult')
        if luci_test_outputs_path and os.path.exists(xcresult_bundle):
          dump_path = os.path.join(luci_test_outputs_path, 'ios_embedding.xcresult')
          # xcresults contain many little files. Archive the bundle before upload.
          shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle)
        raise

  finally:
    delete_simulator(new_simulator_name)


def delete_simulator(simulator_name):
  # Will delete all simulators with this name.
  command = [
      'xcrun',
      'simctl',
      'delete',
      simulator_name,
  ]
  # Let this fail if the simulator was never created.
  run_cmd(command, expect_failure=True)


def gather_dart_tests(build_dir, test_filter):
  dart_tests_dir = os.path.join(
      BUILDROOT_DIR,
      'flutter',
      'testing',
      'dart',
  )

  # Now that we have the Sky packages at the hardcoded location, run `dart pub get`.
  run_engine_executable(
      build_dir,
      os.path.join('dart-sdk', 'bin', 'dart'),
      None,
      flags=['pub', 'get', '--offline'],
      cwd=dart_tests_dir,
  )

  dart_observatory_tests = glob.glob('%s/observatory/*_test.dart' % dart_tests_dir)
  dart_tests = glob.glob('%s/*_test.dart' % dart_tests_dir)

  if 'release' not in build_dir:
    for dart_test_file in dart_observatory_tests:
      if test_filter is not None and os.path.basename(dart_test_file) not in test_filter:
        logger.info("Skipping '%s' due to filter.", dart_test_file)
      else:
        logger.info("Gathering dart test '%s' with observatory enabled", dart_test_file)
        for multithreaded in [False, True]:
          for enable_impeller in [False, True]:
            yield gather_dart_test(
                build_dir, dart_test_file,
                FlutterTesterOptions(
                    multithreaded=multithreaded,
                    enable_impeller=enable_impeller,
                    enable_observatory=True
                )
            )

  for dart_test_file in dart_tests:
    if test_filter is not None and os.path.basename(dart_test_file) not in test_filter:
      logger.info("Skipping '%s' due to filter.", dart_test_file)
    else:
      logger.info("Gathering dart test '%s'", dart_test_file)
      for multithreaded in [False, True]:
        for enable_impeller in [False, True]:
          yield gather_dart_test(
              build_dir, dart_test_file,
              FlutterTesterOptions(multithreaded=multithreaded, enable_impeller=enable_impeller)
          )


def gather_dart_smoke_test(build_dir, test_filter):
  smoke_test = os.path.join(
      BUILDROOT_DIR,
      'flutter',
      'testing',
      'smoke_test_failure',
      'fail_test.dart',
  )
  if test_filter is not None and os.path.basename(smoke_test) not in test_filter:
    logger.info("Skipping '%s' due to filter.", smoke_test)
  else:
    yield gather_dart_test(
        build_dir, smoke_test, FlutterTesterOptions(multithreaded=True, expect_failure=True)
    )
    yield gather_dart_test(
        build_dir, smoke_test, FlutterTesterOptions(multithreaded=False, expect_failure=True)
    )


def gather_dart_package_tests(build_dir, package_path, extra_opts):
  dart_tests = glob.glob('%s/test/*_test.dart' % package_path)
  if not dart_tests:
    raise Exception('No tests found for Dart package at %s' % package_path)
  for dart_test_file in dart_tests:
    opts = ['--disable-dart-dev', dart_test_file] + extra_opts
    yield EngineExecutableTask(
        build_dir, os.path.join('dart-sdk', 'bin', 'dart'), None, flags=opts, cwd=package_path
    )


# Returns a list of Dart packages to test.
#
# The first element of each tuple in the returned list is the path to the Dart
# package to test. It is assumed that the packages follow the convention that
# tests are named as '*_test.dart', and reside under a directory called 'test'.
#
# The second element of each tuple is a list of additional command line
# arguments to pass to each of the packages tests.
def build_dart_host_test_list(build_dir):
  dart_host_tests = [
      (
          os.path.join('flutter', 'ci'),
          [os.path.join(BUILDROOT_DIR, 'flutter')],
      ),
      (
          os.path.join('flutter', 'flutter_frontend_server'),
          [
              build_dir,
              os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
              os.path.join(build_dir, 'flutter_patched_sdk')
          ],
      ),
      (os.path.join('flutter', 'testing', 'litetest'), []),
      (
          os.path.join('flutter', 'tools', 'api_check'),
          [os.path.join(BUILDROOT_DIR, 'flutter')],
      ),
      (os.path.join('flutter', 'tools', 'build_bucket_golden_scraper'), []),
      (os.path.join('flutter', 'tools', 'clang_tidy'), []),
      (
          os.path.join('flutter', 'tools', 'const_finder'),
          [
              os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
              os.path.join(build_dir, 'flutter_patched_sdk'),
              os.path.join(build_dir, 'dart-sdk', 'lib', 'libraries.json'),
          ],
      ),
      (os.path.join('flutter', 'tools', 'githooks'), []),
      (os.path.join('flutter', 'tools', 'header_guard_check'), []),
      (os.path.join('flutter', 'tools', 'pkg', 'engine_build_configs'), []),
      (os.path.join('flutter', 'tools', 'pkg', 'engine_repo_tools'), []),
      (os.path.join('flutter', 'tools', 'pkg', 'git_repo_tools'), []),
  ]
  if not is_asan(build_dir):
    dart_host_tests += [
        (os.path.join('flutter', 'tools', 'path_ops', 'dart'), []),
    ]

  return dart_host_tests


def run_benchmark_tests(build_dir):
  test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'benchmark')
  dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
  for dart_test_file in dart_tests:
    opts = ['--disable-dart-dev', dart_test_file]
    run_engine_executable(
        build_dir, os.path.join('dart-sdk', 'bin', 'dart'), None, flags=opts, cwd=test_dir
    )


def worker_init(queue, level):
  queue_handler = logging.handlers.QueueHandler(queue)
  log = logging.getLogger(__name__)
  log.setLevel(level)
  log.addHandler(queue_handler)


def run_engine_tasks_in_parallel(tasks):
  # Work around a bug in Python.
  #
  # The multiprocessing package relies on the win32 WaitForMultipleObjects()
  # call, which supports waiting on a maximum of MAXIMUM_WAIT_OBJECTS (defined
  # by Windows to be 64) handles, processes in this case. To avoid hitting
  # this, we limit ourselves to 60 handles (since there are a couple extra
  # processes launched for the queue reader and thread wakeup reader).
  #
  # See: https://bugs.python.org/issue26903
  max_processes = multiprocessing.cpu_count()
  if sys_platform.startswith(('cygwin', 'win')) and max_processes > 60:
    max_processes = 60

  queue = multiprocessing.Queue()
  queue_listener = logging.handlers.QueueListener(queue, logger_handler)
  queue_listener.start()

  failures = []
  try:
    with multiprocessing.Pool(max_processes, worker_init,
                              [queue, logger.getEffectiveLevel()]) as pool:
      async_results = [(t, pool.apply_async(t, ())) for t in tasks]
      for task, async_result in async_results:
        try:
          async_result.get()
        except Exception as exn:  # pylint: disable=broad-except
          failures += [(task, exn)]
  finally:
    queue_listener.stop()

  if len(failures) > 0:
    logger.error('The following commands failed:')
    for task, exn in failures:
      logger.error('%s\n  %s\n\n', str(task), str(exn))
    return False

  return True


class DirectoryChange():
  """
  A scoped change in the CWD.
  """
  old_cwd: str = ''
  new_cwd: str = ''

  def __init__(self, new_cwd: str):
    self.new_cwd = new_cwd

  def __enter__(self):
    self.old_cwd = os.getcwd()
    os.chdir(self.new_cwd)

  def __exit__(self, exception_type, exception_value, exception_traceback):
    os.chdir(self.old_cwd)


def generate_dir_listing(dir_path: str) -> str:
  listing = os.listdir(dir_path)
  listing.sort()
  return reduce(lambda a, b: a + '\n' + b, listing)


def str_replace_range(instr: str, start: int, end: int, replacement: str) -> str:
  return instr[:start] + replacement + instr[end:]


def redirect_patch(patch: str) -> str:
  'Makes a diff point its output file to its input file.'
  input_path = re.search(r'^--- a(.*)', patch, re.MULTILINE)
  output_path = re.search(r'^\+\+\+ b(.*)', patch, re.MULTILINE)
  return str_replace_range(
      patch,
      output_path.span(1)[0],
      output_path.span(1)[1], input_path.group(1)
  )


def run_impeller_golden_tests(build_dir: str):
  """
  Executes the impeller golden image tests from in the `variant` build.
  """
  tests_path: str = os.path.join(build_dir, 'impeller_golden_tests')
  if not os.path.exists(tests_path):
    raise Exception(
        'Cannot find the "impeller_golden_tests" executable in "%s". You may need to build it.' %
        (build_dir)
    )
  harvester_path: Path = Path(SCRIPT_DIR).parent.joinpath('tools'
                                                         ).joinpath('golden_tests_harvester')
  with tempfile.TemporaryDirectory(prefix='impeller_golden') as temp_dir:
    run_cmd([tests_path, '--working_dir=%s' % temp_dir], cwd=build_dir)
    with tempfile.NamedTemporaryFile(mode='w',
                                     prefix='impeller_golden_tests_output') as dir_listing_file:
      dir_listing = generate_dir_listing(temp_dir)
      dir_listing_file.write(dir_listing)
      golden_path = os.path.join('testing', 'impeller_golden_tests_output.txt')
      diff_result = subprocess.run(
          f'git diff -p {golden_path} {dir_listing_file.name}',
          check=False,
          shell=True,
          stdout=subprocess.PIPE,
          cwd=os.path.join(BUILDROOT_DIR, 'flutter')
      )
      if diff_result.returncode != 0:
        print_divider('<')
        print(f'Unexpected diff in {golden_path}, use `git apply` with the following patch.')
        print('')
        print(redirect_patch(diff_result.stdout.decode()))
        raise RuntimeError('impeller_golden_tests diff failure')

    with DirectoryChange(harvester_path):
      run_cmd(['dart', 'pub', 'get'])
      bin_path = Path('.').joinpath('bin').joinpath('golden_tests_harvester.dart')
      run_cmd(['dart', 'run', str(bin_path), temp_dir])


def main():
  parser = argparse.ArgumentParser(
      description="""
In order to learn the details of running tests in the engine, please consult the
Flutter Wiki page on the subject: https://github.com/flutter/flutter/wiki/Testing-the-engine
"""
  )
  all_types = [
      'engine',
      'dart',
      'dart-host',
      'benchmarks',
      'java',
      'android',
      'objc',
      'font-subset',
      'impeller-golden',
  ]

  parser.add_argument(
      '--variant',
      dest='variant',
      action='store',
      default='host_debug_unopt',
      help='The engine build variant to run the tests for.'
  )
  parser.add_argument(
      '--type',
      type=str,
      default='all',
      help='A list of test types, default is "all" (equivalent to "%s")' % (','.join(all_types))
  )
  parser.add_argument(
      '--engine-filter', type=str, default='', help='A list of engine test executables to run.'
  )
  parser.add_argument(
      '--dart-filter',
      type=str,
      default='',
      help='A list of Dart test scripts to run in flutter_tester.'
  )
  parser.add_argument(
      '--dart-host-filter',
      type=str,
      default='',
      help='A list of Dart test scripts to run with the Dart CLI.'
  )
  parser.add_argument(
      '--java-filter',
      type=str,
      default='',
      help='A single Java test class to run (example: "io.flutter.SmokeTest")'
  )
  parser.add_argument(
      '--android-variant',
      dest='android_variant',
      action='store',
      default='android_debug_unopt',
      help='The engine build variant to run java or android tests for'
  )
  parser.add_argument(
      '--ios-variant',
      dest='ios_variant',
      action='store',
      default='ios_debug_sim_unopt',
      help='The engine build variant to run objective-c tests for'
  )
  parser.add_argument(
      '--verbose-dart-snapshot',
      dest='verbose_dart_snapshot',
      action='store_true',
      default=False,
      help='Show extra dart snapshot logging.'
  )
  parser.add_argument(
      '--objc-filter',
      type=str,
      default=None,
      help=(
          'Filter parameter for which objc tests to run '
          '(example: "IosUnitTestsTests/SemanticsObjectTest/testShouldTriggerAnnouncement")'
      )
  )
  parser.add_argument(
      '--coverage',
      action='store_true',
      default=None,
      help='Generate coverage reports for each unit test framework run.'
  )
  parser.add_argument(
      '--engine-capture-core-dump',
      dest='engine_capture_core_dump',
      action='store_true',
      default=False,
      help='Capture core dumps from crashes of engine tests.'
  )
  parser.add_argument(
      '--use-sanitizer-suppressions',
      dest='sanitizer_suppressions',
      action='store_true',
      default=False,
      help='Provide the sanitizer suppressions lists to the via environment to the tests.'
  )
  parser.add_argument(
      '--adb-path',
      dest='adb_path',
      action='store',
      default=None,
      help='Provide the path of adb used for android tests. By default it looks on $PATH.'
  )
  parser.add_argument(
      '--quiet',
      dest='quiet',
      action='store_true',
      default=False,
      help='Only emit output when there is an error.'
  )

  args = parser.parse_args()

  logger.addHandler(logger_handler)
  if not args.quiet:
    logger.setLevel(logging.INFO)

  if args.type == 'all':
    types = all_types
  else:
    types = args.type.split(',')

  build_dir = os.path.join(OUT_DIR, args.variant)
  if args.type != 'java' and args.type != 'android':
    assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir

  if args.sanitizer_suppressions:
    assert is_linux() or is_mac(
    ), 'The sanitizer suppressions flag is only supported on Linux and Mac.'
    file_dir = os.path.dirname(os.path.abspath(__file__))
    command = [
        'env', '-i', 'bash', '-c',
        'source {}/sanitizer_suppressions.sh >/dev/null && env'.format(file_dir)
    ]
    process = subprocess.Popen(command, stdout=subprocess.PIPE)
    for line in process.stdout:
      key, _, value = line.decode('utf8').strip().partition('=')
      os.environ[key] = value
    process.communicate()  # Avoid pipe deadlock while waiting for termination.

  success = True

  engine_filter = args.engine_filter.split(',') if args.engine_filter else None
  if 'engine' in types:
    run_cc_tests(build_dir, engine_filter, args.coverage, args.engine_capture_core_dump)

  # Use this type to exclusively run impeller tests.
  if 'impeller' in types:
    build_name = args.variant
    try:
      xvfb.start_virtual_x(build_name, build_dir)
      run_engine_executable(
          build_dir, 'impeller_unittests', engine_filter, shuffle_flags, coverage=args.coverage
      )
    finally:
      xvfb.stop_virtual_x(build_name)

  if 'dart' in types:
    dart_filter = args.dart_filter.split(',') if args.dart_filter else None
    tasks = list(gather_dart_smoke_test(build_dir, dart_filter))
    tasks += list(gather_dart_tests(build_dir, dart_filter))
    success = success and run_engine_tasks_in_parallel(tasks)

  if 'dart-host' in types:
    dart_filter = args.dart_host_filter.split(',') if args.dart_host_filter else None
    dart_host_packages = build_dart_host_test_list(build_dir)
    tasks = []
    for dart_host_package, extra_opts in dart_host_packages:
      if dart_filter is None or dart_host_package in dart_filter:
        tasks += list(
            gather_dart_package_tests(
                build_dir,
                os.path.join(BUILDROOT_DIR, dart_host_package),
                extra_opts,
            )
        )

    success = success and run_engine_tasks_in_parallel(tasks)

  if 'java' in types:
    assert not is_windows(), "Android engine files can't be compiled on Windows."
    java_filter = args.java_filter
    if ',' in java_filter or '*' in java_filter:
      logger.wraning(
          'Can only filter JUnit4 tests by single entire class name, '
          'eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter
      )
      java_filter = None
    run_java_tests(java_filter, args.android_variant)

  if 'android' in types:
    assert not is_windows(), "Android engine files can't be compiled on Windows."
    run_android_tests(args.android_variant, args.adb_path)

  if 'objc' in types:
    assert is_mac(), 'iOS embedding tests can only be run on macOS.'
    run_objc_tests(args.ios_variant, args.objc_filter)

  # https://github.com/flutter/flutter/issues/36300
  if 'benchmarks' in types and not is_windows():
    run_benchmark_tests(build_dir)
    run_engine_benchmarks(build_dir, engine_filter)

  variants_to_skip = ['host_release', 'host_profile']
  if ('engine' in types or 'font-subset' in types) and args.variant not in variants_to_skip:
    run_cmd(['python3', 'test.py'], cwd=FONT_SUBSET_DIR)

  if 'impeller-golden' in types:
    run_impeller_golden_tests(build_dir)

  return 0 if success else 1


if __name__ == '__main__':
  sys_exit(main())