# Copyright 2021 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Flutter Engine builder recipe.

This recipe is used to build flavors of flutter engine identified by lists of
gn flags and ninja configs and targets.


The following are examples of valid configurations passed to builders using
this recipe in the builds property:

 {
    "gn" : [
       "--ios",
       "--runtime-mode",
       "debug",
       "--simulator",
       "--no-lto"
    ],
    "ninja": {
      "config": "ios_debug_sim",
      "targets": ["ios_test_flutter"]
    }
 }
"""
import contextlib
import copy
import re

from google.protobuf import struct_pb2
from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2

DEPS = [
    'depot_tools/depot_tools',
    'flutter/archives',
    'flutter/build_util',
    'flutter/flutter_bcid',
    'flutter/flutter_deps',
    'flutter/logs_util',
    'flutter/monorepo',
    'flutter/os_utils',
    'flutter/osx_sdk',
    'flutter/rbe',
    'flutter/repo_util',
    'flutter/retry',
    'flutter/shard_util_v2',
    'flutter/signing',
    'flutter/test_utils',
    'fuchsia/cas_util',
    'recipe_engine/bcid_reporter',
    'recipe_engine/buildbucket',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
    'recipe_engine/time',
]

ANDROID_ARTIFACTS_BUCKET = 'download.flutter.io'

# Relative paths used to mock paths for testing.
MOCK_JAR_PATH = (
    'io/flutter/x86_debug/'
    '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/'
    'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar'
)
MOCK_POM_PATH = (
    'io/flutter/x86_debug/'
    '1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584/'
    'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom'
)

# Used for mock paths
DIRECTORY = 'DIRECTORY'

# Default timeout for tests running as part of the build.
DEFAULT_TEST_TIMEOUT_SECS = 30 * 60


def run_generators(api, pub_dirs, generator_tasks, checkout, env, env_prefixes):
  """Runs sub-builds generators."""
  # Run pub on all of the pub_dirs.
  for pub in pub_dirs:
    pub_dir = api.path.abs_to_path(api.path.dirname(checkout.join(pub)))
    with api.context(env=env, env_prefixes=env_prefixes, cwd=pub_dir):
      api.step('dart pub get', ['dart', 'pub', 'get'])
  for generator_task in generator_tasks:
    # Generators must run from inside flutter folder.
    cmd = []
    for script in generator_task.get('scripts'):
      full_path_script = checkout.join(script)
      cmd.append(full_path_script)
    cmd.extend(generator_task.get('parameters', []))
    api.step(generator_task.get('name'), cmd)


def _should_run_test(api, test):
  """Whether the current test should run on this branch.

  The current test should run when the `test_if` matches the test branch, for both presubmit
  and postsubmit.
  """
  # Presubmit builds have git_ref property, whereas `api.buildbucket.gitiles_commit.ref` is None:
  #  1) Main branch
  #    * git_ref: "refs/pull/51183/head"
  #    * git_branch: "main"
  #  2) Release branch
  #    * git_ref: "refs/pull/144592/head"
  #    * git_branch: "flutter-3.20-candidate.17"
  if api.properties.get('git_ref', None):
    branch = api.properties.get('git_branch')
  else:
    # Postsubmit builds do not have git_ref property, but have `api.buildbucket.gitiles_commit.ref` defined:
    #  1) Main branch
    #    * api.buildbucket.gitiles_commit.ref: "refs/heads/main"
    #  2) Release branch
    #    * api.buildbucket.gitiles_commit.ref: "refs/heads/flutter-3.20-candidate.17"
    branch = api.buildbucket.gitiles_commit.ref
  test_if = test.get('test_if', '')
  regex = re.compile(f'.*{test_if}')
  return regex.match(branch)


def run_tests(api, tests, checkout, env, env_prefixes):
  """Runs sub-build tests."""
  # Run local tests in the builder to optimize resource usage.
  for test in tests:
    if not _should_run_test(api, test):
      continue
    # Copy and expand env, env_prefixes. This is required to
    # add configuration env variables.
    test_deps = test.get('test_dependencies', [])
    api.flutter_deps.required_deps(env, env_prefixes, test_deps)
    tmp_env = copy.deepcopy(env)
    tmp_env.update(test.get('env', {}))
    # Run tests within a exitStack context
    with contextlib.ExitStack() as exit_stack:
      api.flutter_deps.enter_contexts(
          exit_stack, test.get('contexts', []), tmp_env, env_prefixes
      )
      command = [test.get('language')] if test.get('language') else []
      # Ideally local tests should be completely hermetic and in theory we can run
      # them in parallel using futures. I haven't found a flutter engine
      # configuration with more than one local test but once we find it we
      # should run the list of tests using parallelism.
      # TODO(godofredoc): Optimize to run multiple local tests in parallel.
      command.append(checkout.join(test.get('script')))
      command.extend(test.get('parameters', []))
      step_name = api.test_utils.test_step_name(test.get('name'))
      test_timeout_secs = test.get(
          'test_timeout_secs', DEFAULT_TEST_TIMEOUT_SECS
      )

      # pylint: disable=cell-var-from-loop
      def run_test():
        # Replace MAGIC_ENVS
        updated_command = api.os_utils.replace_magic_envs(command, tmp_env)
        return api.step(step_name, updated_command, timeout=test_timeout_secs)

      # Rerun test step 3 times by default if failing.
      # TODO(keyonghan): notify tree gardener for test failures/flakes:
      # https://github.com/flutter/flutter/issues/89308
      api.logs_util.initialize_logs_collection(tmp_env)
      try:
        # Run within another context to make the logs env variable available to
        # test scripts.
        with api.context(env=tmp_env, env_prefixes=env_prefixes):
          api.retry.wrap(
              run_test,
              max_attempts=test.get('max_attempts', 3),
              step_name=step_name
          )
      finally:
        api.logs_util.upload_logs(test.get('name'))


def Build(api, checkout, env, env_prefixes, outputs, build):
  """Builds a flavor identified as a set of gn and ninja configs."""

  # Mock data for tests. This is required for the archive api to expand the directory to full path
  # of files.
  api.path.mock_add_directory(
      api.path['cache'].join(
          'builder/src/out/android_jit_release_x86/zip_archives/download.flutter.io'
      )
  )

  ninja_tool = {
      "ninja": api.build_util.build,
  }
  deps = build.get('dependencies', [])
  api.flutter_deps.required_deps(env, env_prefixes, deps)
  api.flutter_bcid.report_stage('compile')
  gn = build.get('gn')

  # If prod apply prod gn overrides.
  if api.flutter_bcid.is_prod_build():
    gn = build.get('postsubmit_overrides', {}).get('gn', gn)

  if gn:
    with api.context(env=env, env_prefixes=env_prefixes):
      gn = list(gn)
      if api.flutter_bcid.is_official_build():
        # Goma/RBE is not supported for official builds.
        gn.append('--no-goma')
        gn.append('--no-rbe')
      if api.monorepo.is_monorepo_ci_build:
        version = env['REVISION']
        gn.append(f'--gn-args=engine_version="{version}"')
      if api.monorepo.is_monorepo_try_build:
        version = api.monorepo.try_build_identifier
        gn.append(f'--gn-args=engine_version="{version}"')
      rbe_working_path = api.path.mkdtemp(prefix="rbe")
      if '--rbe' in gn:
        api.rbe.prepare_rbe_gn(rbe_working_path, gn)
      api.build_util.run_gn(gn, checkout)
      ninja = build.get('ninja')
      ninja_tool[ninja.get('tool', 'ninja')](
          ninja.get('config'),
          checkout,
          ninja.get('targets', []),
          env,
          rbe_working_path=rbe_working_path
      )
  generator_tasks = build.get('generators', {}).get('tasks', [])
  pub_dirs = build.get('generators', {}).get('pub_dirs', [])
  archives = build.get('archives', [])
  # Get only local tests.
  tests = build.get('tests', [])
  with api.context(env=env, env_prefixes=env_prefixes,
                   cwd=checkout.join('flutter')), api.depot_tools.on_path():
    run_generators(api, pub_dirs, generator_tasks, checkout, env, env_prefixes)
    run_tests(api, tests, checkout, env, env_prefixes)
    api.flutter_bcid.report_stage('upload')
    for archive_config in archives:
      outputs[archive_config['name']] = Archive(api, checkout, archive_config)
    api.flutter_bcid.report_stage('upload-complete')
    # Allow time for the provenance to upload so it can be validated
    api.time.sleep(60)
    for archive_config in archives:
      if api.flutter_bcid.is_official_build():
        Verify(api, checkout, archive_config)
  # Archive full build. This is inefficient but necessary for global generators.
  if build.get('cas_archive', True):
    full_build_hash = api.shard_util_v2.archive_full_build(
        checkout.join('out', build.get('name')), build.get('name')
    )
    outputs['full_build'] = full_build_hash


def Archive(api, checkout, archive_config):
  paths = api.archives.engine_v2_gcs_paths(checkout, archive_config)
  # Sign artifacts if running on mac and a release candidate branch.
  is_release_branch = api.repo_util.is_release_candidate_branch(
      checkout.join('flutter')
  )
  if api.platform.is_mac and is_release_branch:
    signing_paths = [
        path.local
        for path in paths
        if api.signing.requires_signing(path.local)
    ]
    api.signing.code_sign(signing_paths)
  for path in paths:
    api.archives.upload_artifact(path.local, path.remote)
    api.flutter_bcid.upload_provenance(path.local, path.remote)


def Verify(api, checkout, archive_config):
  """Verifies a set of artifacts through BCID using artifact provenance."""

  paths = api.archives.engine_v2_gcs_paths(checkout, archive_config)

  for path in paths:
    gcs_path = path.remote
    gcs_path_without_prefix = str.lstrip(gcs_path, 'gs://')
    file = api.path.basename(gcs_path)
    bucket = gcs_path_without_prefix.split('/', maxsplit=1)[0]
    gcs_path_without_bucket = '/'.join(gcs_path_without_prefix.split('/')[1:])

    api.flutter_bcid.download_and_verify_provenance(
        file, bucket, gcs_path_without_bucket
    )


def RunSteps(api):
  # Collect memory/cpu/process before task execution.
  api.os_utils.collect_os_info()

  # If on macOS, reset Xcode in case a previous build failed to do so.
  api.osx_sdk.reset_xcode()

  api.flutter_bcid.report_stage('start')
  checkout = api.path['cache'].join('builder', 'src')
  api.file.rmtree('Clobber build output', checkout.join('out'))
  cache_root = api.path['cache'].join('builder')
  api.file.ensure_directory('Ensure checkout cache', cache_root)

  with api.os_utils.make_temp_directory('standalone_repo') as temp_checkout:
    build = api.repo_util.get_build(temp_checkout)
  gclient_variables = build.get('gclient_variables', {})

  # Enable long path support on Windows.
  api.os_utils.enable_long_paths()

  api.flutter_bcid.report_stage('fetch')
  if api.monorepo.is_monorepo_ci_build or api.monorepo.is_monorepo_try_build:
    env, env_prefixes = api.repo_util.monorepo_environment(
        api.path['cache'].join('builder')
    )
    api.repo_util.monorepo_checkout(cache_root, env, env_prefixes)
    checkout = api.path['cache'].join('builder', 'engine', 'src')
  else:
    env, env_prefixes = api.repo_util.engine_environment(
        api.path['cache'].join('builder')
    )
    api.repo_util.engine_checkout(
        cache_root, env, env_prefixes, gclient_variables=gclient_variables
    )
  outputs = {}
  api.logs_util.initialize_logs_collection(env)
  try:
    if api.platform.is_mac:
      with api.osx_sdk('ios'):
        Build(api, checkout, env, env_prefixes, outputs, build)
    else:
      Build(api, checkout, env, env_prefixes, outputs, build)
  finally:
    api.logs_util.upload_logs('builder', type='engine')
  output_props = api.step('Set output properties', None)
  output_props.presentation.properties['cas_output_hash'] = outputs

  # This is to clean up leaked processes.
  api.os_utils.kill_processes()
  # Collect memory/cpu/process after task execution.
  api.os_utils.collect_os_info()


def GenTests(api):
  build = {
      "archives": [{
          "name":
              "android_jit_release_x86", "type":
                  "gcs", "realm":
                      "production", "base_path":
                          "out/android_jit_release_x86/zip_archives/",
          "include_paths": [
              "out/android_jit_release_x86/zip_archives/android-x86-jit-release/artifacts.zip",
              "out/android_jit_release_x86/zip_archives/download.flutter.io"
          ]
      }], "gn": ["--ios", "--rbe"],
      "ninja": {"config": "ios_debug", "targets": []}, "generators": {
          "pub_dirs": ["dev"], "tasks": [{
              "name": "generator1", "scripts": ["script1.sh", "dev/felt.dart"],
              "parameters": ["--argument1"]
          }]
      }, "tests": [{
          "name": "mytest", "script": "myscript.sh",
          "parameters": ["param1", "param2", '${FLUTTER_LOGS_DIR}'],
          "type": "local", "contexts": ["metric_center_token"]
      }]
  }
  yield api.test(
      'basic',
      api.properties(build=build, no_goma=True),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main',
          revision='abcd' * 10,
          build_number=123,
      ),
  )
  yield api.test(
      'config_file',
      api.properties(no_goma=True, config_name='abc'),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main',
          revision='abcd' * 10,
          build_number=123,
      ),
  )
  yield api.test(
      'mac',
      api.properties(build=build, no_goma=True),
      api.platform('mac', 64),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='mac-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main',
          revision='abcd' * 10,
          build_number=123,
      ),
      api.step_data(
          'Identify branches.git branch',
          stdout=api.raw_io.output_text(
              'branch1\nbranch2\nremotes/origin/flutter-3.2-candidate.5'
          )
      ),
  )
  yield api.test(
      'monorepo',
      api.properties(build=build, no_goma=True),
      api.monorepo.ci_build(),
  )
  yield api.test(
      'monorepo_tryjob',
      api.properties(
          build=build, no_goma=True, try_build_identifier='81123491'
      ),
      api.monorepo.try_build(),
  )

  fake_bcid_response_success = '''
  {
    "allowed": true,
    "verificationSummary": "This artifact is definitely legitimate!"
  }
  '''
  build_custom = dict(build)
  build_custom["gclient_variables"] = {"example_custom_var": True}
  build_custom["tests"] = []
  artifacts_location = 'artifacts.zip'
  jar_location = 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.jar'
  pom_location = 'x86_debug-1.0.0-0005149dca9b248663adcde4bdd7c6c915a76584.pom'
  yield api.test(
      'dart-internal-flutter-success',
      api.properties(build=build, no_goma=True),
      api.buildbucket.ci_build(
          project='dart-internal',
          bucket='flutter',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main',
      ),
      api.step_data(
          'Verify {0} provenance.verify {0} provenance'
          .format(artifacts_location),
          stdout=api.raw_io.output_text(fake_bcid_response_success)
      ),
      api.step_data(
          'Verify {0} provenance.verify {0} provenance'.format(jar_location),
          stdout=api.raw_io.output_text(fake_bcid_response_success)
      ),
      api.step_data(
          'Verify {0} provenance.verify {0} provenance'.format(pom_location),
          stdout=api.raw_io.output_text(fake_bcid_response_success)
      ),
  )
  test_if_build = {
      "tests": [{
          "name": "mytest", "script": "myscript.sh",
          "parameters": ["param1", "param2",
                         '${FLUTTER_LOGS_DIR}'], "type": "local",
          "contexts": ["metric_center_token"], "test_if": "main"
      }]
  }
  yield api.test(
      'test_if_skip_postsubmit',
      api.properties(build=test_if_build, no_goma=True),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/flutter-3.17-candidate.0',
          revision='abcd' * 10,
          build_number=123,
      ),
  )
  yield api.test(
      'test_if_not_skip_postsubmit',
      api.properties(build=test_if_build, no_goma=True),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main',
          revision='abcd' * 10,
          build_number=123,
      ),
  )
  yield api.test(
      'test_if_not_skip_presubmit',
      api.properties(
          build=test_if_build,
          no_goma=True,
          git_ref="refs/pull/51183/head",
          git_branch="main"
      ),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          revision='abcd' * 10,
          build_number=123,
      ),
  )
  yield api.test(
      'test_if_skip_presubmit',
      api.properties(
          build=test_if_build,
          no_goma=True,
          git_ref="refs/pull/51183/head",
          git_branch="flutter-3.17-candidate.0"
      ),
      api.buildbucket.ci_build(
          project='flutter',
          bucket='prod',
          builder='linux-host',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          revision='abcd' * 10,
          build_number=123,
      ),
  )