recipes/engine_v2/builder.py

# Copyright 2021 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Flutter Engine builder recipe.

This recipe is used to build flavors of flutter engine identified by lists of
gn flags and ninja configs and targets.


The following are examples of valid configurations passed to builders using
this recipe in the builds property:

 {
    "gn" : [
       "--ios",
       "--runtime-mode",
       "debug",
       "--simulator",
       "--no-lto"
    ],
    "ninja": {
      "config": "ios_debug_sim",
      "targets": ["ios_test_flutter"]
    }
 }
"""
import copy

from contextlib import contextmanager

from google.protobuf import struct_pb2
from PB.recipes.flutter.engine.engine import InputProperties
from PB.recipes.flutter.engine.engine import EnvProperties
from PB.go.chromium.org.luci.buildbucket.proto import build as build_pb2

DEPS = [
    'depot_tools/gsutil',
    'flutter/archives',
    'flutter/flutter_bcid',
    'flutter/build_util',
    'flutter/flutter_deps',
    'flutter/os_utils',
    'flutter/osx_sdk',
    'flutter/repo_util',
    'flutter/retry',
    'flutter/shard_util_v2',
    'flutter/test_utils',
    'fuchsia/cas_util',
    'recipe_engine/bcid_reporter',
    'recipe_engine/buildbucket',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/step',
]

PROPERTIES = InputProperties
ENV_PROPERTIES = EnvProperties
ANDROID_ARTIFACTS_BUCKET = 'download.flutter.io'


def Build(api, checkout, env, env_prefixes, outputs):
  """Builds a flavor identified as a set of gn and ninja configs."""
  ninja_tool = {
      "ninja": api.build_util.build,
      "autoninja": api.build_util.build_autoninja,
  }
  deps = api.properties.get('dependencies', [])
  api.flutter_deps.required_deps(env, env_prefixes, deps)
  build = api.properties.get('build')
  api.flutter_bcid.report_stage('compile')
  api.build_util.run_gn(build.get('gn'), checkout)
  ninja = build.get('ninja')
  ninja_tool[ninja.get('tool', 'ninja')
            ](ninja.get('config'), checkout, ninja.get('targets'))
  # Archive full build. This is inneficient but necessary for global generators.
  full_build_hash = api.shard_util_v2.archive_full_build(
          checkout.join('out', build.get('name')), build.get('name'))
  outputs['full_build'] = full_build_hash
  generator_tasks = build.get('generators', {}).get('tasks', [])
  pub_dirs = build.get('generators', {}).get('pub_dirs', [])
  archives = build.get('archives', [])
  # Get only local tests.
  tests = [t for t in build.get('tests', []) if t.get('type') == 'local']
  with api.context(env=env, env_prefixes=env_prefixes,
                   cwd=checkout.join('flutter')):
    # Run pub on all of the pub_dirs.
    for pub in pub_dirs:
      pub_dir = api.path.abs_to_path(
          api.path.dirname(
              checkout.join(pub))
      )
      with api.context(env=env, env_prefixes=env_prefixes,
                       cwd=pub_dir):
        api.step('dart pub get', ['dart', 'pub', 'get'])
    for generator_task in generator_tasks:
      # Generators must run from inside flutter folder.
      cmd = []
      for script in generator_task.get('scripts'):
        full_path_script = checkout.join(script)
        cmd.append(full_path_script)
      cmd.extend(generator_task.get('parameters', []))
      api.step(generator_task.get('name'), cmd)
    # Run local tests in the builder to optimize resource usage.
    for test in tests:
      command = [test.get('language')] if test.get('language') else []
      # Ideally local tests should be completely hermetic and in theory we can run
      # them in parallel using futures. I haven't found a flutter engine
      # configuration with more than one local test but once we find it we
      # should run the list of tests using parallelism.
      # TODO(godofredoc): Optimize to run multiple local tests in parallel.
      command.append(checkout.join(test.get('script')))
      command.extend(test.get('parameters', []))
      #api.step(test.get('name'), command)
      step_name = api.test_utils.test_step_name(test.get('name'))

      def run_test():
        return api.step(step_name, command)

      # Rerun test step 3 times by default if failing.
      # TODO(keyonghan): notify tree gardener for test failures/flakes:
      # https://github.com/flutter/flutter/issues/89308
      api.retry.wrap(run_test, step_name=test.get('name'))

    api.flutter_bcid.report_stage('upload')
    for archive_config in archives:
      outputs[archive_config['name']] = Archive(api, checkout, archive_config)
    api.flutter_bcid.report_stage('upload-complete')


def Archive(api, checkout,  archive_config):
  paths = api.archives.engine_v2_gcs_paths(checkout, archive_config)
  for path in paths:
      api.archives.upload_artifact(path.local, path.remote)
      api.flutter_bcid.upload_provenance(
          path.local,
          path.remote
      )

def RunSteps(api, properties, env_properties):
  api.flutter_bcid.report_stage('start')
  checkout = api.path['cache'].join('builder', 'src')
  api.file.rmtree('Clobber build output', checkout.join('out'))
  cache_root = api.path['cache'].join('builder')
  api.file.ensure_directory('Ensure checkout cache', cache_root)

  # Enable long path support on Windows.
  api.os_utils.enable_long_paths()
  env, env_prefixes = api.repo_util.engine_environment(api.path['cache'].join('builder'))

  # Engine path is used inconsistently across the engine repo. We'll start
  # with [cache]/builder and will adjust it to start using it consistently.
  env['ENGINE_PATH'] = api.path['cache'].join('builder')

  api.flutter_bcid.report_stage('fetch')
  if api.buildbucket.gitiles_commit.project == 'monorepo':
    api.repo_util.monorepo_checkout(cache_root, env, env_prefixes)
    checkout = api.path['cache'].join('builder', 'engine', 'src')
  else:
    api.repo_util.engine_checkout(cache_root, env, env_prefixes)
  outputs = {}
  if api.platform.is_mac:
    with api.osx_sdk('ios'):
      Build(api, checkout, env, env_prefixes, outputs)
  else:
    Build(api, checkout, env, env_prefixes, outputs)
  output_props = api.step('Set output properties', None)
  output_props.presentation.properties['cas_output_hash'] = outputs


def GenTests(api):
  build = {
      "archives": [
                {
                    "name": "android_jit_release_x86",
                    "type": "gcs",
                    "base_path": "out/android_jit_release_x86/zip_archives/",
                    "include_paths": [
                        "out/android_jit_release_x86/zip_archives/android-x86-jit-release/artifacts.zip",
                        "out/android_jit_release_x86/zip_archives/download.flutter.io"
                    ]
                }
      ],
      "gn": ["--ios"], "ninja": {"config": "ios_debug", "targets": []},
      "generators": {
          "pub_dirs": ["dev"],
          "tasks": [
              {
                  "name": "generator1",
                  "scripts": ["script1.sh", "dev/felt.dart"],
                  "parameters": ["--argument1"]
              }
          ]
      },
      "tests": [
          {
               "name": "mytest", "script": "myscript.sh",
               "parameters": ["param1", "param2"], "type": "local"
          }
      ]
  }
  yield api.test(
     'basic',
     api.properties(build=build, no_goma=True),
  )
  yield api.test(
      'mac', api.properties(build=build, no_goma=True),
      api.platform('mac', 64),
  )
  yield api.test(
      'monorepo', api.properties(build=build, no_goma=True),
      api.buildbucket.ci_build(
          project='dart',
          bucket='ci.sandbox',
          git_repo='https://dart.googlesource.com/monorepo',
          git_ref='refs/heads/main'
      ),
  )
  build_custom = dict(build)
  build_custom["gclient_custom_vars"] = {"example_custom_var": True}
  build_custom["tests"] = []
  yield api.test(
      'dart-internal-flutter', api.properties(build=build, no_goma=True),
      api.buildbucket.ci_build(
          project='dart-internal',
          bucket='flutter',
          git_repo='https://flutter.googlesource.com/mirrors/engine',
          git_ref='refs/heads/main'
      ),
  )
  build_custom = dict(build)
  build_custom["gclient_custom_vars"] = {"example_custom_var": True}
  build_custom["tests"] = []
  yield api.test(
      'basic_custom_vars', api.properties(build=build_custom)
  )
  # gcs archives
  build_gcs = copy.deepcopy(build)
  build_gcs['archives'][0]['type'] = 'gcs'
  yield api.test(
      'basic_gcs', api.properties(build=build_gcs, no_goma=True),
      api.step_data(
          'git rev-parse',
          stdout=api.raw_io
          .output_text('12345abcde12345abcde12345abcde12345abcde\n')
      )
  )
  yield api.test(
      'monorepo_gcs', api.properties(build=build_gcs, no_goma=True, clobber=False),
      api.buildbucket.ci_build(
          project='dart',
          bucket='ci.sandbox',
          git_repo='https://dart.googlesource.com/monorepo',
          git_ref='refs/heads/main',
      ),
      api.step_data(
          'git rev-parse',
          stdout=api.raw_io
          .output_text('12345abcde12345abcde12345abcde12345abcde\n')
      )
  )