recipe_modules/repo_util/api.py

# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

REPOS = {
    'flutter':
        'https://chromium.googlesource.com/external/github.com/flutter/flutter',
    'engine':
        'https://chromium.googlesource.com/external/github.com/flutter/engine',
    'cocoon':
        'https://flutter.googlesource.com/mirrors/cocoon',
    'infra':
      'https://flutter.googlesource.com/infra',
    'packages':
        'https://github.com/flutter/packages',
    'plugins':
        'https://flutter.googlesource.com/mirrors/plugins'
}

import re
from recipe_engine import recipe_api


class RepoUtilApi(recipe_api.RecipeApi):
  """Provides utilities to work with flutter repos."""

  def engine_checkout(
      self, checkout_path, env, env_prefixes, clobber=True, custom_vars={}
  ):
    """Checkout code using gclient.

    Args:
      checkout_path(Path): The path to checkout source code and dependencies.
      env(dict): A dictionary with the environment variables to set.
      env_prefixes(dict): A dictionary with the paths to be added to environment variables.
      clobber(bool): A boolean indicating whether the checkout folder should be cleaned.
      custom_vars(dict): A dictionary with custom variable definitions for gclient solution.
    """
    git_url = REPOS['engine']
    git_id = self.m.buildbucket.gitiles_commit.id
    git_ref = self.m.buildbucket.gitiles_commit.ref
    if 'git_url' in self.m.properties and 'git_ref' in self.m.properties:
      git_url = self.m.properties['git_url']
      git_id = self.m.properties['git_ref']
      git_ref = self.m.properties['git_ref']

    # Inner function to execute code a second time in case of failure.
    def _InnerCheckout():
      with self.m.context(env=env, env_prefixes=env_prefixes,
                          cwd=checkout_path), self.m.depot_tools.on_path():
        src_cfg = self.m.gclient.make_config()
        soln = src_cfg.solutions.add()
        soln.name = 'src/flutter'
        soln.url = git_url
        soln.revision = git_id
        soln.custom_vars = custom_vars
        src_cfg.parent_got_revision_mapping['parent_got_revision'
                                           ] = 'got_revision'
        src_cfg.repo_path_map[git_url] = ('src/flutter', git_ref or 'refs/heads/master')
        self.m.gclient.c = src_cfg
        self.m.gclient.c.got_revision_mapping['src/flutter'
                                             ] = 'got_engine_revision'
        self.m.bot_update.ensure_checkout()
        self.m.gclient.runhooks()

    with self.m.step.nest('Checkout source code'):
      try:
        # Run this out of context
        if clobber:
          self.m.file.rmtree('Clobber cache', checkout_path)
          self.m.file.ensure_directory('Ensure checkout cache', checkout_path)
        _InnerCheckout()
      except (self.m.step.StepFailure, self.m.step.InfraFailure):
        # Run this out of context

        # Ensure depot tools is in the path to prevent problems with vpython not
        # being found after a failure.
        with self.m.depot_tools.on_path():
          self.m.file.rmtree('Clobber cache', checkout_path)
          self.m.file.rmtree(
              'Clobber git cache', self.m.path['cache'].join('git')
          )
          self.m.file.ensure_directory('Ensure checkout cache', checkout_path)
        # Now try a second time
        _InnerCheckout()

  def checkout(self, name, checkout_path, url=None, ref=None):
    """Checks out a repo and returns sha1 of checked out revision.

    The supported repository names and their urls are defined in the global
    REPOS variable.

    Args:
      name (str): name of the supported repository.
      checkout_path (Path): directory to clone into.
      url (str): optional url overwrite of the remote repo.
      ref (str): optional ref overwrite to fetch and check out.
    """
    if name not in REPOS:
      raise ValueError('Unsupported repo: %s' % name)
    with self.m.step.nest('Checkout flutter/%s' % name):
      git_url = url or REPOS[name]
      # gitiles_commit.id is more specific than gitiles_commit.ref, which is
      # branch
      git_ref = ref or self.m.buildbucket.gitiles_commit.id or \
          self.m.buildbucket.gitiles_commit.ref or 'refs/heads/master'

      def do_checkout():
        return self.m.git.checkout(
            git_url,
            dir_path=checkout_path,
            ref=git_ref,
            recursive=True,
            set_got_revision=True,
            tags=True
        )

      return self.m.utils.retry(do_checkout, max_attempts=2)

  def get_branch(self, checkout_path):
    """Get git branch for beta and stable channels.

    Post submit tests for release candidate branches pass the channel stable|beta in
    the git_branch property. As the commits are being tested before they are publised
    is not possible to use the channels to checkout the correct versions of dependencies
    from other repositories, furthermore the channels are only applicable to
    flutter/flutter repository. For tests with dependencies on other repositories we are
    using the release candidate branch to check out the equivalent to the commit under test.
    E.g. if the current commit comes from flutter@flutter-2.8-candiddate.16 then we checkout
    plugins@flutter-2.8-candiddate.16.

    To guess the branch name we get the current commit from the checkout and then we find
    all the different branches the commit exist on, if there is a branch that starts with
    flutter- then we assume that's the release candidate branch under test.
    """
    if self.m.properties.get('git_branch', '') in ['beta', 'stable']:
      with self.m.context(cwd=checkout_path):
        commit = self.m.git(
                'rev-parse', 'HEAD',
                stdout=self.m.raw_io.output_text()).stdout.strip()
        branches = self.m.git(
                'branch', '-a', '--contains', commit,
                stdout=self.m.raw_io.output_text()).stdout.splitlines()
        branches = [b.strip().replace('remotes/origin/', '') for b in branches]
        return next(
                iter(filter(lambda branch: branch.startswith('flutter-'), branches)),
                self.m.properties.get('git_branch', '')
        )
    return self.m.properties.get('git_branch', '')

  def flutter_environment(self, checkout_path):
    """Returns env and env_prefixes of an flutter/dart command environment."""
    dart_bin = checkout_path.join('bin', 'cache', 'dart-sdk', 'bin')
    flutter_bin = checkout_path.join('bin')
    # Fail if flutter bin folder does not exist. dart-sdk/bin folder will be
    # available only after running "flutter doctor" and it needs to be run as
    # the first command on the context using the environment.
    if not self.m.path.exists(flutter_bin):
      msg = (
          'flutter bin folders do not exist,'
          'did you forget to checkout flutter repo?'
      )
      self.m.step.empty('Flutter Environment', status=self.m.step.FAILURE, step_text=msg)
    git_ref = self.m.properties.get('git_ref', '')
    pub_cache_path = self.m.path['start_dir'].join('.pub-cache')
    env = {
        # Setup our own pub_cache to not affect other slaves on this machine,
        # and so that the pre-populated pub cache is contained in the package.
        'PUB_CACHE':
            pub_cache_path,
        # Windows Packaging script assumes this is set.
        'DEPOT_TOOLS':
            str(self.m.depot_tools.root),
        'SDK_CHECKOUT_PATH':
            checkout_path,
        'LUCI_CI':
            True,
        'LUCI_PR':
            re.sub('refs\/pull\/|\/head', '', git_ref),
        'LUCI_BRANCH':
            self.m.properties.get('release_ref', '').replace('refs/heads/', ''),
        'GIT_BRANCH': self.get_branch(checkout_path),
        'OS':
            'linux' if self.m.platform.name == 'linux' else
            ('darwin' if self.m.platform.name == 'mac' else 'win'),
        'REVISION': self.m.buildbucket.gitiles_commit.id or ''
    }
    env_prefixes = {'PATH': ['%s' % str(flutter_bin), '%s' % str(dart_bin)]}
    return env, env_prefixes

  def engine_environment(self, checkout_path):
    """Returns env and env_prefixes of an flutter/dart command environment."""
    dart_bin = checkout_path.join('third_party', 'dart', 'tools', 'sdks', 'dart-sdk', 'bin')
    git_ref = self.m.properties.get('git_ref', '')
    android_home = checkout_path.join('third_party', 'android_tools', 'sdk')
    env = {
        # Windows Packaging script assumes this is set.
        'DEPOT_TOOLS':
            str(self.m.depot_tools.root),
        'ENGINE_CHECKOUT_PATH':
            checkout_path,
        'LUCI_CI':
            True,
        'LUCI_PR':
            re.sub('refs\/pull\/|\/head', '', git_ref),
        'LUCI_BRANCH':
            self.m.properties.get('release_ref', '').replace('refs/heads/', ''),
        'GIT_BRANCH': self.m.properties.get('git_branch', ''),
        'OS':
            'linux' if self.m.platform.name == 'linux' else
            ('darwin' if self.m.platform.name == 'mac' else 'win'),
        'ANDROID_HOME': str(android_home),
        'LUCI_WORKDIR': str(self.m.path['start_dir']),
        'REVISION': self.m.buildbucket.gitiles_commit.id or ''
    }
    env_prefixes = {'PATH': ['%s' % str(dart_bin)]}
    return env, env_prefixes

  def sdk_checkout_path(self):
    """Returns the checkoout path of the current flutter_environment.

    Returns(Path): A path object with the current checkout path.
    """
    checkout_path = self.m.context.env.get('SDK_CHECKOUT_PATH')
    assert checkout_path, 'Outside of a flutter_environment?'
    return self.m.path.abs_to_path(checkout_path)