recipes/firebaselab/firebaselab.py

# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Recipe to run firebase lab tests.
# This recipe uses the standard flutter dependencies model, a property
# task_name to identify the test to run and two more properties
# physical_devices and virtual_devices to specify the devices to use.

from contextlib import contextmanager
import re

DEPS = [
    'depot_tools/gsutil',
    'flutter/flutter_deps',
    'flutter/os_utils',
    'flutter/repo_util',
    'flutter/retry',
    'fuchsia/gcloud',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/step',
    'recipe_engine/swarming',
]


def RunSteps(api):
  api.os_utils.collect_os_info()
  checkout_path = api.path['start_dir'].join('flutter')
  # Bucket to upload apks and logs.
  gcs_bucket = 'flutter_firebase_testlab_staging'
  # Checkout flutter/flutter.
  checkout_path = api.path['start_dir'].join('flutter')
  api.repo_util.checkout(
      'flutter',
      checkout_path=checkout_path,
      url=api.properties.get('git_url'),
      ref=api.properties.get('git_ref')
  )
  # Install dependencies. E.g. android sdk.
  env, env_prefixes = api.repo_util.flutter_environment(checkout_path)
  deps = api.properties.get('dependencies', [])
  api.flutter_deps.required_deps(env, env_prefixes, deps)

  # Get the name of the integration test to run.
  task_name = api.properties.get('task_name')

  # Initialize to empty lists for use cases when physical_devices or virtual_device
  # properties are not provided.
  default_physical_devices = []
  default_virtual_devices = []

  physical_devices = default_physical_devices if api.properties.get(
      'physical_devices'
  ) is None else api.properties.get('physical_devices')
  virtual_devices = default_virtual_devices if api.properties.get(
      'virtual_devices'
  ) is None else api.properties.get('virtual_devices')

  test_configurations = (
      (
          'Build appbundle', [
              'flutter', 'build', 'appbundle', '--target-platform',
              'android-arm,android-arm64'
          ], 'build/app/outputs/bundle/release/app-release.aab',
          list(physical_devices)
      ),
      # Use apk because if you let the virtual device pick, it may pick an ARM binary and use
      # runtime translation. We have seen runtime crashes in ARM translation that don't seem to
      # occur on physical devices with the same binary. In summary we are trying to make sure
      # the virtual device is getting a specific architecture that is less likely to crash/have issues.
      (
          'Build apk', [
              'flutter', 'build', 'apk', '--debug', '--target-platform',
              'android-x86'
          ], 'build/app/outputs/flutter-apk/app-debug.apk',
          list(virtual_devices)
      ),
  )

  with api.context(env=env, env_prefixes=env_prefixes, cwd=checkout_path):
    # Run flutter doctor and update packages.
    api.step('flutter doctor', ['flutter', 'doctor', '-v'])
    api.step(
        'download dependencies',
        ['flutter', 'update-packages', '-v'],
        infra_step=True,
    )

  test_path = checkout_path.join('dev', 'integration_tests', task_name)
  with api.step.nest('test_execution') as presentation:
    with api.context(env=env, env_prefixes=env_prefixes, cwd=test_path):
      # Collect the task id which is used to generate the logs destination path.
      task_id = api.swarming.task_id
      # Set the GCP project to use.
      api.gcloud(
          '--quiet',
          'config',
          'set',
          'project',
          'flutter-infra-staging',
          infra_step=True,
      )
      for step_name, build_command, binary, devices in test_configurations:
        # Skip running gcloud command if no devices were provided.
        if not devices:
          continue
        # Build the app bundle or apk.
        api.step(step_name, build_command)
        # Run run the binary in firebaselab using the provided device configurations.
        firebase_cmd = [
            'firebase', 'test', 'android', 'run', '--type', 'robo', '--app',
            binary, '--timeout', '2m',
            '--results-bucket=gs://%s' % gcs_bucket,
            '--results-dir=%s/%s' % (task_name, task_id)
        ] + devices

        # See https://firebase.google.com/docs/test-lab/android/command-line#script_exit_codes
        # If the firebase command fails with 1, it's likely an HTTP issue that
        # will resolve on a retry. If it fails on 15 or 20, it's explicitly
        # an infra failure on the FTL side, so we should just retry.
        def run_firebase():
          return api.gcloud(*firebase_cmd)

        # Sometimes, infra failures on the FTL side are persistent. We should
        # allow CI to pass in that case rather than block the tree.
        infra_failure_codes = (1, 15, 20)
        try:
          api.retry.wrap(
              run_firebase, max_attempts=3, retriable_codes=infra_failure_codes
          )
        except api.step.StepFailure:
          if api.step.active_result.retcode in infra_failure_codes:
            # FTL is having some infra outage. Don't block the tree. Still
            # check logs for pieces that may have passed.
            pass
          else:
            raise

      # Download the test logcat files.
      logcat_path = '%s/%s/*/logcat' % (task_name, task_id)
      tmp_logcat = api.path['cleanup'].join('logcat')
      api.gsutil.download(gcs_bucket, logcat_path, api.path['cleanup'])
      # Read the logcat files and add them to the step logs.
      content = api.file.read_text('read', tmp_logcat)
      presentation.logs['logcat'] = content
      # Grep logcat files in search of E/flutter log
      # entries, if found then then fail the test.
      api.step('analyze_logcat', ['grep', 'E/flutter', tmp_logcat], ok_ret=(1,))
  # This is to clean up leaked processes.
  api.os_utils.kill_processes()
  # Collect memory/cpu/process after task execution.
  api.os_utils.collect_os_info()

def GenTests(api):
  physical_devices = ['--device', 'model=redfin,version=30']
  yield api.test(
      'basic',
      api.repo_util.flutter_environment_data(),
      api.properties(task_name='the_task', physical_devices=physical_devices),
      # A return code of 1 from grep means not error messages were
      # found in logcat and the only acceptable return code.
      api.step_data('test_execution.analyze_logcat', retcode=1),
  )
  yield api.test(
      'empty_devices',
      api.repo_util.flutter_environment_data(),
      api.properties(
          task_name='the_task',
          virtual_devices=[],
          physical_devices=physical_devices
      ),
      # A return code of 1 from grep means not error messages were
      # found in logcat and the only acceptable return code.
      api.step_data('test_execution.analyze_logcat', retcode=1),
  )
  yield api.test(
      'succeed_on_infra_failure',
      api.repo_util.flutter_environment_data(),
      api.properties(physical_devices=physical_devices),
      api.step_data('test_execution.gcloud firebase', retcode=15),
      api.step_data('test_execution.gcloud firebase (2)', retcode=15),
      api.step_data('test_execution.gcloud firebase (3)', retcode=15),
      status='FAILURE'
  )
  yield api.test(
      'failure 10',
      api.repo_util.flutter_environment_data(),
      api.properties(physical_devices=physical_devices),
      api.step_data('test_execution.gcloud firebase', retcode=10),
      status='FAILURE'
  )