recipe_modules/logs_util/api.py

# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import re
from recipe_engine import recipe_api


class LogUtilsApi(recipe_api.RecipeApi):
  """Utilities to collect logs in a generic way."""

  def initialize_logs_collection(self, env):
    """Initializes log processing.

    The initialization process creates a temp directory and adds it to the
    FLUTTER_LOGS_DIR env variable to make it available to the task steps. Task
    steps can place any files they want to upload to GCS as public logs.

    Args:
      env(dict): Env variables dictionary.
    """
    # Create a temp folder to keep logs until we can upload them to gcs
    # at the end of the execution of the test.
    with self.m.step.nest('Initialize logs'):
      logs_path = self.m.path['cleanup'].join('flutter_logs_dir')
      self.m.file.ensure_directory('Ensure %s' % logs_path, logs_path)
      env['FLUTTER_LOGS_DIR'] = logs_path
      # Ensure that any test outputs, e.g. timelines/timeline summaries are
      # included as logs.
      env['FLUTTER_TEST_OUTPUTS_DIR'] = logs_path
      # Write a noop file to for the creation of the remote folder estructure
      # when there logs folder is empty.
      self.m.file.write_text(
          'Write noop file', logs_path.join('noop.txt'), '', include_log=False
      )

  def upload_logs(self, task):
    """Upload the log files in FLUTTER_LOGS_DIR to GCS.

    Args:
      task(str): A string with the task name the logs belong to.
    """
    git_hash = self.m.buildbucket.gitiles_commit.id
    # gitiles_commit is only populated on post-submits.
    # UUID is used in LED and try jobs.
    uuid = self.m.uuid.random()
    invocation_id = git_hash if git_hash else uuid
    logs_path = self.m.path['cleanup'].join('flutter_logs_dir')
    with self.m.step.nest('process logs'):
      self.m.gsutil.upload(
          bucket='flutter_logs',
          source=logs_path,
          dest='flutter/%s/%s/%s' % (invocation_id, task, uuid),
          link_name='archive logs',
          args=['-r'],
          multithreaded=True,
          name='upload logs %s' % invocation_id,
          unauthenticated_url=True
      )
      log_files = self.m.file.glob_paths(
          'logs', source=logs_path, pattern='*', test_data=['a.txt']
      )
    with self.m.step.nest('log links') as presentation:
      pattern_str = 'https://storage.googleapis.com/%s/flutter/%s/%s/%s/%s'
      log_files = self.m.file.listdir(
          'List logs path', logs_path, True, test_data=('myfile.txt',)
      )
      for log_file in log_files:
        base_name = self.m.path.basename(log_file)

        url = pattern_str % (
            'flutter_logs', invocation_id, task, uuid, base_name
        )
        presentation.links[base_name] = url

  def upload_test_metrics(self, file_path, task_name, git_hash=None):
    """Retrieve the file from the provided path and store it in gcs.

    Upload test metrics only on post-submit. These are metrics about unit test execution time,
    and they will be used to shard tests evenly according to their overall execution time.
    """
    with self.m.step.nest('store metrics'):
      git_hash = self.m.buildbucket.gitiles_commit.id if not git_hash else git_hash
      if not git_hash or not self.m.path.exists(file_path):
        return
      uuid = self.m.uuid.random()
      self.m.gsutil.upload(
        bucket='flutter_logs',
        source=file_path,
        dest='flutter/%s/%s/%s' % (git_hash, task_name, uuid),
        link_name='archive logs',
        args=['-r'],
        multithreaded=True,
        name='upload logs %s' % git_hash,
        unauthenticated_url=True
      )