blob: 3acb2266e05d79382d7cba0f6953a86477a73785 [file] [log] [blame]
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import re
from recipe_engine import recipe_api
class LogUtilsApi(recipe_api.RecipeApi):
"""Utilities to collect logs in a generic way."""
def initialize_logs_collection(self, env):
"""Initializes log processing.
The initialization process creates a temp directory and adds it to the
FLUTTER_LOGS_DIR env variable to make it available to the task steps. Task
steps can place any files they want to upload to GCS as public logs.
Args:
env(dict): Env variables dictionary.
"""
# Create a temp folder to keep logs until we can upload them to gcs
# at the end of the execution of the test.
with self.m.step.nest('Initialize logs'):
logs_path = self.m.path['cleanup'].join('flutter_logs_dir')
self.m.file.ensure_directory('Ensure %s' % logs_path, logs_path)
env['FLUTTER_LOGS_DIR'] = logs_path
# Ensure that any test outputs, e.g. timelines/timeline summaries are
# included as logs.
env['FLUTTER_TEST_OUTPUTS_DIR'] = logs_path
# Write a noop file to for the creation of the remote folder estructure
# when there logs folder is empty.
self.m.file.write_text(
'Write noop file', logs_path.join('noop.txt'), '', include_log=False
)
def upload_logs(self, task):
"""Upload the log files in FLUTTER_LOGS_DIR to GCS.
Args:
task(str): A string with the task name the logs belong to.
"""
git_hash = self.m.buildbucket.gitiles_commit.id
# gitiles_commit is only populated on post-submits.
# UUID is used in LED and try jobs.
uuid = self.m.uuid.random()
invocation_id = git_hash if git_hash else uuid
logs_path = self.m.path['cleanup'].join('flutter_logs_dir')
with self.m.step.nest('process logs'):
self.m.gsutil.upload(
bucket='flutter_logs',
source=logs_path,
dest='flutter/%s/%s/%s' % (invocation_id, task, uuid),
link_name='archive logs',
args=['-r'],
multithreaded=True,
name='upload logs %s' % invocation_id,
unauthenticated_url=True
)
log_files = self.m.file.glob_paths(
'logs', source=logs_path, pattern='*', test_data=['a.txt']
)
with self.m.step.nest('log links') as presentation:
pattern_str = 'https://storage.googleapis.com/%s/flutter/%s/%s/%s/%s'
log_files = self.m.file.listdir(
'List logs path', logs_path, True, test_data=('myfile.txt',)
)
for log_file in log_files:
base_name = self.m.path.basename(log_file)
url = pattern_str % (
'flutter_logs', invocation_id, task, uuid, base_name
)
presentation.links[base_name] = url
def upload_test_metrics(self, file_path, task_name, git_hash=None):
"""Retrieve the file from the provided path and store it in gcs.
Upload test metrics only on post-submit. These are metrics about unit test execution time,
and they will be used to shard tests evenly according to their overall execution time.
"""
with self.m.step.nest('store metrics'):
git_hash = self.m.buildbucket.gitiles_commit.id if not git_hash else git_hash
if not git_hash or not self.m.path.exists(file_path):
return
uuid = self.m.uuid.random()
self.m.gsutil.upload(
bucket='flutter_logs',
source=file_path,
dest='flutter/%s/%s/%s' % (git_hash, task_name, uuid),
link_name='archive logs',
args=['-r'],
multithreaded=True,
name='upload logs %s' % git_hash,
unauthenticated_url=True
)