blob: 153de2fe453a2eef31bd517881dc174b8eeee5ba [file] [log] [blame] [edit]
# Copyright 2020 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import re
from recipe_engine import recipe_api
# The maximum number of characters to be included in the summary markdown.
# Even though the max size for the markdown is 4000 bytes we are saving 500
# bytes for addittional prefixes added automatically by LUCI like the number
# of failed steps out of the total.
MAX_CHARS = 3500
# Default timeout for tests seconds
TIMEOUT_SECS = 3600
# Map between iphone identifier and generation name.
IDENTIFIER_NAME_MAP = {
'iPhone1,1': 'iPhone',
'iPhone1,2': 'iPhone 3G',
'iPhone2,1': 'iPhone 3GS',
'iPhone3,1': 'iPhone 4',
'iPhone3,2': 'iPhone 4',
'iPhone3,3': 'iPhone 4',
'iPhone4,1': 'iPhone 4S',
'iPhone5,1': 'iPhone 5',
'iPhone5,2': 'iPhone 5',
'iPhone5,3': 'iPhone 5c',
'iPhone5,4': 'iPhone 5c',
'iPhone6,1': 'iPhone 5s',
'iPhone6,2': 'iPhone 5s',
'iPhone7,2': 'iPhone 6',
'iPhone7,1': 'iPhone 6 Plus',
'iPhone8,1': 'iPhone 6s',
'iPhone8,2': 'iPhone 6s Plus',
'iPhone8,4': 'iPhone SE (1st generation)',
'iPhone9,1': 'iPhone 7',
'iPhone9,3': 'iPhone 7',
'iPhone9,2': 'iPhone 7 Plus',
'iPhone9,4': 'iPhone 7 Plus',
'iPhone10,1': 'iPhone 8',
'iPhone10,4': 'iPhone 8',
'iPhone10,2': 'iPhone 8 Plus',
'iPhone10,5': 'iPhone 8 Plus',
'iPhone10,3': 'iPhone X',
'iPhone10,6': 'iPhone X',
'iPhone11,8': 'iPhone XR',
'iPhone11,2': 'iPhone XS',
'iPhone11,6': 'iPhone XS Max',
'iPhone12,1': 'iPhone 11',
'iPhone12,3': 'iPhone 11 Pro',
'iPhone12,5': 'iPhone 11 Pro Max',
'iPhone12,8': 'iPhone SE (2nd generation)',
'iPhone13,1': 'iPhone 12 mini',
'iPhone13,2': 'iPhone 12',
'iPhone13,3': 'iPhone 12 Pro',
'iPhone13,4': 'iPhone 12 Pro Max',
'iPhone14,4': 'iPhone 13 mini',
'iPhone14,5': 'iPhone 13',
'iPhone14,2': 'iPhone 13 Pro',
'iPhone14,3': 'iPhone 13 Pro Max',
}
# Regexp for windows os version number
_WINDOWS_OS_RE = r'\[version (\d+\.\d+)\.(\d+(?:\.\d+|))\]'
class TestUtilsApi(recipe_api.RecipeApi):
"""Utilities to run flutter tests."""
def _truncateString(self, string):
"""Truncate the string to MAX_CHARS"""
byte_count = 0
lines = string.splitlines()
output = []
for line in reversed(lines):
# +1 to account for the \n separators.
byte_count += len(line.encode('utf-8')) + 1
if byte_count >= MAX_CHARS:
break
output.insert(0, line)
return '\n'.join(output)
def _is_flaky(self, output):
"""Check if test step is flaky"""
lines = output.splitlines()
lines.reverse()
# The flakiness status message `flaky: true` is expected to be located at the
# end of the stdout file. Check last 10 lines to make sure it is covered if existing.
for line in lines[:10]:
if 'flaky: true' in line:
return True
return False
def is_devicelab_bot(self):
"""Whether the current bot is a devicelab bot or not."""
return (
str(self.m.swarming.bot_id).startswith('flutter-devicelab') or
str(self.m.swarming.bot_id).startswith('flutter-win')
)
def run_test(self, step_name, command_list, timeout_secs=TIMEOUT_SECS, infra_step=False, suppress_log=False):
"""Recipe's step wrapper to collect stdout and add it to step_summary.
Args:
step_name(str): The name of the step.
command_list(list(str)): A list of strings with the command and
parameters to execute.
timeout_secs(int): The timeout in seconds for this step.
infra_step: mark step as an infra step
suppress_log: flag whether test logs are suppressed.
Returns(str): The status of the test step. A str `flaky` or `success` will
be returned when step succeeds, and an exception will be thrown out when
step fails.
"""
try:
step = self.m.step(
step_name,
command_list,
infra_step=infra_step,
stdout=self.m.raw_io.output_text(),
stderr=self.m.raw_io.output_text(),
timeout=timeout_secs
)
except self.m.step.StepFailure as f:
result = f.result
# Truncate stdout
stdout = self._truncateString(result.stdout)
# Truncate stderr
stderr = self._truncateString(result.stderr)
raise self.m.step.StepFailure('\n\n```%s```\n' % (stdout or stderr))
finally:
if not suppress_log:
self.m.step.active_result.presentation.logs[
'test_stdout'] = self.m.step.active_result.stdout
self.m.step.active_result.presentation.logs[
'test_stderr'] = self.m.step.active_result.stderr
if self._is_flaky(step.stdout):
test_run_status = 'flaky'
else:
test_run_status = 'success'
return test_run_status
def test_step_name(self, step_name):
"""Append keyword test to test step name to be consistent.
Args:
step_name(str): The name of the step.
Returns(str): The test step name prefixed with "test".
"""
return 'test: %s' % step_name
def flaky_step(self, step_name):
"""Add a flaky step when test is flaky.
Args:
step_name(str): The name of the step.
"""
if self.m.platform.is_win:
self.m.step(
'step is flaky: %s' % step_name,
['powershell.exe', 'echo "test run is flaky"'],
infra_step=True,
)
else:
self.m.step(
'step is flaky: %s' % step_name,
['echo', 'test run is flaky'],
infra_step=True,
)
def collect_benchmark_tags(self, env, env_prefixes, target_tags):
"""Collect host and device tags for devicelab benchmarks.
Args:
env(dict): Current environment variables.
env_prefixes(dict): Current environment prefixes variables.
target_tags(list(str)): Builder tags defined in .ci.yaml targets.
Returns:
A dictionary representation of the tag names and values.
Examples:
Linux/android:
{
'arch': 'intel',
'host_type': 'linux',
'device_type': 'Moto G Play'
}
Mac/ios:
{
'arch': 'm1',
'host_type': 'mac',
'device_type': 'iPhone 6s'
}
Windows/android:
{
'arch': 'intel',
'host_type': 'win',
'device_type': 'Moto G Play'
}
"""
device_tags = {}
def _get_tag(step_name, commands):
return self.m.step(
step_name,
commands,
stdout=self.m.raw_io.output_text(),
infra_step=True,
).stdout.rstrip()
# Collect device tags.
#
# Mac/iOS testbeds always have builder_name starting with `Mac_ios`.
# The android tests always have builder_name like `%_android %`.
#
# We may need to support other platforms like desktop, and
# https://github.com/flutter/flutter/issues/92296 to track a more
# generic way to collect device tags.
if 'ios' in target_tags:
with self.m.context(env=env, env_prefixes=env_prefixes):
iphone_identifier = _get_tag(
'Find device type', ['ideviceinfo', '--key', 'ProductType']
)
device_tags['device_type'] = IDENTIFIER_NAME_MAP[iphone_identifier]
elif 'android' in target_tags:
with self.m.context(env=env, env_prefixes=env_prefixes):
device_tags['device_type'] = _get_tag(
'Find device type', ['adb', 'shell', 'getprop', 'ro.product.model']
)
else:
device_tags['device_type'] = 'none'
# Use same `none` as device version to consolidate metric data streams.
# Do not use real distinct values: https://github.com/flutter/flutter/issues/112804
device_tags['device_version'] = 'none'
device_tags['host_type'] = self.m.platform.name
device_tags['arch'] = self.m.platform.arch
return device_tags