upload failing screenshots for firefox tests
Change-Id: I1928624db2fe523ad6882f8f428c486de75ba662
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/7420
Reviewed-by: Christopher Fujino <fujino@google.com>
Commit-Queue: Nurhan Turgut <nurhan@google.com>
diff --git a/recipes/web_engine.expected/linux-post-submit.json b/recipes/web_engine.expected/linux-post-submit.json
index 0e56dc2..d0890c2 100644
--- a/recipes/web_engine.expected/linux-post-submit.json
+++ b/recipes/web_engine.expected/linux-post-submit.json
@@ -876,6 +876,44 @@
},
{
"cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "tests for linux",
+ "[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results/info.txt"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter/lib/web_ui",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "CHROME_NO_SANDBOX": "true",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GOMA_DIR": "[CACHE]/goma/client"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "write info file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@info.txt@tests for linux@@@",
+ "@@@STEP_LOG_END@info.txt@@@"
+ ]
+ },
+ {
+ "cmd": [
"cipd",
"ensure",
"-root",
@@ -1126,7 +1164,7 @@
]
},
"infra_step": true,
- "name": "write info file",
+ "name": "write info file (2)",
"~followup_annotations": [
"@@@STEP_LOG_LINE@info.txt@tests for linux@@@",
"@@@STEP_LOG_END@info.txt@@@"
diff --git a/recipes/web_engine.expected/linux-pre-submit.json b/recipes/web_engine.expected/linux-pre-submit.json
index 8535922..6994db5 100644
--- a/recipes/web_engine.expected/linux-pre-submit.json
+++ b/recipes/web_engine.expected/linux-pre-submit.json
@@ -891,6 +891,129 @@
},
{
"cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "copy",
+ "tests for linux",
+ "[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results/info.txt"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter/lib/web_ui",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "CHROME_NO_SANDBOX": "true",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GOMA_DIR": "[CACHE]/goma/client"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "write info file",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@info.txt@tests for linux@@@",
+ "@@@STEP_LOG_END@info.txt@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
+ "--",
+ "RECIPE_REPO[depot_tools]/gsutil.py",
+ "-m",
+ "----",
+ "cp",
+ "-r",
+ "[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results",
+ "gs://mybucket/web_engine/0firefox"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter/lib/web_ui",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "CHROME_NO_SANDBOX": "true",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GOMA_DIR": "[CACHE]/goma/client"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "gsutil upload goldens 0firefox",
+ "~followup_annotations": [
+ "@@@STEP_LINK@archive goldens@https://console.cloud.google.com/storage/browser/mybucket/web_engine/0firefox@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "glob",
+ "[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results",
+ "*.html"
+ ],
+ "cwd": "[CACHE]/builder/src/flutter/lib/web_ui",
+ "env": {
+ "ANDROID_HOME": "[CACHE]/builder/src/third_party/android_tools/sdk",
+ "CHROME_NO_SANDBOX": "true",
+ "ENGINE_PATH": "[CACHE]/builder",
+ "GOMA_DIR": "[CACHE]/goma/client"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[CACHE]/builder/src/third_party/dart/tools/sdks/dart-sdk/bin"
+ ]
+ },
+ "env_suffixes": {
+ "DEPOT_TOOLS_UPDATE": [
+ "0"
+ ],
+ "PATH": [
+ "RECIPE_REPO[depot_tools]"
+ ]
+ },
+ "infra_step": true,
+ "name": "html goldens",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@glob@[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results/a.html@@@",
+ "@@@STEP_LOG_END@glob@@@"
+ ]
+ },
+ {
+ "cmd": [],
+ "name": "Failed golden links",
+ "~followup_annotations": [
+ "@@@STEP_LINK@a.html@https://storage.googleapis.com/mybucket/web_engine/0firefox/a.html@@@"
+ ]
+ },
+ {
+ "cmd": [
"cipd",
"ensure",
"-root",
@@ -1141,7 +1264,7 @@
]
},
"infra_step": true,
- "name": "write info file",
+ "name": "write info file (2)",
"~followup_annotations": [
"@@@STEP_LOG_LINE@info.txt@tests for linux@@@",
"@@@STEP_LOG_END@info.txt@@@"
@@ -1219,7 +1342,7 @@
]
},
"infra_step": true,
- "name": "html goldens",
+ "name": "html goldens (2)",
"~followup_annotations": [
"@@@STEP_LOG_LINE@glob@[CACHE]/builder/src/flutter/lib/web_ui/.dart_tool/test_results/a.html@@@",
"@@@STEP_LOG_END@glob@@@"
@@ -1227,7 +1350,7 @@
},
{
"cmd": [],
- "name": "Failed golden links",
+ "name": "Failed golden links (2)",
"~followup_annotations": [
"@@@STEP_LINK@a.html@https://storage.googleapis.com/mybucket/web_engine/0/a.html@@@"
]
diff --git a/recipes/web_engine.py b/recipes/web_engine.py
index 62e4062..deb3c7a 100644
--- a/recipes/web_engine.py
+++ b/recipes/web_engine.py
@@ -60,6 +60,7 @@
name = 'build %s' % ' '.join([config] + list(targets))
api.step(name, ninja_args)
+
def FormatAndDartTest(api):
checkout = GetCheckoutPath(api)
with api.context(cwd=checkout.join('flutter')):
@@ -89,7 +90,7 @@
checkout = GetCheckoutPath(api)
# Download the driver for Firefox.
firefox_driver_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'drivers', 'firefox')
+ 'drivers', 'firefox')
pkgdriver = api.cipd.EnsureFile()
pkgdriver.add_package('flutter_internal/browser-drivers/firefoxdriver-linux',
'latest')
@@ -110,7 +111,7 @@
# Download the driver for Chrome 84.
chrome_driver_84_path = checkout.join('flutter', 'lib', 'web_ui',
'.dart_tool', 'drivers', 'chrome', '84')
- chrome_pkgdriver_84= api.cipd.EnsureFile()
+ chrome_pkgdriver_84 = api.cipd.EnsureFile()
chrome_pkgdriver_84.add_package(
'flutter_internal/browser-drivers/chrome/${platform}', 'latest-84')
api.cipd.ensure(chrome_driver_84_path, chrome_pkgdriver_84)
@@ -119,19 +120,20 @@
# The binary 741412 has major version 82.
# TODO: remove this version once 84 start working with no issues.
chrome_path_82 = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'chrome', '741412')
+ 'chrome', '741412')
chrome_pkg_82 = api.cipd.EnsureFile()
chrome_pkg_82.add_package('flutter_internal/browsers/chrome-linux', 'latest')
api.cipd.ensure(chrome_path_82, chrome_pkg_82)
# Download the driver for Chrome 82.
# TODO: remove this version once 84 start working with no issues.
- chrome_driver_82_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'drivers', 'chrome')
+ chrome_driver_82_path = checkout.join('flutter', 'lib', 'web_ui',
+ '.dart_tool', 'drivers', 'chrome')
chrome_pkgdriver_82 = api.cipd.EnsureFile()
chrome_pkgdriver_82.add_package(
'flutter_internal/browser-drivers/chromedriver-linux', 'latest')
api.cipd.ensure(chrome_driver_82_path, chrome_pkgdriver_82)
+
def CloneGoldens(api):
builder_root = api.path['cache'].join('builder')
goldens = builder_root.join('goldens')
@@ -160,47 +162,54 @@
revision_number = golden_lock_content['revision']
with api.context(cwd=goldens):
api.git.checkout(
- repo,
- dir_path=goldens,
- ref=revision_number,
- recursive=True,
- set_got_revision=True
- )
+ repo,
+ dir_path=goldens,
+ ref=revision_number,
+ recursive=True,
+ set_got_revision=True)
golden_files = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
'goldens')
api.file.copytree('copy goldens', goldens, golden_files)
-def UploadFailingGoldens(api, checkout):
- logs_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'test_results')
- tests_info_file_path = logs_path.join('info.txt')
- api.file.write_text('write info file',
- tests_info_file_path,
- 'tests for %s' % api.platform.name,
- 'tests for windows',
- )
- if api.properties.get('gcs_goldens_bucket') and not api.runtime.is_experimental:
- api.gsutil.upload(
- bucket=api.properties['gcs_goldens_bucket'],
- source=logs_path,
- dest='%s/%s' % ('web_engine', api.buildbucket.build.id),
- link_name='archive goldens',
- args=['-r'],
- multithreaded=True,
- name='upload goldens %s' % api.buildbucket.build.id,
- unauthenticated_url=True)
- html_files = api.file.glob_paths(
- 'html goldens',
- source=logs_path,
- pattern='*.html',
- test_data=('a.html',)).get_result()
- with api.step.nest('Failed golden links') as presentation:
- for html_file in html_files:
- base_name = api.path.basename(html_file)
- url = 'https://storage.googleapis.com/%s/web_engine/%s/%s' % (
- api.properties['gcs_goldens_bucket'],
- api.buildbucket.build.id, base_name)
- presentation.links[base_name] = url
+
+def UploadFailingGoldens(api, checkout, browser):
+ logs_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
+ 'test_results')
+ tests_info_file_path = logs_path.join('info.txt')
+ api.file.write_text(
+ 'write info file',
+ tests_info_file_path,
+ 'tests for %s' % api.platform.name,
+ 'tests for windows',
+ )
+
+ if api.properties.get('gcs_goldens_bucket'):
+ if browser == 'chrome':
+ bucket_id = api.buildbucket.build.id
+ else:
+ bucket_id = str(api.buildbucket.build.id) + browser
+
+ api.gsutil.upload(
+ bucket=api.properties['gcs_goldens_bucket'],
+ source=logs_path,
+ dest='%s/%s' % ('web_engine', bucket_id),
+ link_name='archive goldens',
+ args=['-r'],
+ multithreaded=True,
+ name='upload goldens %s' % bucket_id,
+ unauthenticated_url=True)
+ html_files = api.file.glob_paths(
+ 'html goldens',
+ source=logs_path,
+ pattern='*.html',
+ test_data=('a.html',)).get_result()
+ with api.step.nest('Failed golden links') as presentation:
+ for html_file in html_files:
+ base_name = api.path.basename(html_file)
+ url = 'https://storage.googleapis.com/%s/web_engine/%s/%s' % (
+ api.properties['gcs_goldens_bucket'], bucket_id, base_name)
+ presentation.links[base_name] = url
+
def RunSteps(api, properties, env_properties):
"""Steps to checkout flutter engine and execute web tests."""
@@ -250,8 +259,9 @@
with api.context(cwd=dart_sdk_dir):
# The default fetch remote is a local dir, so explicitly fetch from
# upstream remote
- api.step('Fetch dart tags',
- ['git', 'fetch', 'https://dart.googlesource.com/sdk.git', '--tags'])
+ api.step(
+ 'Fetch dart tags',
+ ['git', 'fetch', 'https://dart.googlesource.com/sdk.git', '--tags'])
api.step('List all tags', ['git', 'tag', '--list'])
api.gclient.runhooks()
@@ -289,10 +299,10 @@
api.step('felt licenses', felt_licenses)
if api.platform.is_win:
chrome_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'chrome', '768975')
+ 'chrome', '768975')
if api.platform.is_mac:
chrome_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'chrome', '768985')
+ 'chrome', '768985')
additional_args_safari_desktop = ['--browser', 'safari']
felt_test_safari_desktop = copy.deepcopy(felt_cmd)
felt_test_safari_desktop.append('test')
@@ -309,33 +319,37 @@
felt_test_firefox = copy.deepcopy(felt_cmd)
felt_test_firefox.append('test')
felt_test_firefox.extend(additional_args_firefox)
- api.step('felt test firefox', felt_test_firefox)
+ with recipe_api.defer_results():
+ api.step('felt test firefox', felt_test_firefox)
+ UploadFailingGoldens(api, checkout, 'firefox')
chrome_path = checkout.join('flutter', 'lib', 'web_ui', '.dart_tool',
- 'chrome', '768968')
+ 'chrome', '768968')
DownloadChromeAndDriver(api, chrome_path)
felt_test = copy.deepcopy(felt_cmd)
felt_test.append('test')
felt_test.extend(additional_args)
if api.platform.is_mac:
- with SetupXcode(api):
- with recipe_api.defer_results():
- api.step('felt ios-safari test',felt_test)
- UploadFailingGoldens(api, checkout)
+ with SetupXcode(api):
+ with recipe_api.defer_results():
+ api.step('felt ios-safari test', felt_test)
+ UploadFailingGoldens(api, checkout, 'ios-safari')
else:
with recipe_api.defer_results():
api.step('felt test chrome', felt_test)
- UploadFailingGoldens(api, checkout)
+ UploadFailingGoldens(api, checkout, 'chrome')
# Collect memory/cpu/process after task execution.
api.os_utils.collect_os_info()
+
def GenTests(api):
golden_yaml_file = {'repository': 'repo', 'revision': 'b6efc758'}
yield api.test('linux-post-submit') + api.properties(
goma_jobs='200') + api.platform('linux', 64)
yield api.test('windows-post-submit') + api.properties(
goma_jobs='200') + api.platform('win', 32)
- yield api.test('mac-post-submit',
+ yield api.test(
+ 'mac-post-submit',
api.step_data('read yaml.parse', api.json.output(golden_yaml_file)),
api.properties(goma_jobs='200'), api.platform('mac', 64))
yield api.test('linux-pre-submit') + api.properties(
