Ignore infra failures from FTL
Change-Id: I286d5e86bb70388512d8d07e56e73a579a1f4209
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/41606
Commit-Queue: Dan Field <dnfield@google.com>
Reviewed-by: Zach Anderson <zra@google.com>
Reviewed-by: Keyong Han <keyonghan@google.com>
diff --git a/recipes/firebaselab/firebaselab.expected/failure 10.json b/recipes/firebaselab/firebaselab.expected/failure 10.json
index 6f0c2cb..488fdcd 100644
--- a/recipes/firebaselab/firebaselab.expected/failure 10.json
+++ b/recipes/firebaselab/firebaselab.expected/failure 10.json
@@ -177,8 +177,6 @@
"cmd": [],
"name": "test_execution",
"~followup_annotations": [
- "@@@STEP_LOG_LINE@logcat@@@@",
- "@@@STEP_LOG_END@logcat@@@",
"@@@STEP_FAILURE@@@"
]
},
@@ -429,237 +427,9 @@
]
},
{
- "cmd": [
- "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/bin/gcloud",
- "firebase",
- "test",
- "android",
- "run",
- "--type",
- "robo",
- "--app",
- "build/app/outputs/bundle/release/app-release.aab",
- "--timeout",
- "2m",
- "--results-bucket=gs://flutter_firebase_testlab_staging",
- "--results-dir=None/fake-task-id",
- "--device",
- "model=redfin,version=30",
- "--device",
- "model=griffin,version=24"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "test_execution.gcloud firebase (2)",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "flutter",
- "build",
- "apk",
- "--debug",
- "--target-platform",
- "android-x86"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "test_execution.Build apk",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/bin/gcloud",
- "firebase",
- "test",
- "android",
- "run",
- "--type",
- "robo",
- "--app",
- "build/app/outputs/flutter-apk/app-debug.apk",
- "--timeout",
- "2m",
- "--results-bucket=gs://flutter_firebase_testlab_staging",
- "--results-dir=None/fake-task-id",
- "--device",
- "model=Nexus5,version=21",
- "--device",
- "model=Nexus5,version=22",
- "--device",
- "model=Nexus5,version=23",
- "--device",
- "model=Nexus6P,version=26",
- "--device",
- "model=Nexus6P,version=27",
- "--device",
- "model=NexusLowRes,version=29"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "test_execution.gcloud firebase (3)",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "python3",
- "-u",
- "RECIPE_MODULE[depot_tools::gsutil]/resources/gsutil_smart_retry.py",
- "--",
- "RECIPE_REPO[depot_tools]/gsutil.py",
- "----",
- "cp",
- "gs://flutter_firebase_testlab_staging/None/fake-task-id/*/logcat",
- "[CLEANUP]"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "test_execution.gsutil download",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
- ]
- },
- {
- "cmd": [
- "vpython3",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "copy",
- "[CLEANUP]/logcat",
- "/path/to/tmp/"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "infra_step": true,
- "name": "test_execution.read",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_LOG_END@logcat@@@"
- ]
- },
- {
- "cmd": [
- "grep",
- "E/flutter",
- "[CLEANUP]/logcat"
- ],
- "cwd": "[START_DIR]/flutter/dev/integration_tests",
- "env": {
- "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
- "GIT_BRANCH": "",
- "LUCI_BRANCH": "",
- "LUCI_CI": "True",
- "LUCI_PR": "",
- "OS": "linux",
- "PUB_CACHE": "[START_DIR]/.pub-cache",
- "REVISION": "12345abcde12345abcde12345abcde12345abcde",
- "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
- },
- "env_prefixes": {
- "PATH": [
- "[START_DIR]/flutter/bin",
- "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
- ]
- },
- "name": "test_execution.analyze_logcat",
- "~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@",
- "@@@STEP_FAILURE@@@"
- ]
- },
- {
"failure": {
"failure": {},
- "humanReason": "Step('test_execution.analyze_logcat') (retcode: 0)"
+ "humanReason": "Step('test_execution.gcloud firebase') (retcode: 10)"
},
"name": "$result"
}
diff --git a/recipes/firebaselab/firebaselab.expected/failure 15.json b/recipes/firebaselab/firebaselab.expected/failure 15.json
index 6f0c2cb..f6b7416 100644
--- a/recipes/firebaselab/firebaselab.expected/failure 15.json
+++ b/recipes/firebaselab/firebaselab.expected/failure 15.json
@@ -468,7 +468,52 @@
},
"name": "test_execution.gcloud firebase (2)",
"~followup_annotations": [
- "@@@STEP_NEST_LEVEL@1@@@"
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[START_DIR]/cipd_tool/path/to/gcloud/version%3Apinned-version/bin/gcloud",
+ "firebase",
+ "test",
+ "android",
+ "run",
+ "--type",
+ "robo",
+ "--app",
+ "build/app/outputs/bundle/release/app-release.aab",
+ "--timeout",
+ "2m",
+ "--results-bucket=gs://flutter_firebase_testlab_staging",
+ "--results-dir=None/fake-task-id",
+ "--device",
+ "model=redfin,version=30",
+ "--device",
+ "model=griffin,version=24"
+ ],
+ "cwd": "[START_DIR]/flutter/dev/integration_tests",
+ "env": {
+ "DEPOT_TOOLS": "RECIPE_REPO[depot_tools]",
+ "GIT_BRANCH": "",
+ "LUCI_BRANCH": "",
+ "LUCI_CI": "True",
+ "LUCI_PR": "",
+ "OS": "linux",
+ "PUB_CACHE": "[START_DIR]/.pub-cache",
+ "REVISION": "12345abcde12345abcde12345abcde12345abcde",
+ "SDK_CHECKOUT_PATH": "[START_DIR]/flutter"
+ },
+ "env_prefixes": {
+ "PATH": [
+ "[START_DIR]/flutter/bin",
+ "[START_DIR]/flutter/bin/cache/dart-sdk/bin"
+ ]
+ },
+ "name": "test_execution.gcloud firebase (3)",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_FAILURE@@@"
]
},
{
@@ -549,7 +594,7 @@
"[START_DIR]/flutter/bin/cache/dart-sdk/bin"
]
},
- "name": "test_execution.gcloud firebase (3)",
+ "name": "test_execution.gcloud firebase (4)",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
diff --git a/recipes/firebaselab/firebaselab.py b/recipes/firebaselab/firebaselab.py
index 3534d65..ab7f197 100644
--- a/recipes/firebaselab/firebaselab.py
+++ b/recipes/firebaselab/firebaselab.py
@@ -125,7 +125,18 @@
def run_firebase():
return api.gcloud(*firebase_cmd)
- api.retry.wrap(run_firebase, max_attempts=3, retriable_ret=(1, 15, 20))
+ # Sometimes, infra failures on the FTL side are persistent. We should
+ # allow CI to pass in that case rather than block the tree.
+ infra_failure_codes = (1, 15, 20)
+ try:
+ api.retry.wrap(run_firebase, max_attempts=3, retriable_codes=infra_failure_codes)
+ except api.step.StepFailure:
+ if api.step.active_result.retcode in infra_failure_codes:
+ # FTL is having some infra outage. Don't block the tree. Still
+ # check logs for pieces that may have passed.
+ pass
+ else:
+ raise
logcat_path = '%s/%s/*/logcat' % (task_name, task_id)
tmp_logcat = api.path['cleanup'].join('logcat')
@@ -144,6 +155,10 @@
yield api.test('failure 15',
api.repo_util.flutter_environment_data()) + api.step_data(
'test_execution.gcloud firebase', retcode=15
+ ) + api.step_data(
+ 'test_execution.gcloud firebase (2)', retcode=15
+ ) + api.step_data(
+ 'test_execution.gcloud firebase (3)', retcode=15
)
yield api.test('failure 10',
