Add retries to cas uploads.
This is to address failures on windows with hash mismatches.
Bug: https://github.com/flutter/flutter/issues/120372
Change-Id: Ieed73e4c61adb90cc277b240b925838d927f24fa
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/39080
Reviewed-by: Drew Roen <drewroen@google.com>
Commit-Queue: Godofredo Contreras <godofredoc@google.com>
diff --git a/recipe_modules/retry/api.py b/recipe_modules/retry/api.py
index b7658d7..6d9211c 100644
--- a/recipe_modules/retry/api.py
+++ b/recipe_modules/retry/api.py
@@ -73,17 +73,19 @@
sleep (int or float): The initial time to sleep between attempts.
backoff_factor (int or float): The factor by which the sleep time
will be multiplied after each attempt.
+ Returns:
+ The result of executing func.
"""
for attempt in range(max_attempts):
try:
- func()
+ result = func()
# Append an extra step to reflect test flakiness, so that we can easily
# collect flaky test statistics. This can also be used to trigger
# notification when a flake happens.
# This is mainly used for Engine builders for now.
if attempt > 0 and step_name is not None and 'test:' in step_name:
self.m.test_utils.flaky_step(step_name)
- return
+ return result
except self.m.step.StepFailure:
step = self.m.step.active_result
retriable_failure = retriable_codes == 'any' or \
@@ -110,11 +112,13 @@
sleep (int or float): The initial time to sleep between attempts.
backoff_factor (int or float): The factor by which the sleep time
will be multiplied after each attempt.
+ Returns:
+ The result of executing func.
"""
for attempt in range(max_attempts):
try:
- func()
- return
+ result = func()
+ return result
except self.m.step.StepFailure:
if attempt == max_attempts - 1:
raise
diff --git a/recipe_modules/shard_util_v2/__init__.py b/recipe_modules/shard_util_v2/__init__.py
index 1218035..8dbd479 100644
--- a/recipe_modules/shard_util_v2/__init__.py
+++ b/recipe_modules/shard_util_v2/__init__.py
@@ -3,6 +3,7 @@
# found in the LICENSE file.
DEPS = [
+ 'flutter/retry',
'fuchsia/cas_util',
'recipe_engine/buildbucket',
'recipe_engine/cas',
diff --git a/recipe_modules/shard_util_v2/api.py b/recipe_modules/shard_util_v2/api.py
index 10fac45..fe97417 100644
--- a/recipe_modules/shard_util_v2/api.py
+++ b/recipe_modules/shard_util_v2/api.py
@@ -485,4 +485,15 @@
cas_dir = self.m.path.mkdtemp('out-cas-directory')
cas_engine = cas_dir.join(target)
self.m.file.copytree('Copy host_debug_unopt', build_dir, cas_engine)
- return self.m.cas_util.upload(cas_dir, step_name='Archive full build for %s' % target)
+
+ def _upload():
+ return self.m.cas_util.upload(cas_dir, step_name='Archive full build for %s' % target)
+
+ # Windows CAS upload is flaky, hashes are calculated before files are fully synced to disk.
+ return self.m.retry.basic_wrap(
+ _upload,
+ step_name='Archive full build',
+ sleep=10.0,
+ backoff_factor=5,
+ max_attempts=3
+ )
