Use new defer module
Also, don't use any deferred logic in some places, like where there was
only one command being run.
Bug: chromium:1495428
Change-Id: If1221538c7b763a7215f2481c3635031e2b64115
Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/51924
Reviewed-by: Keyong Han <keyonghan@google.com>
Commit-Queue: Rob Mohr <mohrr@google.com>
diff --git a/recipe_modules/goma/__init__.py b/recipe_modules/goma/__init__.py
index 0ee03a0..df453e1 100644
--- a/recipe_modules/goma/__init__.py
+++ b/recipe_modules/goma/__init__.py
@@ -10,6 +10,7 @@
"recipe_engine/buildbucket",
"recipe_engine/cipd",
"recipe_engine/context",
+ "recipe_engine/defer",
"recipe_engine/file",
"recipe_engine/json",
"recipe_engine/path",
diff --git a/recipe_modules/goma/api.py b/recipe_modules/goma/api.py
index 6e3eda5..73a0402 100644
--- a/recipe_modules/goma/api.py
+++ b/recipe_modules/goma/api.py
@@ -169,19 +169,21 @@
self._goma_ctl("start goma", ["restart"])
self._goma_started = True
except self.m.step.StepFailure: # pragma: no cover
- with self.m.step.defer_results():
- self._run_jsonstatus()
- self._goma_ctl("stop goma (start failure)", ["stop"])
+ deferred = []
+ deferred.append(self.m.defer(self._run_jsonstatus))
+ deferred.append(self.m.defer(self._goma_ctl, "stop goma (start failure)", ["stop"]))
+ self.m.defer.collect(deferred)
raise
def _stop(self):
"""Stop goma compiler proxy."""
assert self._goma_started
- with self.m.step.defer_results():
- self._run_jsonstatus()
- self._goma_ctl("goma stats", ["stat"])
- self._goma_ctl("stop goma", ["stop"])
+ deferred = []
+ deferred.append(self.m.defer(self._run_jsonstatus))
+ deferred.append(self.m.defer(self._goma_ctl, "goma stats", ["stat"]))
+ deferred.append(self.m.defer(self._goma_ctl, "stop goma", ["stop"]))
+ self.m.defer.collect(deferred)
self._goma_started = False
diff --git a/recipe_modules/goma/tests/full.expected/linux_stop_goma_failed.json b/recipe_modules/goma/tests/full.expected/linux_stop_goma_failed.json
index 3154436..fdf3e70 100644
--- a/recipe_modules/goma/tests/full.expected/linux_stop_goma_failed.json
+++ b/recipe_modules/goma/tests/full.expected/linux_stop_goma_failed.json
@@ -282,8 +282,38 @@
]
},
{
+ "cmd": [],
+ "name": "teardown goma.collect",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_TEXT@1 deferred failures@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@Traceback (most recent call last):@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_modules/defer/api.py\", in __call__@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return DeferredResult(_api=self.m, _value=func(*args, **kwargs))@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in _inner@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return func(*a, **kw)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[flutter]/recipe_modules/goma/api.py\", line 117, in _goma_ctl@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return self.m.python3(@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in _inner@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return func(*a, **kw)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[fuchsia]/recipe_modules/python3/api.py\", in __call__@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return self.m.step(name, cmd, **kwargs)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_engine/recipe_api.py\", in _inner@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return func(*a, **kw)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_modules/step/api.py\", in __call__@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return self._run_or_raise_step(@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_modules/step/api.py\", in _run_or_raise_step@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ return self._raise_on_disallowed_statuses(ret, allowed_statuses)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ File \"RECIPE_REPO[recipe_engine]/recipe_modules/step/api.py\", in _raise_on_disallowed_statuses@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@ raise exc('.'.join(result.name_tokens), result)@@@",
+ "@@@STEP_LOG_LINE@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@recipe_engine.recipe_api.InfraFailure: Infra Failure: Step('teardown goma.stop goma') (retcode: 1)@@@",
+ "@@@STEP_LOG_END@InfraFailure(\"Infra Failure: Step('teardown goma.stop goma') (retcode: 1)\")@@@",
+ "@@@STEP_FAILURE@@@"
+ ]
+ },
+ {
"failure": {
- "humanReason": "1 out of 3 aggregated steps failed: Infra Failure: Step('teardown goma.stop goma') (retcode: 1)"
+ "humanReason": "Infra Failure: Step('teardown goma.stop goma') (retcode: 1)"
},
"name": "$result"
}
diff --git a/recipes/engine/engine_builder.py b/recipes/engine/engine_builder.py
index 8d86b37..c6dd7db 100644
--- a/recipes/engine/engine_builder.py
+++ b/recipes/engine/engine_builder.py
@@ -17,6 +17,7 @@
'recipe_engine/buildbucket',
'recipe_engine/cas',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/platform',
@@ -55,7 +56,7 @@
api.step('gn %s' % ' '.join(args), gn_cmd)
else:
# Run gn within a context.
- env = {'GOMA_DIR': api.goma.goma_dir.get_result()}
+ env = {'GOMA_DIR': api.goma.goma_dir}
with api.context(env=env):
api.step('gn %s' % ' '.join(args), gn_cmd)
@@ -79,13 +80,19 @@
output_files = []
output_dirs = []
- with api.osx_sdk('ios'), api.depot_tools.on_path(), api.context(
- env=env), api.step.defer_results():
+ with api.osx_sdk('ios'), api.depot_tools.on_path(), api.context(env=env):
+ deferred = []
for build in properties.builds:
with api.step.nest('build %s (%s)' %
(build.dir, ','.join(build.targets))):
- RunGN(api, build.disable_goma, *build.gn_args)
- Build(api, build.dir, build.disable_goma, *build.targets)
+ deferred.append(
+ api.defer(RunGN, api, build.disable_goma, *build.gn_args)
+ )
+ deferred.append(
+ api.defer(
+ Build, api, build.dir, build.disable_goma, *build.targets
+ )
+ )
for output_file in build.output_files:
output_files.append(
cache_root.join('src', 'out', build.dir, output_file)
@@ -95,9 +102,10 @@
cache_root.join('src', 'out', build.dir, output_dir)
)
# This is to clean up leaked processes.
- api.os_utils.kill_processes()
+ deferred.append(api.defer(api.os_utils.kill_processes))
# Collect memory/cpu/process after task execution.
- api.os_utils.collect_os_info()
+ deferred.append(api.defer(api.os_utils.collect_os_info))
+ api.defer.collect(deferred)
cas_hash = CasOutputs(api, output_files, output_dirs)
output_props = api.step('Set output properties', None)
diff --git a/recipes/engine/femu_test.py b/recipes/engine/femu_test.py
index 5e75a5f..20e0224 100644
--- a/recipes/engine/femu_test.py
+++ b/recipes/engine/femu_test.py
@@ -255,8 +255,7 @@
# TODO(http://fxb/121613): Emulator instances are not cleaned up
# when tests fail. Added a clean up step before tests start to
# stop all running emulators.
- with api.step.defer_results():
- api.retry.step('run ffx test', [ffx] + suite['test_command'].split(' '))
+ api.retry.step('run ffx test', [ffx] + suite['test_command'].split(' '))
def ReadLogFiles(api, ffx):
diff --git a/recipes/flutter/android_views.py b/recipes/flutter/android_views.py
index f396b6f..acfe365 100644
--- a/recipes/flutter/android_views.py
+++ b/recipes/flutter/android_views.py
@@ -16,6 +16,7 @@
'flutter/repo_util',
'flutter/retry',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/properties',
@@ -62,25 +63,40 @@
views_test_dir = checkout_path.join(
'dev', 'integration_tests', 'android_views'
)
- with api.step.nest('prepare environment'), api.step.defer_results():
+ with api.step.nest('prepare environment'):
+ deferred = []
# This prevents junk analytics from being sent due to testing
- api.step(
- 'flutter config --no-analytics',
- ['flutter', 'config', '--no-analytics'],
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter config --no-analytics',
+ ['flutter', 'config', '--no-analytics'],
+ )
)
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor'],
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter doctor',
+ ['flutter', 'doctor'],
+ )
)
- api.step(
- 'flutter devices',
- ['flutter', 'devices', '--device-timeout=40', '--verbose'],
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter devices',
+ ['flutter', 'devices', '--device-timeout=40', '--verbose'],
+ )
)
- api.step(
- 'download flutter dependencies',
- ['flutter', 'update-packages', '-v'],
- infra_step=True,
+ deferred.append(
+ api.defer(
+ api.step,
+ 'download flutter dependencies',
+ ['flutter', 'update-packages', '-v'],
+ infra_step=True,
+ )
)
+ api.defer.collect(deferred)
+
# Create gradlew file
with api.context(env=env, env_prefixes=env_prefixes, cwd=views_test_dir):
api.step(
@@ -102,8 +118,7 @@
max_attempts=2,
infra_step=True,
)
- with api.context(env=env, env_prefixes=env_prefixes,
- cwd=views_test_dir), api.step.defer_results():
+ with api.context(env=env, env_prefixes=env_prefixes, cwd=views_test_dir):
api.step(
'Android Views Integration Tests',
[
diff --git a/recipes/flutter/coverage.py b/recipes/flutter/coverage.py
index 922acee..8b91d5b 100644
--- a/recipes/flutter/coverage.py
+++ b/recipes/flutter/coverage.py
@@ -7,6 +7,7 @@
'flutter/flutter_deps',
'flutter/repo_util',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
@@ -31,16 +32,20 @@
packages_path = checkout_path.join('packages', 'flutter')
with api.context(env=env, env_prefixes=env_prefixes, cwd=packages_path):
- with api.step.nest('prepare environment'), api.step.defer_results():
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor'],
+ with api.step.nest('prepare environment'):
+ deferred = []
+ deferred.append(
+ api.defer(api.step, 'flutter doctor',['flutter', 'doctor'])
)
- api.step(
- 'download dependencies',
- ['flutter', 'update-packages', '-v'],
- infra_step=True,
+ deferred.append(
+ api.defer(
+ api.step,
+ 'download dependencies',
+ ['flutter', 'update-packages', '-v'],
+ infra_step=True,
+ )
)
+ api.defer.collect(deferred)
api.step(
'flutter coverage',
diff --git a/recipes/flutter/deferred_components.py b/recipes/flutter/deferred_components.py
index 700a587..f85ede1 100644
--- a/recipes/flutter/deferred_components.py
+++ b/recipes/flutter/deferred_components.py
@@ -19,6 +19,7 @@
'flutter/repo_util',
'recipe_engine/cipd',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/properties',
@@ -62,46 +63,72 @@
bundletool_dir = cache_root.join('bundletool')
bundletool_jar = bundletool_dir.join('bundletool.jar')
with api.context(env=env, env_prefixes=env_prefixes, cwd=checkout_path):
- with api.step.nest('prepare environment'), api.step.defer_results():
+ with api.step.nest('prepare environment'):
+ deferred = []
# This prevents junk analytics from being sent due to testing
- api.step(
- 'flutter config --no-analytics',
- ['flutter', 'config', '--no-analytics'],
- )
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor'],
- )
- api.step(
- 'flutter devices',
- ['flutter', 'devices', '--device-timeout=40', '--verbose'],
- )
- api.step(
- 'download dependencies',
- ['flutter', 'update-packages', '-v'],
- infra_step=True,
- )
- api.cipd.ensure(
- bundletool_dir,
- api.cipd.EnsureFile().add_package(
- 'flutter/android/bundletool',
- '0xeDa85nRhdQfi3iN2dK8PPluwI73z9San_Afuj3CfgC'
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter config --no-analytics',
+ ['flutter', 'config', '--no-analytics'],
)
)
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter doctor',
+ ['flutter', 'doctor'],
+ )
+ )
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter devices',
+ ['flutter', 'devices', '--device-timeout=40', '--verbose'],
+ )
+ )
+ deferred.append(
+ api.defer(
+ api.step,
+ 'download dependencies',
+ ['flutter', 'update-packages', '-v'],
+ infra_step=True,
+ )
+ )
+ deferred.append(
+ api.defer(
+ api.cipd.ensure,
+ bundletool_dir,
+ api.cipd.EnsureFile().add_package(
+ 'flutter/android/bundletool',
+ '0xeDa85nRhdQfi3iN2dK8PPluwI73z9San_Afuj3CfgC'
+ )
+ )
+ )
+ api.defer.collect(deferred)
+
test_dir = checkout_path.join(
'dev', 'integration_tests', 'deferred_components_test'
)
- with api.context(env=env, env_prefixes=env_prefixes,
- cwd=test_dir), api.step.defer_results():
+ with api.context(env=env, env_prefixes=env_prefixes, cwd=test_dir):
+ deferred = []
# These assets are not allowed to be checked into the repo,
# so they are downloaded separately here.
- api.step('download assets script', ['./download_assets.sh'])
- api.step(
- 'Deferred components release tests',
- ['./run_release_test.sh',
- str(bundletool_jar), env['ADB_PATH']],
- timeout=700,
+ deferred.append(
+ api.defer(
+ api.step, 'download assets script', ['./download_assets.sh']
+ )
)
+ deferred.append(
+ api.defer(
+ api.step,
+ 'Deferred components release tests',
+ ['./run_release_test.sh',
+ str(bundletool_jar), env['ADB_PATH']],
+ timeout=700,
+ )
+ )
+ api.defer.collect(deferred)
# This is to clean up leaked processes.
api.os_utils.kill_processes()
diff --git a/recipes/flutter/flutter.py b/recipes/flutter/flutter.py
index 1922c8e..962ef44 100644
--- a/recipes/flutter/flutter.py
+++ b/recipes/flutter/flutter.py
@@ -19,6 +19,7 @@
'flutter/os_utils',
'flutter/repo_util',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
@@ -50,26 +51,35 @@
env, env_prefixes, api.properties.get('dependencies', [])
)
with api.context(env=env, env_prefixes=env_prefixes, cwd=checkout_path):
- with api.step.nest('prepare environment'), api.step.defer_results():
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor'],
+ with api.step.nest('prepare environment'):
+ deferred = []
+ deferred.append(
+ api.defer(api.step, 'flutter doctor', ['flutter', 'doctor'])
)
- api.step(
- 'download dependencies',
- ['flutter', 'update-packages', '-v'],
- infra_step=True,
+ deferred.append(
+ api.defer(
+ api.step,
+ 'download dependencies',
+ ['flutter', 'update-packages', '-v'],
+ infra_step=True,
+ )
)
- with api.step.defer_results():
- api.adhoc_validation.run(
- api.properties.get('validation_name'),
- api.properties.get('validation'), env, env_prefixes,
- api.properties.get('secrets', {})
- )
- # This is to clean up leaked processes.
- api.os_utils.kill_processes()
- # Collect memory/cpu/process after task execution.
- api.os_utils.collect_os_info()
+ api.defer.collect(deferred)
+
+ deferred = []
+ deferred.append(
+ api.defer(
+ api.adhoc_validation.run,
+ api.properties.get('validation_name'),
+ api.properties.get('validation'), env, env_prefixes,
+ api.properties.get('secrets', {})
+ )
+ )
+ # This is to clean up leaked processes.
+ deferred.append(api.defer(api.os_utils.kill_processes))
+ # Collect memory/cpu/process after task execution.
+ deferred.append(api.defer(api.os_utils.collect_os_info))
+ api.defer.collect(deferred)
def GenTests(api):
diff --git a/recipes/flutter/flutter_drone.py b/recipes/flutter/flutter_drone.py
index 7ec3169..19da867 100644
--- a/recipes/flutter/flutter_drone.py
+++ b/recipes/flutter/flutter_drone.py
@@ -21,6 +21,7 @@
'flutter/test_utils',
'flutter/token_util',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/step',
@@ -97,27 +98,35 @@
# Load local engine information if available.
api.flutter_deps.flutter_engine(env, env_prefixes)
if api.properties.get('$flutter/osx_sdk'):
- with api.osx_sdk('ios'), api.step.defer_results():
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor', '-v'],
+ with api.osx_sdk('ios'):
+ deferred = []
+ deferred.append(
+ api.defer(
+ api.step,
+ 'flutter doctor',
+ ['flutter', 'doctor', '-v'],
+ )
)
- RunShard(api, env, env_prefixes, checkout_path)
+ deferred.append(
+ api.defer(RunShard, api, env, env_prefixes, checkout_path))
# This is to clean up leaked processes.
- api.os_utils.kill_processes()
+ deferred.append(api.defer(api.os_utils.kill_processes))
# Collect memory/cpu/process after task execution.
- api.os_utils.collect_os_info()
+ deferred.append(api.defer(api.os_utils.collect_os_info))
+ api.defer.collect(deferred)
else:
- with api.step.defer_results():
- api.step(
- 'flutter doctor',
- ['flutter', 'doctor', '-v'],
- )
- RunShard(api, env, env_prefixes, checkout_path)
- # This is to clean up leaked processes.
- api.os_utils.kill_processes()
- # Collect memory/cpu/process after task execution.
- api.os_utils.collect_os_info()
+ deferred = []
+ deferred.append(
+ api.defer(api.step,
+ 'flutter doctor',
+ ['flutter', 'doctor', '-v']))
+ deferred.append(
+ api.defer(RunShard, api, env, env_prefixes, checkout_path))
+ # This is to clean up leaked processes.
+ deferred.append(api.defer(api.os_utils.kill_processes))
+ # Collect memory/cpu/process after task execution.
+ deferred.append(api.defer(api.os_utils.collect_os_info))
+ api.defer.collect(deferred)
def GenTests(api):
diff --git a/recipes/recipes.py b/recipes/recipes.py
index 5ce43f5..78fec69 100755
--- a/recipes/recipes.py
+++ b/recipes/recipes.py
@@ -19,6 +19,7 @@
'fuchsia/git_checkout',
'recipe_engine/buildbucket',
'recipe_engine/context',
+ 'recipe_engine/defer',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/properties',
@@ -106,9 +107,12 @@
with api.context(cwd=checkout_path):
api.git('log', 'log', '--oneline', '-n', '10')
api.recipe_testing.projects = ('flutter',)
- with api.step.defer_results():
- api.recipe_testing.run_lint(checkout_path)
- api.recipe_testing.run_unit_tests(checkout_path)
+
+ deferred = []
+ deferred.append(api.defer(api.recipe_testing.run_lint, checkout_path))
+ deferred.append(api.defer(api.recipe_testing.run_unit_tests, checkout_path))
+ api.defer.collect(deferred)
+
if not unittest_only:
flutter = options_pb2.Project(name='flutter', include_unrestricted=True)
opts = options_pb2.Options(projects=[flutter])
