diff --git a/recipe_modules/retry/examples/full.expected/pass_with_retries.json b/recipe_modules/retry/examples/full.expected/pass_with_retries.json
index 1cf71a7..4fc83e5 100644
--- a/recipe_modules/retry/examples/full.expected/pass_with_retries.json
+++ b/recipe_modules/retry/examples/full.expected/pass_with_retries.json
@@ -37,8 +37,8 @@
     "cmd": [],
     "name": "step is flaky: test: mytest",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
@@ -77,8 +77,8 @@
     "cmd": [],
     "name": "step is flaky: nested.test: mytest_func",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipe_modules/test_utils/api.py b/recipe_modules/test_utils/api.py
index 5adf025..fb93fb8 100644
--- a/recipe_modules/test_utils/api.py
+++ b/recipe_modules/test_utils/api.py
@@ -159,7 +159,7 @@
         ) if self._test_data.enabled else logs
         if self._is_flaky(step_stdout):
           test_run_status = 'flaky'
-          self.flaky_step(step_name, step_stdout)
+          self.flaky_step(step_name, logs_file)
         else:
           test_run_status = 'success'
       except self.m.step.StepFailure as f:
@@ -185,14 +185,14 @@
     """
     return 'test: %s' % step_name
 
-  def flaky_step(self, step_name, stdout=''):
+  def flaky_step(self, step_name, stdout=None):
     """Add a flaky step when test is flaky.
     Args:
       step_name(str): The name of the step.
     """
     with self.m.step.nest('step is flaky: %s' % step_name) as presentation:
       presentation.links['stdout'] = stdout
-      presentation.status = self.m.step.WARNING
+      presentation.status = self.m.step.FAILURE
 
   def collect_benchmark_tags(self, env, env_prefixes, target_tags):
     """Collect host and device tags for devicelab benchmarks.
diff --git a/recipe_modules/test_utils/examples/full.expected/flaky.json b/recipe_modules/test_utils/examples/full.expected/flaky.json
index 4368bc0..2868667 100644
--- a/recipe_modules/test_utils/examples/full.expected/flaky.json
+++ b/recipe_modules/test_utils/examples/full.expected/flaky.json
@@ -36,16 +36,16 @@
     "cmd": [],
     "name": "step is flaky: mytest",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@#flaky\nthis%20is%20a%20flaky\nflaky:%20true@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@[CLEANUP]/tmp_tmp_1@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
     "cmd": [],
     "name": "step is flaky: test step",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipe_modules/test_utils/examples/full.expected/passing-mac.json b/recipe_modules/test_utils/examples/full.expected/passing-mac.json
index 0e17099..515176a 100644
--- a/recipe_modules/test_utils/examples/full.expected/passing-mac.json
+++ b/recipe_modules/test_utils/examples/full.expected/passing-mac.json
@@ -36,8 +36,8 @@
     "cmd": [],
     "name": "step is flaky: test step",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipe_modules/test_utils/examples/full.expected/passing.json b/recipe_modules/test_utils/examples/full.expected/passing.json
index 1350502..09c4369 100644
--- a/recipe_modules/test_utils/examples/full.expected/passing.json
+++ b/recipe_modules/test_utils/examples/full.expected/passing.json
@@ -36,8 +36,8 @@
     "cmd": [],
     "name": "step is flaky: test step",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipes/devicelab/devicelab_drone.expected/xcode-devicelab.json b/recipes/devicelab/devicelab_drone.expected/xcode-devicelab.json
index 0a60d28..de436ea 100644
--- a/recipes/devicelab/devicelab_drone.expected/xcode-devicelab.json
+++ b/recipes/devicelab/devicelab_drone.expected/xcode-devicelab.json
@@ -2542,8 +2542,8 @@
     "cmd": [],
     "name": "step is flaky: run abc",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@#flaky\nthis%20is%20a%20flaky\nflaky:%20true@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@[CLEANUP]/tmp_tmp_2@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
@@ -3223,8 +3223,8 @@
     "cmd": [],
     "name": "step is flaky: run abc (2)",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipes/devicelab/devicelab_test_drone.expected/basic.json b/recipes/devicelab/devicelab_test_drone.expected/basic.json
index d33245b..16f0823 100644
--- a/recipes/devicelab/devicelab_test_drone.expected/basic.json
+++ b/recipes/devicelab/devicelab_test_drone.expected/basic.json
@@ -622,8 +622,8 @@
     "cmd": [],
     "name": "step is flaky: run abc",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@#flaky\nthis%20is%20a%20flaky\nflaky:%20true@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@[CLEANUP]/tmp_tmp_3@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
@@ -1086,8 +1086,8 @@
     "cmd": [],
     "name": "step is flaky: run abc (2)",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipes/devicelab/devicelab_test_drone.expected/experimental.json b/recipes/devicelab/devicelab_test_drone.expected/experimental.json
index 4c14b6c..9f8a2fe 100644
--- a/recipes/devicelab/devicelab_test_drone.expected/experimental.json
+++ b/recipes/devicelab/devicelab_test_drone.expected/experimental.json
@@ -622,8 +622,8 @@
     "cmd": [],
     "name": "step is flaky: run abc",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@#flaky\nthis%20is%20a%20flaky\nflaky:%20true@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@[CLEANUP]/tmp_tmp_3@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
@@ -1086,8 +1086,8 @@
     "cmd": [],
     "name": "step is flaky: run abc (2)",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
diff --git a/recipes/devicelab/devicelab_test_drone.expected/mac.json b/recipes/devicelab/devicelab_test_drone.expected/mac.json
index 178085d..0ef92cf 100644
--- a/recipes/devicelab/devicelab_test_drone.expected/mac.json
+++ b/recipes/devicelab/devicelab_test_drone.expected/mac.json
@@ -1352,8 +1352,8 @@
     "cmd": [],
     "name": "step is flaky: run abc",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@#flaky\nthis%20is%20a%20flaky\nflaky:%20true@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@[CLEANUP]/tmp_tmp_5@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {
@@ -2047,8 +2047,8 @@
     "cmd": [],
     "name": "step is flaky: run abc (2)",
     "~followup_annotations": [
-      "@@@STEP_LINK@stdout@@@@",
-      "@@@STEP_WARNINGS@@@"
+      "@@@STEP_LINK@stdout@None@@@",
+      "@@@STEP_FAILURE@@@"
     ]
   },
   {