Fix the fail condition by attempting to restart the emulator up to 3 times. This prevents the test from hanging due to the emulator crashing silently (https://b.corp.google.com/issues/319321211). While this is not a fix it should prevent us from hitting the hang. Example of it preventing the fail and the subsequent hang in the setup method: https://ci.chromium.org/ui/p/flutter/builders/prod.shadow/Linux%20Production%20Engine%20Drone/66/overview Bug: https://github.com/flutter/flutter/issues/141292 Change-Id: I83e477a538dccc77aa6b192a891b38b1c2f0dbdd Reviewed-on: https://flutter-review.googlesource.com/c/recipes/+/53961 Commit-Queue: Ricardo Amador <ricardoamador@google.com> Reviewed-by: Yusuf Mohsinally <mohsinally@google.com>
diff --git a/recipe_modules/android_virtual_device/api.py b/recipe_modules/android_virtual_device/api.py index 07a567d..97e0746 100644 --- a/recipe_modules/android_virtual_device/api.py +++ b/recipe_modules/android_virtual_device/api.py
@@ -8,6 +8,7 @@ # Supports 19 though API 34. AVD_CIPD_IDENTIFIER = 'nNnmIzfGCF3wVB1sB14hKaU77TdoTFbq6uq_wXHM-WQC' +RERUN_ATTEMPTS = 3 class AndroidVirtualDeviceApi(recipe_api.RecipeApi): """Installs and manages an Android AVD. @@ -122,17 +123,28 @@ ], stdout=self.m.raw_io.output_text(add_output_log=True) ) - output = self.m.step( - 'Start Android emulator (API level %s)' % self.version, [ - 'xvfb-run', 'vpython3', avd_script_path, 'start', - '--no-read-only', '--wipe-data', '--writable-system', - '--debug-tags', 'all', '--avd-config', avd_config - ], - stdout=self.m.raw_io.output_text(add_output_log=True) - ).stdout - m = re.search(r'.*pid: (\d+)\)', output) - self.emulator_pid = m.group(1) + # rerun this 3 times to see if it will start. This will prevent the + # hour long timeout in the test. + for attempt in range(RERUN_ATTEMPTS): + output = self.m.step( + 'Start Android emulator (API level %s)' % self.version, [ + 'xvfb-run', 'vpython3', avd_script_path, 'start', + '--no-read-only', '--wipe-data', '--writable-system', + '--debug-tags', 'all', '--avd-config', avd_config + ], + stdout=self.m.raw_io.output_text(add_output_log=True) + ).stdout + + # Need to look for the main loop crash that signals incomplete start. + if "Hostapd main loop has stopped" not in output: + m = re.search(r'.*pid: (\d+)\)', output) + self.emulator_pid = m.group(1) + break + if attempt == RERUN_ATTEMPTS - 1: + raise self.m.step.InfraFailure( + 'Emulator has failed to start correctly.' + ) env['EMULATOR_PID'] = self.emulator_pid return self.emulator_pid
diff --git a/recipe_modules/android_virtual_device/examples/full_api_34.expected/emulator fails to start.json b/recipe_modules/android_virtual_device/examples/full_api_34.expected/emulator fails to start.json new file mode 100644 index 0000000..cac28cf --- /dev/null +++ b/recipe_modules/android_virtual_device/examples/full_api_34.expected/emulator fails to start.json
@@ -0,0 +1,385 @@ +[ + { + "cmd": [], + "name": "download avd package" + }, + { + "cmd": [ + "cipd", + "ensure", + "-root", + "[CACHE]/avd", + "-ensure-file", + "chromium/tools/android/avd/linux-amd64 nNnmIzfGCF3wVB1sB14hKaU77TdoTFbq6uq_wXHM-WQC", + "-max-threads", + "0", + "-json-output", + "/path/to/tmp/json" + ], + "env": { + "USE_EMULATOR": "true" + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "download avd package.ensure_installed", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@json.output@{@@@", + "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@", + "@@@STEP_LOG_LINE@json.output@ \"\": [@@@", + "@@@STEP_LOG_LINE@json.output@ {@@@", + "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-nNnmIzfGCF3wVB1s\",@@@", + "@@@STEP_LOG_LINE@json.output@ \"package\": \"chromium/tools/android/avd/linux-amd64\"@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@ ]@@@", + "@@@STEP_LOG_LINE@json.output@ }@@@", + "@@@STEP_LOG_LINE@json.output@}@@@", + "@@@STEP_LOG_END@json.output@@@" + ] + }, + { + "cmd": [], + "name": "Show devices attached - before emulator install/start" + }, + { + "cmd": [ + "chmod", + "755", + "RECIPE_MODULE[flutter::android_virtual_device]/resources/adb_show_devices.sh" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "infra_step": true, + "name": "Show devices attached - before emulator install/start.Set execute permission", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "RECIPE_MODULE[flutter::android_virtual_device]/resources/adb_show_devices.sh", + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "infra_step": true, + "name": "Show devices attached - before emulator install/start.adb_show_devices.sh", + "timeout": 3600, + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@test_stdout@@@@", + "@@@STEP_LOG_END@test_stdout@@@", + "@@@STEP_LOG_LINE@test_stderr@@@@", + "@@@STEP_LOG_END@test_stderr@@@" + ] + }, + { + "cmd": [], + "name": "start avd", + "~followup_annotations": [ + "@@@STEP_EXCEPTION@@@" + ] + }, + { + "cmd": [ + "vpython3", + "[CACHE]/avd/src/tools/android/avd/avd.py", + "install", + "--avd-config", + "[CACHE]/avd/src/tools/android/avd/proto/android_34_google_apis_x64.textpb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "start avd.Install Android emulator (API level 34)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [ + "xvfb-run", + "vpython3", + "[CACHE]/avd/src/tools/android/avd/avd.py", + "start", + "--no-read-only", + "--wipe-data", + "--writable-system", + "--debug-tags", + "all", + "--avd-config", + "[CACHE]/avd/src/tools/android/avd/proto/android_34_google_apis_x64.textpb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "start avd.Start Android emulator (API level 34)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@Hostapd main loop has stopped@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [ + "xvfb-run", + "vpython3", + "[CACHE]/avd/src/tools/android/avd/avd.py", + "start", + "--no-read-only", + "--wipe-data", + "--writable-system", + "--debug-tags", + "all", + "--avd-config", + "[CACHE]/avd/src/tools/android/avd/proto/android_34_google_apis_x64.textpb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "start avd.Start Android emulator (API level 34) (2)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@Hostapd main loop has stopped@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [ + "xvfb-run", + "vpython3", + "[CACHE]/avd/src/tools/android/avd/avd.py", + "start", + "--no-read-only", + "--wipe-data", + "--writable-system", + "--debug-tags", + "all", + "--avd-config", + "[CACHE]/avd/src/tools/android/avd/proto/android_34_google_apis_x64.textpb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "start avd.Start Android emulator (API level 34) (3)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@raw_io.output_text@Hostapd main loop has stopped@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [], + "name": "kill and cleanup avd" + }, + { + "cmd": [ + "kill", + "-9", + "" + ], + "name": "kill and cleanup avd.Kill emulator cleanup", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "ps", + "-axww" + ], + "name": "kill and cleanup avd.list processes", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [], + "name": "uninstall avd" + }, + { + "cmd": [ + "vpython3", + "[CACHE]/avd/src/tools/android/avd/avd.py", + "uninstall", + "--avd-config", + "[CACHE]/avd/src/tools/android/avd/proto/android_34_google_apis_x64.textpb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "env_suffixes": { + "DEPOT_TOOLS_UPDATE": [ + "0" + ], + "PATH": [ + "RECIPE_REPO[depot_tools]" + ] + }, + "name": "uninstall avd.Uninstall Android emulator (API level 34)", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_END@raw_io.output_text@@@" + ] + }, + { + "cmd": [], + "name": "Show devices attached - after emulator uninstall" + }, + { + "cmd": [ + "chmod", + "755", + "RECIPE_MODULE[flutter::android_virtual_device]/resources/adb_show_devices.sh" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "EMULATOR_PID": "", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "infra_step": true, + "name": "Show devices attached - after emulator uninstall.Set execute permission", + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@" + ] + }, + { + "cmd": [ + "RECIPE_MODULE[flutter::android_virtual_device]/resources/adb_show_devices.sh", + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb" + ], + "cwd": "[CACHE]/avd", + "env": { + "ADB_PATH": "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools/adb", + "AVD_ROOT": "[CACHE]/avd", + "EMULATOR_PID": "", + "USE_EMULATOR": "true" + }, + "env_prefixes": { + "PATH": [ + "[CACHE]/avd/src/third_party/android_sdk/public/platform-tools" + ] + }, + "infra_step": true, + "name": "Show devices attached - after emulator uninstall.adb_show_devices.sh", + "timeout": 3600, + "~followup_annotations": [ + "@@@STEP_NEST_LEVEL@1@@@", + "@@@STEP_LOG_LINE@test_stdout@@@@", + "@@@STEP_LOG_END@test_stdout@@@", + "@@@STEP_LOG_LINE@test_stderr@@@@", + "@@@STEP_LOG_END@test_stderr@@@" + ] + }, + { + "failure": { + "humanReason": "Emulator has failed to start correctly." + }, + "name": "$result" + } +] \ No newline at end of file
diff --git a/recipe_modules/android_virtual_device/examples/full_api_34.py b/recipe_modules/android_virtual_device/examples/full_api_34.py index 17d135f..8b83bb8 100644 --- a/recipe_modules/android_virtual_device/examples/full_api_34.py +++ b/recipe_modules/android_virtual_device/examples/full_api_34.py
@@ -44,6 +44,30 @@ ) yield api.test( + 'emulator fails to start', + api.properties(use_emulator="true"), + api.step_data( + 'start avd.Start Android emulator (API level %s)' % avd_api_version, + stdout=api.raw_io.output_text( + 'Hostapd main loop has stopped' + ) + ), + api.step_data( + 'start avd.Start Android emulator (API level %s) (2)' % avd_api_version, + stdout=api.raw_io.output_text( + 'Hostapd main loop has stopped' + ) + ), + api.step_data( + 'start avd.Start Android emulator (API level %s) (3)' % avd_api_version, + stdout=api.raw_io.output_text( + 'Hostapd main loop has stopped' + ) + ), + status='INFRA_FAILURE' + ) + + yield api.test( 'emulator started and stopped, processes killed', api.properties(use_emulator="true"), api.step_data(