| # Copyright 2018 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """The `osx_sdk` module provides safe functions to access a semi-hermetic |
| XCode installation. |
| |
| Available only to Google-run bots.""" |
| |
| from contextlib import contextmanager |
| import re |
| |
| from recipe_engine import recipe_api |
| from datetime import datetime, timedelta |
| |
| _RUNTIMESPATH = ( |
| 'Contents/Developer/Platforms/iPhoneOS.platform/Library/' |
| 'Developer/CoreSimulator/Profiles/Runtimes' |
| ) |
| |
| # This CIPD source contains Xcode and iOS runtimes create and maintained by the |
| # Flutter team. |
| _FLUTTER_XCODE_CIPD = 'flutter_internal/ios/xcode' |
| |
| # This CIPD source contains Xcode and iOS runtimes created and maintained by the |
| # Chrome team. |
| _INFRA_XCODE_CIPD = 'infra_internal/ios/xcode' |
| |
| # Intel bots cannot run macOS 26, which newer Xcodes require, so any Xcode |
| # requested on Intel is replaced by 17c52 (Xcode 26.2), the newest Xcode that |
| # runs there. See https://github.com/flutter/flutter/issues/192754. |
| _INTEL_MAX_XCODE_VERSION = '17c52' |
| |
| # Matches simulator runtime disk image UDIDs as printed by `simctl runtime`, |
| # e.g. `D: 73EA5108-FE5E-4ADC-B6A6-D2FBE4448EA2 iOS (26.2 - 23C54) (Unusable - |
| # Other Failure: ... "Duplicate of 30BC4E5F-DE5F-499A-93AF-22FEF83EF916" ...)`. |
| _RUNTIME_UDID_RE = re.compile( |
| r'\b[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}\b' |
| ) |
| |
| # Substring of the `simctl runtime add` output when the added image is a |
| # duplicate of an already registered (possibly still deleting) image. See |
| # https://github.com/flutter/flutter/issues/180141. |
| _DUPLICATE_RUNTIME_ERROR = 'duplicate of' |
| |
| |
| class OSXSDKApi(recipe_api.RecipeApi): |
| """API for using OS X SDK distributed via CIPD.""" |
| |
| def __init__(self, sdk_properties, *args, **kwargs): |
| super(OSXSDKApi, self).__init__(*args, **kwargs) |
| self._sdk_properties = sdk_properties |
| self._sdk_version = None |
| self._runtime_versions = None |
| self._tool_pkg = 'infra/tools/mac_toolchain/${platform}' |
| self._tool_ver = 'latest' |
| self._cleanup_cache = False |
| self.macos_13_or_later = False |
| self.macos_26_or_later = False |
| self._xcode_cipd_package_source = None |
| self._skip = False |
| |
| def initialize(self): |
| """Initializes xcode, and ios versions. |
| |
| Versions are usually passed as recipe properties but if not then defaults |
| are used. |
| """ |
| if not self.m.platform.is_mac: |
| return |
| |
| if 'skip_xcode_install' in self._sdk_properties: |
| self._skip = self._sdk_properties['skip_xcode_install'] |
| |
| if 'cleanup_cache' in self._sdk_properties: |
| self._cleanup_cache = self._sdk_properties['cleanup_cache'] |
| |
| if 'arm' in self.m.platform.arch and 'toolchain_ver_arm' in self._sdk_properties: |
| self._tool_ver = self._sdk_properties['toolchain_ver_arm'] |
| elif 'intel' in self.m.platform.arch and 'toolchain_ver_intel' in self._sdk_properties: |
| self._tool_ver = self._sdk_properties['toolchain_ver_intel'] |
| |
| if 'runtime_versions' in self._sdk_properties: |
| # Sorts the runtime versions to make xcode cache path deterministic, without |
| # being affected by how user orders the runtime versions. |
| runtime_versions = self._sdk_properties['runtime_versions'] |
| runtime_versions.sort(reverse=True) |
| self._runtime_versions = runtime_versions |
| |
| find_os = self.m.step( |
| "find macOS version", |
| ["sw_vers", "-productVersion"], |
| stdout=self.m.raw_io.output_text(), |
| step_test_data=( |
| lambda: self.m.raw_io.test_api.stream_output_text("14.4") |
| ), |
| ) |
| current_os = self.m.version.parse(find_os.stdout.strip()) |
| if 'sdk_version' in self._sdk_properties: |
| self._sdk_version = self._sdk_properties['sdk_version'].lower() |
| # The `!=` check only avoids a no-op step when 17c52 is already requested. |
| if ('intel' in self.m.platform.arch and |
| self._sdk_version != _INTEL_MAX_XCODE_VERSION): |
| self.m.step.empty( |
| 'Pinning Xcode to %s on Intel' % _INTEL_MAX_XCODE_VERSION, |
| step_text=( |
| 'Requested Xcode %s, but Intel bots cannot run macOS 26. See ' |
| 'https://github.com/flutter/flutter/issues/192754.' |
| % self._sdk_version |
| ), |
| ) |
| self._sdk_version = _INTEL_MAX_XCODE_VERSION |
| |
| self.macos_13_or_later = current_os >= self.m.version.parse('13.0.0') |
| self.macos_26_or_later = current_os >= self.m.version.parse('26.0.0') |
| |
| if 'xcode_cipd_package_source' in self._sdk_properties: |
| self._xcode_cipd_package_source = self._sdk_properties[ |
| 'xcode_cipd_package_source'] |
| else: |
| # TODO(vashworth): Once all bots are on macOS 13, we can remove this |
| # check and always default to _INFRA_XCODE_CIPD. |
| if self.macos_13_or_later: |
| # Starting with macOS 13, Xcode packages that have been altered are |
| # considered "damaged" and will not be usable. Since Xcode CIPD packages |
| # from flutter_internal have been altered up to Xcode 15 beta 6 (15a5219j), |
| # use infra_internal Xcode CIPD packages as the default on macOS 13. |
| self._xcode_cipd_package_source = _INFRA_XCODE_CIPD |
| else: |
| self._xcode_cipd_package_source = _FLUTTER_XCODE_CIPD |
| |
| @contextmanager |
| def __call__(self, kind, devicelab=False): |
| """Sets up the XCode SDK environment. |
| |
| Is a no-op on non-mac platforms. |
| |
| This will deploy the helper tool and the XCode.app bundle at |
| `[START_DIR]/cache/osx_sdk`. |
| |
| To avoid machines rebuilding these on every run, set up a named cache in |
| your cr-buildbucket.cfg file like: |
| |
| caches: [ |
| { |
| name: "flutter_xcode" |
| path: "osx_sdk" |
| } |
| ] |
| |
| The Xcode app will be installed under: |
| osx_sdk/xcode_<xcode-version> |
| |
| If any iOS runtime is needed, the corresponding path will be: |
| osx_sdk/xcode_<xcode-version>_runtime_<runtime-version> |
| |
| These cached packages will be shared across builds/bots. |
| |
| Usage: |
| with api.osx_sdk('mac'): |
| # sdk with mac build bits |
| |
| with api.osx_sdk('ios'): |
| # sdk with mac+iOS build bits |
| |
| Args: |
| kind ('mac'|'ios'): How the SDK should be configured. iOS includes the |
| base XCode distribution, as well as the iOS simulators (which can be |
| quite large). |
| devicelab (bool): whether this is a devicelab tasks. The xcode for devicelab |
| is installed in a fixed location `/opt/flutter/xcode`. |
| |
| Raises: |
| StepFailure or InfraFailure. |
| """ |
| assert kind in ('mac', 'ios'), 'Invalid kind %r' % (kind,) |
| if self._skip or not self.m.platform.is_mac: |
| yield |
| return |
| |
| if self._sdk_version is None: |
| self.m.step.empty('Xcode version is not set. Skipping.',) |
| yield |
| return |
| |
| try: |
| with self.m.context(infra_steps=True): |
| self._diagnose_unresponsive_mac(devicelab) |
| self._setup_osx_sdk(kind, devicelab) |
| runtime_simulators = self.m.step( |
| 'list runtimes', ['xcrun', 'simctl', 'list', 'runtimes'], |
| stdout=self.m.raw_io.output_text(add_output_log=True) |
| ).stdout.splitlines() |
| if self._missing_runtime(runtime_simulators[1:]) or self._unavailable_runtime(runtime_simulators): |
| self._cleanup_cache = True |
| self._setup_osx_sdk(kind, devicelab) |
| yield |
| finally: |
| self.reset_xcode() |
| |
| def reset_xcode(self): |
| '''Unset manually defined Xcode path for Xcode command line tools on macOS.''' |
| if self._skip or not self.m.platform.is_mac: |
| return |
| with self.m.context(infra_steps=True): |
| self.m.step('reset XCode', ['sudo', 'xcode-select', '--reset']) |
| |
| def _missing_runtime(self, runtime_simulators): |
| """Check if there is any missing runtime. |
| |
| If no explicit `_runtime_versions` is specified, we assume `runtime_simulators` |
| at least has the default runtime and should not be empty. |
| |
| If there is explicit `_runtime_versions` defined, we need to check if the number |
| of installed runtime matches the number of required. |
| |
| The runtime_simulators follows: |
| [ |
| "iOS 16.2 (16.2 - 20C52) - com.apple.CoreSimulator.SimRuntime.iOS-16-2", |
| "iOS 16.4 (16.4 - 20E247) - com.apple.CoreSimulator.SimRuntime.iOS-16-4" |
| ] |
| |
| The property `_runtime_versions` follows: |
| [ |
| "ios-16-4_14e300c", |
| "ios-16-2_14c18" |
| ] |
| """ |
| if not self._runtime_versions: |
| return not runtime_simulators |
| return len(self._runtime_versions) != len(runtime_simulators) |
| |
| def _unavailable_runtime(self, runtime_simulators): |
| """Check if any runtime is unavailable due to architecture mismatch. |
| |
| On Intel Macs, an ARM-only runtime may have been cached. When this happens, |
| `simctl list runtimes` reports the runtime as unavailable with a specific |
| error about liblaunch_sim.dylib. |
| |
| Args: |
| runtime_simulators: (list) A list of strings from `simctl list runtimes` output. |
| |
| Returns: |
| True if any runtime is unavailable due to this error, False otherwise. |
| """ |
| unavailable_error = '(unavailable, liblaunch_sim.dylib could not be opened)' |
| for runtime in runtime_simulators: |
| if unavailable_error in runtime: |
| return True |
| return False |
| |
| def _get_xcode_base_cache_path(self, devicelab: bool): |
| """Returns the base cache path for xcode. |
| |
| Args: |
| devicelab - Whether or not the machine this code is running is a devicelab |
| machine. |
| """ |
| if devicelab: |
| return self.m.path.cast_to_path('/opt/flutter/xcode') |
| return self.m.path.cache_dir / 'osx_sdk' |
| |
| def _setup_osx_sdk(self, kind, devicelab): |
| app = None |
| self._clean_xcode_cache(devicelab) |
| # NOTE: cleaning of the cache on devicelab will happen via salt. |
| if not devicelab: |
| self._micro_manage_cache(devicelab=devicelab) |
| app = self._ensure_sdk(kind, devicelab) |
| self.m.os_utils.kill_simulators() |
| self._select_xcode(app) |
| self.m.step('list simulators', ['xcrun', 'simctl', 'list']) |
| |
| def _clean_xcode_cache(self, devicelab): |
| """Cleans up cache when specified or polluted. |
| |
| Cleans up only corresponding versioned xcode instead of the whole `osx_sdk`. |
| |
| If on macOS 13 or later, deletes all mounted runtimes and deletes |
| corresponding cached runtime dmgs. |
| """ |
| if not self._cleanup_cache: |
| return |
| if devicelab or not self.macos_13_or_later: |
| self.m.file.rmtree('Cleaning up Xcode cache', self._xcode_dir(devicelab)) |
| else: |
| # Clean up with `file.rmtree` fails on macOS 13 non-devicelab bots with an |
| # "Operation not permitted" error when deleting XCode.app/Contents/_CodeSignature. |
| # Use `rm -rf` instead. |
| self.m.step( |
| 'Cleaning up Xcode cache', |
| ['rm', '-rf', self._xcode_dir(devicelab)] |
| ) |
| |
| def _ensure_mac_toolchain(self, tool_dir): |
| ef = self.m.cipd.EnsureFile() |
| ef.add_package(self._tool_pkg, self._tool_ver) |
| self.m.cipd.ensure(tool_dir, ef) |
| |
| def _select_xcode(self, sdk_app): |
| self.m.step('select xcode', ['sudo', 'xcode-select', '--switch', sdk_app]) |
| |
| def _verify_xcode(self, sdk_app): |
| """If Xcode is already downloaded, verify that it's not damaged. |
| |
| Args: |
| sdk_app: (Path) Path to installed Xcode app bundle. |
| """ |
| if not self.m.path.exists(sdk_app): |
| return |
| |
| with self.m.step.nest('verify xcode %s' % sdk_app): |
| version_check_failed = False |
| try: |
| self._select_xcode(sdk_app) |
| |
| # This step is expected to timeout if Xcode is damaged. |
| self.m.step( |
| 'check xcode version', |
| [ |
| 'xcrun', |
| 'xcodebuild', |
| '-version', |
| ], |
| timeout=60 * 5, # 5 minutes |
| ) |
| except self.m.step.StepFailure: |
| version_check_failed = True |
| raise |
| finally: |
| self.reset_xcode() |
| self._dimiss_damaged_notification() |
| |
| if version_check_failed: |
| self._diagnose_codesign_failure(sdk_app) |
| |
| def _diagnose_codesign_failure(self, sdk_app): |
| """Check if Xcode verification may have failed due to code not matching |
| original signed code. Used to help debug issues. |
| """ |
| self.m.step( |
| 'verify codesign', |
| [ |
| 'codesign', |
| '-vv', |
| sdk_app, |
| ], |
| ok_ret='any', |
| ) |
| |
| def _dimiss_damaged_notification(self): |
| """If Xcode is damaged, it may show a notification that can cause |
| Xcode processes to hang until it's closed. Kill `CoreServicesUIAgent` |
| to close it. |
| """ |
| self.m.step( |
| 'dismiss damaged notification', |
| ['killall', '-9', 'CoreServicesUIAgent'], |
| ok_ret='any', |
| ) |
| |
| def _try_install_xcode(self, tool_dir, kind, app_dir, sdk_app, devicelab): |
| """Installs xcode using mac_toolchain. If install fails, clear the cache and try again. |
| |
| Args: |
| devicelab: (bool) Whether this is a devicelab tasks. Don't install |
| explicit runtimes for devicelab tasks. |
| app_dir: (Path) Path to Xcode cache directory. |
| tool_dir: (Path) Path to mac_toolchain cache directory. |
| sdk_app: (Path) Path to installed Xcode app bundle. |
| kind ('mac'|'ios'): How the SDK should be configured. |
| """ |
| with self.m.step.nest('install xcode'): |
| try: |
| self._install_xcode(tool_dir, kind, app_dir, sdk_app, devicelab) |
| except self.m.step.StepFailure: |
| self._install_xcode(tool_dir, kind, app_dir, sdk_app, devicelab, True) |
| |
| def _install_xcode( |
| self, tool_dir, kind, app_dir, sdk_app, devicelab, retry=False |
| ): |
| """Installs xcode using mac_toolchain. |
| |
| Args: |
| devicelab: (bool) Whether this is a devicelab tasks. Don't install |
| explicit runtimes for devicelab tasks. |
| app_dir: (Path) Path to Xcode cache directory. |
| tool_dir: (Path) Path to mac_toolchain cache directory. |
| sdk_app: (Path) Path to installed Xcode app bundle. |
| kind ('mac'|'ios'): How the SDK should be configured. |
| retry: (bool) Whether this is the second attempt to install Xcode. |
| """ |
| |
| install_path = sdk_app |
| |
| # On retry, install Xcode to a different path and later move it to the |
| # original path. Although unproven, there is a theory that re-installing |
| # Xcode to the same path as a previously damaged version may cause the new |
| # version to also be considered damaged. |
| if retry: |
| uuid = self.m.uuid.random() |
| install_path = app_dir / f'temp_{uuid}_xcode.app' |
| |
| try: |
| # Verify that existing Xcode is not damaged. If it's damaged, the |
| # 'install xcode from cipd' step may get stuck until it times out. |
| self._verify_xcode(install_path) |
| except self.m.step.StepFailure: |
| self._cleanup_cache = True |
| self._clean_xcode_cache(devicelab) |
| |
| try: |
| self._ensure_mac_toolchain(tool_dir) |
| if self.macos_13_or_later: |
| # TODO(vashworth): Remove macOS 13 specific install steps once |
| # https://github.com/flutter/flutter/issues/138238 is resolved. |
| self.m.step('Show tool_dir cache', ['ls', '-al', tool_dir]) |
| self.m.step( |
| 'install xcode from cipd', |
| [ |
| tool_dir / 'mac_toolchain', |
| 'install', |
| '-kind', |
| kind, |
| '-xcode-version', |
| self._sdk_version, |
| '-output-dir', |
| install_path, |
| '-cipd-package-prefix', |
| self._xcode_cipd_package_source, |
| '-with-runtime=%s' % (not bool(self._runtime_versions)), |
| '-with-metal-toolchain=True', |
| '-verbose', |
| ], |
| timeout=60 * 30 # 30 minutes |
| ) |
| if retry: |
| self.m.step( |
| 'move to final destination', |
| [ |
| 'mv', |
| install_path, |
| sdk_app, |
| ], |
| ) |
| except self.m.step.StepFailure: |
| if self.macos_13_or_later: |
| # TODO(vashworth): Remove macOS 13 specific install steps once |
| # https://github.com/flutter/flutter/issues/138238 is resolved. |
| self.m.step('Show tool_dir cache', ['ls', '-al', tool_dir]) |
| self.m.step('Show app_dir cache', ['ls', '-al', app_dir], ok_ret='any') |
| self._dimiss_damaged_notification() |
| self._diagnose_codesign_failure(sdk_app) |
| self._cleanup_cache = True |
| self._clean_xcode_cache(devicelab) |
| self.m.step.empty( |
| 'Failed to install Xcode', |
| status=self.m.step.INFRA_FAILURE, |
| ) |
| finally: |
| if retry: |
| self.m.step( |
| 'clean temporary install path', |
| [ |
| 'rm', |
| '-rf', |
| install_path, |
| ], |
| ok_ret='any', |
| ) |
| |
| def _diagnose_unresponsive_mac(self, devicelab): |
| """To prevent mac tests from hanging due to unresponsive host, kill Setup |
| Assistant and reset the Launch Services database. |
| |
| Also, check if `xcodebuild` commands may have potentially hung due to Launch |
| Services issues. If so, add entry to Launch Services reset log and print a warning. |
| |
| On macOS 26 and later, `lsregister -kill` was removed so the reset is |
| skipped, but the log check still runs so Launch Services problems are |
| surfaced. |
| |
| Args: |
| devicelab: (bool) tells the module which path we should be working with. |
| """ |
| if devicelab: |
| return |
| |
| # Try to kill setup assistant |
| self._kill_setup_assistant() |
| |
| with self.m.step.nest('verify launch services') as display_step: |
| start_time = self.now().strftime("%Y-%m-%d %H:%M:%S") |
| |
| if self.macos_26_or_later: |
| # macOS 26 removed `lsregister -kill` ("The -kill option has been |
| # removed because it was dangerous and no longer useful"), so the db |
| # can no longer be reset from here. The log check below still runs. |
| # TODO(okorohelijah): Find a replacement for resetting the Launch |
| # Services database on macOS 26+. See |
| # https://github.com/flutter/flutter/issues/192645. |
| self.m.step.empty( |
| 'Skipping Launch Services reset on macOS 26+', |
| step_text='`lsregister -kill` was removed in macOS 26.', |
| ) |
| else: |
| self.m.step( |
| 'Reset and rescan Launch Services db', |
| [ |
| '/System/Library/Frameworks/CoreServices.framework/Frameworks/LaunchServices.framework/Support/lsregister', |
| '-kill', |
| '-r', |
| '-domain', |
| 'local', |
| '-domain', |
| 'system', |
| '-domain', |
| 'user', |
| ], |
| ) |
| |
| launch_services_reset_log = self._get_xcode_base_cache_path( |
| devicelab |
| ) / 'launch_services_reset_log.txt' |
| |
| reset_logs = '' |
| last_entry_time = None |
| if self.m.path.exists(launch_services_reset_log): |
| reset_logs = self.m.file.read_text( |
| 'Check if Launch Services db has been reset recently', |
| launch_services_reset_log, |
| ) |
| try: |
| # Get last entry in log. |
| last_entry = reset_logs.splitlines()[-1].rstrip() |
| last_entry_time = datetime.strptime(last_entry, "%Y-%m-%d %H:%M:%S") |
| except: |
| # If there was an invalid date, reset the log file. |
| reset_logs = '' |
| |
| # If last log entry was within 24 hours, only search logs since that time |
| # (with a 30 minute buffer). Otherwise search the last 24 hours |
| if last_entry_time and (self.now() - last_entry_time).days <= 1: |
| search_time = (last_entry_time + |
| timedelta(minutes=30)).strftime("%Y-%m-%d %H:%M:%S") |
| time_parameters = [ |
| '--start', |
| search_time, |
| ] |
| else: |
| time_parameters = [ |
| '--last', |
| '24h', |
| ] |
| |
| # Check if xcodebuild commands have potentially been slowed by Launch |
| # Services. |
| xcodebuild_logs = self.m.step( |
| 'Check if xcodebuild impacted by Launch Services', |
| [ |
| 'log', |
| 'show', |
| *time_parameters, |
| '--style', |
| 'compact', |
| '--predicate', |
| 'logType == "error" AND process == "xcodebuild" AND ' |
| 'subsystem == "com.apple.launchservices" AND composedMessage ' |
| 'CONTAINS "disconnect event interruption received for service"', |
| ], |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| ) |
| |
| # Remove first line, which is headers. |
| xcodebuild_logs_stdout = xcodebuild_logs.stdout.splitlines()[1:] |
| |
| # Since Launch Services resets on every build (before macOS 26), |
| # xcodebuild should not continue to fail. If there are still errors here, |
| # print a warning and update the reset log. |
| if len(xcodebuild_logs_stdout) == 0: |
| return |
| |
| display_step.status = self.m.step.INFRA_FAILURE |
| if self.macos_26_or_later: |
| display_step.step_text = 'xcodebuild reported Launch Services errors and the Launch Services db cannot be reset on macOS 26+. Please file a bug and see go/flutter-infra-macos-troubleshoot.' |
| else: |
| display_step.step_text = 'Launch Services has already been reset recently. Please file a bug and see go/flutter-infra-macos-troubleshoot.' |
| |
| self.m.file.write_text( |
| 'Update Launch Services reset log', |
| launch_services_reset_log, |
| '%s\n%s' % (reset_logs, start_time), |
| ) |
| |
| def _kill_setup_assistant(self): |
| """Tries to kill Setup Assistant. Sometimes on macOS, tests will be blocked |
| by a Setup Assistant window. This appears to be a bug with macOS |
| (see b/343978626). To workaround, kill Setup Assistant.""" |
| |
| pgrep_result = self.m.step( |
| 'check for Setup Assistant', |
| [ |
| 'pgrep', |
| 'Setup Assistant', |
| ], |
| ok_ret='any', |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| ) |
| |
| if pgrep_result.stdout != '': |
| self.m.step( |
| 'Kill Setup Assistant', |
| [ |
| 'pkill', |
| '-9', |
| 'Setup Assistant', |
| ], |
| ok_ret='any', |
| ) |
| |
| def now(self): |
| """Provide a deterministic date when running tests.""" |
| return datetime.now() # pragma: nocover |
| |
| def _ensure_sdk(self, kind, devicelab): |
| """Ensures the mac_toolchain tool and OSX SDK packages are installed. |
| |
| Returns Path to the installed sdk app bundle.""" |
| app_dir = self._xcode_dir(devicelab) |
| tool_dir = self.m.path.mkdtemp() / 'osx_sdk' if devicelab else app_dir |
| sdk_app = app_dir / 'XCode.app' |
| self._try_install_xcode(tool_dir, kind, app_dir, sdk_app, devicelab) |
| |
| self._cleanup_runtimes_cache(sdk_app) |
| |
| self._install_runtimes(devicelab, app_dir, tool_dir, sdk_app, kind) |
| |
| return sdk_app |
| |
| def _micro_manage_cache(self, devicelab: bool): |
| """Tracks the age of packages in the target cache. If older than a |
| specific date then delete them. |
| |
| Params: |
| devicelab: (bool) tells the module which path we should be working with. |
| """ |
| cache_path = self._get_xcode_base_cache_path(devicelab) |
| app_dir = self._xcode_dir(devicelab) |
| self.m.step("show app_dir", ['echo', app_dir]) |
| self._show_xcode_cache(cache_path) |
| self.m.cache_micro_manager.run(cache_path, [app_dir]) |
| self._show_xcode_cache(cache_path) |
| |
| def _show_xcode_cache(self, cache_path): |
| self.m.step( |
| 'Show xcode cache', |
| ['ls', '-al', cache_path], |
| ok_ret='any', |
| ) |
| |
| def _install_runtimes(self, devicelab, app_dir, tool_dir, sdk_app, kind): |
| """Ensure runtimes are installed. |
| |
| For macOS lower than 13, this involves downloading the runtime and copying |
| it into the Xcode app bundle. |
| |
| For macOS 13 and higher, this involved downloading the runtime dmg and |
| running a `simctl` command to verify and mount it. This is required because |
| copying files into Xcode on macOS 13 may damage it and prevent it from |
| being usable. |
| |
| Args: |
| devicelab: (bool) Whether this is a devicelab tasks. Don't install |
| explicit runtimes for devicelab tasks. |
| app_dir: (Path) Path to Xcode cache directory. |
| tool_dir: (Path) Path to mac_toolchain cache directory. |
| sdk_app: (Path) Path to installed Xcode app bundle. |
| kind ('mac'|'ios'): How the SDK should be configured. |
| """ |
| |
| if not self._runtime_versions: |
| # On macOS 13, when cleanup_cache is True, we first clear the Xcode cache, |
| # install Xcode, and then clean up the runtimes cache. It happens in this |
| # order because removing runtimes requires Xcode developer tools so Xcode |
| # must be installed first. However, when no explicit runtimes are defined, |
| # the runtime is also installed in the `_try_install_xcode` function. So after |
| # cleaning the runtimes, the runtime that was installed may have been |
| # removed. So re-call `_try_install_xcode` to reinstall the removed runtime. |
| if self.macos_13_or_later and self._cleanup_cache: |
| with self.m.step.nest('install runtimes'): |
| self._try_install_xcode(tool_dir, kind, app_dir, sdk_app, devicelab) |
| return |
| |
| if devicelab: |
| return |
| |
| with self.m.step.nest('install runtimes'): |
| if self.macos_13_or_later: |
| self._select_xcode(sdk_app) |
| runtime_simulators = self.m.step( |
| 'list runtimes', ['xcrun', 'simctl', 'list', 'runtimes'], |
| stdout=self.m.raw_io.output_text(add_output_log=True) |
| ).stdout.splitlines() |
| |
| for version in self._runtime_versions: |
| runtime_version_parts = version.split('_') |
| if len(runtime_version_parts) != 2: |
| self.m.step.empty( |
| 'Invalid runtime version %s' % version, |
| status=self.m.step.INFRA_FAILURE, |
| ) |
| runtime_version = runtime_version_parts[0] |
| xcode_version = runtime_version_parts[1] |
| # we can assume devicelab False and build this with base path. |
| runtime_dmg_cache_dir = self._runtime_dmg_dir_cache_path(version) |
| |
| self.m.step( |
| 'install xcode runtime %s' % version.lower(), |
| [ |
| app_dir / '/mac_toolchain', |
| 'install-runtime-dmg', |
| '-cipd-package-prefix', |
| self._xcode_cipd_package_source, |
| '-runtime-version', |
| runtime_version, |
| '-xcode-version', |
| xcode_version, |
| '-output-dir', |
| runtime_dmg_cache_dir, |
| ], |
| ) |
| downloaded_runtime_files = self.m.file.listdir( |
| 'list xcode runtime dmg %s' % version.lower(), |
| runtime_dmg_cache_dir |
| ) |
| |
| # The runtime dmg may not be named consistently so search for the dmg file. |
| runtime_dmg_path = None |
| for file_path in downloaded_runtime_files: |
| if '.dmg' in str(file_path): |
| runtime_dmg_path = str(file_path) |
| |
| if runtime_dmg_path is None: |
| self.m.step.empty( |
| 'Failed to find runtime dmg', |
| status=self.m.step.INFRA_FAILURE, |
| ) |
| |
| # Skip adding the runtime if it's already mounted. |
| ios_runtime_build_version = self._lookup_runtime_build_version( |
| runtime_version, xcode_version |
| ) |
| if not self._is_runtime_build_listed(ios_runtime_build_version, |
| runtime_simulators): |
| self._mount_runtime( |
| version, runtime_dmg_path, ios_runtime_build_version |
| ) |
| |
| else: |
| # Skips runtime installation if it already exists. Otherwise, |
| # installs each runtime version under `osx_sdk` for cache sharing, |
| # and then copies over to the destination. |
| |
| self.m.file.ensure_directory( |
| 'Ensuring runtimes directory', sdk_app / _RUNTIMESPATH |
| ) |
| for version in self._runtime_versions: |
| runtime_name = 'iOS %s.simruntime' % version.lower( |
| ).replace('ios-', '').replace('-', '.') |
| dest = sdk_app / _RUNTIMESPATH / runtime_name |
| if not self.m.path.exists(dest): |
| cache_base_path = self._get_xcode_base_cache_path(False) |
| runtime_cache_dir = ( |
| cache_base_path / ('xcode_runtime_%s' % version.lower()) |
| ) |
| self.m.step( |
| 'install xcode runtime %s' % version.lower(), |
| [ |
| app_dir / 'mac_toolchain', |
| 'install-runtime', |
| '-cipd-package-prefix', |
| _FLUTTER_XCODE_CIPD, |
| '-runtime-version', |
| version.lower(), |
| '-output-dir', |
| runtime_cache_dir, |
| ], |
| ) |
| # Move the runtimes |
| path_with_version = runtime_cache_dir / runtime_name |
| # If the runtime was the default for xcode the cipd bundle contains a directory called iOS.simruntime otherwise |
| # it contains a folder called "iOS <version>.simruntime". |
| source = path_with_version if self.m.path.exists( |
| path_with_version |
| ) else runtime_cache_dir / 'iOS.simruntime' |
| self.m.file.copytree( |
| 'Copy runtime to %s' % dest, source, dest, symlinks=True |
| ) |
| |
| def _xcode_dir(self, devicelab): |
| """Returns the location of the xcode app in the cache dir. |
| |
| For a devicelab task, the path is prefixed at `/opt/flutter/xcode`. |
| |
| For a host only task without runtime, the path looks like |
| xcode_<xcode-version> |
| |
| a host only task with runtimes, the path looks like |
| xcode_<xcode-version>_runtime1_<runtime1-version>_..._runtimeN_<runtimeN-version> |
| """ |
| |
| xcode_cache_base_path = self._get_xcode_base_cache_path(devicelab) |
| if devicelab: |
| return xcode_cache_base_path / self._sdk_version |
| runtime_version = None |
| sdk_version = f"xcode_{self._sdk_version}" |
| if not self.macos_13_or_later and self._runtime_versions: |
| runtime_version = "_".join(self._runtime_versions) |
| sdk_version = f"{sdk_version}_runtime_{runtime_version}" |
| return xcode_cache_base_path / sdk_version |
| |
| def _runtime_dmg_dir_cache_path(self, version): |
| cache_base_path = self._get_xcode_base_cache_path(False) |
| # this method seems to be only used for non-devicelab cache path. |
| return cache_base_path / ('xcode_runtime_dmg_%s' % version.lower()) |
| |
| def _cleanup_runtimes_cache(self, sdk_app): |
| """Deletes all mounted runtimes and deletes corresponding cached runtime dmgs. |
| |
| This is only used for macOS 13+ since runtimes are installed and mounted separately. |
| """ |
| |
| if not self._cleanup_cache or not self.macos_13_or_later: |
| return |
| |
| with self.m.step.nest('Cleaning up runtimes cache'): |
| self._select_xcode(sdk_app) |
| |
| simulator_cleanup_result = self.m.step( |
| 'Cleaning up mounted simulator runtimes', |
| [ |
| 'xcrun', |
| 'simctl', |
| 'runtime', |
| 'delete', |
| 'all', |
| ], |
| raise_on_failure=False, |
| ok_ret='any', |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| stderr=self.m.raw_io.output_text(add_output_log=True), |
| ) |
| |
| simulator_cleanup_stdout = simulator_cleanup_result.stdout.rstrip() |
| simulator_cleanup_stderr = simulator_cleanup_result.stderr.rstrip() |
| if simulator_cleanup_stdout and 'No matching images found to delete' not in simulator_cleanup_stdout: |
| self.m.step.empty( |
| 'Failed to delete runtimes', |
| status=self.m.step.INFRA_FAILURE, |
| step_text=simulator_cleanup_stdout, |
| ) |
| if simulator_cleanup_stderr and 'No matching images found to delete' not in simulator_cleanup_stderr: |
| self.m.step.empty( |
| 'Failed to delete runtimes', |
| status=self.m.step.INFRA_FAILURE, |
| step_text=simulator_cleanup_stderr, |
| ) |
| |
| # Determine how many runtimes can remain to consider them unmounted. For |
| # Xcode versions 15 or greater, 0 are allowed to remain. Otherwise 1 is |
| # allowed. This is because in older versions of Xcode, the runtime is |
| # included in Xcode, which means it won't be unmounted. |
| max_remaining_runtimes = 1 |
| xcode_version = self.m.step( |
| 'check xcode version', |
| [ |
| 'xcrun', |
| 'xcodebuild', |
| '-version', |
| ], |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| ).stdout.splitlines() |
| if len(xcode_version) > 0: |
| xcode_version = xcode_version[0].replace("Xcode", "").strip() |
| parsed_xcode_version = self.m.version.parse(xcode_version) |
| if parsed_xcode_version >= self.m.version.parse('15.0.0'): |
| max_remaining_runtimes = 0 |
| |
| # Wait up to ~5 minutes until runtimes are unmounted and their disk |
| # images are fully deleted. |
| self.m.retry.basic_wrap( |
| lambda timeout: self._is_runtimes_unmounted( |
| max_remaining_runtimes, |
| timeout=timeout, |
| ), |
| step_name='Wait for runtimes to unmount', |
| sleep=5.0, |
| backoff_factor=2, |
| max_attempts=7 |
| ) |
| |
| if not self._runtime_versions: |
| return |
| |
| for version in self._runtime_versions: |
| runtime_dmg_cache_dir = self._runtime_dmg_dir_cache_path(version) |
| |
| self.m.file.rmtree( |
| 'Cleaning up runtime cache %s' % version.lower(), |
| runtime_dmg_cache_dir |
| ) |
| |
| # pylint: disable=unused-argument |
| def _is_runtimes_unmounted(self, max_remaining_runtimes, timeout=None): |
| '''Check if more than one runtime is currently mounted or if any runtime |
| disk image is still being deleted. If so, raise an `InfraFailure` (which |
| `retry.basic_wrap` retries; if it persists, the build is an infra failure |
| rather than a test failure). |
| |
| Args: |
| max_remaining_runtimes: (int) How many runtimes are allowed to remain for |
| it to be considered done unmounting. |
| ''' |
| runtime_simulators = self._list_runtimes() |
| |
| if len(runtime_simulators[1:]) > max_remaining_runtimes: |
| raise self.m.step.InfraFailure('Runtimes not unmounted yet') |
| |
| # `simctl list runtimes` stops listing a runtime as soon as it is unmounted, |
| # but `simctl runtime delete` keeps the disk image registered (in the |
| # `Deleting` state) until it has been fully removed. Re-adding the same |
| # image before that finishes makes simctl register the new image as an |
| # unusable "Duplicate of" the old one, which is the root cause of |
| # https://github.com/flutter/flutter/issues/180141. |
| self._is_runtime_deletion_complete() |
| |
| def _list_runtimes(self): |
| """Runs `xcrun simctl list runtimes` and returns its output lines. |
| |
| Example output: |
| [ |
| "== Runtimes ==", |
| "iOS 16.2 (16.2 - 20C52) - com.apple.CoreSimulator.SimRuntime.iOS-16-2", |
| "iOS 16.4 (16.4 - 20E247) - com.apple.CoreSimulator.SimRuntime.iOS-16-4" |
| ] |
| """ |
| return self.m.step( |
| 'list runtimes', ['xcrun', 'simctl', 'list', 'runtimes'], |
| stdout=self.m.raw_io.output_text(add_output_log=True) |
| ).stdout.splitlines() |
| |
| # pylint: disable=unused-argument |
| def _is_runtime_deletion_complete(self, timeout=None): |
| """Check if any runtime disk image is still being deleted. If so, raise an |
| `InfraFailure`. |
| |
| Uses `xcrun simctl runtime list` (not `simctl list runtimes`), which |
| reports the state of each runtime disk image (e.g. `Ready`, `Deleting`, |
| `Unusable`). |
| """ |
| disk_images = self.m.step( |
| 'list runtime disk images', ['xcrun', 'simctl', 'runtime', 'list'], |
| stdout=self.m.raw_io.output_text(add_output_log=True) |
| ).stdout |
| |
| if 'Deleting' in disk_images: |
| raise self.m.step.InfraFailure('Runtime disk images not deleted yet') |
| |
| def _wait_for_runtime_deletion(self): |
| """Waits up to ~5 minutes until no runtime disk image is being deleted.""" |
| self.m.retry.basic_wrap( |
| lambda timeout: self._is_runtime_deletion_complete(timeout=timeout), |
| sleep=5.0, |
| backoff_factor=2, |
| max_attempts=7 |
| ) |
| |
| def _mount_runtime( |
| self, version, runtime_dmg_path, ios_runtime_build_version |
| ): |
| """Mounts an iOS runtime dmg with `simctl runtime add`. |
| |
| `simctl runtime add` fails with an `Unusable - Other Failure: ... |
| "Duplicate of <UDID>"` error when an image with the same identity is still |
| registered, typically because a previous `simctl runtime delete` of that |
| image has not finished yet (deletion is asynchronous). The runtime may |
| nevertheless be usable, so rather than failing the build outright: |
| 1. wait for any pending deletions to finish, |
| 2. re-check `simctl list runtimes`; if the runtime is now listed, the |
| duplicate error was benign, |
| 3. otherwise delete the duplicate disk image(s) and retry the mount |
| once, failing the build if that also fails. |
| See https://github.com/flutter/flutter/issues/180141. |
| |
| Args: |
| version: (string) The configured runtime version (e.g. |
| "ios-16-4_14e300c"). |
| runtime_dmg_path: (string) Path to the runtime dmg to mount. |
| ios_runtime_build_version: (string) The iOS runtime build (e.g. |
| "20e247") used to verify whether the runtime is mounted. |
| """ |
| step_name = 'verify and mount runtime %s' % version.lower() |
| cmd = ['xcrun', 'simctl', 'runtime', 'add', runtime_dmg_path] |
| result = self.m.step( |
| step_name, |
| cmd, |
| ok_ret='any', |
| stdout=self.m.raw_io.output_text(add_output_log=True), |
| stderr=self.m.raw_io.output_text(add_output_log=True), |
| ) |
| if result.retcode == 0: |
| return |
| |
| output = '%s\n%s' % (result.stdout, result.stderr) |
| if _DUPLICATE_RUNTIME_ERROR not in output.lower(): |
| result.presentation.status = self.m.step.INFRA_FAILURE |
| self.m.step.empty( |
| 'Failed to mount runtime %s' % version.lower(), |
| status=self.m.step.INFRA_FAILURE, |
| step_text=output.strip(), |
| ) |
| |
| # The presentation can only be changed until the next step starts. |
| result.presentation.step_text = ( |
| 'simctl reported a "Duplicate of" error (retcode %d); see the "handle ' |
| 'duplicate runtime %s" step.' % (result.retcode, version.lower()) |
| ) |
| |
| with self.m.step.nest('handle duplicate runtime %s' % |
| version.lower()) as display_step: |
| self._wait_for_runtime_deletion() |
| |
| if self._is_runtime_build_listed(ios_runtime_build_version, |
| self._list_runtimes()): |
| display_step.step_text = ( |
| 'Runtime %s (%s) is already mounted; ignoring benign "Duplicate ' |
| 'of" error from simctl.' % (version, ios_runtime_build_version) |
| ) |
| return |
| |
| # The runtime is not usable, so the unusable duplicate (and the image it |
| # duplicates, which is not usable either since it is not listed) must be |
| # removed before the dmg can be added again. |
| for udid in self._parse_runtime_udids(output): |
| self.m.step( |
| 'delete runtime disk image %s' % udid, |
| ['xcrun', 'simctl', 'runtime', 'delete', udid], |
| ok_ret='any', |
| ) |
| self._wait_for_runtime_deletion() |
| |
| self.m.step('%s (retry)' % step_name, cmd) |
| |
| def _parse_runtime_udids(self, output): |
| """Returns the unique runtime disk image UDIDs found in `output`, in order. |
| |
| Args: |
| output: (string) Output of a `simctl runtime` command. |
| """ |
| udids = [] |
| for udid in _RUNTIME_UDID_RE.findall(output.upper()): |
| if udid not in udids: |
| udids.append(udid) |
| return udids |
| |
| def _lookup_runtime_build_version(self, runtime_version, xcode_version): |
| """Looks up the iOS runtime build version from CIPD. |
| |
| Args: |
| runtime_version: (string) The iOS version (e.g. "ios-16-4"). |
| xcode_version: (string) The Xcode version (e.g. "14e300c"). |
| Returns: |
| A string of the iOS runtime build version (e.g. "20e247"). |
| |
| Raises: |
| InfraFailure if the build version could not be determined. |
| """ |
| |
| with self.m.step.nest('cipd describe %s_%s' % |
| (runtime_version, xcode_version)) as display_step: |
| # First try to get the iOS runtime build version by the Xcode version |
| ios_runtime_build_version = self._get_ios_runtime_build_version( |
| runtime_version, xcode_version |
| ) |
| |
| # If unable to get from the Xcode version, try again by using the iOS runtime version |
| if ios_runtime_build_version is None: |
| ios_runtime_build_version = self._get_ios_runtime_build_version( |
| runtime_version |
| ) |
| |
| if ios_runtime_build_version is None: |
| self.m.step.empty( |
| 'Failed to get runtime build version', |
| status=self.m.step.INFRA_FAILURE, |
| ) |
| else: |
| self.m.step.empty( |
| 'runtime build version', step_text=ios_runtime_build_version |
| ) |
| display_step.status = self.m.step.SUCCESS |
| |
| return ios_runtime_build_version |
| |
| def _is_runtime_build_listed( |
| self, ios_runtime_build_version, runtime_simulators |
| ): |
| """Determine if an iOS runtime build is listed as usable by |
| `simctl list runtimes`. |
| |
| Args: |
| ios_runtime_build_version: (string) The iOS runtime build (e.g. "20e247"). |
| runtime_simulators: (list) Output lines of `simctl list runtimes`. |
| Returns: |
| A boolean for whether or not the runtime is mounted and usable. |
| """ |
| for runtime in runtime_simulators: |
| # Example runtime: `iOS 14.3 (14.3 - 18C61) - com.apple.CoreSimulator.SimRuntime.iOS-14-3` |
| runtime = runtime.lower() |
| # A runtime reported as unavailable/unusable (e.g. an unusable duplicate |
| # disk image) does not count as mounted; it must be (re)mounted. |
| if 'unavailable' in runtime or 'unusable' in runtime: |
| continue |
| if ios_runtime_build_version.lower() in runtime: |
| return True |
| |
| return False |
| |
| def _get_ios_runtime_build_version(self, runtime_version, xcode_version=None): |
| """Gets the iOS runtime build version from CIPD. |
| |
| If `xcode_version` is provided, it will use it to search for the CIPD package. |
| If not provided, the `runtime_version` will be used. In both cases, it ensures |
| the found package matches the `runtime_version`. |
| |
| Args: |
| runtime_version: (string) An iOS version used to find and match the CIPD package (e.g. "ios-16-4") |
| xcode_version: (string) A version of Xcode to use to find CIPD package (e.g. "14e300c"). |
| Returns: |
| A string of the build version or None if unable to get the build version. |
| """ |
| |
| search_ref = runtime_version |
| if xcode_version is not None: |
| search_ref = xcode_version |
| |
| try: |
| description = self.m.cipd.describe( |
| '%s/ios_runtime_dmg' % self._xcode_cipd_package_source, |
| search_ref, |
| ) |
| runtime_tag = None |
| build_tag = None |
| for tag in description.tags: |
| if 'ios_runtime_version' in tag.tag: |
| runtime_tag = self._parse_cipd_description_tag(tag.tag) |
| if 'ios_runtime_build' in tag.tag: |
| build_tag = self._parse_cipd_description_tag(tag.tag) |
| |
| if runtime_tag == runtime_version: |
| return build_tag |
| |
| self.m.step.empty( |
| 'mismatching runtimes', |
| step_text='Found %s, expected %s' % (runtime_tag, runtime_version), |
| status=self.m.step.INFRA_FAILURE, |
| ) |
| |
| except self.m.step.StepFailure: |
| pass |
| |
| return None |
| |
| def _parse_cipd_description_tag(self, tag): |
| """Parse a colon separated CIPD description tag. |
| |
| Args: |
| tag: (string) A colon separated string describing a CIPD tag. |
| (e.g "ios_runtime_build:21A5303d" or "ios_runtime_version:ios-17-0") |
| Returns: |
| A string of the value following the colon or None if unable to parse. |
| """ |
| tag_parts = tag.split(':') |
| if len(tag_parts) == 2: |
| return tag_parts[1] |
| return None |
