Fix SDK->simruntime mapping assumptions in test runners (#2889)

Redux of #2675. 

Refactors simulator_creator.py to discover a recent simruntime when none
is explicitly declared, and removes the SDK version fallback from the
test runner rule implementations. This assumption is problematic in
recent times whenever Apple pushes a new simruntime OTA for an existing
Xcode, or when the simruntime identifier does not align with the
simruntime version and consequently the SDK version (such as OS patch
releases).

---------

Signed-off-by: Aaron Sky <asky@dropbox.com>

Author: aaronsky

apple/testing/default_runner/ios_test_runner.bzl

@@ -14,6 +14,7 @@
 
 """iOS test runner rule."""
 
+load("@build_bazel_apple_support//xcode:providers.bzl", "XcodeVersionPropertiesInfo")
 load(
     "//apple:providers.bzl",
     "AppleDeviceTestRunnerInfo",
@@ -28,6 +29,7 @@ def _get_template_substitutions(
         post_action_binary,
         post_action_determines_exit_code,
         pre_action_binary,
+        sdk_version,
         testrunner):
     """Returns the template substitutions for this runner."""
     subs = {
@@ -37,6 +39,7 @@ def _get_template_substitutions(
         "post_action_binary": post_action_binary,
         "post_action_determines_exit_code": post_action_determines_exit_code,
         "pre_action_binary": pre_action_binary,
+        "sdk_version": sdk_version,
         "testrunner_binary": testrunner,
     }
     return {"%(" + k + ")s": subs[k] for k in subs}
@@ -53,6 +56,8 @@ def _get_execution_environment(*, xcode_config):
 def _ios_test_runner_impl(ctx):
     """Implementation for the ios_test_runner rule."""
 
+    xcode_properties_attr = getattr(apple_common, "XcodeProperties", None) or XcodeVersionPropertiesInfo
+    sdk_version = ctx.attr._xcode_config[xcode_properties_attr].default_ios_sdk_version
     os_version = str(ctx.attr.os_version or ctx.fragments.objc.ios_simulator_version or "")
     device_type = ctx.attr.device_type or ctx.fragments.objc.ios_simulator_device or ""
 
@@ -84,6 +89,7 @@ def _ios_test_runner_impl(ctx):
             post_action_binary = post_action_binary,
             post_action_determines_exit_code = "true" if post_action_determines_exit_code else "false",
             pre_action_binary = pre_action_binary,
+            sdk_version = sdk_version,
             testrunner = ctx.executable._testrunner.short_path,
         ),
     )
@@ -118,6 +124,7 @@ When executed, the binary will have the following environment variables availabl
 <ul>
 <li>`SIMULATOR_DEVICE_TYPE`: The device type of the simulator to create. The supported types correspond to the output of `xcrun simctl list devicetypes`. E.g., iPhone 6, iPad Air. The value will either be the value of the `device_type` attribute, the `--ios_simulator_device` command-line flag, or an empty string that should imply a default device.</li>
 <li>`SIMULATOR_OS_VERSION`: The os version of the simulator to create. The supported os versions correspond to the output of `xcrun simctl list runtimes`. ' 'E.g., 11.2, 9.3. The value will either be the value of the `os_version` attribute, the `--ios_simulator_version` command-line flag, or an empty string that should imply a default OS version for the selected simulator runtime.</li>
+<li>`SIMULATOR_SDK_VERSION`: The SDK version of the simulator to create. The supported SDK builds correspond to the output of `xcrun simctl runtime match list`. E.g., 11.2, 9.3. The value will be derived from the `default_ios_sdk_version` for the current Xcode version.</li>
 </ul>
 """,
         ),

apple/testing/default_runner/ios_test_runner.template.sh

@@ -243,17 +243,12 @@ if [[ -n "${REUSE_GLOBAL_SIMULATOR:-}" ]]; then
     exit 1
   fi
 
-  if [[ -z "%(os_version)s" ]]; then
-    echo "error: to create a re-useable simulator the OS version must always be set on the test runner or with '--ios_simulator_version'" >&2
-    exit 1
-  fi
-
   if [[ -z "%(device_type)s" ]]; then
-    echo "error: to create a re-useable simulator the device type must always be set on the test runner or with '--ios_simulator_device'" >&2
+    echo "error: to create a re-useable simulator; the device type must always be set on the test runner or with '--ios_simulator_device'" >&2
     exit 1
   fi
 
-  id="$(SIMULATOR_DEVICE_TYPE="%(device_type)s" SIMULATOR_OS_VERSION="%(os_version)s" "%(create_simulator_action_binary)s")"
+  id="$(SIMULATOR_DEVICE_TYPE="%(device_type)s" SIMULATOR_OS_VERSION="%(os_version)s" SIMULATOR_SDK_VERSION="%(sdk_version)s" "%(create_simulator_action_binary)s")"
   target_flags=(
     "test"
     "--platform=ios_simulator"

apple/testing/default_runner/ios_xctestrun_runner.bzl

@@ -29,6 +29,7 @@ def _get_template_substitutions(
         pre_action_binary,
         random,
         reuse_simulator,
+        sdk_version,
         xcodebuild_args,
         xctestrun_template,
         xctrunner_entitlements_template):
@@ -45,6 +46,7 @@ def _get_template_substitutions(
         "post_action_determines_exit_code": post_action_determines_exit_code,
         "pre_action_binary": pre_action_binary,
         "reuse_simulator": reuse_simulator,
+        "sdk_version": sdk_version,
         # "ordered" isn't a special string, but anything besides "random" for this field runs in order
         "test_order": "random" if random else "ordered",
         "xcodebuild_args": xcodebuild_args,
@@ -64,16 +66,9 @@ def _get_execution_environment(ctx):
 def _ios_xctestrun_runner_impl(ctx):
     # TODO: Remove this getattr when we drop Bazel 8
     xcode_properties_attr = getattr(apple_common, "XcodeProperties", None) or XcodeVersionPropertiesInfo
-    os_version = str(ctx.attr.os_version or ctx.fragments.objc.ios_simulator_version or
-                     ctx.attr._xcode_config[xcode_properties_attr].default_ios_sdk_version)
-
-    # TODO: Ideally we would be smarter about picking a device, but we don't know what the current version of Xcode supports
-    device_type = ctx.attr.device_type or ctx.fragments.objc.ios_simulator_device or "iPhone 15"
-
-    if not os_version:
-        fail("error: os_version must be set on ios_xctestrun_runner, or passed with --ios_simulator_version")
-    if not device_type:
-        fail("error: device_type must be set on ios_xctestrun_runner, or passed with --ios_simulator_device")
+    sdk_version = ctx.attr._xcode_config[xcode_properties_attr].default_ios_sdk_version
+    os_version = str(ctx.attr.os_version or ctx.fragments.objc.ios_simulator_version or "")
+    device_type = ctx.attr.device_type or ctx.fragments.objc.ios_simulator_device or ""
 
     runfiles = ctx.runfiles(files = [
         ctx.file._xctestrun_template,
@@ -114,6 +109,7 @@ def _ios_xctestrun_runner_impl(ctx):
             pre_action_binary = pre_action_binary,
             random = ctx.attr.random,
             reuse_simulator = "true" if ctx.attr.reuse_simulator else "false",
+            sdk_version = sdk_version,
             xcodebuild_args = " ".join(ctx.attr.xcodebuild_args) if ctx.attr.xcodebuild_args else "",
             xctestrun_template = ctx.file._xctestrun_template.short_path,
             xctrunner_entitlements_template = ctx.file._xctrunner_entitlements_template.short_path,
@@ -176,8 +172,9 @@ When executed, the binary will have the following environment variables availabl
 
 <ul>
 <li>`SIMULATOR_DEVICE_TYPE`: The device type of the simulator to create. The supported types correspond to the output of `xcrun simctl list devicetypes`. E.g., iPhone 6, iPad Air. The value will either be the value of the `device_type` attribute, or the `--ios_simulator_device` command-line flag.</li>
-<li>`SIMULATOR_OS_VERSION`: The os version of the simulator to create. The supported os versions correspond to the output of `xcrun simctl list runtimes`. ' 'E.g., 11.2, 9.3. The value will either be the value of the `os_version` attribute, or the `--ios_simulator_version` command-line flag.</li>
+<li>`SIMULATOR_OS_VERSION`: The OS version of the simulator to create. The supported OS versions correspond to the output of `xcrun simctl list runtimes`. E.g., 11.2, 9.3. The value will either be the value of the `os_version` attribute, or the `--ios_simulator_version` command-line flag.</li>
 <li>`SIMULATOR_REUSE_SIMULATOR`: Whether to reuse an existing simulator or create a new one. The value will be set to "1" if the `reuse_simulator` attribute is true, and unset otherwise. Whether or not this variable is respected should be treated as an implementation detail of the simulator creator tool.</li>
+<li>`SIMULATOR_SDK_VERSION`: The SDK version of the simulator to create. The supported SDK builds correspond to the output of `xcrun simctl runtime match list`. E.g., 11.2, 9.3. The value will be derived from the `default_ios_sdk_version` for the current Xcode version.</li>
 </ul>
 """,
         ),

apple/testing/default_runner/ios_xctestrun_runner.template.sh

@@ -399,7 +399,12 @@ fi
 
 simulator_id="unused"
 if [[ "$build_for_device" == false ]]; then
-  simulator_id="$(SIMULATOR_DEVICE_TYPE="%(device_type)s" SIMULATOR_OS_VERSION="%(os_version)s" SIMULATOR_REUSE_SIMULATOR="${reuse_simulator:-}" "%(create_simulator_action_binary)s")"
+  if [[ -z "%(device_type)s" ]]; then
+    echo "error: to create a simulator; the device type must always be set on the test runner or with '--ios_simulator_device'" >&2
+    exit 1
+  fi
+
+  simulator_id="$(SIMULATOR_DEVICE_TYPE="%(device_type)s" SIMULATOR_OS_VERSION="%(os_version)s" SIMULATOR_REUSE_SIMULATOR="${reuse_simulator:-}" SIMULATOR_SDK_VERSION="%(sdk_version)s" "%(create_simulator_action_binary)s")"
 fi
 
 test_exit_code=0

apple/testing/default_runner/simulator_creator.py

@@ -21,11 +21,12 @@
 import subprocess
 import sys
 import time
+from dataclasses import dataclass
 from typing import List, Optional
 
 
 def _simctl(extra_args: List[str]) -> str:
-    return subprocess.check_output(["xcrun", "simctl"] + extra_args).decode()
+    return subprocess.check_output(["xcrun", "simctl", *extra_args], text=True)
 
 
 def _boot_simulator(simulator_id: str) -> None:
@@ -82,23 +83,64 @@ def _boot_simulator(simulator_id: str) -> None:
     time.sleep(3)
 
 
-def _device_name(device_type: str, os_version: str) -> str:
+@dataclass
+class _Runtime:
+    identifier: str
+    platform: str
+    version: str
+
+
+def _selected_simulator_runtime(
+    os_version: Optional[str], sdk_version: Optional[str]
+) -> _Runtime:
+    runtimes = json.loads(_simctl(["list", "runtimes", "-j"]))["runtimes"]
+    available_runtimes = [
+        runtime
+        for runtime in runtimes
+        if runtime["platform"] == "iOS" and runtime["isAvailable"]
+    ]
+    if not available_runtimes:
+        raise RuntimeError("no available runtimes found")
+    sdk_runtime_match = None
+    if sdk_version:
+        sdk_name = f"iphoneos{sdk_version}"
+        sdk_runtime_matches = json.loads(_simctl(["runtime", "match", "list", "-j"]))
+        if sdk_name not in sdk_runtime_matches:
+            raise RuntimeError(
+                f"no sdk build to runtime mapping found matching {sdk_name}"
+            )
+        sdk_runtime_match = sdk_runtime_matches[sdk_name]
+    for runtime in available_runtimes:
+        if os_version and runtime["version"] == os_version:
+            return _Runtime(
+                identifier=runtime["identifier"],
+                platform=runtime["platform"],
+                version=runtime["version"],
+            )
+        elif (
+            sdk_runtime_match
+            and runtime["buildversion"] == sdk_runtime_match["chosenRuntimeBuild"]
+        ):
+            return _Runtime(
+                identifier=runtime["identifier"],
+                platform=runtime["platform"],
+                version=runtime["version"],
+            )
+    raise RuntimeError("no matching runtimes found")
+
+
+def _default_device_name(device_type: str, os_version: str) -> str:
     return f"BAZEL_TEST_{device_type}_{os_version}"
 
 
 def _create_and_boot_simulator(
-    os_version: str,
+    device_name: str,
     device_type: str,
-    name: Optional[str],
+    runtime_identifier: str,
     reuse_simulator: bool,
 ) -> str:
     devices = json.loads(_simctl(["list", "devices", "-j"]))["devices"]
-    device_name = name or _device_name(device_type, os_version)
-    runtime_identifier = "com.apple.CoreSimulator.SimRuntime.iOS-{}".format(
-        os_version.replace(".", "-")
-    )
-
-    existing_device=None
+    existing_device = None
 
     if reuse_simulator:
         devices_for_os = devices.get(runtime_identifier) or []
@@ -134,6 +176,7 @@ def _create_and_boot_simulator(
 
 class Namespace(argparse.Namespace):
     os_version: Optional[str]
+    sdk_version: Optional[str]
     device_type: Optional[str]
     simulator_name: Optional[str]
     reuse_simulator: bool
@@ -142,16 +185,22 @@ class Namespace(argparse.Namespace):
 def _build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser()
     parser.add_argument(
-        "--os_version",
+        "--os-version",
         required=False,
         default=None,
-        help="The iOS version to run the tests on, ex: 12.1",
+        help="The OS version to run the tests on, ex: 12.1",
     )
     parser.add_argument(
-        "--device_type",
+        "--sdk-version",
         required=False,
         default=None,
-        help="The iOS device to run the tests on, ex: iPhone X",
+        help="The SDK version the tests were built for, ex: 12.1",
+    )
+    parser.add_argument(
+        "--device-type",
+        required=False,
+        default=None,
+        help="The iOS device to run the tests on, ex: iPhone 15",
     )
     parser.add_argument(
         "--name",
@@ -173,24 +222,27 @@ def _main() -> None:
 
     args = parser.parse_args(namespace=Namespace())
 
-    os_version = args.os_version or os.getenv("SIMULATOR_OS_VERSION")
     device_type = args.device_type or os.getenv("SIMULATOR_DEVICE_TYPE")
-    simulator_name = args.name
-    reuse_simulator: bool = args.reuse_simulator or (
-        os.getenv("SIMULATOR_REUSE_SIMULATOR") is not None
-    )
-
-    if not os_version:
-        parser.error(
-            "OS version must be provided either as an argument or through the SIMULATOR_OS_VERSION environment variable"
-        )
     if not device_type:
         parser.error(
             "Device type must be provided either as an argument or through the SIMULATOR_DEVICE_TYPE environment variable"
         )
 
+    os_version = args.os_version or os.getenv("SIMULATOR_OS_VERSION")
+    sdk_version = args.sdk_version or os.getenv("SIMULATOR_SDK_VERSION")
+    reuse_simulator: bool = args.reuse_simulator or (
+        os.getenv("SIMULATOR_REUSE_SIMULATOR") is not None
+    )
+
+    selected_runtime = _selected_simulator_runtime(os_version, sdk_version)
+    device_name = args.name or _default_device_name(
+        device_type, selected_runtime.version
+    )
+
+    print("Selected simulator runtime", selected_runtime.identifier, file=sys.stderr)
+
     simulator_id = _create_and_boot_simulator(
-        os_version, device_type, simulator_name, reuse_simulator
+        device_name, device_type, selected_runtime.identifier, reuse_simulator
     )
 
     print(simulator_id.strip())