global: add Apple SDK metadata to PYTHON.json

indygreg/PyOxidizer#373 has revealed issues
relinking the Python distributions on older Apple SDKs and Clang
toolchains. In order to help downstream consumers validate they are
using a sufficiently modern SDK and toolchain, this commit adds metadata
to the Python distribution to advertise information about the Apple
SDK used to build and its targeting settings.

As part of this, we teach the Rust validation code to ensure fields
are present on Apple triples.

Author: indygreg

cpython-unix/build.py

@@ -120,6 +120,7 @@ def add_target_env(env, build_platform, target_triple, build_env):
             raise Exception("could not find minimum Apple SDK version in cflags")
 
         sdk_platform = settings["apple_sdk_platform"]
+        env["APPLE_SDK_PLATFORM"] = sdk_platform
 
         env["TARGET_TRIPLE"] = target_triple
 
@@ -148,6 +149,13 @@ def add_target_env(env, build_platform, target_triple, build_env):
         if not os.path.exists(sdk_path):
             raise Exception("macOS SDK path %s does not exist" % sdk_path)
 
+        # Grab the version from the SDK so we can put it in PYTHON.json.
+        sdk_settings_path = pathlib.Path(sdk_path) / "SDKSettings.json"
+        with sdk_settings_path.open("rb") as fh:
+            sdk_settings = json.load(fh)
+            env["APPLE_SDK_VERSION"] = sdk_settings["Version"]
+            env["APPLE_SDK_CANONICAL_NAME"] = sdk_settings["CanonicalName"]
+
         extra_target_cflags.extend(["-isysroot", sdk_path])
         extra_target_ldflags.extend(["-isysroot", sdk_path])
 
@@ -854,7 +862,7 @@ def build_cpython(
 
         # Create PYTHON.json file describing this distribution.
         python_info = {
-            "version": "6",
+            "version": "7",
             "target_triple": target_triple,
             "optimizations": optimizations,
             "python_tag": entry["python_tag"],
@@ -890,6 +898,14 @@ def build_cpython(
             "tk8.6",
         ]
 
+        if "-apple" in target_triple:
+            python_info["apple_sdk_platform"] = env["APPLE_SDK_PLATFORM"]
+            python_info["apple_sdk_version"] = env["APPLE_SDK_VERSION"]
+            python_info["apple_sdk_canonical_name"] = env["APPLE_SDK_CANONICAL_NAME"]
+            python_info["apple_sdk_deployment_target"] = env[
+                "APPLE_MIN_DEPLOYMENT_TARGET"
+            ]
+
         # Add metadata derived from built distribution.
         extra_metadata = build_env.get_file("metadata.json")
         python_info.update(json.loads(extra_metadata))

cpython-windows/build.py

@@ -2204,7 +2204,7 @@ def build_cpython(
 
         # Create PYTHON.json file describing this distribution.
         python_info = {
-            "version": "6",
+            "version": "7",
             "target_triple": target_triple,
             "optimizations": optimizations,
             "python_tag": entry["python_tag"],

docs/distributions.rst

@@ -32,7 +32,7 @@ without having to resort to heuristics.
 The file contains a JSON map. This map has the following keys:
 
 version
-   Version number of the file format. Currently ``6``.
+   Version number of the file format. Currently ``7``.
 
 target_triple
    A target triple defining the platform and architecture of the machine
@@ -279,6 +279,44 @@ python_extension_module_loading
 
    (Version 5 or above only.)
 
+apple_sdk_canonical_name
+   Optional canonical name of Apple SDK used to build.
+
+   Should only be present for target triples with ``apple`` in them.
+
+   The canonical name can be used to find a copy of this SDK on another
+   machine.
+
+   (Version 7 or above only.)
+
+apple_sdk_platform
+   Optional name of the platform of the Apple SDK used to build.
+
+   Should only be present for target triples with ``apple`` in them.
+
+   e.g. ``macosx``
+
+   (Version 7 or above only.)
+
+apple_sdk_version
+   Optional version of the Apple SDK used to build.
+
+   Should only be present for target triples with ``apple`` in them.
+
+   If relinking build artifacts, ideally this exact SDK version is used.
+   Newer versions will likely work. Older versions or Clang toolchains
+   associated with older versions may not.
+
+   (Version 7 or above only.)
+
+apple_sdk_deployment_target
+   Optional version of the Apple SDK deployment target used to build.
+
+   This effectively establishes a minimum version of the target operating
+   system this binary is purportedly compatible with.
+
+   (Version 7 or above only.)
+
 crt_features
    Describes C Runtime features/requirements for binaries.
 

src/json.rs

@@ -57,6 +57,10 @@ pub struct PythonBuildInfo {
 #[derive(Debug, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct PythonJsonMain {
+    pub apple_sdk_canonical_name: Option<String>,
+    pub apple_sdk_deployment_target: Option<String>,
+    pub apple_sdk_platform: Option<String>,
+    pub apple_sdk_version: Option<String>,
     pub build_info: PythonBuildInfo,
     pub crt_features: Vec<String>,
     pub libpython_link_mode: String,

src/main.rs

@@ -615,13 +615,29 @@ fn validate_possible_object_file(
 fn validate_json(json: &PythonJsonMain, triple: &str) -> Result<Vec<String>> {
     let mut errors = vec![];
 
-    if json.version != "6" {
+    if json.version != "7" {
         errors.push(format!(
-            "expected version 6 in PYTHON.json; got {}",
+            "expected version 7 in PYTHON.json; got {}",
             json.version
         ));
     }
 
+    // Distributions built with Apple SDKs should have SDK metadata.
+    if triple.contains("-apple-") {
+        if json.apple_sdk_canonical_name.is_none() {
+            errors.push("JSON missing apple_sdk_canonical_name on Apple triple".to_string());
+        }
+        if json.apple_sdk_deployment_target.is_none() {
+            errors.push("JSON missing apple_sdk_deployment_target on Apple triple".to_string());
+        }
+        if json.apple_sdk_platform.is_none() {
+            errors.push("JSON missing apple_sdk_platform on Apple triple".to_string());
+        }
+        if json.apple_sdk_version.is_none() {
+            errors.push("JSON missing apple_sdk_version on Apple triple".to_string());
+        }
+    }
+
     let wanted_platform_tag = PLATFORM_TAG_BY_TRIPLE
         .get(triple)
         .ok_or_else(|| anyhow!("platform tag not defined for triple {}", triple))?