unix: properly define PYTHON_ARCH for more triples

We were inserting malformed symlinks into the archive because we
didn't annotate all target triples. This commit hopefully fixes
that.

As part of debugging this, I also discovered that cross builds
were missing "d" in abiflags. This commit should also fix that.

To prevent this from happening again we teach the validation code to
ensure all symlink targets exist in the archive.

Author: indygreg

cpython-unix/build-cpython.sh

@@ -794,6 +794,12 @@ import os
 import sys
 import sysconfig
 
+# When doing cross builds, sysconfig still picks up abiflags from the
+# host Python, which is never built in debug mode. Patch abiflags accordingly.
+if os.environ.get("CPYTHON_DEBUG") and "d" not in sysconfig.get_config_var("abiflags"):
+    sys.abiflags += "d"
+    sysconfig._CONFIG_VARS["abiflags"] += "d"
+
 metadata = {
     "python_abi_tag": sys.abiflags,
     "python_implementation_cache_tag": sys.implementation.cache_tag,
@@ -817,7 +823,6 @@ metadata = {
     "python_config_vars": {k: str(v) for k, v in sysconfig.get_config_vars().items()},
 }
 
-
 # When cross-compiling, we use a host Python to run this script. There are
 # some hacks to get sysconfig to pick up the correct data file. However,
 # these hacks don't work for sysconfig.get_paths() and we get paths to the host
@@ -864,11 +869,43 @@ touch "${LIB_DYNLOAD}/.empty"
 
 # Symlink libpython so we don't have 2 copies.
 if [ -n "${PYTHON_BINARY_SUFFIX}" ]; then
-    if [ "${PYBUILD_PLATFORM}" = "macos" ]; then
-        PYTHON_ARCH="darwin"
-    else
-        PYTHON_ARCH="x86_64-linux-gnu"
-    fi
+    case "${TARGET_TRIPLE}" in
+        aarch64-unknown-linux-gnu)
+            PYTHON_ARCH="aarch64-linux-gnu"
+            ;;
+        # This is too aggressive. But we don't have patches in place for
+        # setting the platform name properly on non-Darwin.
+        *-apple-*)
+            PYTHON_ARCH="darwin"
+            ;;
+        armv7-unknown-linux-gnueabi)
+            PYTHON_ARCH="armv7-linux-gnueabi"
+            ;;
+        armv7-unknown-linux-gnueabihf)
+            PYTHON_ARCH="armv7-linux-gnueabihf"
+            ;;
+        i686-unknown-linux-gnu)
+            PYTHON_ARCH="i386-linux-gnu"
+            ;;
+        mips-unknown-linux-gnu)
+            PYTHON_ARCH="mips-linux-gnu"
+            ;;
+        mipsel-unknown-linux-gnu)
+            PYTHON_ARCH="mipsel-linux-gnu"
+            ;;
+        mips64el-unknown-linux-gnuabi64)
+            PYTHON_ARCH="mips64el-linux-gnuabi64"
+            ;;
+        s390x-unknown-linux-gnu)
+            PYTHON_ARCH="s390x-linux-gnu"
+            ;;
+        x86_64-unknown-linux-*)
+            PYTHON_ARCH="x86_64-linux-gnu"
+            ;;
+        *)
+            echo "unhandled target triple: ${TARGET_TRIPLE}"
+            exit 1
+    esac
 
     LIBPYTHON=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.a
     ln -sf \

src/main.rs

@@ -612,7 +612,7 @@ fn validate_possible_object_file(
     Ok((errors, seen_dylibs))
 }
 
-fn validate_json(json: &PythonJsonMain, triple: &str) -> Result<Vec<String>> {
+fn validate_json(json: &PythonJsonMain, triple: &str, is_debug: bool) -> Result<Vec<String>> {
     let mut errors = vec![];
 
     if json.version != "7" {
@@ -650,6 +650,16 @@ fn validate_json(json: &PythonJsonMain, triple: &str) -> Result<Vec<String>> {
         ));
     }
 
+    if is_debug
+        && !json
+            .python_config_vars
+            .get("abiflags")
+            .unwrap()
+            .contains('d')
+    {
+        errors.push("abiflags does not contain 'd'".to_string());
+    }
+
     for extension in json.build_info.extensions.keys() {
         if GLOBALLY_BANNED_EXTENSIONS.contains(&extension.as_str()) {
             errors.push(format!("banned extension detected: {}", extension));
@@ -663,6 +673,7 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
     let mut errors = vec![];
     let mut seen_dylibs = BTreeSet::new();
     let mut seen_paths = BTreeSet::new();
+    let mut seen_symlink_targets = BTreeSet::new();
 
     let dist_filename = dist_path
         .file_name()
@@ -696,6 +707,8 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
         return Err(anyhow!("could not parse Python version from filename"));
     };
 
+    let is_debug = dist_filename.contains("-debug-");
+
     let reader = std::io::BufReader::new(fh);
     let dctx = zstd::stream::Decoder::new(reader)?;
     let mut tf = tar::Archive::new(dctx);
@@ -707,10 +720,12 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
 
     let mut entry = entries.next().unwrap()?;
     if entry.path()?.display().to_string() == "python/PYTHON.json" {
+        seen_paths.insert(entry.path()?.to_path_buf());
+
         let mut data = Vec::new();
         entry.read_to_end(&mut data)?;
         let json = parse_python_json(&data).context("parsing PYTHON.json")?;
-        errors.extend(validate_json(&json, triple)?);
+        errors.extend(validate_json(&json, triple, is_debug)?);
 
         wanted_python_paths.extend(json.python_paths.values().map(|x| format!("python/{}", x)));
     } else {
@@ -726,6 +741,10 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
 
         seen_paths.insert(path.clone());
 
+        if let Some(link_name) = entry.link_name()? {
+            seen_symlink_targets.insert(path.parent().unwrap().join(link_name));
+        }
+
         // If this path starts with a path referenced in wanted_python_paths,
         // remove the prefix from wanted_python_paths so we don't error on it
         // later.
@@ -773,7 +792,16 @@ fn validate_distribution(dist_path: &Path) -> Result<Vec<String>> {
 
         if path == PathBuf::from("python/PYTHON.json") {
             let json = parse_python_json(&data).context("parsing PYTHON.json")?;
-            errors.extend(validate_json(&json, triple)?);
+            errors.extend(validate_json(&json, triple, is_debug)?);
+        }
+    }
+
+    for path in seen_symlink_targets {
+        if !seen_paths.contains(&path) {
+            errors.push(format!(
+                "symlink target {} referenced in archive but not found",
+                path.display()
+            ));
         }
     }