unix: add support for s390x Linux cross builds

This was trivial to get working since Debian has a cross toolchain
for it.

Author: indygreg

.github/workflows/linux.yml

@@ -128,6 +128,7 @@ jobs:
           - 'i686-unknown-linux-gnu'
           - 'mips-unknown-linux-gnu'
           - 'mipsel-unknown-linux-gnu'
+          - 's390x-unknown-linux-gnu'
           - 'x86_64-unknown-linux-gnu'
           - 'x86_64-unknown-linux-musl'
         py:
@@ -162,6 +163,10 @@ jobs:
             py: 'cpython-3.8'
           - triple: 'mipsel-unknown-linux-gnu'
             py: 'cpython-3.10'
+          - triple: 's390x-unknown-linux-gnu'
+            py: 'cpython-3.8'
+          - triple: 's390x-unknown-linux-gnu'
+            py: 'cpython-3.10'
           # We don't publish noopt builds for GNU Linux when PGO is available.
           - triple: 'i686-unknown-linux-gnu'
             optimizations: 'noopt'
@@ -193,6 +198,10 @@ jobs:
             optimizations: 'pgo'
           - triple: 'mipsel-unknown-linux-gnu'
             optimizations: 'pgo+lto'
+          - triple: 's390x-unknown-linux-gnu'
+            optimizations: 'pgo'
+          - triple: 's390x-unknown-linux-gnu'
+            optimizations: 'pgo+lto'
 
     needs:
       - pythonbuild

cpython-unix/build-cpython.sh

@@ -622,6 +622,11 @@ if [ "${BUILD_TRIPLE}" != "${TARGET_TRIPLE}" ]; then
             CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_file__dev_ptc=no"
             CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_file__dev_ptmx=no"
             ;;
+        s390x-unknown-linux-gnu)
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_buggy_getaddrinfo=no"
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_file__dev_ptc=no"
+            CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_file__dev_ptmx=no"
+            ;;
         x86_64-unknown-linux-musl)
             ;;
         *)

cpython-unix/build-libX11.sh

@@ -66,6 +66,9 @@ if [ "${BUILD_TRIPLE}" != "${TARGET_TRIPLE}" ]; then
     mips64el-unknown-linux-gnuabi64)
       EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
       ;;
+    s390x-unknown-linux-gnu)
+      EXTRA_FLAGS="${EXTRA_FLAGS} --enable-malloc0returnsnull"
+      ;;
     *)
       echo "cross-compiling but malloc(0) override not set; failures possible"
       ;;
@@ -91,6 +94,9 @@ case "${TARGET_TRIPLE}" in
   mips64el-unknown-linux-gnuabi64)
     CC_FOR_BUILD=gcc
     ;;
+  s390x-unknown-linux-gnu)
+    CC_FOR_BUILD=gcc
+    ;;
   *)
     CC_FOR_BUILD=clang
     ;;

cpython-unix/build.cross.Dockerfile

@@ -52,9 +52,11 @@ RUN apt-get install \
     gcc-mips-linux-gnu \
     gcc-mips64el-linux-gnuabi64 \
     gcc-mipsel-linux-gnu \
+    gcc-s390x-linux-gnu \
     libc6-dev-arm64-cross \
     libc6-dev-armel-cross \
     libc6-dev-armhf-cross \
     libc6-dev-mips-cross \
     libc6-dev-mips64el-cross \
-    libc6-dev-mipsel-cross
+    libc6-dev-mipsel-cross \
+    libc6-dev-s390x-cross

cpython-unix/build.py

@@ -106,6 +106,8 @@ def add_target_env(env, build_platform, target_triple, build_env):
         env["CC"] = "mipsel-linux-gnu-gcc"
     elif target_triple == "mips64el-unknown-linux-gnuabi64":
         env["CC"] = "mips64el-linux-gnuabi64-gcc"
+    elif target_triple == "s390x-unknown-linux-gnu":
+        env["CC"] = "s390x-linux-gnu-gcc"
     else:
         raise Exception("unhandled target triple: %s" % target_triple)
 
@@ -132,6 +134,8 @@ def add_target_env(env, build_platform, target_triple, build_env):
             env["TARGET_TRIPLE"] = "mipsel-unknown-linux-gnu"
         elif target_triple == "mips64el-unknown-linux-gnuabi64":
             env["TARGET_TRIPLE"] = "mips64el-unknown-linux-gnuabi64"
+        elif target_triple == "s390x-unknown-linux-gnu":
+            env["TARGET_TRIPLE"] = "s390x-unknown-linux-gnu"
         elif target_triple in ("x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl"):
             env["TARGET_TRIPLE"] = "x86_64-unknown-linux-gnu"
         elif target_triple == "i686-unknown-linux-gnu":

cpython-unix/static-modules.3.9.s390x-unknown-linux-gnu

@@ -0,0 +1,43 @@
+# Setup.dist doesn't have entries for all modules. This file defines
+# what's missing. The content here is reconstructed from logic in
+# setup.py and what was observed to execute in a normal build via setup.py.
+# We should audit this every time we upgrade CPython.
+
+_bz2 _bz2module.c -lbz2
+_crypt _cryptmodule.c -lcrypt
+_ctypes _ctypes/_ctypes.c _ctypes/callbacks.c _ctypes/callproc.c _ctypes/stgdict.c _ctypes/cfield.c -I/tools/deps/include -L/tools/deps/lib -lffi -ldl
+_ctypes_test _ctypes/_ctypes_test.c -lm
+_curses _cursesmodule.c -DHAVE_NCURSESW=1 -I/tools/deps/include/ncursesw -L/tools/deps/lib -lncursesw
+_curses_panel _curses_panel.c -DHAVE_NCURSESW=1 -I/tools/deps/include/ncursesw -L/tools/deps/lib -lpanelw -lncursesw
+_dbm _dbmmodule.c -DHAVE_BERKDB_H -DDB_DBM_HSEARCH -I/tools/deps/include -L/tools/deps/lib -ldb
+_decimal _decimal/_decimal.c _decimal/libmpdec/basearith.c _decimal/libmpdec/constants.c _decimal/libmpdec/context.c _decimal/libmpdec/convolute.c _decimal/libmpdec/crt.c _decimal/libmpdec/difradix2.c _decimal/libmpdec/fnt.c _decimal/libmpdec/fourstep.c _decimal/libmpdec/io.c _decimal/libmpdec/mpalloc.c _decimal/libmpdec/mpdecimal.c _decimal/libmpdec/numbertheory.c _decimal/libmpdec/sixstep.c _decimal/libmpdec/transpose.c -DCONFIG_64=1 -DANSI=1 -DHAVE_UINT128_T=1 -IModules/_decimal/libmpdec
+_elementtree _elementtree.c -DHAVE_EXPAT_CONFIG_H=1 -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI -IModules/expat
+_gdbm _gdbmmodule.c -DHAVE_NDBM_H -I/tools/deps/include -L/tools/deps/lib -lgdbm
+_hashlib _hashopenssl.c -I/tools/deps/include -L/tools/deps/lib -lssl -lcrypto
+_json _json.c
+_lsprof _lsprof.c rotatingtree.c
+_lzma _lzmamodule.c -I/tools/deps/include -L/tools/deps/lib -llzma
+# TODO check setup.py logic for semaphore.c and possibly fix missing
+# dependency.
+_multiprocessing _multiprocessing/multiprocessing.c _multiprocessing/semaphore.c
+_opcode _opcode.c
+_posixshmem _multiprocessing/posixshmem.c -IModules/_multiprocessing -lrt
+_queue _queuemodule.c
+_sqlite3 _sqlite/cache.c _sqlite/connection.c _sqlite/cursor.c _sqlite/microprotocols.c _sqlite/module.c _sqlite/prepare_protocol.c _sqlite/row.c _sqlite/statement.c _sqlite/util.c -I/tools/deps/include -IModules/_sqlite -DMODULE_NAME=\"sqlite3\" -DSQLITE_OMIT_LOAD_EXTENSION=1 -L/tools/deps/lib -lsqlite3
+_ssl _ssl.c -I/tools/deps/include -lssl -lcrypto
+_testbuffer _testbuffer.c
+_testimportmultiple _testimportmultiple.c
+_testinternalcapi _testinternalcapi.c -DPy_BUILD_CORE_MODULE
+_testmultiphase _testmultiphase.c
+_tkinter _tkinter.c tkappinit.c -DWITH_APPINIT -I/tools/deps/include/X11 -L/tools/deps/lib -ltcl8.6 -ltk8.6 -lX11 -lxcb -lXau
+_uuid _uuidmodule.c -I/tools/deps/include/uuid -luuid
+_xxsubinterpreters _xxsubinterpretersmodule.c
+_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
+ossaudiodev ossaudiodev.c
+pyexpat pyexpat.c expat/xmlparse.c expat/xmlrole.c expat/xmltok.c -DHAVE_EXPAT_CONFIG_H=1 -DXML_POOR_ENTROPY=1 -DUSE_PYEXPAT_CAPI -IModules/expat
+# readline variant needs to come first because libreadline is in /tools/deps and is
+# picked up by build. We /could/ make libedit first. But since we employ a hack to
+# coerce use of libedit on Linux, it seems prudent for the build system to pick
+# up readline.
+readline VARIANT=readline readline.c -I/tools/deps/include -I/tools/deps/include/ncursesw -L/tools/deps/lib -lreadline -lncursesw
+readline VARIANT=libedit readline-libedit.c -DUSE_LIBEDIT=1 -I/tools/deps/libedit/include -I/tools/deps/libedit/include/ncursesw -L/tools/deps/libedit/lib -ledit -lncursesw

cpython-unix/targets.yml

@@ -215,6 +215,32 @@ mipsel-unknown-linux-gnu:
     - zlib
   openssl_target: linux-mips32
 
+s390x-unknown-linux-gnu:
+  host_platforms:
+    - linux
+  docker_image_suffix: .cross
+  needs:
+    - bdb
+    - binutils
+    - bzip2
+    - gdbm
+    - libedit
+    - libffi
+    - libX11
+    - libXau
+    - libxcb
+    - ncurses
+    - readline
+    - sqlite
+    - tcl
+    - tk
+    - tix
+    - uuid
+    - xorgproto
+    - xz
+    - zlib
+  openssl_target: linux64-s390x
+
 thumb7k-apple-watchos:
   host_platforms:
     - darwin

docs/building.rst

@@ -39,7 +39,7 @@ As are various other targets::
     $ ./build-linux.py --target armv7-unknown-linux-gnueabihf
     $ ./build-linux.py --target mips-unknown-linux-gnu
     $ ./build-linux.py --target mipsel-unknown-linux-gnu
-
+    $ ./build-linux.py --target s390x-unknown-linux-gnu
 
 macOS
 =====

src/main.rs

@@ -33,6 +33,7 @@ const RECOGNIZED_TRIPLES: &[&str] = &[
     "mips-unknown-linux-gnu",
     "mipsel-unknown-linux-gnu",
     "mips64el-unknown-linux-gnuabi64",
+    "s390x-unknown-linux-gnu",
     "thumbv7k-apple-watchos",
     "x86_64-apple-darwin",
     "x86_64-apple-ios",
@@ -145,6 +146,10 @@ static GLIBC_MAX_VERSION_BY_TRIPLE: Lazy<HashMap<&'static str, version_compare::
             "mips64el-unknown-linux-gnuabi64",
             version_compare::Version::from("2.17").unwrap(),
         );
+        versions.insert(
+            "s390x-unknown-linux-gnu",
+            version_compare::Version::from("2.17").unwrap(),
+        );
         versions.insert(
             "x86_64-unknown-linux-gnu",
             version_compare::Version::from("2.17").unwrap(),
@@ -173,6 +178,7 @@ static ELF_ALLOWED_LIBRARIES_BY_TRIPLE: Lazy<HashMap<&'static str, Vec<&'static
             ("mips-unknown-linux-gnu", vec!["ld.so.1"]),
             ("mipsel-unknown-linux-gnu", vec!["ld.so.1"]),
             ("mips64el-unknown-linux-gnuabi64", vec![]),
+            ("s390x-unknown-linux-gnu", vec!["ld64.so.1"]),
             ("x86_64-unknown-linux-gnu", vec!["ld-linux-x86-64.so.2"]),
         ]
         .iter()
@@ -333,6 +339,7 @@ static PLATFORM_TAG_BY_TRIPLE: Lazy<HashMap<&'static str, &'static str>> = Lazy:
         ("mips-unknown-linux-gnu", "linux-mips"),
         ("mipsel-unknown-linux-gnu", "linux-mipsel"),
         ("mips64el-unknown-linux-gnuabi64", "todo"),
+        ("s390x-unknown-linux-gnu", "linux-s390x"),
         ("x86_64-apple-darwin", "macosx-10.9-x86_64"),
         ("x86_64-apple-ios", "iOS-x86_64"),
         ("x86_64-pc-windows-msvc", "win-amd64"),
@@ -370,6 +377,7 @@ fn validate_elf(
         "mips-unknown-linux-gnu" => goblin::elf::header::EM_MIPS,
         "mipsel-unknown-linux-gnu" => goblin::elf::header::EM_MIPS,
         "mips64el-unknown-linux-gnuabi64" => 0,
+        "s390x-unknown-linux-gnu" => goblin::elf::header::EM_S390,
         "x86_64-unknown-linux-gnu" => goblin::elf::header::EM_X86_64,
         "x86_64-unknown-linux-musl" => goblin::elf::header::EM_X86_64,
         _ => panic!("unhandled target triple: {}", target_triple),
@@ -400,7 +408,7 @@ fn validate_elf(
         .expect("max glibc version not defined for target triple");
 
     // functionality doesn't yet support mips.
-    if !target_triple.starts_with("mips") {
+    if !target_triple.starts_with("mips") && !target_triple.starts_with("s390x-") {
         let mut undefined_symbols = tugger_binary_analysis::find_undefined_elf_symbols(&bytes, elf);
         undefined_symbols.sort();