feat: add manylinux_2_31_armv7l image (#1699)

This reverts commit e09f5845680c23699efbb240d8e466efe2de11ed.

Author: mayeut

.github/workflows/build.yml

@@ -54,7 +54,7 @@ jobs:
               ("x86_64", "ubuntu-24.04", ("manylinux2014", "manylinux_2_28", "manylinux_2_34", "musllinux_1_2")),
               ("aarch64", "ubuntu-24.04-arm", ("manylinux2014", "manylinux_2_28", "manylinux_2_34", "musllinux_1_2")),
               ("i686", "ubuntu-24.04", ("manylinux2014", "musllinux_1_2")),
-              ("armv7l", "ubuntu-24.04-arm", ("musllinux_1_2",)),
+              ("armv7l", "ubuntu-24.04-arm", ("manylinux_2_31", "musllinux_1_2")),
           ]
           expanded = [{"policy": policy, "platform": platform, "runner": runner} for platform, runner, policies in reduced for policy in policies]
           print(json.dumps(expanded, indent=2))

README.rst

@@ -120,6 +120,37 @@ distros using glibc 2.34 or later, including:
 - CentOS/RHEL 9+
 
 
+
+manylinux_2_31 (Ubuntu 20.04 based) - armv7l only - BETA
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Caveat:
+
+Only Debian derivatives are available for armv7l. They do not provide recent builds of the GCC toolchain
+compatible with a vanilla install of the distribution. As such, we only get the GCC toolchain shipped with
+the base distribution.
+
+The package manager & packages names are different than what is found on other manylinux images.
+Other images are using RHEL derivatives only for now so using yum/dnf as a package manager and RHEL like
+packages names this image is using apt and Debian like packages names.
+
+If one depends on let's say OpenSSL development package, then, the commands to issue to install it are a bit different:
+
+- ``dnf -y install openssl-devel`` on RHEL derivatives
+- ``apt-get update && apt-get install -y libssl-dev`` on Debian derivatives
+
+
+Toolchain: GCC 9
+
+- armv7l image: ``quay.io/pypa/manylinux_2_31_armv7l``
+
+Built wheels are also expected to be compatible with other
+distros using glibc 2.31 or later, including:
+
+- Debian 11+
+- Ubuntu 20.04+
+
+
 manylinux_2_28 (AlmaLinux 8 based)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

build.sh

@@ -37,6 +37,11 @@ elif [ "${POLICY}" == "manylinux_2_28" ]; then
 	DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-14/root"
 	PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
 	LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst"
+elif [ "${POLICY}" == "manylinux_2_31" ]; then
+	BASEIMAGE="ubuntu:20.04"
+	DEVTOOLSET_ROOTPATH=
+	PREPEND_PATH=
+	LD_LIBRARY_PATH_ARG=
 elif [ "${POLICY}" == "manylinux_2_34" ]; then
 	BASEIMAGE="almalinux:9"
 	DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-14/root"

docker/build_scripts/build-cpython.sh

@@ -51,7 +51,9 @@ if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ] ; then
 	export TCLTK_LIBS="-ltk8.6 -ltcl8.6"
 fi
 
-if [ "${BASE_POLICY}_${AUDITWHEEL_ARCH}" == "musllinux_armv7l" ]; then
+if [ "${BASE_POLICY}_${AUDITWHEEL_ARCH}" == "manylinux_armv7l" ]; then
+	CONFIGURE_ARGS+=(--build=armv7l-unknown-linux-gnueabihf)
+elif [ "${BASE_POLICY}_${AUDITWHEEL_ARCH}" == "musllinux_armv7l" ]; then
 	CONFIGURE_ARGS+=(--build=arm-linux-musleabihf)
 fi
 

docker/build_scripts/build-openssl.sh

@@ -32,6 +32,8 @@ fi
 
 if [ "${OS_ID_LIKE}" = "rhel" ];then
 	manylinux_pkg_remove openssl-devel
+elif [ "${OS_ID_LIKE}" = "debian" ];then
+	manylinux_pkg_remove libssl-dev
 elif [ "${OS_ID_LIKE}" = "alpine" ]; then
 	manylinux_pkg_remove openssl-dev
 fi

docker/build_scripts/build_utils.sh

@@ -19,6 +19,8 @@ else
 		PACKAGE_MANAGER=dnf
 	elif command -v yum >/dev/null 2>&1; then
 		PACKAGE_MANAGER=yum
+	elif command -v apt-get >/dev/null 2>&1; then
+		PACKAGE_MANAGER=apt
 	else
 		echo "unsupported image"
 		exit 1
@@ -29,6 +31,7 @@ fi
 OS_ID_LIKE=$(. /etc/os-release; echo "${ID} ${ID_LIKE:-}")
 case "${OS_ID_LIKE}" in
 	*rhel*) OS_ID_LIKE=rhel;;
+	*debian) OS_ID_LIKE=debian;;
 	*alpine*) OS_ID_LIKE=alpine;;
 	*) echo "unsupported image"; exit 1;;
 esac
@@ -89,6 +92,9 @@ function manylinux_pkg_install {
 		yum -y install "$@"
 	elif [ "${PACKAGE_MANAGER}" = "dnf" ]; then
 		dnf -y install --allowerasing "$@"
+	elif  [ "${PACKAGE_MANAGER}" = "apt" ]; then
+		DEBIAN_FRONTEND=noninteractive apt-get update -qq
+		DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends "$@"
 	elif [ "${PACKAGE_MANAGER}" = "apk" ]; then
 		apk add --no-cache "$@"
 	else
@@ -101,6 +107,8 @@ function manylinux_pkg_remove {
 		yum erase -y "$@"
 	elif [ "${PACKAGE_MANAGER}" = "dnf" ];then
 		dnf erase -y "$@"
+	elif [ "${PACKAGE_MANAGER}" = "apt" ];then
+		DEBIAN_FRONTEND=noninteractive apt-get remove -y "$@"
 	elif [ "${PACKAGE_MANAGER}" = "apk" ]; then
 		apk del "$@"
 	else
@@ -115,6 +123,9 @@ function manylinux_pkg_clean {
 	elif [ "${PACKAGE_MANAGER}" = "dnf" ]; then
 		dnf clean all
 		rm -rf /var/cache/dnf
+	elif  [ "${PACKAGE_MANAGER}" = "apt" ]; then
+		DEBIAN_FRONTEND=noninteractive apt-get clean -qq
+		rm -rf /var/lib/apt/lists/*
 	elif [ "${PACKAGE_MANAGER}" = "apk" ]; then
 		:
 	else

docker/build_scripts/install-build-packages.sh

@@ -23,6 +23,8 @@ if [ "${OS_ID_LIKE}" = "rhel" ]; then
 	else
 		COMPILE_DEPS+=(libidn2-devel tk-devel)
 	fi
+elif [ "${OS_ID_LIKE}" == "debian" ]; then
+	COMPILE_DEPS=(libbz2-dev libncurses-dev libreadline-dev tk-dev libgdbm-dev libdb-dev libpcap-dev liblzma-dev openssl libssl-dev libkeyutils-dev libkrb5-dev comerr-dev libidn2-0-dev libcurl4-openssl-dev uuid-dev libffi-dev linux-headers-generic)
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
 	COMPILE_DEPS=(bzip2-dev ncurses-dev readline-dev tk-dev gdbm-dev libpcap-dev xz-dev openssl openssl-dev keyutils-dev krb5-dev libcom_err libidn-dev curl-dev util-linux-dev libffi-dev linux-headers)
 else

docker/build_scripts/install-runtime-packages.sh

@@ -35,6 +35,8 @@ source "${MY_DIR}/build_utils.sh"
 # MANYLINUX_DEPS: Install development packages (except for libgcc which is provided by gcc install)
 if [ "${OS_ID_LIKE}" == "rhel" ]; then
 	MANYLINUX_DEPS=(glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel libICE-devel libSM-devel zlib-devel expat-devel)
+elif [ "${OS_ID_LIKE}" == "debian" ]; then
+  MANYLINUX_DEPS=(libc6-dev libglib2.0-dev libx11-dev libxext-dev libxrender-dev libgl1-mesa-dev libice-dev libsm-dev zlib1g-dev libexpat1-dev libssl-dev)
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
 	MANYLINUX_DEPS=(musl-dev libstdc++ glib-dev libx11-dev libxext-dev libxrender-dev mesa-dev libice-dev libsm-dev zlib-dev expat-dev)
 else
@@ -54,6 +56,13 @@ if [ "${OS_ID_LIKE}" == "rhel" ]; then
 		# for graalpy
 		RUNTIME_DEPS+=(libxcrypt-compat)
 	fi
+elif [ "${OS_ID_LIKE}" == "debian" ]; then
+  RUNTIME_DEPS=(zlib1g libbz2-1.0 libexpat1 libncurses6 libreadline8 tk libgdbm6 libdb5.3 libpcap0.8 liblzma5 libkeyutils1 libkrb5-3 libcom-err2 libidn2-0 libcurl4 uuid)
+  if [ "${AUDITWHEEL_POLICY}" == "manylinux_2_31" ]; then
+  	RUNTIME_DEPS+=(libffi7 libssl1.1)
+  else
+  	RUNTIME_DEPS+=(libffi8 libssl3)
+  fi
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
 	RUNTIME_DEPS=(zlib bzip2 expat ncurses-libs readline tk gdbm db xz openssl keyutils-libs krb5-libs libcom_err libidn2 libcurl libuuid libffi)
 else
@@ -120,6 +129,9 @@ elif [ "${OS_ID_LIKE}" == "rhel" ]; then
 	if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
 		TOOLCHAIN_DEPS+=(yasm)
 	fi
+elif [ "${OS_ID_LIKE}" == "debian" ]; then
+	TOOLCHAIN_DEPS+=(binutils gcc g++ gfortran)
+	BASE_TOOLS+=(gpg gpg-agent hardlink hostname locales xz-utils)
 elif [ "${OS_ID_LIKE}" == "alpine" ]; then
 	TOOLCHAIN_DEPS=(binutils gcc g++ gfortran)
 	BASE_TOOLS+=(gnupg util-linux shadow tar)

docker/build_scripts/update-system-packages.sh

@@ -21,6 +21,9 @@ if [ "${PACKAGE_MANAGER}" == "yum" ]; then
 	fi
 elif [ "${PACKAGE_MANAGER}" == "dnf" ]; then
 	dnf -y upgrade
+elif [ "${PACKAGE_MANAGER}" == "apt" ]; then
+	DEBIAN_FRONTEND=noninteractive apt-get update -qq
+	DEBIAN_FRONTEND=noninteractive apt-get upgrade -qq -y
 elif [ "${PACKAGE_MANAGER}" == "apk" ]; then
 	apk upgrade --no-cache
 else
@@ -31,20 +34,24 @@ manylinux_pkg_clean
 fixup-mirrors
 
 # do we want to update locales ?
-if [ "${OS_ID_LIKE}" == "rhel" ]; then
+if [ "${OS_ID_LIKE}" == "rhel" ] || [ "${OS_ID_LIKE}" == "debian" ]; then
 	LOCALE_ARCHIVE=/usr/lib/locale/locale-archive
 	TIMESTAMP_FILE=${LOCALE_ARCHIVE}.ml.timestamp
 	if [ ! -f "${TIMESTAMP_FILE}" ] || [ "${LOCALE_ARCHIVE}" -nt "${TIMESTAMP_FILE}" ]; then
 		# upgrading glibc-common can end with removal on en_US.UTF-8 locale
 		localedef -i en_US -f UTF-8 en_US.UTF-8
 
 		# if we updated glibc, we need to strip locales again...
-		if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
-			localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
-		fi
-		if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
-			mv -f "${LOCALE_ARCHIVE}" "${LOCALE_ARCHIVE}.tmpl"
-			build-locale-archive --install-langs="en_US.utf8"
+		if [ "${OS_ID_LIKE}" == "rhel" ]; then
+			if localedef --list-archive | grep -sq -v -i ^en_US.utf8; then
+				localedef --list-archive | grep -v -i ^en_US.utf8 | xargs localedef --delete-from-archive
+			fi
+			if [ "${AUDITWHEEL_POLICY}" == "manylinux2014" ]; then
+				mv -f "${LOCALE_ARCHIVE}" "${LOCALE_ARCHIVE}.tmpl"
+				build-locale-archive --install-langs="en_US.utf8"
+			fi
+		elif [ "${OS_ID_LIKE}" == "debian" ]; then
+			update-locale LANG=en_US.UTF-8
 		fi
 		touch ${TIMESTAMP_FILE}
 	fi