Add zstd support on Unix

Author: zanieb

cpython-unix/Makefile

@@ -236,6 +236,9 @@ $(OUTDIR)/xz-$(XZ_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/
 $(OUTDIR)/zlib-$(ZLIB_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-zlib.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) zlib
 
+$(OUTDIR)/zstd-$(ZSTD_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-zstd.sh
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) zstd
+
 PYTHON_HOST_DEPENDS := \
   $(PYTHON_DEP_DEPENDS) \
   $(HERE)/build-cpython-host.sh \
@@ -272,6 +275,7 @@ PYTHON_DEPENDS_$(1) := \
     $$(if $$(NEED_UUID),$$(OUTDIR)/uuid-$$(UUID_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_XZ),$$(OUTDIR)/xz-$$(XZ_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_ZLIB),$$(OUTDIR)/zlib-$$(ZLIB_VERSION)-$$(PACKAGE_SUFFIX).tar) \
+    $$(if $$(NEED_ZSTD),$$(OUTDIR)/zstd-$$(ZSTD_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(NULL)
 
 ALL_PYTHON_DEPENDS_$(1) = \

cpython-unix/build-cpython.sh

@@ -1213,7 +1213,7 @@ ${BUILD_PYTHON} ${ROOT}/fix_shebangs.py ${ROOT}/out/python/install
 # downstream consumers.
 OBJECT_DIRS="Objects Parser Parser/lexer Parser/pegen Parser/tokenizer Programs Python Python/deepfreeze"
 OBJECT_DIRS="${OBJECT_DIRS} Modules"
-for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _sha3 _sqlite _sre _testinternalcapi _xxtestfuzz ; do
+for ext in _blake2 cjkcodecs _ctypes _ctypes/darwin _decimal _expat _hacl _io _multiprocessing _sha3 _sqlite _sre _testinternalcapi _xxtestfuzz _zstd; do
     OBJECT_DIRS="${OBJECT_DIRS} Modules/${ext}"
 done
 

cpython-unix/build-zstd.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=`pwd`
+
+export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
+export PREFIX="/tools/deps"
+
+tar -xf zstd-${ZSTD_VERSION}.tar.gz
+
+pushd cpython-source-deps-zstd-${ZSTD_VERSION}/lib
+
+if [ "${CC}" = "musl-clang" ]; then
+    # In order to build the library with SSE2, BMI, and AVX2 intrinstics, we need musl-clang to find
+    # headers that provide access to the intrinsics, as they are not provided by musl. These are
+    # part of the include files that are part of clang. But musl-clang eliminates them from the
+    # default include path. So copy them into place.
+    for h in ${TOOLS_PATH}/${TOOLCHAIN}/lib/clang/*/include/*intrin.h ${TOOLS_PATH}/${TOOLCHAIN}/lib/clang/*/include/{emmintrin.h,immintrin.h}; do
+        filename=$(basename "$h")
+        if [ -e "${TOOLS_PATH}/host/include/${filename}" ]; then
+            echo "${filename} already exists; don't need to copy!"
+        fi
+        cp "$h" ${TOOLS_PATH}/host/include/
+    done
+
+    EXTRA_TARGET_CFLAGS="${EXTRA_TARGET_CFLAGS} -I${TOOLS_PATH}/host/include/"
+
+    # `qsort_r` is not available in musl and the zstd source provides a fallback implementation, but
+    # they do not check if they actually have glibc, so we patch that to skip the invalid use.
+    patch -p1 <<EOF
+diff --git a/lib/dictBuilder/cover.c b/lib/dictBuilder/cover.c
+index 5e6e8bc..6ca72a1 100644
+--- a/lib/dictBuilder/cover.c
++++ b/lib/dictBuilder/cover.c
+@@ -328,7 +328,7 @@ static void stableSort(COVER_ctx_t *ctx) {
+     qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
+             ctx,
+             (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp));
+-#elif defined(_GNU_SOURCE)
++#elif defined(_GNU_SOURCE) && defined(__GLIBC__)
+     qsort_r(ctx->suffix, ctx->suffixSize, sizeof(U32),
+             (ctx->d <= 8 ? &COVER_strict_cmp8 : &COVER_strict_cmp),
+             ctx);
+EOF
+    # # The SIMD intrinsics for SSE2 require `emmintrin.h` which is not available in musl. We disable
+    # # all intrinsics rather than just SSE2 with `-mno-sse2` because clang will crash. This may be
+    # # resolved when we upgrade the musl builds to user a newer version of clang.
+    # EXTRA_TARGET_CFLAGS="${EXTRA_TARGET_CFLAGS} -DZSTD_NO_INTRINSICS"
+    # # There's code that does not respect `ZSTD_NO_INTRINSICS` and attempts to use AVX2 intrinsics
+    # # via `immintrin.h`.
+    # EXTRA_TARGET_CFLAGS="${EXTRA_TARGET_CFLAGS} -mno-avx2"
+    # # There's legacy code which attempts to use the `bextr` BMI intrinstic via `immintrin.h`, but
+    # # this is gated behind a check for `__GNUC__`, let's let them know we're not using GNU C. We
+    # # could also set `-mno-bmi`.
+    # EXTRA_TARGET_CFLAGS="${EXTRA_TARGET_CFLAGS} -U__GNUC__"
+    # # We unset `_GNU_SOURCE` because it toggles the use of `qsort_r` which is not available in musl,
+    # # the zstd source provides a fallback implementation if needed.
+    # EXTRA_TARGET_CFLAGS="${EXTRA_TARGET_CFLAGS} -U_GNU_SOURCE"
+fi
+
+CFLAGS="${EXTRA_TARGET_CFLAGS} -fPIC" LDFLAGS="${EXTRA_TARGET_LDFLAGS}" make -j ${NUM_CPUS} libzstd.a
+make -j ${NUM_CPUS} install-static DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} install-includes DESTDIR=${ROOT}/out
+make -j ${NUM_CPUS} install-pc DESTDIR=${ROOT}/out

cpython-unix/build.py

@@ -1163,6 +1163,7 @@ def main():
             "xtrans",
             "xz",
             "zlib",
+            "zstd",
         ):
             tools_path = "host" if action in ("m4", "patchelf") else "deps"
 

cpython-unix/extension-modules.yml

@@ -772,15 +772,14 @@ _xxtestfuzz:
     - _xxtestfuzz/fuzzer.c
 
 _zstd:
-  # Disable on all targets until we add a zstd library
-  disabled-targets:
-    - .*
   minimum-python-version: '3.14'
   sources:
     - _zstd/_zstdmodule.c
-    - _zstd/zdict.c
+    - _zstd/zstddict.c
     - _zstd/compressor.c
     - _zstd/decompressor.c
+  links:
+    - zstd
 
 _zoneinfo:
   minimum-python-version: "3.9"

cpython-unix/targets.yml

@@ -108,6 +108,7 @@ aarch64-apple-darwin:
     - tk
     - uuid
     - xz
+    - zstd
   openssl_target: darwin64-arm64-cc
 
 aarch64-apple-ios:
@@ -151,6 +152,7 @@ aarch64-apple-ios:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: ios64-cross
 
 aarch64-unknown-linux-gnu:
@@ -198,6 +200,7 @@ aarch64-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-aarch64
   # Blocked on:
   #   BOLT-ERROR: Cannot relax adr in non-simple function
@@ -245,6 +248,7 @@ arm64-apple-tvos:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: todo
 
 armv7-unknown-linux-gnueabi:
@@ -286,6 +290,7 @@ armv7-unknown-linux-gnueabi:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-armv4
 
 armv7-unknown-linux-gnueabihf:
@@ -327,6 +332,7 @@ armv7-unknown-linux-gnueabihf:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-armv4
 
 mips-unknown-linux-gnu:
@@ -368,6 +374,7 @@ mips-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-mips32
 
 mipsel-unknown-linux-gnu:
@@ -409,6 +416,7 @@ mipsel-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-mips32
 
 ppc64le-unknown-linux-gnu:
@@ -450,6 +458,7 @@ ppc64le-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-ppc64le
 
 riscv64-unknown-linux-gnu:
@@ -491,6 +500,7 @@ riscv64-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux64-riscv64
 
 s390x-unknown-linux-gnu:
@@ -532,6 +542,7 @@ s390x-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux64-s390x
 
 thumb7k-apple-watchos:
@@ -574,6 +585,7 @@ thumb7k-apple-watchos:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: todo
 
 # Intel macOS.
@@ -629,6 +641,7 @@ x86_64-apple-darwin:
     - tk
     - uuid
     - xz
+    - zstd
   openssl_target: darwin64-x86_64-cc
 
 x86_64-apple-ios:
@@ -672,6 +685,7 @@ x86_64-apple-ios:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: darwin64-x86_64-cc
 
 x86_64-apple-tvos:
@@ -714,6 +728,7 @@ x86_64-apple-tvos:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: todo
 
 x86_64-apple-watchos:
@@ -756,6 +771,7 @@ x86_64-apple-watchos:
     - openssl-3.0
     - sqlite
     - xz
+    - zstd
   openssl_target: todo
 
 x86_64-unknown-linux-gnu:
@@ -801,6 +817,7 @@ x86_64-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
   bolt_capable: true
 
@@ -848,6 +865,7 @@ x86_64_v2-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
   bolt_capable: true
 
@@ -895,6 +913,7 @@ x86_64_v3-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
   bolt_capable: true
 
@@ -942,6 +961,7 @@ x86_64_v4-unknown-linux-gnu:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
   bolt_capable: true
 
@@ -987,6 +1007,7 @@ x86_64-unknown-linux-musl:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
 
 x86_64_v2-unknown-linux-musl:
@@ -1032,6 +1053,7 @@ x86_64_v2-unknown-linux-musl:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
 
 x86_64_v3-unknown-linux-musl:
@@ -1077,6 +1099,7 @@ x86_64_v3-unknown-linux-musl:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64
 
 x86_64_v4-unknown-linux-musl:
@@ -1122,4 +1145,5 @@ x86_64_v4-unknown-linux-musl:
     - xorgproto
     - xz
     - zlib
+    - zstd
   openssl_target: linux-x86_64

src/validation.rs

@@ -778,6 +778,7 @@ const GLOBAL_EXTENSIONS_PYTHON_3_14: &[&str] = &[
     "_zoneinfo",
     "_hmac",
     "_types",
+    "_zstd",
 ];
 
 const GLOBAL_EXTENSIONS_MACOS: &[&str] = &["_scproxy"];
@@ -813,8 +814,7 @@ const GLOBAL_EXTENSIONS_WINDOWS: &[&str] = &[
     "winsound",
 ];
 
-// TODO(zanieb): Move `_zstd` to non-Windows specific once we add support on Unix.
-const GLOBAL_EXTENSIONS_WINDOWS_3_14: &[&str] = &["_wmi", "_zstd"];
+const GLOBAL_EXTENSIONS_WINDOWS_3_14: &[&str] = &["_wmi"];
 
 const GLOBAL_EXTENSIONS_WINDOWS_PRE_3_13: &[&str] = &["_msi"];