Add support for free-threading builds of CPython

.github/workflows/test.yml

@@ -20,6 +20,7 @@ jobs:
           - '3.11'
           - '3.12'
           - '3.13'
+          - '3.13t'
         arch:
           - 'arm64'
           - 'x86'
@@ -59,47 +60,60 @@ jobs:
       PYTHONDEVMODE: '1'
     steps:
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: Quansight-Labs/setup-python@v5
         with:
           python-version: ${{ matrix.py }}
           architecture: ${{ matrix.arch }}
 
       - name: Install Rust
-        if: matrix.arch == 'x64'
+        if: matrix.arch == 'x64' && !startsWith(matrix.py, '3.13')
         uses: dtolnay/rust-toolchain@v1
         with:
           toolchain: stable
 
       - uses: actions/checkout@v4
 
       - name: Install Dependencies
+        if: "!endsWith(matrix.py, 't')"
         shell: bash
         run: |
           python -m pip install --require-hashes -r ci/requirements.txt
 
+      - name: Install Dependencies (free-threading)
+        if: "endsWith(matrix.py, 't')"
+        shell: bash
+        run: |
+          python -m pip install --require-hashes -r ci/requirements.freethreading.txt
+
       # TODO enable once PyO3 supports 3.13.
       - name: Build (Rust)
-        if: matrix.arch == 'x64' && matrix.py != '3.13'
+        if: matrix.arch == 'x64' && !startsWith(matrix.py, '3.13')
         run: |
           python -m pip install --config-settings='--build-option=--rust-backend' -e .
 
       - name: Build (No Rust)
-        if: matrix.arch != 'x64' || matrix.py == '3.13'
+        if: matrix.arch != 'x64' || startsWith(matrix.py, '3.13')
         run: |
           python -m pip install -e .
 
       - name: Test C Backend
         run: |
           pytest --numprocesses=auto --hypothesis-profile=${HYPOTHESIS_PROFILE} -v tests/
 
+      - name: Test in Parallel
+        if: "endsWith(matrix.py, 't')"
+        run: |
+          pytest --numprocesses=auto --hypothesis-profile=${HYPOTHESIS_PROFILE} --parallel-threads=10 -v tests/
+
       - name: Test CFFI Backend
+        if: "!startsWith(matrix.py, '3.13')"  # see pyproject.toml:4
         env:
           PYTHON_ZSTANDARD_IMPORT_POLICY: 'cffi'
         run: |
           pytest --numprocesses=auto --hypothesis-profile=${HYPOTHESIS_PROFILE} -v tests/
 
       - name: Test Rust Backend
-        if: matrix.arch == 'x64'
+        if: matrix.arch == 'x64' && !startsWith(matrix.py, '3.13')
         # Rust backend is currently experimental. So ignore failures in it.
         continue-on-error: true
         env:

c-ext/_pyzstd_atomics.h

@@ -0,0 +1,88 @@
+/*
+ * Provides wrappers around C11 standard library atomics and MSVC intrinsics
+ * to provide basic atomic load and store functionality. This is based on
+ * code in CPython's pyatomic.h, pyatomic_std.h, and pyatomic_msc.h
+ *
+ * Adapted from:
+ * - numpy/_core/src/common/npy_atomic.h
+ * - cpython/Include/cpython/pyatomic.h
+ */
+
+#ifndef PYZSTD_ATOMICS_H
+#define PYZSTD_ATOMICS_H
+
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
+    && !defined(__STDC_NO_ATOMICS__)
+// TODO: support C++ atomics as well if this header is ever needed in C++
+    #include <stdatomic.h>
+    #include <stdint.h>
+    #define STDC_ATOMICS
+#elif _MSC_VER
+    #include <intrin.h>
+    #define MSC_ATOMICS
+    #if !defined(_M_X64) && !defined(_M_IX86) && !defined(_M_ARM64)
+        #error "Unsupported MSVC build configuration, neither x86 or ARM"
+    #endif
+#elif defined(__GNUC__) && (__GNUC__ > 4)
+    #define GCC_ATOMICS
+#elif defined(__clang__)
+    #if __has_builtin(__atomic_load)
+        #define GCC_ATOMICS
+    #endif
+#else
+    #error "no supported atomic implementation for this platform/compiler"
+#endif
+
+
+static inline int8_t
+pyzstd_atomic_load_int8(const int8_t *obj) {
+#ifdef STDC_ATOMICS
+    return (int8_t)atomic_load((const _Atomic(int8_t)*)obj);
+#elif defined(MSC_ATOMICS)
+#if defined(_M_X64) || defined(_M_IX86)
+    return *(volatile int8_t *)obj;
+#else // defined(_M_ARM64)
+    return (int8_t)__ldar8((unsigned __int8 volatile *)obj);
+#endif
+#elif defined(GCC_ATOMICS)
+    return __atomic_load_n(obj, __ATOMIC_SEQ_CST);
+#endif
+}
+
+static inline void
+pyzstd_atomic_store_int8(int8_t *obj, int8_t value) {
+#ifdef STDC_ATOMICS
+    atomic_store((_Atomic(int8_t)*)obj, value);
+#elif defined(MSC_ATOMICS)
+    _InterlockedExchange8((volatile char *)obj, (char)value);
+#elif defined(GCC_ATOMICS)
+    __atomic_store_n(obj, value, __ATOMIC_SEQ_CST);
+#endif
+}
+
+static inline int
+pyzstd_atomic_compare_exchange_int8(int8_t *obj, int8_t expected, int8_t desired) {
+#ifdef STDC_ATOMICS
+    return atomic_compare_exchange_strong((_Atomic(int8_t)*)obj,
+                                          &expected, desired);
+#elif defined(MSC_ATOMICS)
+    int8_t initial = (int8_t)_InterlockedCompareExchange8(
+                                       (volatile char *)obj,
+                                       (char)value,
+                                       (char)expected);
+    if (initial == *expected) {
+        return 1;
+    }
+    *expected = initial;
+    return 0;
+#elif defined(GCC_ATOMICS)
+    return __atomic_compare_exchange_n(obj, &expected, desired, 0,
+                                       __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#undef MSC_ATOMICS
+#undef STDC_ATOMICS
+#undef GCC_ATOMICS
+
+#endif // PYZSTD_ATOMICS_H

c-ext/backend_c.c

@@ -210,6 +210,10 @@ void zstd_module_init(PyObject *m) {
     Py_DECREF(feature);
 #endif
 
+#ifdef Py_GIL_DISABLED
+    PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED);
+#endif
+
     if (PyObject_SetAttrString(m, "backend_features", features) == -1) {
         return;
     }
@@ -313,7 +317,7 @@ size_t roundpow2(size_t i) {
 int safe_pybytes_resize(PyObject **obj, Py_ssize_t size) {
     PyObject *tmp;
 
-    if ((*obj)->ob_refcnt == 1) {
+    if (Py_REFCNT(*obj) == 1) {
         return _PyBytes_Resize(obj, size);
     }
 

c-ext/compressor.c

@@ -7,6 +7,7 @@
  */
 
 #include "python-zstandard.h"
+#include "_pyzstd_atomics.h"
 
 extern PyObject *ZstdError;
 
@@ -238,6 +239,8 @@ static int ZstdCompressor_init(ZstdCompressor *self, PyObject *args,
         Py_INCREF(dict);
     }
 
+    self->in_use = 0;
+
     if (setup_cctx(self)) {
         return -1;
     }
@@ -522,6 +525,14 @@ static PyObject *ZstdCompressor_compress(ZstdCompressor *self, PyObject *args,
         return NULL;
     }
 
+    if (pyzstd_atomic_load_int8(&self->in_use)) {
+        goto concurrent_use;
+    }
+
+    if (!pyzstd_atomic_compare_exchange_int8(&self->in_use, 0, 1)) {
+        goto concurrent_use;
+    }
+
     ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only);
 
     destSize = ZSTD_compressBound(source.len);
@@ -569,8 +580,16 @@ static PyObject *ZstdCompressor_compress(ZstdCompressor *self, PyObject *args,
     Py_SET_SIZE(output, outBuffer.pos);
 
 finally:
+    pyzstd_atomic_store_int8(&self->in_use, 0);
     PyBuffer_Release(&source);
     return output;
+
+concurrent_use:
+    PyErr_SetString(ZstdError, "concurrent use is not allowed. "
+                               "See https://python-zstandard.readthedocs.io"
+                               "/en/latest/api_usage.html"
+                               "#thread-and-object-reuse-safety");
+    return NULL;
 }
 
 static ZstdCompressionObj *ZstdCompressor_compressobj(ZstdCompressor *self,

c-ext/python-zstandard.h

@@ -104,6 +104,8 @@ typedef struct {
     ZSTD_CCtx *cctx;
     /* Compression parameters in use. */
     ZSTD_CCtx_params *params;
+    /* Is this compressor being used by a thread? */
+    int8_t in_use;
 } ZstdCompressor;
 
 extern PyTypeObject *ZstdCompressorType;

ci/requirements.freethreading.in

@@ -0,0 +1,12 @@
+# This is a dependency of pytest on Windows but isn't picked up by pip-compile.
+atomicwrites
+cibuildwheel
+#cffi
+colorama
+hypothesis
+mypy
+pycparser
+pytest-xdist
+pytest-run-parallel
+pytest
+wheel

ci/requirements.freethreading.txt

@@ -0,0 +1,132 @@
+#
+# This file is autogenerated by pip-compile with Python 3.13
+# by the following command:
+#
+#    pip-compile --generate-hashes --output-file=ci/requirements.freethreading.txt --pre ci/requirements.freethreading.in
+#
+atomicwrites==1.4.1 \
+    --hash=sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11
+    # via -r ci/requirements.freethreading.in
+attrs==24.2.0 \
+    --hash=sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346 \
+    --hash=sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2
+    # via hypothesis
+bashlex==0.18 \
+    --hash=sha256:5bb03a01c6d5676338c36fd1028009c8ad07e7d61d8a1ce3f513b7fff52796ee \
+    --hash=sha256:91d73a23a3e51711919c1c899083890cdecffc91d8c088942725ac13e9dcfffa
+    # via cibuildwheel
+bracex==2.5.post1 \
+    --hash=sha256:12c50952415bfa773d2d9ccb8e79651b8cdb1f31a42f6091b804f6ba2b4a66b6 \
+    --hash=sha256:13e5732fec27828d6af308628285ad358047cec36801598368cb28bc631dbaf6
+    # via cibuildwheel
+certifi==2024.8.30 \
+    --hash=sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8 \
+    --hash=sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9
+    # via cibuildwheel
+cibuildwheel==2.21.3 \
+    --hash=sha256:3ce23a9e5406b3eeb80039d7a6fdb218a2450932a8037c0bf76511cd88dfb74e \
+    --hash=sha256:f1d036a13603a6ce4019d8b1bd52c296cf32461a3b3be8441434b60b8b378b80
+    # via -r ci/requirements.freethreading.in
+colorama==0.4.6 \
+    --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
+    --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
+    # via -r ci/requirements.freethreading.in
+execnet==2.1.1 \
+    --hash=sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc \
+    --hash=sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3
+    # via pytest-xdist
+filelock==3.16.1 \
+    --hash=sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0 \
+    --hash=sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435
+    # via cibuildwheel
+hypothesis==6.116.0 \
+    --hash=sha256:9c1ac9a2edb77aacae1950d8ded6b3f40dbf8483097c88336265c348d2132c71 \
+    --hash=sha256:d30271214eae0d4758b72b408e9777405c7c7f687e14e8a42853adea887b2891
+    # via -r ci/requirements.freethreading.in
+iniconfig==2.0.0 \
+    --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
+    --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
+    # via pytest
+mypy==1.13.0 \
+    --hash=sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc \
+    --hash=sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e \
+    --hash=sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f \
+    --hash=sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74 \
+    --hash=sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a \
+    --hash=sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2 \
+    --hash=sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b \
+    --hash=sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73 \
+    --hash=sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e \
+    --hash=sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d \
+    --hash=sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d \
+    --hash=sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6 \
+    --hash=sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca \
+    --hash=sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d \
+    --hash=sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5 \
+    --hash=sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62 \
+    --hash=sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a \
+    --hash=sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc \
+    --hash=sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7 \
+    --hash=sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb \
+    --hash=sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7 \
+    --hash=sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732 \
+    --hash=sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80 \
+    --hash=sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a \
+    --hash=sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc \
+    --hash=sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2 \
+    --hash=sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0 \
+    --hash=sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24 \
+    --hash=sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7 \
+    --hash=sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b \
+    --hash=sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372 \
+    --hash=sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8
+    # via -r ci/requirements.freethreading.in
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via mypy
+packaging==24.1 \
+    --hash=sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002 \
+    --hash=sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124
+    # via
+    #   cibuildwheel
+    #   pytest
+platformdirs==4.3.6 \
+    --hash=sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907 \
+    --hash=sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb
+    # via cibuildwheel
+pluggy==1.5.0 \
+    --hash=sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1 \
+    --hash=sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669
+    # via pytest
+pycparser==2.22 \
+    --hash=sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6 \
+    --hash=sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc
+    # via -r ci/requirements.freethreading.in
+pytest==8.3.3 \
+    --hash=sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181 \
+    --hash=sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2
+    # via
+    #   -r ci/requirements.freethreading.in
+    #   pytest-run-parallel
+    #   pytest-xdist
+pytest-run-parallel==0.1.0 \
+    --hash=sha256:13d8579d39d60d5d77695e6bc292daa3352a5974eb446819f52fba4e20bb0d0f \
+    --hash=sha256:271854a2919aaff4e2a39bc2094bd2f96aa32fba9e51a995405ead35b74cc062
+    # via -r ci/requirements.freethreading.in
+pytest-xdist==3.6.1 \
+    --hash=sha256:9ed4adfb68a016610848639bb7e02c9352d5d9f03d04809919e2dafc3be4cca7 \
+    --hash=sha256:ead156a4db231eec769737f57668ef58a2084a34b2e55c4a8fa20d861107300d
+    # via -r ci/requirements.freethreading.in
+sortedcontainers==2.4.0 \
+    --hash=sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88 \
+    --hash=sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0
+    # via hypothesis
+typing-extensions==4.12.2 \
+    --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
+    --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
+    # via mypy
+wheel==0.44.0 \
+    --hash=sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f \
+    --hash=sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49
+    # via -r ci/requirements.freethreading.in

pyproject.toml

@@ -28,7 +28,7 @@ Documentation = "https://python-zstandard.readthedocs.io/en/latest/"
 
 [build-system]
 requires = [
-    "cffi>=1.17.0",
+    # "cffi>=1.17.0",  # ok for default, nok for free-threading
     # 69.0.0 breaks handling of --config-settings=--build-option, which our CI
     # relies on. So constrained to an older version until we figure out a
     # workaround. See comment at