From 5477c05df6c2d1819bf87d321d378e34f583c6af Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 19 Aug 2025 05:04:27 +0000 Subject: [PATCH 1/2] Squashed commit of the following: commit 7170133015127685271403326e7eeba45c103668 Author: Mark Kittisopikul Date: Tue Aug 19 04:28:08 2025 +0000 Add zstd package to Alpine commit 6d9b6eb9513f87377d92e7451b537cc143511eea Author: Mark Kittisopikul Date: Tue Aug 19 04:12:14 2025 +0000 uv pip list commit 538256b34054368f58b374ea219f434e135509f7 Author: Mark Kittisopikul Date: Tue Aug 19 04:01:10 2025 +0000 Add py3-numpy, remove zfp commit 311ebae333cb7ff8cc0af463566ed9f034f19903 Author: Mark Kittisopikul Date: Tue Aug 19 03:44:24 2025 +0000 Add python3-dev commit b646ff8d73edca4b363c5dbbf269661e66bd2440 Author: Mark Kittisopikul Date: Tue Aug 19 03:37:16 2025 +0000 Add meson ans pkgconf to Alpine commit 6a8e5d2f746ba25ad7e591c00c984c872af56cce Author: Mark Kittisopikul Date: Tue Aug 19 03:33:21 2025 +0000 Remove docker commit 5a3586059f4a599aaa9c4334211889d647b26468 Author: Mark Kittisopikul Date: Tue Aug 19 03:31:50 2025 +0000 Use uv venv commit 80c1df09c60d1f0ffb1fb8df41503c9882f767bb Author: Mark Kittisopikul Date: Tue Aug 19 03:28:48 2025 +0000 Remove uv init commit fca994e2ed885ab50135c616b087bb6517df3b9a Author: Mark Kittisopikul Date: Tue Aug 19 03:24:56 2025 +0000 Use uv commit b9db990b863768a30d5b7271e7b4a6b7d40b6db8 Author: Mark Kittisopikul Date: Tue Aug 19 03:14:28 2025 +0000 Do not need manual checkout commit 98df0a8aa0f1af7780261b8da347c704b85c32e3 Author: Mark Kittisopikul Date: Tue Aug 19 03:12:05 2025 +0000 Expand Alpine commit 1c7f3052cf6078357ee799db609119ad0aab33f5 Author: Mark Kittisopikul Date: Tue Aug 19 02:58:14 2025 +0000 Use Alpinr platform x86 commit bc447855f716110772e120572eff211d903e21ec Author: Mark Kittisopikul Date: Tue Aug 19 02:56:19 2025 +0000 Try Alpine Linux commit 4282cdf062f959cee60205e054639d08180ed01c Author: Mark Kittisopikul Date: Tue Aug 19 02:37:51 2025 +0000 Try to install docker commit c013eab4355fed06bf540f71e388f71746940c13 Author: Mark Kittisopikul Date: Tue Aug 19 02:09:06 2025 +0000 Try i386/debian commit eeef1d2383037af7ee3a37023ca3fecb90d10ea8 Author: Mark Kittisopikul Date: Tue Aug 19 02:05:01 2025 +0000 Try i386 container --- .github/workflows/ci-i386.yml | 64 +++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 .github/workflows/ci-i386.yml diff --git a/.github/workflows/ci-i386.yml b/.github/workflows/ci-i386.yml new file mode 100644 index 00000000..1cf1f20f --- /dev/null +++ b/.github/workflows/ci-i386.yml @@ -0,0 +1,64 @@ +name: Tests on i386 + +on: [push, pull_request] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-i386: + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout source + uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 # required for version resolution + + - name: Setup Alpine Linux environment + uses: jirutka/setup-alpine@v1.2.0 + with: + arch: x86 + packages: > + build-base + python3 + python3-dev + git + py3-pip + py3-pytest + uv + meson + pkgconf + py3-numpy + py3-numpy-dev + zstd + + - name: Install numcodecs + run: | + export DISABLE_NUMCODECS_AVX2="" + uv venv + uv pip install -v -e .[test,test_extras,msgpack,crc32c] + shell: alpine.sh {0} + + + - name: Install zarr-python + # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out + # so we can have some tests of our minimum version of numpy (1.24) + if: matrix.python-version != '3.11' + run: uv add zarr>=3 + shell: alpine.sh {0} + + + - name: List installed packages + run: uv pip list + shell: alpine.sh {0} + + - name: Run tests + run: uv run pytest -v + shell: alpine.sh {0} From 42b55956c8c645def63763d9f5f8d9e0e63e1423 Mon Sep 17 00:00:00 2001 From: Mark Kittisopikul Date: Tue, 19 Aug 2025 05:07:30 +0000 Subject: [PATCH 2/2] Squashed commit of the following: commit 786d31fcde2366bfb93df430f0b0222964b650ab Author: Mark Kittisopikul Date: Mon Aug 18 21:03:19 2025 +0000 Add pass, comment for dest_size commit 1349304b83c7b4e150d0f0d9341a317ddc9d3e1e Author: Mark Kittisopikul Date: Mon Aug 18 17:31:17 2025 +0000 Fix LLM issues .. commit b0fe5560b3980631457d6652f8d3d4116d8b4452 Author: Mark Kittisopikul Date: Mon Aug 18 12:01:19 2025 -0400 Apply suggestion from @mkitti commit 10a01d726fb2ad300755e4a3f20f9bef5a494843 Merge: 41150f2 f42a233 Author: Mark Kittisopikul Date: Mon Aug 18 11:50:52 2025 -0400 Merge branch 'main' into zstd-frame-content-size-ull commit 41150f2cf5ac8eb4eecea09fc30a3519e6636bd6 Author: Mark Kittisopikul Date: Mon Aug 18 11:43:10 2025 -0400 Update zstd.pyx with new content --- numcodecs/zstd.pyx | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/numcodecs/zstd.pyx b/numcodecs/zstd.pyx index f93da633..7b88a8da 100644 --- a/numcodecs/zstd.pyx +++ b/numcodecs/zstd.pyx @@ -15,6 +15,9 @@ from .abc import Codec from libc.stdlib cimport malloc, realloc, free +cdef extern from "stdint.h": + cdef size_t SIZE_MAX + cdef extern from "zstd.h": unsigned ZSTD_versionNumber() nogil @@ -202,8 +205,8 @@ def decompress(source, dest=None): Py_buffer* dest_pb char* dest_ptr size_t source_size, dest_size, decompressed_size - size_t nbytes, cbytes, blocksize size_t dest_nbytes + unsigned long long content_size # obtain source memoryview source_mv = ensure_continguous_memoryview(source) @@ -214,14 +217,20 @@ def decompress(source, dest=None): source_size = source_pb.len try: - - # determine uncompressed size - dest_size = ZSTD_getFrameContentSize(source_ptr, source_size) - if dest_size == 0 or dest_size == ZSTD_CONTENTSIZE_ERROR: + # determine uncompressed size using unsigned long long for full range + content_size = ZSTD_getFrameContentSize(source_ptr, source_size) + if content_size == ZSTD_CONTENTSIZE_UNKNOWN and dest is None: + return stream_decompress(source_pb) + elif content_size == ZSTD_CONTENTSIZE_UNKNOWN: + # dest is not None + # set dest_size based on dest + pass + elif content_size == ZSTD_CONTENTSIZE_ERROR or content_size == 0: raise RuntimeError('Zstd decompression error: invalid input data') + elif content_size > (SIZE_MAX): + raise RuntimeError('Zstd decompression error: content size too large for platform') - if dest_size == ZSTD_CONTENTSIZE_UNKNOWN and dest is None: - return stream_decompress(source_pb) + dest_size = content_size # setup destination buffer if dest is None: @@ -236,7 +245,7 @@ def decompress(source, dest=None): dest_ptr = dest_pb.buf dest_nbytes = dest_pb.len - if dest_size == ZSTD_CONTENTSIZE_UNKNOWN: + if content_size == ZSTD_CONTENTSIZE_UNKNOWN: dest_size = dest_nbytes # validate output buffer