Merge branch 'main' into mkitti-multi-frame-zstd-clean

Author: mkitti

.github/workflows/ci-i386.yml

@@ -0,0 +1,64 @@
+name: Tests on i386
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-i386:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0 # required for version resolution
+
+      - name: Setup Alpine Linux environment
+        uses: jirutka/[email protected]
+        with:
+          arch: x86
+          packages: >
+            build-base
+            python3
+            python3-dev
+            git
+            py3-pip
+            py3-pytest
+            uv
+            meson
+            pkgconf
+            py3-numpy
+            py3-numpy-dev
+            zstd
+
+      - name: Install numcodecs
+        run: |
+          export DISABLE_NUMCODECS_AVX2=""
+          uv venv
+          uv pip install -v -e .[test,test_extras,msgpack,crc32c]
+        shell: alpine.sh {0}
+
+
+      - name: Install zarr-python
+        # Since zarr v3 requires numpy >= 1.25, on Python 3.11 leave it out
+        # so we can have some tests of our minimum version of numpy (1.24)
+        if: matrix.python-version != '3.11'
+        run: uv add zarr>=3
+        shell: alpine.sh {0}
+
+
+      - name: List installed packages
+        run: uv pip list
+        shell: alpine.sh {0}
+
+      - name: Run tests
+        run: uv run pytest -v
+        shell: alpine.sh {0}

c-blosc

@@ -15,6 +15,9 @@ from .abc import Codec
 
 from libc.stdlib cimport malloc, realloc, free
 
+cdef extern from "stdint.h":
+    cdef size_t SIZE_MAX
+
 cdef extern from "zstd.h":
 
     unsigned ZSTD_versionNumber() nogil
@@ -204,8 +207,8 @@ def decompress(source, dest=None):
         Py_buffer* dest_pb
         char* dest_ptr
         size_t source_size, dest_size, decompressed_size
-        size_t nbytes, cbytes, blocksize
         size_t dest_nbytes
+        unsigned long long content_size
 
     # obtain source memoryview
     source_mv = ensure_continguous_memoryview(source)
@@ -216,18 +219,24 @@ def decompress(source, dest=None):
     source_size = source_pb.len
 
     try:
-
-        # determine uncompressed size
+        # determine uncompressed size using unsigned long long for full range
         try:
-            dest_size = findTotalContentSize(source_ptr, source_size)
+            content_size = findTotalContentSize(source_ptr, source_size)
         except RuntimeError:
             raise RuntimeError('Zstd decompression error: invalid input data')
 
-        if dest_size == 0 or dest_size == ZSTD_CONTENTSIZE_ERROR:
+        if content_size == ZSTD_CONTENTSIZE_UNKNOWN and dest is None:
+            return stream_decompress(source_pb)
+        elif content_size == ZSTD_CONTENTSIZE_UNKNOWN:
+            # dest is not None
+            # set dest_size based on dest
+            pass
+        elif content_size == ZSTD_CONTENTSIZE_ERROR or content_size == 0:
             raise RuntimeError('Zstd decompression error: invalid input data')
+        elif content_size > (<unsigned long long>SIZE_MAX):
+            raise RuntimeError('Zstd decompression error: content size too large for platform')
 
-        if dest_size == ZSTD_CONTENTSIZE_UNKNOWN and dest is None:
-            return stream_decompress(source_pb)
+        dest_size = <size_t>content_size
 
         # setup destination buffer
         if dest is None:
@@ -242,7 +251,7 @@ def decompress(source, dest=None):
         dest_ptr = <char*>dest_pb.buf
         dest_nbytes = dest_pb.len
 
-        if dest_size == ZSTD_CONTENTSIZE_UNKNOWN:
+        if content_size == ZSTD_CONTENTSIZE_UNKNOWN:
             dest_size = dest_nbytes
 
         # validate output buffer