Merge branch 'transpose' of github.com:ricardosp4/python-blosc2 into transpose

Author: ricardosp4

.github/workflows/cibuildwheels.yml

@@ -14,39 +14,50 @@ env:
   # Skip testing on aarch64 for now, as it is emulated on GitHub Actions and takes too long
   # Now that github provides native arm64 runners, we can enable tests again
   # CIBW_TEST_SKIP: "*linux*aarch64*"
+  # Skip PyPy wheels for now (numexpr needs some adjustments first)
+  # musllinux takes too long to build, and it's not worth it for now
+  CIBW_SKIP: "pp* *musllinux* *-win32"
 
 jobs:
 
   build_wheels:
-    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }} - ${{ matrix.p_ver }}
-    runs-on: ${{ matrix.os }}
+    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
+    runs-on: ${{ matrix.runs-on || matrix.os }}
     permissions:
       contents: write
     env:
       CIBW_BUILD: ${{ matrix.cibw_build }}
       CIBW_ARCHS_LINUX: ${{ matrix.arch }}
       CIBW_ARCHS_MACOS: "x86_64 arm64"
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest]
-        arch: [x86_64, aarch64]
-        # aarch64 is emulated, and it is giving issues with cibuildwheel
-        # Just build for x86_64 for now (Mac arm64 is already covered by cibuildwheel)
-        # arch: [x86_64]
-        cibw_build: ["cp3{11,12,13}-*"]
-        p_ver: ["3.11-3.13"]
-        exclude:
-          - os: windows-latest
-            arch: aarch64
-          # cibuild is already in charge to build aarch64 (see CIBW_ARCHS_MACOS)
-          - os: macos-latest
-            arch: aarch64
-          # Mutually exclude aarch64 and x86_64 for ubuntu-24.04-arm and ubuntu-latest
-          - os: ubuntu-24.04-arm
-            arch: x86_64
+        include:
+          # Linux x86_64 builds
           - os: ubuntu-latest
+            arch: x86_64
+            cibw_pattern: "cp3{11,12,13}-manylinux*"
+            artifact_name: "linux-x86_64"
+
+          # Linux ARM64 builds (native runners)
+          - os: ubuntu-24.04-arm
             arch: aarch64
+            cibw_pattern: "cp3{11,12,13}-manylinux*"
+            artifact_name: "linux-aarch64"
+            # Don't use native runners for now (looks like wait times are too long)
+            #runs-on: ["ubuntu-latest", "arm64"]
+
+          # Windows builds
+          - os: windows-latest
+            arch: x86_64
+            cibw_pattern: "cp3{11,12,13}-win64"
+            artifact_name: "windows-x86_64"
 
+          # macOS builds (universal2)
+          - os: macos-latest
+            arch: x86_64
+            cibw_pattern: "cp3{11,12,13}-macosx*"
+            artifact_name: "macos-universal2"
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -58,9 +69,9 @@ jobs:
           python-version: '3.x'
 
       # For some reason, this is still needed, even when using new arm64 runners
-      - name: Set up QEMU
-        if: ${{ matrix.arch == 'aarch64' }}
-        uses: docker/setup-qemu-action@v3
+#      - name: Set up QEMU
+#        if: ${{ matrix.arch == 'aarch64' }}
+#        uses: docker/setup-qemu-action@v3
 
       - name: Install Ninja
         id: ninja
@@ -74,68 +85,52 @@ jobs:
       - name: Build wheels
         uses: pypa/[email protected]
 
-      - name: Upload wheels
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.p_ver }}
-          path: ./wheelhouse/*.whl
-
-  build_sdist:
-    name: Build sdist
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.12]
-        os: [ubuntu-latest]
-        arch: [auto]
-        exclude:
-          - os: [ubuntu-latest]
-            # We don't support 32-bit platforms in python-blosc2
-            arch: x86
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: actions/setup-python@v5
-        name: Setup Python ${{ matrix.python-version }}
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Build sdist
-        run: pipx run build --sdist --outdir ./wheelhouse
-
-      - name: Upload sdist package
-        uses: actions/upload-artifact@v4
-        with:
-          name: wheels-source
-          path: wheelhouse/*.tar.gz
+      - name: Make sdist
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          python -m pip install build
+          python -m build --sdist --outdir wheelhouse .
 
       - name: Build building extension from sdist package
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
           cd ./wheelhouse
           tar -xzf blosc2-*.tar.gz
           cd ./blosc2-*/
+          python -m venv sdist_test_env
+          source sdist_test_env/bin/activate
           pip install pip --upgrade
           pip install --break-system-packages -e .[test]
 
       - name: Test sdist package with pytest
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        timeout-minutes: 10
         run: |
           cd ./wheelhouse/blosc2-*/
-          pytest
+          source sdist_test_env/bin/activate
+          python -m pytest tests/test_open.py tests/test_vlmeta.py tests/ndarray/test_evaluate.py
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.artifact_name }}
+          path: |
+            ./wheelhouse/*.whl
+            ./wheelhouse/*.tar.gz
+
 
   upload_pypi:
-    needs: [ build_wheels, build_sdist ]  # last but not least
+    needs: [ build_wheels]
     runs-on: ubuntu-latest
     # Only upload wheels when tagging (typically a release)
     if: startsWith(github.event.ref, 'refs/tags')
     steps:
       - uses: actions/download-artifact@v4
         with:
-          name: wheels-*
           path: ./wheelhouse
           merge-multiple: true  # Merge all the wheels artifacts into one directory
 
       - uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.blosc_pypi_secret }}
+          packages-dir: wheelhouse/

.pre-commit-config.yaml

@@ -16,7 +16,7 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.4
+    rev: v0.11.4
     hooks:
       - id: ruff
         args: ["--fix", "--show-fixes"]

ANNOUNCE.rst

@@ -1,13 +1,16 @@
-Announcing Python-Blosc2 3.2.0
+Announcing Python-Blosc2 3.2.1
 ==============================
 
-This is a minor release where we fixed some bugs related with the
-maximum type size supported by Blosc2; now, the maximum size is
-512 MB (compared to 255 bytes in previous versions).  We are also
-introducing WASM32 wheels for the first time.
+Here, all array containers in Blosc2 implement the ``__array_interface__``
+protocol to expose the data in the array.  This allows for better
+interoperability with other libraries like NumPy, CuPy, etc.  Now, the range
+of functions spans to most of NumPy functions, including reductions.
 
-We also added new ``blosc2.matmul()`` function for computing matrix
-multiplication on NDArray instances.
+See examples at: https://github.com/Blosc/python-blosc2/blob/main/examples/ndarray/jit-numpy-funcs.py
+See benchmarks at: https://github.com/Blosc/python-blosc2/blob/main/bench/ndarray/jit-numpy-funcs.py
+
+We have also improved the performance of constructors like ``blosc2.linspace()``
+or ``blosc2.arange()`` by a factor of up to 3x for large arrays.
 
 You can think of Python-Blosc2 3.x as an extension of NumPy/numexpr that:
 
@@ -17,7 +20,6 @@ You can think of Python-Blosc2 3.x as an extension of NumPy/numexpr that:
 - Supports NumPy ufunc mechanism: mix and match NumPy and Blosc2 computations.
 - Integrates with Numba and Cython via UDFs (User Defined Functions).
 - Adheres to modern NumPy casting rules way better than numexpr.
-- Computes expressions only when needed. They can also be stored for later use.
 - Supports linear algebra operations (like ``blosc2.matmul()``).
 
 Install it with::

CMakeLists.txt

@@ -50,7 +50,7 @@ else()
     include(FetchContent)
     FetchContent_Declare(blosc2
         GIT_REPOSITORY https://github.com/Blosc/c-blosc2
-        GIT_TAG 55a103239b24f8b4e07dafa5a289ad62bfade524  # v2.17.0
+        GIT_TAG 5fcd6fbf9ffcf613fabdb1eb3a90eeb12f7c04fe  # v2.17.1
     )
     FetchContent_MakeAvailable(blosc2)
     include_directories("${blosc2_SOURCE_DIR}/include")

README.rst

@@ -23,43 +23,31 @@ A fast & compressed ndarray library with a flexible compute engine
         :target: https://github.com/Blosc/python-blosc2/actions/workflows/build.yml
 
 
-What it is
-==========
+What is Python-Blosc2?
+=======================
 
 Python-Blosc2 is a high-performance compressed ndarray library with a flexible
-compute engine.  It uses the C-Blosc2 library as the compression backend.
-`C-Blosc2 <https://github.com/Blosc/c-blosc2>`_ is the next generation of
-Blosc, an `award-winning <https://www.blosc.org/posts/prize-push-Blosc2/>`_
-library that has been around for more than a decade, and that is been used
-by many projects, including `PyTables <https://www.pytables.org/>`_ or
-`Zarr <https://zarr.readthedocs.io/en/stable/>`_.
-
-Python-Blosc2 is Python wrapper that exposes the C-Blosc2 API, *plus* an
-integrated compute engine. This allows to perform complex calculations on
-compressed data in a way that operands do not need to be in-memory, but can be
-stored on disk or on `the network <https://github.com/ironArray/Caterva2>`_.
-This makes possible to work with data no matter how large it is, and that
-can be stored in a distributed fashion.
-
-Most importantly, Python-Blosc2 uses the `C-Blosc2 simple and open format
+compute engine, using `C-Blosc2 <https://www.blosc.org/c-blosc2/c-blosc2.html>`_
+as its compression backend. It allows complex calculations on compressed data,
+whether stored in memory, on disk, or over the network (e.g., via
+`Caterva2 <https://github.com/ironArray/Caterva2>`_).  It uses the
+`C-Blosc2 simple and open format
 <https://github.com/Blosc/c-blosc2/blob/main/README_FORMAT.rst>`_ for storing
-compressed data, making it easy to integrate with other systems and tools.
-
-You can find more introductory info about Python-Blosc2 at:
+compressed data.
 
-https://www.blosc.org/python-blosc2/getting_started/overview.html
+More info: https://www.blosc.org/python-blosc2/getting_started/overview.html
 
 Installing
 ==========
 
-Blosc2 now provides Python wheels for the major OS (Win, Mac and Linux) and platforms.
-You can install the binary packages from PyPi using ``pip``:
+Binary packages are available for major OSes (Win, Mac, Linux) and platforms.
+Install from PyPi using ``pip``:
 
 .. code-block:: console
 
     pip install blosc2 --upgrade
 
-For conda users, you can install the package from the conda-forge channel:
+Conda users can install from conda-forge:
 
 .. code-block:: console
 
@@ -72,13 +60,15 @@ The documentation is available here:
 
 https://blosc.org/python-blosc2/python-blosc2.html
 
-Additionally, you can find some examples at:
+You can find examples at:
 
 https://github.com/Blosc/python-blosc2/tree/main/examples
 
-Finally, we taught a tutorial at the `PyData Global 2024 <https://pydata.org/global2024/>`_
-that you can find at: https://github.com/Blosc/Python-Blosc2-3.0-tutorial.  There you will
-find different Jupyter notebook that explains the main features of Python-Blosc2.
+A tutorial from PyData Global 2024 is available at:
+
+https://github.com/Blosc/Python-Blosc2-3.0-tutorial
+
+It contains Jupyter notebooks explaining the main features of Python-Blosc2.
 
 License
 =======
@@ -118,6 +108,7 @@ to the core development of the Blosc2 library:
 - Juan David Ibáñez
 - Ivan Vilata i Balaguer
 - Oumaima Ech.Chdig
+- Ricardo Sales Piquer
 
 In addition, other people have participated to the project in different
 aspects:

RELEASE_NOTES.md

@@ -1,9 +1,26 @@
 # Release notes
 
-## Changes from 3.2.0 to 3.2.1
+## Changes from 3.2.1 to 3.2.2
 
 XXX version-specific blurb XXX
 
+## Changes from 3.2.0 to 3.2.1
+
+* The array containers are now using the `__array_interface__` protocol to
+  expose the data in the array.  This allows for better interoperability with
+  other libraries that support the `__array_interface__` protocol, like NumPy,
+  CuPy, etc.  Now, the range of functions that can be used within the `blosc2.jit`
+  decorator is way larger, and essentially all NumPy functions should work now.
+
+  See examples at: https://github.com/Blosc/python-blosc2/blob/main/examples/ndarray/jit-numpy-funcs.py
+  See benchmarks at: https://github.com/Blosc/python-blosc2/blob/main/bench/ndarray/jit-numpy-funcs.py
+
+* The performance of constructors like `arange()`, `linspace()` and `fromiter()`
+  has been improved.  Now, they can be up to 3x faster, specially with large
+  arrays.
+
+* C-Blosc2 updated to 2.17.1.  This fixes various UB as well as compiler warnings.
+
 ## Changes from 3.1.1 to 3.2.0
 
 * Structured arrays can be larger than 255 bytes now.  This was a limitation

bench/ndarray/jit-expr.py

@@ -23,7 +23,7 @@
 cparams = blosc2.CParams(clevel=1, codec=blosc2.Codec.LZ4)
 cparams_out = blosc2.CParams(clevel=1, codec=blosc2.Codec.LZ4)
 print("Using cparams: ", cparams)
-check_result = True
+check_result = False
 # Lossy compression
 # filters = [blosc2.Filter.TRUNC_PREC, blosc2.Filter.SHUFFLE]
 # filters_meta = [8, 0]  # keep 8 bits of precision in mantissa
@@ -150,5 +150,4 @@ def compute_expression_compr(a, b, c):
 print(f"Speedup: {tref / t1:.2f}x, out cratio: {cratio:.2f}x")
 if check_result:
     np.testing.assert_allclose(out, nout)
-
-print("All results are equal!")
+    print("All results are equal!")