BLD/RLS: Update wheels to include GDAL 3.9.2 and add manylinux_2_28 wheels (#466)

brendan-ward · web-flow · commit a5fccad2f35d · 2024-09-05T08:23:40.000+02:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -20,7 +20,7 @@ jobs:
       - run:
           name: Build the Linux aarch64 wheels.
           command: |
-            python3 -m pip install --user cibuildwheel==2.17.0
+            python3 -m pip install --user cibuildwheel==2.20.0
             python3 -m cibuildwheel --output-dir wheelhouse
       - run:
           name: Test the wheels
diff --git a/.github/workflows/docker-gdal.yml b/.github/workflows/docker-gdal.yml
@@ -20,8 +20,9 @@ jobs:
       fail-fast: false
       matrix:
         container:
-          - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.10.6
-          - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.3" # python 3.10.12
+          - "ghcr.io/osgeo/gdal:ubuntu-small-latest" # >= python 3.12.3
+          - "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2" # python 3.12.3
+          - "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.7.3" # python 3.10.12
           - "ghcr.io/osgeo/gdal:ubuntu-small-3.6.4" # python 3.10.6
           - "osgeo/gdal:ubuntu-small-3.5.3" # python 3.8.10
@@ -38,6 +39,9 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Create virtual environment
+        # install uv and use it to create a virtual environment, then add it to
+        # environment variables so that it is automatically activated and can be
+        # used for tests below
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
           . $HOME/.cargo/env
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -54,36 +54,60 @@ jobs:
     needs: [build-sdist]
     runs-on: ubuntu-latest
     container:
-      image: "ghcr.io/osgeo/gdal:ubuntu-small-3.8.5"
+      image: "ghcr.io/osgeo/gdal:ubuntu-small-3.9.2"
 
     steps:
+      - name: Install packages
+        run: |
+          apt-get update && apt-get install -y build-essential python3-dev
+
+      - name: Create virtual environment
+        # install uv and use it to create a virtual environment, then add it to
+        # environment variables so that it is automatically activated and can be
+        # used for tests below
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          . $HOME/.cargo/env
+          uv venv .venv
+          echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+
       - name: Download sdist from artifacts
         uses: actions/download-artifact@v4
         with:
           name: pyogrio-sdist
           path: wheelhouse
 
-      - name: Install pip
-        run: |
-          apt-get update && apt-get install -y python3-pip
-
       - name: Build from sdist and install test dependencies
         shell: bash
         run: |
-          python3 -m pip install --no-cache-dir wheelhouse/*.tar.gz
-          python3 -m pip install --no-cache-dir pytest pandas pyproj shapely>=2
-          python3 -m pip install --no-cache-dir --no-deps geopandas
-          python3 -m pip list
+          uv pip install --no-cache wheelhouse/*.tar.gz
+          uv pip install pytest pandas pyproj shapely>=2
+          uv pip install --no-deps geopandas
+          uv pip list
 
       - name: Run tests
         shell: bash
+        # virtual environment is automatically activated
         run: |
           cd ..
-          python3 -m pytest --pyargs pyogrio.tests -v
+          uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')"
+          uv run python -m pytest --pyargs pyogrio.tests -v
 
   build-wheels-linux:
     name: Build wheels on Linux
-    runs-on: "ubuntu-20.04"
+    runs-on: "ubuntu-latest"
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # use manylinux2014 for older glibc platforms until discontinued
+          - wheel_name: "pyogrio-wheel-linux-manylinux2014_x86_64"
+            container: "ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile"
+
+          # use manylinux_2_28 for any platforms with glibc>=2.28
+          - wheel_name: "pyogrio-wheel-linux-manylinux_2_28_x86_64"
+            container: "ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile"
 
     steps:
       - name: Checkout
@@ -103,7 +127,7 @@ jobs:
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile
+          file: ${{ matrix.container }}
           tags: manylinux-vcpkg-gdal:latest
           push: false
           load: true
@@ -117,7 +141,7 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: pyogrio-wheel-linux
+          name: ${{ matrix.wheel_name }}
           path: ./wheelhouse/*.whl
           compression-level: 0
 
@@ -129,7 +153,7 @@ jobs:
       matrix:
         include:
           - os: "macos-12"
-            triplet: "x64-osx-dynamic"
+            triplet: "x64-osx-dynamic-release"
             arch: x86_64
             vcpkg_cache: "/Users/runner/.cache/vcpkg/archives"
             vcpkg_logs: "/usr/local/share/vcpkg/buildtrees/**/*.log"
@@ -160,7 +184,7 @@ jobs:
           path: |
             ${{ matrix.vcpkg_cache }}
           # bump the last digit to avoid using previous build cache
-          key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.8.5-cache0
+          key: ${{ matrix.os }}-${{ matrix.arch }}-vcpkg-gdal3.9.2-cache0
 
       # MacOS build requires aclocal, which is part of automake, but appears
       # to be missing in default image
@@ -176,9 +200,9 @@ jobs:
           cd $VCPKG_INSTALLATION_ROOT
           # on mac the clone is not clean, otherwise git pull fails
           git reset --hard
-          # pull latest version to ensure the required commit with GDAL 3.8 is available
+          # pull specific commit with desired GDAL version
           git pull
-          git checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049
+          git checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
 
       - name: Install GDAL
         env:
@@ -215,17 +239,28 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-20.04", "windows-latest", "macos-12", "macos-latest"]
+        os:
+          [
+            "ubuntu-latest",
+            "ubuntu-20.04",
+            "windows-latest",
+            "macos-12",
+            "macos-latest",
+          ]
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
         include:
+          - os: "ubuntu-latest"
+            artifact: pyogrio-wheel-linux-manylinux2014_x86_64
+          - os: "ubuntu-latest"
+            artifact: pyogrio-wheel-linux-manylinux_2_28_x86_64
           - os: "ubuntu-20.04"
-            artifact: pyogrio-wheel-linux
+            artifact: pyogrio-wheel-linux-manylinux_2_28_x86_64
           - os: "windows-latest"
             artifact: pyogrio-wheel-x64-windows-dynamic-release
           - os: "macos-12"
-            artifact: pyogrio-wheel-x64-osx-dynamic
+            artifact: pyogrio-wheel-x64-osx-dynamic-release
           - os: "macos-latest"
-            artifact: pyogrio-wheel-x64-osx-dynamic
+            artifact: pyogrio-wheel-arm64-osx-dynamic-release
 
     steps:
       - name: Checkout
@@ -236,8 +271,27 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           allow-prereleases: true
-          cache: "pip"
-          cache-dependency-path: "ci/requirements-wheel-test.txt"
+
+      - name: Create virtual environment (Linux / MacOS)
+        # install uv and use it to create a virtual environment, then add it to
+        # environment variables so that it is automatically activated and can be
+        # used for tests below
+        if: ${{ runner.os != 'Windows' }}
+        shell: bash
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          . $HOME/.cargo/env
+          uv venv .venv
+          echo "VIRTUAL_ENV=.venv" >> $GITHUB_ENV
+          echo "$PWD/.venv/bin" >> $GITHUB_PATH
+
+      - name: Create virtual environment (Windows)
+        if: ${{ runner.os == 'Windows' }}
+        run: |
+          irm https://astral.sh/uv/install.ps1 | iex
+          uv venv .venv
+          "VIRTUAL_ENV=.venv" | Out-File -FilePath $env:GITHUB_ENV -Append
+          "$PWD/.venv/Scripts" | Out-File -FilePath $env:GITHUB_PATH -Append
 
       - name: Download wheels from artifacts
         uses: actions/download-artifact@v4
@@ -248,19 +302,20 @@ jobs:
       - name: Install dependencies and pyogrio wheel
         shell: bash
         run: |
-          python -m pip install -r ci/requirements-wheel-test.txt
+          uv pip install -r ci/requirements-wheel-test.txt
+          uv pip install --no-cache --pre --no-index --find-links wheelhouse pyogrio
           if [ ${{ matrix.python-version }} != "3.12" ]; then
-            python -m pip install --no-deps geopandas
+            uv pip install --no-deps geopandas
           fi
-          python -m pip install --pre --find-links wheelhouse pyogrio
-          python -m pip list
+          uv pip list
 
       - name: Run tests
         shell: bash
+        # virtual environment is automatically activated
         run: |
           cd ..
-          python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')"
-          python -m pytest --pyargs pyogrio.tests -v
+          uv run python -c "import pyogrio; print(f'GDAL version: {pyogrio.__gdal_version__}\nGEOS version: {pyogrio.__gdal_geos_version__}')"
+          uv run python -m pytest --pyargs pyogrio.tests -v
 
   publish:
     name: Publish pyogrio to GitHub / PyPI
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,6 +1,13 @@
 # CHANGELOG
 
-## 0.9.1 (yyyy-mm-dd)
+## 0.10.0 (yyyy-mm-dd)
+
+### Bug fixes
+
+-   Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435).
+-   Enable mask & bbox filter when geometry column not read (#431).
+-   Raise NotImplmentedError when user attempts to write to an open file handle (#442).
+-   Prevent seek on read from compressed inputs (#443).
 
 ### Packaging
 
@@ -10,13 +17,7 @@
     packages if needed: more info [here](https://gdal.org/download.html#conda).
     This also leads to `pyproj` becoming an optional dependency; you will need
     to install `pyproj` in order to support spatial reference systems (#452).
-
-### Bug fixes
-
--   Silence warning from `write_dataframe` with `GeoSeries.notna()` (#435).
--   Enable mask & bbox filter when geometry column not read (#431).
--   Raise NotImplmentedError when user attempts to write to an open file handle (#442).
--   Prevent seek on read from compressed inputs (#443).
+-   The GDAL library included in the wheels is updated from 3.8.5 to GDAL 3.9.2 (#466).
 
 ## 0.9.0 (2024-06-17)
 
diff --git a/ci/custom-triplets/x64-linux-dynamic-release.cmake b/ci/custom-triplets/x64-linux-dynamic-release.cmake
diff --git a/ci/custom-triplets/x64-osx-dynamic-release.cmake b/ci/custom-triplets/x64-osx-dynamic-release.cmake
diff --git a/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux2014_x86_64-vcpkg-gdal.Dockerfile
@@ -3,16 +3,16 @@ FROM quay.io/pypa/manylinux2014_x86_64:2024-08-12-7fde9b1
 # building openssl needs IPC-Cmd (https://github.com/microsoft/vcpkg/issues/24988)
 RUN yum install -y curl unzip zip tar perl-IPC-Cmd
 
-# require python >= 3.7 (python 3.6 is default on base image) for meson 
+# require python >= 3.7 (python 3.6 is default on base image) for meson
 RUN ln -s /opt/python/cp38-cp38/bin/python3 /usr/bin/python3
 
 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
-    git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049
+    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
 
 ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
 ENV PATH="${PATH}:/opt/vcpkg"
 
-ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic"
+ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic-release"
 
 # mkdir & touch -> workaround for https://github.com/microsoft/vcpkg/issues/27786
 RUN bootstrap-vcpkg.sh && \
@@ -21,11 +21,11 @@ RUN bootstrap-vcpkg.sh && \
     vcpkg integrate install && \
     vcpkg integrate bash
 
-COPY ci/custom-triplets/x64-linux-dynamic.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic.cmake
+COPY ci/custom-triplets/x64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic-release.cmake
 COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/
 COPY ci/vcpkg.json opt/vcpkg/
 
-ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic/lib"
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic-release/lib"
 RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \
     --overlay-ports=opt/vcpkg/custom-ports \
     --feature-flags="versions,manifests" \
diff --git a/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_aarch64-vcpkg-gdal.Dockerfile
@@ -4,7 +4,7 @@ FROM quay.io/pypa/manylinux_2_28_aarch64:2024-08-12-7fde9b1
 RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd
 
 RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
-    git -C /opt/vcpkg checkout 4f4a1821b2e8c7a2863e4df65a4d514f84144049
+    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
 
 ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
 ENV PATH="${PATH}:/opt/vcpkg"
@@ -24,7 +24,6 @@ RUN bootstrap-vcpkg.sh && \
     vcpkg integrate bash
 
 COPY ci/custom-triplets/arm64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/arm64-linux-dynamic-release.cmake
-COPY ci/vcpkg-custom-ports/ opt/vcpkg/custom-ports/
 COPY ci/vcpkg.json opt/vcpkg/
 
 # temporary workaround for https://github.com/microsoft/vcpkg/issues/36094
@@ -33,7 +32,6 @@ RUN git -C /opt/vcpkg apply vcpkg_linux_crosscompiling.patch
 
 ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/arm64-linux-dynamic-release/lib"
 RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \
-    --overlay-ports=opt/vcpkg/custom-ports \
     --feature-flags="versions,manifests" \
     --x-manifest-root=opt/vcpkg \
     --x-install-root=opt/vcpkg/installed && \
diff --git a/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile b/ci/manylinux_2_28_x86_64-vcpkg-gdal.Dockerfile
@@ -0,0 +1,33 @@
+FROM quay.io/pypa/manylinux_2_28_x86_64:2024-08-12-7fde9b1
+
+# building openssl needs IPC-Cmd (https://github.com/microsoft/vcpkg/issues/24988)
+RUN dnf -y install curl zip unzip tar ninja-build perl-IPC-Cmd
+
+RUN git clone https://github.com/Microsoft/vcpkg.git /opt/vcpkg && \
+    git -C /opt/vcpkg checkout 73794ce5f63fd138fab999a22959ca7c6305d93c
+
+ENV VCPKG_INSTALLATION_ROOT="/opt/vcpkg"
+ENV PATH="${PATH}:/opt/vcpkg"
+
+ENV VCPKG_DEFAULT_TRIPLET="x64-linux-dynamic-release"
+
+# mkdir & touch -> workaround for https://github.com/microsoft/vcpkg/issues/27786
+RUN bootstrap-vcpkg.sh && \
+    mkdir -p /root/.vcpkg/ $HOME/.vcpkg && \
+    touch /root/.vcpkg/vcpkg.path.txt $HOME/.vcpkg/vcpkg.path.txt && \
+    vcpkg integrate install && \
+    vcpkg integrate bash
+
+COPY ci/custom-triplets/x64-linux-dynamic-release.cmake opt/vcpkg/custom-triplets/x64-linux-dynamic-release.cmake
+COPY ci/vcpkg.json opt/vcpkg/
+
+ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/opt/vcpkg/installed/x64-linux-dynamic-release/lib"
+RUN vcpkg install --overlay-triplets=opt/vcpkg/custom-triplets \
+    --feature-flags="versions,manifests" \
+    --x-manifest-root=opt/vcpkg \
+    --x-install-root=opt/vcpkg/installed && \
+    vcpkg list
+
+# setting git safe directory is required for properly building wheels when
+# git >= 2.35.3
+RUN git config --global --add safe.directory "*"
diff --git a/ci/vcpkg.json b/ci/vcpkg.json
@@ -1,12 +1,12 @@
 {
     "name": "pyogrio",
-    "version": "0.8.0",
+    "version": "0.10.0",
     "dependencies": [
         {
             "name": "gdal",
             "default-features": false,
             "features": ["recommended-features", "curl", "geos", "iconv"]
         }
     ],
-    "builtin-baseline": "4f4a1821b2e8c7a2863e4df65a4d514f84144049"
+    "builtin-baseline": "73794ce5f63fd138fab999a22959ca7c6305d93c"
 }
diff --git a/pyproject.toml b/pyproject.toml

Original file line number	Diff line number	Diff line change
`@@ -1,12 +1,12 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "pyogrio",`
`3`		`- "version": "0.8.0",`
	`3`	`+ "version": "0.10.0",`
`4`	`4`	`"dependencies": [`
`5`	`5`	`{`
`6`	`6`	`"name": "gdal",`
`7`	`7`	`"default-features": false,`
`8`	`8`	`"features": ["recommended-features", "curl", "geos", "iconv"]`
`9`	`9`	`}`
`10`	`10`	`],`
`11`		`- "builtin-baseline": "4f4a1821b2e8c7a2863e4df65a4d514f84144049"`
	`11`	`+ "builtin-baseline": "73794ce5f63fd138fab999a22959ca7c6305d93c"`
`12`	`12`	`}`