Enable PYPI upload workflow without Windows packages (#239)

* Enable pypi upload in CI/CD

Author: dudoslav

.github/workflows/build-wheels.yml

@@ -5,12 +5,12 @@ on:
   push:
     branches:
       - release-*
-      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
+      - '*wheel*'  # must quote since "*" is a YAML reserved character; we want a string
     tags:
-      - "*"
+      - '*'
   pull_request:
     branches:
-      - "*wheel*" # must quote since "*" is a YAML reserved character; we want a string
+      - '*wheel*'  # must quote since "*" is a YAML reserved character; we want a string
 
 jobs:
   generate_backwards_compatibility_data:
@@ -19,7 +19,7 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
-
+      
       # Based on https://github.com/TileDB-Inc/conda-forge-nightly-controller/blob/51519a0f8340b32cf737fcb59b76c6a91c42dc47/.github/workflows/activity.yml#L19C10-L19C10
       - name: Setup git
         run: |
@@ -62,48 +62,93 @@ jobs:
           GH_TOKEN: ${{ github.token }}
 
   build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+    name: Wheel ${{ matrix.buildplat[0] }}-${{ matrix.buildplat[1] }}-${{ matrix.python }}
     # TODO(paris): Add this back once generate_backwards_compatibility_data is confirmed to work.
     # needs: generate_backwards_compatibility_data
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.buildplat[0] }}
     strategy:
       matrix:
-        os: [ubuntu-22.04, macos-12, windows-2022]
-        # `windows-2022, ` blocked by https://github.com/pybind/pybind11/issues/3445#issuecomment-1525500927
+        buildplat:
+          - [ ubuntu-22.04, manylinux_x86_64 ]
+          - [ macos-13, macosx_x86_64 ]
+          - [ macos-13, macosx_arm64 ]
+          # Windows build is currently disabled since the C++ build fails on multiple errors e.g. file manipulation
+          # and std::filesystem::path implicit to string conversion.
+          # Follow up task https://app.shortcut.com/tiledb-inc/story/41119/enable-windows-build-for-vector-search-repository
+        python: [ "cp39", "cp310", "cp311", "cp312", "pp39" ]
+        exclude:
+          - buildplat: [ macos-13, macosx_arm64 ]
+            python: "pp39"
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: "Brew setup on macOS" # x-ref c8e49ba8f8b9ce
+      - name: 'Brew setup on macOS' # x-ref c8e49ba8f8b9ce
         if: ${{ startsWith(matrix.os, 'macos-') == true }}
         run: |
           set -e pipefail
-          brew install automake pkg-config ninja
+          brew update
+          brew install automake pkg-config ninja llvm
 
       - name: Build wheels
-        uses: pypa/[email protected].2
+        uses: pypa/[email protected].5
         env:
-          CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
-          CIBW_SKIP: "*-win32 cp27-* cp35-* cp36-* cp37-* pp* *_i686 *musllinux*"
-          CIBW_ARCHS_MACOS: "x86_64 arm64"
           CIBW_BUILD_VERBOSITY: 3
+          CIBW_ENVIRONMENT_MACOS: >
+            CC=clang
+            CXX=clang++
           MACOSX_DEPLOYMENT_TARGET: "12.0"
+          CIBW_ARCHS: all
+          CIBW_PRERELEASE_PYTHONS: True
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.buildplat[1] }}
         with:
           output-dir: wheelhouse
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: './wheelhouse/*.whl'
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          path: dist/*.tar.gz
+
+
+  upload_pypi:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+    outputs:
+      package_version: ${{ steps.get_package_version.outputs.package_version }}
+    steps:
+      - uses: actions/download-artifact@v4
         with:
-          path: ./wheelhouse/*.whl
-# TODO: Needs support for pulling in the root directory
-#  build_sdist:
-#    name: Build source distribution
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@v3
-#
-#      - name: Build sdist
-#        run: pipx run build --sdist
-#
-#      - uses: actions/upload-artifact@v3
-#        with:
-#          path: dist/*.tar.gz
+          # unpacks all CIBW artifacts into dist/
+          pattern: 'cibw-*'
+          path: dist
+          merge-multiple: true
+
+      - id: get_package_version
+        run: |
+          echo "package_version=$(ls dist/ | head -n 1 | cut -d - -f 2)" >> "$GITHUB_OUTPUT"
+
+      - name: Upload to test-pypi
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+      - name: Upload to pypi
+        if: ${{ github.event_name != 'workflow_dispatch' }}
+        uses: pypa/gh-action-pypi-publish@release/v1

apis/python/requirements-py.txt

@@ -1,3 +1,4 @@
 numpy==1.24.3
 tiledb-cloud==0.10.24
 tiledb==0.25.0
+scikit-learn==1.3.2