[WIP] feat(c/sedona-libgpuspatial) Add GPU-accelerated spatial joins

.github/workflows/rust-gpu.yml

@@ -0,0 +1,256 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This workflow compiles CUDA code on GitHub-hosted runners (ubuntu-latest).
+# CUDA compilation (nvcc) works WITHOUT GPU hardware - only needs CUDA toolkit.
+# GPU runtime execution requires actual GPU, so tests are commented out.
+#
+name: rust-gpu
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'c/sedona-libgpuspatial/**'
+      - 'rust/sedona-spatial-join-gpu/**'
+      - '.github/workflows/rust-gpu.yml'
+
+  push:
+    branches:
+      - main
+    paths:
+      - 'c/sedona-libgpuspatial/**'
+      - 'rust/sedona-spatial-join-gpu/**'
+      - '.github/workflows/rust-gpu.yml'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.ref }}-${{ github.workflow }}-rust-gpu
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash -l -eo pipefail {0}
+
+# Set workflow timeout to 90 minutes for CUDA compilation
+# Expected: ~45-60 minutes first time, ~10-15 minutes cached
+env:
+  WORKFLOW_TIMEOUT_MINUTES: 90
+  # At GEOS updated to 3.14.0
+  VCPKG_REF: 5a01de756c28279ddfdd2b061d1c75710a6255fa
+
+jobs:
+  rust-gpu-build:
+    # Using GitHub-hosted runner to compile CUDA code
+    # CUDA compilation works without GPU hardware (only needs CUDA toolkit)
+    # GPU tests are skipped (no GPU hardware for runtime execution)
+    # TODO: Once GPU runner is ready, enable GPU tests with:
+    #   runs-on: [self-hosted, gpu, linux, cuda]
+    strategy:
+      fail-fast: false
+      matrix:
+        name: [ "clippy", "docs", "test", "build" ]
+
+    name: "${{ matrix.name }}"
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      CARGO_INCREMENTAL: 0
+      # Disable debug info completely to save disk space
+      CARGO_PROFILE_DEV_DEBUG: 0
+      CARGO_PROFILE_TEST_DEBUG: 0
+      # Limit parallel compilation to reduce memory pressure (GPU compilation is intensive)
+      CARGO_BUILD_JOBS: 4
+
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Clone vcpkg
+        uses: actions/checkout@v4
+        with:
+          repository: microsoft/vcpkg
+          ref: ${{ env.VCPKG_REF }}
+          path: vcpkg
+
+      # Set up environment variables for vcpkg and CUDA
+      - name: Set up environment variables and bootstrap vcpkg
+        env:
+          VCPKG_ROOT: ${{ github.workspace }}/vcpkg
+          CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
+          # CUDA configuration (CUDA 12.4 installs to /usr/local/cuda-12.4)
+          CUDA_HOME: /usr/local/cuda-12.4
+        run: |
+          cd vcpkg
+          ./bootstrap-vcpkg.sh
+          cd ..
+
+          echo "VCPKG_ROOT=$VCPKG_ROOT" >> $GITHUB_ENV
+          echo "PATH=$VCPKG_ROOT:$PATH" >> $GITHUB_ENV
+          echo "CMAKE_TOOLCHAIN_FILE=$CMAKE_TOOLCHAIN_FILE" >> $GITHUB_ENV
+          echo "/usr/local/cuda/bin" >> $GITHUB_PATH
+
+      # Free up disk space before build
+      - name: Free disk space
+        run: |
+          # Remove unnecessary packages to free up ~10GB
+          sudo apt-get remove -y '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri || true
+          sudo apt-get autoremove -y
+          sudo apt-get clean
+
+          # Remove large directories
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL || true
+
+      # Install system dependencies including CUDA toolkit for compilation
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+
+          # Install transport tools for Kitware CMake (needed for newer CMake)
+          sudo apt-get install -y apt-transport-https ca-certificates gnupg software-properties-common wget
+
+          # Add Kitware repository for CMake
+          wget -qO - https://apt.kitware.com/keys/kitware-archive-latest.asc | sudo apt-key add -
+          sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main'
+          sudo apt-get update
+
+          # Install build tools
+          sudo apt-get install -y build-essential pkg-config cmake flex bison
+
+          # Install libclang for bindgen (Rust FFI binding generator)
+          sudo apt-get install -y libclang-dev
+
+          # Verify compiler and CMake versions
+          gcc --version
+          g++ --version
+          cmake --version
+
+          # Install GEOS for spatial operations
+          sudo apt-get install -y libgeos-dev libzstd-dev
+
+          # Install CUDA toolkit for compilation (nvcc)
+          # Note: CUDA compilation works without GPU hardware
+          # GPU runtime tests still require actual GPU
+          # CUDA 12.4 supports GCC 11 (default on Ubuntu 22.04)
+          if ! command -v nvcc &> /dev/null; then
+            echo "Installing CUDA 12.4 toolkit for compilation..."
+
+            # Add NVIDIA CUDA repository
+            wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.1-1_all.deb
+            sudo dpkg -i cuda-keyring_1.1-1_all.deb
+            sudo apt-get update
+
+            # Remove any existing CUDA toolkit
+            sudo apt purge cuda-toolkit* -y || true
+
+            # Install CUDA 12.4 specifically
+            sudo apt-get install -y cuda-toolkit-12-4
+
+            # Set CUDA path
+            echo "/usr/local/cuda-12.4/bin" >> $GITHUB_PATH
+
+            nvcc --version
+          else
+            echo "CUDA toolkit already installed: $(nvcc --version)"
+          fi
+
+      # Cache vcpkg installed packages (expensive to rebuild)
+      - name: Cache vcpkg binaries
+        id: cache-vcpkg
+        uses: actions/cache@v4
+        with:
+          path: vcpkg/packages
+          # Bump the number at the end of this line to force a new dependency build
+          key: vcpkg-installed-${{ runner.os }}-${{ runner.arch }}-${{ env.VCPKG_REF }}-2
+
+      # Install vcpkg dependencies from vcpkg.json manifest
+      - name: Install vcpkg dependencies
+        if: steps.cache-vcpkg.outputs.cache-hit != 'true'
+        run: |
+          ./vcpkg/vcpkg install abseil openssl gtest
+          # Clean up vcpkg buildtrees and downloads to save space
+          rm -rf vcpkg/buildtrees
+          rm -rf vcpkg/downloads
+
+      - name: Use stable Rust
+        id: rust
+        run: |
+          rustup toolchain install stable --no-self-update
+          rustup default stable
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "rust-gpu-v3"
+          # Cache key includes GPU packages and vcpkg config
+          key: "${{ runner.os }}-${{ hashFiles('c/sedona-libgpuspatial/**', 'vcpkg.json') }}"
+
+      # Build WITH GPU feature to compile CUDA code
+      # CUDA compilation (nvcc) works without GPU hardware
+      # Only GPU runtime execution requires actual GPU
+      - name: Build libgpuspatial (with CUDA compilation)
+        run: |
+          echo "=== Building libgpuspatial WITH GPU feature ==="
+          echo "Compiling CUDA code using nvcc (no GPU hardware needed for compilation)"
+          echo "Note: First build with CUDA takes 45-60 minutes (CMake + CUDA compilation)"
+          echo "Subsequent builds: 10-15 minutes (cached)"
+          echo ""
+          echo "Build started at: $(date)"
+          # Build library only (skip tests - they require CUDA driver which isn't available)
+          # --lib builds only the library, not test binaries
+          cargo build --locked --package sedona-libgpuspatial --lib --features gpu --verbose
+
+      - name: Build GPU spatial join (with GPU feature)
+        run: |
+          echo "=== Building GPU spatial join package WITH GPU feature ==="
+          echo "Building Rust GPU spatial join (depends on libgpuspatial)"
+          echo ""
+          # Build library only (skip tests - they require CUDA driver)
+          cargo build --locked --package sedona-spatial-join-gpu --lib --features gpu --verbose
+
+      - name: Build entire workspace with GPU features
+        run: |
+          echo "=== Building entire SedonaDB workspace WITH GPU features ==="
+          echo "Verifying GPU packages integrate with rest of codebase"
+          echo ""
+          # Build entire workspace with GPU features enabled
+          # Exclude sedonadb (Python extension, requires maturin)
+          # Exclude sedona-s2geography (has GCC 11 compatibility issues, unrelated to GPU)
+          # Build libs only (skip tests - they require CUDA driver)
+          cargo build --workspace --exclude sedonadb --exclude sedona-s2geography --lib --features gpu --verbose
+
+      # GPU tests commented out - no GPU hardware on GitHub runners
+      # Uncomment these when running on self-hosted GPU runner
+
+      # - name: Test libgpuspatial
+      #   run: |
+      #     echo "Running libgpuspatial tests with GPU..."
+      #     cargo test --package sedona-libgpuspatial --features gpu -- --nocapture
+
+      # - name: Test GPU spatial join (structure tests)
+      #   run: |
+      #     echo "Running structure tests (don't require GPU execution)..."
+      #     cargo test --package sedona-spatial-join-gpu --features gpu
+
+      # - name: Test GPU functional tests (require GPU)
+      #   run: |
+      #     echo "Running GPU functional tests (require actual GPU)..."
+      #     cargo test --package sedona-spatial-join-gpu --features gpu -- --ignored --nocapture

.gitignore

@@ -49,3 +49,7 @@ __pycache__
 
 # .env file for release management
 dev/release/.env
+
+
+venv/
+