cu130.yml

name: fVDB CUDA 13.0.1 Build and Test

on:
  pull_request_target:
    branches:
      - '**'
    paths-ignore:
        - 'CODEOWNERS'
        - '**.md'
        - 'debug/**'
        - 'examples/**'
        - 'notebooks/**'
        - 'scripts/**'

# Allow subsequent pushes to the same PR or REF to cancel any previous jobs.
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  deployments: read
  pull-requests: read
  issues: read
  # Need ID token write permission to use OIDC
  id-token: write

jobs:
  ##############################################################################
  # START FVDB BUILD RUNNER
  ##############################################################################
  start-build-runner:
    name: Start CPU-only EC2 runner for build
    runs-on: ubuntu-latest
    if: github.event.pull_request.draft == false || github.event_name != 'pull_request_target'
    outputs:
      label: ${{ steps.start-build-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-build-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Start EC2 runner
        id: start-build-runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ami-0e14a711dad782a70
          ec2-instance-type: m6a.8xlarge
          subnet-id: subnet-03f2320d6e6e0005b
          security-group-id: sg-0cd08bd89d6212223

  ##############################################################################
  # BUILD FVDB
  ##############################################################################
  fvdb-build:
    name: fVDB Build
    needs: start-build-runner
    runs-on: ${{ needs.start-build-runner.outputs.label }}
    container:
      image: nvidia/cuda:13.0.1-cudnn-devel-ubuntu22.04
      env:
        PYTHONPATH: ""
        CPM_SOURCE_CACHE: "/__w/cpm_cache"
        FORCE_CUDA: 1
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Apt update and install git and wget
        run: apt update && apt install -y git wget

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          # For pull requests, this automatically checks out the merge commit
          # For pushes, this checks out the pushed commit
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref }}
      - name: Fetch PR branch
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global --add safe.directory "$(pwd)"
          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_branch

      - name: Merge PR branch into base
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git merge pr_branch

      # - name: Simulate merge for push events
      #   if: github.event_name == 'push'
      #   run: |
      #     echo "Push event detected - simulating merge with target branch"
      #     git config --global user.name "github-actions[bot]"
      #     git config --global user.email "github-actions[bot]@users.noreply.github.com"
      #     git config --global --add safe.directory "$(pwd)"

      #     # Get the target branch (main in this case, since push is only on main)
      #     TARGET_BRANCH="main"
      #     CURRENT_COMMIT="${{ github.sha }}"

      #     echo "Creating merge simulation: merging $CURRENT_COMMIT into $TARGET_BRANCH"
      #     git fetch origin $TARGET_BRANCH
      #     git checkout $TARGET_BRANCH
      #     git merge --no-ff $CURRENT_COMMIT --message "CI: Simulating merge of $CURRENT_COMMIT into $TARGET_BRANCH"

      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "0.7.5"
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
        env:
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Install CMake
        run: >
          wget -nv https://github.com/Kitware/CMake/releases/download/v4.0.3/cmake-4.0.3-linux-x86_64.sh &&
          mkdir /opt/cmake &&
          sh cmake-4.0.3-linux-x86_64.sh --prefix=/usr/local --skip-license &&
          cmake --version

      - name: Install apt dependencies
        run: apt install -y zlib1g-dev libpng-dev pkg-config

      - name: Install pip dependencies
        run: |
          uv venv
          uv pip install --no-cache-dir -r env/build_requirements.txt --extra-index-url https://download.pytorch.org/whl/cu130

      - name: Build fvdb
        run: |
          source .venv/bin/activate
          ./build.sh wheel verbose gtests benchmarks --cuda-arch-list '8.9+PTX'

      - name: Upload wheel
        uses: actions/upload-artifact@v4
        with:
            name: fvdb-test-package
            path: dist/*.whl
            retention-days: 2

      #NOTE: This tar step is here because directly uploading the build directory
      #      wasn't working due to losing executable permissions on the files.
      - name: Tar build directory
        run: tar -cvf fvdb-gtest.tar build/

      - name: Upload gtests
        uses: actions/upload-artifact@v4
        with:
            name: fvdb-gtest
            path: fvdb-gtest.tar
            retention-days: 2

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          rm -rf /__w/_temp/*
          echo "Cleanup completed"

  ##############################################################################
  # STOP FVDB BUILD RUNNER
  ##############################################################################
  fvdb-build-stop-runner:
    name: Stop CPU-only EC2 runner for build
    needs:
      - start-build-runner # required to get output from the start-build-runner job
      - fvdb-build # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-build-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}

  ##############################################################################
  # START FVDB TESTS GPU RUNNER
  ##############################################################################
  start-tests-gpu-runner:
    name: Start EC2 GPU runner for gtests
    needs: fvdb-build
    runs-on: ubuntu-latest
    outputs:
      label: ${{ steps.start-tests-gpu-runner.outputs.label }}
      ec2-instance-id: ${{ steps.start-tests-gpu-runner.outputs.ec2-instance-id }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Start EC2 GPU runner
        id: start-tests-gpu-runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: start
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          ec2-image-id: ami-0f5c0b65fde08ae43
          ec2-instance-type: g6.2xlarge # 8 CPU-core, L4 GPU
          subnet-id: subnet-03f2320d6e6e0005b
          security-group-id: sg-0cd08bd89d6212223

  ##############################################################################
  # RUN FVDB GTESTS
  ##############################################################################
  fvdb-gtests:
    needs: start-tests-gpu-runner # required to start the main job when the runner is ready
    name: fVDB GTests
    runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
    container:
      image: nvidia/cuda:13.0.1-cudnn-runtime-ubuntu22.04
      env:
        PYTHONPATH: ""
        CPM_SOURCE_CACHE: "/__w/cpm_cache"
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Apt update and install git and wget
        run: apt update && apt install -y git wget

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          # For pull requests, this automatically checks out the merge commit
          # For pushes, this checks out the pushed commit
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref }}
      - name: Fetch PR branch
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global --add safe.directory "$(pwd)"
          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_branch

      - name: Merge PR branch into base
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git merge pr_branch


      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "0.7.5"
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
        env:
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Install CMake
        run: >
          wget -nv https://github.com/Kitware/CMake/releases/download/v4.0.3/cmake-4.0.3-linux-x86_64.sh &&
          mkdir /opt/cmake &&
          sh cmake-4.0.3-linux-x86_64.sh --prefix=/usr/local --skip-license &&
          cmake --version

      - name: Install apt dependencies
        run: apt install -y zlib1g-dev libpng-dev mesa-vulkan-drivers vulkan-tools make

      - name: Install pip dependencies
        run: |
          uv venv
          uv pip install --no-cache-dir -r env/build_requirements.txt --extra-index-url https://download.pytorch.org/whl/cu130

      - name: Download package with nanovdb_editor
        uses: actions/download-artifact@v4
        with:
            name: fvdb-test-package
            path: ./dist

      - name: Install nanovdb_editor
        run: |
            source .venv/bin/activate
            pip install ./dist/nanovdb_editor*.whl

      - name: Download gtests
        uses: actions/download-artifact@v4
        with:
            name: fvdb-gtest
            path: .

      - name: Extract tar
        run: |
            tar -xvf fvdb-gtest.tar

      - name: Run tests
        run: |
          source .venv/bin/activate
          ./build.sh ctest

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          rm -rf /__w/_temp/*
          echo "Cleanup completed"


  ##############################################################################
  # RUN FVDB UNIT TESTS
  ##############################################################################
  fvdb-unit-test:
    needs: start-tests-gpu-runner # required to start the main job when the runner is ready
    name: fVDB PyTests
    runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
    container:
      image: nvidia/cuda:13.0.1-cudnn-devel-ubuntu22.04
      env:
        PYTHONPATH: ""
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Apt update and install git and wget
        run: apt update && apt install -y git wget

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          # For pull requests, this automatically checks out the merge commit
          # For pushes, this checks out the pushed commit
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref }}

      - name: Fetch PR branch
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global --add safe.directory "$(pwd)"
          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_branch

      - name: Merge PR branch into base
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git merge pr_branch

      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "0.7.5"
      - name: Set up Python
        uses: actions/setup-python@v5
        id: setup-python
        with:
          python-version: '3.11'
        env:
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Install apt dependencies
        run: apt install -y zlib1g-dev libpng-dev libsparsehash-dev

      - name: Install pip dependencies
        run: |
          uv venv
          uv pip install --no-cache-dir -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/cu130 --index-strategy unsafe-best-match
          uv pip install --no-cache-dir setuptools
          TORCH_CUDA_ARCH_LIST="8.9+PTX" uv pip install -v --no-build-isolation git+https://github.com/rusty1s/pytorch_scatter.git
          TORCH_CUDA_ARCH_LIST="8.9+PTX" uv pip install -v --no-build-isolation git+https://github.com/heiwang1997/torchsparse.git

      - name: Download package
        uses: actions/download-artifact@v4
        with:
            name: fvdb-test-package
            path: ./dist

      - name: Install package
        run: |
          source .venv/bin/activate
          pip install ./dist/*.whl

      - name: Run tests
        run: |
          source .venv/bin/activate
          cd tests;
          pytest -v unit

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          rm -rf /__w/_temp/*
          echo "Cleanup completed"

  ##############################################################################
  # RUN FVDB DOCUMENTATION TESTS
  ##############################################################################
  fvdb-docs-test:
    needs: start-tests-gpu-runner # required to start the main job when the runner is ready
    name: fVDB Documentation Tests
    runs-on: ${{ needs.start-tests-gpu-runner.outputs.label }} # run the job on the newly created runner
    container:
      image: nvidia/cuda:13.0.1-cudnn-runtime-ubuntu22.04
      env:
        PYTHONPATH: ""
      options: --rm
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Apt update and install git and wget
        run: apt update && apt install -y git wget

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # Fetch all history for all branches
          # For pull requests, this automatically checks out the merge commit
          # For pushes, this checks out the pushed commit
          ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.base.ref || github.ref }}

      - name: Fetch PR branch
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git config --global user.name "github-actions[bot]"
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git config --global --add safe.directory "$(pwd)"
          git fetch origin pull/${{ github.event.pull_request.number }}/head:pr_branch

      - name: Merge PR branch into base
        if: github.event_name == 'pull_request_target'
        run: |
          cd $GITHUB_WORKSPACE
          git merge pr_branch
      - name: Install uv
        uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86
        with:
          version: "0.7.5"
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
        env:
          AGENT_TOOLSDIRECTORY: "/opt/hostedtoolcache"

      - name: Install apt dependencies
        run: apt install -y zlib1g-dev libpng-dev

      - name: Install pip dependencies
        run: |
          uv venv
          uv pip install --no-cache-dir -r env/test_requirements.txt --extra-index-url https://download.pytorch.org/whl/cu130 --index-strategy unsafe-best-match

      - name: Download package
        uses: actions/download-artifact@v4
        with:
            name: fvdb-test-package
            path: ./dist

      - name: Install package
        run: |
          source .venv/bin/activate
          pip install ./dist/*.whl

      - name: Run tests
        run: |
          source .venv/bin/activate
          pytest --markdown-docs docs --ignore-glob="**/wip/**"

      - name: Cleanup
        if: always()
        run: |
          echo "Cleaning up /__w/_temp directory"
          rm -rf /__w/_temp/*
          echo "Cleanup completed"

  ##############################################################################
  # STOP FVDB TESTS GPU RUNNER
  ##############################################################################
  fvdb-tests-stop-gpu-runner:
    name: Stop GPU EC2 runner for gtests
    needs:
      - start-tests-gpu-runner # required to get output from the start-tests-gpu-runner job
      - fvdb-gtests # required to wait when the main job is done
      - fvdb-unit-test # required to wait when the main job is done
      - fvdb-docs-test # required to wait when the main job is done
    runs-on: ubuntu-latest
    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::420032683002:role/openvdb-fvdb-github-actions-role
          aws-region: us-east-2
      - name: Stop EC2 runner
        uses: machulav/ec2-github-runner@v2.4.3
        with:
          mode: stop
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          label: ${{ needs.start-tests-gpu-runner.outputs.label }}
          ec2-instance-id: ${{ needs.start-tests-gpu-runner.outputs.ec2-instance-id }}