paimon-python-checks.yml

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

name: Python Check Code Style and Test

on:
  push:
    paths:
      - 'paimon-python/**'
      - '!**/*.md'
      - '.github/workflows/paimon-python-checks.yml'
  pull_request:
    paths:
      - 'paimon-python/**'
      - '!**/*.md'
      - '.github/workflows/paimon-python-checks.yml'

env:
  PYTHON_VERSIONS: "['3.6.15', '3.10']"
  JDK_VERSION: 8
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true


concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
  cancel-in-progress: true

jobs:
  build_native:
    uses: ./.github/workflows/build-faiss-native.yml
    with:
      platform: linux-amd64
      jdk-version: '8'

  lint-python:
    runs-on: ubuntu-latest
    needs: build_native
    container: "python:${{ matrix.python-version }}-slim"
    strategy:
      matrix:
        python-version: [ '3.6.15', '3.10' ]

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v5
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Set up Maven
        uses: stCarolas/setup-maven@v5
        with:
          maven-version: 3.8.8

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Verify requirements.txt dependencies can be installed
        shell: bash
        run: |
          cd paimon-python
          python -m pip install --upgrade pip
          # Use --target to install to a temporary directory to verify dependencies
          # This works for both Python 3.6 (pip 21.3.1) and Python 3.10+ (pip 22.2+)
          # since --target is supported in all pip versions
          TEMP_DIR=$(mktemp -d)
          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || {
            echo "ERROR: Failed to resolve dependencies from dev/requirements.txt"
            echo "This indicates a version conflict or unavailable package version"
            rm -rf "$TEMP_DIR"
            exit 1
          }
          rm -rf "$TEMP_DIR"
          echo "✓ All dependencies in dev/requirements.txt can be resolved"

      - name: Download native library artifact
        uses: actions/download-artifact@v7
        with:
          name: faiss-native-linux-amd64
          path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/

      - name: List downloaded native library
        run: |
          echo "=== Downloaded native libraries ==="
          ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B -ntp clean install -DskipTests

      - name: Build Java Paimon Faiss
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B -ntp clean install -DskipTests -Ppaimon-faiss

      - name: Install Python dependencies
        shell: bash
        run: |
          df -h
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            python -m pip install --upgrade pip==21.3.1
            python --version
            python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 faiss-cpu==1.7.2 2>&1 >/dev/null
          else
            python -m pip install --upgrade pip
            pip install torch --index-url https://download.pytorch.org/whl/cpu
            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 faiss-cpu==1.7.4
          fi
          df -h
      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -e pytest_torch

  torch_test:
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && apt-get clean \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Python version
        run: python --version

      - name: Install Python dependencies
        shell: bash
        run: |
            python -m pip install --upgrade pip
            pip install torch --index-url https://download.pytorch.org/whl/cpu
            python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh -i pytest_torch

  requirement_version_compatible_test:
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Python version
        run: python --version

      - name: Install base Python dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          pip install torch --index-url https://download.pytorch.org/whl/cpu
          python -m pip install --no-cache-dir \
            pyroaring \
            readerwriterlock==1.0.9 \
            fsspec==2024.3.1 \
            cachetools==5.3.3 \
            ossfs==2023.12.0 \
            fastavro==1.11.1 \
            pyarrow==16.0.0 \
            zstandard==0.24.0 \
            polars==1.32.0 \
            duckdb==1.3.2 \
            numpy==1.24.3 \
            pandas==2.0.3 \
            cramjam \
            pytest~=7.0 \
            py4j==0.10.9.9 \
            requests \
            parameterized==0.9.0 \
            packaging
            

      - name: Test requirement version compatibility
        shell: bash
        run: |
          cd paimon-python

          # Test Ray version compatibility
          echo "=========================================="
          echo "Testing Ray version compatibility"
          echo "=========================================="
          for ray_version in 2.44.0 2.48.0 2.53.0; do
            echo "Testing Ray version: $ray_version"

            # Install specific Ray version
            python -m pip install --no-cache-dir -q ray==$ray_version

            # Verify Ray version
            python -c "import ray; print(f'Ray version: {ray.__version__}')"
            python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"

            # Run tests
            python -m pytest pypaimon/tests/ray_data_test.py::RayDataTest -v --tb=short || {
              echo "Tests failed for Ray $ray_version"
              exit 1
            }

            # Uninstall Ray to avoid conflicts
            python -m pip uninstall -y ray
          done

          # Add other dependency version tests here in the future
          # Example:
          # echo "=========================================="
          # echo "Testing PyArrow version compatibility"
          # echo "=========================================="
          # for pyarrow_version in 16.0.0 17.0.0 18.0.0; do
          #   ...
          # done
        env:
          PYTHONPATH: ${{ github.workspace }}/paimon-python