paimon-python-checks.yml

################################################################################
#  Licensed to the Apache Software Foundation (ASF) under one
#  or more contributor license agreements.  See the NOTICE file
#  distributed with this work for additional information
#  regarding copyright ownership.  The ASF licenses this file
#  to you under the Apache License, Version 2.0 (the
#  "License"); you may not use this file except in compliance
#  with the License.  You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
#  Unless required by applicable law or agreed to in writing, software
#  distributed under the License is distributed on an "AS IS" BASIS,
#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  See the License for the specific language governing permissions and
# limitations under the License.
################################################################################

name: Python Check Code Style and Test

on:
  push:
    paths:
      - 'paimon-python/**'
      - '!**/*.md'
      - '.github/workflows/paimon-python-checks.yml'
  pull_request:
    paths:
      - 'paimon-python/**'
      - '!**/*.md'
      - '.github/workflows/paimon-python-checks.yml'

env:
  PYTHON_VERSIONS: "['3.6.15', '3.10']"
  JDK_VERSION: 8
  MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true


concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
  cancel-in-progress: true

jobs:
  lint-python:
    runs-on: ubuntu-latest
    container: "python:${{ matrix.python-version }}-slim"
    strategy:
      matrix:
        python-version: ['3.6.15', '3.10']

    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Set up Maven
        uses: stCarolas/setup-maven@v4.5
        with:
          maven-version: 3.8.8

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B clean install -DskipTests

      - name: Install Python dependencies
        shell: bash
        run: |
          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
            python -m pip install --upgrade pip==21.3.1
            python --version
            python -m pip install -q pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
          else
            python -m pip install --upgrade pip
            python -m pip install -q pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 2>&1 >/dev/null
          fi
      - name: Run lint-python.sh
        shell: bash
        run: |
          chmod +x paimon-python/dev/lint-python.sh
          ./paimon-python/dev/lint-python.sh

  requirement_version_compatible_test:
    runs-on: ubuntu-latest
    container: "python:3.10-slim"

    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Set up JDK ${{ env.JDK_VERSION }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ env.JDK_VERSION }}
          distribution: 'temurin'

      - name: Set up Maven
        uses: stCarolas/setup-maven@v4.5
        with:
          maven-version: 3.8.8

      - name: Install system dependencies
        shell: bash
        run: |
          apt-get update && apt-get install -y \
            build-essential \
            git \
            curl \
            && rm -rf /var/lib/apt/lists/*

      - name: Verify Java and Maven installation
        run: |
          java -version
          mvn -version

      - name: Verify Python version
        run: python --version

      - name: Build Java
        run: |
          echo "Start compiling modules"
          mvn -T 2C -B clean install -DskipTests

      - name: Install base Python dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install -q \
            pyroaring \
            readerwriterlock==1.0.9 \
            fsspec==2024.3.1 \
            cachetools==5.3.3 \
            ossfs==2023.12.0 \
            fastavro==1.11.1 \
            pyarrow==16.0.0 \
            zstandard==0.24.0 \
            polars==1.32.0 \
            duckdb==1.3.2 \
            numpy==1.24.3 \
            pandas==2.0.3 \
            pytest~=7.0 \
            py4j==0.10.9.9 \
            requests \
            parameterized==0.9.0 \
            packaging

      - name: Test requirement version compatibility
        shell: bash
        run: |
          cd paimon-python
          
          # Test Ray version compatibility
          echo "=========================================="
          echo "Testing Ray version compatibility"
          echo "=========================================="
          for ray_version in 2.44.0 2.48.0 2.53.0; do
            echo "Testing Ray version: $ray_version"
            
            # Install specific Ray version
            python -m pip install -q ray==$ray_version
            
            # Verify Ray version
            python -c "import ray; print(f'Ray version: {ray.__version__}')"
            python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
            
            # Run tests
            python -m pytest pypaimon/tests/ray_data_test.py::RayDataTest -v --tb=short || {
              echo "Tests failed for Ray $ray_version"
              exit 1
            }
            
            # Uninstall Ray to avoid conflicts
            python -m pip uninstall -y ray
          done
          
          # Add other dependency version tests here in the future
          # Example:
          # echo "=========================================="
          # echo "Testing PyArrow version compatibility"
          # echo "=========================================="
          # for pyarrow_version in 16.0.0 17.0.0 18.0.0; do
          #   ...
          # done
        env:
          PYTHONPATH: ${{ github.workspace }}/paimon-python