Skip to content
Python Check Code Style and Test

Merge branch 'master' into 6880 #4

Sign in to view logs

Merge branch 'master' into 6880

Merge branch 'master' into 6880 #4

Workflow file for this run

.github/workflows/paimon-python-checks.yml at 4c02bb3
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
name: Python Check Code Style and Test
on:
push:
paths:
- 'paimon-python/**'
- '!**/*.md'
- '.github/workflows/paimon-python-checks.yml'
pull_request:
paths:
- 'paimon-python/**'
- '!**/*.md'
- '.github/workflows/paimon-python-checks.yml'
env:
PYTHON_VERSIONS: "['3.6.15', '3.10']"
JDK_VERSION: 8
MAVEN_OPTS: -Dmaven.wagon.httpconnectionManager.ttlSeconds=30 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.number || github.run_id }}
cancel-in-progress: true
jobs:
build_native:
uses: ./.github/workflows/build-faiss-native.yml
with:
platform: linux-amd64
jdk-version: '8'
lint-python:
runs-on: ubuntu-latest
needs: build_native
container: "python:${{ matrix.python-version }}-slim"
strategy:
matrix:
python-version: [ '3.6.15', '3.10' ]
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Set up JDK ${{ env.JDK_VERSION }}
uses: actions/setup-java@v5
with:
java-version: ${{ env.JDK_VERSION }}
distribution: 'temurin'
- name: Set up Maven
uses: stCarolas/setup-maven@v5
with:
maven-version: 3.8.8
- name: Install system dependencies
shell: bash
run: |
apt-get update && apt-get install -y \
build-essential \
git \
curl \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
- name: Verify Java and Maven installation
run: |
java -version
mvn -version
- name: Verify Python version
run: python --version
- name: Verify requirements.txt dependencies can be installed
shell: bash
run: |
cd paimon-python
python -m pip install --upgrade pip
# Use --target to install to a temporary directory to verify dependencies
# This works for both Python 3.6 (pip 21.3.1) and Python 3.10+ (pip 22.2+)
# since --target is supported in all pip versions
TEMP_DIR=$(mktemp -d)
python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || {
echo "ERROR: Failed to resolve dependencies from dev/requirements.txt"
echo "This indicates a version conflict or unavailable package version"
rm -rf "$TEMP_DIR"
exit 1
}
rm -rf "$TEMP_DIR"
echo "✓ All dependencies in dev/requirements.txt can be resolved"
- name: Download native library artifact
uses: actions/download-artifact@v7
with:
name: faiss-native-linux-amd64
path: paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
- name: List downloaded native library
run: |
echo "=== Downloaded native libraries ==="
ls -la paimon-faiss/paimon-faiss-jni/src/main/resources/linux/amd64/
- name: Build Java
run: |
echo "Start compiling modules"
mvn -T 2C -B -ntp clean install -DskipTests
- name: Build Java Paimon Faiss
run: |
echo "Start compiling modules"
mvn -T 2C -B -ntp clean install -DskipTests -Ppaimon-faiss
- name: Install Python dependencies
shell: bash
run: |
df -h
if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
python -m pip install --upgrade pip==21.3.1
python --version
python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 faiss-cpu==1.7.2 2>&1 >/dev/null
else
python -m pip install --upgrade pip
pip install torch --index-url https://download.pytorch.org/whl/cpu
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 cramjam flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 faiss-cpu==1.7.4
fi
df -h
- name: Run lint-python.sh
shell: bash
run: |
chmod +x paimon-python/dev/lint-python.sh
./paimon-python/dev/lint-python.sh -e pytest_torch
torch_test:
runs-on: ubuntu-latest
container: "python:3.10-slim"
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Install system dependencies
shell: bash
run: |
apt-get update && apt-get install -y \
build-essential \
git \
curl \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
- name: Verify Python version
run: python --version
- name: Install Python dependencies
shell: bash
run: |
python -m pip install --upgrade pip
pip install torch --index-url https://download.pytorch.org/whl/cpu
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
- name: Run lint-python.sh
shell: bash
run: |
chmod +x paimon-python/dev/lint-python.sh
./paimon-python/dev/lint-python.sh -i pytest_torch
requirement_version_compatible_test:
runs-on: ubuntu-latest
container: "python:3.10-slim"
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Install system dependencies
shell: bash
run: |
apt-get update && apt-get install -y \
build-essential \
git \
curl \
&& rm -rf /var/lib/apt/lists/*
- name: Verify Python version
run: python --version
- name: Install base Python dependencies
shell: bash
run: |
python -m pip install --upgrade pip
pip install torch --index-url https://download.pytorch.org/whl/cpu
python -m pip install --no-cache-dir \
pyroaring \
readerwriterlock==1.0.9 \
fsspec==2024.3.1 \
cachetools==5.3.3 \
ossfs==2023.12.0 \
fastavro==1.11.1 \
pyarrow==16.0.0 \
zstandard==0.24.0 \
polars==1.32.0 \
duckdb==1.3.2 \
numpy==1.24.3 \
pandas==2.0.3 \
cramjam \
pytest~=7.0 \
py4j==0.10.9.9 \
requests \
parameterized==0.9.0 \
packaging
- name: Test requirement version compatibility
shell: bash
run: |
cd paimon-python
# Test Ray version compatibility
echo "=========================================="
echo "Testing Ray version compatibility"
echo "=========================================="
for ray_version in 2.44.0 2.48.0 2.53.0; do
echo "Testing Ray version: $ray_version"
# Install specific Ray version
python -m pip install --no-cache-dir -q ray==$ray_version
# Verify Ray version
python -c "import ray; print(f'Ray version: {ray.__version__}')"
python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
# Run tests
python -m pytest pypaimon/tests/ray_data_test.py::RayDataTest -v --tb=short || {
echo "Tests failed for Ray $ray_version"
exit 1
}
# Uninstall Ray to avoid conflicts
python -m pip uninstall -y ray
done
# Add other dependency version tests here in the future
# Example:
# echo "=========================================="
# echo "Testing PyArrow version compatibility"
# echo "=========================================="
# for pyarrow_version in 16.0.0 17.0.0 18.0.0; do
# ...
# done
env:
PYTHONPATH: ${{ github.workspace }}/paimon-python