Skip to content

Commit fdd01cb

Browse files
author
umi
committed
fixDevice
1 parent 7c20467 commit fdd01cb

File tree

1 file changed

+88
-109
lines changed

1 file changed

+88
-109
lines changed

‎.github/workflows/paimon-python-checks.yml‎

Lines changed: 88 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -85,129 +85,108 @@ jobs:
8585
run: |
8686
echo "Start compiling modules"
8787
mvn -T 2C -B clean install -DskipTests
88-
df -h
89-
# Clean up Maven build artifacts to save space
90-
echo "Cleaning up Maven artifacts..."
91-
rm -rf ~/.m2/repository/org/apache/paimon/*/target
92-
df -h
9388
9489
- name: Install Python dependencies
9590
shell: bash
9691
run: |
97-
df -h / && du -sh /tmp 2>/dev/null || echo "/tmp empty or not exist"
92+
df -h
9893
if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
9994
python -m pip install --upgrade pip==21.3.1
10095
python --version
101-
python -m pip install --no-cache-dir -q pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
96+
python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
10297
else
103-
echo $TMPDIR
104-
python -c "import tempfile; print('tempfile.gettempdir():', tempfile.gettempdir())"
10598
python -m pip install --upgrade pip
106-
python -m pip install --no-cache-dir \
107-
readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 \
108-
fastavro==1.11.1 zstandard==0.24.0 pylance==0.39.0 flake8==4.0.1 \
109-
pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 \
110-
2>&1 >/dev/null
111-
ls -la /tmp/ | grep pip || echo "No pip temp dirs in /tmp"
112-
python -m pip install --no-cache-dir \
113-
numpy==1.24.3 pandas==2.0.3 duckdb==1.3.2 polars==1.32.0 \
114-
2>&1 >/dev/null
115-
rm -rf /tmp/pip-*
116-
python -m pip install --no-cache-dir \
117-
torch ray==2.48.0 pyarrow==16.0.0 \
118-
2>&1 >/dev/null
119-
rm -rf /tmp/pip-* /tmp/torch* /tmp/ray*
99+
pip install torch --index-url https://download.pytorch.org/whl/cpu
100+
python -m pip install pyroaring readerwriterlock==1.0.9 fsspec==2024.3.1 cachetools==5.3.3 ossfs==2023.12.0 ray==2.48.0 fastavro==1.11.1 pyarrow==16.0.0 zstandard==0.24.0 polars==1.32.0 duckdb==1.3.2 numpy==1.24.3 pandas==2.0.3 pylance==0.39.0 flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0
120101
fi
121-
df -h / && du -sh /tmp 2>/dev/null || echo "/tmp empty or not exist"
122-
# Clean up pip cache after installation
123-
rm -rf ~/.cache/pip
124102
df -h
125103
- name: Run lint-python.sh
126104
shell: bash
127105
run: |
128106
chmod +x paimon-python/dev/lint-python.sh
129107
./paimon-python/dev/lint-python.sh
130108
131-
# requirement_version_compatible_test:
132-
# runs-on: ubuntu-latest
133-
# container: "python:3.10-slim"
134-
#
135-
# steps:
136-
# - name: Checkout code
137-
# uses: actions/checkout@v2
138-
#
139-
# - name: Install system dependencies
140-
# shell: bash
141-
# run: |
142-
# apt-get update && apt-get install -y \
143-
# build-essential \
144-
# git \
145-
# curl \
146-
# && rm -rf /var/lib/apt/lists/*
147-
#
148-
# - name: Verify Python version
149-
# run: python --version
150-
#
151-
# - name: Install base Python dependencies
152-
# shell: bash
153-
# run: |
154-
# python -m pip install --upgrade pip
155-
# python -m pip install --no-cache-dir -q \
156-
# pyroaring \
157-
# readerwriterlock==1.0.9 \
158-
# fsspec==2024.3.1 \
159-
# cachetools==5.3.3 \
160-
# ossfs==2023.12.0 \
161-
# fastavro==1.11.1 \
162-
# pyarrow==16.0.0 \
163-
# zstandard==0.24.0 \
164-
# polars==1.32.0 \
165-
# duckdb==1.3.2 \
166-
# numpy==1.24.3 \
167-
# pandas==2.0.3 \
168-
# pytest~=7.0 \
169-
# py4j==0.10.9.9 \
170-
# requests \
171-
# parameterized==0.9.0 \
172-
# packaging \
173-
# torch
174-
#
175-
# - name: Test requirement version compatibility
176-
# shell: bash
177-
# run: |
178-
# cd paimon-python
179-
#
180-
# # Test Ray version compatibility
181-
# echo "=========================================="
182-
# echo "Testing Ray version compatibility"
183-
# echo "=========================================="
184-
# for ray_version in 2.44.0 2.48.0 2.53.0; do
185-
# echo "Testing Ray version: $ray_version"
186-
#
187-
# # Install specific Ray version
188-
# python -m pip install --no-cache-dir -q ray==$ray_version
189-
#
190-
# # Verify Ray version
191-
# python -c "import ray; print(f'Ray version: {ray.__version__}')"
192-
# python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
193-
#
194-
# # Run tests
195-
# python -m pytest pypaimon/tests/ray_data_test.py::RayDataTest -v --tb=short || {
196-
# echo "Tests failed for Ray $ray_version"
197-
# exit 1
198-
# }
199-
#
200-
# # Uninstall Ray to avoid conflicts
201-
# python -m pip uninstall -y ray
202-
# done
203-
#
204-
# # Add other dependency version tests here in the future
205-
# # Example:
206-
# # echo "=========================================="
207-
# # echo "Testing PyArrow version compatibility"
208-
# # echo "=========================================="
209-
# # for pyarrow_version in 16.0.0 17.0.0 18.0.0; do
210-
# # ...
211-
# # done
212-
# env:
213-
# PYTHONPATH: ${{ github.workspace }}/paimon-python
109+
requirement_version_compatible_test:
110+
runs-on: ubuntu-latest
111+
container: "python:3.10-slim"
112+
113+
steps:
114+
- name: Checkout code
115+
uses: actions/checkout@v2
116+
117+
- name: Install system dependencies
118+
shell: bash
119+
run: |
120+
apt-get update && apt-get install -y \
121+
build-essential \
122+
git \
123+
curl \
124+
&& rm -rf /var/lib/apt/lists/*
125+
126+
- name: Verify Python version
127+
run: python --version
128+
129+
- name: Install base Python dependencies
130+
shell: bash
131+
run: |
132+
python -m pip install --upgrade pip
133+
pip install torch --index-url https://download.pytorch.org/whl/cpu
134+
python -m pip install --no-cache-dir \
135+
pyroaring \
136+
readerwriterlock==1.0.9 \
137+
fsspec==2024.3.1 \
138+
cachetools==5.3.3 \
139+
ossfs==2023.12.0 \
140+
fastavro==1.11.1 \
141+
pyarrow==16.0.0 \
142+
zstandard==0.24.0 \
143+
polars==1.32.0 \
144+
duckdb==1.3.2 \
145+
numpy==1.24.3 \
146+
pandas==2.0.3 \
147+
pytest~=7.0 \
148+
py4j==0.10.9.9 \
149+
requests \
150+
parameterized==0.9.0 \
151+
packaging
152+
153+
154+
- name: Test requirement version compatibility
155+
shell: bash
156+
run: |
157+
cd paimon-python
158+
159+
# Test Ray version compatibility
160+
echo "=========================================="
161+
echo "Testing Ray version compatibility"
162+
echo "=========================================="
163+
for ray_version in 2.44.0 2.48.0 2.53.0; do
164+
echo "Testing Ray version: $ray_version"
165+
166+
# Install specific Ray version
167+
python -m pip install --no-cache-dir -q ray==$ray_version
168+
169+
# Verify Ray version
170+
python -c "import ray; print(f'Ray version: {ray.__version__}')"
171+
python -c "from packaging.version import parse; import ray; assert parse(ray.__version__) == parse('$ray_version'), f'Expected Ray $ray_version, got {ray.__version__}'"
172+
173+
# Run tests
174+
python -m pytest pypaimon/tests/ray_data_test.py::RayDataTest -v --tb=short || {
175+
echo "Tests failed for Ray $ray_version"
176+
exit 1
177+
}
178+
179+
# Uninstall Ray to avoid conflicts
180+
python -m pip uninstall -y ray
181+
done
182+
183+
# Add other dependency version tests here in the future
184+
# Example:
185+
# echo "=========================================="
186+
# echo "Testing PyArrow version compatibility"
187+
# echo "=========================================="
188+
# for pyarrow_version in 16.0.0 17.0.0 18.0.0; do
189+
# ...
190+
# done
191+
env:
192+
PYTHONPATH: ${{ github.workspace }}/paimon-python

0 commit comments

Comments
 (0)