Skip to content

Commit c6f6f39

Browse files
tjtanaatjtanaa
committed
add simple ga to tests my pypi server is setup correctly
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
1 parent ffa0965 commit c6f6f39

File tree

2 files changed

+219
-1
lines changed

2 files changed

+219
-1
lines changed

‎.github/scripts/generate_s3_index.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,7 +182,7 @@ def generate_package_index(
182182
except Exception as e:
183183
print(f"Warning: Could not extract metadata from {wheel_name}: {e}", file=sys.stderr)
184184

185-
if add_hashes:
185+
if add_metadata:
186186
try:
187187
sha256 = calculate_sha256(wheel_path)
188188
attributes.append(f'data-dist-info-metadata="sha256={sha256}"')

‎.github/workflows/test-s3-pypi.yml‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
name: Test S3 PyPI Repository
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
test_package:
7+
description: 'Package to test with (e.g., requests, flask, numpy)'
8+
required: false
9+
default: 'requests'
10+
python_version:
11+
description: 'Python version'
12+
required: false
13+
default: '3.12'
14+
15+
jobs:
16+
test-s3-pypi:
17+
runs-on: ubuntu-22.04
18+
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@v4
22+
23+
- name: Set up Python
24+
uses: actions/setup-python@v5
25+
with:
26+
python-version: ${{ github.event.inputs.python_version || '3.12' }}
27+
28+
- name: Install dependencies
29+
run: |
30+
python -m pip install --upgrade pip
31+
pip install packaging
32+
33+
- name: Download test package and dependencies
34+
run: |
35+
TEST_PACKAGE="${{ github.event.inputs.test_package || 'requests' }}"
36+
37+
echo "📦 Downloading $TEST_PACKAGE and its dependencies..."
38+
mkdir -p test-wheels
39+
40+
# Download the package and all its dependencies
41+
pip download "$TEST_PACKAGE" \
42+
--dest test-wheels \
43+
--python-version ${{ github.event.inputs.python_version || '3.12' }} \
44+
--platform manylinux2014_x86_64 \
45+
--only-binary=:all:
46+
47+
echo ""
48+
echo "Downloaded wheels:"
49+
ls -lh test-wheels/
50+
echo ""
51+
echo "Total wheels: $(ls test-wheels/*.whl 2>/dev/null | wc -l)"
52+
53+
- name: Generate PyPI index
54+
run: |
55+
echo "🔨 Generating PyPI index..."
56+
57+
# Use the website endpoint URL
58+
S3_URL="http://${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com"
59+
60+
python3 .github/scripts/generate_s3_index.py \
61+
--wheels-dir test-wheels \
62+
--output-dir test-index \
63+
--s3-url "$S3_URL" \
64+
--no-metadata \
65+
--rocm-version "N/A" \
66+
--python-version "${{ github.event.inputs.python_version || '3.12' }}" \
67+
--gpu-arch "Test" \
68+
--vllm-version "Test Repository"
69+
70+
echo ""
71+
echo "Generated index structure:"
72+
tree -L 3 test-index/ || find test-index/ -type f
73+
74+
echo ""
75+
echo "Sample package index:"
76+
TEST_PACKAGE="${{ github.event.inputs.test_package || 'requests' }}"
77+
NORMALIZED_NAME=$(echo "$TEST_PACKAGE" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
78+
if [ -f "test-index/simple/$NORMALIZED_NAME/index.html" ]; then
79+
echo "=== test-index/simple/$NORMALIZED_NAME/index.html ==="
80+
cat "test-index/simple/$NORMALIZED_NAME/index.html"
81+
fi
82+
83+
- name: Configure AWS credentials
84+
uses: aws-actions/configure-aws-credentials@v4
85+
with:
86+
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
87+
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
88+
aws-region: ${{ secrets.AWS_REGION }}
89+
90+
- name: Upload to S3 (test prefix)
91+
run: |
92+
echo "☁️ Uploading to S3 with 'test/' prefix..."
93+
94+
# Upload wheels to test/packages/
95+
echo "Uploading wheels..."
96+
aws s3 sync test-wheels/ "s3://${{ secrets.AWS_S3_BUCKET }}/test/packages/" \
97+
--region "${{ secrets.AWS_REGION }}" \
98+
--content-type "application/octet-stream" \
99+
--cache-control "public, max-age=31536000" \
100+
--size-only \
101+
--exclude "*" \
102+
--include "*.whl"
103+
104+
# Upload index to test/ (root of test prefix)
105+
echo "Uploading index..."
106+
aws s3 sync test-index/ "s3://${{ secrets.AWS_S3_BUCKET }}/test/" \
107+
--region "${{ secrets.AWS_REGION }}" \
108+
--content-type "text/html" \
109+
--cache-control "no-cache, no-store, must-revalidate" \
110+
--size-only \
111+
--delete \
112+
--exclude "packages/*"
113+
114+
echo "✅ Upload complete"
115+
116+
- name: Verify S3 structure
117+
run: |
118+
echo "🔍 Verifying S3 structure..."
119+
120+
TEST_PACKAGE="${{ github.event.inputs.test_package || 'requests' }}"
121+
NORMALIZED_NAME=$(echo "$TEST_PACKAGE" | tr '[:upper:]' '[:lower:]' | tr '_' '-')
122+
S3_URL="http://${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com"
123+
124+
echo ""
125+
echo "Testing URLs:"
126+
echo " Main index: $S3_URL/test/simple/"
127+
echo " Package index: $S3_URL/test/simple/$NORMALIZED_NAME/"
128+
echo ""
129+
130+
# Test main index
131+
echo "1. Checking main index..."
132+
curl -f "$S3_URL/test/simple/index.html" > /tmp/main-index.html
133+
if grep -q "$NORMALIZED_NAME" /tmp/main-index.html; then
134+
echo " ✅ Main index contains $NORMALIZED_NAME"
135+
else
136+
echo " ❌ Main index does NOT contain $NORMALIZED_NAME"
137+
exit 1
138+
fi
139+
140+
# Test package index
141+
echo "2. Checking package index..."
142+
curl -f "$S3_URL/test/simple/$NORMALIZED_NAME/index.html" > /tmp/pkg-index.html
143+
echo " Package index contents:"
144+
cat /tmp/pkg-index.html
145+
146+
# Check for data-dist-info-metadata (should NOT be present)
147+
if grep -q "data-dist-info-metadata" /tmp/pkg-index.html; then
148+
echo " ⚠️ WARNING: data-dist-info-metadata still present (--no-metadata not working)"
149+
else
150+
echo " ✅ No data-dist-info-metadata attributes (correct)"
151+
fi
152+
153+
# Test a wheel URL
154+
echo "3. Testing wheel accessibility..."
155+
WHEEL_URL=$(grep -oP 'href="\K[^"]+' /tmp/pkg-index.html | head -1)
156+
if [ -n "$WHEEL_URL" ]; then
157+
echo " Testing: $WHEEL_URL"
158+
if curl -I -f "$WHEEL_URL" > /dev/null 2>&1; then
159+
echo " ✅ Wheel is accessible"
160+
else
161+
echo " ❌ Wheel is NOT accessible"
162+
exit 1
163+
fi
164+
fi
165+
166+
- name: Test pip installation
167+
run: |
168+
echo "🧪 Testing pip installation..."
169+
170+
TEST_PACKAGE="${{ github.event.inputs.test_package || 'requests' }}"
171+
S3_URL="http://${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com"
172+
173+
# Create a clean virtual environment
174+
python -m venv test-venv
175+
source test-venv/bin/activate
176+
177+
echo ""
178+
echo "Installing $TEST_PACKAGE from S3 repository..."
179+
pip install -v "$TEST_PACKAGE" \
180+
--index-url "$S3_URL/test/simple/" \
181+
--trusted-host "${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com"
182+
183+
echo ""
184+
echo "✅ Installation successful!"
185+
186+
# Verify the package works
187+
echo ""
188+
echo "Testing import..."
189+
python -c "import $(echo $TEST_PACKAGE | tr '-' '_'); print('✅ Import successful')"
190+
191+
- name: Cleanup test files from S3
192+
if: always()
193+
run: |
194+
echo "🧹 Cleaning up test files from S3..."
195+
aws s3 rm "s3://${{ secrets.AWS_S3_BUCKET }}/test/" \
196+
--region "${{ secrets.AWS_REGION }}" \
197+
--recursive
198+
echo "✅ Cleanup complete"
199+
200+
- name: Summary
201+
if: always()
202+
run: |
203+
TEST_PACKAGE="${{ github.event.inputs.test_package || 'requests' }}"
204+
S3_URL="http://${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com"
205+
206+
echo "## Test Summary" >> $GITHUB_STEP_SUMMARY
207+
echo "" >> $GITHUB_STEP_SUMMARY
208+
echo "**Test Package:** $TEST_PACKAGE" >> $GITHUB_STEP_SUMMARY
209+
echo "**Python Version:** ${{ github.event.inputs.python_version || '3.12' }}" >> $GITHUB_STEP_SUMMARY
210+
echo "" >> $GITHUB_STEP_SUMMARY
211+
echo "**Test Repository URL:** \`$S3_URL/test/simple/\`" >> $GITHUB_STEP_SUMMARY
212+
echo "" >> $GITHUB_STEP_SUMMARY
213+
echo "**Installation Command:**" >> $GITHUB_STEP_SUMMARY
214+
echo '```bash' >> $GITHUB_STEP_SUMMARY
215+
echo "pip install $TEST_PACKAGE \\" >> $GITHUB_STEP_SUMMARY
216+
echo " --index-url $S3_URL/test/simple/ \\" >> $GITHUB_STEP_SUMMARY
217+
echo " --trusted-host ${{ secrets.AWS_S3_BUCKET }}.s3-website-${{ secrets.AWS_REGION }}.amazonaws.com" >> $GITHUB_STEP_SUMMARY
218+
echo '```' >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)