Refactor: rename Java packages from org.greenplum to org.apache.cloudberry
#66
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: PXF CI Pipeline | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| types: [opened, synchronize, reopened, edited] | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| JAVA_VERSION: "11" | |
| JAVA_HOME: "/usr/lib/jvm/java-11-openjdk" | |
| GO_VERSION: "1.21" | |
| GPHOME: "/usr/local/cloudberry-db" | |
| CLOUDBERRY_VERSION: "main" | |
| PXF_HOME: "/usr/local/pxf" | |
| jobs: | |
| # Stage 1: Build artifacts (runs in parallel) | |
| build-cloudberry-deb: | |
| name: Build Cloudberry DEB Package | |
| runs-on: ubuntu-latest | |
| container: | |
| image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest | |
| options: --user root | |
| steps: | |
| - name: Checkout Cloudberry source | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: apache/cloudberry | |
| ref: ${{ env.CLOUDBERRY_VERSION }} | |
| path: workspace/cloudberry | |
| submodules: true | |
| - name: Checkout PXF source (for build script) | |
| uses: actions/checkout@v4 | |
| with: | |
| path: cloudberry-pxf | |
| - name: Build Cloudberry DEB | |
| run: | | |
| export WORKSPACE=$PWD/workspace | |
| export CLOUDBERRY_VERSION=99.0.0 | |
| export CLOUDBERRY_BUILD=1 | |
| bash cloudberry-pxf/ci/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh | |
| - name: Package Cloudberry source | |
| run: | | |
| cd workspace | |
| tar czf cloudberry-source.tar.gz cloudberry/ | |
| - name: Upload DEB artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cloudberry-deb | |
| path: workspace/cloudberry-deb/*.deb | |
| retention-days: 7 | |
| - name: Upload Cloudberry source artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: cloudberry-source | |
| path: workspace/cloudberry-source.tar.gz | |
| retention-days: 7 | |
| build-docker-images: | |
| name: Build Docker Images | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout PXF source | |
| uses: actions/checkout@v4 | |
| with: | |
| path: cloudberry-pxf | |
| - name: Build singlecluster image | |
| run: | | |
| cd cloudberry-pxf/ci/singlecluster | |
| docker build -t pxf/singlecluster:3 . | |
| docker save pxf/singlecluster:3 > /tmp/singlecluster-image.tar | |
| - name: Upload singlecluster image | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: singlecluster-image | |
| path: /tmp/singlecluster-image.tar | |
| retention-days: 1 | |
| # Stage 2: Parallel test jobs using matrix strategy | |
| pxf-test: | |
| name: Test PXF - ${{ matrix.test_group }} | |
| needs: [build-cloudberry-deb, build-docker-images] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_group: | |
| - cli | |
| - external-table | |
| - server | |
| - sanity | |
| - smoke | |
| - hdfs | |
| - hcatalog | |
| - hcfs | |
| - hive | |
| - hbase | |
| - profile | |
| - jdbc | |
| - proxy | |
| - unused | |
| - s3 | |
| - features | |
| - gpdb | |
| - load | |
| - pxf_extension | |
| steps: | |
| - name: Free disk space | |
| run: | | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /usr/local/share/boost | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/hostedtoolcache | |
| sudo docker system prune -af | |
| df -h | |
| - name: Checkout PXF source | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 1 | |
| path: cloudberry-pxf | |
| submodules: true | |
| - name: Download Cloudberry DEB | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: cloudberry-deb | |
| path: /tmp | |
| - name: Download Cloudberry source | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: cloudberry-source | |
| path: /tmp | |
| - name: Download singlecluster image | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: singlecluster-image | |
| path: /tmp | |
| - name: Load singlecluster image | |
| run: | | |
| docker load < /tmp/singlecluster-image.tar | |
| - name: Prepare Cloudberry source | |
| run: | | |
| tar xzf /tmp/cloudberry-source.tar.gz | |
| chmod -R u+rwX,go+rX cloudberry | |
| - name: Start Services | |
| id: start_services | |
| run: | | |
| cd cloudberry-pxf | |
| docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true | |
| docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml build | |
| docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml up -d | |
| docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry | |
| docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/ | |
| docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb | |
| docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/ci/docker/pxf-cbdb-dev/ubuntu && ./script/entrypoint.sh" | |
| - name: Run Test - ${{ matrix.test_group }} | |
| id: run_test | |
| continue-on-error: true | |
| timeout-minutes: 120 | |
| run: | | |
| docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../ci/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../ci/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}" | |
| - name: Collect artifacts and generate stats | |
| if: always() | |
| id: collect_artifacts | |
| run: | | |
| mkdir -p artifacts/logs | |
| TEST_GROUP="${{ matrix.test_group }}" | |
| TEST_RESULT="${{ steps.run_test.outcome }}" | |
| # Initialize counters | |
| TOTAL=0 | |
| PASSED=0 | |
| FAILED=0 | |
| SKIPPED=0 | |
| # Copy test artifacts | |
| cp -r cloudberry-pxf/automation/test_artifacts/* artifacts/ 2>/dev/null || true | |
| docker exec pxf-cbdb-dev bash -c "cp -r /usr/local/pxf/logs/* /tmp/pxf-logs/ 2>/dev/null || true" || true | |
| docker cp pxf-cbdb-dev:/tmp/pxf-logs artifacts/logs/ 2>/dev/null || true | |
| # Parse surefire reports for automation tests | |
| if [[ "$TEST_GROUP" != "cli" && "$TEST_GROUP" != "server" ]]; then | |
| for xml in cloudberry-pxf/automation/target/surefire-reports/TEST-*.xml; do | |
| if [ -f "$xml" ]; then | |
| tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") | |
| failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") | |
| errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") | |
| skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0") | |
| TOTAL=$((TOTAL + tests)) | |
| FAILED=$((FAILED + failures + errors)) | |
| SKIPPED=$((SKIPPED + skipped)) | |
| fi | |
| done | |
| PASSED=$((TOTAL - FAILED - SKIPPED)) | |
| fi | |
| # Generate stats JSON | |
| cat > artifacts/test_stats.json <<EOF | |
| { | |
| "group": "$TEST_GROUP", | |
| "result": "$TEST_RESULT", | |
| "total": $TOTAL, | |
| "passed": $PASSED, | |
| "failed": $FAILED, | |
| "skipped": $SKIPPED | |
| } | |
| EOF | |
| echo "failed_count=$FAILED" >> $GITHUB_OUTPUT | |
| echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT | |
| echo "Test stats for $TEST_GROUP: total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED" | |
| - name: Cleanup containers | |
| if: always() | |
| run: | | |
| cd cloudberry-pxf | |
| docker compose -f ci/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true | |
| - name: Upload test artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-results-${{ matrix.test_group }} | |
| path: artifacts/** | |
| if-no-files-found: ignore | |
| retention-days: 7 | |
| - name: Check test result | |
| if: always() | |
| run: | | |
| FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}" | |
| SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}" | |
| if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then | |
| echo "Test group ${{ matrix.test_group }} failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)" | |
| exit 1 | |
| fi | |
| # Stage 3: Summary job | |
| test-summary: | |
| name: Test Summary | |
| needs: [pxf-test] | |
| if: always() | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download all test artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: all-artifacts | |
| pattern: test-results-* | |
| - name: Generate summary | |
| run: | | |
| echo "## PXF Test Results Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Overall counters | |
| OVERALL_TOTAL=0 | |
| OVERALL_PASSED=0 | |
| OVERALL_FAILED=0 | |
| OVERALL_SKIPPED=0 | |
| GROUPS_PASSED=0 | |
| GROUPS_FAILED=0 | |
| FAILED_GROUP_NAMES="" | |
| # Collect all test stats | |
| declare -A GROUP_STATS | |
| for dir in all-artifacts/test-results-*; do | |
| if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then | |
| group=$(cat "$dir/test_stats.json" | grep -oP '"group":\s*"\K[^"]+' || basename "$dir" | sed 's/test-results-//') | |
| result=$(cat "$dir/test_stats.json" | grep -oP '"result":\s*"\K[^"]+' || echo "unknown") | |
| total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+' || echo "0") | |
| passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+' || echo "0") | |
| failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+' || echo "0") | |
| skipped=$(cat "$dir/test_stats.json" | grep -oP '"skipped":\s*\K\d+' || echo "0") | |
| GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped" | |
| OVERALL_TOTAL=$((OVERALL_TOTAL + total)) | |
| OVERALL_PASSED=$((OVERALL_PASSED + passed)) | |
| OVERALL_FAILED=$((OVERALL_FAILED + failed)) | |
| OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped)) | |
| if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then | |
| GROUPS_PASSED=$((GROUPS_PASSED + 1)) | |
| else | |
| GROUPS_FAILED=$((GROUPS_FAILED + 1)) | |
| FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} " | |
| fi | |
| fi | |
| done | |
| # Overall summary | |
| echo "### Overall Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| if [ $GROUPS_FAILED -eq 0 ]; then | |
| echo "✅ **All ${GROUPS_PASSED} test groups passed**" >> $GITHUB_STEP_SUMMARY | |
| else | |
| echo "❌ **${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED)) test groups failed**" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY | |
| echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY | |
| echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY | |
| echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Detailed table | |
| echo "### Test Results by Group" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY | |
| echo "|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY | |
| for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do | |
| IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}" | |
| if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then | |
| status="✅ PASS" | |
| else | |
| status="❌ FAIL" | |
| fi | |
| echo "| $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY | |
| done | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| # Check if any group failed | |
| if [ $GROUPS_FAILED -gt 0 ]; then | |
| echo "::error::${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}" | |
| exit 1 | |
| fi |