Add a pxf ci test

.github/workflows/pxf-ci.yml

@@ -0,0 +1,347 @@
+name: PXF CI Pipeline
+
+on:
+  push:
+    branches: [ merge-with-upstream ]
+  pull_request:
+    branches: [ merge-with-upstream ]
+    types: [opened, synchronize, reopened, edited]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  JAVA_VERSION: "11"
+  JAVA_HOME: "/usr/lib/jvm/java-11-openjdk"
+  GO_VERSION: "1.21"
+  GPHOME: "/usr/local/cloudberry-db"
+  CLOUDBERRY_VERSION: "main"
+  PXF_HOME: "/usr/local/pxf"
+
+jobs:
+  # Stage 1: Build artifacts (runs in parallel)
+  build-cloudberry-deb:
+    name: Build Cloudberry DEB Package
+    runs-on: ubuntu-latest
+    container:
+      image: apache/incubator-cloudberry:cbdb-build-ubuntu22.04-latest
+      options: --user root
+    steps:
+    - name: Checkout Cloudberry source
+      uses: actions/checkout@v4
+      with:
+        repository: apache/cloudberry
+        ref: ${{ env.CLOUDBERRY_VERSION }}
+        path: workspace/cloudberry
+        submodules: true
+
+    - name: Checkout PXF source (for build script)
+      uses: actions/checkout@v4
+      with:
+        path: cloudberry-pxf
+
+    - name: Build Cloudberry DEB
+      run: |
+        export WORKSPACE=$PWD/workspace
+        export CLOUDBERRY_VERSION=99.0.0
+        export CLOUDBERRY_BUILD=1
+        bash cloudberry-pxf/concourse/docker/pxf-cbdb-dev/ubuntu/script/build_cloudberry_deb.sh
+
+    - name: Package Cloudberry source
+      run: |
+        cd workspace
+        tar czf cloudberry-source.tar.gz cloudberry/
+
+    - name: Upload DEB artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloudberry-deb
+        path: workspace/cloudberry-deb/*.deb
+        retention-days: 7
+
+    - name: Upload Cloudberry source artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloudberry-source
+        path: workspace/cloudberry-source.tar.gz
+        retention-days: 7
+
+  build-docker-images:
+    name: Build Docker Images
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout PXF source
+      uses: actions/checkout@v4
+      with:
+        path: cloudberry-pxf
+
+    - name: Build singlecluster image
+      run: |
+        cd cloudberry-pxf/concourse/singlecluster
+        docker build -t pxf/singlecluster:3 .
+        docker save pxf/singlecluster:3 > /tmp/singlecluster-image.tar
+
+    - name: Upload singlecluster image
+      uses: actions/upload-artifact@v4
+      with:
+        name: singlecluster-image
+        path: /tmp/singlecluster-image.tar
+        retention-days: 1
+
+  # Stage 2: Parallel test jobs using matrix strategy
+  pxf-test:
+    name: Test PXF - ${{ matrix.test_group }}
+    needs: [build-cloudberry-deb, build-docker-images]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test_group:
+          - cli
+          - external-table
+          - server
+          - sanity
+          - smoke
+          - hdfs
+          - hcatalog
+          - hcfs
+          - hive
+          - hbase
+          - profile
+          - jdbc
+          - proxy
+          - unused
+          - s3
+          - features
+          - gpdb
+    steps:
+    - name: Free disk space
+      run: |
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /usr/local/share/boost
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/hostedtoolcache
+        sudo docker system prune -af
+        df -h
+
+    - name: Checkout PXF source
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        path: cloudberry-pxf
+        submodules: true
+
+    - name: Download Cloudberry DEB
+      uses: actions/download-artifact@v4
+      with:
+        name: cloudberry-deb
+        path: /tmp
+
+    - name: Download Cloudberry source
+      uses: actions/download-artifact@v4
+      with:
+        name: cloudberry-source
+        path: /tmp
+
+    - name: Download singlecluster image
+      uses: actions/download-artifact@v4
+      with:
+        name: singlecluster-image
+        path: /tmp
+
+    - name: Load singlecluster image
+      run: |
+        docker load < /tmp/singlecluster-image.tar
+
+    - name: Prepare Cloudberry source
+      run: |
+        tar xzf /tmp/cloudberry-source.tar.gz
+        chmod -R u+rwX,go+rX cloudberry
+
+    - name: Start Services
+      id: start_services
+      run: |
+        cd cloudberry-pxf
+        docker compose -f concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true
+        docker compose -f concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml build
+        docker compose -f concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml up -d
+        docker exec pxf-cbdb-dev sudo chown -R gpadmin:gpadmin /home/gpadmin/workspace/cloudberry
+        docker cp /tmp/*.deb pxf-cbdb-dev:/tmp/
+        docker exec pxf-cbdb-dev sudo chown gpadmin:gpadmin /tmp/*.deb
+        docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/concourse/docker/pxf-cbdb-dev/ubuntu && ./script/entrypoint.sh"
+
+    - name: Run Test - ${{ matrix.test_group }}
+      id: run_test
+      continue-on-error: true
+      run: |
+        docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}"
+
+    - name: Collect artifacts and generate stats
+      if: always()
+      run: |
+        mkdir -p artifacts/logs
+        TEST_GROUP="${{ matrix.test_group }}"
+        TEST_RESULT="${{ steps.run_test.outcome }}"
+
+        # Initialize counters
+        TOTAL=0
+        PASSED=0
+        FAILED=0
+        SKIPPED=0
+
+        # Copy test artifacts
+        cp -r cloudberry-pxf/automation/test_artifacts/* artifacts/ 2>/dev/null || true
+        docker exec pxf-cbdb-dev bash -c "cp -r /usr/local/pxf/logs/* /tmp/pxf-logs/ 2>/dev/null || true" || true
+        docker cp pxf-cbdb-dev:/tmp/pxf-logs artifacts/logs/ 2>/dev/null || true
+
+        # Parse surefire reports for automation tests
+        if [[ "$TEST_GROUP" != "cli" && "$TEST_GROUP" != "server" ]]; then
+          for xml in cloudberry-pxf/automation/target/surefire-reports/TEST-*.xml; do
+            if [ -f "$xml" ]; then
+              tests=$(grep -oP 'tests="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              failures=$(grep -oP 'failures="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              errors=$(grep -oP 'errors="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+              skipped=$(grep -oP 'skipped="\K\d+' "$xml" 2>/dev/null | head -1 || echo "0")
+
+              TOTAL=$((TOTAL + tests))
+              FAILED=$((FAILED + failures + errors))
+              SKIPPED=$((SKIPPED + skipped))
+            fi
+          done
+          PASSED=$((TOTAL - FAILED - SKIPPED))
+        fi
+
+        # Generate stats JSON
+        cat > artifacts/test_stats.json <<EOF
+        {
+          "group": "$TEST_GROUP",
+          "result": "$TEST_RESULT",
+          "total": $TOTAL,
+          "passed": $PASSED,
+          "failed": $FAILED,
+          "skipped": $SKIPPED
+        }
+        EOF
+
+        echo "Test stats for $TEST_GROUP: total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
+
+    - name: Cleanup containers
+      if: always()
+      run: |
+        cd cloudberry-pxf
+        docker compose -f concourse/docker/pxf-cbdb-dev/ubuntu/docker-compose.yml down -v || true
+
+    - name: Upload test artifacts
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: test-results-${{ matrix.test_group }}
+        path: artifacts/**
+        if-no-files-found: ignore
+        retention-days: 7
+
+    - name: Check test result
+      if: always()
+      run: |
+        if [ "${{ steps.run_test.outcome }}" == "failure" ]; then
+          echo "Test group ${{ matrix.test_group }} failed"
+          exit 1
+        fi
+
+  # Stage 3: Summary job
+  test-summary:
+    name: Test Summary
+    needs: [pxf-test]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+    - name: Download all test artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: all-artifacts
+        pattern: test-results-*
+
+    - name: Generate summary
+      run: |
+        echo "## PXF Test Results Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+        # Overall counters
+        OVERALL_TOTAL=0
+        OVERALL_PASSED=0
+        OVERALL_FAILED=0
+        OVERALL_SKIPPED=0
+        GROUPS_PASSED=0
+        GROUPS_FAILED=0
+
+        # Collect all test stats
+        declare -A GROUP_STATS
+
+        for dir in all-artifacts/test-results-*; do
+          if [ -d "$dir" ] && [ -f "$dir/test_stats.json" ]; then
+            group=$(cat "$dir/test_stats.json" | grep -oP '"group":\s*"\K[^"]+' || basename "$dir" | sed 's/test-results-//')
+            result=$(cat "$dir/test_stats.json" | grep -oP '"result":\s*"\K[^"]+' || echo "unknown")
+            total=$(cat "$dir/test_stats.json" | grep -oP '"total":\s*\K\d+' || echo "0")
+            passed=$(cat "$dir/test_stats.json" | grep -oP '"passed":\s*\K\d+' || echo "0")
+            failed=$(cat "$dir/test_stats.json" | grep -oP '"failed":\s*\K\d+' || echo "0")
+            skipped=$(cat "$dir/test_stats.json" | grep -oP '"skipped":\s*\K\d+' || echo "0")
+
+            GROUP_STATS[$group]="$result,$total,$passed,$failed,$skipped"
+
+            OVERALL_TOTAL=$((OVERALL_TOTAL + total))
+            OVERALL_PASSED=$((OVERALL_PASSED + passed))
+            OVERALL_FAILED=$((OVERALL_FAILED + failed))
+            OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped))
+
+            if [ "$result" == "success" ]; then
+              GROUPS_PASSED=$((GROUPS_PASSED + 1))
+            else
+              GROUPS_FAILED=$((GROUPS_FAILED + 1))
+            fi
+          fi
+        done
+
+        # Overall summary
+        echo "### Overall Summary" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        if [ $GROUPS_FAILED -eq 0 ]; then
+          echo "✅ **All ${GROUPS_PASSED} test groups passed**" >> $GITHUB_STEP_SUMMARY
+        else
+          echo "❌ **${GROUPS_FAILED} of $((GROUPS_PASSED + GROUPS_FAILED)) test groups failed**" >> $GITHUB_STEP_SUMMARY
+        fi
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "- Total Tests: $OVERALL_TOTAL" >> $GITHUB_STEP_SUMMARY
+        echo "- Passed: $OVERALL_PASSED" >> $GITHUB_STEP_SUMMARY
+        echo "- Failed: $OVERALL_FAILED" >> $GITHUB_STEP_SUMMARY
+        echo "- Skipped: $OVERALL_SKIPPED" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+        # Detailed table
+        echo "### Test Results by Group" >> $GITHUB_STEP_SUMMARY
+        echo "" >> $GITHUB_STEP_SUMMARY
+        echo "| Test Group | Status | Passed | Failed | Skipped | Total |" >> $GITHUB_STEP_SUMMARY
+        echo "|------------|--------|-------:|-------:|--------:|------:|" >> $GITHUB_STEP_SUMMARY
+
+        for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
+          IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}"
+          if [ "$result" == "success" ]; then
+            status="✅ PASS"
+          else
+            status="❌ FAIL"
+          fi
+          echo "| $group | $status | $passed | $failed | $skipped | $total |" >> $GITHUB_STEP_SUMMARY
+        done
+
+        echo "" >> $GITHUB_STEP_SUMMARY
+
+        # Check if any group failed
+        if [ $GROUPS_FAILED -gt 0 ]; then
+          echo "::error::${GROUPS_FAILED} test group(s) failed"
+          exit 1
+        fi

.gitignore

@@ -10,3 +10,9 @@
 build/
 server/*/out
 server/tmp
+/.metals/metals.mv.db
+/.vscode/c_cpp_properties.json
+/.vscode/launch.json
+/.vscode/settings.json
+/automation/dataTempFolder/
+/cli/go/pkg/

automation/.gitignore

@@ -8,3 +8,5 @@ tempClusterConfDirectory/
 output/
 automation_logs/
 regression.diffs
+/dataTempFolder/
+/jsystem0.log.lck

automation/Makefile

@@ -8,6 +8,8 @@
 SHELL := bash
 UNAME_S := $(shell uname -s)
 MAVEN_TEST_OPTS+= -B -e
+MAVEN_OPTS ?= -Xmx2g -Xms512m
+export MAVEN_OPTS
 PXF_TMP_LIB := $(HOME)/automation_tmp_lib
 BASE_PATH   ?= /mnt/nfs/var/nfsshare
 USE_FDW     ?= false
@@ -20,7 +22,7 @@ ifneq "$(GROUP)" ""
 	MAVEN_TEST_OPTS+= -Dgroups=$(GROUP)
 endif
 
-MAVEN_TEST_OPTS+= -Djava.awt.headless=true -DuseFDW=$(USE_FDW)
+MAVEN_TEST_OPTS+= -Djava.awt.headless=true -DuseFDW=$(USE_FDW) -Duser.timezone=UTC
 
 ifneq "$(OFFLINE)" "true"
 	MAVEN_TEST_OPTS+= -U
@@ -98,7 +100,8 @@ symlink_pxf_jars: check-env
 	@if [ -d "$(PXF_HOME)/application" ]; then \
 		rm -rf $(PXF_TMP_LIB) && \
 		mkdir -p $(PXF_TMP_LIB)/tmp && \
-		unzip -qq -j $(PXF_HOME)/application/pxf-app-*.jar 'BOOT-INF/lib/pxf-*.jar' -d $(PXF_TMP_LIB)/tmp && \
+		pxf_app=$$(ls -1v $(PXF_HOME)/application/pxf-app-*.jar | grep -v 'plain.jar' | tail -n 1) && \
+		unzip -qq -j "$${pxf_app}" 'BOOT-INF/lib/pxf-*.jar' -d $(PXF_TMP_LIB)/tmp && \
 		for jar in $(PXF_TMP_LIB)/tmp/pxf-*.jar; do \
 			jar_name="$${jar##*/}"; \
 			if [[ $${jar_name} =~ ^pxf-[A-Za-z0-9]+(-[0-9.]+.*).jar$$ ]]; then \

automation/jsystem.properties

@@ -13,5 +13,5 @@ reporter.classes=jsystem.extensions.report.html.LevelHtmlTestReporter;jsystem.fr
 resources.src=/home/gpadmin/workspace/pxf/automation/src/main/resources
 sutClassName=jsystem.framework.sut.SutImpl
 sutFile=default.xml
-tests.dir=/home/gpadmin/workspace/pxf/automation/target/test-classes
+tests.dir=/home/gpadmin/workspace/cloudberry-pxf/automation/target/test-classes
 tests.src=/home/gpadmin/workspace/pxf/automation/src/main/java