Autmation: update pipeline & fix hive file copy

.github/workflows/pxf-ci.yml

@@ -179,11 +179,13 @@ jobs:
     - name: Run Test - ${{ matrix.test_group }}
       id: run_test
       continue-on-error: true
+      timeout-minutes: 120
       run: |
         docker exec pxf-cbdb-dev bash -lc "cd /home/gpadmin/workspace/cloudberry-pxf/automation && source ../concourse/docker/pxf-cbdb-dev/ubuntu/script/pxf-env.sh && ../concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh ${{ matrix.test_group }}"
-
+        
     - name: Collect artifacts and generate stats
       if: always()
+      id: collect_artifacts
       run: |
         mkdir -p artifacts/logs
         TEST_GROUP="${{ matrix.test_group }}"
@@ -229,6 +231,8 @@ jobs:
         }
         EOF
 
+        echo "failed_count=$FAILED" >> $GITHUB_OUTPUT
+        echo "skipped_count=$SKIPPED" >> $GITHUB_OUTPUT
         echo "Test stats for $TEST_GROUP: total=$TOTAL, passed=$PASSED, failed=$FAILED, skipped=$SKIPPED"
 
     - name: Cleanup containers
@@ -249,8 +253,11 @@ jobs:
     - name: Check test result
       if: always()
       run: |
-        if [ "${{ steps.run_test.outcome }}" == "failure" ]; then
-          echo "Test group ${{ matrix.test_group }} failed"
+        FAILED_COUNT="${{ steps.collect_artifacts.outputs.failed_count || 0 }}"
+        SKIPPED_COUNT="${{ steps.collect_artifacts.outputs.skipped_count || 0 }}"
+
+        if [ "${{ steps.run_test.outcome }}" == "failure" ] || [ "$FAILED_COUNT" -gt 0 ]; then
+          echo "Test group ${{ matrix.test_group }} failed (Failures: $FAILED_COUNT, Skipped: $SKIPPED_COUNT)"
           exit 1
         fi
 
@@ -279,6 +286,7 @@ jobs:
         OVERALL_SKIPPED=0
         GROUPS_PASSED=0
         GROUPS_FAILED=0
+        FAILED_GROUP_NAMES=""
 
         # Collect all test stats
         declare -A GROUP_STATS
@@ -299,10 +307,11 @@ jobs:
             OVERALL_FAILED=$((OVERALL_FAILED + failed))
             OVERALL_SKIPPED=$((OVERALL_SKIPPED + skipped))
 
-            if [ "$result" == "success" ]; then
+            if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
               GROUPS_PASSED=$((GROUPS_PASSED + 1))
             else
               GROUPS_FAILED=$((GROUPS_FAILED + 1))
+              FAILED_GROUP_NAMES="${FAILED_GROUP_NAMES}${group} "
             fi
           fi
         done
@@ -330,7 +339,7 @@ jobs:
 
         for group in $(echo "${!GROUP_STATS[@]}" | tr ' ' '\n' | sort); do
           IFS=',' read -r result total passed failed skipped <<< "${GROUP_STATS[$group]}"
-          if [ "$result" == "success" ]; then
+          if [ "$result" == "success" ] && [ "$failed" -eq 0 ]; then
             status="✅ PASS"
           else
             status="❌ FAIL"
@@ -342,6 +351,6 @@ jobs:
 
         # Check if any group failed
         if [ $GROUPS_FAILED -gt 0 ]; then
-          echo "::error::${GROUPS_FAILED} test group(s) failed"
+          echo "::error::${GROUPS_FAILED} test group(s) failed: ${FAILED_GROUP_NAMES}"
           exit 1
         fi

automation/pom.xml

@@ -163,7 +163,7 @@
         <dependency>
             <groupId>org.testng</groupId>
             <artifactId>testng</artifactId>
-            <version>6.8.7</version>
+            <version>6.14.3</version>
         </dependency>
 
         <dependency>

automation/src/test/java/org/greenplum/pxf/automation/features/hive/HiveBaseTest.java

@@ -384,9 +384,7 @@ void loadDataIntoHive(Hdfs hdfs, Hive hive, String fileName, HiveTable tableName
         hdfs.copyFromLocal(localPath, hdfsPath);
 
         // Verify file was copied to HDFS
-        if (!hdfs.doesFileExist(hdfsPath)) {
-            throw new RuntimeException("File was not copied to HDFS: " + hdfsPath);
-        }
+        hdfs.waitForFile(hdfsPath, 3);
 
         // load to hive table
         hive.loadData(tableName, hdfsPath, false);

concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh

@@ -555,6 +555,7 @@ gpdb_test() {
 save_test_reports() {
   local group="$1"
   local surefire_dir="${REPO_ROOT}/automation/target/surefire-reports"
+  local logs_dir="${REPO_ROOT}/automation/automation_logs"
   local artifacts_dir="${REPO_ROOT}/automation/test_artifacts"
   local group_dir="${artifacts_dir}/${group}"
 
@@ -566,6 +567,13 @@ save_test_reports() {
   else
     echo "[run_tests] No surefire reports found for $group"
   fi
+
+  if [ -d "$logs_dir" ] && [ "$(ls -A "$logs_dir" 2>/dev/null)" ]; then
+    echo "[run_tests] Saving $group test logs to $group_dir"
+    cp -r "$logs_dir" "$group_dir/" 2>/dev/null || true
+  else
+    echo "[run_tests] No automation logs found for $group"
+  fi
 }
 
 # Generate test summary from surefire reports