Fix: Add load/performance/extension test groups and align env/test expectations

Mulily0513 · Mulily0513 · commit 17c8ad68f000 · 2025-12-29T16:53:38.000+08:00
- add bench_test and pxf_extension_test in run_tests.sh, plus matrix entries for bench and pxf_extension in CI
- bump surefire heap to 4G to avoid OOM
- update gpupgrade expected outputs to new PXF_HOME paths and JSON formatter error text
- make ProtocolUtils/HiveBaseTest/JdbcHiveTest/OrcWriteTest/ParquetWriteTest more robust to env defaults (protocol, creds, hive JDBC URL)
- keep MultiServerTest running under HDFS with a safe working directory fallback
- set distribution key and INSERT pattern for performance test data load
diff --git a/.github/workflows/pxf-ci.yml b/.github/workflows/pxf-ci.yml
@@ -119,6 +119,8 @@ jobs:
           - s3
           - features
           - gpdb
+          - load
+          - pxf_extension
     steps:
     - name: Free disk space
       run: |
diff --git a/.gitignore b/.gitignore
@@ -16,3 +16,4 @@ server/tmp
 /.vscode/settings.json
 /automation/dataTempFolder/
 /cli/go/pkg/
+/automation/test_artifacts
diff --git a/automation/pom.xml b/automation/pom.xml
@@ -59,7 +59,7 @@
                 <version>2.15</version>
                 <configuration>
                     <testFailureIgnore>true</testFailureIgnore>
-                    <argLine>-Xmx2048m -XX:MaxPermSize=512m</argLine>
+                    <argLine>-Xmx4096m</argLine>
                     <forkCount>1</forkCount>
                     <reuseForks>false</reuseForks>
                 </configuration>
diff --git a/automation/sqlrepo/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/sqlrepo/features/gpupgrade/extension2_0/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans
@@ -40,11 +40,11 @@ FROM pg_catalog.pg_extension AS e
     INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
 WHERE d.deptype = 'e' AND e.extname = 'pxf'
 ORDER BY 1;
-       proname      |            prosrc            |   probin
---------------------+------------------------------+-------------
- pxf_read           | pxfprotocol_import           | $libdir/pxf
- pxf_validate       | pxfprotocol_validate_urls    | $libdir/pxf
- pxf_write          | pxfprotocol_export           | $libdir/pxf
- pxfwritable_export | gpdbwritableformatter_export | $libdir/pxf
- pxfwritable_import | gpdbwritableformatter_import | $libdir/pxf
+       proname      |            prosrc            |             probin
+--------------------+------------------------------+----------------------------------
+ pxf_read           | pxfprotocol_import           | $PXF_HOME/gpextable/pxf
+ pxf_validate       | pxfprotocol_validate_urls    | $PXF_HOME/gpextable/pxf
+ pxf_write          | pxfprotocol_export           | $PXF_HOME/gpextable/pxf
+ pxfwritable_export | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf
+ pxfwritable_import | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf
 (5 rows)
diff --git a/automation/sqlrepo/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans b/automation/sqlrepo/features/gpupgrade/extension2_1/step_1_before_running_pxf_pre_gpupgrade/expected/query01.ans
@@ -40,13 +40,13 @@ FROM pg_catalog.pg_extension AS e
     INNER JOIN pg_catalog.pg_proc AS p ON (p.oid = d.objid)
 WHERE d.deptype = 'e' AND e.extname = 'pxf'
 ORDER BY 1;
-       proname       |            prosrc            |   probin
----------------------+------------------------------+-------------
- pxf_read            | pxfprotocol_import           | $libdir/pxf
- pxf_validate        | pxfprotocol_validate_urls    | $libdir/pxf
- pxf_write           | pxfprotocol_export           | $libdir/pxf
- pxfdelimited_import | pxfdelimited_import          | $libdir/pxf
- pxfwritable_export  | gpdbwritableformatter_export | $libdir/pxf
- pxfwritable_import  | gpdbwritableformatter_import | $libdir/pxf
+       proname       |            prosrc            |             probin
+---------------------+------------------------------+----------------------------------
+ pxf_read            | pxfprotocol_import           | $PXF_HOME/gpextable/pxf
+ pxf_validate        | pxfprotocol_validate_urls    | $PXF_HOME/gpextable/pxf
+ pxf_write           | pxfprotocol_export           | $PXF_HOME/gpextable/pxf
+ pxfdelimited_import | pxfdelimited_import          | $PXF_HOME/gpextable/pxf
+ pxfwritable_export  | gpdbwritableformatter_export | $PXF_HOME/gpextable/pxf
+ pxfwritable_import  | gpdbwritableformatter_import | $PXF_HOME/gpextable/pxf
 (6 rows)
 
diff --git a/automation/sqlrepo/features/hdfs/writable/json/invalid_encoding/expected/query01.ans b/automation/sqlrepo/features/hdfs/writable/json/invalid_encoding/expected/query01.ans
@@ -7,4 +7,4 @@
 -- end_matchsubs
 
 INSERT INTO pxf_invalid_encoding_json_write SELECT * from gpdb_primitive_types;
-ERROR:  gpdbwritable formatter can only export UTF8 formatted data. Define the external table with ENCODING UTF8
+ERROR:  pxfwritable_export formatter can only export UTF8 formatted data. Define the external table with ENCODING UTF8
diff --git a/automation/sqlrepo/features/jdbc/session_params/expected/query01.ans b/automation/sqlrepo/features/jdbc/session_params/expected/query01.ans
@@ -5,7 +5,7 @@
 SELECT * FROM pxf_jdbc_read_view_no_params WHERE name='client_min_messages' OR name='default_statistics_target' ORDER BY name;
            name            | setting
 ---------------------------+---------
- client_min_messages       | error
+ client_min_messages       | notice
  default_statistics_target | 100
 (2 rows)
 
diff --git a/automation/src/main/java/org/greenplum/pxf/automation/utils/system/ProtocolUtils.java b/automation/src/main/java/org/greenplum/pxf/automation/utils/system/ProtocolUtils.java
@@ -14,7 +14,14 @@ public static ProtocolEnum getProtocol() {
 
         ProtocolEnum result;
         try {
-            result = ProtocolEnum.valueOf(System.getProperty(PROTOCOL_KEY, ProtocolEnum.HDFS.name()).toUpperCase());
+            String protocol = System.getProperty(PROTOCOL_KEY);
+            if (protocol == null) {
+                protocol = System.getenv(PROTOCOL_KEY);
+            }
+            if (protocol == null) {
+                protocol = ProtocolEnum.HDFS.name();
+            }
+            result = ProtocolEnum.valueOf(protocol.toUpperCase());
         } catch (Exception e) {
             result = ProtocolEnum.HDFS; // use HDFS as default mode
         }
@@ -23,15 +30,19 @@ public static ProtocolEnum getProtocol() {
     }
 
     public static String getSecret() {
-        return System.getProperty(AWS_SECRET_ACCESS_KEY);
+        String secret = System.getProperty(AWS_SECRET_ACCESS_KEY);
+        return secret != null ? secret : System.getenv(AWS_SECRET_ACCESS_KEY);
     }
 
     public static String getAccess() {
-        return System.getProperty(AWS_ACCESS_KEY_ID);
+        String access = System.getProperty(AWS_ACCESS_KEY_ID);
+        String result = access != null ? access : System.getenv(AWS_ACCESS_KEY_ID);
+        return result;
     }
 
     public static String getPxfTestKeepData() {
-        return System.getProperty(PXF_TEST_KEEP_DATA, "false");
+        String keepData = System.getProperty(PXF_TEST_KEEP_DATA);
+        return keepData != null ? keepData : System.getenv().getOrDefault(PXF_TEST_KEEP_DATA, "false");
     }
 
 
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/jdbc/JdbcHiveTest.java
@@ -166,7 +166,7 @@ protected void prepareData(Hive hive, Hdfs hdfs, String hiveTypesFileName) throw
     }
 
     protected void createTables(Hive hive, String serverName, String gpdbTypesTableName, String gpdbQueryTableName) throws Exception {
-        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default;auth=noSasl";
+        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default";
         String user = null;
 
         // On kerberized cluster, enabled then we need the hive/hiveserver2_hostname principal in the connection string.
@@ -219,7 +219,7 @@ protected void createTablesForWriteTest(Hive hive, String hiverServerName, Strin
             hiveReadable = TableFactory.getPxfJdbcReadableTable(
                     hiveReadableName, GPDB_WRITE_TYPES_TABLE_FIELDS, targetHiveTable.getFullName(), serverName);
         } else {
-            String jdbcUrl = String.format("%s%s:10000/default;auth=noSasl", HIVE_JDBC_URL_PREFIX, hive.getHost());
+            String jdbcUrl = String.format("%s%s:10000/default", HIVE_JDBC_URL_PREFIX, hive.getHost());
             // create GPDB external table for writing data from GPDB to Hive with JDBC profile
             hiveWritable = TableFactory.getPxfJdbcWritableTable(
                     hiveWritableName, GPDB_WRITE_TYPES_TABLE_FIELDS, targetHiveTable.getFullName(),
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/multiserver/MultiServerTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/multiserver/MultiServerTest.java
@@ -50,9 +50,6 @@ public class MultiServerTest extends BaseFeature {
      */
     @Override
     public void beforeClass() throws Exception {
-        if (ProtocolUtils.getProtocol() == ProtocolEnum.HDFS) {
-            return;
-        }
         // Initialize an additional HDFS system object (optional system object)
         hdfs2 = (Hdfs) systemManager.
                 getSystemObject("/sut", "hdfs2", -1, null, false, null, SutFactory.getInstance().getSutInstance());
@@ -71,6 +68,10 @@ public void beforeClass() throws Exception {
         }
 
         String hdfsWorkingDirectory = hdfs.getWorkingDirectory();
+        if (hdfsWorkingDirectory == null) {
+            // Fallback to the default automation working directory to avoid NPE when protocol is HDFS
+            hdfsWorkingDirectory = "/tmp/pxf_automation_data";
+        }
         defaultPath = hdfsWorkingDirectory + "/" + fileName;
 
         // Initialize server objects
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/orc/OrcWriteTest.java
@@ -207,7 +207,7 @@ public void orcWritePrimitivesReadWithHive() throws Exception {
         hive.runQuery(ctasHiveQuery);
 
         // use the Hive JDBC profile to avoid using the PXF ORC reader implementation
-        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default;auth=noSasl";
+        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default";
         ExternalTable exHiveJdbcTable = TableFactory.getPxfJdbcReadableTable(
                 gpdbTableNamePrefix + "_readable", ORC_PRIMITIVE_TABLE_COLUMNS_READ_FROM_HIVE,
                 hiveTable.getName() + "_ctas", HIVE_JDBC_DRIVER_CLASS, jdbcUrl, null);
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java b/automation/src/test/java/org/greenplum/pxf/automation/features/parquet/ParquetWriteTest.java
@@ -369,7 +369,7 @@ public void parquetWriteListsReadWithHive() throws Exception {
         }
 
         // use the Hive JDBC profile to avoid using the PXF Parquet reader implementation
-        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default;auth=noSasl";
+        String jdbcUrl = HIVE_JDBC_URL_PREFIX + hive.getHost() + ":10000/default";
 
         ExternalTable exHiveJdbcTable = TableFactory.getPxfJdbcReadableTable(
                 readTableName, PARQUET_PRIMITIVE_ARRAYS_TABLE_COLUMNS_READ_FROM_HIVE,
diff --git a/automation/src/test/java/org/greenplum/pxf/automation/performance/PerformanceTest.java b/automation/src/test/java/org/greenplum/pxf/automation/performance/PerformanceTest.java
@@ -170,9 +170,11 @@ private void prepareRcData() throws Exception {
 
     private void prepareNativeGpdbData() throws Exception {
         gpdbNativeTable = new Table("perf_test", getColumnTypeGpdb());
+        gpdbNativeTable.setDistributionFields(new String[] { "int0" });
         gpdb.createTableAndVerify(gpdbNativeTable);
 
-        gpdb.insertData(gpdbTextHiveProfile, gpdbNativeTable);
+        gpdb.runQuery(String.format("INSERT INTO %s SELECT * FROM %s",
+                gpdbNativeTable.getName(), gpdbTextHiveProfile.getName()));
     }
 
     @Override
diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/entrypoint.sh
@@ -389,10 +389,15 @@ start_hive_services() {
   export PATH="${JAVA_HOME}/bin:${HIVE_ROOT}/bin:${HADOOP_ROOT}/bin:${PATH}"
   export HIVE_HOME="${HIVE_ROOT}"
   export HADOOP_HOME="${HADOOP_ROOT}"
+  local tez_root="${TEZ_ROOT:-${GPHD_ROOT}/tez}"
   # bump HS2 heap to reduce Tez OOMs during tests
   export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-1024}
   export HADOOP_CLIENT_OPTS="-Xmx${HADOOP_HEAPSIZE}m -Xms512m ${HADOOP_CLIENT_OPTS:-}"
 
+  # ensure Tez libs are available on HDFS for hive.execution.engine=tez
+  "${HADOOP_ROOT}/bin/hadoop" fs -mkdir -p /apps/tez
+  "${HADOOP_ROOT}/bin/hadoop" fs -copyFromLocal -f "${tez_root}"/* /apps/tez
+
   # ensure clean state
   pkill -f HiveServer2 || true
   pkill -f HiveMetaStore || true
diff --git a/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh b/concourse/docker/pxf-cbdb-dev/ubuntu/script/run_tests.sh
@@ -538,10 +538,6 @@ feature_test(){
   make GROUP="features" || true
   save_test_reports "features"
   echo "[run_tests] GROUP=features finished"
-
-  make GROUP="gpdb" || true
-  save_test_reports "gpdb"
-  echo "[run_tests] GROUP=gpdb finished"
 }
 
 gpdb_test() {
@@ -551,6 +547,81 @@ gpdb_test() {
   echo "[run_tests] GROUP=gpdb finished"
 }
 
+pxf_extension_test(){
+  local sudo_cmd=""
+  if [ "$(id -u)" -ne 0 ]; then
+    sudo_cmd="sudo -n"
+  fi
+  local extension_dir="${GPHOME}/share/postgresql/extension"
+  local pxf_fdw_control="${extension_dir}/pxf_fdw.control"
+  if [ -d "${REPO_ROOT}/fdw" ] && [ -d "${extension_dir}" ]; then
+    for sql in pxf_fdw--2.0.sql pxf_fdw--1.0--2.0.sql pxf_fdw--2.0--1.0.sql; do
+      if [ -f "${REPO_ROOT}/fdw/${sql}" ]; then
+        ${sudo_cmd} cp -f "${REPO_ROOT}/fdw/${sql}" "${extension_dir}/${sql}"
+      fi
+    done
+  fi
+
+  set_pxf_fdw_default_version() {
+    local version="$1"
+    if [ -f "${pxf_fdw_control}" ]; then
+      ${sudo_cmd} sed -i "s/^default_version = '.*'/default_version = '${version}'/" "${pxf_fdw_control}"
+    fi
+  }
+
+  set_pxf_fdw_default_version "2.0"
+  make GROUP="pxfExtensionVersion2" || true
+  save_test_reports "pxfExtensionVersion2"
+  make GROUP="pxfExtensionVersion2_1" || true
+  save_test_reports "pxfExtensionVersion2_1"
+
+  set_pxf_fdw_default_version "1.0"
+  make GROUP="pxfFdwExtensionVersion1" || true
+  save_test_reports "pxfFdwExtensionVersion1"
+
+  set_pxf_fdw_default_version "2.0"
+  make GROUP="pxfFdwExtensionVersion2" || true
+  save_test_reports "pxfFdwExtensionVersion2"
+}
+
+bench_prepare_env() {
+  export HADOOP_HEAPSIZE=${HADOOP_HEAPSIZE:-2048}
+  export JAVA_HOME="${JAVA_HADOOP}"
+  export PATH="${JAVA_HOME}/bin:${HADOOP_HOME}/bin:${PATH}"
+
+  hdfs dfs -rm -r -f /tmp/pxf_automation_data /gpdb-ud-scratch/tmp/pxf_automation_data >/dev/null 2>&1 || true
+  for scratch in /tmp/pxf_automation_data /gpdb-ud-scratch/tmp/pxf_automation_data; do
+    hdfs dfs -mkdir -p "${scratch}" >/dev/null 2>&1 || true
+    hdfs dfs -chmod -R 775 "$(dirname "${scratch}")" >/dev/null 2>&1 || true
+  done
+  hdfs dfs -mkdir -p /tmp/hive >/dev/null 2>&1 || true
+  hdfs dfs -chmod -R 777 /tmp/hive >/dev/null 2>&1 || true
+
+  export PROTOCOL=
+  export PXF_TEST_KEEP_DATA=${PXF_TEST_KEEP_DATA:-true}
+
+  ensure_hive_ready
+}
+
+load_test(){
+  bench_prepare_env
+  make GROUP="load" || true
+  save_test_reports "load"
+  echo "[run_tests] GROUP=load finished"
+}
+
+performance_test(){
+  bench_prepare_env
+  make GROUP="performance" || true
+  save_test_reports "performance"
+  echo "[run_tests] GROUP=performance finished"
+}
+
+bench_test(){
+  load_test
+  performance_test
+}
+
 # Save test reports for a specific group to avoid overwriting
 save_test_reports() {
   local group="$1"
@@ -599,7 +670,7 @@ generate_test_summary() {
 
     local group=$(basename "$group_dir")
     # Skip if it's not a test group directory
-    [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features)$ ]] || continue
+    [[ "$group" =~ ^(smoke|hcatalog|hcfs|hdfs|hive|gpdb|sanity|hbase|profile|jdbc|proxy|unused|s3|features|load|performance|pxfExtensionVersion2|pxfExtensionVersion2_1|pxfFdwExtensionVersion1|pxfFdwExtensionVersion2)$ ]] || continue
 
     echo "Processing $group test reports from $group_dir"
 
@@ -759,16 +830,22 @@ run_single_group() {
       make GROUP="s3"
       save_test_reports "s3"
       ;;
-    features|gpdb)
-      ensure_gpupgrade_helpers
-      ensure_testplugin_jar
-      ensure_minio_bucket
-      ensure_hadoop_s3a_config
-      configure_pxf_s3_server
-      configure_pxf_default_hdfs_server
-      export PROTOCOL=
-      make GROUP="$group"
-      save_test_reports "$group"
+    features)
+      feature_test
+      ;;
+    gpdb)
+      gpdb_test
+      ;;
+    pxf_extension)
+      pxf_extension_test
+      ;;
+    load)
+      bench_prepare_env
+      load_test
+      ;;
+    performance)
+      bench_prepare_env
+      performance_test
       ;;
     proxy)
       export PROTOCOL=
@@ -782,7 +859,7 @@ run_single_group() {
       ;;
     *)
       echo "Unknown test group: $group"
-      echo "Available groups: cli, external-table, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features, gpdb"
+      echo "Available groups: cli, external-table, server, sanity, smoke, hdfs, hcatalog, hcfs, hive, hbase, profile, jdbc, proxy, unused, s3, features, gpdb, load, performance, bench, pxf_extension"
       exit 1
       ;;
   esac
@@ -809,6 +886,9 @@ main() {
     # Run feature tests (includes features, gpdb)
     feature_test
 
+    # Run bench tests (includes load, performance)
+    bench_test
+
     echo "[run_tests] All test groups completed, generating summary..."
 
     # Generate test summary and return appropriate exit code
