[GLUTEN-11088][VL] Fix GlutenDatasetSuite in Spark-4.0 (#11197)

* [VL] Fix GlutenDatasetSuite in Spark-4.0

Author: zhli1142015

backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxBackend.scala

@@ -583,4 +583,7 @@ object VeloxBackendSettings extends BackendSettingsApi {
   override def supportOverwriteByExpression(): Boolean = enableEnhancedFeatures()
 
   override def supportOverwritePartitionsDynamic(): Boolean = enableEnhancedFeatures()
+
+  /** Velox does not support columnar shuffle with empty schema. */
+  override def supportEmptySchemaColumnarShuffle(): Boolean = false
 }

backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxValidatorApi.scala

@@ -88,13 +88,15 @@ class VeloxValidatorApi extends ValidatorApi {
       outputAttributes: Seq[Attribute],
       outputPartitioning: Partitioning,
       child: SparkPlan): Option[String] = {
-    if (outputAttributes.isEmpty) {
-      // See: https://github.com/apache/incubator-gluten/issues/7600.
-      return Some("Shuffle with empty output schema is not supported")
-    }
-    if (child.output.isEmpty) {
-      // See: https://github.com/apache/incubator-gluten/issues/7600.
-      return Some("Shuffle with empty input schema is not supported")
+    if (!BackendsApiManager.getSettings.supportEmptySchemaColumnarShuffle()) {
+      if (outputAttributes.isEmpty) {
+        // See: https://github.com/apache/incubator-gluten/issues/7600.
+        return Some("Shuffle with empty output schema is not supported")
+      }
+      if (child.output.isEmpty) {
+        // See: https://github.com/apache/incubator-gluten/issues/7600.
+        return Some("Shuffle with empty input schema is not supported")
+      }
     }
     doSchemaValidate(child.schema)
   }

gluten-substrait/src/main/scala/org/apache/gluten/backendsapi/BackendSettingsApi.scala

@@ -162,4 +162,7 @@ trait BackendSettingsApi {
   def supportOverwriteByExpression(): Boolean = false
 
   def supportOverwritePartitionsDynamic(): Boolean = false
+
+  /** Whether the backend supports columnar shuffle with empty schema. */
+  def supportEmptySchemaColumnarShuffle(): Boolean = true
 }

gluten-substrait/src/main/scala/org/apache/gluten/extension/columnar/CollectLimitTransformerRule.scala

@@ -30,7 +30,10 @@ case class CollectLimitTransformerRule() extends Rule[SparkPlan] {
     }
 
     val transformed = plan.transformUp {
-      case exec: CollectLimitExec if exec.child.supportsColumnar =>
+      case exec: CollectLimitExec
+          if exec.child.supportsColumnar &&
+            (exec.child.output.nonEmpty ||
+              BackendsApiManager.getSettings.supportEmptySchemaColumnarShuffle()) =>
         val offset = SparkShimLoader.getSparkShims.getCollectLimitOffset(exec)
         BackendsApiManager.getSparkPlanExecApiInstance
           .genColumnarCollectLimitExec(exec.limit, exec.child, offset)

gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala

@@ -839,8 +839,6 @@ class VeloxTestSettings extends BackendTestSettings {
     // Rewrite the following two tests in GlutenDatasetSuite.
     .exclude("dropDuplicates: columns with same column name")
     .exclude("groupBy.as")
-    // TODO: fix in Spark-4.0
-    .exclude("SPARK-23627: provide isEmpty in DataSet")
   enableSuite[GlutenDateFunctionsSuite]
     // The below two are replaced by two modified versions.
     .exclude("unix_timestamp")