[Build] Remove the delta-hudi and delta-contribs spark suffix for artifacts. (#6090)

#### Which Delta project/connector is this regarding?

- [x] Spark
- [ ] Standalone
- [ ] Flink
- [ ] Kernel
- [ ] Other (fill in here)

## Description

Currently, after we publish the `./build/sbt release` artifacts to the
staging service, and it has the following artifacts

```
io/delta/delta-connect-client_2.13/4.1.0/
io/delta/delta-connect-client_4.0_2.13/4.1.0/
io/delta/delta-connect-client_4.1_2.13/4.1.0/
io/delta/delta-connect-common_2.13/4.1.0/
io/delta/delta-connect-common_4.0_2.13/4.1.0/
io/delta/delta-connect-common_4.1_2.13/4.1.0/
io/delta/delta-connect-server_2.13/4.1.0/
io/delta/delta-connect-server_4.0_2.13/4.1.0/
io/delta/delta-connect-server_4.1_2.13/4.1.0/
io/delta/delta-contribs_2.13/4.1.0/
io/delta/delta-contribs_4.0_2.13/4.1.0/
io/delta/delta-contribs_4.1_2.13/4.1.0/
io/delta/delta-hudi_2.13/4.1.0/
io/delta/delta-hudi_4.0_2.13/4.1.0/
io/delta/delta-hudi_4.1_2.13/4.1.0/
io/delta/delta-iceberg_2.13/4.1.0/
io/delta/delta-kernel-api/4.1.0/
io/delta/delta-kernel-defaults/4.1.0/
io/delta/delta-kernel-unitycatalog/4.1.0/
io/delta/delta-sharing-spark_2.13/4.1.0/
io/delta/delta-sharing-spark_4.0_2.13/4.1.0/
io/delta/delta-sharing-spark_4.1_2.13/4.1.0/
io/delta/delta-spark_2.13/4.1.0/
io/delta/delta-spark_4.0_2.13/4.1.0/
io/delta/delta-spark_4.1_2.13/4.1.0/
io/delta/delta-storage/4.1.0/
io/delta/delta-storage-s3-dynamodb/4.1.0/
```

For `delta-hudi`, we should remove the `_4.0` and `_4.1` suffix , since
we don't need the spark suffix for `delta-hudi`.

For `delta-contribs`, it only have two `LogStore` classes inside it, we
should also remove the spark suffix for them.

## How was this patch tested?

Use the `test_cross_spark_publish.py` to test this. 

## Does this PR introduce _any_ user-facing changes?

Yes. remove the spark suffix for `delta-hudi`. then
`delta-spark_4.0_2.13` artifacts won't be available any more.

---------

Signed-off-by: openinx <openinx@gmail.com>

Author: openinx

build.sbt

@@ -662,7 +662,10 @@ lazy val contribs = (project in file("contribs"))
     commonSettings,
     scalaStyleSettings,
     releaseSettings,
-    CrossSparkVersions.sparkDependentModuleName(sparkVersion),
+    // Set sparkVersion directly (not sparkDependentModuleName) so that
+    // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark
+    // suffix to the artifact name. delta-contribs is only published as delta-contribs_2.13.
+    sparkVersion := CrossSparkVersions.getSparkVersion(),
     Compile / packageBin / mappings := (Compile / packageBin / mappings).value ++
       listPythonFiles(baseDirectory.value.getParentFile / "python"),
 
@@ -1315,15 +1318,30 @@ lazy val hudi = (project in file("hudi"))
     commonSettings,
     scalaStyleSettings,
     releaseSettings,
-    CrossSparkVersions.sparkDependentSettings(sparkVersion),
-    libraryDependencies ++= Seq(
-      "org.apache.hudi" % "hudi-java-client" % "0.15.0" % "compile" excludeAll(
-        ExclusionRule(organization = "org.apache.hadoop"),
-        ExclusionRule(organization = "org.apache.zookeeper"),
-      ),
-      "org.apache.spark" %% "spark-avro" % sparkVersion.value % "test" excludeAll ExclusionRule(organization = "org.apache.hadoop"),
-      "org.apache.parquet" % "parquet-avro" % "1.12.3" % "compile"
-    ),
+    // Set sparkVersion directly (not sparkDependentModuleName) so that
+    // runOnlyForReleasableSparkModules discovers this module, but without adding a Spark
+    // suffix to the artifact name. delta-hudi is only published as delta-hudi_2.13.
+    sparkVersion := CrossSparkVersions.getSparkVersion(),
+    libraryDependencies ++= {
+      if (supportHudi) {
+        Seq(
+          "org.apache.hudi" % "hudi-java-client" % "0.15.0" % "compile" excludeAll(
+            ExclusionRule(organization = "org.apache.hadoop"),
+            ExclusionRule(organization = "org.apache.zookeeper"),
+          ),
+          "org.apache.spark" %% "spark-avro" % sparkVersion.value % "test" excludeAll ExclusionRule(organization = "org.apache.hadoop"),
+          "org.apache.parquet" % "parquet-avro" % "1.12.3" % "compile"
+        )
+      } else {
+        Seq.empty
+      }
+    },
+    // Skip compilation and publishing when supportHudi is false
+    Compile / skip := !supportHudi,
+    Test / skip := !supportHudi,
+    publish / skip := !supportHudi,
+    publishLocal / skip := !supportHudi,
+    publishM2 / skip := !supportHudi,
     assembly / assemblyJarName := s"${name.value}-assembly_${scalaBinaryVersion.value}-${version.value}.jar",
     assembly / logLevel := Level.Info,
     assembly / test := {},

project/CrossSparkVersions.scala

@@ -82,7 +82,7 @@ import Unidoc._
  * 1. Spark-Dependent Published Modules:
  *    - Use CrossSparkVersions.sparkDependentSettings(sparkVersion)
  *    - Include releaseSettings (publishable)
- *    - Examples: delta-spark, delta-connect-*, delta-sharing-spark, delta-iceberg
+ *    - Examples: delta-spark, delta-connect-*, delta-sharing-spark, delta-iceberg, delta-hudi, delta-contribs
  *    - These modules get version-specific artifact names for non-default Spark versions
  *    - Automatically included in cross-Spark publishing
  *
@@ -277,7 +277,6 @@ object SparkVersionSpec {
     targetJvm = "17",
     additionalSourceDir = Some("scala-shims/spark-4.1"),
     supportIceberg = false,
-    supportHudi = false,
     antlr4Version = "4.13.1",
     additionalJavaOptions = java17TestSettings,
     jacksonVersion = "2.18.2"
@@ -288,7 +287,6 @@ object SparkVersionSpec {
     targetJvm = "17",
     additionalSourceDir = Some("scala-shims/spark-4.2"),
     supportIceberg = false,
-    supportHudi = false,
     antlr4Version = "4.13.1",
     additionalJavaOptions = java17TestSettings,
     jacksonVersion = "2.18.2",

project/tests/test_cross_spark_publish.py

@@ -35,8 +35,6 @@
     "delta-connect-client{suffix}_2.13-{version}.jar",
     "delta-connect-server{suffix}_2.13-{version}.jar",
     "delta-sharing-spark{suffix}_2.13-{version}.jar",
-    "delta-contribs{suffix}_2.13-{version}.jar",
-    "delta-hudi{suffix}_2.13-{version}.jar",
 ]
 
 # Iceberg-related modules - only built for Spark versions with supportIceberg=true
@@ -45,15 +43,21 @@
     "delta-iceberg_2.13-{version}.jar",
 ]
 
+# Hudi-related modules - only built for Spark versions with supportHudi=true
+# delta-hudi has no Spark suffix (always delta-hudi_2.13)
+DELTA_HUDI_JAR_TEMPLATES = [
+    "delta-hudi_2.13-{version}.jar",
+]
+
 # Non-spark-related modules (built once, same for all Spark versions)
 # Template format: {version} = Delta version (e.g., "3.4.0-SNAPSHOT")
 NON_SPARK_RELATED_JAR_TEMPLATES = [
-    # Java-only modules (no Scala version)
     "delta-storage-{version}.jar",
     "delta-kernel-api-{version}.jar",
     "delta-kernel-defaults-{version}.jar",
     "delta-storage-s3-dynamodb-{version}.jar",
-    "delta-kernel-unitycatalog-{version}.jar"
+    "delta-kernel-unitycatalog-{version}.jar",
+    "delta-contribs_2.13-{version}.jar",
 ]
 
 
@@ -65,6 +69,7 @@ class SparkVersionSpec:
     """
     suffix: str  # e.g., "" for default, "_X.Y" for other versions
     support_iceberg: bool = False  # Whether this Spark version supports iceberg integration
+    support_hudi: bool = True  # Whether this Spark version supports hudi integration
 
     def __post_init__(self):
         """Generate JAR templates with the suffix applied."""
@@ -80,22 +85,28 @@ def __post_init__(self):
         else:
             self.iceberg_jars = []
 
+        # Hudi JARs have no Spark suffix (always delta-hudi_2.13)
+        if self.support_hudi:
+            self.hudi_jars = list(DELTA_HUDI_JAR_TEMPLATES)
+        else:
+            self.hudi_jars = []
+
         # Non-Spark-related JAR templates are the same for all Spark versions
         self.non_spark_related_jars = list(NON_SPARK_RELATED_JAR_TEMPLATES)
 
     @property
     def all_jars(self) -> List[str]:
-        """All JAR templates for this Spark version (Spark-related + non-Spark-related + iceberg if supported)."""
-        return self.spark_related_jars + self.non_spark_related_jars + self.iceberg_jars
+        """All JAR templates for this Spark version."""
+        return self.spark_related_jars + self.non_spark_related_jars + self.iceberg_jars + self.hudi_jars
 
 
 # Spark versions to test (key = full version string, value = spec with suffix)
 # By default, ALL versions get a Spark suffix (e.g., delta-spark_4.0_2.13)
 # skipSparkSuffix=true removes the suffix (used during release for backward compat)
 # These should mirror CrossSparkVersions.scala
 SPARK_VERSIONS: Dict[str, SparkVersionSpec] = {
-    "4.0.1": SparkVersionSpec(suffix="_4.0", support_iceberg=True),
-    "4.1.0": SparkVersionSpec(suffix="_4.1", support_iceberg=False)
+    "4.0.1": SparkVersionSpec(suffix="_4.0", support_iceberg=True, support_hudi=True),
+    "4.1.0": SparkVersionSpec(suffix="_4.1", support_iceberg=False, support_hudi=True)
 }
 
 # The default Spark version
@@ -223,7 +234,7 @@ def test_backward_compat_publish(self) -> bool:
         # Create a spec without suffix for backward compatibility
         # Uses the same iceberg support as the default Spark version
         default_spark_spec = SPARK_VERSIONS[DEFAULT_SPARK]
-        spark_spec_no_suffix = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg)
+        spark_spec_no_suffix = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg, support_hudi=default_spark_spec.support_hudi)
 
         print("\n" + "="*70)
         print(f"TEST: skipSparkSuffix=true (backward compatibility - no suffix)")
@@ -271,12 +282,12 @@ def test_cross_spark_workflow(self) -> bool:
         # Build expected JARs:
         # 1. All modules WITHOUT suffix (from Step 1 - backward compat)
         # 2. Spark-dependent modules WITH suffix for each non-master version (from Step 2)
-        # 3. Iceberg JARs for supported versions (with suffix only)
+        # 3. Iceberg/Hudi JARs for supported versions (no Spark suffix)
         expected = set()
 
         # Step 1: All modules without suffix (uses default Spark version's iceberg support)
         default_spark_spec = SPARK_VERSIONS[DEFAULT_SPARK]
-        no_suffix_spec = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg)
+        no_suffix_spec = SparkVersionSpec(suffix="", support_iceberg=default_spark_spec.support_iceberg, support_hudi=default_spark_spec.support_hudi)
         expected.update(substitute_xversion(no_suffix_spec.all_jars, self.delta_version))
 
         # Step 2: Spark-dependent modules WITH suffix for each non-master version
@@ -286,6 +297,7 @@ def test_cross_spark_workflow(self) -> bool:
 
             expected.update(substitute_xversion(spark_spec.spark_related_jars, self.delta_version))
             expected.update(substitute_xversion(spark_spec.iceberg_jars, self.delta_version))
+            expected.update(substitute_xversion(spark_spec.hudi_jars, self.delta_version))
 
         return self.validate_jars(expected, "Cross-Spark Workflow")