feat: publish vortex-spark-all assembly JAR (#5435)

To allow using Vortex as a read/write datasource with Spark, we publish
an assembly JAR that shades all runtime dependencies that may conflict
with Spark.

<img width="1840" height="1195" alt="image"
src="https://github.com/user-attachments/assets/f65f8c32-fca6-41cd-b54f-965b8149bc97"
/>

Unfortunately it's a bit hard to test publishing, I've just modeled this
on the existing setup from vortex-jni-all JAR

Signed-off-by: Andrew Duffy <[email protected]>

Author: a10y

java/testfiles/Cargo.lock

@@ -1,16 +1,19 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
 apply(plugin = "com.vanniktech.maven.publish")
 
 plugins {
     `java-library`
     `jvm-test-suite`
+    id("com.gradleup.shadow") version "9.2.2"
 }
 
 dependencies {
-    api("org.apache.spark:spark-catalyst_2.12")
-    api("org.apache.spark:spark-sql_2.12")
+    compileOnly("org.apache.spark:spark-catalyst_2.12")
+    compileOnly("org.apache.spark:spark-sql_2.12")
     api(project(":vortex-jni", configuration = "shadow"))
 
     compileOnly("org.immutables:value")
@@ -73,6 +76,22 @@ mavenPublishing {
     }
 }
 
+// shade guava and protobuf dependencies
+tasks.withType<ShadowJar> {
+    relocate("com.google.protobuf", "dev.vortex.relocated.com.google.protobuf")
+    relocate("com.google.common", "dev.vortex.relocated.com.google.common")
+    relocate("org.apache.arrow", "dev.vortex.relocated.org.apache.arrow") {
+        // exclude C Data Interface since JNI cannot be relocated
+        exclude("org.apache.arrow.c.jni.JniWrapper")
+        exclude("org.apache.arrow.c.jni.PrivateData")
+        exclude("org.apache.arrow.c.jni.CDataJniException")
+        // Also used by JNI: https://github.com/apache/arrow/blob/apache-arrow-11.0.0/java/c/src/main/cpp/jni_wrapper.cc#L341
+        // Note this class is not used by us, but required when loading the native lib
+        exclude("org.apache.arrow.c.ArrayStreamExporter\$ExportedArrayStreamPrivateData")
+    }
+}
+
+
 tasks.withType<Test>().all {
     classpath +=
         project(":vortex-jni")
@@ -88,4 +107,8 @@ tasks.withType<Test>().all {
     )
 }
 
+tasks.build {
+    dependsOn("shadowJar")
+}
+
 description = "Apache Spark bindings for reading Vortex file datasets"

java/vortex-spark/build.gradle.kts

@@ -140,6 +140,8 @@ public static DataType toDataType(DType dType) {
 
                 // TODO(aduffy): other extension types
                 throw new IllegalArgumentException("Unsupported non-temporal extension type");
+            case DECIMAL:
+                return DataTypes.createDecimalType(dType.getPrecision(), dType.getScale());
             default:
                 throw new IllegalArgumentException("unreachable");
         }

java/vortex-spark/src/main/java/dev/vortex/spark/SparkTypes.java

@@ -55,7 +55,9 @@ where
             debug_assert_eq!(
                 array.dtype(),
                 this.dtype,
-                "ArrayStreamAdapter received an array with unexpected dtype"
+                "ArrayStreamAdapter expected array with type {}, actual {}",
+                this.dtype,
+                array.dtype(),
             );
         }