feat(java): use JNI instead of JNA (#2773)

Switch from JNA -> JNI. On some microbenchmarks I've seen this result in
a ~3x speedup for simple string-passing.

I wired this into the Iceberg fork for Vortex and am seeing an immediate
~40% speedup on Citibike scan queries

Subsequently #2781 gives us another 2x speedup on Citibike.

Author: a10y

Cargo.lock

@@ -18,6 +18,7 @@ members = [
     "vortex-flatbuffers",
     "vortex-io",
     "vortex-ipc",
+    "vortex-jni",
     "vortex-layout",
     "vortex-mask",
     "vortex-metrics",

Cargo.toml

@@ -57,8 +57,6 @@ jakarta.validation:jakarta.validation-api:2.0.2 (1 constraints: fd10b6c3)
 jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 (5 constraints: 8c647f61)
 javax.activation:activation:1.1.1 (1 constraints: 150dc536)
 joda-time:joda-time:2.12.5 (1 constraints: 5f0ca305)
-net.java.dev.jna:jna:5.17.0 (1 constraints: c50c2618)
-net.java.dev.jna:jna-platform:5.17.0 (1 constraints: 3f05563b)
 net.razorvine:pickle:1.3 (1 constraints: b80cce1c)
 net.sf.py4j:py4j:0.10.9.7 (1 constraints: b10d385f)
 org.antlr:antlr4-runtime:4.9.3 (1 constraints: 300ea766)

java/versions.lock

@@ -3,7 +3,6 @@ com.google.guava:failureaccess = 1.0.3
 com.google.guava:guava = 33.4.5-jre
 com.google.guava:listenablefuture = 9999.0-empty-to-avoid-conflict-with-guava
 com.jakewharton.nopen:* = 1.0.1
-net.java.dev.jna:* = 5.17.0
 org.apache.spark:* = 3.5.5
 org.immutables:value = 2.10.1
 com.google.protobuf:protobuf-java = 4.30.1

java/versions.props

@@ -9,16 +9,14 @@ plugins {
 }
 
 dependencies {
-    api("net.java.dev.jna:jna-platform")
-    api("com.google.protobuf:protobuf-java")
-
     compileOnly("org.immutables:value")
     annotationProcessor("org.immutables:value")
 
     errorprone("com.google.errorprone:error_prone_core")
     errorprone("com.jakewharton.nopen:nopen-checker")
 
     implementation("com.google.guava:guava")
+    implementation("com.google.protobuf:protobuf-java")
     compileOnly("com.google.errorprone:error_prone_annotations")
     compileOnly("com.jakewharton.nopen:nopen-annotations")
 }
@@ -48,16 +46,64 @@ tasks.build {
     dependsOn("shadowJar")
 }
 
+tasks.register("generateJniHeaders") {
+    description = "Generates JNI header files for Java classes with native methods"
+    group = "build"
+
+    // Define input and output properties
+    val jniClasses =
+        fileTree("src/main/java") {
+            // Adjust this include pattern to match only files that need JNI headers
+            include("**/JNI*.java")
+        }
+
+    inputs.files(jniClasses)
+    outputs.dir("$buildDir/generated/jni")
+
+    doLast {
+        // Create output directory if it doesn't exist
+        val headerDir = file("$buildDir/generated/jni")
+        headerDir.mkdirs()
+
+        val classesDir =
+            sourceSets["main"]
+                .java.destinationDirectory
+                .get()
+                .asFile
+
+        // Compile only the selected files with -h option
+        ant.withGroovyBuilder {
+            "javac"(
+                "classpath" to sourceSets["main"].compileClasspath.asPath,
+                "srcdir" to "src/main/java",
+                "includes" to jniClasses.includes.joinToString(","),
+                "destdir" to classesDir,
+                "includeantruntime" to false,
+                "debug" to true,
+                "source" to java.sourceCompatibility,
+                "target" to java.targetCompatibility,
+            ) {
+                "compilerarg"("line" to "-h ${headerDir.absolutePath}")
+            }
+        }
+
+        println("JNI headers generated in ${headerDir.absolutePath}")
+    }
+
+    // Make this task run after the compileJava task
+    dependsOn("compileJava")
+}
+
 publishing {
     publications {
         create<MavenPublication>("mavenJava") {
-            from(components["java"]) // Publishes the compiled JAR
+            artifact(tasks.shadowJar.get())
             artifactId = "vortex-jni"
         }
     }
 }
 
-val vortexFFI = projectDir.parentFile.parentFile.resolve("vortex-ffi")
+val vortexJNI = projectDir.parentFile.parentFile.resolve("vortex-jni")
 
 val platformLibSuffix =
     if (System.getProperty("os.name").contains("Mac")) {
@@ -67,15 +113,15 @@ val platformLibSuffix =
     }
 
 val targetDir = projectDir.parentFile.parentFile.resolve("target")
-val libraryFile = targetDir.resolve("release/libvortex_ffi.$platformLibSuffix")
+val libraryFile = targetDir.resolve("release/libvortex_jni.$platformLibSuffix")
 
 val cargoCheck by tasks.registering(Exec::class) {
-    workingDir = vortexFFI
+    workingDir = vortexJNI
     commandLine("cargo", "check")
 }
 
 val cargoBuild by tasks.registering(Exec::class) {
-    workingDir = vortexFFI
+    workingDir = vortexJNI
     commandLine(
         "cargo",
         "build",
@@ -90,7 +136,7 @@ val cargoBuild by tasks.registering(Exec::class) {
 }
 
 val cargoClean by tasks.registering(Exec::class) {
-    workingDir = vortexFFI
+    workingDir = vortexJNI
     commandLine("cargo", "clean")
 }
 
@@ -123,7 +169,7 @@ val copySharedLibrary by tasks.registering(Copy::class) {
     dependsOn(cargoBuild)
 
     from(libraryFile)
-    into(projectDir.resolve("src/main/resources/$resourceDir"))
+    into(projectDir.resolve("src/main/resources/native/$resourceDir"))
 
     doLast {
         println("Copied $libraryFile into resource directory")
@@ -133,3 +179,8 @@ val copySharedLibrary by tasks.registering(Copy::class) {
 tasks.withType<ProcessResources>().configureEach {
     dependsOn(copySharedLibrary)
 }
+
+// Remove the JAR task, replace it with shadowJar
+tasks.named("jar").configure {
+    enabled = false
+}