WIP - 2250 - Tx compressor

.github/workflows/prover-native-lib-blob-compressor-release.yml

@@ -46,12 +46,15 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_x86_64.so ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_COMPRESSOR}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_DECOMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_TX_COMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4
@@ -81,12 +84,15 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_arm64.so ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_COMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_DECOMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_TX_COMPRESSOR}
       - name: Cache built binaries
         uses: actions/upload-artifact@v4
         with:
@@ -113,6 +119,8 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
@@ -122,6 +130,8 @@ jobs:
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_COMPRESSOR}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_DECOMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_DECOMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_TX_COMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_TX_COMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4

jvm-libs/linea/tx-compressor/build.gradle

@@ -0,0 +1,83 @@
+plugins {
+  id 'net.consensys.zkevm.kotlin-common-minimal-conventions'
+  id 'net.consensys.zkevm.linea-native-libs-helper'
+}
+
+description = 'Java JNA wrapper for Linea Transaction Compressor Library implemented in GO Lang'
+
+dependencies {
+  compileOnly "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  compileOnly project(":jvm-libs:generic:extensions:kotlin")
+  compileOnly "org.apache.logging.log4j:log4j-api:${libs.versions.log4j.get()}"
+  compileOnly "org.apache.logging.log4j:log4j-core:${libs.versions.log4j.get()}"
+
+  testImplementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  testImplementation project(":jvm-libs:generic:extensions:kotlin")
+  testImplementation project(":jvm-libs:linea:blob-compressor")
+  testImplementation testFixtures(project(":jvm-libs:linea:blob-compressor"))
+  testImplementation project(':jvm-libs:linea:besu-libs')
+  testImplementation project(':jvm-libs:linea:besu-rlp-and-mappers')
+}
+
+jar {
+  dependsOn configurations.runtimeClasspath
+}
+
+test {
+  // we cannot have more 1 compressor per JVM, hence we disable parallel execution
+  // because multiple threads would cause issues with the native library
+  systemProperties["junit.jupiter.execution.parallel.enabled"] = false
+  maxParallelForks = 1
+}
+
+def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get().absolutePath
+
+task downloadNativeLibs {
+  doLast {
+    // TODO: Update URL when tx-compressor native libs are released
+    // fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-vX.Y.Z/linea-blob-libs-vX.Y.Z.zip", "tx_compressor", libsZipDownloadOutputDir)
+  }
+}
+
+// Task to copy locally compiled native library for testing
+// Usage: ./gradlew :jvm-libs:linea:tx-compressor:copyLocalNativeLib
+// Requires: make lib/txcompressor in prover directory first
+task copyLocalNativeLib {
+  doLast {
+    def osName = System.getProperty("os.name").toLowerCase()
+    def osArch = System.getProperty("os.arch").toLowerCase()
+
+    def resourceDir
+    def libExtension
+    if (osName.contains("mac") || osName.contains("darwin")) {
+      libExtension = "dylib"
+      resourceDir = osArch.contains("aarch64") || osArch.contains("arm64") ? "darwin-aarch64" : "darwin-x86-64"
+    } else if (osName.contains("linux")) {
+      libExtension = "so"
+      resourceDir = osArch.contains("aarch64") || osArch.contains("arm64") ? "linux-aarch64" : "linux-x86-64"
+    } else {
+      throw new GradleException("Unsupported OS: ${osName}")
+    }
+
+    def srcFile = rootProject.file("prover/lib/compressor/libtxcompressor/build/libtx_compressor_native_jna")
+    def destDir = file("src/main/resources/${resourceDir}")
+    def destFile = new File(destDir, "libtx_compressor_jna_v1.0.0.${libExtension}")
+
+    if (!srcFile.exists()) {
+      throw new GradleException("Native library not found at ${srcFile}. Run 'make lib/txcompressor' in prover directory first.")
+    }
+
+    destDir.mkdirs()
+    copy {
+      from srcFile
+      into destDir
+      rename { "libtx_compressor_jna_v1.0.0.${libExtension}" }
+    }
+    println("Copied native library to ${destFile}")
+  }
+}
+
+compileKotlin {
+  dependsOn tasks.downloadNativeLibs
+  dependsOn tasks.copyLocalNativeLib
+}

jvm-libs/linea/tx-compressor/src/main/kotlin/linea/blob/GoNativeTxCompressor.kt

@@ -0,0 +1,123 @@
+package linea.blob
+
+import com.sun.jna.Library
+import com.sun.jna.Native
+import linea.jvm.ResourcesUtil.copyResourceToTmpDir
+
+/**
+ * JNA interface for the Go native transaction compressor library.
+ *
+ * This compressor operates at the transaction level rather than the block level,
+ * maintaining compression context across transactions for better compression ratios.
+ * It is designed for sequencer block building where transactions are added one by one
+ * until the compressed size threshold is reached.
+ */
+interface GoNativeTxCompressor {
+
+  /**
+   * TxInit initializes the transaction compressor.
+   *
+   * @param dataLimit Maximum size of compressed data in bytes. The caller should
+   *                  account for blob overhead (~100 bytes) when setting this limit.
+   * @param dictPath Path to the compression dictionary
+   * @return true if the compressor was successfully initialized, false otherwise
+   */
+  fun TxInit(dataLimit: Int, dictPath: String): Boolean
+
+  /**
+   * TxReset resets the compressor to its initial state.
+   * Must be called between each block being built.
+   */
+  fun TxReset()
+
+  /**
+   * TxWrite appends an RLP-encoded transaction to the compressed data.
+   *
+   * @param data bytes of the RLP encoded transaction
+   * @param data_len number of bytes
+   * @return true if the transaction was appended, false if it would exceed the limit
+   *         or if an error occurred (check TxError() for details)
+   */
+  fun TxWrite(data: ByteArray, data_len: Int): Boolean
+
+  /**
+   * TxCanWrite checks if an RLP-encoded transaction can be appended without actually appending it.
+   *
+   * @param data bytes of the RLP encoded transaction
+   * @param data_len number of bytes
+   * @return true if the transaction could be appended, false otherwise
+   */
+  fun TxCanWrite(data: ByteArray, data_len: Int): Boolean
+
+  /**
+   * TxLen returns the current length of the compressed data.
+   *
+   * @return number of bytes of compressed data
+   */
+  fun TxLen(): Int
+
+  /**
+   * TxWritten returns the number of uncompressed bytes written to the compressor.
+   *
+   * @return number of uncompressed bytes written
+   */
+  fun TxWritten(): Int
+
+  /**
+   * TxBytes fills out with the compressed data.
+   * The caller must allocate out and ensure that len(out) == TxLen()
+   *
+   * @param out The ByteArray to be filled with compressed data
+   */
+  fun TxBytes(out: ByteArray)
+
+  /**
+   * TxError returns the last error message.
+   * Should be checked if TxWrite or TxCanWrite returns false.
+   *
+   * @return error message string, or null if no error
+   */
+  fun TxError(): String?
+}
+
+interface GoNativeTxCompressorJnaLib : GoNativeTxCompressor, Library
+
+enum class TxCompressorVersion(val version: String) {
+  V1("v1.0.0"),
+}
+
+class GoNativeTxCompressorFactory {
+  companion object {
+    private const val DICTIONARY_NAME = "compressor-dictionaries/v2025-04-21.bin"
+    val dictionaryPath =
+      copyResourceToTmpDir(DICTIONARY_NAME, GoNativeTxCompressorFactory::class.java.classLoader)
+
+    private fun getLibFileName(version: String) = "tx_compressor_jna_$version"
+
+    @JvmStatic
+    private val loadedVersions = mutableMapOf<TxCompressorVersion, GoNativeTxCompressor>()
+
+    @JvmStatic
+    fun getInstance(version: TxCompressorVersion): GoNativeTxCompressor {
+      synchronized(loadedVersions) {
+        return loadedVersions[version]
+          ?: loadLib(version)
+            .also { loadedVersions[version] = it }
+      }
+    }
+
+    private fun loadLib(version: TxCompressorVersion): GoNativeTxCompressor {
+      val extractedLibFile = Native.extractFromResourcePath(
+        getLibFileName(version.version),
+        GoNativeTxCompressorFactory::class.java.classLoader,
+      )
+
+      return Native.load(
+        /* name = */
+        extractedLibFile.toString(),
+        /* interfaceClass = */
+        GoNativeTxCompressorJnaLib::class.java,
+      )
+    }
+  }
+}