WIP - 2250 - Tx compressor

.github/workflows/prover-native-lib-blob-compressor-release.yml

@@ -46,12 +46,15 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_x86_64.so ${SRC_SHNARF}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_COMPRESSOR}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_DECOMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_linux_x86_64.so ${SRC_TX_COMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4
@@ -81,12 +84,15 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_SHNARF}_${VERSION}_linux_arm64.so ${SRC_SHNARF}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_COMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_DECOMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_linux_arm64.so ${SRC_TX_COMPRESSOR}
       - name: Cache built binaries
         uses: actions/upload-artifact@v4
         with:
@@ -113,6 +119,8 @@ jobs:
           TARGET_COMPRESSOR: "blob_compressor"
           SRC_DECOMPRESSOR: "./lib/compressor/libdecompressor/libdecompressor.go"
           TARGET_DECOMPRESSOR: "blob_decompressor"
+          SRC_TX_COMPRESSOR: "./lib/compressor/libtxcompressor/libtxcompressor.go"
+          TARGET_TX_COMPRESSOR: "tx_compressor"
         run: |
           cd prover
           mkdir target
@@ -122,6 +130,8 @@ jobs:
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_COMPRESSOR}
           GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_DECOMPRESSOR}
           GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_DECOMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_DECOMPRESSOR}
+          GOARCH="amd64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_darwin_x86_64.dylib ${SRC_TX_COMPRESSOR}
+          GOARCH="arm64" go build -tags=nocorset -buildmode=c-shared -o ./target/${TARGET_TX_COMPRESSOR}_${VERSION}_darwin_arm64.dylib ${SRC_TX_COMPRESSOR}
 
       - name: Cache built binaries
         uses: actions/upload-artifact@v4

jvm-libs/linea/tx-compressor/build.gradle

@@ -0,0 +1,45 @@
+plugins {
+  id 'net.consensys.zkevm.kotlin-common-minimal-conventions'
+  id 'net.consensys.zkevm.linea-native-libs-helper'
+}
+
+description = 'Java JNA wrapper for Linea Transaction Compressor Library implemented in GO Lang'
+
+dependencies {
+  compileOnly "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  compileOnly project(":jvm-libs:generic:extensions:kotlin")
+  compileOnly "org.apache.logging.log4j:log4j-api:${libs.versions.log4j.get()}"
+  compileOnly "org.apache.logging.log4j:log4j-core:${libs.versions.log4j.get()}"
+
+  testImplementation "net.java.dev.jna:jna:${libs.versions.jna.get()}"
+  testImplementation project(":jvm-libs:generic:extensions:kotlin")
+  testImplementation project(":jvm-libs:linea:blob-compressor")
+  testImplementation testFixtures(project(":jvm-libs:linea:blob-compressor"))
+  testImplementation project(':jvm-libs:linea:besu-libs')
+  testImplementation project(':jvm-libs:linea:besu-rlp-and-mappers')
+}
+
+jar {
+  dependsOn configurations.runtimeClasspath
+}
+
+test {
+  // we cannot have more 1 compressor per JVM, hence we disable parallel execution
+  // because multiple threads would cause issues with the native library
+  systemProperties["junit.jupiter.execution.parallel.enabled"] = false
+  maxParallelForks = 1
+}
+
+def libsZipDownloadOutputDir = project.parent.layout.buildDirectory.asFile.get().absolutePath
+
+task downloadNativeLibs {
+  doLast {
+    // TODO: Update URL when tx-compressor native libs are released
+    // For now, we'll use the same release as blob-compressor since they share the dictionary
+    fetchLibFromZip("https://github.com/Consensys/linea-monorepo/releases/download/blob-libs-v2.1.0-rc1/linea-blob-libs-v2.1.0-rc1.zip", "tx_compressor", libsZipDownloadOutputDir)
+  }
+}
+
+compileKotlin {
+  dependsOn tasks.downloadNativeLibs
+}

jvm-libs/linea/tx-compressor/src/main/kotlin/linea/blob/GoNativeTxCompressor.kt

@@ -0,0 +1,123 @@
+package linea.blob
+
+import com.sun.jna.Library
+import com.sun.jna.Native
+import linea.jvm.ResourcesUtil.copyResourceToTmpDir
+
+/**
+ * JNA interface for the Go native transaction compressor library.
+ *
+ * This compressor operates at the transaction level rather than the block level,
+ * maintaining compression context across transactions for better compression ratios.
+ * It is designed for sequencer block building where transactions are added one by one
+ * until the compressed size threshold is reached.
+ */
+interface GoNativeTxCompressor {
+
+  /**
+   * TxInit initializes the transaction compressor.
+   *
+   * @param dataLimit Maximum size of compressed data in bytes. The caller should
+   *                  account for blob overhead (~100 bytes) when setting this limit.
+   * @param dictPath Path to the compression dictionary
+   * @return true if the compressor was successfully initialized, false otherwise
+   */
+  fun TxInit(dataLimit: Int, dictPath: String): Boolean
+
+  /**
+   * TxReset resets the compressor to its initial state.
+   * Must be called between each block being built.
+   */
+  fun TxReset()
+
+  /**
+   * TxWrite appends an RLP-encoded transaction to the compressed data.
+   *
+   * @param data bytes of the RLP encoded transaction
+   * @param data_len number of bytes
+   * @return true if the transaction was appended, false if it would exceed the limit
+   *         or if an error occurred (check TxError() for details)
+   */
+  fun TxWrite(data: ByteArray, data_len: Int): Boolean
+
+  /**
+   * TxCanWrite checks if an RLP-encoded transaction can be appended without actually appending it.
+   *
+   * @param data bytes of the RLP encoded transaction
+   * @param data_len number of bytes
+   * @return true if the transaction could be appended, false otherwise
+   */
+  fun TxCanWrite(data: ByteArray, data_len: Int): Boolean
+
+  /**
+   * TxLen returns the current length of the compressed data.
+   *
+   * @return number of bytes of compressed data
+   */
+  fun TxLen(): Int
+
+  /**
+   * TxWritten returns the number of uncompressed bytes written to the compressor.
+   *
+   * @return number of uncompressed bytes written
+   */
+  fun TxWritten(): Int
+
+  /**
+   * TxBytes fills out with the compressed data.
+   * The caller must allocate out and ensure that len(out) == TxLen()
+   *
+   * @param out The ByteArray to be filled with compressed data
+   */
+  fun TxBytes(out: ByteArray)
+
+  /**
+   * TxError returns the last error message.
+   * Should be checked if TxWrite or TxCanWrite returns false.
+   *
+   * @return error message string, or null if no error
+   */
+  fun TxError(): String?
+}
+
+interface GoNativeTxCompressorJnaLib : GoNativeTxCompressor, Library
+
+enum class TxCompressorVersion(val version: String) {
+  V1("v1.0.0"),
+}
+
+class GoNativeTxCompressorFactory {
+  companion object {
+    private const val DICTIONARY_NAME = "compressor-dictionaries/v2025-04-21.bin"
+    val dictionaryPath =
+      copyResourceToTmpDir(DICTIONARY_NAME, GoNativeTxCompressorFactory::class.java.classLoader)
+
+    private fun getLibFileName(version: String) = "tx_compressor_jna_$version"
+
+    @JvmStatic
+    private val loadedVersions = mutableMapOf<TxCompressorVersion, GoNativeTxCompressor>()
+
+    @JvmStatic
+    fun getInstance(version: TxCompressorVersion): GoNativeTxCompressor {
+      synchronized(loadedVersions) {
+        return loadedVersions[version]
+          ?: loadLib(version)
+            .also { loadedVersions[version] = it }
+      }
+    }
+
+    private fun loadLib(version: TxCompressorVersion): GoNativeTxCompressor {
+      val extractedLibFile = Native.extractFromResourcePath(
+        getLibFileName(version.version),
+        GoNativeTxCompressorFactory::class.java.classLoader,
+      )
+
+      return Native.load(
+        /* name = */
+        extractedLibFile.toString(),
+        /* interfaceClass = */
+        GoNativeTxCompressorJnaLib::class.java,
+      )
+    }
+  }
+}

jvm-libs/linea/tx-compressor/src/main/kotlin/linea/blob/TxCompressor.kt

@@ -0,0 +1,151 @@
+package linea.blob
+
+import linea.kotlin.encodeHex
+import org.apache.logging.log4j.LogManager
+
+class TxCompressionException(message: String) : RuntimeException(message)
+
+/**
+ * Interface for transaction-level compression.
+ *
+ * This compressor operates at the transaction level rather than the block level,
+ * maintaining compression context across transactions for better compression ratios.
+ * It is designed for sequencer block building where transactions are added one by one
+ * until the compressed size threshold is reached.
+ */
+interface TxCompressor {
+
+  /**
+   * Checks if an RLP-encoded transaction can be appended without actually appending it.
+   *
+   * @param rlpEncodedTx RLP-encoded transaction bytes
+   * @return true if the transaction could be appended, false if it would exceed the limit
+   * @throws TxCompressionException if the transaction is invalid
+   */
+  fun canAppendTransaction(rlpEncodedTx: ByteArray): Boolean
+
+  /**
+   * Appends an RLP-encoded transaction to the compressed data.
+   *
+   * @param rlpEncodedTx RLP-encoded transaction bytes
+   * @return AppendResult containing whether the transaction was appended and size information
+   * @throws TxCompressionException if the transaction is invalid
+   */
+  fun appendTransaction(rlpEncodedTx: ByteArray): AppendResult
+
+  /**
+   * Returns the current compressed size in bytes.
+   */
+  fun getCompressedSize(): Int
+
+  /**
+   * Returns the number of uncompressed bytes written.
+   */
+  fun getUncompressedSize(): Int
+
+  /**
+   * Returns the compressed data.
+   */
+  fun getCompressedData(): ByteArray
+
+  /**
+   * Resets the compressor to its initial state.
+   * Must be called between each block being built.
+   */
+  fun reset()
+
+  /**
+   * Returns the compressed data and resets the compressor.
+   */
+  fun getCompressedDataAndReset(): ByteArray {
+    val compressedData = getCompressedData()
+    reset()
+    return compressedData
+  }
+
+  data class AppendResult(
+    /** Whether the transaction was appended (false if it would exceed the limit) */
+    val txAppended: Boolean,
+    /** Compressed size before attempting to append */
+    val compressedSizeBefore: Int,
+    /** Compressed size after (same as before if not appended) */
+    val compressedSizeAfter: Int,
+  )
+}
+
+/**
+ * Go-backed implementation of TxCompressor using JNA bindings.
+ */
+class GoBackedTxCompressor private constructor(
+  internal val goNativeTxCompressor: GoNativeTxCompressor,
+) : TxCompressor {
+
+  companion object {
+    @JvmStatic
+    fun getInstance(compressorVersion: TxCompressorVersion, dataLimit: Int): GoBackedTxCompressor {
+      require(dataLimit > 0) { "dataLimit=$dataLimit must be greater than 0" }
+
+      val goNativeTxCompressor = GoNativeTxCompressorFactory.getInstance(compressorVersion)
+      val initialized = goNativeTxCompressor.TxInit(
+        dataLimit = dataLimit,
+        dictPath = GoNativeTxCompressorFactory.dictionaryPath.toString(),
+      )
+      if (!initialized) {
+        throw InstantiationException(goNativeTxCompressor.TxError())
+      }
+      return GoBackedTxCompressor(goNativeTxCompressor)
+    }
+  }
+
+  private val log = LogManager.getLogger(GoBackedTxCompressor::class.java)
+
+  override fun canAppendTransaction(rlpEncodedTx: ByteArray): Boolean {
+    val canWrite = goNativeTxCompressor.TxCanWrite(rlpEncodedTx, rlpEncodedTx.size)
+    val error = goNativeTxCompressor.TxError()
+    if (error != null) {
+      log.error("Failure while checking transaction: {}", rlpEncodedTx.encodeHex())
+      throw TxCompressionException(error)
+    }
+    return canWrite
+  }
+
+  override fun appendTransaction(rlpEncodedTx: ByteArray): TxCompressor.AppendResult {
+    val compressionSizeBefore = goNativeTxCompressor.TxLen()
+    val appended = goNativeTxCompressor.TxWrite(rlpEncodedTx, rlpEncodedTx.size)
+    val compressedSizeAfter = goNativeTxCompressor.TxLen()
+
+    log.trace(
+      "transaction compressed: txRlpSize={} compressionDataBefore={} compressionDataAfter={} compressionRatio={}",
+      rlpEncodedTx.size,
+      compressionSizeBefore,
+      compressedSizeAfter,
+      1.0 - ((compressedSizeAfter - compressionSizeBefore).toDouble() / rlpEncodedTx.size),
+    )
+
+    val error = goNativeTxCompressor.TxError()
+    if (error != null) {
+      log.error("Failure while writing the following RLP encoded transaction: {}", rlpEncodedTx.encodeHex())
+      throw TxCompressionException(error)
+    }
+
+    return TxCompressor.AppendResult(appended, compressionSizeBefore, compressedSizeAfter)
+  }
+
+  override fun getCompressedSize(): Int {
+    return goNativeTxCompressor.TxLen()
+  }
+
+  override fun getUncompressedSize(): Int {
+    return goNativeTxCompressor.TxWritten()
+  }
+
+  override fun getCompressedData(): ByteArray {
+    val compressedData = ByteArray(goNativeTxCompressor.TxLen())
+    goNativeTxCompressor.TxBytes(compressedData)
+    return compressedData
+  }
+
+  override fun reset() {
+    goNativeTxCompressor.TxReset()
+  }
+}