Upgrade lz4 dependency to 1.10.1 (elastic#139221)

slobodanadamovic · web-flow · commit 00bdb63d629d · 2025-12-23T17:45:24.000+01:00
Upgrading lz4 to the latest 1.10.1 version.
diff --git a/docs/changelog/139221.yaml b/docs/changelog/139221.yaml
@@ -0,0 +1,5 @@
+pr: 139221
+summary: Upgrade lz4 dependency to 1.10.1
+area: Infra/Core
+type: upgrade
+issues: []
diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml
@@ -62,9 +62,9 @@
             <sha256 value="333ff5369043975b7e031b8b27206937441854738e038c1f47f98d072a20437a" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="at.yawk.lz4" name="lz4-java" version="1.8.1">
-         <artifact name="lz4-java-1.8.1.jar">
-            <sha256 value="1de8ff91b90fba9d4896b8617ed3594c2e4390d321d5a92c34f80e62e406c1b0" origin="Generated by Gradle"/>
+      <component group="at.yawk.lz4" name="lz4-java" version="1.10.1">
+         <artifact name="lz4-java-1.10.1.jar">
+            <sha256 value="a58a84c4271e50df4c96ed916ccb7e48a869f8ed9cdcda1ad5d3d4c33b0214a3" origin="Generated by Gradle"/>
          </artifact>
       </component>
       <component group="cglib" name="cglib-nodep" version="3.3.0">
diff --git a/libs/lz4/build.gradle b/libs/lz4/build.gradle
@@ -9,7 +9,7 @@
 apply plugin: 'elasticsearch.publish'
 
 dependencies {
-  api 'at.yawk.lz4:lz4-java:1.8.1'
+  api 'at.yawk.lz4:lz4-java:1.10.1'
   api project(':libs:core')
 
   testImplementation(project(":test:framework")) {
diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/ESLZ4Decompressor.java
@@ -20,98 +20,169 @@
 
 import net.jpountz.lz4.LZ4Exception;
 import net.jpountz.lz4.LZ4FastDecompressor;
+import net.jpountz.util.ByteBufferUtils;
 
 import java.nio.ByteBuffer;
 
+import static org.elasticsearch.lz4.LZ4Constants.COPY_LENGTH;
+import static org.elasticsearch.lz4.LZ4Constants.ML_BITS;
+import static org.elasticsearch.lz4.LZ4Constants.RUN_MASK;
+import static org.elasticsearch.lz4.LZ4Utils.notEnoughSpace;
+
 /**
- * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
- * net.jpountz.lz4.LZ4JavaSafeFastDecompressor.
- *
- * It modifies the original implementation to use custom LZ4SafeUtils and SafeUtils implementations which
- * include performance improvements.
+ * This file is a vendored version of {@code net.jpountz.lz4.LZ4JavaSafeFastDecompressor} from
+ * <a href="https://github.com/yawkat/lz4-java">yawkat/lz4-java</a>. To obtain the original file, check out the {@code lz4-java} repository
+ * and run {@code mvn clean install} which will generate the original source of this class at
+ * {@code target/generated-sources/mvel/net/jpountz/lz4/LZ4JavaSafeFastDecompressor.java}.
+ * <p>
+ * It modifies the original implementation to use local {@link LZ4SafeUtils} and {@link SafeUtils} implementations which include some
+ * performance optimisations, and it also drops support for decompressing data from a direct (non-heap) {@link ByteBuffer}.
+ * <p>
+ * Differences from the original are annotated with [ES change from upstream]
  */
 public class ESLZ4Decompressor extends LZ4FastDecompressor {
     public static final LZ4FastDecompressor INSTANCE = new ESLZ4Decompressor();
 
-    ESLZ4Decompressor() {}
+    private ESLZ4Decompressor() {}
+
+    @Override
+    public int decompress(byte[] src, final int srcOff, byte[] dest, final int destOff, int destLen) {
+
+        final int srcEnd = src.length;
+
+        return decompress(src, srcOff, srcEnd - srcOff, dest, destOff, destLen);
+    }
 
-    public int decompress(byte[] src, int srcOff, byte[] dest, int destOff, int destLen) {
-        SafeUtils.checkRange(src, srcOff);
+    private int decompress(byte[] src, final int srcOff, final int srcLen, byte[] dest, final int destOff, int destLen) {
+        SafeUtils.checkRange(src, srcOff, srcLen);
         SafeUtils.checkRange(dest, destOff, destLen);
+
         if (destLen == 0) {
-            if (SafeUtils.readByte(src, srcOff) != 0) {
+            // Allow `srcLen > 1` despite just one byte being consumed since this 'fast' decompressor does not have to fully consume the src
+            if (srcLen < 1 || SafeUtils.readByte(src, srcOff) != 0) {
                 throw new LZ4Exception("Malformed input at " + srcOff);
-            } else {
-                return 1;
             }
-        } else {
-            int destEnd = destOff + destLen;
-            int sOff = srcOff;
-            int dOff = destOff;
-
-            while (true) {
-                int token = SafeUtils.readByte(src, sOff) & 255;
-                ++sOff;
-                int literalLen = token >>> 4;
-                if (literalLen == 15) {
-                    byte len;
-                    for (boolean var11 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; literalLen += 255) {
-                    }
+            return 1;
+        }
 
-                    literalLen += len & 255;
-                }
+        final int srcEnd = srcOff + srcLen;
+        final int destEnd = destOff + destLen;
 
-                int literalCopyEnd = dOff + literalLen;
-                if (literalCopyEnd > destEnd - 8) {
-                    if (literalCopyEnd != destEnd) {
-                        throw new LZ4Exception("Malformed input at " + sOff);
-                    } else {
-                        LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
-                        sOff += literalLen;
-                        return sOff - srcOff;
+        int sOff = srcOff;
+        int dOff = destOff;
+
+        while (true) {
+            if (sOff >= srcEnd) {
+                throw new LZ4Exception("Malformed input at " + sOff);
+            }
+            final int token = SafeUtils.readByte(src, sOff) & 0xFF;
+            ++sOff;
+
+            // literals
+            int literalLen = token >>> ML_BITS;
+            if (literalLen == RUN_MASK) {
+                byte len = (byte) 0xFF;
+                while (sOff < srcEnd && (len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
+                    literalLen += 0xFF;
+                    if (literalLen < 0) {
+                        throw new LZ4Exception("Too large literalLen");
                     }
                 }
+                literalLen += len & 0xFF;
+            }
+
+            final int literalCopyEnd = dOff + literalLen;
+            // Check for overflow
+            if (literalCopyEnd < dOff) {
+                throw new LZ4Exception("Too large literalLen");
+            }
 
-                LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
-                sOff += literalLen;
-                int matchDec = SafeUtils.readShortLE(src, sOff);
-                sOff += 2;
-                int matchOff = literalCopyEnd - matchDec;
-                if (matchOff < destOff) {
+            if (notEnoughSpace(destEnd - literalCopyEnd, COPY_LENGTH) || notEnoughSpace(srcEnd - sOff, COPY_LENGTH + literalLen)) {
+
+                if (literalCopyEnd != destEnd) {
+                    throw new LZ4Exception("Malformed input at " + sOff);
+                } else if (notEnoughSpace(srcEnd - sOff, literalLen)) {
                     throw new LZ4Exception("Malformed input at " + sOff);
+
+                } else {
+                    LZ4SafeUtils.safeArraycopy(src, sOff, dest, dOff, literalLen);
+                    sOff += literalLen;
+                    dOff = literalCopyEnd;
+                    break; // EOF
                 }
+            }
 
-                int matchLen = token & 15;
-                if (matchLen == 15) {
-                    byte len;
-                    for (boolean var15 = true; (len = SafeUtils.readByte(src, sOff++)) == -1; matchLen += 255) {
-                    }
+            LZ4SafeUtils.wildArraycopy(src, sOff, dest, dOff, literalLen);
+            sOff += literalLen;
+            dOff = literalCopyEnd;
 
-                    matchLen += len & 255;
-                }
+            // matchs
+            final int matchDec = SafeUtils.readShortLE(src, sOff);
+            sOff += 2;
+            int matchOff = dOff - matchDec;
 
-                matchLen += 4;
-                int matchCopyEnd = literalCopyEnd + matchLen;
-                if (matchCopyEnd > destEnd - 8) {
-                    if (matchCopyEnd > destEnd) {
-                        throw new LZ4Exception("Malformed input at " + sOff);
-                    }
+            if (matchOff < destOff) {
+                throw new LZ4Exception("Malformed input at " + sOff);
+            }
 
-                    LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, literalCopyEnd, matchLen);
-                } else {
-                    LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, literalCopyEnd, matchCopyEnd);
+            int matchLen = token & LZ4Constants.ML_MASK;
+            if (matchLen == LZ4Constants.ML_MASK) {
+                byte len = (byte) 0xFF;
+                while (sOff < srcEnd && (len = SafeUtils.readByte(src, sOff++)) == (byte) 0xFF) {
+                    matchLen += 0xFF;
+                    if (matchLen < 0) {
+                        throw new LZ4Exception("Too large matchLen");
+                    }
                 }
+                matchLen += len & 0xFF;
+            }
+            matchLen += LZ4Constants.MIN_MATCH;
+
+            final int matchCopyEnd = dOff + matchLen;
+            // Check for overflow
+            if (matchCopyEnd < dOff) {
+                throw new LZ4Exception("Too large matchLen");
+            }
 
-                dOff = matchCopyEnd;
+            if (matchDec == 0) {
+                if (matchCopyEnd > destEnd) {
+                    throw new LZ4Exception("Malformed input at " + sOff);
+                }
+                // With matchDec == 0, matchOff == dOff, so we'd copy in place. Zero the data instead. (CVE-2025-66566)
+                assert matchOff == dOff; // should always hold, but this extra check will trigger during fuzzing if my logic is wrong
+                LZ4Utils.zero(dest, dOff, matchCopyEnd);
+            } else if (notEnoughSpace(destEnd - matchCopyEnd, COPY_LENGTH)) {
+                if (matchCopyEnd > destEnd) {
+                    throw new LZ4Exception("Malformed input at " + sOff);
+                }
+                LZ4SafeUtils.safeIncrementalCopy(dest, matchOff, dOff, matchLen);
+            } else {
+                LZ4SafeUtils.wildIncrementalCopy(dest, matchOff, dOff, matchCopyEnd);
             }
+            dOff = matchCopyEnd;
         }
+
+        return sOff - srcOff;
+
     }
 
-    public int decompress(ByteBuffer src, int srcOff, ByteBuffer dest, int destOff, int destLen) {
+    @Override
+    public int decompress(ByteBuffer src, final int srcOff, ByteBuffer dest, final int destOff, int destLen) {
+
+        final int srcEnd = src.capacity();
+
+        return decompress(src, srcOff, srcEnd - srcOff, dest, destOff, destLen);
+    }
+
+    private int decompress(ByteBuffer src, final int srcOff, final int srcLen, ByteBuffer dest, final int destOff, int destLen) {
+        ByteBufferUtils.checkRange(src, srcOff, srcLen);
+        ByteBufferUtils.checkRange(dest, destOff, destLen);
+
         if (src.hasArray() && dest.hasArray()) {
-            return this.decompress(src.array(), srcOff + src.arrayOffset(), dest.array(), destOff + dest.arrayOffset(), destLen);
-        } else {
-            throw new AssertionError("Do not support decompression on direct buffers");
+            return decompress(src.array(), srcOff + src.arrayOffset(), srcLen, dest.array(), destOff + dest.arrayOffset(), destLen);
         }
+
+        // [ES change from upstream]: remove unused code
+        throw new AssertionError("Do not support decompression on direct buffers");
     }
 }
diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4SafeUtils.java
@@ -30,18 +30,21 @@
 import static org.elasticsearch.lz4.LZ4Constants.ML_BITS;
 import static org.elasticsearch.lz4.LZ4Constants.ML_MASK;
 import static org.elasticsearch.lz4.LZ4Constants.RUN_MASK;
+import static org.elasticsearch.lz4.LZ4Utils.lengthOfEncodedInteger;
+import static org.elasticsearch.lz4.LZ4Utils.notEnoughSpace;
 
 /**
- * This file is forked from https://github.com/lz4/lz4-java. In particular, it forks the following file
- * net.jpountz.lz4.LZ4SafeUtils.
+ * This file is a vendored version of {@code net.jpountz.lz4.LZ4SafeUtils} from
+ * <a href="https://github.com/yawkat/lz4-java">yawkat/lz4-java</a>.
+ * <p>
+ * Differences from the original are annotated with [ES change from upstream]
  *
- * It modifies the original implementation to use Java9 array mismatch method and varhandle performance
- * improvements. Comments are included to mark the changes.
+ * @see <a href="https://github.com/yawkat/lz4-java/blob/v1.10.1/src/java/net/jpountz/lz4/LZ4SafeUtils.java">LZ4SafeUtils.java</a>
  */
 enum LZ4SafeUtils {
     ;
 
-    // Added VarHandle
+    // [ES change from upstream]: define VarHandles allowing us to do operations without direct byte[] operations
     private static final VarHandle intPlatformNative = MethodHandles.byteArrayViewVarHandle(int[].class, Utils.NATIVE_BYTE_ORDER);
     private static final VarHandle longPlatformNative = MethodHandles.byteArrayViewVarHandle(long[].class, Utils.NATIVE_BYTE_ORDER);
 
@@ -54,6 +57,7 @@ static int hash64k(byte[] buf, int i) {
     }
 
     static boolean readIntEquals(byte[] buf, int i, int j) {
+        // [ES change from upstream]: use SafeUtils.readInt instead of comparing byte-by-byte
         return SafeUtils.readInt(buf, i) == SafeUtils.readInt(buf, j);
     }
 
@@ -63,16 +67,20 @@ static void safeIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchLe
         }
     }
 
-    // Modified wildIncrementalCopy to mirror version in LZ4UnsafeUtils
     static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCopyEnd) {
+        // [ES change from upstream]: use the implementation from LZ4UnsafeUtils#wildIncrementalCopy
+        // see https://github.com/yawkat/lz4-java/blob/v1.10.1/src/java-unsafe/net/jpountz/lz4/LZ4UnsafeUtils.java#L55-L99
+        // with further modifications as noted with [ES change from upstream] comments
         if (dOff - matchOff < 4) {
             for (int i = 0; i < 4; ++i) {
+                // [ES change from upstream]: just copy the byte directly, don't use readByte and writeByte
                 dest[dOff + i] = dest[matchOff + i];
             }
             dOff += 4;
             matchOff += 4;
             int dec = 0;
             assert dOff >= matchOff && dOff - matchOff < 8;
+            // [ES change from upstream]: use enhanced switch
             switch (dOff - matchOff) {
                 case 1 -> matchOff -= 3;
                 case 2 -> matchOff -= 2;
@@ -85,32 +93,35 @@ static void wildIncrementalCopy(byte[] dest, int matchOff, int dOff, int matchCo
                 case 7 -> dec = 3;
             }
 
+            // [ES change from upstream]: use copy4Bytes instead of readInt and writeInt
             copy4Bytes(dest, matchOff, dest, dOff);
             dOff += 4;
             matchOff -= dec;
         } else if (dOff - matchOff < LZ4Constants.COPY_LENGTH) {
+            // [ES change from upstream]: use copy8Bytes instead of readLong and writeLong
             copy8Bytes(dest, matchOff, dest, dOff);
             dOff += dOff - matchOff;
         }
         while (dOff < matchCopyEnd) {
+            // [ES change from upstream]: use copy8Bytes instead of readLong and writeLong
             copy8Bytes(dest, matchOff, dest, dOff);
             dOff += 8;
             matchOff += 8;
         }
     }
 
-    // Modified to use VarHandle
     static void copy8Bytes(byte[] src, int sOff, byte[] dest, int dOff) {
+        // [ES change from upstream]: use VarHandles instead of a byte-by-byte loop
         longPlatformNative.set(dest, dOff, (long) longPlatformNative.get(src, sOff));
     }
 
-    // Added to copy single int
+    // [ES change from upstream]: add this method to copy a single int (using VarHandles similarly to copy8Bytes)
     static void copy4Bytes(byte[] src, int sOff, byte[] dest, int dOff) {
         intPlatformNative.set(dest, dOff, (int) intPlatformNative.get(src, sOff));
     }
 
-    // Modified to use Arrays.mismatch
     static int commonBytes(byte[] b, int o1, int o2, int limit) {
+        // [ES change from upstream]: use Arrays#mismatch instead of a byte-by-byte loop
         int mismatch = Arrays.mismatch(b, o1, limit, b, o2, limit);
         return mismatch == -1 ? limit : mismatch;
     }
@@ -132,8 +143,8 @@ static void wildArraycopy(byte[] src, int sOff, byte[] dest, int dOff, int len)
             for (int i = 0; i < len; i += 8) {
                 copy8Bytes(src, sOff + i, dest, dOff + i);
             }
-            // Modified to catch IndexOutOfBoundsException instead of ArrayIndexOutOfBoundsException.
-            // VarHandles throw IndexOutOfBoundsException
+            // [ES change from upstream]: Varhandles throw an IndexOutOfBoundsException instead of ArrayIndexOutOfBoundsException so we
+            // change the type of exception caught here.
         } catch (IndexOutOfBoundsException e) {
             throw new LZ4Exception("Malformed input at offset " + sOff);
         }
@@ -184,7 +195,7 @@ static int encodeSequence(byte[] src, int anchor, int matchOff, int matchRef, in
     static int lastLiterals(byte[] src, int sOff, int srcLen, byte[] dest, int dOff, int destEnd) {
         final int runLen = srcLen;
 
-        if (dOff + runLen + 1 + (runLen + 255 - RUN_MASK) / 255 > destEnd) {
+        if (notEnoughSpace(destEnd - dOff, 1 + lengthOfEncodedInteger(runLen) + runLen)) {
             throw new LZ4Exception();
         }
 
diff --git a/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java b/libs/lz4/src/main/java/org/elasticsearch/lz4/LZ4Utils.java
diff --git a/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java b/libs/lz4/src/test/java/org/elasticsearch/lz4/ESLZ4DecompressorTests.java
