Skip to content

Compression libraries: FastLZ, LZ4 and Snappy #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Compression libraries: FastLZ, LZ4 and Snappy #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions contracts/utils/compression/FastLZ.sol
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
// SPDX-License-Identifier: MIT

pragma solidity ^0.8.24;

/**
* @dev Library for decompressing data using FastLZ.
*
* See https://ariya.github.io/FastLZ/
*/
library FastLZ {
/**
* @dev FastLZ level 1 decompression.
*
* Based on the reference implementation available here:
* https://github.com/ariya/FastLZ?tab=readme-ov-file#decompressor-reference-implementation
*/
function decompress(bytes memory input) internal pure returns (bytes memory output) {
assembly ("memory-safe") {
let inputPtr := add(input, 0x20)
let inputEnd := add(add(input, 0x20), mload(input))

// Use new memory allocate at the FMP
output := mload(0x40)
let outputPtr := add(output, 0x20)

for {} lt(inputPtr, inputEnd) {} {
let chunk := mload(inputPtr)
let first := byte(0, chunk)
let type_ := shr(5, first)

switch type_
case 0 {
mstore(outputPtr, mload(add(inputPtr, 1)))
inputPtr := add(inputPtr, add(2, first))
outputPtr := add(outputPtr, add(1, first))
}
case 7 {
let len := add(9, byte(1, chunk))
for {
let i := 0
let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1)
let ref := sub(outputPtr, ofs)
let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 3)
outputPtr := add(outputPtr, len)
}
Comment on lines +37 to +51
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Fix FastLZ type‑7 byte ordering.

For type‑7 matches the format is ctrl | offsetLow | lengthExtra. The code currently reads byte(1, chunk) as the length byte and byte(2, chunk) as the offset byte, then advances inputPtr by 3. This swaps the order: valid streams produced by the reference encoder/LibZip place the offset immediately after ctrl, so we subtract the wrong offset (using the length byte) and read the real offset as the next chunk’s first byte. Result: long matches (>=9 bytes) decode garbage or revert when the pointer check at Line 68 trips.

Reorder the reads so we pull the offset from byte(1, chunk), the length extension from byte(2, chunk), and only consume the extra length byte when we actually use it. One possible fix:

-                case 7 {
-                    let len := add(9, byte(1, chunk))
-                    for {
-                        let i := 0
-                        let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1)
+                case 7 {
+                    let extra := byte(2, chunk)
+                    let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1)
+                    let len := add(9, extra)
+                    for {
+                        let i := 0
                         let ref := sub(outputPtr, ofs)
                         let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
                     } lt(i, len) {
                         i := add(i, step)
                     } {
                         mcopy(add(outputPtr, i), add(ref, i), step)
                     }
-                    inputPtr := add(inputPtr, 3)
+                    inputPtr := add(inputPtr, 3)

(and mirror the change in the calldata loop).

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
case 7 {
let len := add(9, byte(1, chunk))
for {
let i := 0
let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1)
let ref := sub(outputPtr, ofs)
let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 3)
outputPtr := add(outputPtr, len)
}
case 7 {
// pull offset low first, then length extension
let extra := byte(2, chunk)
let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1)
let len := add(9, extra)
for {
let i := 0
let ref := sub(outputPtr, ofs)
let step:= xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 3)
outputPtr := add(outputPtr, len)
}
🤖 Prompt for AI Agents 
In contracts/utils/compression/FastLZ.sol around lines 37 to 51, the type-7
branch reads the bytes in the wrong order (using byte(1, chunk) as the length
and byte(2, chunk) as the offset) and always advances inputPtr by 3; update it
to read the offset from byte(1, chunk) and the length-extension from byte(2,
chunk), compute ofs using the correct offset byte, only advance inputPtr by 2
normally and by 3 when the extra length byte is actually consumed, and mirror
the same byte-order and pointer-advance fix in the calldata decoding loop so
offsets and lengths match the reference encoder/LibZip format.

default {
let len := add(2, type_)
for {
let i := 0
let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1)
let ref := sub(outputPtr, ofs)
let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 2)
outputPtr := add(outputPtr, len)
}
}
if iszero(eq(inputPtr, inputEnd)) {
revert(0, 0)
}

mstore(output, sub(outputPtr, add(output, 0x20)))
mstore(0x40, outputPtr)
}
}

function decompressCalldata(bytes calldata input) internal pure returns (bytes memory output) {
assembly ("memory-safe") {
let inputPtr := input.offset
let inputEnd := add(input.offset, input.length)

// Use new memory allocate at the FMP
output := mload(0x40)
let outputPtr := add(output, 0x20)

for {} lt(inputPtr, inputEnd) {} {
let chunk := calldataload(inputPtr)
let first := byte(0, chunk)
let type_ := shr(5, first)

switch type_
case 0 {
mstore(outputPtr, calldataload(add(inputPtr, 1)))
inputPtr := add(inputPtr, add(2, first))
outputPtr := add(outputPtr, add(1, first))
}
case 7 {
let len := add(9, byte(1, chunk))
for {
let i := 0
let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1)
let ref := sub(outputPtr, ofs)
let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 3)
outputPtr := add(outputPtr, len)
}
default {
let len := add(2, type_)
for {
let i := 0
let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1)
let ref := sub(outputPtr, ofs)
let step := xor(len, mul(lt(ofs, len), xor(ofs, len)))
} lt(i, len) {
i := add(i, step)
} {
mcopy(add(outputPtr, i), add(ref, i), step)
}
inputPtr := add(inputPtr, 2)
outputPtr := add(outputPtr, len)
}
}
if iszero(eq(inputPtr, inputEnd)) {
revert(0, 0)
}

mstore(output, sub(outputPtr, add(output, 0x20)))
mstore(0x40, outputPtr)
}
}
}
Loading
Loading