From a206a5032efef7735c4cb1a0e1959da823405062 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Mon, 28 Jul 2025 17:11:12 +0200 Subject: [PATCH 01/20] wip --- contracts/utils/Compression.sol | 77 ++++++++++++++++++ test/utils/Compression.t.sol | 137 ++++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+) create mode 100644 contracts/utils/Compression.sol create mode 100644 test/utils/Compression.t.sol diff --git a/contracts/utils/Compression.sol b/contracts/utils/Compression.sol new file mode 100644 index 00000000000..ec28537ddbc --- /dev/null +++ b/contracts/utils/Compression.sol @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +/** + * @dev Library for compressing and decompressing buffers. Supported compression algorithm: + * * FastLZ (level1): WIP + * * Calldata optimized: todo + * * ZIP: todo + */ +library Compression { + /** + * @dev FastLZ level 1 decompression. + * + * Based on the reference implementation available here: + * https://github.com/ariya/FastLZ?tab=readme-ov-file#decompressor-reference-implementation + */ + function flzDecompress(bytes memory input) internal pure returns (bytes memory output) { + assembly ("memory-safe") { + // Use new memory allocate at the FMP + output := mload(0x40) + + // Decrypted data location + let ptr := add(output, 0x20) + + // end of the input data (input.length after the beginning of the data) + let end := add(add(input, 0x20), mload(input)) + + for { + let data := add(input, 0x20) + } lt(data, end) {} { + let chunk := mload(data) + let first := byte(0, chunk) + let type_ := shr(5, first) + + switch type_ + case 0 { + mstore(ptr, mload(add(data, 1))) + data := add(data, add(2, first)) + ptr := add(ptr, add(1, first)) + } + case 7 { + let ofs := add(shl(8, and(first, 31)), byte(2, chunk)) + let len := add(9, byte(1, chunk)) + let ref := sub(sub(ptr, ofs), 1) + let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) + for { + let i := 0 + } lt(i, len) { + i := add(i, step) + } { + mstore(add(ptr, i), mload(add(ref, i))) + } + data := add(data, 3) + ptr := add(ptr, len) + } + default { + let ofs := add(shl(8, and(first, 31)), byte(1, chunk)) + let len := add(2, type_) + let ref := sub(sub(ptr, ofs), 1) + let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) + for { + let i := 0 + } lt(i, len) { + i := add(i, step) + } { + mstore(add(ptr, i), mload(add(ref, i))) + } + data := add(data, 2) + ptr := add(ptr, len) + } + } + mstore(output, sub(ptr, add(output, 0x20))) + mstore(0x40, ptr) + } + } +} diff --git a/test/utils/Compression.t.sol b/test/utils/Compression.t.sol new file mode 100644 index 00000000000..d956c0d0c86 --- /dev/null +++ b/test/utils/Compression.t.sol @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +import {Test} from "forge-std/Test.sol"; +import {Compression} from "@openzeppelin/contracts/utils/Compression.sol"; + +contract CompressionTest is Test { + using Compression for bytes; + + function testEncodeDecode(bytes memory input) external pure { + assertEq(_flzCompress(input).flzDecompress(), input); + } + + /// Copied from solady + function _flzCompress(bytes memory input) private pure returns (bytes memory output) { + assembly ("memory-safe") { + // store 8 bytes (value) at ptr, and return updated ptr + function ms8(ptr, value) -> ret { + mstore8(ptr, value) + ret := add(ptr, 1) + } + // load 24 bytes from a given location in memory, right aligned and in reverse order + function u24(ptr) -> value { + value := mload(ptr) + value := or(shl(16, byte(2, value)), or(shl(8, byte(1, value)), byte(0, value))) + } + function cmp(p_, q_, e_) -> _l { + for { + e_ := sub(e_, q_) + } lt(_l, e_) { + _l := add(_l, 1) + } { + e_ := mul(iszero(byte(0, xor(mload(add(p_, _l)), mload(add(q_, _l))))), e_) + } + } + function literals(runs_, src_, dest_) -> _o { + for { + _o := dest_ + } iszero(lt(runs_, 0x20)) { + runs_ := sub(runs_, 0x20) + } { + mstore(ms8(_o, 31), mload(src_)) + _o := add(_o, 0x21) + src_ := add(src_, 0x20) + } + if iszero(runs_) { + leave + } + mstore(ms8(_o, sub(runs_, 1)), mload(src_)) + _o := add(1, add(_o, runs_)) + } + function mt(l_, d_, o_) -> _o { + for { + d_ := sub(d_, 1) + } iszero(lt(l_, 263)) { + l_ := sub(l_, 262) + } { + o_ := ms8(ms8(ms8(o_, add(224, shr(8, d_))), 253), and(0xff, d_)) + } + if iszero(lt(l_, 7)) { + _o := ms8(ms8(ms8(o_, add(224, shr(8, d_))), sub(l_, 7)), and(0xff, d_)) + leave + } + _o := ms8(ms8(o_, add(shl(5, l_), shr(8, d_))), and(0xff, d_)) + } + function setHash(i_, v_) { + let p_ := add(mload(0x40), shl(2, i_)) + mstore(p_, xor(mload(p_), shl(224, xor(shr(224, mload(p_)), v_)))) + } + function getHash(i_) -> _h { + _h := shr(224, mload(add(mload(0x40), shl(2, i_)))) + } + function hash(v_) -> _r { + _r := and(shr(19, mul(2654435769, v_)), 0x1fff) + } + function setNextHash(ip_, ipStart_) -> _ip { + setHash(hash(u24(ip_)), sub(ip_, ipStart_)) + _ip := add(ip_, 1) + } + + output := mload(0x40) + + calldatacopy(output, calldatasize(), 0x8000) // Zeroize the hashmap. + let op := add(output, 0x8000) + + let a := add(input, 0x20) + + let ipStart := a + let ipLimit := sub(add(ipStart, mload(input)), 13) + for { + let ip := add(2, a) + } lt(ip, ipLimit) {} { + let r := 0 + let d := 0 + for {} 1 {} { + let s := u24(ip) + let h := hash(s) + r := add(ipStart, getHash(h)) + setHash(h, sub(ip, ipStart)) + d := sub(ip, r) + if iszero(lt(ip, ipLimit)) { + break + } + ip := add(ip, 1) + if iszero(gt(d, 0x1fff)) { + if eq(s, u24(r)) { + break + } + } + } + if iszero(lt(ip, ipLimit)) { + break + } + ip := sub(ip, 1) + if gt(ip, a) { + op := literals(sub(ip, a), a, op) + } + let l := cmp(add(r, 3), add(ip, 3), add(ipLimit, 9)) + op := mt(l, d, op) + ip := setNextHash(setNextHash(add(ip, l), ipStart), ipStart) + a := ip + } + // Copy the result to compact the memory, overwriting the hashmap. + let end := sub(literals(sub(add(ipStart, mload(input)), a), a, op), 0x7fe0) + let o := add(output, 0x20) + mstore(output, sub(end, o)) // Store the length. + for {} iszero(gt(o, end)) { + o := add(o, 0x20) + } { + mstore(o, mload(add(o, 0x7fe0))) + } + + mstore(0x40, end) + } + } +} From bf58a07b38de2607feda7c7641e1abb36069618c Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Tue, 29 Jul 2025 11:03:40 +0200 Subject: [PATCH 02/20] Update Compression.sol --- contracts/utils/Compression.sol | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contracts/utils/Compression.sol b/contracts/utils/Compression.sol index ec28537ddbc..683709dae52 100644 --- a/contracts/utils/Compression.sol +++ b/contracts/utils/Compression.sol @@ -4,9 +4,11 @@ pragma solidity ^0.8.20; /** * @dev Library for compressing and decompressing buffers. Supported compression algorithm: - * * FastLZ (level1): WIP - * * Calldata optimized: todo - * * ZIP: todo + * * FastLZ (level1): (WIP) Do we want this? We should have an NodeJS implementation. + * * Other LZ77/LZSS/LZ4: Do we want this? + * * Deflate: Do we want this? The Huffman part is probably going to be a pain, and not worth it for "small" inputs. + * * Brotli, bzip, gzip: Same as Deflate? + * * Calldata optimized: Do we want this? */ library Compression { /** From 5984e6c629448dba607e67b2d881fcd789ea9003 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:00:49 +0200 Subject: [PATCH 03/20] Add snappy uncompress --- .../FastLZ.sol} | 11 +- contracts/utils/compression/Snappy.sol | 240 +++++++++++ package-lock.json | 399 ++++++++++++++++++ package.json | 1 + .../FastLZ.t.sol} | 8 +- test/utils/compression/Snappy.test.js | 63 +++ 6 files changed, 709 insertions(+), 13 deletions(-) rename contracts/utils/{Compression.sol => compression/FastLZ.sol} (82%) create mode 100644 contracts/utils/compression/Snappy.sol rename test/utils/{Compression.t.sol => compression/FastLZ.t.sol} (95%) create mode 100644 test/utils/compression/Snappy.test.js diff --git a/contracts/utils/Compression.sol b/contracts/utils/compression/FastLZ.sol similarity index 82% rename from contracts/utils/Compression.sol rename to contracts/utils/compression/FastLZ.sol index 683709dae52..61ed41ca869 100644 --- a/contracts/utils/Compression.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -3,21 +3,16 @@ pragma solidity ^0.8.20; /** - * @dev Library for compressing and decompressing buffers. Supported compression algorithm: - * * FastLZ (level1): (WIP) Do we want this? We should have an NodeJS implementation. - * * Other LZ77/LZSS/LZ4: Do we want this? - * * Deflate: Do we want this? The Huffman part is probably going to be a pain, and not worth it for "small" inputs. - * * Brotli, bzip, gzip: Same as Deflate? - * * Calldata optimized: Do we want this? + * @dev Library for compressing and decompressing buffers using FastLZ. */ -library Compression { +library FastLZ { /** * @dev FastLZ level 1 decompression. * * Based on the reference implementation available here: * https://github.com/ariya/FastLZ?tab=readme-ov-file#decompressor-reference-implementation */ - function flzDecompress(bytes memory input) internal pure returns (bytes memory output) { + function decompress(bytes memory input) internal pure returns (bytes memory output) { assembly ("memory-safe") { // Use new memory allocate at the FMP output := mload(0x40) diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol new file mode 100644 index 00000000000..24d8b7e75c1 --- /dev/null +++ b/contracts/utils/compression/Snappy.sol @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.20; + +/** + * @dev Library for compressing and decompressing buffers using Snappy. + * + * See https://github.com/google/snappy + */ +library Snappy { + error DecodingFailure(); + + /** + * @dev Implementation of Snappy's uncompress function + */ + function uncompress(bytes memory input) internal pure returns (bytes memory output) { + bytes4 errorSelector = DecodingFailure.selector; + + assembly ("memory-safe") { + // helper: revert with custom error (without args) if boolean isn't true + function assert(b, e) { + if iszero(b) { + mstore(0, e) + revert(0, 0x04) + } + } + + // input buffer bounds + let inputBegin := add(input, 0x20) + let inputEnd := add(inputBegin, mload(input)) + + // input traversal pointer + let inputPtr := inputBegin + // read length of the decompressed buffer + let outputLength := 0 + for {} lt(inputPtr, inputEnd) { + inputPtr := add(inputPtr, 1) + } { + let c := byte(0, mload(inputPtr)) + outputLength := add(outputLength, shl(mul(7, sub(inputPtr, inputBegin)), and(c, 0x7f))) + if iszero(and(c, 0x80)) { + break + } + } + inputPtr := add(inputPtr, 1) + + // allocated output buffer + output := mload(0x40) + let outputPtr := add(output, 0x20) + mstore(output, outputLength) + mstore(0x40, add(outputPtr, outputLength)) + + // decompress input buffer into output buffer + for { + let len, offset + } lt(inputPtr, inputEnd) {} { + // get next (compressed) word -- used as a cache for further reads + let w := mload(inputPtr) + inputPtr := add(inputPtr, 1) + + let c := byte(0, w) + + // consider different cases based on the lower 2 bits of c + // - 0: litteral + // - 1,2,3: offset copy + switch and(c, 0x3) + case 0 { + len := add(shr(2, c), 1) + if gt(len, 60) { + assert(lt(add(inputPtr, 3), inputEnd), errorSelector) + let smallLen := sub(len, 60) + len := or(or(byte(1, w), shl(8, byte(2, w))), or(shl(16, byte(3, w)), shl(24, byte(4, w)))) + len := add(and(len, shr(sub(256, mul(8, smallLen)), not(0))), 1) + inputPtr := add(inputPtr, smallLen) + } + assert(not(gt(add(inputPtr, len), inputEnd)), errorSelector) + // copy len bytes from input to output in chunks of 32 bytes + for { + let i := 0 + } lt(i, len) { + i := add(i, 0x20) + } { + mstore(add(outputPtr, i), mload(add(inputPtr, i))) + } + inputPtr := add(inputPtr, len) + outputPtr := add(outputPtr, len) + + // continue to skip the offset copy logic that is shared by the other 3 cases + continue + } + case 1 { + assert(lt(inputPtr, inputEnd), errorSelector) + len := add(and(shr(2, c), 0x7), 4) + offset := add(byte(1, w), shl(8, shr(5, c))) + inputPtr := add(inputPtr, 1) + } + case 2 { + assert(lt(add(inputPtr, 1), inputEnd), errorSelector) + len := add(shr(2, c), 1) + offset := add(byte(1, w), shl(8, byte(2, w))) + inputPtr := add(inputPtr, 2) + } + case 3 { + assert(lt(add(inputPtr, 3), inputEnd), errorSelector) + len := add(shr(2, c), 1) + offset := add(add(byte(1, w), shl(8, byte(2, w))), add(shl(16, byte(3, w)), shl(24, byte(4, w)))) + inputPtr := add(inputPtr, 4) + } + assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) + + // copying in chunks will not work if the offset is larger than the chunk length, so we compute + // `chunkSize = Math.min(0x20, offset)` and use it for the memory copy + let chunkSize := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + + // copy len bytes from output to itself. + for { + let i := 0 + } lt(i, len) { + i := add(i, chunkSize) + } { + mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) + } + outputPtr := add(outputPtr, len) + } + + // sanity check, FMP is at the right location + assert(eq(outputPtr, mload(0x40)), errorSelector) + } + } + + /// @dev Variant of {uncompress} that takes a buffer from calldata. + function uncompressCalldata(bytes calldata input) internal pure returns (bytes memory output) { + bytes4 errorSelector = DecodingFailure.selector; + + assembly ("memory-safe") { + // helper: revert with custom error (without args) if boolean isn't true + function assert(b, e) { + if iszero(b) { + mstore(0, e) + revert(0, 0x04) + } + } + + // input buffer bounds + let inputBegin := input.offset + let inputEnd := add(inputBegin, input.length) + + // input traversal pointer + let inputPtr := inputBegin + // read length of the decompressed buffer + let outputLength := 0 + for {} lt(inputPtr, inputEnd) { + inputPtr := add(inputPtr, 1) + } { + let c := byte(0, calldataload(inputPtr)) + outputLength := add(outputLength, shl(mul(7, sub(inputPtr, inputBegin)), and(c, 0x7f))) + if iszero(and(c, 0x80)) { + break + } + } + inputPtr := add(inputPtr, 1) + + // allocated output buffer + output := mload(0x40) + let outputPtr := add(output, 0x20) + mstore(output, outputLength) + mstore(0x40, add(outputPtr, outputLength)) + + // decompress input buffer into output buffer + for { + let len, offset + } lt(inputPtr, inputEnd) {} { + // get next (compressed) word -- used as a cache for further reads + let w := calldataload(inputPtr) + inputPtr := add(inputPtr, 1) + + let c := byte(0, w) + + // consider different cases based on the lower 2 bits of c + // - 0: litteral + // - 1,2,3: offset copy + switch and(c, 0x3) + case 0 { + len := add(shr(2, c), 1) + if gt(len, 60) { + assert(lt(add(inputPtr, 3), inputEnd), errorSelector) + let smallLen := sub(len, 60) + len := or(or(byte(1, w), shl(8, byte(2, w))), or(shl(16, byte(3, w)), shl(24, byte(4, w)))) + len := add(and(len, shr(sub(256, mul(8, smallLen)), not(0))), 1) + inputPtr := add(inputPtr, smallLen) + } + assert(not(gt(add(inputPtr, len), inputEnd)), errorSelector) + // copy len bytes from input to output in chunks of 32 bytes + calldatacopy(outputPtr, inputPtr, len) + inputPtr := add(inputPtr, len) + outputPtr := add(outputPtr, len) + + // continue to skip the offset copy logic that is shared by the other 3 cases + continue + } + case 1 { + assert(lt(inputPtr, inputEnd), errorSelector) + len := add(and(shr(2, c), 0x7), 4) + offset := add(byte(1, w), shl(8, shr(5, c))) + inputPtr := add(inputPtr, 1) + } + case 2 { + assert(lt(add(inputPtr, 1), inputEnd), errorSelector) + len := add(shr(2, c), 1) + offset := add(byte(1, w), shl(8, byte(2, w))) + inputPtr := add(inputPtr, 2) + } + case 3 { + assert(lt(add(inputPtr, 3), inputEnd), errorSelector) + len := add(shr(2, c), 1) + offset := add(add(byte(1, w), shl(8, byte(2, w))), add(shl(16, byte(3, w)), shl(24, byte(4, w)))) + inputPtr := add(inputPtr, 4) + } + assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) + + // copying in chunks will not work if the offset is larger than the chunk length, so we compute + // `chunkSize = Math.min(0x20, offset)` and use it for the memory copy + let chunkSize := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + + // copy len bytes from output to itself. + for { + let i := 0 + } lt(i, len) { + i := add(i, chunkSize) + } { + mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) + } + outputPtr := add(outputPtr, len) + } + + // sanity check, FMP is at the right location + assert(eq(outputPtr, mload(0x40)), errorSelector) + } + } +} diff --git a/package-lock.json b/package-lock.json index 072b6072d59..4806ffb5adf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "prettier-plugin-solidity": "^2.0.0", "rimraf": "^6.0.0", "semver": "^7.3.5", + "snappy": "^7.3.0", "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", @@ -425,6 +426,40 @@ "node": ">=0.1.90" } }, + "node_modules/@emnapi/core": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.4.5.tgz", + "integrity": "sha512-XsLw1dEOpkSX/WucdqUhPWP7hDxSvZiY+fsUC14h+FtQ2Ifni4znbBt8punRX+Uj2JG/uDb8nEHVKvrVlvdZ5Q==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.0.4", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.4.5.tgz", + "integrity": "sha512-++LApOtY0pEEz1zrd9vy1/zXVaVJJ/EbAF3u0fXIzPJEDtnITsBGbbK0EkM72amhl/R5b+5xx0Y/QhcVOpuulg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.0.4.tgz", + "integrity": "sha512-PJR+bOmMOPH8AtcTGAyYNiuJ3/Fcoj2XN/gBEWzDIKh254XO+mM9XoXHk5GNEhodxeMznbg7BlRojVbKN+gC6g==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.7.0.tgz", @@ -1639,6 +1674,325 @@ "node": ">=16.0.0" } }, + "node_modules/@napi-rs/snappy-android-arm-eabi": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.3.0.tgz", + "integrity": "sha512-KgD+8wNtS4/w7JqpiyONTwPAF74mhLnH7up+Ke4l5+jS9WFC1UexFsp+g62ugCIAmsdWfWKIpNTPpxkV7yrdYw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-android-arm64": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.3.0.tgz", + "integrity": "sha512-HUX9icvbWPgBDNwdxrFt1+lDfOmVsxAOMnJ+jzSYFDp2EclQjxAhPUV/rSRtS23xakHZgi6aBiJ+aY1p9Z/Fkg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-arm64": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.3.0.tgz", + "integrity": "sha512-shU1IOgMJRBLxqNvuqRvIr3lP8i2q/UJiwHfkipN05wzjlLN327ej0E98TarsfDgn8SDX92ovsc6zVfb4bhHbg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-darwin-x64": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.3.0.tgz", + "integrity": "sha512-j/gaU69biRWyYq46DocHYBLjJezQhLNEjaWW3J7Y/dbil3P/+iCkvJkA0uHnIG2KXJk/QSyu2kzuy6YnMSk9Qw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-freebsd-x64": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.3.0.tgz", + "integrity": "sha512-JnZDZDoWrUO2E8hmiU6oyrD34xazB8CVoZYCsHmkF5Yrl/5jvr/0BG3jd4SQ10wAdhQDHVwS8iaEB6GXdQxZGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.3.0.tgz", + "integrity": "sha512-qjEItUn91kuR6oLhXrLXSiUSB1LEz9L4lHCidmGmcIvvdM9FrceS0ny8seYuiU5fa6pZTwTO/6O3nQFQlgI+tg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-gnu": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.3.0.tgz", + "integrity": "sha512-+v3IjUgphndbzRQ/PXoEDyORW3X4xfMPBAoQfziW1g+f+7nms+PCobWoAbGMneIGf6QP4KuLsv3clVa9+noW9g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-musl": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.3.0.tgz", + "integrity": "sha512-8XFJIYXYrmesQ4m8xHX7jfon7HFdxY1n3eykfQaAWrKLpLMQL2x+caS6So7wpRFJI/2NyYWiY7FWYTAuMDbFVw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-arm64-ohos": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-ohos/-/snappy-linux-arm64-ohos-7.3.0.tgz", + "integrity": "sha512-PlKHoHxeiNOQakbdMoQ0nF6GOKC52wHNCJ0Ii/JJHF8YfPIOv/BRZBEY32b8IidGj9RMq8vCa/ftBvNjIYdF5Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-ppc64-gnu": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-ppc64-gnu/-/snappy-linux-ppc64-gnu-7.3.0.tgz", + "integrity": "sha512-OJTssYn5nZC6UX9O85z4/Mtwr4ya09P0ebatb7IVNRN8GKcnkZbDOZX4HosSNNAllVQgn3n18gP45RSbtXzcGQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-riscv64-gnu": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-riscv64-gnu/-/snappy-linux-riscv64-gnu-7.3.0.tgz", + "integrity": "sha512-3O0QwzFtUm9sxDVBPjYgvAeHDw+omNJdtvamS60Yn87UZMk8V50vgDDr5NKcksmyfgdftdZfR/bFqDYCucTQuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-s390x-gnu": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-s390x-gnu/-/snappy-linux-s390x-gnu-7.3.0.tgz", + "integrity": "sha512-hGgPn0M1tX4gUsJ/FieLsmKQgiI5p/TOlxh9WDqtIp52k9m965V9lRTEQrlUL//CndlEz3m6bk0PVgd4klNF8w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-gnu": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.3.0.tgz", + "integrity": "sha512-pBDHRUOJqC/6ZG7zy068pTItGeF4gJwO9zcZVgt951WXCvn7cIEjHhZSRim2ud8IoeH0Oumew1QYxZIxOFACJg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-linux-x64-musl": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.3.0.tgz", + "integrity": "sha512-S+lWMBIvNYbTqGx1V2ndmcXyzIqsBTCBiU0cgtWxDNMTea5LCC749k+xxcCms/D0vwEe98qYkqAbs8IIEh1hKA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-wasm32-wasi": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-wasm32-wasi/-/snappy-wasm32-wasi-7.3.0.tgz", + "integrity": "sha512-8K6OVGRzJ21BVXCYmbOa0wZrpalOWT8k/v8PbFHWcLLlghPEMJYGuGMlkzTd+VgKYjkjX1qBLaDA51+UMRWPug==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@napi-rs/wasm-runtime": "^0.2.12" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@napi-rs/snappy-win32-arm64-msvc": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.3.0.tgz", + "integrity": "sha512-TzBl/dmHt+8JGZrv0eQ1j7dL6XuSor7K/1EZ2hwX1vxe/rdygZfIlMJlYbUuEqiK2ZJD1AVuc5WhqyQADu0t+A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-ia32-msvc": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.3.0.tgz", + "integrity": "sha512-UVIIzEcxSo4TKNs8PGrfAjCHKlbxCljnQFjYPEwv8O9YaNpQoSy6pZU4SdO5aB4SRbqhSoqtkJlaKFBiHLa8Zw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/snappy-win32-x64-msvc": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.3.0.tgz", + "integrity": "sha512-4vI5r2QY0NRIe0l/Vsgs+cqZDIFqtsqwSJVVBuCfcf1G39AXQ7g1bstdxnE4s6MHh/Xis7h2vsajYNnQh6xX4Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-0.2.12.tgz", + "integrity": "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.4.3", + "@emnapi/runtime": "^1.4.3", + "@tybys/wasm-util": "^0.10.0" + } + }, "node_modules/@noble/ciphers": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", @@ -2422,6 +2776,17 @@ "node": ">=14.16" } }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.0.tgz", + "integrity": "sha512-VyyPYFlOMNylG45GoAe0xDoLwWuowvf92F9kySqzYh8vmYm7D2u4iUJKa1tOUpS70Ku13ASrOkS4ScXFsTaCNQ==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@types/bn.js": { "version": "5.1.6", "resolved": "https://registry.npmjs.org/@types/bn.js/-/bn.js-5.1.6.tgz", @@ -9339,6 +9704,40 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/snappy": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.3.0.tgz", + "integrity": "sha512-Qd1XxFO71HOOA6RxWkO5yHcYrQyBZOqGFKv99DD75bS34I3J6HEudlO4Lo617B6A6fJJ87YS5oYY9NZXxlXGkw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 10" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "optionalDependencies": { + "@napi-rs/snappy-android-arm-eabi": "7.3.0", + "@napi-rs/snappy-android-arm64": "7.3.0", + "@napi-rs/snappy-darwin-arm64": "7.3.0", + "@napi-rs/snappy-darwin-x64": "7.3.0", + "@napi-rs/snappy-freebsd-x64": "7.3.0", + "@napi-rs/snappy-linux-arm-gnueabihf": "7.3.0", + "@napi-rs/snappy-linux-arm64-gnu": "7.3.0", + "@napi-rs/snappy-linux-arm64-musl": "7.3.0", + "@napi-rs/snappy-linux-arm64-ohos": "7.3.0", + "@napi-rs/snappy-linux-ppc64-gnu": "7.3.0", + "@napi-rs/snappy-linux-riscv64-gnu": "7.3.0", + "@napi-rs/snappy-linux-s390x-gnu": "7.3.0", + "@napi-rs/snappy-linux-x64-gnu": "7.3.0", + "@napi-rs/snappy-linux-x64-musl": "7.3.0", + "@napi-rs/snappy-wasm32-wasi": "7.3.0", + "@napi-rs/snappy-win32-arm64-msvc": "7.3.0", + "@napi-rs/snappy-win32-ia32-msvc": "7.3.0", + "@napi-rs/snappy-win32-x64-msvc": "7.3.0" + } + }, "node_modules/solc": { "version": "0.8.26", "resolved": "https://registry.npmjs.org/solc/-/solc-0.8.26.tgz", diff --git a/package.json b/package.json index 65def77e53e..cecde3025ae 100644 --- a/package.json +++ b/package.json @@ -85,6 +85,7 @@ "prettier-plugin-solidity": "^2.0.0", "rimraf": "^6.0.0", "semver": "^7.3.5", + "snappy": "^7.3.0", "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", diff --git a/test/utils/Compression.t.sol b/test/utils/compression/FastLZ.t.sol similarity index 95% rename from test/utils/Compression.t.sol rename to test/utils/compression/FastLZ.t.sol index d956c0d0c86..bc4cc4c359e 100644 --- a/test/utils/Compression.t.sol +++ b/test/utils/compression/FastLZ.t.sol @@ -3,13 +3,11 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; -import {Compression} from "@openzeppelin/contracts/utils/Compression.sol"; - -contract CompressionTest is Test { - using Compression for bytes; +import {FastLZ} from "@openzeppelin/contracts/utils/compression/FastLZ.sol"; +contract FastLZTest is Test { function testEncodeDecode(bytes memory input) external pure { - assertEq(_flzCompress(input).flzDecompress(), input); + assertEq(FastLZ.decompress(_flzCompress(input)), input); } /// Copied from solady diff --git a/test/utils/compression/Snappy.test.js b/test/utils/compression/Snappy.test.js new file mode 100644 index 00000000000..536d0f98288 --- /dev/null +++ b/test/utils/compression/Snappy.test.js @@ -0,0 +1,63 @@ +const { ethers } = require('hardhat'); +const { expect } = require('chai'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const snappy = require('snappy'); + +async function fixture() { + const mock = await ethers.deployContract('$Snappy'); + return { mock }; +} + +// From https://github.com/google/snappy/blob/main/snappy_unittest.cc +const unittests = [ + '', + 'a', + 'ab', + 'abc', + 'aaaaaaa' + 'b'.repeat(16) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(256) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(2047) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcaaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcabcabcabcabcabcab', + 'abcabcabcabcabcabcab0123456789ABCDEF', + 'abcabcabcabcabcabcabcabcabcabcabcabc', + 'abcabcabcabcabcabcabcabcabcabcabcabc0123456789ABCDEF', +]; + +describe('Snappy', function () { + before(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + describe("Google's unit tests", function () { + for (const [i, str] of Object.entries(unittests)) { + it(`#${i}: length ${str.length}`, async function () { + this.input = str; + }); + } + }); + + it('Lorem ipsum...', async function () { + this.input = + '\ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ +Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ +Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ +Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ +Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ +'; + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const compressed = snappy.compressSync(this.input); + const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + await expect(this.mock.$uncompress(compressed)).to.eventually.equal(hex); + await expect(this.mock.$uncompressCalldata(compressed)).to.eventually.equal(hex); + }); +}); From c45990d8244f2d397cacd1ef9eac22c19a5baf87 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:07:17 +0200 Subject: [PATCH 04/20] add FastLZ.decompressCalldata --- contracts/utils/compression/FastLZ.sol | 98 +++++++++++++++++++++----- contracts/utils/compression/Snappy.sol | 12 ++-- test/utils/compression/FastLZ.t.sol | 8 +++ 3 files changed, 93 insertions(+), 25 deletions(-) diff --git a/contracts/utils/compression/FastLZ.sol b/contracts/utils/compression/FastLZ.sol index 61ed41ca869..79e8ef1b771 100644 --- a/contracts/utils/compression/FastLZ.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -17,58 +17,118 @@ library FastLZ { // Use new memory allocate at the FMP output := mload(0x40) - // Decrypted data location - let ptr := add(output, 0x20) + // Decrypted inputPtr location + let outputPtr := add(output, 0x20) - // end of the input data (input.length after the beginning of the data) + // end of the input inputPtr (input.length after the beginning of the inputPtr) let end := add(add(input, 0x20), mload(input)) for { - let data := add(input, 0x20) - } lt(data, end) {} { - let chunk := mload(data) + let inputPtr := add(input, 0x20) + } lt(inputPtr, end) {} { + let chunk := mload(inputPtr) let first := byte(0, chunk) let type_ := shr(5, first) switch type_ case 0 { - mstore(ptr, mload(add(data, 1))) - data := add(data, add(2, first)) - ptr := add(ptr, add(1, first)) + mstore(outputPtr, mload(add(inputPtr, 1))) + inputPtr := add(inputPtr, add(2, first)) + outputPtr := add(outputPtr, add(1, first)) } case 7 { let ofs := add(shl(8, and(first, 31)), byte(2, chunk)) let len := add(9, byte(1, chunk)) - let ref := sub(sub(ptr, ofs), 1) + let ref := sub(sub(outputPtr, ofs), 1) let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 } lt(i, len) { i := add(i, step) } { - mstore(add(ptr, i), mload(add(ref, i))) + mstore(add(outputPtr, i), mload(add(ref, i))) } - data := add(data, 3) - ptr := add(ptr, len) + inputPtr := add(inputPtr, 3) + outputPtr := add(outputPtr, len) } default { let ofs := add(shl(8, and(first, 31)), byte(1, chunk)) let len := add(2, type_) - let ref := sub(sub(ptr, ofs), 1) + let ref := sub(sub(outputPtr, ofs), 1) let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 } lt(i, len) { i := add(i, step) } { - mstore(add(ptr, i), mload(add(ref, i))) + mstore(add(outputPtr, i), mload(add(ref, i))) } - data := add(data, 2) - ptr := add(ptr, len) + inputPtr := add(inputPtr, 2) + outputPtr := add(outputPtr, len) } } - mstore(output, sub(ptr, add(output, 0x20))) - mstore(0x40, ptr) + mstore(output, sub(outputPtr, add(output, 0x20))) + mstore(0x40, outputPtr) + } + } + + function decompressCallinputPtr(bytes calldata input) internal pure returns (bytes memory output) { + assembly ("memory-safe") { + // Use new memory allocate at the FMP + output := mload(0x40) + + // Decrypted inputPtr location + let outputPtr := add(output, 0x20) + + // end of the input inputPtr (input.length after the beginning of the inputPtr) + let end := add(input.offset, input.length) + + for { + let inputPtr := input.offset + } lt(inputPtr, end) {} { + let chunk := calldataload(inputPtr) + let first := byte(0, chunk) + let type_ := shr(5, first) + + switch type_ + case 0 { + mstore(outputPtr, calldataload(add(inputPtr, 1))) + inputPtr := add(inputPtr, add(2, first)) + outputPtr := add(outputPtr, add(1, first)) + } + case 7 { + let ofs := add(shl(8, and(first, 31)), byte(2, chunk)) + let len := add(9, byte(1, chunk)) + let ref := sub(sub(outputPtr, ofs), 1) + let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) + for { + let i := 0 + } lt(i, len) { + i := add(i, step) + } { + mstore(add(outputPtr, i), mload(add(ref, i))) + } + inputPtr := add(inputPtr, 3) + outputPtr := add(outputPtr, len) + } + default { + let ofs := add(shl(8, and(first, 31)), byte(1, chunk)) + let len := add(2, type_) + let ref := sub(sub(outputPtr, ofs), 1) + let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) + for { + let i := 0 + } lt(i, len) { + i := add(i, step) + } { + mstore(add(outputPtr, i), mload(add(ref, i))) + } + inputPtr := add(inputPtr, 2) + outputPtr := add(outputPtr, len) + } + } + mstore(output, sub(outputPtr, add(output, 0x20))) + mstore(0x40, outputPtr) } } } diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index 24d8b7e75c1..972ff6196b8 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -109,14 +109,14 @@ library Snappy { assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) // copying in chunks will not work if the offset is larger than the chunk length, so we compute - // `chunkSize = Math.min(0x20, offset)` and use it for the memory copy - let chunkSize := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + // `step = Math.min(0x20, offset)` and use it for the memory copy + let step := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) // copy len bytes from output to itself. for { let i := 0 } lt(i, len) { - i := add(i, chunkSize) + i := add(i, step) } { mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) } @@ -219,14 +219,14 @@ library Snappy { assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) // copying in chunks will not work if the offset is larger than the chunk length, so we compute - // `chunkSize = Math.min(0x20, offset)` and use it for the memory copy - let chunkSize := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + // `step = Math.min(0x20, offset)` and use it for the memory copy + let step := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) // copy len bytes from output to itself. for { let i := 0 } lt(i, len) { - i := add(i, chunkSize) + i := add(i, step) } { mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) } diff --git a/test/utils/compression/FastLZ.t.sol b/test/utils/compression/FastLZ.t.sol index bc4cc4c359e..51367595146 100644 --- a/test/utils/compression/FastLZ.t.sol +++ b/test/utils/compression/FastLZ.t.sol @@ -10,6 +10,14 @@ contract FastLZTest is Test { assertEq(FastLZ.decompress(_flzCompress(input)), input); } + function testEncodeDecodeCalldata(bytes memory input) external view { + assertEq(this.decompressCalldata(_flzCompress(input)), input); + } + + function decompressCalldata(bytes calldata input) external pure returns (bytes memory) { + return FastLZ.decompress(input); + } + /// Copied from solady function _flzCompress(bytes memory input) private pure returns (bytes memory output) { assembly ("memory-safe") { From 73102f847655ce39d18166d9464afd19e713dfb9 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:11:10 +0200 Subject: [PATCH 05/20] doc --- contracts/utils/compression/Snappy.sol | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index 972ff6196b8..b3a85dc223d 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -11,7 +11,9 @@ library Snappy { error DecodingFailure(); /** - * @dev Implementation of Snappy's uncompress function + * @dev Implementation of Snappy's uncompress function. + * + * Based on https://github.com/zhipeng-jia/snappyjs/blob/v0.7.0/snappy_decompressor.js[snappyjs javascript implementation]. */ function uncompress(bytes memory input) internal pure returns (bytes memory output) { bytes4 errorSelector = DecodingFailure.selector; From d09ba3cbf03bd01cb0a38f300a94170683308d8d Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:15:13 +0200 Subject: [PATCH 06/20] codespell --- contracts/utils/compression/Snappy.sol | 4 ++-- test/utils/compression/Snappy.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index b3a85dc223d..f0f009437e8 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -63,7 +63,7 @@ library Snappy { let c := byte(0, w) // consider different cases based on the lower 2 bits of c - // - 0: litteral + // - 0: literal // - 1,2,3: offset copy switch and(c, 0x3) case 0 { @@ -179,7 +179,7 @@ library Snappy { let c := byte(0, w) // consider different cases based on the lower 2 bits of c - // - 0: litteral + // - 0: literal // - 1,2,3: offset copy switch and(c, 0x3) case 0 { diff --git a/test/utils/compression/Snappy.test.js b/test/utils/compression/Snappy.test.js index 536d0f98288..6b6b07ab5bc 100644 --- a/test/utils/compression/Snappy.test.js +++ b/test/utils/compression/Snappy.test.js @@ -47,7 +47,7 @@ Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhon Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ -'; +'; // codespell:ignore }); it('Random buffer', async function () { From 3641f34ae04cb396a163cf5835b0da2ed96e81d5 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:26:35 +0200 Subject: [PATCH 07/20] add LiZip helpers --- contracts/utils/compression/FastLZ.sol | 2 +- test/helpers/LibZip.js | 166 +++++++++++++++++++++++++ test/utils/compression/FastLZ.test.js | 63 ++++++++++ 3 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 test/helpers/LibZip.js create mode 100644 test/utils/compression/FastLZ.test.js diff --git a/contracts/utils/compression/FastLZ.sol b/contracts/utils/compression/FastLZ.sol index 79e8ef1b771..e47c819584e 100644 --- a/contracts/utils/compression/FastLZ.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -72,7 +72,7 @@ library FastLZ { } } - function decompressCallinputPtr(bytes calldata input) internal pure returns (bytes memory output) { + function decompressCalldata(bytes calldata input) internal pure returns (bytes memory output) { assembly ("memory-safe") { // Use new memory allocate at the FMP output := mload(0x40) diff --git a/test/helpers/LibZip.js b/test/helpers/LibZip.js new file mode 100644 index 00000000000..c8efc3cbaa0 --- /dev/null +++ b/test/helpers/LibZip.js @@ -0,0 +1,166 @@ +// See: https://github.com/vectorized/solady/blob/main/src/utils/LibZip.sol + +/** + * FastLZ and calldata compression / decompression functions. + * @namespace + * @alias module:solady.LibZip + */ +var LibZip = {}; + +function hexString(data) { + if (typeof data === "string" || data instanceof String) { + if (data = data.match(/^[\s\uFEFF\xA0]*(0[Xx])?([0-9A-Fa-f]*)[\s\uFEFF\xA0]*$/)) { + if (data[2].length % 2) { + throw new Error("Hex string length must be a multiple of 2."); + } + return data[2]; + } + } + throw new Error("Data must be a hex string."); +} + +function byteToString(b) { + return (b | 0x100).toString(16).slice(1); +} + +function parseByte(data, i) { + return parseInt(data.substr(i, 2), 16); +} + +function hexToBytes(data) { + var a = [], i = 0; + for (; i < data.length; i += 2) a.push(parseByte(data, i)); + return a; +} + +function bytesToHex(a) { + var o = "0x", i = 0; + for (; i < a.length; o += byteToString(a[i++])) ; + return o; +} + +/** + * Compresses hex encoded data with the FastLZ LZ77 algorithm. + * @param {string} data A hex encoded string representing the original data. + * @returns {string} The compressed result as a hex encoded string. + */ +LibZip.flzCompress = function(data) { + var ib = hexToBytes(hexString(data)), b = ib.length - 4; + var ht = [], ob = [], a = 0, i = 2, o = 0, j, s, h, d, c, l, r, p, q, e; + + function u24(i) { + return ib[i] | (ib[++i] << 8) | (ib[++i] << 16); + } + + function hash(x) { + return ((2654435769 * x) >> 19) & 8191; + } + + function literals(r, s) { + while (r >= 32) for (ob[o++] = 31, j = 32; j--; r--) ob[o++] = ib[s++]; + if (r) for (ob[o++] = r - 1; r--; ) ob[o++] = ib[s++]; + } + + while (i < b - 9) { + do { + r = ht[h = hash(s = u24(i))] || 0; + c = (d = (ht[h] = i) - r) < 8192 ? u24(r) : 0x1000000; + } while (i < b - 9 && i++ && s != c); + if (i >= b - 9) break; + if (--i > a) literals(i - a, a); + for (l = 0, p = r + 3, q = i + 3, e = b - q; l < e; l++) e *= ib[p + l] === ib[q + l]; + i += l; + for (--d; l > 262; l -= 262) ob[o++] = 224 + (d >> 8), ob[o++] = 253, ob[o++] = d & 255; + if (l < 7) ob[o++] = (l << 5) + (d >> 8), ob[o++] = d & 255; + else ob[o++] = 224 + (d >> 8), ob[o++] = l - 7, ob[o++] = d & 255; + ht[hash(u24(i))] = i++, ht[hash(u24(i))] = i++, a = i; + } + literals(b + 4 - a, a); + return bytesToHex(ob); +} + +/** + * Decompresses hex encoded data with the FastLZ LZ77 algorithm. + * @param {string} data A hex encoded string representing the compressed data. + * @returns {string} The decompressed result as a hex encoded string. + */ +LibZip.flzDecompress = function(data) { + var ib = hexToBytes(hexString(data)), i = 0, o = 0, l, f, t, r, h, ob = []; + while (i < ib.length) { + if (!(t = ib[i] >> 5)) { + for (l = 1 + ib[i++]; l--;) ob[o++] = ib[i++]; + } else { + f = 256 * (ib[i] & 31) + ib[i + 2 - (t = t < 7)]; + l = t ? 2 + (ib[i] >> 5) : 9 + ib[i + 1]; + i = i + 3 - t; + r = o - f - 1; + while (l--) ob[o++] = ob[r++]; + } + } + return bytesToHex(ob); +} + +/** + * Compresses hex encoded calldata. + * @param {string} data A hex encoded string representing the original data. + * @returns {string} The compressed result as a hex encoded string. + */ +LibZip.cdCompress = function(data) { + data = hexString(data); + var o = "0x", z = 0, y = 0, i = 0, c; + + function pushByte(b) { + o += byteToString(((o.length < 4 * 2 + 2) * 0xff) ^ b); + } + + function rle(v, d) { + pushByte(0x00); + pushByte(d - 1 + v * 0x80); + } + + for (; i < data.length; i += 2) { + c = parseByte(data, i); + if (!c) { + if (y) rle(1, y), y = 0; + if (++z === 0x80) rle(0, 0x80), z = 0; + continue; + } + if (c === 0xff) { + if (z) rle(0, z), z = 0; + if (++y === 0x20) rle(1, 0x20), y = 0; + continue; + } + if (y) rle(1, y), y = 0; + if (z) rle(0, z), z = 0; + pushByte(c); + } + if (y) rle(1, y), y = 0; + if (z) rle(0, z), z = 0; + return o; +} + +/** + * Decompresses hex encoded calldata. + * @param {string} data A hex encoded string representing the compressed data. + * @returns {string} The decompressed result as a hex encoded string. + */ +LibZip.cdDecompress = function(data) { + data = hexString(data); + var o = "0x", i = 0, j, c, s; + + while (i < data.length) { + c = ((i < 4 * 2) * 0xff) ^ parseByte(data, i); + i += 2; + if (!c) { + c = ((i < 4 * 2) * 0xff) ^ parseByte(data, i); + s = (c & 0x7f) + 1; + i += 2; + for (j = 0; j < s; ++j) o += byteToString((c >> 7 && j < 32) * 0xff); + continue; + } + o += byteToString(c); + } + return o; +} + +module.exports = LibZip; \ No newline at end of file diff --git a/test/utils/compression/FastLZ.test.js b/test/utils/compression/FastLZ.test.js new file mode 100644 index 00000000000..abdc4ef68a3 --- /dev/null +++ b/test/utils/compression/FastLZ.test.js @@ -0,0 +1,63 @@ +const { ethers } = require('hardhat'); +const { expect } = require('chai'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const LibZip = require('../../helpers/LibZip'); + +async function fixture() { + const mock = await ethers.deployContract('$FastLZ'); + return { mock }; +} + +// From https://github.com/google/snappy/blob/main/snappy_unittest.cc +const unittests = [ + '', + 'a', + 'ab', + 'abc', + 'aaaaaaa' + 'b'.repeat(16) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(256) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(2047) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcaaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcabcabcabcabcabcab', + 'abcabcabcabcabcabcab0123456789ABCDEF', + 'abcabcabcabcabcabcabcabcabcabcabcabc', + 'abcabcabcabcabcabcabcabcabcabcabcabc0123456789ABCDEF', +]; + +describe('FastLZ', function () { + before(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + describe("Google's unit tests", function () { + for (const [i, str] of Object.entries(unittests)) { + it(`#${i}: length ${str.length}`, async function () { + this.input = str; + }); + } + }); + + it('Lorem ipsum...', async function () { + this.input = + '\ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ +Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ +Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ +Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ +Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ +'; // codespell:ignore + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + const compressed = LibZip.flzCompress(hex); + await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); + await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); + }); +}); From 4d2eeb3dbcf28bac2ff9121242048717b139e93b Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:55:55 +0200 Subject: [PATCH 08/20] Apply suggestions from code review --- contracts/utils/compression/FastLZ.sol | 4 +++- contracts/utils/compression/Snappy.sol | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/contracts/utils/compression/FastLZ.sol b/contracts/utils/compression/FastLZ.sol index e47c819584e..77ed1425cdf 100644 --- a/contracts/utils/compression/FastLZ.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -3,7 +3,9 @@ pragma solidity ^0.8.20; /** - * @dev Library for compressing and decompressing buffers using FastLZ. + * @dev Library for decompressing data using FastLZ. + * + * See https://fr.wikipedia.org/wiki/FastLZ */ library FastLZ { /** diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index f0f009437e8..a181483d23f 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -3,7 +3,7 @@ pragma solidity ^0.8.20; /** - * @dev Library for compressing and decompressing buffers using Snappy. + * @dev Library for decompressing data using Snappy. * * See https://github.com/google/snappy */ From b341240043450f8831f4c55db628df58cc2e7282 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 11:57:00 +0200 Subject: [PATCH 09/20] Update contracts/utils/compression/FastLZ.sol --- contracts/utils/compression/FastLZ.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/utils/compression/FastLZ.sol b/contracts/utils/compression/FastLZ.sol index 77ed1425cdf..3d20f56a482 100644 --- a/contracts/utils/compression/FastLZ.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -5,7 +5,7 @@ pragma solidity ^0.8.20; /** * @dev Library for decompressing data using FastLZ. * - * See https://fr.wikipedia.org/wiki/FastLZ + * See https://ariya.github.io/FastLZ/ */ library FastLZ { /** From 92a232e262bbcf7d95f8b1c6b5fd139b0f815050 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 14:46:53 +0200 Subject: [PATCH 10/20] use mcopy --- contracts/utils/compression/Snappy.sol | 41 +++++++++++--------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index a181483d23f..78b2d415876 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -1,6 +1,6 @@ // SPDX-License-Identifier: MIT -pragma solidity ^0.8.20; +pragma solidity ^0.8.24; /** * @dev Library for decompressing data using Snappy. @@ -76,14 +76,7 @@ library Snappy { inputPtr := add(inputPtr, smallLen) } assert(not(gt(add(inputPtr, len), inputEnd)), errorSelector) - // copy len bytes from input to output in chunks of 32 bytes - for { - let i := 0 - } lt(i, len) { - i := add(i, 0x20) - } { - mstore(add(outputPtr, i), mload(add(inputPtr, i))) - } + mcopy(outputPtr, inputPtr, len) inputPtr := add(inputPtr, len) outputPtr := add(outputPtr, len) @@ -110,17 +103,18 @@ library Snappy { } assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) - // copying in chunks will not work if the offset is larger than the chunk length, so we compute - // `step = Math.min(0x20, offset)` and use it for the memory copy - let step := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + // copying in will not work if the offset is larger than the len being copied, so we compute + // `step = Math.min(len, offset)` and use it for the memory copy in chunks + let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) // copy len bytes from output to itself. for { - let i := 0 - } lt(i, len) { - i := add(i, step) + let ptr := outputPtr + let end := add(outputPtr, len) + } lt(ptr, end) { + ptr := add(ptr, step) } { - mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) + mcopy(ptr, sub(ptr, offset), step) } outputPtr := add(outputPtr, len) } @@ -220,17 +214,18 @@ library Snappy { } assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) - // copying in chunks will not work if the offset is larger than the chunk length, so we compute - // `step = Math.min(0x20, offset)` and use it for the memory copy - let step := xor(0x20, mul(lt(offset, 0x20), xor(0x20, offset))) + // copying in will not work if the offset is larger than the len being copied, so we compute + // `step = Math.min(len, offset)` and use it for the memory copy in chunks + let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) // copy len bytes from output to itself. for { - let i := 0 - } lt(i, len) { - i := add(i, step) + let ptr := outputPtr + let end := add(outputPtr, len) + } lt(ptr, end) { + ptr := add(ptr, step) } { - mstore(add(outputPtr, i), mload(sub(add(outputPtr, i), offset))) + mcopy(ptr, sub(ptr, offset), step) } outputPtr := add(outputPtr, len) } From c5c85534a1ed428a0191a15a30ee3fb53f3e3d24 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Wed, 30 Jul 2025 14:54:27 +0200 Subject: [PATCH 11/20] up --- contracts/utils/compression/Snappy.sol | 28 +++----------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index 78b2d415876..12e7d936daa 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -26,11 +26,9 @@ library Snappy { revert(0, 0x04) } } - // input buffer bounds let inputBegin := add(input, 0x20) let inputEnd := add(inputBegin, mload(input)) - // input traversal pointer let inputPtr := inputBegin // read length of the decompressed buffer @@ -45,13 +43,11 @@ library Snappy { } } inputPtr := add(inputPtr, 1) - // allocated output buffer output := mload(0x40) let outputPtr := add(output, 0x20) mstore(output, outputLength) mstore(0x40, add(outputPtr, outputLength)) - // decompress input buffer into output buffer for { let len, offset @@ -59,9 +55,7 @@ library Snappy { // get next (compressed) word -- used as a cache for further reads let w := mload(inputPtr) inputPtr := add(inputPtr, 1) - let c := byte(0, w) - // consider different cases based on the lower 2 bits of c // - 0: literal // - 1,2,3: offset copy @@ -79,7 +73,6 @@ library Snappy { mcopy(outputPtr, inputPtr, len) inputPtr := add(inputPtr, len) outputPtr := add(outputPtr, len) - // continue to skip the offset copy logic that is shared by the other 3 cases continue } @@ -102,15 +95,12 @@ library Snappy { inputPtr := add(inputPtr, 4) } assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) - // copying in will not work if the offset is larger than the len being copied, so we compute // `step = Math.min(len, offset)` and use it for the memory copy in chunks - let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) - - // copy len bytes from output to itself. for { let ptr := outputPtr let end := add(outputPtr, len) + let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) } lt(ptr, end) { ptr := add(ptr, step) } { @@ -118,7 +108,6 @@ library Snappy { } outputPtr := add(outputPtr, len) } - // sanity check, FMP is at the right location assert(eq(outputPtr, mload(0x40)), errorSelector) } @@ -136,11 +125,9 @@ library Snappy { revert(0, 0x04) } } - // input buffer bounds let inputBegin := input.offset let inputEnd := add(inputBegin, input.length) - // input traversal pointer let inputPtr := inputBegin // read length of the decompressed buffer @@ -155,13 +142,11 @@ library Snappy { } } inputPtr := add(inputPtr, 1) - // allocated output buffer output := mload(0x40) let outputPtr := add(output, 0x20) mstore(output, outputLength) mstore(0x40, add(outputPtr, outputLength)) - // decompress input buffer into output buffer for { let len, offset @@ -169,12 +154,10 @@ library Snappy { // get next (compressed) word -- used as a cache for further reads let w := calldataload(inputPtr) inputPtr := add(inputPtr, 1) - let c := byte(0, w) - // consider different cases based on the lower 2 bits of c // - 0: literal - // - 1,2,3: offset copy + // - 1, 2, 3: offset copy switch and(c, 0x3) case 0 { len := add(shr(2, c), 1) @@ -190,7 +173,6 @@ library Snappy { calldatacopy(outputPtr, inputPtr, len) inputPtr := add(inputPtr, len) outputPtr := add(outputPtr, len) - // continue to skip the offset copy logic that is shared by the other 3 cases continue } @@ -213,15 +195,12 @@ library Snappy { inputPtr := add(inputPtr, 4) } assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) - // copying in will not work if the offset is larger than the len being copied, so we compute // `step = Math.min(len, offset)` and use it for the memory copy in chunks - let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) - - // copy len bytes from output to itself. for { let ptr := outputPtr let end := add(outputPtr, len) + let step := xor(offset, mul(lt(len, offset), xor(len, offset))) // min(len, offset) } lt(ptr, end) { ptr := add(ptr, step) } { @@ -229,7 +208,6 @@ library Snappy { } outputPtr := add(outputPtr, len) } - // sanity check, FMP is at the right location assert(eq(outputPtr, mload(0x40)), errorSelector) } From b06e8164168d961540820fae8114c211af127e01 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 13:37:59 +0200 Subject: [PATCH 12/20] fix tests --- test/utils/compression/FastLZ.test.js | 36 +++++++++++++-------------- test/utils/compression/Snappy.test.js | 36 +++++++++++++-------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/test/utils/compression/FastLZ.test.js b/test/utils/compression/FastLZ.test.js index abdc4ef68a3..340ffcfff18 100644 --- a/test/utils/compression/FastLZ.test.js +++ b/test/utils/compression/FastLZ.test.js @@ -31,33 +31,33 @@ describe('FastLZ', function () { Object.assign(this, await loadFixture(fixture)); }); - describe("Google's unit tests", function () { + describe("uncompress", function () { for (const [i, str] of Object.entries(unittests)) { - it(`#${i}: length ${str.length}`, async function () { + it(`Google's unit tests #${i}: length ${str.length}`, async function () { this.input = str; }); } - }); - it('Lorem ipsum...', async function () { - this.input = - '\ + it('Lorem ipsum...', async function () { + this.input = +'\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ -'; // codespell:ignore - }); - - it('Random buffer', async function () { - this.input = ethers.randomBytes(4096); - }); - - afterEach(async function () { - const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); - const compressed = LibZip.flzCompress(hex); - await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); - await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); +'; + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + const compressed = LibZip.flzCompress(hex); + await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); + await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); + }); }); }); diff --git a/test/utils/compression/Snappy.test.js b/test/utils/compression/Snappy.test.js index 6b6b07ab5bc..0656f36c25d 100644 --- a/test/utils/compression/Snappy.test.js +++ b/test/utils/compression/Snappy.test.js @@ -31,33 +31,33 @@ describe('Snappy', function () { Object.assign(this, await loadFixture(fixture)); }); - describe("Google's unit tests", function () { + describe("uncompress", function () { for (const [i, str] of Object.entries(unittests)) { - it(`#${i}: length ${str.length}`, async function () { + it(`Google's unit tests #${i}: length ${str.length}`, async function () { this.input = str; }); } - }); - it('Lorem ipsum...', async function () { - this.input = - '\ + it('Lorem ipsum...', async function () { + this.input = +'\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ -'; // codespell:ignore - }); - - it('Random buffer', async function () { - this.input = ethers.randomBytes(4096); - }); - - afterEach(async function () { - const compressed = snappy.compressSync(this.input); - const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); - await expect(this.mock.$uncompress(compressed)).to.eventually.equal(hex); - await expect(this.mock.$uncompressCalldata(compressed)).to.eventually.equal(hex); +'; + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const compressed = snappy.compressSync(this.input); + const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + await expect(this.mock.$uncompress(compressed)).to.eventually.equal(hex); + await expect(this.mock.$uncompressCalldata(compressed)).to.eventually.equal(hex); + }); }); }); From e8139bf23bfc62b084340ca8b1131bc6b0770d80 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 15:37:14 +0200 Subject: [PATCH 13/20] Add LZ4 --- contracts/utils/compression/LZ4.sol | 173 ++++++++++++++++++++++++++ package-lock.json | 8 ++ package.json | 1 + test/utils/compression/FastLZ.test.js | 7 +- test/utils/compression/LZ4.test.js | 64 ++++++++++ test/utils/compression/Snappy.test.js | 9 +- 6 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 contracts/utils/compression/LZ4.sol create mode 100644 test/utils/compression/LZ4.test.js diff --git a/contracts/utils/compression/LZ4.sol b/contracts/utils/compression/LZ4.sol new file mode 100644 index 00000000000..1658ac09045 --- /dev/null +++ b/contracts/utils/compression/LZ4.sol @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: MIT + +pragma solidity ^0.8.24; + +/** + * @dev Library for decompressing data using LZ4. + * + * See https://lz4.org/ + */ +library LZ4 { + // Compression format parameters/constants. + uint256 private constant MIN_MATCH = 4; + // Frame constants + uint32 private constant MAGIC_NUM = 0x04224d18; // reversed endianness (first 4 bytes of the frame) + // Frame descriptor flags + uint8 private constant FD_CONTENT_SIZE = 0x08; + uint8 private constant FD_BLOCK_CHKSUM = 0x10; + uint8 private constant FD_VERSION_MASK = 0xC0; + uint8 private constant FD_VERSION = 0x40; + // Block sizes + uint32 private constant BS_UNCOMPRESSED = 0x80000000; + uint8 private constant BS_SHIFT = 0x04; + uint8 private constant BS_MASK = 0x07; + + /** + * @dev Implementation of LZ4's decompress function. + * + * See https://github.com/Benzinga/lz4js/blob/master/lz4.js + */ + function decompress(bytes memory input) internal pure returns (bytes memory output) { + assembly ("memory-safe") { + function assert(b, e) { + if iszero(b) { + mstore(0, e) + revert(0, 0x04) + } + } + // load 16 bytes from a given location in memory, right aligned and in reverse order + function readU16(ptr) -> value { + value := mload(ptr) + value := or(byte(0, value), shl(8, byte(1, value))) + } + // load 32 bytes from a given location in memory, right aligned and in reverse order + function readU32(ptr) -> value { + value := mload(ptr) + value := or( + or(byte(0, value), shl(8, byte(1, value))), + or(shl(16, byte(2, value)), shl(24, byte(3, value))) + ) + } + + // input buffer + let inputPtr := add(input, 0x20) + // let inputEnd := add(inputPtr, mload(input)) // TODO: use to check bounds + + // output buffer + output := mload(0x40) + let outputPtr := add(output, 0x20) + + // ========================================== decompress frame =========================================== + // TODO CHECK LENGTH BEFORE + + // Get header (size must be at least 7 / 15 depending on useContentSize ) + // [ magic (4) ++ descriptor (1) ++ bsIds (1) ++ contentsize (0 or 8) ++ ??? (1) ] + let header := mload(inputPtr) + + // read magic number (first 4 bytes, realigned right) + assert(eq(shr(224, header), MAGIC_NUM), 0x00000001) // TODO: error code + + // read descriptor and check version + let descriptor := byte(4, header) + assert(eq(and(descriptor, FD_VERSION_MASK), FD_VERSION), 0x00000002) // TODO: error code + + // read flags + let useBlockSum := eq(and(descriptor, FD_BLOCK_CHKSUM), FD_BLOCK_CHKSUM) + let useContentSize := eq(and(descriptor, FD_CONTENT_SIZE), FD_CONTENT_SIZE) + + // read block size + // let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) + // TODO: check bsMap? value should probably be in [4, 7], otherwise unused + + // move forward 7 bytes + inputPtr := add(inputPtr, add(7, mul(useContentSize, 8))) + + // read blocks + for {} 1 {} { + let compSize := readU32(inputPtr) + + if iszero(compSize) { + break + } + + // read block checksum if "useBlockSum" is true? + inputPtr := add(inputPtr, add(4, mul(useBlockSum, 4))) + + // check if block is compressed + switch iszero(and(compSize, BS_UNCOMPRESSED)) + case 0 { + // mask off the 'uncompressed' bit + compSize := and(compSize, not(BS_UNCOMPRESSED)) + // copy uncompressed chunk + mcopy(outputPtr, inputPtr, compSize) + inputPtr := add(inputPtr, compSize) + outputPtr := add(outputPtr, compSize) + } + case 1 { + for { + let blockEnd := add(inputPtr, compSize) + } lt(inputPtr, blockEnd) {} { + let token := byte(0, mload(inputPtr)) + + // copy literals. + let literalCount := shr(4, token) + if literalCount { + // Parse length. + if eq(literalCount, 0xf) { + for {} 1 {} { + let count := byte(0, mload(inputPtr)) + inputPtr := add(inputPtr, 1) + literalCount := add(literalCount, count) + if lt(count, 0xff) { + break + } + } + } + mcopy(outputPtr, inputPtr, literalCount) + outputPtr := add(outputPtr, literalCount) + } + // 1 for the token + literalCount (possibly 0) read bytes + inputPtr := add(inputPtr, add(literalCount, 1)) + + if lt(inputPtr, blockEnd) { + // Copy match. + let mLength := and(token, 0xf) + + // Parse offset. + let mOffset := readU16(inputPtr) + inputPtr := add(inputPtr, 2) + + // Parse length. + if eq(mLength, 0xf) { + for {} 1 {} { + let count := byte(0, mload(inputPtr)) + inputPtr := add(inputPtr, 1) + mLength := add(mLength, count) + if lt(count, 0xff) { + break + } + } + } + mLength := add(mLength, MIN_MATCH) + + for { + let ptr := outputPtr + let end := add(outputPtr, mLength) + let step := xor(mOffset, mul(lt(mLength, mOffset), xor(mLength, mOffset))) // min(mLength, mOffset) + } lt(ptr, end) { + ptr := add(ptr, step) + } { + mcopy(ptr, sub(ptr, mOffset), step) + } + outputPtr := add(outputPtr, mLength) + } + } + } + } + + // reserve used memory + mstore(output, sub(outputPtr, add(output, 0x20))) + mstore(0x40, outputPtr) + } + } +} diff --git a/package-lock.json b/package-lock.json index 4806ffb5adf..7f666de7edc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "interoperable-addresses": "^0.1.3", "lint-staged": "^16.0.0", "lodash.startcase": "^4.4.0", + "lz4js": "^0.2.0", "micromatch": "^4.0.2", "p-limit": "^6.0.0", "prettier": "^3.0.0", @@ -7648,6 +7649,13 @@ "node": "20 || >=22" } }, + "node_modules/lz4js": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/lz4js/-/lz4js-0.2.0.tgz", + "integrity": "sha512-gY2Ia9Lm7Ep8qMiuGRhvUq0Q7qUereeldZPP1PMEJxPtEWHJLqw9pgX68oHajBH0nzJK4MaZEA/YNV3jT8u8Bg==", + "dev": true, + "license": "ISC" + }, "node_modules/markdown-table": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-2.0.0.tgz", diff --git a/package.json b/package.json index cecde3025ae..ea7b0c17ac6 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "interoperable-addresses": "^0.1.3", "lint-staged": "^16.0.0", "lodash.startcase": "^4.4.0", + "lz4js": "^0.2.0", "micromatch": "^4.0.2", "p-limit": "^6.0.0", "prettier": "^3.0.0", diff --git a/test/utils/compression/FastLZ.test.js b/test/utils/compression/FastLZ.test.js index 340ffcfff18..be71652724f 100644 --- a/test/utils/compression/FastLZ.test.js +++ b/test/utils/compression/FastLZ.test.js @@ -31,7 +31,7 @@ describe('FastLZ', function () { Object.assign(this, await loadFixture(fixture)); }); - describe("uncompress", function () { + describe('decompress', function () { for (const [i, str] of Object.entries(unittests)) { it(`Google's unit tests #${i}: length ${str.length}`, async function () { this.input = str; @@ -40,7 +40,7 @@ describe('FastLZ', function () { it('Lorem ipsum...', async function () { this.input = -'\ + '\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ @@ -54,7 +54,8 @@ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula }); afterEach(async function () { - const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + const raw = ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input); + const hex = ethers.hexlify(raw); const compressed = LibZip.flzCompress(hex); await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); diff --git a/test/utils/compression/LZ4.test.js b/test/utils/compression/LZ4.test.js new file mode 100644 index 00000000000..b9d18d50065 --- /dev/null +++ b/test/utils/compression/LZ4.test.js @@ -0,0 +1,64 @@ +const { ethers } = require('hardhat'); +const { expect } = require('chai'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const lz4js = require('lz4js'); + +async function fixture() { + const mock = await ethers.deployContract('$LZ4'); + return { mock }; +} + +// From https://github.com/google/snappy/blob/main/snappy_unittest.cc +const unittests = [ + '', + 'a', + 'ab', + 'abc', + 'aaaaaaa' + 'b'.repeat(16) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(256) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(2047) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcaaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcabcabcabcabcabcab', + 'abcabcabcabcabcabcab0123456789ABCDEF', + 'abcabcabcabcabcabcabcabcabcabcabcabc', + 'abcabcabcabcabcabcabcabcabcabcabcabc0123456789ABCDEF', +]; + +describe('LZ4', function () { + before(async function () { + Object.assign(this, await loadFixture(fixture)); + }); + + describe('decompress', function () { + for (const [i, str] of Object.entries(unittests)) { + it(`Google's unit tests #${i}: length ${str.length}`, async function () { + this.input = str; + }); + } + + it('Lorem ipsum...', async function () { + this.input = + '\ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ +Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ +Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ +Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ +Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ +'; + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const raw = ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input); + const hex = ethers.hexlify(raw); + const compressed = lz4js.compress(raw); + await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); + // await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); + }); + }); +}); diff --git a/test/utils/compression/Snappy.test.js b/test/utils/compression/Snappy.test.js index 0656f36c25d..fea3c1d51ff 100644 --- a/test/utils/compression/Snappy.test.js +++ b/test/utils/compression/Snappy.test.js @@ -31,7 +31,7 @@ describe('Snappy', function () { Object.assign(this, await loadFixture(fixture)); }); - describe("uncompress", function () { + describe('uncompress', function () { for (const [i, str] of Object.entries(unittests)) { it(`Google's unit tests #${i}: length ${str.length}`, async function () { this.input = str; @@ -40,7 +40,7 @@ describe('Snappy', function () { it('Lorem ipsum...', async function () { this.input = -'\ + '\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ @@ -54,8 +54,9 @@ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula }); afterEach(async function () { - const compressed = snappy.compressSync(this.input); - const hex = ethers.hexlify(ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input)); + const raw = ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input); + const hex = ethers.hexlify(raw); + const compressed = snappy.compressSync(raw); await expect(this.mock.$uncompress(compressed)).to.eventually.equal(hex); await expect(this.mock.$uncompressCalldata(compressed)).to.eventually.equal(hex); }); From ffdc0d21f1814ba510920b648e9a11d40c867dd4 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 15:58:02 +0200 Subject: [PATCH 14/20] fix LZ4 + gas comparaison --- contracts/utils/compression/LZ4.sol | 4 +- test/utils/compression/gas.test.js | 86 +++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 test/utils/compression/gas.test.js diff --git a/contracts/utils/compression/LZ4.sol b/contracts/utils/compression/LZ4.sol index 1658ac09045..9bf10b78e0f 100644 --- a/contracts/utils/compression/LZ4.sol +++ b/contracts/utils/compression/LZ4.sol @@ -108,6 +108,7 @@ library LZ4 { let blockEnd := add(inputPtr, compSize) } lt(inputPtr, blockEnd) {} { let token := byte(0, mload(inputPtr)) + inputPtr := add(inputPtr, 1) // copy literals. let literalCount := shr(4, token) @@ -124,10 +125,9 @@ library LZ4 { } } mcopy(outputPtr, inputPtr, literalCount) + inputPtr := add(inputPtr, literalCount) outputPtr := add(outputPtr, literalCount) } - // 1 for the token + literalCount (possibly 0) read bytes - inputPtr := add(inputPtr, add(literalCount, 1)) if lt(inputPtr, blockEnd) { // Copy match. diff --git a/test/utils/compression/gas.test.js b/test/utils/compression/gas.test.js new file mode 100644 index 00000000000..179757cffb0 --- /dev/null +++ b/test/utils/compression/gas.test.js @@ -0,0 +1,86 @@ +const { ethers } = require('hardhat'); +const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); + +const lz4js = require('lz4js'); +const snappy = require('snappy'); +const LibZip = require('../../helpers/LibZip'); +const { min } = require('../../helpers/math'); + +async function fixture() { + const fastlz = await ethers.deployContract('$FastLZ'); + const lz4 = await ethers.deployContract('$LZ4'); + const snappy = await ethers.deployContract('$Snappy'); + return { fastlz, lz4, snappy }; +} + +// From https://github.com/google/snappy/blob/main/snappy_unittest.cc +const unittests = [ + '', + 'a', + 'ab', + 'abc', + 'aaaaaaa' + 'b'.repeat(16) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(256) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(2047) + 'aaaaa' + 'abc', + 'aaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcaaaaaaa' + 'b'.repeat(65536) + 'aaaaa' + 'abc', + 'abcabcabcabcabcabcab', + 'abcabcabcabcabcabcab0123456789ABCDEF', + 'abcabcabcabcabcabcabcabcabcabcabcabc', + 'abcabcabcabcabcabcabcabcabcabcabcabc0123456789ABCDEF', +]; + +describe('Gas comparaison of decompression algorithms', function () { + before(async function () { + Object.assign(this, await loadFixture(fixture)); + this.results = []; + }); + + describe('decompress', function () { + for (const [i, str] of Object.entries(unittests)) { + it(`Google's unit tests #${i}: length ${str.length}`, async function () { + this.input = str; + }); + } + + it('Lorem ipsum...', async function () { + this.input = + '\ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas ligula urna, bibendum sagittis eleifend non, rutrum sit amet lectus. Donec eu pellentesque dolor, varius lobortis erat. In viverra diam in nunc porta, at pretium orci hendrerit. Duis suscipit lacus eu sodales imperdiet. Donec rhoncus tincidunt sem sed laoreet. Suspendisse potenti. Suspendisse a dictum diam, a porttitor augue. Praesent sodales quis nisi sed auctor. Nullam efficitur est eros, a tincidunt velit faucibus consequat. Praesent urna leo, imperdiet ut mi eu, pellentesque mattis ante. Suspendisse cursus lacus ac urna egestas, vitae ultricies ante porttitor. In sed risus vitae nunc faucibus tristique.\ +Quisque aliquet bibendum augue, et tristique lorem pellentesque quis. Nulla rhoncus erat sed velit luctus, in cursus neque suscipit. Quisque sit amet mauris nec enim congue sagittis eu nec diam. Quisque a enim a leo aliquam vestibulum a ut risus. In hendrerit cursus nisl, et porttitor dolor volutpat non. Donec rhoncus, nisl ut blandit porta, libero felis vulputate ante, et pharetra ex risus et enim. Vestibulum eu ultricies ipsum, quis auctor odio. Morbi ornare metus nec purus elementum, eu interdum magna dapibus. Aliquam odio ipsum, semper in nisl tristique, fermentum porta risus. Curabitur facilisis felis a molestie dignissim. Pellentesque aliquet sagittis sodales. Fusce at dignissim mi. Nulla a tempus quam.\ +Nam et egestas quam. Aliquam bibendum iaculis mauris a sagittis. Suspendisse tincidunt, magna vitae scelerisque pharetra, orci nisi venenatis est, sit amet consequat ligula dolor eu felis. Nulla suscipit eleifend augue, et commodo elit lobortis eget. Integer pharetra commodo metus, at accumsan arcu porttitor sed. Ut eu nulla sit amet diam imperdiet fermentum id in erat. Curabitur at neque ornare neque dictum malesuada a nec enim. Ut ac aliquam mauris, eu pretium urna. Donec vitae leo eros. Phasellus et purus rhoncus, accumsan ligula vel, sagittis lectus. Mauris sed lectus elementum, porta nisl eget, convallis ligula. Aenean pellentesque arcu ac lacus scelerisque sollicitudin. Nunc vitae enim egestas, sollicitudin ipsum vulputate, fringilla urna. Aenean eget libero sollicitudin, sagittis lorem in, convallis nibh.\ +Cras cursus luctus malesuada. Sed dictum, sem feugiat placerat placerat, nisl neque blandit enim, quis semper mauris augue quis lacus. Interdum et malesuada fames ac ante ipsum primis in faucibus. Pellentesque dignissim quis est et auctor. Etiam porttitor facilisis nibh eget luctus. Etiam at congue neque. Donec a odio varius, rhoncus metus ac, bibendum est. Nullam nisl tortor, egestas id quam sed, hendrerit lobortis diam. Phasellus eros sapien, hendrerit nec ex nec, convallis ullamcorper nibh. Integer tempor hendrerit auctor. Duis ut orci iaculis, tincidunt dui eget, faucibus magna. Pellentesque sit amet eros ac nibh pulvinar volutpat. In ligula felis, hendrerit non congue finibus, tincidunt a nibh. Morbi suscipit dui orci, eget volutpat odio malesuada in.\ +Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula lectus. Vestibulum commodo massa nec turpis viverra, nec tempor velit convallis. Etiam egestas quam ut justo rhoncus porta. Morbi viverra mi dui, mattis feugiat neque pulvinar laoreet. Curabitur pulvinar mi vitae nisi sodales tristique. Nunc vulputate maximus ante ac venenatis.\ +'; + }); + + it('Random buffer', async function () { + this.input = ethers.randomBytes(4096); + }); + + afterEach(async function () { + const raw = ethers.isBytesLike(this.input) ? this.input : ethers.toUtf8Bytes(this.input); + const hex = ethers.hexlify(raw); + + const compressedFastlz = LibZip.flzCompress(hex); + const compressedSnappy = snappy.compressSync(raw); + const compressedLz4 = lz4js.compress(raw); + + const gasUsedFastlz = await this.fastlz.$decompress.estimateGas(compressedFastlz).then(Number); + const gasUsedSnappy = await this.snappy.$uncompress.estimateGas(compressedSnappy).then(Number); + const gasUsedLz4 = await this.lz4.$decompress.estimateGas(compressedLz4).then(Number); + const lowest = min(gasUsedFastlz, gasUsedSnappy, gasUsedLz4); + + this.results.push({ + lowest, + extraGasUsedPercentageFastlz: (100 * (gasUsedFastlz - lowest)) / lowest, + extraGasUsedPercentageSnappy: (100 * (gasUsedSnappy - lowest)) / lowest, + extraGasUsedPercentageLz4: (100 * (gasUsedLz4 - lowest)) / lowest, + }); + }); + + after(async function () { + console.table(this.results); + }); + }); +}); From 036679712a12e8e05f078fa93ef4dfd1844eae81 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 16:35:31 +0200 Subject: [PATCH 15/20] add stateless --- contracts/mocks/Stateless.sol | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contracts/mocks/Stateless.sol b/contracts/mocks/Stateless.sol index fd0cf398eb0..bc346351edd 100644 --- a/contracts/mocks/Stateless.sol +++ b/contracts/mocks/Stateless.sol @@ -31,9 +31,12 @@ import {ERC4337Utils} from "../account/utils/draft-ERC4337Utils.sol"; import {ERC7579Utils} from "../account/utils/draft-ERC7579Utils.sol"; import {ERC7913P256Verifier} from "../utils/cryptography/verifiers/ERC7913P256Verifier.sol"; import {ERC7913RSAVerifier} from "../utils/cryptography/verifiers/ERC7913RSAVerifier.sol"; +import {FastLZ} from "../utils/compression/FastLZ.sol"; import {Heap} from "../utils/structs/Heap.sol"; import {InteroperableAddress} from "../utils/draft-InteroperableAddress.sol"; +import {LZ4} from "../utils/compression/LZ4.sol"; import {Math} from "../utils/math/Math.sol"; +import {Memory} from "../utils/Memory.sol"; import {MerkleProof} from "../utils/cryptography/MerkleProof.sol"; import {MessageHashUtils} from "../utils/cryptography/MessageHashUtils.sol"; import {Nonces} from "../utils/Nonces.sol"; @@ -47,9 +50,9 @@ import {SafeERC20} from "../token/ERC20/utils/SafeERC20.sol"; import {ShortStrings} from "../utils/ShortStrings.sol"; import {SignatureChecker} from "../utils/cryptography/SignatureChecker.sol"; import {SignedMath} from "../utils/math/SignedMath.sol"; +import {Snappy} from "../utils/compression/Snappy.sol"; import {StorageSlot} from "../utils/StorageSlot.sol"; import {Strings} from "../utils/Strings.sol"; -import {Memory} from "../utils/Memory.sol"; import {Time} from "../utils/types/Time.sol"; contract Dummy1234 {} From 775a991b9d9ec7e737f90620708e81c4112bd430 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 16:38:37 +0200 Subject: [PATCH 16/20] add solady as a dev dependency to use LibZip from solady/js/solady --- package-lock.json | 8 ++ package.json | 1 + test/helpers/LibZip.js | 166 -------------------------- test/utils/compression/FastLZ.test.js | 2 +- test/utils/compression/gas.test.js | 2 +- 5 files changed, 11 insertions(+), 168 deletions(-) delete mode 100644 test/helpers/LibZip.js diff --git a/package-lock.json b/package-lock.json index 7f666de7edc..d1769492028 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "rimraf": "^6.0.0", "semver": "^7.3.5", "snappy": "^7.3.0", + "solady": "^0.1.24", "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", @@ -9746,6 +9747,13 @@ "@napi-rs/snappy-win32-x64-msvc": "7.3.0" } }, + "node_modules/solady": { + "version": "0.1.24", + "resolved": "https://registry.npmjs.org/solady/-/solady-0.1.24.tgz", + "integrity": "sha512-uFTtYane4KMn2Tbth+7f8svTOrQ5+SUksFyTA9Vqwffwxak7OduHZxBYxSz34foBGnbsKtleM2FbgiAP1NYB1A==", + "dev": true, + "license": "MIT" + }, "node_modules/solc": { "version": "0.8.26", "resolved": "https://registry.npmjs.org/solc/-/solc-0.8.26.tgz", diff --git a/package.json b/package.json index ea7b0c17ac6..100fde02b63 100644 --- a/package.json +++ b/package.json @@ -87,6 +87,7 @@ "rimraf": "^6.0.0", "semver": "^7.3.5", "snappy": "^7.3.0", + "solady": "^0.1.24", "solhint": "^6.0.0", "solhint-plugin-openzeppelin": "file:scripts/solhint-custom", "solidity-ast": "^0.4.50", diff --git a/test/helpers/LibZip.js b/test/helpers/LibZip.js deleted file mode 100644 index c8efc3cbaa0..00000000000 --- a/test/helpers/LibZip.js +++ /dev/null @@ -1,166 +0,0 @@ -// See: https://github.com/vectorized/solady/blob/main/src/utils/LibZip.sol - -/** - * FastLZ and calldata compression / decompression functions. - * @namespace - * @alias module:solady.LibZip - */ -var LibZip = {}; - -function hexString(data) { - if (typeof data === "string" || data instanceof String) { - if (data = data.match(/^[\s\uFEFF\xA0]*(0[Xx])?([0-9A-Fa-f]*)[\s\uFEFF\xA0]*$/)) { - if (data[2].length % 2) { - throw new Error("Hex string length must be a multiple of 2."); - } - return data[2]; - } - } - throw new Error("Data must be a hex string."); -} - -function byteToString(b) { - return (b | 0x100).toString(16).slice(1); -} - -function parseByte(data, i) { - return parseInt(data.substr(i, 2), 16); -} - -function hexToBytes(data) { - var a = [], i = 0; - for (; i < data.length; i += 2) a.push(parseByte(data, i)); - return a; -} - -function bytesToHex(a) { - var o = "0x", i = 0; - for (; i < a.length; o += byteToString(a[i++])) ; - return o; -} - -/** - * Compresses hex encoded data with the FastLZ LZ77 algorithm. - * @param {string} data A hex encoded string representing the original data. - * @returns {string} The compressed result as a hex encoded string. - */ -LibZip.flzCompress = function(data) { - var ib = hexToBytes(hexString(data)), b = ib.length - 4; - var ht = [], ob = [], a = 0, i = 2, o = 0, j, s, h, d, c, l, r, p, q, e; - - function u24(i) { - return ib[i] | (ib[++i] << 8) | (ib[++i] << 16); - } - - function hash(x) { - return ((2654435769 * x) >> 19) & 8191; - } - - function literals(r, s) { - while (r >= 32) for (ob[o++] = 31, j = 32; j--; r--) ob[o++] = ib[s++]; - if (r) for (ob[o++] = r - 1; r--; ) ob[o++] = ib[s++]; - } - - while (i < b - 9) { - do { - r = ht[h = hash(s = u24(i))] || 0; - c = (d = (ht[h] = i) - r) < 8192 ? u24(r) : 0x1000000; - } while (i < b - 9 && i++ && s != c); - if (i >= b - 9) break; - if (--i > a) literals(i - a, a); - for (l = 0, p = r + 3, q = i + 3, e = b - q; l < e; l++) e *= ib[p + l] === ib[q + l]; - i += l; - for (--d; l > 262; l -= 262) ob[o++] = 224 + (d >> 8), ob[o++] = 253, ob[o++] = d & 255; - if (l < 7) ob[o++] = (l << 5) + (d >> 8), ob[o++] = d & 255; - else ob[o++] = 224 + (d >> 8), ob[o++] = l - 7, ob[o++] = d & 255; - ht[hash(u24(i))] = i++, ht[hash(u24(i))] = i++, a = i; - } - literals(b + 4 - a, a); - return bytesToHex(ob); -} - -/** - * Decompresses hex encoded data with the FastLZ LZ77 algorithm. - * @param {string} data A hex encoded string representing the compressed data. - * @returns {string} The decompressed result as a hex encoded string. - */ -LibZip.flzDecompress = function(data) { - var ib = hexToBytes(hexString(data)), i = 0, o = 0, l, f, t, r, h, ob = []; - while (i < ib.length) { - if (!(t = ib[i] >> 5)) { - for (l = 1 + ib[i++]; l--;) ob[o++] = ib[i++]; - } else { - f = 256 * (ib[i] & 31) + ib[i + 2 - (t = t < 7)]; - l = t ? 2 + (ib[i] >> 5) : 9 + ib[i + 1]; - i = i + 3 - t; - r = o - f - 1; - while (l--) ob[o++] = ob[r++]; - } - } - return bytesToHex(ob); -} - -/** - * Compresses hex encoded calldata. - * @param {string} data A hex encoded string representing the original data. - * @returns {string} The compressed result as a hex encoded string. - */ -LibZip.cdCompress = function(data) { - data = hexString(data); - var o = "0x", z = 0, y = 0, i = 0, c; - - function pushByte(b) { - o += byteToString(((o.length < 4 * 2 + 2) * 0xff) ^ b); - } - - function rle(v, d) { - pushByte(0x00); - pushByte(d - 1 + v * 0x80); - } - - for (; i < data.length; i += 2) { - c = parseByte(data, i); - if (!c) { - if (y) rle(1, y), y = 0; - if (++z === 0x80) rle(0, 0x80), z = 0; - continue; - } - if (c === 0xff) { - if (z) rle(0, z), z = 0; - if (++y === 0x20) rle(1, 0x20), y = 0; - continue; - } - if (y) rle(1, y), y = 0; - if (z) rle(0, z), z = 0; - pushByte(c); - } - if (y) rle(1, y), y = 0; - if (z) rle(0, z), z = 0; - return o; -} - -/** - * Decompresses hex encoded calldata. - * @param {string} data A hex encoded string representing the compressed data. - * @returns {string} The decompressed result as a hex encoded string. - */ -LibZip.cdDecompress = function(data) { - data = hexString(data); - var o = "0x", i = 0, j, c, s; - - while (i < data.length) { - c = ((i < 4 * 2) * 0xff) ^ parseByte(data, i); - i += 2; - if (!c) { - c = ((i < 4 * 2) * 0xff) ^ parseByte(data, i); - s = (c & 0x7f) + 1; - i += 2; - for (j = 0; j < s; ++j) o += byteToString((c >> 7 && j < 32) * 0xff); - continue; - } - o += byteToString(c); - } - return o; -} - -module.exports = LibZip; \ No newline at end of file diff --git a/test/utils/compression/FastLZ.test.js b/test/utils/compression/FastLZ.test.js index be71652724f..3859941c1f5 100644 --- a/test/utils/compression/FastLZ.test.js +++ b/test/utils/compression/FastLZ.test.js @@ -2,7 +2,7 @@ const { ethers } = require('hardhat'); const { expect } = require('chai'); const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); -const LibZip = require('../../helpers/LibZip'); +const { LibZip } = require('solady/js/solady'); async function fixture() { const mock = await ethers.deployContract('$FastLZ'); diff --git a/test/utils/compression/gas.test.js b/test/utils/compression/gas.test.js index 179757cffb0..c7d8cbf6b1e 100644 --- a/test/utils/compression/gas.test.js +++ b/test/utils/compression/gas.test.js @@ -3,7 +3,7 @@ const { loadFixture } = require('@nomicfoundation/hardhat-network-helpers'); const lz4js = require('lz4js'); const snappy = require('snappy'); -const LibZip = require('../../helpers/LibZip'); +const { LibZip } = require('solady/js/solady'); const { min } = require('../../helpers/math'); async function fixture() { From 5b39ebff0558127f9a8beae44f75f3f65d86d669 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 23:20:35 +0200 Subject: [PATCH 17/20] LZ4: add memory access checks and calldataDecompress --- contracts/utils/compression/LZ4.sol | 294 ++++++++++++++++++++-------- test/utils/compression/LZ4.test.js | 2 +- 2 files changed, 214 insertions(+), 82 deletions(-) diff --git a/contracts/utils/compression/LZ4.sol b/contracts/utils/compression/LZ4.sol index 9bf10b78e0f..2fa49fa8fef 100644 --- a/contracts/utils/compression/LZ4.sol +++ b/contracts/utils/compression/LZ4.sol @@ -22,149 +22,281 @@ library LZ4 { uint8 private constant BS_SHIFT = 0x04; uint8 private constant BS_MASK = 0x07; + error InvalidMagicNumber(); + error InvalidVersion(); + error DecodingFailure(); + /** * @dev Implementation of LZ4's decompress function. * * See https://github.com/Benzinga/lz4js/blob/master/lz4.js */ function decompress(bytes memory input) internal pure returns (bytes memory output) { + bytes4 invalidMagicNumberCode = InvalidMagicNumber.selector; + bytes4 invalidVersionCode = InvalidVersion.selector; + bytes4 decodingFailureCode = DecodingFailure.selector; + assembly ("memory-safe") { function assert(b, e) { if iszero(b) { mstore(0, e) - revert(0, 0x04) + revert(0, 4) } } - // load 16 bytes from a given location in memory, right aligned and in reverse order - function readU16(ptr) -> value { - value := mload(ptr) - value := or(byte(0, value), shl(8, byte(1, value))) - } - // load 32 bytes from a given location in memory, right aligned and in reverse order - function readU32(ptr) -> value { - value := mload(ptr) - value := or( - or(byte(0, value), shl(8, byte(1, value))), - or(shl(16, byte(2, value)), shl(24, byte(3, value))) - ) + function adv(ptr, end, l, e) -> ptr_ { + ptr_ := add(ptr, l) + if gt(ptr_, end) { + mstore(0, e) + revert(0, 4) + } } - // input buffer let inputPtr := add(input, 0x20) - // let inputEnd := add(inputPtr, mload(input)) // TODO: use to check bounds - + let inputEnd := add(inputPtr, mload(input)) // output buffer output := mload(0x40) let outputPtr := add(output, 0x20) - // ========================================== decompress frame =========================================== - // TODO CHECK LENGTH BEFORE - // Get header (size must be at least 7 / 15 depending on useContentSize ) // [ magic (4) ++ descriptor (1) ++ bsIds (1) ++ contentsize (0 or 8) ++ ??? (1) ] let header := mload(inputPtr) - // read magic number (first 4 bytes, realigned right) - assert(eq(shr(224, header), MAGIC_NUM), 0x00000001) // TODO: error code - + assert(eq(shr(224, header), MAGIC_NUM), invalidMagicNumberCode) // read descriptor and check version let descriptor := byte(4, header) - assert(eq(and(descriptor, FD_VERSION_MASK), FD_VERSION), 0x00000002) // TODO: error code - + assert(eq(and(descriptor, FD_VERSION_MASK), FD_VERSION), invalidVersionCode) // read flags let useBlockSum := eq(and(descriptor, FD_BLOCK_CHKSUM), FD_BLOCK_CHKSUM) let useContentSize := eq(and(descriptor, FD_CONTENT_SIZE), FD_CONTENT_SIZE) - // read block size - // let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) - // TODO: check bsMap? value should probably be in [4, 7], otherwise unused - - // move forward 7 bytes - inputPtr := add(inputPtr, add(7, mul(useContentSize, 8))) - + let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) + assert(and(gt(bsIdx, 3), lt(bsIdx, 8)), decodingFailureCode) + // move forward 7 or 15 bytes depending on "useContentSize" + inputPtr := adv(inputPtr, inputEnd, add(7, mul(useContentSize, 8)), decodingFailureCode) // read blocks for {} 1 {} { - let compSize := readU32(inputPtr) - - if iszero(compSize) { + let chunk := mload(inputPtr) + // read block length (32 bits = 4 bytes reverse endianness) + let blockLength := or( + or(byte(0, chunk), shl(8, byte(1, chunk))), + or(shl(16, byte(2, chunk)), shl(24, byte(3, chunk))) + ) + inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + // empty block means we are done with decoding + if iszero(blockLength) { break } - - // read block checksum if "useBlockSum" is true? - inputPtr := add(inputPtr, add(4, mul(useBlockSum, 4))) - + // read block checksum if "useBlockSum" (from chunk) ? + if useBlockSum { + inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + } // check if block is compressed - switch iszero(and(compSize, BS_UNCOMPRESSED)) + switch iszero(and(blockLength, BS_UNCOMPRESSED)) + // uncompressed block case case 0 { // mask off the 'uncompressed' bit - compSize := and(compSize, not(BS_UNCOMPRESSED)) - // copy uncompressed chunk - mcopy(outputPtr, inputPtr, compSize) - inputPtr := add(inputPtr, compSize) - outputPtr := add(outputPtr, compSize) + blockLength := and(blockLength, not(BS_UNCOMPRESSED)) + // copy uncompressed data to the output buffer + mcopy(outputPtr, inputPtr, blockLength) + inputPtr := adv(inputPtr, inputEnd, blockLength, decodingFailureCode) + outputPtr := add(outputPtr, blockLength) } + // compressed block case case 1 { - for { - let blockEnd := add(inputPtr, compSize) - } lt(inputPtr, blockEnd) {} { + let blockEnd := add(inputPtr, blockLength) + for {} lt(inputPtr, blockEnd) {} { let token := byte(0, mload(inputPtr)) - inputPtr := add(inputPtr, 1) - - // copy literals. - let literalCount := shr(4, token) - if literalCount { + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + // literals to copy + let literalLength := shr(4, token) + if literalLength { // Parse length. - if eq(literalCount, 0xf) { + if eq(literalLength, 0xf) { for {} 1 {} { let count := byte(0, mload(inputPtr)) - inputPtr := add(inputPtr, 1) - literalCount := add(literalCount, count) + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + literalLength := add(literalLength, count) if lt(count, 0xff) { break } } } - mcopy(outputPtr, inputPtr, literalCount) - inputPtr := add(inputPtr, literalCount) - outputPtr := add(outputPtr, literalCount) + mcopy(outputPtr, inputPtr, literalLength) + inputPtr := adv(inputPtr, blockEnd, literalLength, decodingFailureCode) + outputPtr := add(outputPtr, literalLength) } + // if we are done reading the block, break the switch (continue the loop) + if iszero(lt(inputPtr, blockEnd)) { + break + } + // read offset (32 bits = 4 bytes reverse endianness) + chunk := mload(inputPtr) + let offset := or(byte(0, chunk), shl(8, byte(1, chunk))) + inputPtr := adv(inputPtr, blockEnd, 2, decodingFailureCode) + // parse length of the copy section + let copyLength := and(token, 0xf) + if eq(copyLength, 0xf) { + for {} 1 {} { + let count := byte(0, mload(inputPtr)) + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + copyLength := add(copyLength, count) + if lt(count, 0xff) { + break + } + } + } + copyLength := add(copyLength, MIN_MATCH) + // do the copy + for { + let ptr := outputPtr + let end := add(outputPtr, copyLength) + let step := xor(offset, mul(lt(copyLength, offset), xor(copyLength, offset))) // min(copyLength, offset) + } lt(ptr, end) { + ptr := add(ptr, step) + } { + mcopy(ptr, sub(ptr, offset), step) + } + outputPtr := add(outputPtr, copyLength) + } + assert(eq(inputPtr, blockEnd), decodingFailureCode) + } + } + // reserve used memory + mstore(output, sub(outputPtr, add(output, 0x20))) + mstore(0x40, outputPtr) + } + } - if lt(inputPtr, blockEnd) { - // Copy match. - let mLength := and(token, 0xf) - - // Parse offset. - let mOffset := readU16(inputPtr) - inputPtr := add(inputPtr, 2) + function decompressCalldata(bytes calldata input) internal pure returns (bytes memory output) { + bytes4 invalidMagicNumberCode = InvalidMagicNumber.selector; + bytes4 invalidVersionCode = InvalidVersion.selector; + bytes4 decodingFailureCode = DecodingFailure.selector; + assembly ("memory-safe") { + function assert(b, e) { + if iszero(b) { + mstore(0, e) + revert(0, 4) + } + } + function adv(ptr, end, l, e) -> ptr_ { + ptr_ := add(ptr, l) + if gt(ptr_, end) { + mstore(0, e) + revert(0, 4) + } + } + // input buffer + let inputPtr := input.offset + let inputEnd := add(inputPtr, input.length) + // output buffer + output := mload(0x40) + let outputPtr := add(output, 0x20) + // ========================================== decompress frame =========================================== + // Get header (size must be at least 7 / 15 depending on useContentSize ) + // [ magic (4) ++ descriptor (1) ++ bsIds (1) ++ contentsize (0 or 8) ++ ??? (1) ] + let header := calldataload(inputPtr) + // read magic number (first 4 bytes, realigned right) + assert(eq(shr(224, header), MAGIC_NUM), invalidMagicNumberCode) + // read descriptor and check version + let descriptor := byte(4, header) + assert(eq(and(descriptor, FD_VERSION_MASK), FD_VERSION), invalidVersionCode) + // read flags + let useBlockSum := eq(and(descriptor, FD_BLOCK_CHKSUM), FD_BLOCK_CHKSUM) + let useContentSize := eq(and(descriptor, FD_CONTENT_SIZE), FD_CONTENT_SIZE) + // read block size + let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) + assert(and(gt(bsIdx, 3), lt(bsIdx, 8)), decodingFailureCode) + // move forward 7 or 15 bytes depending on "useContentSize" + inputPtr := adv(inputPtr, inputEnd, add(7, mul(useContentSize, 8)), decodingFailureCode) + // read blocks + for {} 1 {} { + let chunk := calldataload(inputPtr) + // read block length (32 bits = 4 bytes reverse endianness) + let blockLength := or( + or(byte(0, chunk), shl(8, byte(1, chunk))), + or(shl(16, byte(2, chunk)), shl(24, byte(3, chunk))) + ) + inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + // empty block means we are done with decoding + if iszero(blockLength) { + break + } + // read block checksum if "useBlockSum" (from chunk) ? + if useBlockSum { + inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + } + // check if block is compressed + switch iszero(and(blockLength, BS_UNCOMPRESSED)) + // uncompressed block case + case 0 { + // mask off the 'uncompressed' bit + blockLength := and(blockLength, not(BS_UNCOMPRESSED)) + // copy uncompressed data to the output buffer + calldatacopy(outputPtr, inputPtr, blockLength) + inputPtr := adv(inputPtr, inputEnd, blockLength, decodingFailureCode) + outputPtr := add(outputPtr, blockLength) + } + // compressed block case + case 1 { + let blockEnd := add(inputPtr, blockLength) + for {} lt(inputPtr, blockEnd) {} { + let token := byte(0, calldataload(inputPtr)) + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + // literals to copy + let literalLength := shr(4, token) + if literalLength { // Parse length. - if eq(mLength, 0xf) { + if eq(literalLength, 0xf) { for {} 1 {} { - let count := byte(0, mload(inputPtr)) - inputPtr := add(inputPtr, 1) - mLength := add(mLength, count) + let count := byte(0, calldataload(inputPtr)) + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + literalLength := add(literalLength, count) if lt(count, 0xff) { break } } } - mLength := add(mLength, MIN_MATCH) - - for { - let ptr := outputPtr - let end := add(outputPtr, mLength) - let step := xor(mOffset, mul(lt(mLength, mOffset), xor(mLength, mOffset))) // min(mLength, mOffset) - } lt(ptr, end) { - ptr := add(ptr, step) - } { - mcopy(ptr, sub(ptr, mOffset), step) + calldatacopy(outputPtr, inputPtr, literalLength) + inputPtr := adv(inputPtr, blockEnd, literalLength, decodingFailureCode) + outputPtr := add(outputPtr, literalLength) + } + // if we are done reading the block, break the switch (continue the loop) + if iszero(lt(inputPtr, blockEnd)) { + break + } + // read offset (32 bits = 4 bytes reverse endianness) + chunk := calldataload(inputPtr) + let offset := or(byte(0, chunk), shl(8, byte(1, chunk))) + inputPtr := adv(inputPtr, blockEnd, 2, decodingFailureCode) + // parse length of the copy section + let copyLength := and(token, 0xf) + if eq(copyLength, 0xf) { + for {} 1 {} { + let count := byte(0, calldataload(inputPtr)) + inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + copyLength := add(copyLength, count) + if lt(count, 0xff) { + break + } } - outputPtr := add(outputPtr, mLength) } + copyLength := add(copyLength, MIN_MATCH) + // do the copy + for { + let ptr := outputPtr + let end := add(outputPtr, copyLength) + let step := xor(offset, mul(lt(copyLength, offset), xor(copyLength, offset))) // min(copyLength, offset) + } lt(ptr, end) { + ptr := add(ptr, step) + } { + mcopy(ptr, sub(ptr, offset), step) + } + outputPtr := add(outputPtr, copyLength) } + assert(eq(inputPtr, blockEnd), decodingFailureCode) } } - // reserve used memory mstore(output, sub(outputPtr, add(output, 0x20))) mstore(0x40, outputPtr) diff --git a/test/utils/compression/LZ4.test.js b/test/utils/compression/LZ4.test.js index b9d18d50065..370cbd8d371 100644 --- a/test/utils/compression/LZ4.test.js +++ b/test/utils/compression/LZ4.test.js @@ -58,7 +58,7 @@ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula const hex = ethers.hexlify(raw); const compressed = lz4js.compress(raw); await expect(this.mock.$decompress(compressed)).to.eventually.equal(hex); - // await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); + await expect(this.mock.$decompressCalldata(compressed)).to.eventually.equal(hex); }); }); }); From b35e095ee41f9dfd641aecb68f561b7b541f681a Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 23:32:20 +0200 Subject: [PATCH 18/20] up --- test/utils/compression/gas.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/utils/compression/gas.test.js b/test/utils/compression/gas.test.js index c7d8cbf6b1e..697a3734a77 100644 --- a/test/utils/compression/gas.test.js +++ b/test/utils/compression/gas.test.js @@ -73,9 +73,9 @@ Nullam eget pharetra mauris. Cras nec ultricies mi. Suspendisse sit amet ligula this.results.push({ lowest, - extraGasUsedPercentageFastlz: (100 * (gasUsedFastlz - lowest)) / lowest, - extraGasUsedPercentageSnappy: (100 * (gasUsedSnappy - lowest)) / lowest, - extraGasUsedPercentageLz4: (100 * (gasUsedLz4 - lowest)) / lowest, + extraGasUsedPercentageFastlz: `+${((100 * (gasUsedFastlz - lowest)) / lowest).toFixed(2)}%`, + extraGasUsedPercentageSnappy: `+${((100 * (gasUsedSnappy - lowest)) / lowest).toFixed(2)}%`, + extraGasUsedPercentageLz4: `+${((100 * (gasUsedLz4 - lowest)) / lowest).toFixed(2)}%`, }); }); From 96745a0fabf3ed42fc85621f7a08550ecd664635 Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 23:35:50 +0200 Subject: [PATCH 19/20] up --- test/utils/compression/FastLZ.t.sol | 128 +--------------------------- 1 file changed, 3 insertions(+), 125 deletions(-) diff --git a/test/utils/compression/FastLZ.t.sol b/test/utils/compression/FastLZ.t.sol index 51367595146..1df946003c3 100644 --- a/test/utils/compression/FastLZ.t.sol +++ b/test/utils/compression/FastLZ.t.sol @@ -4,140 +4,18 @@ pragma solidity ^0.8.20; import {Test} from "forge-std/Test.sol"; import {FastLZ} from "@openzeppelin/contracts/utils/compression/FastLZ.sol"; +import {LibZip} from "solady/src/utils/LibZip.sol"; contract FastLZTest is Test { function testEncodeDecode(bytes memory input) external pure { - assertEq(FastLZ.decompress(_flzCompress(input)), input); + assertEq(FastLZ.decompress(LibZip.flzCompress(input)), input); } function testEncodeDecodeCalldata(bytes memory input) external view { - assertEq(this.decompressCalldata(_flzCompress(input)), input); + assertEq(this.decompressCalldata(LibZip.flzCompress(input)), input); } function decompressCalldata(bytes calldata input) external pure returns (bytes memory) { return FastLZ.decompress(input); } - - /// Copied from solady - function _flzCompress(bytes memory input) private pure returns (bytes memory output) { - assembly ("memory-safe") { - // store 8 bytes (value) at ptr, and return updated ptr - function ms8(ptr, value) -> ret { - mstore8(ptr, value) - ret := add(ptr, 1) - } - // load 24 bytes from a given location in memory, right aligned and in reverse order - function u24(ptr) -> value { - value := mload(ptr) - value := or(shl(16, byte(2, value)), or(shl(8, byte(1, value)), byte(0, value))) - } - function cmp(p_, q_, e_) -> _l { - for { - e_ := sub(e_, q_) - } lt(_l, e_) { - _l := add(_l, 1) - } { - e_ := mul(iszero(byte(0, xor(mload(add(p_, _l)), mload(add(q_, _l))))), e_) - } - } - function literals(runs_, src_, dest_) -> _o { - for { - _o := dest_ - } iszero(lt(runs_, 0x20)) { - runs_ := sub(runs_, 0x20) - } { - mstore(ms8(_o, 31), mload(src_)) - _o := add(_o, 0x21) - src_ := add(src_, 0x20) - } - if iszero(runs_) { - leave - } - mstore(ms8(_o, sub(runs_, 1)), mload(src_)) - _o := add(1, add(_o, runs_)) - } - function mt(l_, d_, o_) -> _o { - for { - d_ := sub(d_, 1) - } iszero(lt(l_, 263)) { - l_ := sub(l_, 262) - } { - o_ := ms8(ms8(ms8(o_, add(224, shr(8, d_))), 253), and(0xff, d_)) - } - if iszero(lt(l_, 7)) { - _o := ms8(ms8(ms8(o_, add(224, shr(8, d_))), sub(l_, 7)), and(0xff, d_)) - leave - } - _o := ms8(ms8(o_, add(shl(5, l_), shr(8, d_))), and(0xff, d_)) - } - function setHash(i_, v_) { - let p_ := add(mload(0x40), shl(2, i_)) - mstore(p_, xor(mload(p_), shl(224, xor(shr(224, mload(p_)), v_)))) - } - function getHash(i_) -> _h { - _h := shr(224, mload(add(mload(0x40), shl(2, i_)))) - } - function hash(v_) -> _r { - _r := and(shr(19, mul(2654435769, v_)), 0x1fff) - } - function setNextHash(ip_, ipStart_) -> _ip { - setHash(hash(u24(ip_)), sub(ip_, ipStart_)) - _ip := add(ip_, 1) - } - - output := mload(0x40) - - calldatacopy(output, calldatasize(), 0x8000) // Zeroize the hashmap. - let op := add(output, 0x8000) - - let a := add(input, 0x20) - - let ipStart := a - let ipLimit := sub(add(ipStart, mload(input)), 13) - for { - let ip := add(2, a) - } lt(ip, ipLimit) {} { - let r := 0 - let d := 0 - for {} 1 {} { - let s := u24(ip) - let h := hash(s) - r := add(ipStart, getHash(h)) - setHash(h, sub(ip, ipStart)) - d := sub(ip, r) - if iszero(lt(ip, ipLimit)) { - break - } - ip := add(ip, 1) - if iszero(gt(d, 0x1fff)) { - if eq(s, u24(r)) { - break - } - } - } - if iszero(lt(ip, ipLimit)) { - break - } - ip := sub(ip, 1) - if gt(ip, a) { - op := literals(sub(ip, a), a, op) - } - let l := cmp(add(r, 3), add(ip, 3), add(ipLimit, 9)) - op := mt(l, d, op) - ip := setNextHash(setNextHash(add(ip, l), ipStart), ipStart) - a := ip - } - // Copy the result to compact the memory, overwriting the hashmap. - let end := sub(literals(sub(add(ipStart, mload(input)), a), a, op), 0x7fe0) - let o := add(output, 0x20) - mstore(output, sub(end, o)) // Store the length. - for {} iszero(gt(o, end)) { - o := add(o, 0x20) - } { - mstore(o, mload(add(o, 0x7fe0))) - } - - mstore(0x40, end) - } - } } From cd20e5096509b578b9188dd9dd05d9b9d06743eb Mon Sep 17 00:00:00 2001 From: Hadrien Croubois Date: Thu, 31 Jul 2025 23:59:51 +0200 Subject: [PATCH 20/20] check inputPtr at then end (can only move forward anyway) --- contracts/utils/compression/FastLZ.sol | 64 +++++++++++++------------- contracts/utils/compression/LZ4.sol | 56 +++++++++------------- contracts/utils/compression/Snappy.sol | 43 ++++------------- 3 files changed, 63 insertions(+), 100 deletions(-) diff --git a/contracts/utils/compression/FastLZ.sol b/contracts/utils/compression/FastLZ.sol index 3d20f56a482..5bcec878425 100644 --- a/contracts/utils/compression/FastLZ.sol +++ b/contracts/utils/compression/FastLZ.sol @@ -16,18 +16,14 @@ library FastLZ { */ function decompress(bytes memory input) internal pure returns (bytes memory output) { assembly ("memory-safe") { + let inputPtr := add(input, 0x20) + let inputEnd := add(add(input, 0x20), mload(input)) + // Use new memory allocate at the FMP output := mload(0x40) - - // Decrypted inputPtr location let outputPtr := add(output, 0x20) - // end of the input inputPtr (input.length after the beginning of the inputPtr) - let end := add(add(input, 0x20), mload(input)) - - for { - let inputPtr := add(input, 0x20) - } lt(inputPtr, end) {} { + for {} lt(inputPtr, inputEnd) {} { let chunk := mload(inputPtr) let first := byte(0, chunk) let type_ := shr(5, first) @@ -39,36 +35,40 @@ library FastLZ { outputPtr := add(outputPtr, add(1, first)) } case 7 { - let ofs := add(shl(8, and(first, 31)), byte(2, chunk)) let len := add(9, byte(1, chunk)) - let ref := sub(sub(outputPtr, ofs), 1) - let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 + let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1) + let ref := sub(outputPtr, ofs) + let step := xor(len, mul(lt(ofs, len), xor(ofs, len))) } lt(i, len) { i := add(i, step) } { - mstore(add(outputPtr, i), mload(add(ref, i))) + mcopy(add(outputPtr, i), add(ref, i), step) } inputPtr := add(inputPtr, 3) outputPtr := add(outputPtr, len) } default { - let ofs := add(shl(8, and(first, 31)), byte(1, chunk)) let len := add(2, type_) - let ref := sub(sub(outputPtr, ofs), 1) - let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 + let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1) + let ref := sub(outputPtr, ofs) + let step := xor(len, mul(lt(ofs, len), xor(ofs, len))) } lt(i, len) { i := add(i, step) } { - mstore(add(outputPtr, i), mload(add(ref, i))) + mcopy(add(outputPtr, i), add(ref, i), step) } inputPtr := add(inputPtr, 2) outputPtr := add(outputPtr, len) } } + if iszero(eq(inputPtr, inputEnd)) { + revert(0, 0) + } + mstore(output, sub(outputPtr, add(output, 0x20))) mstore(0x40, outputPtr) } @@ -76,18 +76,14 @@ library FastLZ { function decompressCalldata(bytes calldata input) internal pure returns (bytes memory output) { assembly ("memory-safe") { + let inputPtr := input.offset + let inputEnd := add(input.offset, input.length) + // Use new memory allocate at the FMP output := mload(0x40) - - // Decrypted inputPtr location let outputPtr := add(output, 0x20) - // end of the input inputPtr (input.length after the beginning of the inputPtr) - let end := add(input.offset, input.length) - - for { - let inputPtr := input.offset - } lt(inputPtr, end) {} { + for {} lt(inputPtr, inputEnd) {} { let chunk := calldataload(inputPtr) let first := byte(0, chunk) let type_ := shr(5, first) @@ -99,36 +95,40 @@ library FastLZ { outputPtr := add(outputPtr, add(1, first)) } case 7 { - let ofs := add(shl(8, and(first, 31)), byte(2, chunk)) let len := add(9, byte(1, chunk)) - let ref := sub(sub(outputPtr, ofs), 1) - let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 + let ofs := add(add(shl(8, and(first, 31)), byte(2, chunk)), 1) + let ref := sub(outputPtr, ofs) + let step := xor(len, mul(lt(ofs, len), xor(ofs, len))) } lt(i, len) { i := add(i, step) } { - mstore(add(outputPtr, i), mload(add(ref, i))) + mcopy(add(outputPtr, i), add(ref, i), step) } inputPtr := add(inputPtr, 3) outputPtr := add(outputPtr, len) } default { - let ofs := add(shl(8, and(first, 31)), byte(1, chunk)) let len := add(2, type_) - let ref := sub(sub(outputPtr, ofs), 1) - let step := sub(0x20, mul(lt(ofs, 0x20), sub(0x1f, ofs))) // min(ofs+1, 0x20) for { let i := 0 + let ofs := add(add(shl(8, and(first, 31)), byte(1, chunk)), 1) + let ref := sub(outputPtr, ofs) + let step := xor(len, mul(lt(ofs, len), xor(ofs, len))) } lt(i, len) { i := add(i, step) } { - mstore(add(outputPtr, i), mload(add(ref, i))) + mcopy(add(outputPtr, i), add(ref, i), step) } inputPtr := add(inputPtr, 2) outputPtr := add(outputPtr, len) } } + if iszero(eq(inputPtr, inputEnd)) { + revert(0, 0) + } + mstore(output, sub(outputPtr, add(output, 0x20))) mstore(0x40, outputPtr) } diff --git a/contracts/utils/compression/LZ4.sol b/contracts/utils/compression/LZ4.sol index 2fa49fa8fef..f1a73b8a63c 100644 --- a/contracts/utils/compression/LZ4.sol +++ b/contracts/utils/compression/LZ4.sol @@ -43,13 +43,6 @@ library LZ4 { revert(0, 4) } } - function adv(ptr, end, l, e) -> ptr_ { - ptr_ := add(ptr, l) - if gt(ptr_, end) { - mstore(0, e) - revert(0, 4) - } - } // input buffer let inputPtr := add(input, 0x20) let inputEnd := add(inputPtr, mload(input)) @@ -72,7 +65,7 @@ library LZ4 { let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) assert(and(gt(bsIdx, 3), lt(bsIdx, 8)), decodingFailureCode) // move forward 7 or 15 bytes depending on "useContentSize" - inputPtr := adv(inputPtr, inputEnd, add(7, mul(useContentSize, 8)), decodingFailureCode) + inputPtr := add(inputPtr, add(7, mul(useContentSize, 8))) // read blocks for {} 1 {} { let chunk := mload(inputPtr) @@ -81,14 +74,14 @@ library LZ4 { or(byte(0, chunk), shl(8, byte(1, chunk))), or(shl(16, byte(2, chunk)), shl(24, byte(3, chunk))) ) - inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + inputPtr := add(inputPtr, 4) // empty block means we are done with decoding if iszero(blockLength) { break } // read block checksum if "useBlockSum" (from chunk) ? if useBlockSum { - inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + inputPtr := add(inputPtr, 4) } // check if block is compressed switch iszero(and(blockLength, BS_UNCOMPRESSED)) @@ -98,7 +91,7 @@ library LZ4 { blockLength := and(blockLength, not(BS_UNCOMPRESSED)) // copy uncompressed data to the output buffer mcopy(outputPtr, inputPtr, blockLength) - inputPtr := adv(inputPtr, inputEnd, blockLength, decodingFailureCode) + inputPtr := add(inputPtr, blockLength) outputPtr := add(outputPtr, blockLength) } // compressed block case @@ -106,7 +99,7 @@ library LZ4 { let blockEnd := add(inputPtr, blockLength) for {} lt(inputPtr, blockEnd) {} { let token := byte(0, mload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) // literals to copy let literalLength := shr(4, token) if literalLength { @@ -114,7 +107,7 @@ library LZ4 { if eq(literalLength, 0xf) { for {} 1 {} { let count := byte(0, mload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) literalLength := add(literalLength, count) if lt(count, 0xff) { break @@ -122,7 +115,7 @@ library LZ4 { } } mcopy(outputPtr, inputPtr, literalLength) - inputPtr := adv(inputPtr, blockEnd, literalLength, decodingFailureCode) + inputPtr := add(inputPtr, literalLength) outputPtr := add(outputPtr, literalLength) } // if we are done reading the block, break the switch (continue the loop) @@ -132,13 +125,13 @@ library LZ4 { // read offset (32 bits = 4 bytes reverse endianness) chunk := mload(inputPtr) let offset := or(byte(0, chunk), shl(8, byte(1, chunk))) - inputPtr := adv(inputPtr, blockEnd, 2, decodingFailureCode) + inputPtr := add(inputPtr, 2) // parse length of the copy section let copyLength := and(token, 0xf) if eq(copyLength, 0xf) { for {} 1 {} { let count := byte(0, mload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) copyLength := add(copyLength, count) if lt(count, 0xff) { break @@ -161,7 +154,8 @@ library LZ4 { assert(eq(inputPtr, blockEnd), decodingFailureCode) } } - // reserve used memory + assert(eq(inputPtr, inputEnd), decodingFailureCode) + // allocate used memory mstore(output, sub(outputPtr, add(output, 0x20))) mstore(0x40, outputPtr) } @@ -179,13 +173,6 @@ library LZ4 { revert(0, 4) } } - function adv(ptr, end, l, e) -> ptr_ { - ptr_ := add(ptr, l) - if gt(ptr_, end) { - mstore(0, e) - revert(0, 4) - } - } // input buffer let inputPtr := input.offset let inputEnd := add(inputPtr, input.length) @@ -208,7 +195,7 @@ library LZ4 { let bsIdx := and(shr(BS_SHIFT, byte(5, header)), BS_MASK) assert(and(gt(bsIdx, 3), lt(bsIdx, 8)), decodingFailureCode) // move forward 7 or 15 bytes depending on "useContentSize" - inputPtr := adv(inputPtr, inputEnd, add(7, mul(useContentSize, 8)), decodingFailureCode) + inputPtr := add(inputPtr, add(7, mul(useContentSize, 8))) // read blocks for {} 1 {} { let chunk := calldataload(inputPtr) @@ -217,14 +204,14 @@ library LZ4 { or(byte(0, chunk), shl(8, byte(1, chunk))), or(shl(16, byte(2, chunk)), shl(24, byte(3, chunk))) ) - inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + inputPtr := add(inputPtr, 4) // empty block means we are done with decoding if iszero(blockLength) { break } // read block checksum if "useBlockSum" (from chunk) ? if useBlockSum { - inputPtr := adv(inputPtr, inputEnd, 4, decodingFailureCode) + inputPtr := add(inputPtr, 4) } // check if block is compressed switch iszero(and(blockLength, BS_UNCOMPRESSED)) @@ -234,7 +221,7 @@ library LZ4 { blockLength := and(blockLength, not(BS_UNCOMPRESSED)) // copy uncompressed data to the output buffer calldatacopy(outputPtr, inputPtr, blockLength) - inputPtr := adv(inputPtr, inputEnd, blockLength, decodingFailureCode) + inputPtr := add(inputPtr, blockLength) outputPtr := add(outputPtr, blockLength) } // compressed block case @@ -242,7 +229,7 @@ library LZ4 { let blockEnd := add(inputPtr, blockLength) for {} lt(inputPtr, blockEnd) {} { let token := byte(0, calldataload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) // literals to copy let literalLength := shr(4, token) if literalLength { @@ -250,7 +237,7 @@ library LZ4 { if eq(literalLength, 0xf) { for {} 1 {} { let count := byte(0, calldataload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) literalLength := add(literalLength, count) if lt(count, 0xff) { break @@ -258,7 +245,7 @@ library LZ4 { } } calldatacopy(outputPtr, inputPtr, literalLength) - inputPtr := adv(inputPtr, blockEnd, literalLength, decodingFailureCode) + inputPtr := add(inputPtr, literalLength) outputPtr := add(outputPtr, literalLength) } // if we are done reading the block, break the switch (continue the loop) @@ -268,13 +255,13 @@ library LZ4 { // read offset (32 bits = 4 bytes reverse endianness) chunk := calldataload(inputPtr) let offset := or(byte(0, chunk), shl(8, byte(1, chunk))) - inputPtr := adv(inputPtr, blockEnd, 2, decodingFailureCode) + inputPtr := add(inputPtr, 2) // parse length of the copy section let copyLength := and(token, 0xf) if eq(copyLength, 0xf) { for {} 1 {} { let count := byte(0, calldataload(inputPtr)) - inputPtr := adv(inputPtr, blockEnd, 1, decodingFailureCode) + inputPtr := add(inputPtr, 1) copyLength := add(copyLength, count) if lt(count, 0xff) { break @@ -297,7 +284,8 @@ library LZ4 { assert(eq(inputPtr, blockEnd), decodingFailureCode) } } - // reserve used memory + assert(eq(inputPtr, inputEnd), decodingFailureCode) + // allocate used memory mstore(output, sub(outputPtr, add(output, 0x20))) mstore(0x40, outputPtr) } diff --git a/contracts/utils/compression/Snappy.sol b/contracts/utils/compression/Snappy.sol index 12e7d936daa..10aa644f50d 100644 --- a/contracts/utils/compression/Snappy.sol +++ b/contracts/utils/compression/Snappy.sol @@ -16,16 +16,7 @@ library Snappy { * Based on https://github.com/zhipeng-jia/snappyjs/blob/v0.7.0/snappy_decompressor.js[snappyjs javascript implementation]. */ function uncompress(bytes memory input) internal pure returns (bytes memory output) { - bytes4 errorSelector = DecodingFailure.selector; - assembly ("memory-safe") { - // helper: revert with custom error (without args) if boolean isn't true - function assert(b, e) { - if iszero(b) { - mstore(0, e) - revert(0, 0x04) - } - } // input buffer bounds let inputBegin := add(input, 0x20) let inputEnd := add(inputBegin, mload(input)) @@ -63,13 +54,11 @@ library Snappy { case 0 { len := add(shr(2, c), 1) if gt(len, 60) { - assert(lt(add(inputPtr, 3), inputEnd), errorSelector) let smallLen := sub(len, 60) len := or(or(byte(1, w), shl(8, byte(2, w))), or(shl(16, byte(3, w)), shl(24, byte(4, w)))) len := add(and(len, shr(sub(256, mul(8, smallLen)), not(0))), 1) inputPtr := add(inputPtr, smallLen) } - assert(not(gt(add(inputPtr, len), inputEnd)), errorSelector) mcopy(outputPtr, inputPtr, len) inputPtr := add(inputPtr, len) outputPtr := add(outputPtr, len) @@ -77,24 +66,20 @@ library Snappy { continue } case 1 { - assert(lt(inputPtr, inputEnd), errorSelector) len := add(and(shr(2, c), 0x7), 4) offset := add(byte(1, w), shl(8, shr(5, c))) inputPtr := add(inputPtr, 1) } case 2 { - assert(lt(add(inputPtr, 1), inputEnd), errorSelector) len := add(shr(2, c), 1) offset := add(byte(1, w), shl(8, byte(2, w))) inputPtr := add(inputPtr, 2) } case 3 { - assert(lt(add(inputPtr, 3), inputEnd), errorSelector) len := add(shr(2, c), 1) offset := add(add(byte(1, w), shl(8, byte(2, w))), add(shl(16, byte(3, w)), shl(24, byte(4, w)))) inputPtr := add(inputPtr, 4) } - assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) // copying in will not work if the offset is larger than the len being copied, so we compute // `step = Math.min(len, offset)` and use it for the memory copy in chunks for { @@ -108,23 +93,16 @@ library Snappy { } outputPtr := add(outputPtr, len) } - // sanity check, FMP is at the right location - assert(eq(outputPtr, mload(0x40)), errorSelector) + // sanity check, we did not read more than the input and FMP is at the right location + if iszero(and(eq(inputPtr, inputEnd), eq(outputPtr, mload(0x40)))) { + revert(0, 0) + } } } /// @dev Variant of {uncompress} that takes a buffer from calldata. function uncompressCalldata(bytes calldata input) internal pure returns (bytes memory output) { - bytes4 errorSelector = DecodingFailure.selector; - assembly ("memory-safe") { - // helper: revert with custom error (without args) if boolean isn't true - function assert(b, e) { - if iszero(b) { - mstore(0, e) - revert(0, 0x04) - } - } // input buffer bounds let inputBegin := input.offset let inputEnd := add(inputBegin, input.length) @@ -162,13 +140,11 @@ library Snappy { case 0 { len := add(shr(2, c), 1) if gt(len, 60) { - assert(lt(add(inputPtr, 3), inputEnd), errorSelector) let smallLen := sub(len, 60) len := or(or(byte(1, w), shl(8, byte(2, w))), or(shl(16, byte(3, w)), shl(24, byte(4, w)))) len := add(and(len, shr(sub(256, mul(8, smallLen)), not(0))), 1) inputPtr := add(inputPtr, smallLen) } - assert(not(gt(add(inputPtr, len), inputEnd)), errorSelector) // copy len bytes from input to output in chunks of 32 bytes calldatacopy(outputPtr, inputPtr, len) inputPtr := add(inputPtr, len) @@ -177,24 +153,21 @@ library Snappy { continue } case 1 { - assert(lt(inputPtr, inputEnd), errorSelector) len := add(and(shr(2, c), 0x7), 4) offset := add(byte(1, w), shl(8, shr(5, c))) inputPtr := add(inputPtr, 1) } case 2 { - assert(lt(add(inputPtr, 1), inputEnd), errorSelector) len := add(shr(2, c), 1) offset := add(byte(1, w), shl(8, byte(2, w))) inputPtr := add(inputPtr, 2) } case 3 { - assert(lt(add(inputPtr, 3), inputEnd), errorSelector) + len := add(shr(2, c), 1) len := add(shr(2, c), 1) offset := add(add(byte(1, w), shl(8, byte(2, w))), add(shl(16, byte(3, w)), shl(24, byte(4, w)))) inputPtr := add(inputPtr, 4) } - assert(and(iszero(iszero(offset)), not(gt(offset, sub(outputPtr, add(output, 0x20))))), errorSelector) // copying in will not work if the offset is larger than the len being copied, so we compute // `step = Math.min(len, offset)` and use it for the memory copy in chunks for { @@ -208,8 +181,10 @@ library Snappy { } outputPtr := add(outputPtr, len) } - // sanity check, FMP is at the right location - assert(eq(outputPtr, mload(0x40)), errorSelector) + // sanity check, we did not read more than the input and FMP is at the right location + if iszero(and(eq(inputPtr, inputEnd), eq(outputPtr, mload(0x40)))) { + revert(0, 0) + } } } }