diff --git a/README.md b/README.md index b3d71c6..28e6f8f 100644 --- a/README.md +++ b/README.md @@ -19,4 +19,46 @@ A: `linux/amd64`, `linux/arm64`, `darwin/arm64`. Pull requests for other platfor **Q: I don't trust `libscroll_zstd*.a` binary files from the repo or these files don't work on my OS/ARCH. How to rebuild them?** -A: Just run `cd libzstd && make libzstd` if your OS/ARCH is supported. +A: To rebuild the libraries for your platform: + +1. Build the legacy encoder: + + ```bash + cd libzstd/encoder-legacy + make install + ``` + +2. Build the standard encoder: + + ```bash + cd libzstd/encoder-standard + make install + ``` + +3. Add symbol prefixes to avoid conflicts: + + ```bash + cd encoding/zstd + ./add_symbol_prefix.sh + ``` + + **Note**: The symbol prefix script currently only works on macOS. For Linux builds, perform steps 1-2 in Docker, then run step 3 on macOS. + + For macOS builds, ensure you have Rust and necessary build tools installed: + + ```bash + # Install Rust if not already installed + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + ``` + + For Linux builds, use Docker with build dependencies: + + ```bash + # Linux ARM64 + docker run -it --rm --platform linux/arm64 -v $(pwd):/workspace -w /workspace rust:1.75-slim bash + apt update && apt install -y build-essential + + # Linux AMD64 + docker run -it --rm --platform linux/amd64 -v $(pwd):/workspace -w /workspace rust:1.75-slim bash + apt update && apt install -y build-essential + ``` diff --git a/encoding/codecv0.go b/encoding/codecv0.go index 0e8b70f..617d34c 100644 --- a/encoding/codecv0.go +++ b/encoding/codecv0.go @@ -430,3 +430,7 @@ func (d *DACodecV0) computeBatchDataHash(chunks []*Chunk, totalL1MessagePoppedBe dataHash := crypto.Keccak256Hash(dataBytes) return dataHash, nil } + +func (d *DACodecV0) CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { + return batchBytes, nil +} diff --git a/encoding/codecv2.go b/encoding/codecv2.go index fe2d338..1c90f86 100644 --- a/encoding/codecv2.go +++ b/encoding/codecv2.go @@ -154,7 +154,7 @@ func (d *DACodecV2) constructBlobPayload(chunks []*Chunk, maxNumChunksPerBatch i copy(challengePreimage[0:], hash[:]) // blobBytes represents the compressed blob payload (batchBytes) - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return nil, common.Hash{}, nil, nil, common.Hash{}, err } @@ -236,7 +236,7 @@ func (d *DACodecV2) EstimateChunkL1CommitBatchSizeAndBlobSize(c *Chunk) (uint64, if err != nil { return 0, 0, fmt.Errorf("failed to construct batch payload in blob: %w", err) } - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return 0, 0, fmt.Errorf("failed to compress scroll batch bytes: %w", err) } @@ -249,7 +249,7 @@ func (d *DACodecV2) EstimateBatchL1CommitBatchSizeAndBlobSize(b *Batch) (uint64, if err != nil { return 0, 0, err } - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return 0, 0, err } @@ -263,7 +263,7 @@ func (d *DACodecV2) checkCompressedDataCompatibility(chunks []*Chunk) (bool, err if err != nil { return false, fmt.Errorf("failed to construct batch payload in blob: %w", err) } - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return false, fmt.Errorf("failed to compress scroll batch bytes: %w", err) } @@ -289,3 +289,8 @@ func (d *DACodecV2) CheckChunkCompressedDataCompatibility(c *Chunk) (bool, error func (d *DACodecV2) CheckBatchCompressedDataCompatibility(b *Batch) (bool, error) { return d.checkCompressedDataCompatibility(b.Chunks) } + +// CompressScrollBatchBytes compresses the batch bytes using zstd compression. +func (d *DACodecV2) CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { + return zstd.CompressScrollBatchBytesLegacy(batchBytes) +} diff --git a/encoding/codecv4.go b/encoding/codecv4.go index 8ab6d20..396018d 100644 --- a/encoding/codecv4.go +++ b/encoding/codecv4.go @@ -14,8 +14,6 @@ import ( "github.com/scroll-tech/go-ethereum/crypto" "github.com/scroll-tech/go-ethereum/crypto/kzg4844" "github.com/scroll-tech/go-ethereum/log" - - "github.com/scroll-tech/da-codec/encoding/zstd" ) type DACodecV4 struct { @@ -205,7 +203,7 @@ func (d *DACodecV4) constructBlobPayload(chunks []*Chunk, maxNumChunksPerBatch i if enableCompression { // blobBytes represents the compressed blob payload (batchBytes) var err error - blobBytes, err = zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err = d.CompressScrollBatchBytes(batchBytes) if err != nil { return nil, common.Hash{}, nil, nil, common.Hash{}, err } @@ -267,7 +265,7 @@ func (d *DACodecV4) estimateL1CommitBatchSizeAndBlobSize(chunks []*Chunk) (uint6 return 0, 0, fmt.Errorf("failed to compress scroll batch bytes: %w", err) } if enableCompression { - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return 0, 0, err } @@ -295,7 +293,7 @@ func (d *DACodecV4) checkCompressedDataCompatibility(chunks []*Chunk) (bool, err if err != nil { return false, fmt.Errorf("failed to construct batch payload in blob: %w", err) } - blobBytes, err := zstd.CompressScrollBatchBytes(batchBytes) + blobBytes, err := d.CompressScrollBatchBytes(batchBytes) if err != nil { return false, fmt.Errorf("failed to compress scroll batch bytes: %w", err) } diff --git a/encoding/codecv7.go b/encoding/codecv7.go index f4615dc..8cacb0c 100644 --- a/encoding/codecv7.go +++ b/encoding/codecv7.go @@ -17,10 +17,15 @@ import ( "github.com/scroll-tech/da-codec/encoding/zstd" ) -type DACodecV7 struct{} +type DACodecV7 struct { + forcedVersion *CodecVersion +} // Version returns the codec version. func (d *DACodecV7) Version() CodecVersion { + if d.forcedVersion != nil { + return *d.forcedVersion + } return CodecV7 } @@ -86,7 +91,7 @@ func (d *DACodecV7) NewDABatch(batch *Batch) (DABatch, error) { return nil, fmt.Errorf("failed to construct blob: %w", err) } - daBatch, err := newDABatchV7(CodecV7, batch.Index, blobVersionedHash, batch.ParentBatchHash, blob, blobBytes, challengeDigest) + daBatch, err := newDABatchV7(d.Version(), batch.Index, blobVersionedHash, batch.ParentBatchHash, blob, blobBytes, challengeDigest) if err != nil { return nil, fmt.Errorf("failed to construct DABatch: %w", err) } @@ -115,7 +120,7 @@ func (d *DACodecV7) constructBlob(batch *Batch) (*kzg4844.Blob, common.Hash, []b sizeSlice := encodeSize3Bytes(uint32(len(payloadBytes))) - blobBytes[blobEnvelopeV7OffsetVersion] = uint8(CodecV7) + blobBytes[blobEnvelopeV7OffsetVersion] = uint8(d.Version()) copy(blobBytes[blobEnvelopeV7OffsetByteSize:blobEnvelopeV7OffsetCompressedFlag], sizeSlice) blobBytes[blobEnvelopeV7OffsetCompressedFlag] = isCompressedFlag blobBytes = append(blobBytes, payloadBytes...) @@ -166,15 +171,15 @@ func (d *DACodecV7) NewDABatchFromBytes(data []byte) (DABatch, error) { return nil, fmt.Errorf("failed to decode DA batch: %w", err) } - if daBatch.version != CodecV7 { - return nil, fmt.Errorf("codec version mismatch: expected %d but found %d", CodecV7, daBatch.version) + if daBatch.version != d.Version() { + return nil, fmt.Errorf("codec version mismatch: expected %d but found %d", d.Version(), daBatch.version) } return daBatch, nil } func (d *DACodecV7) NewDABatchFromParams(batchIndex uint64, blobVersionedHash, parentBatchHash common.Hash) (DABatch, error) { - return newDABatchV7(CodecV7, batchIndex, blobVersionedHash, parentBatchHash, nil, nil, common.Hash{}) + return newDABatchV7(d.Version(), batchIndex, blobVersionedHash, parentBatchHash, nil, nil, common.Hash{}) } func (d *DACodecV7) DecodeDAChunksRawTx(_ [][]byte) ([]*DAChunkRawTx, error) { @@ -186,8 +191,8 @@ func (d *DACodecV7) DecodeBlob(blob *kzg4844.Blob) (DABlobPayload, error) { // read the blob envelope header version := rawBytes[blobEnvelopeV7OffsetVersion] - if CodecVersion(version) != CodecV7 { - return nil, fmt.Errorf("codec version mismatch: expected %d but found %d", CodecV7, version) + if CodecVersion(version) != d.Version() { + return nil, fmt.Errorf("codec version mismatch: expected %d but found %d", d.Version(), version) } // read the data size @@ -229,7 +234,7 @@ func (d *DACodecV7) DecodeTxsFromBlob(blob *kzg4844.Blob, chunks []*DAChunkRawTx // If checkLength is true, this function returns if compression is needed based on the compressed data's length, which is used when doing batch bytes encoding. // If checkLength is false, this function returns the result of the compatibility check, which is used when determining the chunk and batch contents. func (d *DACodecV7) checkCompressedDataCompatibility(payloadBytes []byte, checkLength bool) ([]byte, bool, error) { - compressedPayloadBytes, err := zstd.CompressScrollBatchBytes(payloadBytes) + compressedPayloadBytes, err := d.CompressScrollBatchBytes(payloadBytes) if err != nil { return nil, false, fmt.Errorf("failed to compress blob payload: %w", err) } @@ -383,3 +388,8 @@ func (d *DACodecV7) JSONFromBytes(data []byte) ([]byte, error) { return jsonBytes, nil } + +// CompressScrollBatchBytes compresses the batch bytes using zstd compression. +func (d *DACodecV7) CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { + return zstd.CompressScrollBatchBytesLegacy(batchBytes) +} diff --git a/encoding/codecv8.go b/encoding/codecv8.go new file mode 100644 index 0000000..fe93048 --- /dev/null +++ b/encoding/codecv8.go @@ -0,0 +1,227 @@ +package encoding + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/crypto" + "github.com/scroll-tech/go-ethereum/crypto/kzg4844" + "github.com/scroll-tech/go-ethereum/log" + + "github.com/scroll-tech/da-codec/encoding/zstd" +) + +// DACodecV8 uses CompressScrollBatchBytesStandard for compression instead of CompressScrollBatchBytesLegacy. +// +// Note: Due to Go's method receiver behavior, we need to override all methods that call checkCompressedDataCompatibility. +// When a method in DACodecV7 calls d.checkCompressedDataCompatibility(), it will always use DACodecV7's version, +// even if the instance is actually a DACodecV8. Therefore, we must override: +// - checkCompressedDataCompatibility (core method using the new compression) +// - constructBlob (calls checkCompressedDataCompatibility) +// - NewDABatch (calls constructBlob) +// - CheckChunkCompressedDataCompatibility (calls checkCompressedDataCompatibility) +// - CheckBatchCompressedDataCompatibility (calls checkCompressedDataCompatibility) +// - estimateL1CommitBatchSizeAndBlobSize (calls checkCompressedDataCompatibility) +// - EstimateChunkL1CommitBatchSizeAndBlobSize (calls estimateL1CommitBatchSizeAndBlobSize) +// - EstimateBatchL1CommitBatchSizeAndBlobSize (calls estimateL1CommitBatchSizeAndBlobSize) +type DACodecV8 struct { + DACodecV7 +} + +func NewDACodecV8() *DACodecV8 { + v := CodecV8 + return &DACodecV8{ + DACodecV7: DACodecV7{ + forcedVersion: &v, + }, + } +} + +// checkCompressedDataCompatibility checks the compressed data compatibility for a batch. +// It constructs a blob payload, compresses the data, and checks the compressed data compatibility. +// flag checkLength indicates whether to check the length of the compressed data against the original data. +// If checkLength is true, this function returns if compression is needed based on the compressed data's length, which is used when doing batch bytes encoding. +// If checkLength is false, this function returns the result of the compatibility check, which is used when determining the chunk and batch contents. +func (d *DACodecV8) checkCompressedDataCompatibility(payloadBytes []byte, checkLength bool) ([]byte, bool, error) { + compressedPayloadBytes, err := d.CompressScrollBatchBytes(payloadBytes) + if err != nil { + return nil, false, fmt.Errorf("failed to compress blob payload: %w", err) + } + + if err = checkCompressedDataCompatibilityV7(compressedPayloadBytes); err != nil { + log.Warn("Compressed data compatibility check failed", "err", err, "payloadBytes", hex.EncodeToString(payloadBytes), "compressedPayloadBytes", hex.EncodeToString(compressedPayloadBytes)) + return nil, false, nil + } + + // check if compressed data is bigger or equal to the original data -> no need to compress + if checkLength && len(compressedPayloadBytes) >= len(payloadBytes) { + log.Warn("Compressed data is bigger or equal to the original data", "payloadBytes", hex.EncodeToString(payloadBytes), "compressedPayloadBytes", hex.EncodeToString(compressedPayloadBytes)) + return nil, false, nil + } + + return compressedPayloadBytes, true, nil +} + +// NewDABatch creates a DABatch including blob from the provided Batch. +func (d *DACodecV8) NewDABatch(batch *Batch) (DABatch, error) { + if len(batch.Blocks) == 0 { + return nil, errors.New("batch must contain at least one block") + } + + if err := checkBlocksBatchVSChunksConsistency(batch); err != nil { + return nil, fmt.Errorf("failed to check blocks batch vs chunks consistency: %w", err) + } + + blob, blobVersionedHash, blobBytes, challengeDigest, err := d.constructBlob(batch) + if err != nil { + return nil, fmt.Errorf("failed to construct blob: %w", err) + } + + daBatch, err := newDABatchV7(d.Version(), batch.Index, blobVersionedHash, batch.ParentBatchHash, blob, blobBytes, challengeDigest) + if err != nil { + return nil, fmt.Errorf("failed to construct DABatch: %w", err) + } + + return daBatch, nil +} + +func (d *DACodecV8) constructBlob(batch *Batch) (*kzg4844.Blob, common.Hash, []byte, common.Hash, error) { + blobBytes := make([]byte, blobEnvelopeV7OffsetPayload) + + payloadBytes, err := d.constructBlobPayload(batch) + if err != nil { + return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to construct blob payload: %w", err) + } + + compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes, true /* checkLength */) + if err != nil { + return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) + } + + isCompressedFlag := uint8(0x0) + if enableCompression { + isCompressedFlag = 0x1 + payloadBytes = compressedPayloadBytes + } + + sizeSlice := encodeSize3Bytes(uint32(len(payloadBytes))) + + blobBytes[blobEnvelopeV7OffsetVersion] = uint8(d.Version()) + copy(blobBytes[blobEnvelopeV7OffsetByteSize:blobEnvelopeV7OffsetCompressedFlag], sizeSlice) + blobBytes[blobEnvelopeV7OffsetCompressedFlag] = isCompressedFlag + blobBytes = append(blobBytes, payloadBytes...) + + if len(blobBytes) > maxEffectiveBlobBytes { + log.Error("ConstructBlob: Blob payload exceeds maximum size", "size", len(blobBytes), "blobBytes", hex.EncodeToString(blobBytes)) + return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("blob exceeds maximum size: got %d, allowed %d", len(blobBytes), maxEffectiveBlobBytes) + } + + // convert raw data to BLSFieldElements + blob, err := makeBlobCanonical(blobBytes) + if err != nil { + return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to convert blobBytes to canonical form: %w", err) + } + + // compute blob versioned hash + c, err := kzg4844.BlobToCommitment(blob) + if err != nil { + return nil, common.Hash{}, nil, common.Hash{}, fmt.Errorf("failed to create blob commitment: %w", err) + } + blobVersionedHash := kzg4844.CalcBlobHashV1(sha256.New(), &c) + + // compute challenge digest for codecv7, different from previous versions, + // the blob bytes are padded to the max effective blob size, which is 131072 / 32 * 31 due to the blob encoding + paddedBlobBytes := make([]byte, maxEffectiveBlobBytes) + copy(paddedBlobBytes, blobBytes) + + challengeDigest := crypto.Keccak256Hash(crypto.Keccak256(paddedBlobBytes), blobVersionedHash[:]) + + return blob, blobVersionedHash, blobBytes, challengeDigest, nil +} + +// CheckChunkCompressedDataCompatibility checks the compressed data compatibility for a batch built from a single chunk. +func (d *DACodecV8) CheckChunkCompressedDataCompatibility(c *Chunk) (bool, error) { + // filling the needed fields for the batch used in the check + b := &Batch{ + Chunks: []*Chunk{c}, + PrevL1MessageQueueHash: c.PrevL1MessageQueueHash, + PostL1MessageQueueHash: c.PostL1MessageQueueHash, + Blocks: c.Blocks, + } + + return d.CheckBatchCompressedDataCompatibility(b) +} + +// CheckBatchCompressedDataCompatibility checks the compressed data compatibility for a batch. +func (d *DACodecV8) CheckBatchCompressedDataCompatibility(b *Batch) (bool, error) { + if len(b.Blocks) == 0 { + return false, errors.New("batch must contain at least one block") + } + + if err := checkBlocksBatchVSChunksConsistency(b); err != nil { + return false, fmt.Errorf("failed to check blocks batch vs chunks consistency: %w", err) + } + + payloadBytes, err := d.constructBlobPayload(b) + if err != nil { + return false, fmt.Errorf("failed to construct blob payload: %w", err) + } + + // This check is only used for sanity checks. If the check fails, it means that the compression did not work as expected. + // rollup-relayer will try popping the last chunk of the batch (or last block of the chunk when in proposing chunks) and try again to see if it works as expected. + // Since length check is used for DA and proving efficiency, it does not need to be checked here. + _, compatible, err := d.checkCompressedDataCompatibility(payloadBytes, false /* checkLength */) + if err != nil { + return false, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) + } + + return compatible, nil +} + +func (d *DACodecV8) estimateL1CommitBatchSizeAndBlobSize(batch *Batch) (uint64, uint64, error) { + if len(batch.Blocks) == 0 { + return 0, 0, errors.New("batch must contain at least one block") + } + + blobBytes := make([]byte, blobEnvelopeV7OffsetPayload) + + payloadBytes, err := d.constructBlobPayload(batch) + if err != nil { + return 0, 0, fmt.Errorf("failed to construct blob payload: %w", err) + } + + compressedPayloadBytes, enableCompression, err := d.checkCompressedDataCompatibility(payloadBytes, true /* checkLength */) + if err != nil { + return 0, 0, fmt.Errorf("failed to check batch compressed data compatibility: %w", err) + } + + if enableCompression { + blobBytes = append(blobBytes, compressedPayloadBytes...) + } else { + blobBytes = append(blobBytes, payloadBytes...) + } + + return blobEnvelopeV7OffsetPayload + uint64(len(payloadBytes)), calculatePaddedBlobSize(uint64(len(blobBytes))), nil +} + +// EstimateChunkL1CommitBatchSizeAndBlobSize estimates the L1 commit batch size and blob size for a single chunk. +func (d *DACodecV8) EstimateChunkL1CommitBatchSizeAndBlobSize(chunk *Chunk) (uint64, uint64, error) { + return d.estimateL1CommitBatchSizeAndBlobSize(&Batch{ + Blocks: chunk.Blocks, + PrevL1MessageQueueHash: chunk.PrevL1MessageQueueHash, + PostL1MessageQueueHash: chunk.PostL1MessageQueueHash, + }) +} + +// EstimateBatchL1CommitBatchSizeAndBlobSize estimates the L1 commit batch size and blob size for a batch. +func (d *DACodecV8) EstimateBatchL1CommitBatchSizeAndBlobSize(batch *Batch) (uint64, uint64, error) { + return d.estimateL1CommitBatchSizeAndBlobSize(batch) +} + +// CompressScrollBatchBytes compresses the batch bytes using zstd compression. +func (d *DACodecV8) CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { + return zstd.CompressScrollBatchBytesStandard(batchBytes) +} diff --git a/encoding/codecv8_test.go b/encoding/codecv8_test.go new file mode 100644 index 0000000..bd075ba --- /dev/null +++ b/encoding/codecv8_test.go @@ -0,0 +1,388 @@ +package encoding + +import ( + "encoding/hex" + "math/big" + "strings" + "testing" + + "github.com/agiledragon/gomonkey/v2" + "github.com/scroll-tech/go-ethereum/common" + "github.com/scroll-tech/go-ethereum/common/hexutil" + "github.com/scroll-tech/go-ethereum/core/types" + "github.com/stretchr/testify/require" +) + +// TestCodecV8DABatchHashEncodeDecode tests the hash, encoding and decoding of daBatchV8. +// It also tests the creation of daBatchV8 FromBytes and FromParams. +func TestCodecV8DABatchHashEncodeDecode(t *testing.T) { + codecV8, err := CodecFromVersion(CodecV8) + require.NoError(t, err) + + testCases := []struct { + name string + batch *Batch + expectedEncode string + expectedHash string + creationErr string + }{ + { + name: "Empty Batch, creation error=no blocks", + batch: &Batch{}, + creationErr: "batch must contain at least one block", + }, + { + name: "Batch with 1 block,blocktrace 02", + batch: &Batch{ + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json")}, + }, + expectedEncode: "080000000000000000018a909478d405142cd69d5caa3f98d118d4b362d0433ebb7f9b039a3fe781c90000000000000000000000000000000000000000000000000000000000000000", + expectedHash: "0x496e1cf0b8b26cf352d1a3150823795e805e133258b15dc14a4a4e6ad1cae0d1", + }, + { + name: "Batch with 1 block, blocktrace 06, creation error=L1 messages not consecutive", + batch: &Batch{ + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_06.json")}, + }, + creationErr: "unexpected queue index", + }, + { + name: "Batch with 3 blocks, blocktrace 02, 03, 04", + batch: &Batch{ + PostL1MessageQueueHash: common.HexToHash("0xc7436aaec2cfaf39d5be02a02c6ac2089ab264c3e0fd142db682f1cc00000000"), + Blocks: []*Block{ + readBlockFromJSON(t, "testdata/blockTrace_02.json"), + readBlockFromJSON(t, "testdata/blockTrace_03.json"), + replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_04.json"), 4), + }, + }, + expectedEncode: "08000000000000000001fa8374627fa9c30f3e858435f32fab36e403857ddd2a5f5b5fb693a08f38790000000000000000000000000000000000000000000000000000000000000000", + expectedHash: "0xea506ec9ebbda7d43aa76fae1a0c105258dd2ebdb20a5b53e2227312517aa510", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + daBatchV8i, err := codecV8.NewDABatch(tc.batch) + if tc.creationErr != "" { + require.ErrorContains(t, err, tc.creationErr) + return + } + + require.NoError(t, err) + daBatchV8c := daBatchV8i.(*daBatchV7) + + encoded := daBatchV8c.Encode() + require.Equal(t, tc.expectedEncode, hex.EncodeToString(encoded)) + require.Equal(t, tc.expectedHash, daBatchV8c.Hash().Hex()) + + // test DABatchFromBytes + batchDecoded, err := codecV8.NewDABatchFromBytes(encoded) + batchDecodedV8 := batchDecoded.(*daBatchV7) + require.NoError(t, err) + require.Equal(t, daBatchV8c.version, batchDecodedV8.version) + require.Equal(t, daBatchV8c.batchIndex, batchDecodedV8.batchIndex) + require.Equal(t, daBatchV8c.blobVersionedHash, batchDecodedV8.blobVersionedHash) + require.Equal(t, daBatchV8c.parentBatchHash, batchDecodedV8.parentBatchHash) + require.Nil(t, batchDecodedV8.blob) + require.Nil(t, batchDecodedV8.blobBytes) + require.Equal(t, daBatchV8c.Hash(), batchDecoded.Hash()) + require.Equal(t, daBatchV8c.Encode(), batchDecoded.Encode()) + + // test DABatchFromParams + batchFromParams, err := codecV8.NewDABatchFromParams(daBatchV8c.batchIndex, daBatchV8c.blobVersionedHash, daBatchV8c.parentBatchHash) + require.NoError(t, err) + batchFromParamsV8 := batchFromParams.(*daBatchV7) + require.Equal(t, daBatchV8c.version, batchFromParamsV8.version) + require.Equal(t, daBatchV8c.batchIndex, batchFromParamsV8.batchIndex) + require.Equal(t, daBatchV8c.blobVersionedHash, batchFromParamsV8.blobVersionedHash) + require.Equal(t, daBatchV8c.parentBatchHash, batchFromParamsV8.parentBatchHash) + require.Nil(t, batchFromParamsV8.blob) + require.Nil(t, batchFromParamsV8.blobBytes) + require.Equal(t, daBatchV8c.Hash(), batchFromParams.Hash()) + require.Equal(t, daBatchV8c.Encode(), batchFromParams.Encode()) + }) + } +} + +func TestCodecV8BlobEncodingAndHashing(t *testing.T) { + codecV8, err := CodecFromVersion(CodecV8) + require.NoError(t, err) + require.EqualValues(t, CodecV8, codecV8.Version()) + + testCases := []struct { + name string + batch *Batch + creationErr string + expectedBlobEncode string + expectedBlobVersionedHash string + }{ + { + name: "Empty batch", + batch: &Batch{ + Index: 1, + ParentBatchHash: common.Hash{}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.Hash{}, + Blocks: []*Block{}, + }, + creationErr: "batch must contain at least one block", + }, + { + name: "Batch with 1 block, blocktrace 02", + batch: &Batch{ + Index: 1, + ParentBatchHash: common.Hash{}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.Hash{}, + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json")}, + }, + expectedBlobEncode: "00080000f9016064009d0700240e000002000163807b2a1de9000355418d1e81008400020000f87180843b9aec2e8307a12094c0c4c8baea3f6acb49b6e1fb9e002adeceeacb0ca28a152d02c7e14af60000008083019ecea0ab07ae99c67aa7008e7ba5cf6781e90cc32b219b1de102513d56548a41e86df514a034cbd19fea00cd73e8ce64d00c4d1996b9b5243c578fd7f51bfaec288bbaf42a8bf871010100bae6bf68e9a03fb2bc0615b1bf0d69ce9411edf039985866d8256f10c1be4f007b2cace28d8f20bde27e2604393eb095b7f77316a05a3e6e81065f2b4604bc00ec5bd4aba684835996fc3f879380aac1c09c6eed32f105006032821d6009420094b00e410116", + expectedBlobVersionedHash: "0x018a909478d405142cd69d5caa3f98d118d4b362d0433ebb7f9b039a3fe781c9", + }, + { + name: "Batch with 1 blocks, blocktrace 04 - 1 L1 message + 1 L2 tx", + batch: &Batch{ + Index: 1, + ParentBatchHash: common.Hash{}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.HexToHash("0xc7436aaec2cfaf39d5be02a02c6ac2089ab264c3e0fd142db682f1cc00000000"), + Blocks: []*Block{replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_04.json"), 4)}, + }, + expectedBlobEncode: "000800006201209eed0200d4040000c7436aaec2cfaf39d5be02a02c6ac2089a00b264c3e0fd142db682f1cc00040001646b6e137a120000020001df0b80825d00c0941a258d17bf244c4df02d40343a7626a9d321e105808080808005003906006e16790923b039116001", + expectedBlobVersionedHash: "0x01e37c62837a7232130ca356c3eac9db27533a6136eb6fd1433df9418cf22716", + }, + { + name: "Batch with 3 blocks, blocktrace 02 + 03 + 04", + batch: &Batch{ + Index: 1, + ParentBatchHash: common.Hash{}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.HexToHash("0xc7436aaec2cfaf39d5be02a02c6ac2089ab264c3e0fd142db682f1cc00000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), readBlockFromJSON(t, "testdata/blockTrace_03.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_04.json"), 4)}, + }, + expectedBlobEncode: "0008000c6601601d1705630094960000c7436aaec2cfaf39d5be02a02c6ac208009ab264c3e0fd142db682f1cc0002000363807b2a1de9000355418d1e818400000263807b2d1a2c0003546c3cbb39e50001646b6e137a120000020001f8718000843b9aec2e8307a12094c0c4c8baea3f6acb49b6e1fb9e2adeceeacb0ca28a00152d02c7e14af60000008083019ecea0ab07ae99c67aa78e7ba5cf6781e90c00c32b219b1de102513d56548a41e86df514a034cbd19feacd73e8ce64d00c4d001996b9b5243c578fd7f51bfaec288bbaf42a8bf8710101bae6bf68e9a03fb200bc0615b1bf0d69ce9411edf039985866d8256f10c1be4f7b2cace28d8f20bd00e27e2604393eb095b7f77316a05a3e6e81065f2b4604bcec5bd4aba68483590096fc3f879380aac1c09c6eed32f102f9162d82cf5502843b9b0a17831197e2008080b915d260806040523480156200001157600080fd5b50604051620014b200380380833981810160405260a08110378151602083015160408085018051910051939592948301929184648211639083019060208201858179825181118282000188101794825250918201929091019080838360005b83c357818101518382000152602001620000a9565b50505050905090810190601f16f1578082038051006001836020036101000a0319168191508051604051939291900115012b0146000175015b01a39081015185519093508592508491620001c891600391850190006200026b565b508051620001de90600490602084506005805461ff001960ff001990911660121716905550600680546001600160a01b0380881619928316170090925560078054928716929091169190911790556200023081620002556201000000600160b01b03191633021790555062000307915050565b60ff191660ff00929092565b828160011615610100020316600290049060005260206000209000601f016020900481019282601f10620002ae5780518380011785de016001010085558215620002de579182015b8202de57825182559160200191906001c156005b50620002ec9291f0565b5090565b5b8002ec5760008155600101620002f100565b61119b80620003176000396000f3fe61001004361061010b576000356000e01c80635c975abb116100a257806395d89b4111610071146103015780639d00c29fac14610309578063a457c2d714610335578063a9059cbb1461036157800063dd62ed3e1461038d5761010b565b1461029d57806370a08231146102a5570080638456cb59146102cb5780638e50817a146102d3313ce567116100de57140061021d578063395093511461023b5780633f4ba83a1461026757806340c10f00191461027106fdde0314610110578063095ea7b31461018d57806318160ddd00146101cd57806323b872e7575b6101186103bb565b6040805160208082528300518183015283519192839290830161015261013a61017f92505080910390f3005b6101b9600480360360408110156101a381351690602001356104519115150082525190819003602001d561046e60fd81169160208101359091169060407400565b6102256104fb60ff90921640025105046f610552565b005b61026f02870005a956610654d520bb3516610662067d56e90135166106d218610757031f0700b856034b085f77c7d5a308db565b6003805420601f600260001961010060010088161502019095169490940493840181900481028201810190925282815260006093909290918301828280156104475780601f1061041c57610100808354040002835291610447565b825b8154815260200180831161042a57829003601f1600820191565b600061046561045e610906565b848461090a565b506001920254008184f6565b6104f18461048d6104ec8560405180606080602861108560289100398a166000908152600160205260408120906104cb81019190915260400160000020549190610b51565b935460ff160511016000610522908116825260208000830193909352604091820120918c168152925290205490610be8565b60071600331461059f5762461bcd60e51b60040b60248201526a1b9bdd08185b1b1bdd00d95960aa1b604482015290640190fd5b6105a7610c49565b610100900460ff0016156105f9106f14185d5cd8589b194e881c185d5cd9596082600606460650008282610ced909052604006ca0ddd900407260c6b6f6e6c7920466163746f72007960a0079283918216179091559390921660041561080808550e65086c251100176025006108968dd491824080832093909416825233831661094f5704018000806020018281038252602401806110f36024913960400191fd821661099422003d60228084166000819487168084529482529182902085905581518581529100517f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c700c3b92592819003a3508316610a3b25ce8216610a80230ff86023610a8b83830083610f61565b610ac881265f60268685808220939093559084168152205461000af7908220409490945580905191937fddf252ad1be2c89b69c2b068fc378d00aa952ba7f163c4a11628f55a4df523b3ef9291829003008184841115610be0008381815191508051900ba50b8d0bd2fd900300828201610c421b7f53616665004d6174683a206164646974696f6e206f766572666c6f7700610c9c147362160090557f5db9ee0a495bf2e6ff9c91a7834c1ba4fdd244a5e8aa4e537bd38aea00e4b073aa610cd0a18216610d481f7f45524332303a206d696e7420746f2074006865207a65726f72657373610d546000600254610d610255902054610d878000838393519293910e2d6101001790557f62e78cea01bee320cd4e420270b5ea0074000d11b0c9f74754ebdbfc544b05a2588216610eaa6021ad6021610eb6820060000ef3221b85839020550f199082610fb540805182600091851691912056005b610f6cb07415610fb02a113c602a00610c428383401e7375627472616381005250fe7472616e736665726275726e20616d6f756e742065786365656473200062616c616e6365617070726f7665616c6c6f7766726f6d646563726561736500642062656c6f775061757361626c653a20746f6b656e7768696c652070617500736564a2646970667358221220e96342bec8f6c2bf72815a39998973b64c3b00ed57770f402e9a7b7eeda0265d4c64736f6c634300060c00331c5a77d9fa7e00f466951b2f01f724bca3a5820b63a0e012095745544820636f696e04c001a000235c1a8d40e8c347890397f1a92e6eadbd6422cf7c210e3e1737f0553c63310072a02f7c0384ddd06970446e74229cd96216da62196dc62395bda52095d44b008a9af7df0b80825dc0941a258d17bf244c4df02d40343a7626a9d321e105800080808080814aa8d130a9149a111111110549d2741105c418e61894eb01122000132dcb629c42c818e2c88850202223220549523307f06170f01bb60bc5a52b00d26b2c50bad4e2b2035c47a34038481f5c57890a393e7e010458ce81f09e490072a01121c9ac5b68d650819d732c48a37a11abde6747197969db3cb19efb34000e5599ae00e23d0c1e49e79382d1b7c50c62a112c690260d768f1c65149c24001c46f8708563b416f0f7a0dd98b6e0f685c551512e38ef2e20fe9f579ca4c6006146e0985928fbf8d1a3a8f3fac0c3a31ed348ff91848772604bcff17431fd0084da90ceacd54940452eea0ba22994bea2805624c107ee8ec4dec3a15897b000e2a6aa06fc4bd43eb964968fda4fd4ee7ee46d906ce01a8541022f5f0ab7da0067a5c348265c5652e06f484c92ff6d3158ec85cf1092226b8485c75ab799fc0012c9466daee43ce9132560a312094d536f7803541ca00845e3d84ea9b273c2004c15a17411e1331288b705778e3b7dc7830db1848b47113243da2e0d1012ba002a04a32898f050a8b61a02a7e6dc85eef13556ef249af7dd1d57508d3f7d2900af03f968d912ae39d330fcb9741b8010d4b0f0412f812401338eae5d10ef58001f23b0deea72f11e23aa1325a65d4d19e42222a0ef8b27aa7025bba4688c7900ccd826fa2c1ef64745e50f55e969672665104aed27e28c7adb52800be035d9003cf8163c31b92c81d7e14f7ab6ae1bf743b28fe7977924ce24fef45db39748001d0ce8628719fec729b5aa534c3df0e711d489ce2708547335ef2802fc757e00c3e7cb7b5bbf2ddd70d69a2ea5b9c345f34c9870d0a9f91cac0933a6a9b1e2005db9667ae51967d0165d126e7efc943894231266bafa034b808cd857a8972c00f5b2a95e79f52ec06822723b406fa7fe93a9fd522a9d5aa548c98ecc78d66b0084fd4f11a50c68c03b65c11af468801b23e9f1937dd4a5e9e38157d596e09a00526d4cda44f62b86afc0a88cfeb0e65aa8dd623c9174dce1044907ecc6b9b700837e720e198f2c60f3b0d946652c4e87c6324e1d5895b0beb72535e48e48f400180fbb8c40200e7ab8387fb224c9ef820e", + expectedBlobVersionedHash: "0x01fa8374627fa9c30f3e858435f32fab36e403857ddd2a5f5b5fb693a08f3879", + }, + { + name: "Batch with 3 blocks, blocktrace 02 + 05 (L1 messages only) + 03", + batch: &Batch{ + Index: 3, + ParentBatchHash: common.Hash{2}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.HexToHash("0x3d35d6b71c2769de1a4eb8f603e20f539c53a10c6764a6f5836cf13100000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_05.json"), 3), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_03.json"), 4)}, + }, + expectedBlobEncode: "0008000c4d0160fd163d6200e49400003d35d6b71c2769de1a4eb8f603e20f53009c53a10c6764a6f5836cf1310002000363807b2a1de9000355418d1e8184000002646b6ed07a12000005000563807b2d1a2c0003546c3cbb39e500010000f8007180843b9aec2e8307a12094c0c4c8baea3f6acb49b6e1fb9e2adeceeacb0c00a28a152d02c7e14af60000008083019ecea0ab07ae99c67aa78e7ba5cf678100e90cc32b219b1de102513d56548a41e86df514a034cbd19feacd73e8ce64d0000c4d1996b9b5243c578fd7f51bfaec288bbaf42a8bf8710101bae6bf68e9a0003fb2bc0615b1bf0d69ce9411edf039985866d8256f10c1be4f7b2cace28d8f0020bde27e2604393eb095b7f77316a05a3e6e81065f2b4604bcec5bd4aba68400835996fc3f879380aac1c09c6eed32f102f9162d82cf5502843b9b0a1783110097e28080b915d260806040523480156200001157600080fd5b5060405162000014b2380380833981810160405260a0811037815160208301516040808501800051915193959294830192918464018211639083019060208201858179825181001182820188101794825250918201929091019080838360005b83c357818101005183820152602001620000a9565b50505050905090810190601f16f1578082000380516001836020036101000a031916819150805160405193929190011501002b01460175015b01a39081015185519093508592508491620001c891600391008501906200026b565b508051620001de90600490602084506005805461ff00001960ff1990911660121716905550600680546001600160a01b0380881619920083161790925560078054928716929091169190911790556200023081620002005562010000600160b01b03191633021790555062000307915050565b60ff19001660ff929092565b828160011615610100020316600290049060005260206000002090601f016020900481019282601f10620002ae5780518380011785de010060010185558215620002de579182015b8202de5782518255916020019190600001c1565b50620002ec9291f0565b5090565b5b8002ec576000815560010162000002f1565b61119b80620003176000396000f3fe61001004361061010b576000003560e01c80635c975abb116100a257806395d89b411161007114610301570080639dc29fac14610309578063a457c2d714610335578063a9059cbb1461030061578063dd62ed3e1461038d5761010b565b1461029d57806370a0823114610002a55780638456cb59146102cb5780638e50817a146102d3313ce56711610000de571461021d578063395093511461023b5780633f4ba83a146102675780630040c10f191461027106fdde0314610110578063095ea7b31461018d5780631800160ddd146101cd57806323b872e7575b6101186103bb565b6040805160208000825283518183015283519192839290830161015261013a61017f9250508091000390f35b6101b9600480360360408110156101a381351690602001356104510091151582525190819003602001d561046e60fd81169160208101359091169000604074565b6102256104fb60ff90921640025105046f610552565b005b6102006f028705a956610654d520bb3516610662067d56e90135166106d21861075700031f07b856034b085f77c7d5a308db565b6003805420601f600260001961010000600188161502019095169490940493840181900481028201810190925282008152606093909290918301828280156104475780601f1061041c57610100800083540402835291610447565b825b8154815260200180831161042a5782900300601f16820191565b600061046561045e610906565b848461090a565b506001009202548184f6565b6104f18461048d6104ec8560405180606080602861108500602891398a166000908152600160205260408120906104cb81019190915260004001600020549190610b51565b935460ff160511016000610522908116825200602080830193909352604091820120918c168152925290205490610be8565b00600716331461059f5762461bcd60e51b60040b60248201526a1b9bdd08185b001b1bddd95960aa1b604482015290640190fd5b6105a7610c49565b61010090000460ff16156105f9106f14185d5cd8589b194e881c185d5cd9596082600606004606508282610ced909052604006ca0ddd900407260c6b6f6e6c792046616300746f727960a0079283918216179091559390921660041561080808550e6508006c2511176025006108968dd491824080832093909416825233831661094f5700040180806020018281038252602401806110f36024913960400191fd821661000994223d60228084166000819487168084529482529182902085905581518500815291517f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b20000ac8c7c3b92592819003a3508316610a3b25ce8216610a80230ff86023610a008b838383610f61565b610ac881265f60268685808220939093559084168152002054610af7908220409490945580905191937fddf252ad1be2c89b69c2b06800fc378daa952ba7f163c4a11628f55a4df523b3ef929182900300818484111500610be08381815191508051900ba50b8d0bd2fd900300828201610c421b7f53006166654d6174683a206164646974696f6e206f766572666c6f7700610c9c140073621690557f5db9ee0a495bf2e6ff9c91a7834c1ba4fdd244a5e8aa4e537b00d38aeae4b073aa610cd0a18216610d481f7f45524332303a206d696e742074006f20746865207a65726f72657373610d546000600254610d61025590205461000d8780838393519293910e2d6101001790557f62e78cea01bee320cd4e42020070b5ea74000d11b0c9f74754ebdbfc544b05a2588216610eaa6021ad602161000eb68260000ef3221b85839020550f199082610fb540805182600091851691009120565b610f6cb07415610fb02a113c602a00610c428383401e7375627472006163815250fe7472616e736665726275726e20616d6f756e742065786365650064732062616c616e6365617070726f7665616c6c6f7766726f6d646563726500617365642062656c6f775061757361626c653a20746f6b656e7768696c652000706175736564a2646970667358221220e96342bec8f6c2bf72815a3999897300b64c3bed57770f402e9a7b7eeda0265d4c64736f6c634300060c003300001c005a77d9fa7ef466951b2f01f724bca3a5820b63a0e012095745544820636f69006e04c001a0235c1a8d40e8c347890397f1a92e6eadbd6422cf7c210e3e173700f0553c633172a02f7c0384ddd06970446e74229cd96216da62196dc62395bd00a52095d44b8a9af7814aa8c130a9143223222222122449930e1105c418e6180094eb0112200329cb6220859031c49111a1404466440a92a4660ef06170a037000416b7372a8c6e8610e806c5190aca8c469770741f3857ad8a9c183f16023c00ce819dcea439088d90346b7d9a15a9c08c3943a5b9b68857ffb9a320bf34c2005c209c3b3e0ebd4cbf01f19e046f26314fc1c4b7112ef67e09034893e2368c001c51141c130ef37bb8ee31aa0bf8ecd36e18b420a265035131bd73402ec0fd009157a4a7c65bb6e198d950e6e8a34151b7f5518a472d6623dd20097c94a32200ddd1d39ffa49b6e106b38e3b0167b8883163a684f495285ad1013e6477ccfb001e8e635d765b325513f0d7a2d6889d5977d4e6a276f4239f46b2f9cd0006a100bc7c1e6e359fe80ea39fd8594965fd86ef24613116ebda5e580f2175b3665800e8af559a89410280a84d3fb92d7d4a10b0a244bf186a0b8f0065661efb681c009a2bf5344000a982942e227c46caf376d69d7b8edac418405c59f18089cc9100edf20642c6ad8a63943113fe16eaa48610a99975e1f9d79ac13b3f27f1eebe00e9b88fdf62b6d75b7f346f09d739533afcb9341a80c4d4f0f0d197408580b900c48d118b87591f466003d5e5626f8c7027764cc99a02c7454640df14cf555100f376fd420accb9c756f3b300b03f2595cda88abc9df93383f168df218ea8a3002d015c205e13c383b7e0919247101825fc43cf76b071975276fb0c378fe74c00023ebd85ec65a70e46ba3866863b774a49ed14bac7e759187ca07bb2cdcb5700938e8243b6e4eb7eeebd9f34d7d2ba6795e9c285bbbb689aa9130eb49b9c230068f2c6dc08d6e22bc4a7473f9322f6ef928de6074f3143d52401d1556b59c20066c44fa19eadd403a57aa2abd700a32dc96d007ae1fb0ff9b6ea5201df2ad5005f763ce3ad5e23307d8ad2659503262f0bbe412f032c1a49849f96a3d64d5f0004dc552d08ae29d5c6f84d64bc6a920b988a5119d8ab05a98bc77b97ce249c00d07440f0b8f53ce8e74e8a95cbb05c1f88fca60135d2d9b68c7c07922a2d11006c6f77bc58631eacad8402292db87611f103ccd5de4517", + expectedBlobVersionedHash: "0x01bde0a113f62b38977bd51f0e33b5d1b64c6d14441d016f040c395229d7e549", + }, + // test error cases + { + name: "Batch with 3 blocks, blocktrace 02 + 05 (L1 messages only) + 03, but with wrong (not consecutive) block number", + batch: &Batch{ + Index: 3, + ParentBatchHash: common.Hash{2}, + PrevL1MessageQueueHash: common.Hash{}, + PostL1MessageQueueHash: common.HexToHash("0xfaa13a9ed8937474556dd2ea36be845199e823322cd63279a3ba300000000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), readBlockFromJSON(t, "testdata/blockTrace_05.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_03.json"), 4)}, + }, + creationErr: "invalid block number", + }, + { + name: "Batch with 3 blocks, blocktrace 02 + 05 (L1 messages only) + 03, but with wrong PostL1MessageQueueHash", + batch: &Batch{ + Index: 3, + ParentBatchHash: common.Hash{2}, + PrevL1MessageQueueHash: common.Hash{1}, + PostL1MessageQueueHash: common.HexToHash("0xfaa13a9ed8937474556dd2ea36be845199e823322cd63279a3ba300000000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_05.json"), 3), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_03.json"), 4)}, + }, + creationErr: "failed to sanity check postL1MessageQueueHash", + }, + { + name: "Batch with 3 blocks, blocktrace 02, 04 + 05 (L1 messages only), but with non-consecutive L1 messages number across blocks 04 and 05", + batch: &Batch{ + Index: 3, + ParentBatchHash: common.Hash{2}, + PrevL1MessageQueueHash: common.Hash{1}, + PostL1MessageQueueHash: common.HexToHash("0xfaa13a9ed8937474556dd2ea36be845199e823322cd63279a3ba300000000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_04.json"), 3), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_05.json"), 4)}, + }, + creationErr: "failed to sanity check L1 messages count", + }, + { + name: "Batch with 3 blocks, blocktrace 02, 06, but with non-consecutive L1 messages number within block 06", + batch: &Batch{ + Index: 3, + ParentBatchHash: common.Hash{2}, + PrevL1MessageQueueHash: common.Hash{1}, + PostL1MessageQueueHash: common.HexToHash("0xfaa13a9ed8937474556dd2ea36be845199e823322cd63279a3ba300000000000"), + Blocks: []*Block{readBlockFromJSON(t, "testdata/blockTrace_02.json"), replaceBlockNumber(readBlockFromJSON(t, "testdata/blockTrace_06.json"), 3)}, + }, + creationErr: "unexpected queue index", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var daBatch DABatch + daBatch, err := codecV8.NewDABatch(tc.batch) + if tc.creationErr != "" { + require.ErrorContains(t, err, tc.creationErr) + return + } + require.NoError(t, err) + + // check correctness of blob and blob hash + require.Equal(t, tc.expectedBlobEncode, strings.TrimRight(hex.EncodeToString(daBatch.Blob()[:]), "0")) + require.Equal(t, common.HexToHash(tc.expectedBlobVersionedHash), daBatch.(*daBatchV7).blobVersionedHash) + + // check correctness of blob decoding: blobPayload metadata + blobPayload, err := codecV8.DecodeBlob(daBatch.Blob()) + require.NoError(t, err) + + require.Equal(t, tc.batch.PrevL1MessageQueueHash, blobPayload.PrevL1MessageQueueHash()) + require.Equal(t, tc.batch.PostL1MessageQueueHash, blobPayload.PostL1MessageQueueHash()) + + // check correctness of decoded blocks and transactions + require.Equal(t, len(tc.batch.Blocks), len(blobPayload.Blocks())) + decodedBlocks := blobPayload.Blocks() + for i, block := range tc.batch.Blocks { + numL1Messages, lowestQueueIndex, _, err := block.NumL1MessagesNoSkipping() + require.NoError(t, err) + + daBlock := newDABlockV7(block.Header.Number.Uint64(), block.Header.Time, block.Header.BaseFee, block.Header.GasLimit, uint16(block.NumL2Transactions())+numL1Messages, numL1Messages, lowestQueueIndex) + assertEqualDABlocks(t, daBlock, decodedBlocks[i]) + + txDataDecoded := TxsToTxsData(blobPayload.Transactions()[i]) + var j int + for _, txData := range block.Transactions { + // Decoded blob contains only L2 transactions, L1 transactions need to be read from L1 (by using initialQueueIndex) + // So in this test we skip checking them. + if txData.Type == types.L1MessageTxType { + continue + } + + assertEqualTransactionData(t, txData, txDataDecoded[j]) + j++ + } + } + }) + } +} +func TestCodecV8BatchStandardTestCasesEnableCompression(t *testing.T) { + codecV8, err := CodecFromVersion(CodecV8) + require.NoError(t, err) + + // Apply patches to functions to replace behavior for testing. + { + patches := gomonkey.NewPatches() + defer patches.Reset() + + patches.ApplyFunc(convertTxDataToRLPEncoding, func(txData *types.TransactionData) ([]byte, error) { + data, err := hexutil.Decode(txData.Data) + if err != nil { + return nil, err + } + return data, nil + }) + + patches.ApplyFunc(checkCompressedDataCompatibility, func(_ []byte) error { + return nil + }) + } + + maxAvailableBytesIncompressable := maxEffectiveBlobBytes - 5 - blobPayloadV7MinEncodedLength + // 52 bytes for each block as per daBlockV7 encoding. + bytesPerBlock := 52 + + testCases := []struct { + name string + numBlocks int + txData []string + creationErr string + + expectedBlobVersionedHash string + }{ + { + name: "no blocks", + txData: []string{}, + creationErr: "no blocks", + }, + { + name: "single block, single tx", + numBlocks: 1, + txData: []string{"0x010203"}, + expectedBlobVersionedHash: "0x0184fd3d7edf3ea50c76c1751fcc0c4b605ef1f8e7c434ec1c1a1e0e57226cce", + }, + { + name: "single block, multiple tx", + numBlocks: 1, + txData: []string{"0x010203", "0x040506", "0x070809"}, + expectedBlobVersionedHash: "0x01aa8fde33c446276224f47187c7b30f53df61b3a56f5c8876f9c00828642d13", + }, + { + name: "multiple blocks, single tx per block", + numBlocks: 3, + txData: []string{"0x010203"}, + expectedBlobVersionedHash: "0x01b7c6888a192ee1d221eb5fe1e6d15927903541eb178964d851b742c518ccb0", + }, + { + name: "multiple blocks, multiple tx per block", + numBlocks: 3, + txData: []string{"0x010203", "0x040506", "0x070809"}, + expectedBlobVersionedHash: "0x017cd27686cbe8f92b596f0e21f355be1371979624b1a72e6c7471cd5fa782e4", + }, + { + name: "thousands of blocks, multiple tx per block", + numBlocks: 10000, + txData: []string{"0x010203", "0x040506", "0x070809"}, + expectedBlobVersionedHash: "0x016889370d5706071080111716b0f65b6ff9df7d7b1621103b10b0e3eabc9282", + }, + { + name: "single block, single tx, full blob random data -> data bigger compressed than uncompressed", + numBlocks: 1, + txData: []string{generateRandomData(maxAvailableBytesIncompressable - bytesPerBlock)}, + expectedBlobVersionedHash: "0x0128c6dcaa56600132bc09c26af86c31370d0a3dfb8bece776d28291ca3a721e", + }, + { + name: "2 blocks, single tx, full blob random data", + numBlocks: 2, + txData: []string{generateRandomData(maxAvailableBytesIncompressable/2 - bytesPerBlock*2)}, + expectedBlobVersionedHash: "0x018e8240255f00a81140ae445d1784c16f77791b4fe55c07fffa31c9bc15b3a4", + }, + { + name: "single block, single tx, full blob random data -> error because 1 byte too big", + numBlocks: 1, + txData: []string{generateRandomData(maxAvailableBytesIncompressable - bytesPerBlock + 1)}, + creationErr: "blob exceeds maximum size", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + var blocks []*Block + for i := 0; i < tc.numBlocks; i++ { + block := &Block{ + Header: &types.Header{ + Number: big.NewInt(int64(i)), + }, + Transactions: []*types.TransactionData{}, + } + for _, data := range tc.txData { + tx := &types.TransactionData{Type: 0xff, Data: data} + block.Transactions = append(block.Transactions, tx) + } + blocks = append(blocks, block) + } + + _, blobVersionedHash, _, _, err := codecV8.(*DACodecV8).constructBlob(&Batch{Blocks: blocks}) + if tc.creationErr != "" { + require.ErrorContains(t, err, tc.creationErr) + return + } + require.NoError(t, err) + require.Equal(t, common.HexToHash(tc.expectedBlobVersionedHash), blobVersionedHash) + }) + } +} diff --git a/encoding/da.go b/encoding/da.go index c72c128..de19118 100644 --- a/encoding/da.go +++ b/encoding/da.go @@ -110,7 +110,7 @@ type Block struct { type Chunk struct { Blocks []*Block `json:"blocks"` - // CodecV7. Used for chunk creation in relayer. + // CodecV7, CodecV8. Used for chunk creation in relayer. PrevL1MessageQueueHash common.Hash PostL1MessageQueueHash common.Hash } @@ -122,7 +122,7 @@ type Batch struct { ParentBatchHash common.Hash Chunks []*Chunk - // CodecV7 + // CodecV7, CodecV8. PrevL1MessageQueueHash common.Hash PostL1MessageQueueHash common.Hash Blocks []*Block @@ -766,8 +766,10 @@ func GetHardforkName(config *params.ChainConfig, blockHeight, blockTimestamp uin return "darwinV2" } else if !config.IsEuclidV2(blockTimestamp) { return "euclid" - } else { + } else if !config.IsFeynman(blockTimestamp) { return "euclidV2" + } else { + return "feynman" } } @@ -787,8 +789,10 @@ func GetCodecVersion(config *params.ChainConfig, blockHeight, blockTimestamp uin } else if !config.IsEuclidV2(blockTimestamp) { // V5 is skipped, because it is only used for the special Euclid transition batch that we handle explicitly return CodecV6 - } else { + } else if !config.IsFeynman(blockTimestamp) { return CodecV7 + } else { + return CodecV8 } } @@ -817,7 +821,7 @@ func GetChunkEnableCompression(codecVersion CodecVersion, chunk *Chunk) (bool, e return false, nil case CodecV2, CodecV3: return true, nil - case CodecV4, CodecV5, CodecV6, CodecV7: + case CodecV4, CodecV5, CodecV6, CodecV7, CodecV8: return CheckChunkCompressedDataCompatibility(chunk, codecVersion) default: return false, fmt.Errorf("unsupported codec version: %v", codecVersion) @@ -831,7 +835,7 @@ func GetBatchEnableCompression(codecVersion CodecVersion, batch *Batch) (bool, e return false, nil case CodecV2, CodecV3: return true, nil - case CodecV4, CodecV5, CodecV6, CodecV7: + case CodecV4, CodecV5, CodecV6, CodecV7, CodecV8: return CheckBatchCompressedDataCompatibility(batch, codecVersion) default: return false, fmt.Errorf("unsupported codec version: %v", codecVersion) diff --git a/encoding/da_test.go b/encoding/da_test.go index a0fa6b2..e2eb004 100644 --- a/encoding/da_test.go +++ b/encoding/da_test.go @@ -129,7 +129,7 @@ func TestBlobCompressDecompress(t *testing.T) { blobBytes, err := hex.DecodeString(blobString) assert.NoError(t, err) - compressed, err := zstd.CompressScrollBatchBytes(blobBytes) + compressed, err := zstd.CompressScrollBatchBytesLegacy(blobBytes) assert.NoError(t, err) blob, err := makeBlobCanonical(compressed) diff --git a/encoding/interfaces.go b/encoding/interfaces.go index a7e6098..e4ca3c8 100644 --- a/encoding/interfaces.go +++ b/encoding/interfaces.go @@ -76,6 +76,9 @@ type Codec interface { EstimateBatchL1CommitCalldataSize(*Batch) (uint64, error) JSONFromBytes([]byte) ([]byte, error) // convert batch header bytes to JSON, this is only used to provide witness data for the prover. + + // CompressScrollBatchBytes compresses batch bytes using the appropriate compression method for this codec version + CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) } // CodecVersion represents the version of the codec. @@ -90,6 +93,7 @@ const ( CodecV5 CodecV6 CodecV7 + CodecV8 ) // CodecFromVersion returns the appropriate codec for the given version. @@ -111,6 +115,8 @@ func CodecFromVersion(version CodecVersion) (Codec, error) { return NewDACodecV6(), nil case CodecV7: return &DACodecV7{}, nil + case CodecV8: + return NewDACodecV8(), nil default: return nil, fmt.Errorf("unsupported codec version: %v", version) } @@ -118,7 +124,9 @@ func CodecFromVersion(version CodecVersion) (Codec, error) { // CodecFromConfig determines and returns the appropriate codec based on chain configuration, block number, and timestamp. func CodecFromConfig(chainCfg *params.ChainConfig, startBlockNumber *big.Int, startBlockTimestamp uint64) Codec { - if chainCfg.IsEuclidV2(startBlockTimestamp) { + if chainCfg.IsFeynman(startBlockTimestamp) { + return NewDACodecV8() + } else if chainCfg.IsEuclidV2(startBlockTimestamp) { return &DACodecV7{} } else if chainCfg.IsEuclid(startBlockTimestamp) { // V5 is skipped, because it is only used for the special Euclid transition batch that we handle explicitly @@ -135,3 +143,9 @@ func CodecFromConfig(chainCfg *params.ChainConfig, startBlockNumber *big.Int, st return &DACodecV0{} } } + +// CompressScrollBatchBytes compresses batch bytes using the appropriate codec based on block number and timestamp +func CompressScrollBatchBytes(batchBytes []byte, blockNumber uint64, blockTimestamp uint64, chainCfg *params.ChainConfig) ([]byte, error) { + codec := CodecFromConfig(chainCfg, big.NewInt(int64(blockNumber)), blockTimestamp) + return codec.CompressScrollBatchBytes(batchBytes) +} diff --git a/encoding/interfaces_test.go b/encoding/interfaces_test.go index c965781..77c4aa7 100644 --- a/encoding/interfaces_test.go +++ b/encoding/interfaces_test.go @@ -24,6 +24,7 @@ func TestCodecFromVersion(t *testing.T) { {"CodecV5", CodecV5, &DACodecV5{}, false}, {"CodecV6", CodecV6, &DACodecV6{}, false}, {"CodecV7", CodecV7, &DACodecV7{}, false}, + {"CodecV8", CodecV8, &DACodecV8{}, false}, {"InvalidCodec", CodecVersion(99), nil, true}, } diff --git a/encoding/zstd/add_scroll_prefix_in_zstd_related_symbols.sh b/encoding/zstd/add_scroll_prefix_in_zstd_related_symbols.sh deleted file mode 100755 index 10a0498..0000000 --- a/encoding/zstd/add_scroll_prefix_in_zstd_related_symbols.sh +++ /dev/null @@ -1,29 +0,0 @@ -# Generate redefine.syms for linux_amd64 -/opt/homebrew/opt/llvm/bin/llvm-nm libscroll_zstd_linux_amd64.a | awk '/ZSTD|HUF|FSE|ZBUFF/ {if ($3 != "") print $3 " scroll_" $3}' | sort | uniq > redefine_linux_amd64.syms - -# Use llvm-objcopy to modify symbols for linux_amd64 -llvm-objcopy --redefine-syms=redefine_linux_amd64.syms libscroll_zstd_linux_amd64.a libscroll_zstd_linux_amd64_new.a - -# Move the new file to replace the original and clean up -mv libscroll_zstd_linux_amd64_new.a libscroll_zstd_linux_amd64.a -rm redefine_linux_amd64.syms - -# Generate redefine.syms for linux_arm64 -/opt/homebrew/opt/llvm/bin/llvm-nm libscroll_zstd_linux_arm64.a | awk '/ZSTD|HUF|FSE|ZBUFF/ {if ($3 != "") print $3 " scroll_" $3}' | sort | uniq > redefine_linux_arm64.syms - -# Use llvm-objcopy to modify symbols for linux_arm64 -llvm-objcopy --redefine-syms=redefine_linux_arm64.syms libscroll_zstd_linux_arm64.a libscroll_zstd_linux_arm64_new.a - -# Move the new file to replace the original and clean up -mv libscroll_zstd_linux_arm64_new.a libscroll_zstd_linux_arm64.a -rm redefine_linux_arm64.syms - -# Generate redefine.syms for darwin_arm64 -/opt/homebrew/opt/llvm/bin/llvm-nm libscroll_zstd_darwin_arm64.a | awk '/ZSTD|HUF|FSE|ZBUFF/ {if ($3 != "") print $3 " scroll_" $3}' | sort | uniq > redefine_darwin_arm64.syms - -# Use llvm-objcopy to modify symbols for darwin_arm64 -llvm-objcopy --redefine-syms=redefine_darwin_arm64.syms libscroll_zstd_darwin_arm64.a libscroll_zstd_darwin_arm64_new.a - -# Move the new file to replace the original and clean up -mv libscroll_zstd_darwin_arm64_new.a libscroll_zstd_darwin_arm64.a -rm redefine_darwin_arm64.syms diff --git a/encoding/zstd/add_symbol_prefix.sh b/encoding/zstd/add_symbol_prefix.sh new file mode 100755 index 0000000..0165006 --- /dev/null +++ b/encoding/zstd/add_symbol_prefix.sh @@ -0,0 +1,256 @@ +#!/bin/bash + +set -e + +# Fixed macOS paths +LLVM_NM="/opt/homebrew/opt/llvm/bin/llvm-nm" +LLVM_OBJCOPY="/opt/homebrew/opt/llvm/bin/llvm-objcopy" + +# List of library files to process +LIBRARIES=( + "libencoder_legacy_darwin_arm64.a:scroll_legacy_" + "libencoder_legacy_linux_amd64.a:scroll_legacy_" + "libencoder_legacy_linux_arm64.a:scroll_legacy_" + "libencoder_standard_darwin_arm64.a:scroll_standard_" + "libencoder_standard_linux_amd64.a:scroll_standard_" + "libencoder_standard_linux_arm64.a:scroll_standard_" +) + +echo "=== Adding Symbol Prefixes ===" +echo + +for lib_info in "${LIBRARIES[@]}"; do + IFS=':' read -r LIB_FILE PREFIX <<< "$lib_info" + REDEFINE_FILE="redefine_${LIB_FILE%.*}.syms" + + echo "Processing $LIB_FILE with prefix '$PREFIX'" + + # Check if library file exists + if [ ! -f "$LIB_FILE" ]; then + echo "Warning: Library file not found: $LIB_FILE, skipping..." + continue + fi + + # Check if library is already processed by looking for our prefix + if "$LLVM_NM" "$LIB_FILE" 2>/dev/null | grep -q "${PREFIX}"; then + echo "Library $LIB_FILE already processed (found ${PREFIX} symbols), skipping..." + continue + fi + + # Generate redefine.syms for all potential conflicting symbols + "$LLVM_NM" "$LIB_FILE" | awk ' + /ZSTD|HUF|FSE|ZBUFF|HIST|ERROR|MEM_|XXH|COVER|DICT|POOL|PARAM/ { + if ($3 != "" && $3 !~ /^\./ && $3 !~ /^'"$PREFIX"'/) { + print $3 " '"$PREFIX"'" $3 + } + } + /^[0-9a-fA-F]+ [TDBS] / { + if ($3 != "" && $3 !~ /^\./ && $3 !~ /^__/ && $3 !~ /^'"$PREFIX"'/) { + # Original patterns + if ($3 ~ /^(entropy|fse|huf|zstd|hist|error|mem_|pool|param|cover|dict)/) { + print $3 " '"$PREFIX"'" $3 + } + # Add conflict symbols found by verification logic below + if ($3 == "_atomic_flag_test_and_set" || + $3 == "_atomic_signal_fence" || + $3 == "_divbwt" || + $3 == "divsufsort" || + $3 == "g_debuglevel" || + $3 == "init_cpu_features" || + $3 == "_ERR_getErrorString" || + $3 == "_atomic_flag_clear" || + $3 == "_atomic_flag_clear_explicit" || + $3 == "_atomic_flag_test_and_set_explicit" || + $3 == "_atomic_thread_fence" || + $3 == "_divsufsort" || + $3 == "_g_debuglevel" || + $3 == "divbwt" || + $3 == "ERR_getErrorString" || + $3 == "init_cpu_features_resolver") { + print $3 " '"$PREFIX"'" $3 + } + } + } + ' | sort | uniq > "$REDEFINE_FILE" + + # Check if there are symbols to redefine + if [ ! -s "$REDEFINE_FILE" ]; then + echo "No symbols found to redefine in $LIB_FILE" + rm -f "$REDEFINE_FILE" + continue + fi + + echo "Found $(wc -l < "$REDEFINE_FILE") symbols to redefine in $LIB_FILE" + + # Show sample symbols being renamed + echo "Sample symbols to be renamed:" + head -3 "$REDEFINE_FILE" | while read old new; do + echo " $old -> $new" + done + + # Use llvm-objcopy to modify symbols + "$LLVM_OBJCOPY" --redefine-syms="$REDEFINE_FILE" "$LIB_FILE" "${LIB_FILE%.*}_new.a" + + # Move the new file to replace the original and clean up + mv "${LIB_FILE%.*}_new.a" "$LIB_FILE" + rm "$REDEFINE_FILE" + + echo "Successfully processed $LIB_FILE" + echo +done + +echo "=== Symbol Processing Complete ===" +echo +echo "=== Checking for Symbol Conflicts ===" +echo + +# Extract library files for conflict checking +LIB_FILES=() +for lib_info in "${LIBRARIES[@]}"; do + IFS=':' read -r LIB_FILE PREFIX <<< "$lib_info" + LIB_FILES+=("$LIB_FILE") +done + +# Temporary file to store all symbols +temp_file=$(mktemp) + +# Collect all exported symbols from all libraries +echo "Collecting symbols from all libraries..." +for LIB_FILE in "${LIB_FILES[@]}"; do + if [ ! -f "$LIB_FILE" ]; then + echo "Warning: $LIB_FILE not found, skipping..." + continue + fi + + "$LLVM_NM" "$LIB_FILE" 2>/dev/null | awk -v lib="$LIB_FILE" ' + /^[0-9a-fA-F]+ [TDBS] / { + if ($3 != "" && $3 !~ /^\./ && $3 !~ /^__/) { + print $3 "\t" lib + } + } + ' >> "$temp_file" +done + +echo "1. Checking for duplicate symbols across libraries (by architecture):" +echo " (Only checking within same architecture - cross-architecture conflicts are expected and ignored)" +echo + +# Define architectures +architectures=("darwin_arm64" "linux_amd64" "linux_arm64") + +total_conflicts=0 +for arch in "${architectures[@]}"; do + echo " Architecture: $arch" + + # Create temp file for this architecture + arch_temp_file=$(mktemp) + + # Collect symbols for this architecture only + for lib_info in "${LIBRARIES[@]}"; do + IFS=':' read -r LIB_FILE PREFIX <<< "$lib_info" + + # Skip if file doesn't exist or doesn't match current architecture + if [ ! -f "$LIB_FILE" ] || [[ "$LIB_FILE" != *"$arch"* ]]; then + continue + fi + + "$LLVM_NM" "$LIB_FILE" 2>/dev/null | awk -v lib="$LIB_FILE" ' + /^[0-9a-fA-F]+ [TDBS] / { + if ($3 != "" && $3 !~ /^\./ && $3 !~ /^__/ && $3 !~ /^_ZN/) { + print $3 "\t" lib + } + }' >> "$arch_temp_file" + done + + # Check for conflicts within this architecture + if [ -s "$arch_temp_file" ]; then + arch_conflicts=$(awk '{symbols[$1] = symbols[$1] "\n" $2} END { + conflicts = 0 + conflict_list = "" + for (sym in symbols) { + count = gsub(/\n/, "&", symbols[sym]) + if (count > 1) { + # Skip Rust runtime symbols - these are expected to be identical + if (sym ~ /^rust_(begin_unwind|eh_personality|panic)$/) { + continue + } + conflicts++ + print " ❌ CONFLICT: " sym + libs = symbols[sym] + gsub(/\n/, ", ", libs) + print " Found in: " libs + } + } + if (conflicts == 0) { + print " ✅ No symbol conflicts found within this architecture!" + } else { + print " ❌ Found " conflicts " conflicting symbols (all shown above)" + } + print "CONFLICT_COUNT:" conflicts + }' "$arch_temp_file") + + echo "$arch_conflicts" + arch_conflict_count=$(echo "$arch_conflicts" | grep "CONFLICT_COUNT:" | cut -d: -f2 || echo 0) + total_conflicts=$((total_conflicts + arch_conflict_count)) + else + echo " ✅ No libraries found for this architecture" + fi + + rm "$arch_temp_file" + echo +done + +# Summary +if [ "$total_conflicts" -eq 0 ]; then + echo "🎉 All architectures passed symbol conflict check!" +else + echo "⚠️ Found $total_conflicts total symbol conflicts across all architectures" +fi + +conflict_count=$total_conflicts + +echo "2. Sample prefixed symbols from each library:" +for lib_info in "${LIBRARIES[@]}"; do + IFS=':' read -r LIB_FILE PREFIX <<< "$lib_info" + + if [ ! -f "$LIB_FILE" ]; then + continue + fi + + echo " $LIB_FILE (sample ${PREFIX}* symbols):" + "$LLVM_NM" "$LIB_FILE" 2>/dev/null | awk -v prefix="$PREFIX" ' + /^[0-9a-fA-F]+ [TDBS] / { + if ($3 ~ ("^" prefix)) { + print " " $3 + } + }' | head -3 + echo +done + +echo "3. Preserved original functions:" +for LIB_FILE in "${LIB_FILES[@]}"; do + if [ ! -f "$LIB_FILE" ]; then + continue + fi + + echo " $LIB_FILE:" + "$LLVM_NM" "$LIB_FILE" 2>/dev/null | grep -E "(compress_scroll_batch_bytes_)" | awk '{print " " $2 " " $3}' || echo " No original functions found" + echo +done + +# Cleanup +rm "$temp_file" + +echo "=== Final Analysis ===" + +if [ "$conflict_count" -eq 0 ]; then + echo "🎉 SUCCESS: All libraries processed successfully with no symbol conflicts!" + echo "✅ All target symbols have been properly prefixed" + echo "✅ Original functions preserved" +else + echo "⚠️ WARNING: Found $conflict_count symbol conflicts that need attention." + echo "📋 Please review the conflicts listed above" +fi + +echo +echo "=== Process Complete ===" diff --git a/encoding/zstd/libscroll_zstd_darwin_arm64.a b/encoding/zstd/libencoder_legacy_darwin_arm64.a similarity index 96% rename from encoding/zstd/libscroll_zstd_darwin_arm64.a rename to encoding/zstd/libencoder_legacy_darwin_arm64.a index 9642681..2058763 100644 Binary files a/encoding/zstd/libscroll_zstd_darwin_arm64.a and b/encoding/zstd/libencoder_legacy_darwin_arm64.a differ diff --git a/encoding/zstd/libscroll_zstd_linux_amd64.a b/encoding/zstd/libencoder_legacy_linux_amd64.a similarity index 90% rename from encoding/zstd/libscroll_zstd_linux_amd64.a rename to encoding/zstd/libencoder_legacy_linux_amd64.a index c4385a5..97d048c 100644 Binary files a/encoding/zstd/libscroll_zstd_linux_amd64.a and b/encoding/zstd/libencoder_legacy_linux_amd64.a differ diff --git a/encoding/zstd/libscroll_zstd_linux_arm64.a b/encoding/zstd/libencoder_legacy_linux_arm64.a similarity index 96% rename from encoding/zstd/libscroll_zstd_linux_arm64.a rename to encoding/zstd/libencoder_legacy_linux_arm64.a index 02183c1..446b342 100644 Binary files a/encoding/zstd/libscroll_zstd_linux_arm64.a and b/encoding/zstd/libencoder_legacy_linux_arm64.a differ diff --git a/encoding/zstd/libencoder_standard_darwin_arm64.a b/encoding/zstd/libencoder_standard_darwin_arm64.a new file mode 100644 index 0000000..0712abd Binary files /dev/null and b/encoding/zstd/libencoder_standard_darwin_arm64.a differ diff --git a/encoding/zstd/libencoder_standard_linux_amd64.a b/encoding/zstd/libencoder_standard_linux_amd64.a new file mode 100644 index 0000000..ae4efbf Binary files /dev/null and b/encoding/zstd/libencoder_standard_linux_amd64.a differ diff --git a/encoding/zstd/libencoder_standard_linux_arm64.a b/encoding/zstd/libencoder_standard_linux_arm64.a new file mode 100644 index 0000000..8971727 Binary files /dev/null and b/encoding/zstd/libencoder_standard_linux_arm64.a differ diff --git a/encoding/zstd/libscroll_zstd_darwin_arm64.go b/encoding/zstd/libscroll_zstd_darwin_arm64.go index d83ec17..65b8660 100644 --- a/encoding/zstd/libscroll_zstd_darwin_arm64.go +++ b/encoding/zstd/libscroll_zstd_darwin_arm64.go @@ -1,6 +1,9 @@ +//go:build darwin && arm64 && !musl +// +build darwin,arm64,!musl + package zstd /* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_darwin_arm64.a +#cgo LDFLAGS: ${SRCDIR}/libencoder_legacy_darwin_arm64.a ${SRCDIR}/libencoder_standard_darwin_arm64.a */ import "C" diff --git a/encoding/zstd/libscroll_zstd_linux_amd64.go b/encoding/zstd/libscroll_zstd_linux_amd64.go index f1a686e..1b030c1 100644 --- a/encoding/zstd/libscroll_zstd_linux_amd64.go +++ b/encoding/zstd/libscroll_zstd_linux_amd64.go @@ -1,9 +1,9 @@ -//go:build !musl -// +build !musl +//go:build linux && amd64 && !musl +// +build linux,amd64,!musl package zstd /* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_linux_amd64.a +#cgo LDFLAGS: ${SRCDIR}/libencoder_legacy_linux_amd64.a ${SRCDIR}/libencoder_standard_linux_amd64.a */ import "C" diff --git a/encoding/zstd/libscroll_zstd_linux_arm64.go b/encoding/zstd/libscroll_zstd_linux_arm64.go index f3775d2..e577556 100644 --- a/encoding/zstd/libscroll_zstd_linux_arm64.go +++ b/encoding/zstd/libscroll_zstd_linux_arm64.go @@ -1,9 +1,9 @@ -//go:build !musl -// +build !musl +//go:build linux && arm64 && !musl +// +build linux,arm64,!musl package zstd /* -#cgo LDFLAGS: ${SRCDIR}/libscroll_zstd_linux_arm64.a +#cgo LDFLAGS: ${SRCDIR}/libencoder_legacy_linux_arm64.a ${SRCDIR}/libencoder_standard_linux_arm64.a */ import "C" diff --git a/encoding/zstd/zstd.go b/encoding/zstd/zstd.go index aab718f..79de46c 100644 --- a/encoding/zstd/zstd.go +++ b/encoding/zstd/zstd.go @@ -2,7 +2,8 @@ package zstd /* #include -char* compress_scroll_batch_bytes(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size); +char* compress_scroll_batch_bytes_legacy(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size); +char* compress_scroll_batch_bytes_standard(uint8_t* src, uint64_t src_size, uint8_t* output_buf, uint64_t *output_buf_size); */ import "C" @@ -13,10 +14,11 @@ import ( const compressBufferOverhead = 128 -// CompressScrollBatchBytes compresses the given batch of bytes using zstd compression. +// CompressScrollBatchBytesLegacy compresses the given batch of bytes using zstd compression. +// This function uses the customized scroll-tech/zstd-rs fork version for codec v2-v7. // The output buffer is allocated with an extra compressBufferOverhead bytes to accommodate // potential metadata overhead or error messages from the underlying C function. -func CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { +func CompressScrollBatchBytesLegacy(batchBytes []byte) ([]byte, error) { if len(batchBytes) == 0 { return nil, fmt.Errorf("input batch is empty") } @@ -25,9 +27,30 @@ func CompressScrollBatchBytes(batchBytes []byte) ([]byte, error) { outbufSize := C.uint64_t(len(batchBytes) + compressBufferOverhead) outbuf := make([]byte, outbufSize) - if err := C.compress_scroll_batch_bytes((*C.uchar)(unsafe.Pointer(&batchBytes[0])), srcSize, + if err := C.compress_scroll_batch_bytes_legacy((*C.uchar)(unsafe.Pointer(&batchBytes[0])), srcSize, (*C.uchar)(unsafe.Pointer(&outbuf[0])), &outbufSize); err != nil { - return nil, fmt.Errorf("failed to compress scroll batch bytes: %s", C.GoString(err)) + return nil, fmt.Errorf("failed to compress scroll batch bytes (legacy): %s", C.GoString(err)) + } + + return outbuf[:int(outbufSize)], nil +} + +// CompressScrollBatchBytesStandard compresses the given batch of bytes using zstd compression. +// This function uses the standard zstd 0.13 experimental version for codec v8 and later. +// The output buffer is allocated with an extra compressBufferOverhead bytes to accommodate +// potential metadata overhead or error messages from the underlying C function. +func CompressScrollBatchBytesStandard(batchBytes []byte) ([]byte, error) { + if len(batchBytes) == 0 { + return nil, fmt.Errorf("input batch is empty") + } + + srcSize := C.uint64_t(len(batchBytes)) + outbufSize := C.uint64_t(len(batchBytes) + compressBufferOverhead) + outbuf := make([]byte, outbufSize) + + if err := C.compress_scroll_batch_bytes_standard((*C.uchar)(unsafe.Pointer(&batchBytes[0])), srcSize, + (*C.uchar)(unsafe.Pointer(&outbuf[0])), &outbufSize); err != nil { + return nil, fmt.Errorf("failed to compress scroll batch bytes (standard): %s", C.GoString(err)) } return outbuf[:int(outbufSize)], nil diff --git a/go.mod b/go.mod index 510ae39..55ce260 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/agiledragon/gomonkey/v2 v2.12.0 - github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 + github.com/scroll-tech/go-ethereum v1.10.14-0.20250625112225-a67863c65587 github.com/stretchr/testify v1.9.0 ) diff --git a/go.sum b/go.sum index 0163bf8..72f9102 100644 --- a/go.sum +++ b/go.sum @@ -78,8 +78,8 @@ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601 h1:NEsjCG6uSvLRBlsP3+x6PL1kM+Ojs3g8UGotIPgJSz8= -github.com/scroll-tech/go-ethereum v1.10.14-0.20250305151038-478940e79601/go.mod h1:OblWe1+QrZwdpwO0j/LY3BSGuKT3YPUFBDQQgvvfStQ= +github.com/scroll-tech/go-ethereum v1.10.14-0.20250625112225-a67863c65587 h1:wG1+gb+K4iLtxAHhiAreMdIjP5x9hB64duraN2+u1QU= +github.com/scroll-tech/go-ethereum v1.10.14-0.20250625112225-a67863c65587/go.mod h1:YyfB2AyAtphlbIuDQgaxc2b9mo0zE4EBA1+qtXvzlmg= github.com/scroll-tech/zktrie v0.8.4 h1:UagmnZ4Z3ITCk+aUq9NQZJNAwnWl4gSxsLb2Nl7IgRE= github.com/scroll-tech/zktrie v0.8.4/go.mod h1:XvNo7vAk8yxNyTjBDj5WIiFzYW4bx/gJ78+NK6Zn6Uk= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= diff --git a/libzstd/Makefile b/libzstd/Makefile deleted file mode 100644 index fc65390..0000000 --- a/libzstd/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -.PHONY: libzstd libzstddbg - -clean: - rm -rf *.a *.so target - cargo clean - -libzstd: - cargo build --release - -libzstddbg: - cargo build diff --git a/libzstd/.gitignore b/libzstd/encoder-legacy/.gitignore similarity index 100% rename from libzstd/.gitignore rename to libzstd/encoder-legacy/.gitignore diff --git a/libzstd/Cargo.lock b/libzstd/encoder-legacy/Cargo.lock similarity index 94% rename from libzstd/Cargo.lock rename to libzstd/encoder-legacy/Cargo.lock index 46480d5..4c5375b 100644 --- a/libzstd/Cargo.lock +++ b/libzstd/encoder-legacy/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ ] [[package]] -name = "encoder" +name = "encoder-legacy" version = "0.1.0" dependencies = [ "zstd", @@ -47,13 +47,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "scroll-zstd" -version = "0.1.0" -dependencies = [ - "encoder", -] - [[package]] name = "zstd" version = "0.13.0" diff --git a/libzstd/encoder/Cargo.toml b/libzstd/encoder-legacy/Cargo.toml similarity index 64% rename from libzstd/encoder/Cargo.toml rename to libzstd/encoder-legacy/Cargo.toml index 468f863..781bb8e 100644 --- a/libzstd/encoder/Cargo.toml +++ b/libzstd/encoder-legacy/Cargo.toml @@ -1,10 +1,12 @@ [package] -name = "encoder" +name = "encoder-legacy" version = "0.1.0" edition = "2021" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + [lib] +name = "encoder_legacy" +crate-type = ["staticlib"] [dependencies] -zstd = { git = "https://github.com/scroll-tech/zstd-rs", branch = "hack/mul-block", features = ["experimental"]} \ No newline at end of file +zstd = { git = "https://github.com/scroll-tech/zstd-rs", branch = "hack/mul-block", features = ["experimental"] } diff --git a/libzstd/encoder-legacy/Makefile b/libzstd/encoder-legacy/Makefile new file mode 100644 index 0000000..05f43b5 --- /dev/null +++ b/libzstd/encoder-legacy/Makefile @@ -0,0 +1,56 @@ +.PHONY: all clean build install + +# Detect platform +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + ifeq ($(UNAME_M),x86_64) + PLATFORM := linux_amd64 + else ifeq ($(UNAME_M),aarch64) + PLATFORM := linux_arm64 + endif +else ifeq ($(UNAME_S),Darwin) + ifeq ($(UNAME_M),arm64) + PLATFORM := darwin_arm64 + # Set macOS deployment target to avoid version warnings + export MACOSX_DEPLOYMENT_TARGET := 14.0 + else ifeq ($(UNAME_M),x86_64) + PLATFORM := darwin_amd64 + # Set macOS deployment target to avoid version warnings + export MACOSX_DEPLOYMENT_TARGET := 14.0 + endif +endif + +TARGET_DIR := ../../encoding/zstd + +all: build + +build: + @echo "Building legacy encoder for $(PLATFORM)..." +ifeq ($(UNAME_S),Darwin) + @echo "Setting macOS deployment target to $(MACOSX_DEPLOYMENT_TARGET)" + MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) cargo build --release +else + cargo build --release +endif + +install: build + @echo "Installing legacy library to $(TARGET_DIR)..." + cp target/release/libencoder_legacy.a $(TARGET_DIR)/libencoder_legacy_$(PLATFORM).a +ifeq ($(UNAME_S),Darwin) + @echo "Fixing library symbol table..." + ranlib $(TARGET_DIR)/libencoder_legacy_$(PLATFORM).a +endif + @echo "Legacy installation complete!" + +clean: + cargo clean + rm -f $(TARGET_DIR)/libencoder_legacy_$(PLATFORM).a + +info: + @echo "Legacy encoder - Platform: $(PLATFORM)" + @echo "Target directory: $(TARGET_DIR)" +ifeq ($(UNAME_S),Darwin) + @echo "macOS deployment target: $(MACOSX_DEPLOYMENT_TARGET)" +endif diff --git a/libzstd/rust-toolchain b/libzstd/encoder-legacy/rust-toolchain similarity index 100% rename from libzstd/rust-toolchain rename to libzstd/encoder-legacy/rust-toolchain diff --git a/libzstd/encoder-legacy/src/lib.rs b/libzstd/encoder-legacy/src/lib.rs new file mode 100644 index 0000000..ae268e7 --- /dev/null +++ b/libzstd/encoder-legacy/src/lib.rs @@ -0,0 +1,101 @@ +use core::slice; +use std::io::Write; +use std::os::raw::{c_char, c_uchar}; +use std::ptr::null; +use zstd::stream::Encoder; +use zstd::zstd_safe::{CParameter, ParamSwitch}; + +// re-export zstd +pub use zstd; + +// we use offset window no more than = 17 +// TODO: use for multi-block zstd. +#[allow(dead_code)] +pub const CL_WINDOW_LIMIT: usize = 17; + +/// zstd block size target. +pub const N_BLOCK_SIZE_TARGET: u32 = 124 * 1024; + +/// Maximum number of blocks that we can expect in the encoded data. +pub const N_MAX_BLOCKS: u64 = 10; + +/// Zstd encoder configuration +pub fn init_zstd_encoder(target_block_size: u32) -> Encoder<'static, Vec> { + let mut encoder = Encoder::new(Vec::new(), 0).expect("infallible"); + + // disable compression of literals, i.e. literals will be raw bytes. + encoder + .set_parameter(CParameter::LiteralCompressionMode(ParamSwitch::Disable)) + .expect("infallible"); + // with a hack in zstd we can set window log <= CL_WINDOW_LIMIT with single segment kept + encoder + .set_parameter(CParameter::WindowLog(CL_WINDOW_LIMIT.try_into().unwrap())) + .expect("infallible"); + // set target block size to fit within a single block. + encoder + .set_parameter(CParameter::TargetCBlockSize(target_block_size)) + .expect("infallible"); + // do not include the checksum at the end of the encoded data. + encoder.include_checksum(false).expect("infallible"); + // do not include magic bytes at the start of the frame since we will have a single + // frame. + encoder.include_magicbytes(false).expect("infallible"); + // do not include dictionary id so we have more simple content + encoder.include_dictid(false).expect("infallible"); + // include the content size to know at decode time the expected size of decoded + // data. + encoder.include_contentsize(true).expect("infallible"); + + encoder +} + +/// Helper function to convert error message to C-style string in output buffer +fn out_as_err(err: &str, out: &mut [u8]) -> *const c_char { + let msg = if err.len() + 1 > out.len() { + "compress_scroll_batch_bytes_legacy: not enough output buffer for the error message" + } else { + err + }; + + let cpy_src = unsafe { slice::from_raw_parts(msg.as_ptr(), msg.len()) }; + out[..cpy_src.len()].copy_from_slice(cpy_src); + out[cpy_src.len()] = 0; // build the c-style string + out.as_ptr() as *const c_char +} + +/// Legacy compression function for codec v2-v7 +/// Uses the customized scroll-tech/zstd-rs implementation +#[no_mangle] +pub unsafe extern "C" fn compress_scroll_batch_bytes_legacy( + src: *const c_uchar, + src_size: u64, + output_buf: *mut c_uchar, + output_buf_size: *mut u64, +) -> *const c_char { + let buf_size = *output_buf_size; + let src = unsafe { slice::from_raw_parts(src, src_size as usize) }; + let out = unsafe { slice::from_raw_parts_mut(output_buf, buf_size as usize) }; + + let mut encoder = init_zstd_encoder(N_BLOCK_SIZE_TARGET); + encoder.set_pledged_src_size(Some(src.len() as u64)).expect( + "compress_scroll_batch_bytes_legacy: failed to set pledged src size, should be infallible", + ); + + let ret = encoder.write_all(src); + let ret = ret.and_then(|_| encoder.finish()); + if let Err(e) = ret { + return out_as_err(e.to_string().as_str(), out); + } + + let ret = ret.unwrap(); + if ret.len() > buf_size as usize { + return out_as_err( + "compress_scroll_batch_bytes_legacy: not enough output buffer for compressed data", + out, + ); + } + out[..ret.len()].copy_from_slice(&ret); + *output_buf_size = ret.len() as u64; + + null() +} diff --git a/libzstd/encoder-standard/.gitignore b/libzstd/encoder-standard/.gitignore new file mode 100644 index 0000000..dd635c6 --- /dev/null +++ b/libzstd/encoder-standard/.gitignore @@ -0,0 +1,2 @@ +/target +/_obj diff --git a/libzstd/encoder-standard/Cargo.lock b/libzstd/encoder-standard/Cargo.lock new file mode 100644 index 0000000..1e0124e --- /dev/null +++ b/libzstd/encoder-standard/Cargo.lock @@ -0,0 +1,76 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "encoder-standard" +version = "0.1.0" +dependencies = [ + "zstd", +] + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "libc" +version = "0.2.153" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/libzstd/Cargo.toml b/libzstd/encoder-standard/Cargo.toml similarity index 56% rename from libzstd/Cargo.toml rename to libzstd/encoder-standard/Cargo.toml index 0298b15..6887315 100644 --- a/libzstd/Cargo.toml +++ b/libzstd/encoder-standard/Cargo.toml @@ -1,21 +1,12 @@ -[workspace] -members = [ - "encoder", -] - [package] -name = "scroll-zstd" +name = "encoder-standard" version = "0.1.0" edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] +name = "encoder_standard" crate-type = ["staticlib"] - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] -zstd-encoder = { package = "encoder", path = "encoder"} - -[features] -scroll = [ ] +zstd = { version = "0.13", features = ["experimental"] } diff --git a/libzstd/encoder-standard/Makefile b/libzstd/encoder-standard/Makefile new file mode 100644 index 0000000..a25d742 --- /dev/null +++ b/libzstd/encoder-standard/Makefile @@ -0,0 +1,56 @@ +.PHONY: all clean build install + +# Detect platform +UNAME_S := $(shell uname -s) +UNAME_M := $(shell uname -m) + +ifeq ($(UNAME_S),Linux) + ifeq ($(UNAME_M),x86_64) + PLATFORM := linux_amd64 + else ifeq ($(UNAME_M),aarch64) + PLATFORM := linux_arm64 + endif +else ifeq ($(UNAME_S),Darwin) + ifeq ($(UNAME_M),arm64) + PLATFORM := darwin_arm64 + # Set macOS deployment target to avoid version warnings + export MACOSX_DEPLOYMENT_TARGET := 14.0 + else ifeq ($(UNAME_M),x86_64) + PLATFORM := darwin_amd64 + # Set macOS deployment target to avoid version warnings + export MACOSX_DEPLOYMENT_TARGET := 14.0 + endif +endif + +TARGET_DIR := ../../encoding/zstd + +all: build + +build: + @echo "Building standard encoder for $(PLATFORM)..." +ifeq ($(UNAME_S),Darwin) + @echo "Setting macOS deployment target to $(MACOSX_DEPLOYMENT_TARGET)" + MACOSX_DEPLOYMENT_TARGET=$(MACOSX_DEPLOYMENT_TARGET) cargo build --release +else + cargo build --release +endif + +install: build + @echo "Installing standard library to $(TARGET_DIR)..." + cp target/release/libencoder_standard.a $(TARGET_DIR)/libencoder_standard_$(PLATFORM).a +ifeq ($(UNAME_S),Darwin) + @echo "Fixing library symbol table..." + ranlib $(TARGET_DIR)/libencoder_standard_$(PLATFORM).a +endif + @echo "Standard installation complete!" + +clean: + cargo clean + rm -f $(TARGET_DIR)/libencoder_standard_$(PLATFORM).a + +info: + @echo "Standard encoder - Platform: $(PLATFORM)" + @echo "Target directory: $(TARGET_DIR)" +ifeq ($(UNAME_S),Darwin) + @echo "macOS deployment target: $(MACOSX_DEPLOYMENT_TARGET)" +endif diff --git a/libzstd/encoder-standard/rust-toolchain b/libzstd/encoder-standard/rust-toolchain new file mode 100644 index 0000000..27c108b --- /dev/null +++ b/libzstd/encoder-standard/rust-toolchain @@ -0,0 +1 @@ +nightly-2023-12-03 diff --git a/libzstd/encoder-standard/src/lib.rs b/libzstd/encoder-standard/src/lib.rs new file mode 100644 index 0000000..4cd41a1 --- /dev/null +++ b/libzstd/encoder-standard/src/lib.rs @@ -0,0 +1,101 @@ +use core::slice; +use std::io::Write; +use std::os::raw::{c_char, c_uchar}; +use std::ptr::null; +use zstd::stream::Encoder; +use zstd::zstd_safe::{CParameter, ParamSwitch}; + +// re-export zstd +pub use zstd; + +// we use offset window no more than = 22 +// TODO: use for multi-block zstd. +#[allow(dead_code)] +pub const CL_WINDOW_LIMIT: usize = 22; + +/// zstd block size target. +pub const N_BLOCK_SIZE_TARGET: u32 = 124 * 1024; + +/// Maximum number of blocks that we can expect in the encoded data. +pub const N_MAX_BLOCKS: u64 = 10; + +/// Zstd encoder configuration +pub fn init_zstd_encoder(target_block_size: u32) -> Encoder<'static, Vec> { + let mut encoder = Encoder::new(Vec::new(), 0).expect("infallible"); + + // disable compression of literals, i.e. literals will be raw bytes. + encoder + .set_parameter(CParameter::LiteralCompressionMode(ParamSwitch::Disable)) + .expect("infallible"); + // with a hack in zstd we can set window log <= CL_WINDOW_LIMIT with single segment kept + encoder + .set_parameter(CParameter::WindowLog(CL_WINDOW_LIMIT.try_into().unwrap())) + .expect("infallible"); + // set target block size to fit within a single block. + encoder + .set_parameter(CParameter::TargetCBlockSize(target_block_size)) + .expect("infallible"); + // do not include the checksum at the end of the encoded data. + encoder.include_checksum(false).expect("infallible"); + // do not include magic bytes at the start of the frame since we will have a single + // frame. + encoder.include_magicbytes(false).expect("infallible"); + // do not include dictionary id so we have more simple content + encoder.include_dictid(false).expect("infallible"); + // include the content size to know at decode time the expected size of decoded + // data. + encoder.include_contentsize(true).expect("infallible"); + + encoder +} + +/// Helper function to convert error message to C-style string in output buffer +fn out_as_err(err: &str, out: &mut [u8]) -> *const c_char { + let msg = if err.len() + 1 > out.len() { + "compress_scroll_batch_bytes_standard: not enough output buffer for the error message" + } else { + err + }; + + let cpy_src = unsafe { slice::from_raw_parts(msg.as_ptr(), msg.len()) }; + out[..cpy_src.len()].copy_from_slice(cpy_src); + out[cpy_src.len()] = 0; // build the c-style string + out.as_ptr() as *const c_char +} + +/// Standard compression function for codec v8 and later. +/// Uses the customized scroll-tech/zstd-rs implementation +#[no_mangle] +pub unsafe extern "C" fn compress_scroll_batch_bytes_standard( + src: *const c_uchar, + src_size: u64, + output_buf: *mut c_uchar, + output_buf_size: *mut u64, +) -> *const c_char { + let buf_size = *output_buf_size; + let src = unsafe { slice::from_raw_parts(src, src_size as usize) }; + let out = unsafe { slice::from_raw_parts_mut(output_buf, buf_size as usize) }; + + let mut encoder = init_zstd_encoder(N_BLOCK_SIZE_TARGET); + encoder.set_pledged_src_size(Some(src.len() as u64)).expect( + "compress_scroll_batch_bytes_standard: failed to set pledged src size, should be infallible", + ); + + let ret = encoder.write_all(src); + let ret = ret.and_then(|_| encoder.finish()); + if let Err(e) = ret { + return out_as_err(e.to_string().as_str(), out); + } + + let ret = ret.unwrap(); + if ret.len() > buf_size as usize { + return out_as_err( + "compress_scroll_batch_bytes_standard: not enough output buffer for compressed data", + out, + ); + } + out[..ret.len()].copy_from_slice(&ret); + *output_buf_size = ret.len() as u64; + + null() +} diff --git a/libzstd/encoder/src/lib.rs b/libzstd/encoder/src/lib.rs deleted file mode 100644 index 30bb075..0000000 --- a/libzstd/encoder/src/lib.rs +++ /dev/null @@ -1,46 +0,0 @@ -use zstd::stream::Encoder; -use zstd::zstd_safe::{CParameter, ParamSwitch}; - -// re-export zstd -pub use zstd; - -// we use offset window no more than = 17 -// TODO: use for multi-block zstd. -#[allow(dead_code)] -pub const CL_WINDOW_LIMIT: usize = 17; - -/// zstd block size target. -pub const N_BLOCK_SIZE_TARGET: u32 = 124 * 1024; - -/// Maximum number of blocks that we can expect in the encoded data. -pub const N_MAX_BLOCKS: u64 = 10; - -/// Zstd encoder configuration -pub fn init_zstd_encoder(target_block_size: u32) -> Encoder<'static, Vec> { - let mut encoder = Encoder::new(Vec::new(), 0).expect("infallible"); - - // disable compression of literals, i.e. literals will be raw bytes. - encoder - .set_parameter(CParameter::LiteralCompressionMode(ParamSwitch::Disable)) - .expect("infallible"); - // with a hack in zstd we can set window log <= 17 with single segment kept - encoder - .set_parameter(CParameter::WindowLog(17)) - .expect("infallible"); - // set target block size to fit within a single block. - encoder - .set_parameter(CParameter::TargetCBlockSize(target_block_size)) - .expect("infallible"); - // do not include the checksum at the end of the encoded data. - encoder.include_checksum(false).expect("infallible"); - // do not include magic bytes at the start of the frame since we will have a single - // frame. - encoder.include_magicbytes(false).expect("infallible"); - // do not include dictionary id so we have more simple content - encoder.include_dictid(false).expect("infallible"); - // include the content size to know at decode time the expected size of decoded - // data. - encoder.include_contentsize(true).expect("infallible"); - - encoder -} diff --git a/libzstd/src/lib.rs b/libzstd/src/lib.rs deleted file mode 100644 index 34331ab..0000000 --- a/libzstd/src/lib.rs +++ /dev/null @@ -1,54 +0,0 @@ -use core::slice; -use std::io::Write; -use std::os::raw::{c_char, c_uchar}; -use std::ptr::null; -use zstd_encoder::{init_zstd_encoder, N_BLOCK_SIZE_TARGET}; - -fn out_as_err(err: &str, out: &mut [u8]) -> *const c_char { - let msg = if err.len() + 1 > out.len() { - "compress_scroll_batch_bytes: not enough output buffer for the error message" - } else { - err - }; - - let cpy_src = unsafe { slice::from_raw_parts(msg.as_ptr(), msg.len()) }; - out[..cpy_src.len()].copy_from_slice(cpy_src); - out[cpy_src.len()] = 0; // build the c-style string - out.as_ptr() as *const c_char -} - -/// Entry -#[no_mangle] -pub unsafe extern "C" fn compress_scroll_batch_bytes( - src: *const c_uchar, - src_size: u64, - output_buf: *mut c_uchar, - output_buf_size: *mut u64, -) -> *const c_char { - let buf_size = *output_buf_size; - let src = unsafe { slice::from_raw_parts(src, src_size as usize) }; - let out = unsafe { slice::from_raw_parts_mut(output_buf, buf_size as usize) }; - - let mut encoder = init_zstd_encoder(N_BLOCK_SIZE_TARGET); - encoder.set_pledged_src_size(Some(src.len() as u64)).expect( - "compress_scroll_batch_bytes: failed to set pledged src size, should be infallible", - ); - - let ret = encoder.write_all(src); - let ret = ret.and_then(|_| encoder.finish()); - if let Err(e) = ret { - return out_as_err(e.to_string().as_str(), out); - } - - let ret = ret.unwrap(); - if ret.len() > buf_size as usize { - return out_as_err( - "compress_scroll_batch_bytes: not enough output buffer for compressed data", - out, - ); - } - out[..ret.len()].copy_from_slice(&ret); - *output_buf_size = ret.len() as u64; - - null() -}