refactor: canonical serialization

.gitignore

@@ -19,6 +19,7 @@ contracts/broadcast
 lightnode/docker/build-info.txt
 lightnode/docker/args.sh
 
+.cache
 .idea
 .env
 .vscode

api/hashing/blob_header_v2_compat_test.go

@@ -0,0 +1,58 @@
+package hashing
+
+import (
+	"testing"
+	"time"
+
+	commonv1 "github.com/Layr-Labs/eigenda/api/grpc/common"
+	commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2"
+	grpc "github.com/Layr-Labs/eigenda/api/grpc/validator"
+	hashingv2 "github.com/Layr-Labs/eigenda/api/hashing/v2"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/crypto/sha3"
+)
+
+func TestV2BlobHeaderHashMatchesLegacyHashBlobHeader(t *testing.T) {
+	tsNanos := int64(1234567890123456)
+	header := &commonv2.BlobHeader{
+		Version:       1,
+		QuorumNumbers: []uint32{0, 2, 7},
+		Commitment: &commonv1.BlobCommitment{
+			Commitment:       []byte{0xaa, 0xbb},
+			LengthCommitment: []byte{0x10, 0x11, 0x12},
+			LengthProof:      []byte{0x20},
+			Length:           123,
+		},
+		PaymentHeader: &commonv2.PaymentHeader{
+			AccountId:         "0xabc",
+			Timestamp:         tsNanos,
+			CumulativePayment: []byte{0x09, 0x08, 0x07},
+		},
+	}
+
+	// "Old" blob header hash: use the legacy streaming serializer+hash used by node_hashing.go.
+	legacyHasher := sha3.NewLegacyKeccak256()
+	err := hashBlobHeader(legacyHasher, header)
+	require.NoError(t, err)
+	legacyHash := legacyHasher.Sum(nil)
+
+	req := &grpc.StoreChunksRequest{
+		Batch: &commonv2.Batch{
+			Header: &commonv2.BatchHeader{BatchRoot: []byte{0x01}, ReferenceBlockNumber: 1},
+			BlobCertificates: []*commonv2.BlobCertificate{{
+				BlobHeader: header,
+				Signature:  []byte{0x00},
+				RelayKeys:  []uint32{1},
+			}},
+		},
+		DisperserID: 1,
+		Timestamp:   1,
+	}
+
+	got, err := hashingv2.BlobHeadersHashesAndTimestamps(req)
+	require.NoError(t, err)
+	require.Len(t, got, 1)
+
+	require.Equal(t, legacyHash, got[0].Hash)
+	require.True(t, got[0].Timestamp.Equal(time.Unix(0, tsNanos)))
+}

api/hashing/store_chunk_request_v2_compat_test.go

@@ -0,0 +1,72 @@
+package hashing
+
+import (
+	"testing"
+
+	commonv1 "github.com/Layr-Labs/eigenda/api/grpc/common"
+	commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2"
+	grpc "github.com/Layr-Labs/eigenda/api/grpc/validator"
+	hashingv2 "github.com/Layr-Labs/eigenda/api/hashing/v2"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHashStoreChunksRequestMatchesLegacyHashStoreChunksRequest(t *testing.T) {
+	req := &grpc.StoreChunksRequest{
+		Batch: &commonv2.Batch{
+			Header: &commonv2.BatchHeader{
+				BatchRoot:            []byte{0x01, 0x02, 0x03, 0x04},
+				ReferenceBlockNumber: 42,
+			},
+			BlobCertificates: []*commonv2.BlobCertificate{
+				{
+					BlobHeader: &commonv2.BlobHeader{
+						Version:       1,
+						QuorumNumbers: []uint32{0, 2, 7},
+						Commitment: &commonv1.BlobCommitment{
+							Commitment:       []byte{0xaa, 0xbb},
+							LengthCommitment: []byte{0x10, 0x11, 0x12},
+							LengthProof:      []byte{0x20},
+							Length:           123,
+						},
+						PaymentHeader: &commonv2.PaymentHeader{
+							AccountId:         "0xabc",
+							Timestamp:         999,
+							CumulativePayment: []byte{0x09, 0x08, 0x07},
+						},
+					},
+					Signature: []byte{0xde, 0xad, 0xbe, 0xef},
+					RelayKeys: []uint32{5, 6},
+				},
+				{
+					BlobHeader: &commonv2.BlobHeader{
+						Version:       2,
+						QuorumNumbers: []uint32{1},
+						Commitment: &commonv1.BlobCommitment{
+							Commitment:       []byte{0x01},
+							LengthCommitment: []byte{},
+							LengthProof:      []byte{0xff, 0xee},
+							Length:           0,
+						},
+						PaymentHeader: &commonv2.PaymentHeader{
+							AccountId:         "0xdef",
+							Timestamp:         123456789,
+							CumulativePayment: []byte{0x01, 0x00},
+						},
+					},
+					Signature: []byte{0x00},
+					RelayKeys: []uint32{0},
+				},
+			},
+		},
+		DisperserID: 7,
+		Timestamp:   55,
+	}
+
+	h1, err := HashStoreChunksRequest(req)
+	require.NoError(t, err)
+
+	h2, err := hashingv2.HashStoreChunksRequest(req)
+	require.NoError(t, err)
+
+	require.Equal(t, h1, h2, "legacy (manual) serializer hash must match canonical (struc) serializer hash")
+}

api/hashing/v2/blob_header.go

@@ -0,0 +1,52 @@
+package hashing
+
+import (
+	"fmt"
+	"time"
+
+	grpc "github.com/Layr-Labs/eigenda/api/grpc/validator"
+	"github.com/Layr-Labs/eigenda/api/hashing/v2/serialize"
+	"golang.org/x/crypto/sha3"
+)
+
+// BlobHeaderHashWithTimestamp is a tuple of a blob header hash and the timestamp of the blob header.
+type BlobHeaderHashWithTimestamp struct {
+	// Hash is canonical serialized blob header hash.
+	Hash []byte
+	// Timestamp is derived from PaymentHeader.Timestamp (nanoseconds since epoch).
+	Timestamp time.Time
+}
+
+// BlobHeadersHashesAndTimestamps returns a list of per-BlobHeader hashes (one per BlobCertificate)
+// with the timestamp.
+func BlobHeadersHashesAndTimestamps(request *grpc.StoreChunksRequest) ([]BlobHeaderHashWithTimestamp, error) {
+	certs := request.GetBatch().GetBlobCertificates()
+	out := make([]BlobHeaderHashWithTimestamp, len(certs))
+	for i, cert := range certs {
+		if cert == nil {
+			return nil, fmt.Errorf("nil BlobCertificate at index %d", i)
+		}
+		header := cert.GetBlobHeader()
+		if header == nil {
+			return nil, fmt.Errorf("nil BlobHeader at index %d", i)
+		}
+		paymentHeader := header.GetPaymentHeader()
+		if paymentHeader == nil {
+			return nil, fmt.Errorf("nil PaymentHeader at index %d", i)
+		}
+
+		headerBytes, err := serialize.SerializeBlobHeader(header)
+		if err != nil {
+			return nil, fmt.Errorf("failed to serialize blob header at index %d: %w", i, err)
+		}
+		// Must match legacy hashing (Keccak-256, not SHA3-256).
+		hasher := sha3.NewLegacyKeccak256()
+		_, _ = hasher.Write(headerBytes)
+		out[i] = BlobHeaderHashWithTimestamp{
+			Hash:      hasher.Sum(nil),
+			Timestamp: time.Unix(0, paymentHeader.GetTimestamp()),
+		}
+	}
+
+	return out, nil
+}

api/hashing/v2/serialize/store_chunk_request.go

@@ -0,0 +1,194 @@
+package serialize
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+
+	commonv2 "github.com/Layr-Labs/eigenda/api/grpc/common/v2"
+	grpc "github.com/Layr-Labs/eigenda/api/grpc/validator"
+	"github.com/lunixbochs/struc"
+)
+
+// This file provides a struc-based encoder that preserves the *exact* byte layout of
+// node_hashing.go's manual serializer+hash and separates the serialization from the hashing.
+//
+// Wire-format invariants preserved:
+// - Big-endian integers (struc defaults to big-endian)
+// - For []byte and []uint32: a uint32 length prefix, followed by elements/bytes
+// - Domain is written as raw bytes (no length prefix) at the start
+// - Redundant QuorumNumbersLength field is preserved (it appears before the slice length prefix)
+
+// initialBufCap is a preallocation hint to reduce allocations.
+const initialBufCap = 512
+
+// validatorStoreChunksRequestDomain is the StoreChunksRequest hash domain prefix.
+// Kept here to avoid an import cycle (hashing <-> serialize).
+const validatorStoreChunksRequestDomain = "validator.StoreChunksRequest"
+
+type canonicalStoreChunksRequestBody struct {
+	BatchHeader canonicalBatchHeader
+
+	BlobCertificatesLen uint32 `struc:"uint32,sizeof=BlobCertificates"`
+	BlobCertificates    []canonicalBlobCertificate
+
+	DisperserID uint32
+	Timestamp   uint32
+}
+
+type canonicalBatchHeader struct {
+	RootLen uint32 `struc:"uint32,sizeof=Root"`
+	Root    []byte
+
+	ReferenceBlockNumber uint64
+}
+
+type canonicalBlobCertificate struct {
+	BlobHeader canonicalBlobHeader
+
+	SignatureLen uint32 `struc:"uint32,sizeof=Signature"`
+	Signature    []byte
+
+	RelayKeysLen uint32 `struc:"uint32,sizeof=RelayKeys"`
+	RelayKeys    []uint32
+}
+
+type canonicalBlobHeader struct {
+	Version uint32
+
+	// Kept for backwards-compatible encoding: this is written first...
+	QuorumNumbersLength uint32
+	// ...then the real slice length prefix (same value) followed by elements.
+	QuorumNumbersLen uint32 `struc:"uint32,sizeof=QuorumNumbers"`
+	QuorumNumbers    []uint32
+
+	Commitment    canonicalBlobCommitment
+	PaymentHeader canonicalPaymentHeader
+}
+
+type canonicalBlobCommitment struct {
+	CommitmentLen uint32 `struc:"uint32,sizeof=Commitment"`
+	Commitment    []byte
+
+	LengthCommitmentLen uint32 `struc:"uint32,sizeof=LengthCommitment"`
+	LengthCommitment    []byte
+
+	LengthProofLen uint32 `struc:"uint32,sizeof=LengthProof"`
+	LengthProof    []byte
+
+	Length uint32
+}
+
+type canonicalPaymentHeader struct {
+	// store_chunk.go encodes AccountId as serializeBytes([]byte(string))
+	AccountIdLen uint32 `struc:"uint32,sizeof=AccountId"`
+	AccountId    []byte
+
+	Timestamp int64
+
+	CumulativePaymentLen uint32 `struc:"uint32,sizeof=CumulativePayment"`
+	CumulativePayment    []byte
+}
+
+func SerializeStoreChunksRequest(request *grpc.StoreChunksRequest) ([]byte, error) {
+	if request.GetBatch() == nil || request.GetBatch().GetHeader() == nil {
+		return nil, fmt.Errorf("missing batch/header")
+	}
+
+	certs := request.GetBatch().GetBlobCertificates()
+	if len(certs) > math.MaxUint32 {
+		return nil, fmt.Errorf("array is too long: %d", len(certs))
+	}
+
+	body := canonicalStoreChunksRequestBody{
+		BatchHeader: canonicalBatchHeader{
+			Root:                 request.GetBatch().GetHeader().GetBatchRoot(),
+			ReferenceBlockNumber: request.GetBatch().GetHeader().GetReferenceBlockNumber(),
+		},
+		BlobCertificates: make([]canonicalBlobCertificate, len(certs)),
+		DisperserID:      request.GetDisperserID(),
+		Timestamp:        request.GetTimestamp(),
+	}
+
+	for i, cert := range certs {
+		if cert == nil || cert.GetBlobHeader() == nil ||
+			cert.GetBlobHeader().GetCommitment() == nil ||
+			cert.GetBlobHeader().GetPaymentHeader() == nil {
+			return nil, fmt.Errorf("missing blob certificate fields at index %d", i)
+		}
+
+		bh := cert.GetBlobHeader()
+		commitment := bh.GetCommitment()
+		payment := bh.GetPaymentHeader()
+
+		qnums := bh.GetQuorumNumbers()
+		qnLen := uint32(len(qnums))
+
+		body.BlobCertificates[i] = canonicalBlobCertificate{
+			BlobHeader: canonicalBlobHeader{
+				Version:             bh.GetVersion(),
+				QuorumNumbersLength: qnLen,
+				QuorumNumbers:       qnums,
+				Commitment: canonicalBlobCommitment{
+					Commitment:       commitment.GetCommitment(),
+					LengthCommitment: commitment.GetLengthCommitment(),
+					LengthProof:      commitment.GetLengthProof(),
+					Length:           commitment.GetLength(),
+				},
+				PaymentHeader: canonicalPaymentHeader{
+					AccountId:         []byte(payment.GetAccountId()),
+					Timestamp:         payment.GetTimestamp(),
+					CumulativePayment: payment.GetCumulativePayment(),
+				},
+			},
+			Signature: cert.GetSignature(),
+			RelayKeys: cert.GetRelayKeys(),
+		}
+	}
+
+	var buf bytes.Buffer
+	buf.Grow(initialBufCap)
+
+	_, _ = buf.WriteString(validatorStoreChunksRequestDomain)
+
+	if err := struc.Pack(&buf, &body); err != nil {
+		return nil, fmt.Errorf("failed to pack canonical StoreChunksRequest: %w", err)
+	}
+	return buf.Bytes(), nil
+}
+
+func SerializeBlobHeader(header *commonv2.BlobHeader) ([]byte, error) {
+	if header == nil || header.GetCommitment() == nil || header.GetPaymentHeader() == nil {
+		return nil, fmt.Errorf("missing blob header fields")
+	}
+
+	qnums := header.GetQuorumNumbers()
+	qnLen := uint32(len(qnums))
+
+	// Preserve current SerializeBlobHeader behavior from store_chunk.go:
+	// it only sets Commitment.Commitment and leaves the rest empty/zero.
+	ch := canonicalBlobHeader{
+		Version:             header.GetVersion(),
+		QuorumNumbersLength: qnLen,
+		QuorumNumbers:       qnums,
+		Commitment: canonicalBlobCommitment{
+			Commitment:       header.GetCommitment().GetCommitment(),
+			LengthCommitment: header.GetCommitment().GetLengthCommitment(),
+			LengthProof:      header.GetCommitment().GetLengthProof(),
+			Length:           header.GetCommitment().GetLength(),
+		},
+		PaymentHeader: canonicalPaymentHeader{
+			AccountId:         []byte(header.GetPaymentHeader().GetAccountId()),
+			Timestamp:         header.GetPaymentHeader().GetTimestamp(),
+			CumulativePayment: header.GetPaymentHeader().GetCumulativePayment(),
+		},
+	}
+
+	var buf bytes.Buffer
+	buf.Grow(initialBufCap)
+
+	if err := struc.Pack(&buf, &ch); err != nil {
+		return nil, fmt.Errorf("failed to pack canonical BlobHeader: %w", err)
+	}
+	return buf.Bytes(), nil
+}