From ec8f294c7cbf76047f70736237ad8ae1f1a23253 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 22 Apr 2025 10:39:48 +0200 Subject: [PATCH 1/2] Use pull-style iterators to build index and tree Closes #49 --- ca/ca.go | 76 ++++++++++++++-------- cmd/mtc/main.go | 13 ++-- internal/index.go | 86 +++++++++++++------------ mirror/mirror.go | 28 ++++++-- mtc.go | 148 ++++++++++++++++++++++++++++-------------- utils.go | 160 +++++++++++++++++++++++++++++++++++----------- 6 files changed, 346 insertions(+), 165 deletions(-) diff --git a/ca/ca.go b/ca/ca.go index 865b5f1..5ad2124 100644 --- a/ca/ca.go +++ b/ca/ca.go @@ -739,19 +739,44 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { } } + // Prepare index builder + indexPath := gopath.Join(dir, "index") + indexW, err := os.Create(indexPath) + if err != nil { + return fmt.Errorf("creating %s: %w", indexPath, err) + } + + defer indexW.Close() + ib := internal.NewIndexBuilder(indexW) + + // Prepare tree builder + tb := batch.NewTreeBuilder() + if !empty { + entryOffset := 0 + evidenceOffset := 0 + var entryKey [mtc.HashLen]byte + err = h.walkQueue(func(ar mtc.AssertionRequest) error { + oldEvidenceOffset := evidenceOffset + oldEntryOffset := entryOffset + // Skip assertions that are already expired. if start, _ := batch.ValidityInterval(); ar.NotAfter.Before(start) { return nil } be := mtc.NewBatchEntry(ar.Assertion, ar.NotAfter) + if err := be.Key(entryKey[:]); err != nil { + return fmt.Errorf("Computing key for %x: %w", ar.Checksum, err) + } + buf, err := be.MarshalBinary() if err != nil { return fmt.Errorf("Marshalling assertion %x: %w", ar.Checksum, err) } + // Write out BatchEntry _, err = besBW.Write(buf) if err != nil { return fmt.Errorf( @@ -761,7 +786,10 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { err, ) } + entryOffset += len(buf) + // Prepare evidence when applicable: for instance by deduplicating + // intermediates in umbilical chains. evs := ar.Evidence if ucBuilder != nil { for i := range len(evs) { @@ -777,6 +805,7 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { } } + // Write out Evidence buf, err = evs.MarshalBinary() if err != nil { return fmt.Errorf("Marshalling evidence %x: %w", ar.Checksum, err) @@ -791,6 +820,21 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { err, ) } + evidenceOffset += len(buf) + + // Feed entry to tree builder, and entry and evidence to + // index builder. + if err := tb.Push(&be); err != nil { + return fmt.Errorf("Building tree: %w", err) + } + + if err := ib.Push(internal.IndexBuildEntry{ + EvidenceOffset: uint64(oldEvidenceOffset), + Offset: uint64(oldEntryOffset), + Key: entryKey, + }); err != nil { + return fmt.Errorf("Building index: %w", err) + } return nil }) @@ -808,11 +852,6 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { if err != nil { return fmt.Errorf("closing %s: %w", besPath, err) } - besR, err := os.OpenFile(besPath, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("opening %s: %w", besPath, err) - } - defer besR.Close() err = evBW.Flush() if err != nil { @@ -823,11 +862,6 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { if err != nil { return fmt.Errorf("closing %s: %w", evPath, err) } - evR, err := os.OpenFile(evPath, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("opening %s: %w", evPath, err) - } - defer evR.Close() if ucBuilder != nil { err = ucBuilder.Finish() @@ -841,9 +875,9 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { } // Compute tree - tree, err := batch.ComputeTree(bufio.NewReader(besR)) + tree, err := tb.Finish() if err != nil { - return fmt.Errorf("computing tree: %w", err) + return fmt.Errorf("finishing tree: %w", err) } treePath := gopath.Join(dir, "tree") @@ -865,22 +899,9 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { } // Compute index - _, err = besR.Seek(0, io.SeekStart) - if err != nil { - return fmt.Errorf("seeking %s to start: %w", besPath, err) - } - - indexPath := gopath.Join(dir, "index") - indexW, err := os.Create(indexPath) - if err != nil { - return fmt.Errorf("creating %s: %w", indexPath, err) - } - - defer indexW.Close() - - err = internal.ComputeIndex(besR, evR, indexW) + err = ib.Finish() if err != nil { - return fmt.Errorf("computing %s to start: %w", indexPath, err) + return fmt.Errorf("finishing index: %w", err) } // Sign validity window @@ -899,6 +920,7 @@ func (h *Handle) issueBatchTo(dir string, batch mtc.Batch, empty bool) error { if err != nil { return fmt.Errorf("writing to %s: %w", wPath, err) } + return nil } diff --git a/cmd/mtc/main.go b/cmd/mtc/main.go index 518ab61..df1b6e2 100644 --- a/cmd/mtc/main.go +++ b/cmd/mtc/main.go @@ -961,9 +961,10 @@ func handleInspectEvidence(cc *cli.Context) error { defer r.Close() count := 0 - err = mtc.UnmarshalEvidenceLists( - bufio.NewReader(r), - func(_ int, el *mtc.EvidenceList) error { + + err = mtc.ForEach( + mtc.UnmarshalEvidenceLists(bufio.NewReader(r)), + func(el *mtc.EvidenceList) error { count++ w := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0) err := writeEvidenceList(w, *el) @@ -990,9 +991,9 @@ func handleInspectEntries(cc *cli.Context) error { defer r.Close() count := 0 - err = mtc.UnmarshalBatchEntries( - bufio.NewReader(r), - func(_ int, be *mtc.BatchEntry) error { + err = mtc.ForEach( + mtc.UnmarshalBatchEntries(bufio.NewReader(r)), + func(be *mtc.BatchEntry) error { count++ cs := be.Claims subj := be.Subject diff --git a/internal/index.go b/internal/index.go index ed56d98..699b226 100644 --- a/internal/index.go +++ b/internal/index.go @@ -44,6 +44,12 @@ type IndexSearchResult struct { EvidenceOffset uint64 } +type IndexBuildEntry struct { + Key [mtc.HashLen]byte + Offset uint64 + EvidenceOffset uint64 +} + // Opens an index func OpenIndex(path string) (*Index, error) { r, err := mmap.Open(path) @@ -163,64 +169,60 @@ type indexEntry struct { evidenceOffset uint64 } -// Reads a streams of BatchEntry Evidence from beReader and evReader, -// and writes the index to w. -func ComputeIndex(beReader, evReader io.Reader, w io.Writer) error { - // First compute keys - seqno := uint64(0) - entries := []indexEntry{} - - var key [mtc.HashLen]byte - err := mtc.UnmarshalBatchEntries(beReader, func(offset int, - be *mtc.BatchEntry) error { - err := be.Key(key[:]) - if err != nil { - return err - } - entries = append(entries, indexEntry{ - seqno: seqno, - key: key, - offset: uint64(offset), - }) - seqno++ - return nil - }) +type IndexBuilder struct { + err error + w io.Writer + seqno uint64 + entries []indexEntry +} - seqno = uint64(0) - err = mtc.UnmarshalEvidenceLists(evReader, func(offset int, _ *mtc.EvidenceList) error { - entries[seqno].evidenceOffset = uint64(offset) - seqno++ - return nil - }) +func NewIndexBuilder(w io.Writer) *IndexBuilder { + return &IndexBuilder{ + w: w, + entries: []indexEntry{}, + } +} - if err != nil { - return fmt.Errorf("computing keys: %w", err) +func (b *IndexBuilder) Push(in IndexBuildEntry) error { + if b.err != nil { + return b.err } + b.entries = append(b.entries, indexEntry{ + seqno: b.seqno, + key: in.Key, + offset: in.Offset, + evidenceOffset: in.EvidenceOffset, + }) + + b.seqno++ + return nil +} + +func (b *IndexBuilder) Finish() error { // Sort by key - slices.SortFunc(entries, func(a, b indexEntry) int { + slices.SortFunc(b.entries, func(a, b indexEntry) int { return bytes.Compare(a.key[:], b.key[:]) }) // Write out - bw := bufio.NewWriter(w) + bw := bufio.NewWriter(b.w) var lastKey [mtc.HashLen]byte - for _, entry := range entries { + for _, entry := range b.entries { if lastKey == entry.key { // skip duplicate entries continue } lastKey = entry.key - var b cryptobyte.Builder - b.AddBytes(entry.key[:]) - b.AddUint64(entry.seqno) - b.AddUint64(entry.offset) - b.AddUint64(entry.evidenceOffset) - buf, _ := b.Bytes() - - _, err = bw.Write(buf) - if err != nil { + var cb cryptobyte.Builder + cb.AddBytes(entry.key[:]) + cb.AddUint64(entry.seqno) + cb.AddUint64(entry.offset) + cb.AddUint64(entry.evidenceOffset) + buf, _ := cb.Bytes() + + if _, err := bw.Write(buf); err != nil { return fmt.Errorf("writing index: %w", err) } } diff --git a/mirror/mirror.go b/mirror/mirror.go index fe4e03d..d423f27 100644 --- a/mirror/mirror.go +++ b/mirror/mirror.go @@ -303,15 +303,35 @@ func (h *Handle) fetchBatch(number uint32) error { } // Recompute tree - aasR, err := os.OpenFile(besPath, os.O_RDONLY, 0) + tb := batch.NewTreeBuilder() + besR, err := os.OpenFile(besPath, os.O_RDONLY, 0) if err != nil { return fmt.Errorf("opening %s: %w", besPath, err) } - defer aasR.Close() + defer besR.Close() - tree, err := batch.ComputeTree(bufio.NewReader(aasR)) + besC := mtc.UnmarshalBatchEntries(bufio.NewReader(besR)) + defer besC.Close() + + var be mtc.BatchEntry + for { + err := besC.Pull(&be) + if err == mtc.EOF { + break + } + + if err != nil { + return fmt.Errorf("reading %s: %w", besPath, err) + } + + if err := tb.Push(&be); err != nil { + return fmt.Errorf("building tree: %w", err) + } + } + + tree, err := tb.Finish() if err != nil { - return fmt.Errorf("computing tree: %w", err) + return fmt.Errorf("finishing tree: %w", err) } treePath := gopath.Join(dir, "tree") diff --git a/mtc.go b/mtc.go index 9b22ed6..9a45eed 100644 --- a/mtc.go +++ b/mtc.go @@ -1275,40 +1275,14 @@ func (t *Tree) AuthenticationPath(index uint64) ([]byte, error) { return ret.Bytes(), nil } -// Reads a stream of BatchEntry from in, hashes them, and -// returns the concatenated hashes. -func (batch *Batch) hashLeaves(r io.Reader) ([]byte, error) { - ret := &bytes.Buffer{} - - // First read all batch entries and hash them. - var index uint64 - hash := make([]byte, HashLen) - - err := unmarshal(r, func(_ int, be *BatchEntry) error { - err := be.Hash(hash, batch, index) - if err != nil { - return err - } - _, _ = ret.Write(hash) - index++ - return nil - }) - - if err != nil { - return nil, err - } - - return ret.Bytes(), nil +// Unmarshals BatchEntry from r. +func UnmarshalBatchEntries(r io.Reader) Cursor[*BatchEntry] { + return unmarshal[*BatchEntry](r) } -// Unmarshals BatchEntry from r and calls f for each, with -// the offset in the stream as first argument, and the batch entry -// as second argument. -// -// Returns early on error. -func UnmarshalBatchEntries(r io.Reader, - f func(int, *BatchEntry) error) error { - return unmarshal(r, f) +// Unmarshals BatchEntry from r, keeping note of the offset of each. +func UnmarshalBatchEntriesWithOffset(r io.Reader) Cursor[*BatchEntryWithOffset] { + return unmarshal[*BatchEntryWithOffset](r) } // Unmarshals a single BatchEntry from r. @@ -1421,14 +1395,44 @@ func (batch *Batch) hashEmpty(out []byte, index uint64, level uint8) error { return nil } -// Compute Merkle tree from a stream of BatchEntry from in. -func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { - // First hash the leaves - leaves, err := batch.hashLeaves(r) - if err != nil { - return nil, fmt.Errorf("HashLeaves: %w", err) +type TreeBuilder struct { + leafHashes *bytes.Buffer + index uint64 + batch *Batch + err error +} + +func (batch *Batch) NewTreeBuilder() *TreeBuilder { + return &TreeBuilder{ + leafHashes: &bytes.Buffer{}, + batch: batch, + } +} + +func (b *TreeBuilder) Push(be *BatchEntry) error { + var hash [HashLen]byte + + if b.err != nil { + return b.err + } + + if err := be.Hash(hash[:], b.batch, b.index); err != nil { + b.err = err + return err + } + + _, _ = b.leafHashes.Write(hash[:]) + b.index++ + + return nil +} + +func (b *TreeBuilder) Finish() (*Tree, error) { + if b.err != nil { + return nil, b.err } + leaves := b.leafHashes.Bytes() nLeaves := uint64(len(leaves)) / uint64(HashLen) buf := bytes.NewBuffer(leaves) @@ -1437,7 +1441,7 @@ func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { nLeaves: 0, buf: make([]byte, HashLen), } - if err := batch.hashEmpty(tree.buf[:], 0, 0); err != nil { + if err := b.batch.hashEmpty(tree.buf[:], 0, 0); err != nil { return nil, err } return tree, nil @@ -1454,7 +1458,7 @@ func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { for nNodes != 1 { // Add empty node if number of leaves on this level is odd if nNodes&1 == 1 { - if err := batch.hashEmpty(h, nNodes, level); err != nil { + if err := b.batch.hashEmpty(h, nNodes, level); err != nil { return nil, err } _, _ = buf.Write(h) @@ -1468,7 +1472,7 @@ func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { leftRight := buf.Bytes()[offset+2*HashLen*int(i):] left := leftRight[:HashLen] right := leftRight[HashLen : 2*HashLen] - if err := batch.hashNode(h, left, right, i, level); err != nil { + if err := b.batch.hashNode(h, left, right, i, level); err != nil { return nil, err } _, _ = buf.Write(h) @@ -1480,6 +1484,20 @@ func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { return &Tree{buf: buf.Bytes(), nLeaves: nLeaves}, nil } +// Convenience function to compute Merkle tree from +// a stream of BatchEntry from in. +func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { + tb := batch.NewTreeBuilder() + err := ForEach( + UnmarshalBatchEntries(r), + tb.Push, + ) + if err != nil { + return nil, err + } + return tb.Finish() +} + // Computes the key a BatchEntry for this assertion would have in the index. func (a *Assertion) EntryKey(out []byte) error { // We use dummy not_after, as it's ignored in the key. @@ -1493,6 +1511,9 @@ func (a *Assertion) EntryKey(out []byte) error { // computing the key. This allows us to look up a BatchEntry for some // assertion that does not contain the not_after field. func (be *BatchEntry) Key(out []byte) error { + if len(out) != 32 { + return errors.New("BatchEntry keys are 32 bytes") + } buf, err := be.MarshalBinary() if err != nil { return err @@ -1912,14 +1933,9 @@ func (el *EvidenceList) MarshalBinary() ([]byte, error) { return b.Bytes() } -// Unmarshals EvidenceLists from r and calls f for each, with -// the offset in the stream as first argument, and the EvidenceList -// as second argument. -// -// Returns early on error. -func UnmarshalEvidenceLists(r io.Reader, - f func(int, *EvidenceList) error) error { - return unmarshal(r, f) +// Unmarshals EvidenceLists from r. +func UnmarshalEvidenceLists(r io.Reader) Cursor[*EvidenceList] { + return unmarshal[*EvidenceList](r) } // Unmarshals single EvidenceList from r. @@ -2141,3 +2157,37 @@ func (tai *TrustAnchorIdentifier) unmarshal(s *cryptobyte.String) error { tai.Issuer = oid return nil } + +// Same as BatchEntry, but keeps track of offset within stream it was +// unmarshalled from. Used to create index. +type BatchEntryWithOffset struct { + BatchEntry + Offset int +} + +func (be *BatchEntryWithOffset) unmarshal(s *cryptobyte.String) error { + return be.BatchEntry.unmarshal(s) +} +func (be *BatchEntryWithOffset) maxSize() int { + return be.BatchEntry.maxSize() +} +func (be *BatchEntryWithOffset) recordOffset(offset int) { + be.Offset = offset +} + +// Same as EvidenceList, but keeps track of offset within stream it was +// unmarshalled from. Used to create index. +type EvidenceListWithOffset struct { + EvidenceList + Offset int +} + +func (ev *EvidenceListWithOffset) unmarshal(s *cryptobyte.String) error { + return ev.EvidenceList.unmarshal(s) +} +func (ev *EvidenceListWithOffset) maxSize() int { + return ev.EvidenceList.maxSize() +} +func (ev *EvidenceListWithOffset) recordOffset(offset int) { + ev.Offset = offset +} diff --git a/utils.go b/utils.go index e166f28..f776a54 100644 --- a/utils.go +++ b/utils.go @@ -8,6 +8,47 @@ import ( "reflect" ) +// Pull-style iterator similar to io.ReadCloser but for general T and +// only pulls one value at a time. Assumes T is refernece. +type Cursor[T any] interface { + // Pull one value and write to out. + Pull(out T) error + + // Release underlying resources. Closing twice is no-op. + Close() error +} + +// Pull from c and call f on each. +// +// Abort early if f returns an error. Closes c. +func ForEach[T any](c Cursor[T], f func(T) error) error { + var t T + defer c.Close() + + // T is a pointer type, so by default nil. We need to allocate t + // first. The obvious t = new(T) is wrong, as new(T) is of type *T + // instead of T. The following does what we want. + reflect.ValueOf(&t).Elem().Set(reflect.New(reflect.TypeOf(t).Elem())) + + for { + err := c.Pull(t) + if err == EOF { + return nil + } + + if err != nil { + return err + } + + err = f(t) + if err != nil { + return err + } + } + + panic("shouldn't reach") +} + var ( // ErrTruncated is a parsing error returned when the input seems to have // been truncated. @@ -21,8 +62,8 @@ var ( // match the corresponding data. ErrChecksumInvalid = errors.New("Invalid checksum") - // Used to stop unmarshalling early - errShortCircuit = errors.New("Short circuit") + // Used to indicate end of stream for Cursor[T]. + EOF = errors.New("EOF") ) type unmarshaler interface { @@ -38,74 +79,119 @@ type unmarshaler interface { maxSize() int } +// If an object (that implements unmarshaler) implements this +// interface, when unmarshalling a list of objects, we'll call the +// recordOffset() function with the offset of the object in the list. +// +// This allows us to implement both UnmarshalBatchEntries() and +// UnmarshalBatchEntriesWithOffsets() without too much hassle. +// The latter is required to create an index into the entries file. +type offsetRecorder interface { + recordOffset(offset int) +} + // Unmarshals a single T from r. -func unmarshalOne[T unmarshaler](r io.Reader) (ret T, err error) { - err = unmarshal(r, func(_ int, msg T) error { - ret = msg - return errShortCircuit - }) - if err == errShortCircuit { - err = nil - } - return +func unmarshalOne[T unmarshaler](r io.Reader) (T, error) { + c := unmarshal[T](r) + defer c.Close() + var t T + + // T is a pointer type, so by default nil. We need to allocate t + // first. The obvious t = new(T) is wrong, as new(T) is of type *T + // instead of T. The following does what we want. + reflect.ValueOf(&t).Elem().Set(reflect.New(reflect.TypeOf(t).Elem())) + + err := c.Pull(t) + return t, err } -// Unmarshals a stream of T from r, and call f on each of them as second -// argument, with the offset in the stream as the first argument. -// -// If f returns an error, break. -func unmarshal[T unmarshaler](r io.Reader, f func(int, T) error) error { - // Create a new instance of T - var msg T - reflect.ValueOf(&msg).Elem().Set(reflect.New(reflect.TypeOf(msg).Elem())) +type streamingUnmarshaler[T unmarshaler] struct { + r io.Reader + buf []byte + s cryptobyte.String + maxSize int + err error + offset int + recordOffset bool +} +// Unmarshals a stream of T from r. +func unmarshal[T unmarshaler](r io.Reader) Cursor[T] { + var dummy T buf := make([]byte, 512) - s := cryptobyte.String(buf[:0]) - maxSize := msg.maxSize() - offset := 0 + _, recordOffset := any(dummy).(offsetRecorder) + return &streamingUnmarshaler[T]{ + r: r, + buf: buf, + s: cryptobyte.String(buf[:0]), + maxSize: dummy.maxSize(), + recordOffset: recordOffset, + } +} +func (c *streamingUnmarshaler[T]) Close() error { + return nil +} + +func (c *streamingUnmarshaler[T]) pull(out T) error { for { - oldS := s - err := msg.unmarshal(&s) + oldS := c.s + err := out.unmarshal(&c.s) - // Success? Call f and continue if err == nil { - if err := f(offset, msg); err != nil { - return err + if c.recordOffset { + any(out).(offsetRecorder).recordOffset(c.offset) } - offset += len(oldS) - len(s) - continue + + c.offset += len(oldS) - len(c.s) + return nil } if err != ErrTruncated { return err } + // Ok, we need to extend the buffer. // Did we have sucecss in the last iteration? - if cap(oldS) != cap(buf) { + if cap(oldS) != cap(c.buf) { // Yes, we need to move the remaining data to the front. - copy(buf[:len(oldS)], oldS) + copy(c.buf[:len(oldS)], oldS) } else { // No. We grow the buffer. No need to move the remaining data: // it's still in front. - if len(buf) > maxSize { + if len(c.buf) > c.maxSize { // This shouldn't be possible, but let's error gracefully. return errors.New("Unexpected ErrTruncated") } - buf = append(buf, make([]byte, len(buf))...) + c.buf = append(c.buf, make([]byte, len(c.buf))...) } - n, err := r.Read(buf[len(oldS):]) + n, err := c.r.Read(c.buf[len(oldS):]) if n == 0 && err == io.EOF { - break + return EOF } if n == 0 { return err } - s = cryptobyte.String(buf[:len(oldS)+n]) + c.s = cryptobyte.String(c.buf[:len(oldS)+n]) } - return nil + + panic("shouldn't reach") +} + +func (c *streamingUnmarshaler[T]) Pull(out T) error { + if c.err != nil { + return c.err + } + + err := c.pull(out) + + if err != nil { + c.err = err + } + + return err } func copyUint8LengthPrefixed(s *cryptobyte.String, out *[]byte) bool { From 032e8cd1fe52376e973f15cf09e8348b6820c236 Mon Sep 17 00:00:00 2001 From: Bas Westerbaan Date: Tue, 22 Apr 2025 14:52:31 +0200 Subject: [PATCH 2/2] Luke's suggestions on #55 Co-authored-by: Luke Valenta --- internal/index.go | 22 +++++++++++----------- mtc.go | 2 +- utils.go | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/internal/index.go b/internal/index.go index 699b226..f5d0bcb 100644 --- a/internal/index.go +++ b/internal/index.go @@ -183,32 +183,32 @@ func NewIndexBuilder(w io.Writer) *IndexBuilder { } } -func (b *IndexBuilder) Push(in IndexBuildEntry) error { - if b.err != nil { - return b.err +func (ib *IndexBuilder) Push(in IndexBuildEntry) error { + if ib.err != nil { + return ib.err } - b.entries = append(b.entries, indexEntry{ - seqno: b.seqno, + ib.entries = append(ib.entries, indexEntry{ + seqno: ib.seqno, key: in.Key, offset: in.Offset, evidenceOffset: in.EvidenceOffset, }) - b.seqno++ + ib.seqno++ return nil } -func (b *IndexBuilder) Finish() error { +func (ib *IndexBuilder) Finish() error { // Sort by key - slices.SortFunc(b.entries, func(a, b indexEntry) int { - return bytes.Compare(a.key[:], b.key[:]) + slices.SortFunc(ib.entries, func(a, ib indexEntry) int { + return bytes.Compare(a.key[:], ib.key[:]) }) // Write out - bw := bufio.NewWriter(b.w) + bw := bufio.NewWriter(ib.w) var lastKey [mtc.HashLen]byte - for _, entry := range b.entries { + for _, entry := range ib.entries { if lastKey == entry.key { // skip duplicate entries continue diff --git a/mtc.go b/mtc.go index 9a45eed..dd9241e 100644 --- a/mtc.go +++ b/mtc.go @@ -1485,7 +1485,7 @@ func (b *TreeBuilder) Finish() (*Tree, error) { } // Convenience function to compute Merkle tree from -// a stream of BatchEntry from in. +// a stream of BatchEntry from r. func (batch *Batch) ComputeTree(r io.Reader) (*Tree, error) { tb := batch.NewTreeBuilder() err := ForEach( diff --git a/utils.go b/utils.go index f776a54..68bf213 100644 --- a/utils.go +++ b/utils.go @@ -9,7 +9,7 @@ import ( ) // Pull-style iterator similar to io.ReadCloser but for general T and -// only pulls one value at a time. Assumes T is refernece. +// only pulls one value at a time. Assumes T is a reference. type Cursor[T any] interface { // Pull one value and write to out. Pull(out T) error @@ -152,7 +152,7 @@ func (c *streamingUnmarshaler[T]) pull(out T) error { } // Ok, we need to extend the buffer. - // Did we have sucecss in the last iteration? + // Did we have success in the last iteration? if cap(oldS) != cap(c.buf) { // Yes, we need to move the remaining data to the front. copy(c.buf[:len(oldS)], oldS)