internal/keyspan: add special SPANSTART, SPANEND key kinds

Author: jbowens

batch.go

@@ -2284,7 +2284,7 @@ func (i *flushableBatchIter) extractValue() base.LazyValue {
 		return base.LazyValue{}
 	}
 	kind := InternalKeyKind(p[0])
-	if kind > InternalKeyKindMax {
+	if kind > base.InternalKeyKindDurableMax {
 		i.err = base.CorruptionErrorf("corrupted batch")
 		return base.LazyValue{}
 	}

batchrepr/reader.go

@@ -87,7 +87,7 @@ func (r *Reader) Next() (kind base.InternalKeyKind, ukey []byte, value []byte, o
 		return 0, nil, nil, false, nil
 	}
 	kind = base.InternalKeyKind((*r)[0])
-	if kind > base.InternalKeyKindMax {
+	if kind > base.InternalKeyKindDurableMax {
 		return 0, nil, nil, false, errors.Wrapf(ErrInvalidBatch, "invalid key kind 0x%x", (*r)[0])
 	}
 	*r, ukey, ok = DecodeStr((*r)[1:])

ingest.go

@@ -139,7 +139,7 @@ func ingestSynthesizeShared(
 		// a.RANGEDEL.100, with a.RANGEDEL.100 being the smallest key. To create a
 		// correct bound, we just use the maximum key kind (which sorts first).
 		// Similarly, we use the smallest key kind for the largest key.
-		smallestPointKey := base.MakeInternalKey(sm.SmallestPointKey.UserKey, 0, base.InternalKeyKindMax)
+		smallestPointKey := base.MakeInternalKey(sm.SmallestPointKey.UserKey, 0, base.InternalKeyKindDurableMax)
 		largestPointKey := base.MakeInternalKey(sm.LargestPointKey.UserKey, 0, 0)
 		if sm.LargestPointKey.IsExclusiveSentinel() {
 			largestPointKey = base.MakeRangeDeleteSentinelKey(sm.LargestPointKey.UserKey)
@@ -220,12 +220,12 @@ func ingestLoad1External(
 		if e.EndKeyIsInclusive {
 			meta.ExtendPointKeyBounds(
 				opts.Comparer.Compare,
-				base.MakeInternalKey(smallestCopy, 0, InternalKeyKindMax),
+				base.MakeInternalKey(smallestCopy, 0, base.InternalKeyKindDurableMax),
 				base.MakeInternalKey(largestCopy, 0, 0))
 		} else {
 			meta.ExtendPointKeyBounds(
 				opts.Comparer.Compare,
-				base.MakeInternalKey(smallestCopy, 0, InternalKeyKindMax),
+				base.MakeInternalKey(smallestCopy, 0, base.InternalKeyKindDurableMax),
 				base.MakeRangeDeleteSentinelKey(largestCopy))
 		}
 	}

internal/base/internal.go

@@ -96,6 +96,9 @@ const (
 	// heuristics, but is not required to be accurate for correctness.
 	InternalKeyKindDeleteSized InternalKeyKind = 23
 
+	InternalKeyKindSpanStart InternalKeyKind = 24
+	InternalKeyKindSpanEnd   InternalKeyKind = 25
+
 	// This maximum value isn't part of the file format. Future extensions may
 	// increase this value.
 	//
@@ -105,7 +108,12 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 23
+	InternalKeyKindMax InternalKeyKind = 25
+
+	// NB: This is less than InternalKeyKindSpanStart and InternalKeyKindSpanEnd
+	// because those key kinds are never used in durable formats; only as
+	// special in-memory indicators.
+	InternalKeyKindDurableMax InternalKeyKind = 23
 
 	// Internal to the sstable format. Not exposed by any sstable iterator.
 	// Declared here to prevent definition of valid key kinds that set this bit.
@@ -157,6 +165,8 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
 	InternalKeyKindIngestSST:      "INGESTSST",
 	InternalKeyKindDeleteSized:    "DELSIZED",
+	InternalKeyKindSpanStart:      "SPANSTART",
+	InternalKeyKindSpanEnd:        "SPANEND",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -249,6 +259,8 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
 	"INGESTSST":     InternalKeyKindIngestSST,
 	"DELSIZED":      InternalKeyKindDeleteSized,
+	"SPANSTART":     InternalKeyKindSpanStart,
+	"SPANEND":       InternalKeyKindSpanEnd,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The
@@ -476,7 +488,8 @@ func (k InternalKey) IsExclusiveSentinel() bool {
 	}
 	switch kind := k.Kind(); kind {
 	case InternalKeyKindRangeDelete, InternalKeyKindRangeKeyDelete,
-		InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet:
+		InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet,
+		InternalKeyKindSpanStart, InternalKeyKindSpanEnd:
 		return true
 	default:
 		return false

internal/base/internal_test.go

@@ -41,7 +41,7 @@ func TestInvalidInternalKey(t *testing.T) {
 		"\x01\x02\x03\x04\x05\x06\x07",
 		"foo",
 		"foo\x08\x07\x06\x05\x04\x03\x02",
-		"foo\x18\x07\x06\x05\x04\x03\x02\x01",
+		"foo\x1a\x07\x06\x05\x04\x03\x02\x01",
 	}
 	for _, tc := range testCases {
 		k := DecodeInternalKey([]byte(tc))

internal/keyspan/interleaving_iter.go

@@ -199,6 +199,10 @@ type InterleavingIterOpts struct {
 	// end keys of spans (in addition to the start keys, which are always
 	// interleaved).
 	InterleaveEndKeys bool
+	// UseBoundaryKeyKinds configures the interleaving iterator to interleave
+	// keys using SPANSTART and SPANEND key kinds for start and end boundaries
+	// respectively, rather than the key kind of the first key in the Span.
+	UseBoundaryKeyKinds bool
 }
 
 // Init initializes the InterleavingIter to interleave point keys from pointIter
@@ -979,7 +983,12 @@ func (i *InterleavingIter) yieldPointKey() *base.InternalKV {
 
 func (i *InterleavingIter) yieldSyntheticSpanStartMarker(lowerBound []byte) *base.InternalKV {
 	i.spanMarker.K.UserKey = i.startKey()
-	i.spanMarker.K.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, i.span.Keys[0].Kind())
+
+	kind := base.InternalKeyKindSpanStart
+	if !i.opts.UseBoundaryKeyKinds {
+		kind = i.span.Keys[0].Kind()
+	}
+	i.spanMarker.K.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, kind)
 
 	// Truncate the key we return to our lower bound if we have one. Note that
 	// we use the lowerBound function parameter, not i.lower. The lowerBound
@@ -1015,8 +1024,12 @@ func (i *InterleavingIter) yieldSyntheticSpanStartMarker(lowerBound []byte) *bas
 }
 
 func (i *InterleavingIter) yieldSyntheticSpanEndMarker() *base.InternalKV {
+	kind := base.InternalKeyKindSpanEnd
+	if !i.opts.UseBoundaryKeyKinds {
+		kind = i.span.Keys[0].Kind()
+	}
 	i.spanMarker.K.UserKey = i.endKey()
-	i.spanMarker.K.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, i.span.Keys[0].Kind())
+	i.spanMarker.K.Trailer = base.MakeTrailer(base.InternalKeySeqNumMax, kind)
 	return i.verify(&i.spanMarker)
 }
 

internal/keyspan/interleaving_iter_test.go

@@ -123,6 +123,7 @@ func runInterleavingIterTest(t *testing.T, filename string) {
 				hooks.threshold = []byte(strings.Join(cmdArg.Vals, ""))
 			}
 			opts.InterleaveEndKeys = td.HasArg("interleave-end-keys")
+			opts.UseBoundaryKeyKinds = td.HasArg("use-boundary-key-kinds")
 			iter.Init(testkeys.Comparer, &pointIter, keyspanIter, opts)
 			// Clear any previous bounds.
 			pointIter.SetBounds(nil, nil)

internal/keyspan/testdata/interleaving_iter

@@ -1152,6 +1152,58 @@ Span: f-g:{(#6,RANGEDEL)}
 -- SpanChanged(nil)
 .
 
+iter interleave-end-keys use-boundary-key-kinds
+first
+next
+next
+next
+next
+next
+next
+next
+next
+next
+next
+----
+-- SpanChanged(nil)
+-- SpanChanged(nil)
+PointKey: a@4#8,SET
+Span: <invalid>
+-
+-- SpanChanged(b-e:{(#5,RANGEDEL)})
+PointKey: b#72057594037927935,SPANSTART
+Span: b-e:{(#5,RANGEDEL)}
+-
+PointKey: c@11#8,SET
+Span: b-e:{(#5,RANGEDEL)}
+-
+PointKey: c@3#8,SET
+Span: b-e:{(#5,RANGEDEL)}
+-
+PointKey: c@1#4,SET
+Span: b-e:{(#5,RANGEDEL)}
+-
+PointKey: d@5#3,SET
+Span: b-e:{(#5,RANGEDEL)}
+-
+PointKey: e#72057594037927935,SPANEND
+Span: b-e:{(#5,RANGEDEL)}
+-
+-- SpanChanged(nil)
+PointKey: e@9#2,SET
+Span: <invalid>
+-
+-- SpanChanged(f-g:{(#6,RANGEDEL)})
+PointKey: f#72057594037927935,SPANSTART
+Span: f-g:{(#6,RANGEDEL)}
+-
+PointKey: g#72057594037927935,SPANEND
+Span: f-g:{(#6,RANGEDEL)}
+-
+-- SpanChanged(nil)
+.
+
+
 iter interleave-end-keys
 last
 prev