db: ignore closed EFOS in check

Fixes #5390

Author: sumeerbhola

db.go

@@ -494,7 +494,11 @@ type DB struct {
 		}
 
 		snapshots struct {
-			// The list of active snapshots.
+			// The list of active snapshots, including EFOS snapshots that have not
+			// transitioned. An EFOS snapshot that transitions to FOS is removed
+			// from this list atomically with the transition (using DB.mu). A closed
+			// snapshot (including an EFOS snapshot) may still be in this list
+			// transiently, since the removal is not atomic with the close.
 			snapshotList
 
 			// The cumulative count and size of snapshot-pinned keys written to

ingest.go

@@ -2216,8 +2216,16 @@ func (d *DB) ingestApply(
 	if exciseSpan.Valid() {
 		for s := d.mu.snapshots.root.next; s != &d.mu.snapshots.root; s = s.next {
 			// Skip non-EFOS snapshots, and also skip any EFOS that were created
-			// *after* the excise.
-			if s.efos == nil || base.Visible(exciseSeqNum, s.efos.seqNum, base.SeqNumMax) {
+			// *after* the excise, or are already closed.
+			//
+			// NB: this code can race with EventuallyFileOnlySnapshot.Close, which
+			// closes the channel that isClosed() reads, without holding DB.mu. This
+			// is harmless: if this code observes !isClosed, that means the EFOS was
+			// not closed until after the excise has already updated the LSM state.
+			// It is then fair to confirm that the EFOS did not overlap with the
+			// excise.
+			if s.efos == nil || base.Visible(exciseSeqNum, s.efos.seqNum, base.SeqNumMax) ||
+				s.efos.isClosed() {
 				continue
 			}
 			efos := s.efos
@@ -2227,7 +2235,13 @@ func (d *DB) ingestApply(
 			// snapshot.
 			for i := range efos.protectedRanges {
 				if efos.protectedRanges[i].OverlapsKeyRange(d.cmp, exciseSpan) {
-					panic("unexpected excise of an EventuallyFileOnlySnapshot's bounds")
+					panic(errors.AssertionFailedf(
+						"unexpected excise of an EventuallyFileOnlySnapshot's bounds: "+
+							"excise [%s, %s) seqnum: %s, efos [%s, %s) seqnum: %s transitioned: %t closed: %t visible: %s",
+						d.opts.Comparer.FormatKey(exciseSpan.Start), d.opts.Comparer.FormatKey(exciseSpan.End),
+						exciseSeqNum, d.opts.Comparer.FormatKey(efos.protectedRanges[i].Start),
+						d.opts.Comparer.FormatKey(efos.protectedRanges[i].End), efos.seqNum,
+						efos.hasTransitioned(), efos.isClosed(), d.mu.versions.visibleSeqNum.Load()))
 				}
 			}
 		}

snapshot.go

@@ -377,6 +377,15 @@ func (es *EventuallyFileOnlySnapshot) hasTransitioned() bool {
 	return es.mu.vers != nil
 }
 
+func (es *EventuallyFileOnlySnapshot) isClosed() bool {
+	select {
+	case <-es.closed:
+		return true
+	default:
+		return false
+	}
+}
+
 // waitForFlush waits for a flush on any memtables that need to be flushed
 // before this EFOS can transition to a file-only snapshot. If this EFOS is
 // waiting on a flush of the mutable memtable, it forces a rotation within