Remove test skips and disabled BLIP replication codepaths for delta sync when using legacy revs (missing follow-up for CBG-3748 / #7718)

Some remaining test cases remain skipped until we store legacy rev revision body backups (non-delta sync case - tracked with another ticket)

Author: bbrks

db/blip_handler.go

@@ -795,8 +795,7 @@ func (bh *blipHandler) handleChanges(rq *blip.Message) error {
 	}
 	output.Write([]byte("]"))
 	response := rq.Response()
-	// Disable delta sync for protocol versions < 4, CBG-3748 (backwards compatibility for revID delta sync)
-	if bh.sgCanUseDeltas && bh.useHLV() {
+	if bh.sgCanUseDeltas {
 		base.DebugfCtx(bh.loggingCtx, base.KeyAll, "Setting deltas=true property on handleChanges response")
 		response.Properties[ChangesResponseDeltas] = trueProperty
 		bh.replicationStats.HandleChangesDeltaRequestedCount.Add(int64(nRequested))
@@ -845,7 +844,6 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error {
 	defer func() {
 		bh.replicationStats.HandleChangesTime.Add(time.Since(startTime).Nanoseconds())
 	}()
-	changesContainLegacyRevs := false // keep track if proposed changes have legacy revs for delta sync purposes
 	versionVectorProtocol := bh.useHLV()
 
 	for i, change := range changeList {
@@ -867,7 +865,6 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error {
 			proposedVersionStr := ExtractCVFromProposeChangesRev(rev)
 			status, currentRev = bh.collection.CheckProposedVersion(bh.loggingCtx, docID, proposedVersionStr, parentRevID, rev)
 		} else {
-			changesContainLegacyRevs = true
 			status, currentRev = bh.collection.CheckProposedRev(bh.loggingCtx, docID, rev, parentRevID)
 		}
 		if status == ProposedRev_OK_IsNew {
@@ -899,8 +896,7 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error {
 	}
 	output.Write([]byte("]"))
 	response := rq.Response()
-	// Disable delta sync for protocol versions < 4 or changes batches that have legacy revs in them, CBG-3748 (backwards compatibility for revID delta sync)
-	if bh.sgCanUseDeltas && bh.useHLV() && !changesContainLegacyRevs {
+	if bh.sgCanUseDeltas {
 		base.DebugfCtx(bh.loggingCtx, base.KeyAll, "Setting deltas=true property on proposeChanges response")
 		response.Properties[ChangesResponseDeltas] = trueProperty
 	}

db/blip_sync_context.go

@@ -387,8 +387,7 @@ func (bsc *BlipSyncContext) handleChangesResponse(ctx context.Context, sender *b
 
 			var err error
 
-			// fallback to sending full revisions for non hlv aware peers, CBG-3748
-			if deltaSrcRevID != "" && bsc.useHLV() {
+			if deltaSrcRevID != "" {
 				err = bsc.sendRevAsDelta(ctx, sender, docID, rev, deltaSrcRevID, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx)
 			} else {
 				err = bsc.sendRevision(ctx, sender, docID, rev, seq, knownRevs, maxHistory, handleChangesResponseDbCollection, collectionIdx, legacyRev)

db/crud_test.go

@@ -1081,8 +1081,15 @@ func BenchmarkHandleRevDelta(b *testing.B) {
 }
 
 func TestGetAvailableRevAttachments(t *testing.T) {
-	t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)")
 	db, ctx := setupTestDB(t)
+	db.Options.StoreLegacyRevTreeData = true
+
+	// TODO: CBG-4840 Only requires delta sync until restoration of non-delta sync RevTree ID revision body backups"
+	if !base.IsEnterpriseEdition() {
+		t.Skip("CBG-4840 - temp skip see above comment")
+	}
+	db.Options.DeltaSyncOptions = DeltaSyncOptions{Enabled: true, RevMaxAgeSeconds: 300}
+
 	defer db.Close(ctx)
 	collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)
 
@@ -1104,19 +1111,19 @@ func TestGetAvailableRevAttachments(t *testing.T) {
 
 	// Get available attachments by immediate ancestor revision or parent revision
 	meta, found := collection.getAvailableRevAttachments(ctx, doc, parent)
+	require.True(t, found, "Ancestor should exists")
 	attachment := meta["camera.txt"].(map[string]interface{})
 	assert.Equal(t, "sha1-VoSNiNQGHE1HirIS5HMxj6CrlHI=", attachment["digest"])
 	assert.Equal(t, json.Number("20"), attachment["length"])
 	assert.Equal(t, json.Number("1"), attachment["revpos"])
-	assert.True(t, found, "Ancestor should exists")
 
 	// Get available attachments by immediate ancestor revision
 	meta, found = collection.getAvailableRevAttachments(ctx, doc, ancestor)
+	require.True(t, found, "Ancestor should exists")
 	attachment = meta["camera.txt"].(map[string]interface{})
 	assert.Equal(t, "sha1-VoSNiNQGHE1HirIS5HMxj6CrlHI=", attachment["digest"])
 	assert.Equal(t, json.Number("20"), attachment["length"])
 	assert.Equal(t, json.Number("1"), attachment["revpos"])
-	assert.True(t, found, "Ancestor should exists")
 }
 
 func TestGet1xRevAndChannels(t *testing.T) {

db/database.go

@@ -70,13 +70,13 @@ const (
 	DefaultSGReplicateEnabled               = true
 	DefaultSGReplicateWebsocketPingInterval = time.Minute * 5
 	DefaultCompactInterval                  = 24 * time.Hour
+	DefaultStoreLegacyRevTreeData           = true // for 4.0, this is opt-out - we can switch to opt-in at a later date
 )
 
 // Default values for delta sync
 var (
 	DefaultDeltaSyncEnabled   = false
 	DefaultDeltaSyncRevMaxAge = uint32(60 * 60 * 24) // 24 hours in seconds
-	DefaultStoreLegacyRevs    = true                 // for 4.0, this is opt-out - we can switch to opt-in at a later date
 )
 
 var (
@@ -206,6 +206,7 @@ type DatabaseContextOptions struct {
 	NumIndexPartitions            *uint32           // Number of partitions for GSI indexes, if not set will default to 1
 	ImportVersion                 uint64            // Version included in import DCP checkpoints, incremented when collections added to db
 	DisablePublicAllDocs          bool              // Disable public access to the _all_docs endpoint for this database
+	StoreLegacyRevTreeData        bool              // Whether to store additional data for legacy rev tree support in delta sync and replication backup revs
 }
 
 type ConfigPrincipals struct {
@@ -241,7 +242,6 @@ type UserViewsOptions struct {
 type DeltaSyncOptions struct {
 	Enabled          bool   // Whether delta sync is enabled (EE only)
 	RevMaxAgeSeconds uint32 // The number of seconds deltas for old revs are available for
-	StoreLegacyRevs  bool   // Whether to store additional data to allow legacy RevTree ID support for delta sync
 }
 
 type APIEndpoints struct {

db/database_collection.go

@@ -198,7 +198,7 @@ func (c *DatabaseCollection) deltaSyncRevMaxAgeSeconds() uint32 {
 
 // storeLegacyRevTreeData returns true if legacy revision tree pointer data is stored. This is controlled at the database level.
 func (c *DatabaseCollection) storeLegacyRevTreeData() bool {
-	return c.dbCtx.Options.DeltaSyncOptions.StoreLegacyRevs
+	return c.dbCtx.Options.StoreLegacyRevTreeData
 }
 
 // eventMgr manages nofication events. This is controlled at database level.

db/database_test.go

@@ -978,8 +978,8 @@ func TestDeltaSyncWhenFromRevIsLegacyRevTreeID(t *testing.T) {
 	db.Options.DeltaSyncOptions = DeltaSyncOptions{
 		Enabled:          true,
 		RevMaxAgeSeconds: 300,
-		StoreLegacyRevs:  true,
 	}
+	db.Options.StoreLegacyRevTreeData = true
 
 	defer db.Close(ctx)
 	collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)

db/revision_cache_test.go

@@ -623,10 +623,17 @@ func TestRevisionCacheInternalProperties(t *testing.T) {
 }
 
 func TestBypassRevisionCache(t *testing.T) {
-	t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)")
-	base.SetUpTestLogging(t, base.LevelInfo, base.KeyAll)
+	base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD)
 
 	db, ctx := setupTestDB(t)
+	db.Options.StoreLegacyRevTreeData = true
+
+	// TODO: CBG-4840 Only requires delta sync until restoration of non-delta sync RevTree ID revision body backups"
+	if !base.IsEnterpriseEdition() {
+		t.Skip("CBG-4840 - temp skip see above comment")
+	}
+	db.Options.DeltaSyncOptions = DeltaSyncOptions{Enabled: true, RevMaxAgeSeconds: 300}
+
 	defer db.Close(ctx)
 
 	collection, ctx := GetSingleDatabaseCollectionWithUser(ctx, t, db)

rest/attachment_test.go

@@ -687,12 +687,18 @@ func TestBulkGetBadAttachmentReproIssue2528(t *testing.T) {
 }
 
 func TestConflictWithInvalidAttachment(t *testing.T) {
-	t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)")
-	rt := NewRestTester(t, nil)
+	rt := NewRestTesterPersistentConfigNoDB(t)
 	defer rt.Close()
 
 	dbConfig := rt.NewDbConfig()
 	dbConfig.AllowConflicts = base.Ptr(true)
+
+	// TODO: CBG-4840 Only requires delta sync until restoration of non-delta sync RevTree ID revision body backups"
+	if !base.IsEnterpriseEdition() {
+		t.Skip("CBG-4840 - temp skip see above comment")
+	}
+	dbConfig.DeltaSync = &DeltaSyncConfig{Enabled: base.Ptr(true), RevMaxAgeSeconds: base.Ptr(uint32(300))}
+
 	RequireStatus(t, rt.CreateDatabase("db", dbConfig), http.StatusCreated)
 	// Create Doc
 	version := rt.CreateTestDoc("doc1")

rest/blip_api_crud_test.go

@@ -1822,8 +1822,7 @@ func TestPutRevV4(t *testing.T) {
 // Actual:
 // - Same as Expected (this test is unable to repro SG #3281, but is being left in as a regression test)
 func TestGetRemovedDoc(t *testing.T) {
-	t.Skip("Revs are backed up by hash of CV now, test needs to fetch backup rev by revID, CBG-3748 (backwards compatibility for revID)")
-	base.SetUpTestLogging(t, base.LevelInfo, base.KeyHTTP, base.KeySync, base.KeySyncMsg)
+	base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll)
 
 	rt := NewRestTester(t, &RestTesterConfig{SyncFn: channels.DocChannelsSyncFunction})
 	defer rt.Close()
@@ -1894,7 +1893,8 @@ func TestGetRemovedDoc(t *testing.T) {
 	// Delete any temp revisions in case this prevents the bug from showing up (didn't make a difference)
 	tempRevisionDocID := base.RevPrefix + "foo:5:3-cde"
 	err = rt.GetSingleDataStore().Delete(tempRevisionDocID)
-	assert.NoError(t, err, "Unexpected Error")
+	// TODO: CBG-4840 - Requires restoration of non-delta sync RevTree revision backups
+	// assert.NoError(t, err, "Unexpected Error")
 
 	// Try to get rev 3 via BLIP API and assert that _removed == true
 	resultDoc, err = bt2.GetDocAtRev("foo", "3-cde")

rest/blip_api_delta_sync_test.go

@@ -331,8 +331,7 @@ func TestBlipDeltaSyncNewAttachmentPull(t *testing.T) {
 		// Check EE is delta, and CE is full-body replication
 		// msg, ok = client.pullReplication.WaitForMessage(5)
 		msg = btcRunner.WaitForBlipRevMessage(client.id, doc1ID, version2)
-		// Delta sync only works for Version vectors, CBG-3748 (backwards compatibility for revID)
-		sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV()
+		sgCanUseDeltas := base.IsEnterpriseEdition()
 		if sgCanUseDeltas {
 			// Check the request was sent with the correct deltaSrc property
 			client.AssertDeltaSrcProperty(t, msg, version)
@@ -417,8 +416,7 @@ func TestBlipDeltaSyncPull(t *testing.T) {
 		msg := btcRunner.WaitForBlipRevMessage(client.id, docID, version2)
 
 		// Check EE is delta, and CE is full-body replication
-		// Delta sync only works for Version vectors, CBG-3748 (backwards compatibility for revID)
-		sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV()
+		sgCanUseDeltas := base.IsEnterpriseEdition()
 		if sgCanUseDeltas {
 			// Check the request was sent with the correct deltaSrc property
 			client.AssertDeltaSrcProperty(t, msg, version)
@@ -464,7 +462,6 @@ func TestBlipDeltaSyncPullResend(t *testing.T) {
 		GuestEnabled: true,
 	}
 	btcRunner := NewBlipTesterClientRunner(t)
-	btcRunner.SkipSubtest[RevtreeSubtestName] = true // delta sync not implemented for rev tree replication, CBG-3748
 	btcRunner.Run(func(t *testing.T) {
 		rt := NewRestTester(t,
 			&rtConfig)
@@ -659,8 +656,7 @@ func TestBlipDeltaSyncPullTombstoned(t *testing.T) {
 			deltasRequestedEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasRequested.Value()
 			deltasSentEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value()
 		}
-		// delta sync not implemented for rev tree replication, CBG-3748
-		sgCanUseDelta := base.IsEnterpriseEdition() && client.UseHLV()
+		sgCanUseDelta := base.IsEnterpriseEdition()
 		if sgCanUseDelta {
 			assert.Equal(t, deltaCacheHitsStart, deltaCacheHitsEnd)
 			assert.Equal(t, deltaCacheMissesStart+1, deltaCacheMissesEnd)
@@ -804,8 +800,7 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) {
 			deltasSentEnd = rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value()
 		}
 
-		// delta sync not implemented for rev tree replication, CBG-3748
-		sgCanUseDelta := base.IsEnterpriseEdition() && client1.UseHLV()
+		sgCanUseDelta := base.IsEnterpriseEdition()
 		if sgCanUseDelta {
 			assert.Equal(t, deltaCacheHitsStart+1, deltaCacheHitsEnd)
 			assert.Equal(t, deltaCacheMissesStart+1, deltaCacheMissesEnd)
@@ -851,7 +846,7 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) {
 		defer client.Close()
 
 		client.ClientDeltas = true
-		sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV()
+		sgCanUseDeltas := base.IsEnterpriseEdition()
 		btcRunner.StartPull(client.id)
 
 		// create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4
@@ -878,7 +873,6 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) {
 
 		// Check EE is delta
 		// Check the request was sent with the correct deltaSrc property
-		// delta sync not implemented for rev tree replication, CBG-3748
 		if sgCanUseDeltas {
 			client.AssertDeltaSrcProperty(t, msg, version1)
 		} else {
@@ -919,7 +913,6 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) {
 		updatedDeltaCacheHits := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheHit.Value()
 		updatedDeltaCacheMisses := rt.GetDatabase().DbStats.DeltaSync().DeltaCacheMiss.Value()
 
-		// delta sync not implemented for rev tree replication, CBG-3748
 		if sgCanUseDeltas {
 			assert.Equal(t, deltaCacheHits+1, updatedDeltaCacheHits)
 			assert.Equal(t, deltaCacheMisses, updatedDeltaCacheMisses)
@@ -955,7 +948,7 @@ func TestBlipDeltaSyncPush(t *testing.T) {
 		client := btcRunner.NewBlipTesterClientOptsWithRT(rt, nil)
 		defer client.Close()
 		client.ClientDeltas = true
-		sgCanUseDeltas := base.IsEnterpriseEdition() && client.UseHLV()
+		sgCanUseDeltas := base.IsEnterpriseEdition()
 
 		btcRunner.StartPull(client.id)