Merge remote-tracking branch 'origin/main' into ci-improve-shell-scripts

Author: torcolvin

.editorconfig

@@ -1,3 +1,16 @@
+; https://editorconfig.org/
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[manifest/**.{json,xml}]
+indent_size = 4
+
 [*.sh]
 indent_style = space
 indent_size = 4

.github/workflows/manifestvalidation.yml

@@ -0,0 +1,46 @@
+# Copyright 2022-Present Couchbase, Inc.
+#
+# Use of this software is governed by the Business Source License included in
+# the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
+# file, in accordance with the Business Source License, use of this software
+# will be governed by the Apache License, Version 2.0, included in the file
+# licenses/APL2.txt.
+
+name: manifestvalidation
+permissions:
+  contents: read
+
+on:
+  push:
+    paths:
+      - 'manifest/**'
+      - '.github/workflows/manifestvalidation.yml'
+    branches:
+      - 'master'
+      - 'main'
+      - 'release/*'
+      - 'feature/*'
+      - 'CBG*'
+      - 'ci-*'
+  pull_request:
+    paths:
+      - 'manifest/**'
+      - '.github/workflows/manifestvalidation.yml'
+    branches:
+      - 'master'
+      - 'main'
+      - 'release/*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ !contains(github.ref, 'release/')}}
+
+jobs:
+  manifest_validation:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Ensure manifest product-config properties are sorted
+        id: validate
+        run: |
+          diff <(jq --sort-keys . manifest/product-config.json) <(jq . manifest/product-config.json)

base/collection.go

@@ -357,12 +357,19 @@ func (b *GocbV2Bucket) GetCCVSettings(ctx context.Context) (ccvEnabled bool, max
 	if err != nil {
 		return false, nil, fmt.Errorf("error getting vbucket count: %v", err)
 	}
-	// we'd always expect a CAS value per vbucket if CCV is enabled and has propagated correctly - so fail if that's not the case
+
+	highCAS := make(map[VBNo]uint64, numVBuckets)
+	// we'd always expect a CAS value per vbucket if CCV is enabled and has propagated correctly
+	// except after a bucket flushed in Server < 7.6.8 see MB-64705
+	// Treating this as ECCV=true with startingCas=0, which will mean imports will all get tagged with bucket SourceID.
 	if len(response.VBucketsMaxCas) != int(numVBuckets) {
-		return false, nil, fmt.Errorf("error getting vbucket CAS, expected %d vbucket CAS values, got %q", numVBuckets, response.VBucketsMaxCas)
+		InfofCtx(ctx, KeyBucket, "Bucket %q has enableCrossClusterVersioning=true but unexpected number of vbucket CAS values - expected %d, got %+v. Treating all imports as originating on this Couchbase Server cluster.", MD(b.GetName()), numVBuckets, response.VBucketsMaxCas)
+		for i := range numVBuckets {
+			highCAS[VBNo(i)] = 0
+		}
+		return true, highCAS, nil
 	}
 
-	highCAS := make(map[VBNo]uint64, numVBuckets)
 	for i, casStr := range response.VBucketsMaxCas {
 		cas, err := strconv.ParseUint(casStr, 10, 64)
 		if err != nil {

base/version.go

@@ -20,7 +20,7 @@ const (
 	ProductName = "Couchbase Sync Gateway"
 
 	ProductAPIVersionMajor = "4"
-	ProductAPIVersionMinor = "0"
+	ProductAPIVersionMinor = "1"
 	ProductAPIVersion      = ProductAPIVersionMajor + "." + ProductAPIVersionMinor
 )
 

db/crud.go

@@ -1051,7 +1051,7 @@ func (db *DatabaseCollectionWithUser) updateHLV(ctx context.Context, d *Document
 
 // MigrateAttachmentMetadata will move any attachment metadata defined in sync data to global sync xattr
 func (c *DatabaseCollectionWithUser) MigrateAttachmentMetadata(ctx context.Context, docID string, cas uint64, syncData *SyncData) error {
-	xattrs, _, err := c.dataStore.GetXattrs(ctx, docID, []string{base.GlobalXattrName})
+	xattrs, _, err := c.dataStore.GetXattrs(ctx, docID, []string{base.GlobalXattrName, base.VirtualXattrRevSeqNo})
 	if err != nil && !base.IsXattrNotFoundError(err) {
 		return err
 	}
@@ -1073,14 +1073,32 @@ func (c *DatabaseCollectionWithUser) MigrateAttachmentMetadata(ctx context.Conte
 	if err != nil {
 		return base.RedactErrorf("Failed to Marshal sync data when attempting to migrate sync data attachments to global xattr with id: %s. Error: %v", base.UD(docID), err)
 	}
+	revSeqNo, err := unmarshalRevSeqNo(xattrs[base.VirtualXattrRevSeqNo])
+	if err != nil {
+		base.InfofCtx(ctx, base.KeyCRUD, "Could not determine revSeqNo found when attempting to migrate sync data attachments to global xattr for doc %q. Assuming 0. Error: %v", base.UD(docID), err)
+	}
+
+	metadataOnlyUpdate := &MetadataOnlyUpdate{
+		HexCAS:           expandMacroCASValueString, // when non-empty, this is replaced with cas macro expansion
+		PreviousHexCAS:   syncData.Cas,
+		PreviousRevSeqNo: revSeqNo,
+	}
+	rawMouXattr, err := base.JSONMarshal(metadataOnlyUpdate)
+	if err != nil {
+		return base.RedactErrorf("Failed to marshal _mou when attempting to migrate sync data attachments to global xattr with id: %s. Error: %v", base.UD(docID), err)
+	}
 
 	// build macro expansion for sync data. This will avoid the update to xattrs causing an extra import event (i.e. sync cas will be == to doc cas)
 	opts := &sgbucket.MutateInOptions{}
-	spec := macroExpandSpec(base.SyncXattrName)
+	spec := append(macroExpandSpec(base.SyncXattrName), sgbucket.NewMacroExpansionSpec(XattrMouCasPath(), sgbucket.MacroCas))
 	opts.MacroExpansion = spec
 	opts.PreserveExpiry = true // if doc has expiry, we should preserve this
 
-	updatedXattr := map[string][]byte{base.SyncXattrName: rawSyncXattr, base.GlobalXattrName: globalXattr}
+	updatedXattr := map[string][]byte{
+		base.SyncXattrName:   rawSyncXattr,
+		base.GlobalXattrName: globalXattr,
+		base.MouXattrName:    rawMouXattr,
+	}
 	_, err = c.dataStore.UpdateXattrs(ctx, docID, 0, cas, updatedXattr, opts)
 	return err
 }
@@ -2667,6 +2685,11 @@ func (db *DatabaseCollectionWithUser) updateAndReturnDoc(ctx context.Context, do
 	skipObsoleteAttachmentsRemoval := false
 	isNewDocCreation := false
 
+	// Don't remove obsolete attachments if using ECCV - the other cluster may still need them!
+	if db.dbCtx.CachedCCVEnabled.Load() {
+		skipObsoleteAttachmentsRemoval = true
+	}
+
 	if db.UseXattrs() || upgradeInProgress {
 		var casOut uint64
 		// Update the document, storing metadata in extended attribute

db/database.go

@@ -1681,7 +1681,11 @@ func (db *DatabaseContext) GetVersionPruningWindow(ctx context.Context, forceRef
 func (db *DatabaseContext) updateCCVSettings(ctx context.Context) error {
 	cbStore, ok := base.AsCouchbaseBucketStore(db.Bucket)
 	if !ok {
-		db.CachedCCVEnabled.Store(false)
+		// for rosmar, ECCV is always enabled, mark starting cas as 0
+		db.CachedCCVEnabled.Store(true)
+		for vbNo := range db.numVBuckets {
+			db.CachedCCVStartingCas.Store(base.VBNo(vbNo), 0)
+		}
 		return nil
 	}
 

db/import_test.go

@@ -119,9 +119,6 @@ func TestFeedImport(t *testing.T) {
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			if base.UnitTestUrlIsWalrus() {
-				t.Skip("rosmar does not support vBuckets and unknownSource")
-			}
 			docID := SafeDocumentName(t, t.Name())
 			db.CachedCCVEnabled.Store(testCase.eccv)
 			for vBucket := range db.numVBuckets {
@@ -282,9 +279,6 @@ func TestOnDemandImport(t *testing.T) {
 	}
 	for _, testCase := range testCases {
 		t.Run(testCase.name, func(t *testing.T) {
-			if base.UnitTestUrlIsWalrus() {
-				t.Skip("rosmar does not support vBuckets and unknownSource")
-			}
 			docID := SafeDocumentName(t, t.Name())
 			db.CachedCCVEnabled.Store(testCase.eccv)
 			for vBucket := range db.numVBuckets {

docs/api/admin-capella.yaml

@@ -10,7 +10,7 @@ openapi: 3.0.3
 info:
   title: App Services Admin API
   description: 'App Services manages access and synchronization between Couchbase Lite and Couchbase Capella'
-  version: '4.0'
+  version: '4.1'
   license:
     name: Business Source License 1.1 (BSL)
     url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE'

docs/api/admin.yaml

@@ -13,7 +13,7 @@ info:
     # Introduction
 
     The Sync Gateway Admin REST API is used to administer user accounts and roles, and to run administrative tasks in superuser mode.
-  version: '4.0'
+  version: '4.1'
   license:
     name: Business Source License 1.1 (BSL)
     url: 'https://github.com/couchbase/sync_gateway/blob/master/LICENSE'

docs/api/components/schemas.yaml

@@ -1577,9 +1577,7 @@ Database:
           $ref: '#/Replication'
     revs_limit:
       description: |-
-        The maximum depth a document's revision tree can grow too.
-
-        The minimum is `20` if conflicts are allowed and 0 if not. It is not recommended to go below `100` when conflicts are allowed. The default is `100` if conflicts are allowed and `50` if not.
+        The maximum depth a document's revision tree can grow to.
       type: number
       default: 50
       minimum: 0