From 21f3b3f8ac675e4e9eb641272b0aa88fb8c8ca13 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Wed, 20 Aug 2025 14:28:12 +0100 Subject: [PATCH 01/14] Allow CV to be used in OCC writes (Document Update/Delete and the associated attachment endpoints) using existing `rev` parameter - which can automatically detect RevTreeID/CV. Switch test helpers to use CV in writes when available for coverage, and allow fallback to RevTreeID. Prevent CV OCC value in doc updates for docs in conflict or against non-current versions, since we don't maintain linear version history like we do for RevTrees and can't correlate an old CV with a branched revision ID. --- base/util.go | 15 ++ base/util_test.go | 26 +++ db/blip_handler.go | 2 + db/changes_test.go | 2 +- db/crud.go | 74 +++++-- db/crud_test.go | 2 +- db/database_test.go | 10 +- db/document.go | 58 +++++ db/utilities_hlv_testing.go | 54 ----- docs/api/components/parameters.yaml | 6 +- docs/api/components/schemas.yaml | 13 +- docs/api/paths/admin/keyspace-_bulk_docs.yaml | 4 +- docs/api/paths/admin/keyspace-_changes.yaml | 4 +- .../paths/admin/keyspace-_config-sync.yaml | 2 +- .../paths/admin/keyspace-_local-docid.yaml | 8 +- docs/api/paths/admin/keyspace-_raw-docid.yaml | 2 +- docs/api/paths/admin/keyspace-_revs_diff.yaml | 2 +- docs/api/paths/admin/keyspace-docid.yaml | 8 +- .../api/paths/public/keyspace-_bulk_docs.yaml | 4 +- docs/api/paths/public/keyspace-_changes.yaml | 2 +- .../paths/public/keyspace-_local-docid.yaml | 2 +- docs/api/paths/public/keyspace-docid.yaml | 6 +- rest/api_test.go | 42 ++++ rest/bootstrap_test.go | 3 +- rest/doc_api.go | 209 ++++++++++++++---- rest/utilities_testing.go | 29 ++- rest/utilities_testing_resttester.go | 40 +++- topologytest/sync_gateway_peer_test.go | 8 +- 28 files changed, 458 insertions(+), 179 deletions(-) diff --git a/base/util.go b/base/util.go index e3fbd889b5..47fa8502bc 100644 --- a/base/util.go +++ b/base/util.go @@ -1825,3 +1825,18 @@ func KeysPresent[K comparable, V any](m map[K]V, keys []K) []K { } return result } + +// IsRevTreeID checks if the string looks like a RevTree ID. +func IsRevTreeID(s string) bool { + // If we scan forwards past each digit until we hit `-`, we know this is a RevTree ID. + for i, r := range s { + if r >= '0' && r <= '9' { + continue + } + if r == '-' && i > 0 { + return true + } + break + } + return false +} diff --git a/base/util_test.go b/base/util_test.go index 4209c94dcf..925962a57c 100644 --- a/base/util_test.go +++ b/base/util_test.go @@ -1854,3 +1854,29 @@ func TestKeysPresent(t *testing.T) { }) } } + +func TestIsRevTreeID(t *testing.T) { + tests := []struct { + value string + expected bool + }{ + {"1-abc", true}, + {"1-abc123", true}, + {"1-abc123def456", true}, + {"1-abc123def456ghi789", true}, + {"1-abc123def456ghi789jkl012", true}, + {"123-abc123", true}, + {"1234567890-a", true}, + {"0-a", true}, + {"1", false}, + {"abc", false}, + {"a-abc", false}, + {"-abc", false}, + {"1a@bc", false}, + } + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + assert.Equalf(t, tt.expected, IsRevTreeID(tt.value), "IsRevTreeID(%v)", tt.value) + }) + } +} diff --git a/db/blip_handler.go b/db/blip_handler.go index 292f6b9d50..117b78fa14 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -852,6 +852,7 @@ func (bh *blipHandler) handleProposeChanges(rq *blip.Message) error { changeIsVector := false if versionVectorProtocol { + // TODO: CBG-4812 Use base.IsRevTreeID changeIsVector = strings.Contains(rev, "@") } if versionVectorProtocol && changeIsVector { @@ -1094,6 +1095,7 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err var incomingHLV *HybridLogicalVector // Build history/HLV var legacyRevList []string + // TODO: CBG-4812 Use base.IsRevTreeID changeIsVector := strings.Contains(rev, "@") if !bh.useHLV() || !changeIsVector { newDoc.RevID = rev diff --git a/db/changes_test.go b/db/changes_test.go index b2bc8d2d19..f294d3a7b7 100644 --- a/db/changes_test.go +++ b/db/changes_test.go @@ -418,7 +418,7 @@ func TestActiveOnlyCacheUpdate(t *testing.T) { // Tombstone 5 documents for i := 2; i <= 6; i++ { key := fmt.Sprintf("%s_%d", t.Name(), i) - _, _, err = collection.DeleteDoc(ctx, key, revId) + _, _, err = collection.DeleteDoc(ctx, key, DocVersion{RevTreeID: revId}) require.NoError(t, err, "Couldn't delete document") } diff --git a/db/crud.go b/db/crud.go index bc952f04ad..5fd73053c2 100644 --- a/db/crud.go +++ b/db/crud.go @@ -310,13 +310,13 @@ func (db *DatabaseCollectionWithUser) Get1xBody(ctx context.Context, docid strin } // Get Rev with all-or-none history based on specified 'history' flag -func (db *DatabaseCollectionWithUser) Get1xRevBody(ctx context.Context, docid, revid string, history bool, attachmentsSince []string) (Body, error) { +func (db *DatabaseCollectionWithUser) Get1xRevBody(ctx context.Context, docid, revOrCV string, history bool, attachmentsSince []string) (Body, error) { maxHistory := 0 if history { maxHistory = math.MaxInt32 } - return db.Get1xRevBodyWithHistory(ctx, docid, revid, Get1xRevBodyOptions{ + return db.Get1xRevBodyWithHistory(ctx, docid, revOrCV, Get1xRevBodyOptions{ MaxHistory: maxHistory, HistoryFrom: nil, AttachmentsSince: attachmentsSince, @@ -333,8 +333,8 @@ type Get1xRevBodyOptions struct { } // Retrieves rev with request history specified as collection of revids (historyFrom) -func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revtreeid string, opts Get1xRevBodyOptions) (Body, error) { - rev, err := db.getRev(ctx, docid, revtreeid, opts.MaxHistory, opts.HistoryFrom) +func (db *DatabaseCollectionWithUser) Get1xRevBodyWithHistory(ctx context.Context, docid, revOrCV string, opts Get1xRevBodyOptions) (Body, error) { + rev, err := db.getRev(ctx, docid, revOrCV, opts.MaxHistory, opts.HistoryFrom) if err != nil { return nil, err } @@ -1093,7 +1093,11 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod generation++ delete(body, BodyRev) - // Not extracting it yet because we need this property around to generate a rev ID + // remove CV before RevTreeID generation + matchCV, _ := body[BodyCV].(string) + delete(body, BodyCV) + + // Not extracting it yet because we need this property around to generate a RevTreeID deleted, _ := body[BodyDeleted].(bool) expiry, err := body.ExtractExpiry() @@ -1146,21 +1150,42 @@ func (db *DatabaseCollectionWithUser) Put(ctx context.Context, docid string, bod } var conflictErr error - // Make sure matchRev matches an existing leaf revision: - if matchRev == "" { - matchRev = doc.CurrentRev - if matchRev != "" { - // PUT with no parent rev given, but there is an existing current revision. - // This is OK as long as the current one is deleted. - if !doc.History[matchRev].Deleted { - conflictErr = base.HTTPErrorf(http.StatusConflict, "Document exists") - } else { - generation, _ = ParseRevID(ctx, matchRev) - generation++ + + // OCC check of matchCV against CV on the doc + if matchCV != "" { + if matchCV == doc.HLV.GetCurrentVersionString() { + // set matchRev to the current revision ID and allow existing codepaths to perform RevTree-based update. + matchRev = doc.CurrentRev + // bump generation based on retrieved RevTree ID + generation, _ = ParseRevID(ctx, matchRev) + generation++ + } else if doc.hasFlag(channels.Conflict | channels.Hidden) { + // Can't use CV as an OCC Value when a document is in conflict, or we're updating the non-winning leaf + // There's no way to get from a given old CV to a RevTreeID to perform the update correctly, since we don't maintain linear history for a given SourceID. + // Reject the request and force the user to resolve the conflict using RevTree IDs which does have linear history available. + conflictErr = base.HTTPErrorf(http.StatusBadRequest, "Cannot use CV to modify a document in conflict - resolve first with RevTree ID") + } else { + conflictErr = base.HTTPErrorf(http.StatusConflict, "Document revision conflict") + } + } + + if conflictErr == nil { + // Make sure matchRev matches an existing leaf revision: + if matchRev == "" { + matchRev = doc.CurrentRev + if matchRev != "" { + // PUT with no parent rev given, but there is an existing current revision. + // This is OK as long as the current one is deleted. + if !doc.History[matchRev].Deleted { + conflictErr = base.HTTPErrorf(http.StatusConflict, "Document exists") + } else { + generation, _ = ParseRevID(ctx, matchRev) + generation++ + } } + } else if !doc.History.isLeaf(matchRev) || db.IsIllegalConflict(ctx, doc, matchRev, deleted, false, nil) { + conflictErr = base.HTTPErrorf(http.StatusConflict, "Document revision conflict") } - } else if !doc.History.isLeaf(matchRev) || db.IsIllegalConflict(ctx, doc, matchRev, deleted, false, nil) { - conflictErr = base.HTTPErrorf(http.StatusConflict, "Document revision conflict") } // Make up a new _rev, and add it to the history: @@ -2893,7 +2918,8 @@ func (db *DatabaseCollectionWithUser) MarkPrincipalsChanged(ctx context.Context, // Creates a new document, assigning it a random doc ID. func (db *DatabaseCollectionWithUser) Post(ctx context.Context, body Body) (docid string, rev string, doc *Document, err error) { - if body[BodyRev] != nil { + // This error isn't very accurate, you just _cannot_ use POST to update an existing document - even if it does exist. We don't even bother checking for existence. + if body[BodyRev] != nil || body[BodyCV] != nil { return "", "", nil, base.HTTPErrorf(http.StatusNotFound, "No previous revision to replace") } @@ -2914,8 +2940,13 @@ func (db *DatabaseCollectionWithUser) Post(ctx context.Context, body Body) (doci } // Deletes a document, by adding a new revision whose _deleted property is true. -func (db *DatabaseCollectionWithUser) DeleteDoc(ctx context.Context, docid string, revid string) (string, *Document, error) { - body := Body{BodyDeleted: true, BodyRev: revid} +func (db *DatabaseCollectionWithUser) DeleteDoc(ctx context.Context, docid string, docVersion DocVersion) (string, *Document, error) { + body := Body{BodyDeleted: true} + if !docVersion.CV.IsEmpty() { + body[BodyCV] = docVersion.CV.String() + } else { + body[BodyRev] = docVersion.RevTreeID + } newRevID, doc, err := db.Put(ctx, docid, body) return newRevID, doc, err } @@ -3342,6 +3373,7 @@ func (db *DatabaseCollectionWithUser) CheckProposedVersion(ctx context.Context, // previousRev may be revTreeID or version var previousVersion Version previousRevFormat := "version" + // TODO: CBG-4812 Use base.IsRevTreeID if !strings.Contains(previousRev, "@") { previousRevFormat = "revTreeID" } diff --git a/db/crud_test.go b/db/crud_test.go index ea71228307..6d81191a8b 100644 --- a/db/crud_test.go +++ b/db/crud_test.go @@ -1158,7 +1158,7 @@ func TestGet1xRevAndChannels(t *testing.T) { assert.Equal(t, []interface{}{"a"}, revisions[RevisionsIds]) // Delete the document, creating tombstone revision rev3 - rev3, _, err := collection.DeleteDoc(ctx, docId, rev2) + rev3, _, err := collection.DeleteDoc(ctx, docId, DocVersion{RevTreeID: rev2}) require.NoError(t, err) bodyBytes, removed, err = collection.get1xRevFromDoc(ctx, doc2, rev3, true) assert.False(t, removed) diff --git a/db/database_test.go b/db/database_test.go index 59fdbf0473..615275af6f 100644 --- a/db/database_test.go +++ b/db/database_test.go @@ -731,7 +731,7 @@ func TestGetDeleted(t *testing.T) { rev1id, _, err := collection.Put(ctx, "doc1", body) assert.NoError(t, err, "Put") - rev2id, _, err := collection.DeleteDoc(ctx, "doc1", rev1id) + rev2id, _, err := collection.DeleteDoc(ctx, "doc1", DocVersion{RevTreeID: rev1id}) assert.NoError(t, err, "DeleteDoc") // Get the deleted doc with its history; equivalent to GET with ?revs=true @@ -1395,7 +1395,7 @@ func TestAllDocsOnly(t *testing.T) { } // Now delete one document and try again: - _, _, err = collection.DeleteDoc(ctx, ids[23].DocID, ids[23].RevID) + _, _, err = collection.DeleteDoc(ctx, ids[23].DocID, DocVersion{RevTreeID: ids[23].RevID}) assert.NoError(t, err, "Couldn't delete doc 23") alldocs, err = allDocIDs(ctx, collection.DatabaseCollection) @@ -1726,7 +1726,7 @@ func TestConflicts(t *testing.T) { ) // Delete 2-b; verify this makes 2-a current: - rev3, _, err := collection.DeleteDoc(ctx, "doc", "2-b") + rev3, _, err := collection.DeleteDoc(ctx, "doc", DocVersion{RevTreeID: "2-b"}) assert.NoError(t, err, "delete 2-b") rawBody, _, _ = collection.dataStore.GetRaw("doc") @@ -3369,7 +3369,7 @@ func TestTombstoneCompactionStopWithManager(t *testing.T) { docID := fmt.Sprintf("doc%d", i) rev, _, err := collection.Put(ctx, docID, Body{}) assert.NoError(t, err) - _, _, err = collection.DeleteDoc(ctx, docID, rev) + _, _, err = collection.DeleteDoc(ctx, docID, DocVersion{RevTreeID: rev}) assert.NoError(t, err) } @@ -3780,7 +3780,7 @@ func TestImportCompactPanic(t *testing.T) { // Create a document, then delete it, to create a tombstone rev, doc, err := collection.Put(ctx, "test", Body{}) require.NoError(t, err) - _, _, err = collection.DeleteDoc(ctx, doc.ID, rev) + _, _, err = collection.DeleteDoc(ctx, doc.ID, DocVersion{RevTreeID: rev}) require.NoError(t, err) require.NoError(t, collection.WaitForPendingChanges(ctx)) diff --git a/db/document.go b/db/document.go index f4e60f8ac2..694149e934 100644 --- a/db/document.go +++ b/db/document.go @@ -19,6 +19,7 @@ import ( "math" "net/http" "strconv" + "strings" "time" sgbucket "github.com/couchbase/sg-bucket" @@ -1484,3 +1485,60 @@ func (doc *Document) ExtractDocVersion() DocVersion { CV: *doc.HLV.ExtractCurrentVersionFromHLV(), } } + +// DocVersion represents a specific version of a document in an revID/HLV agnostic manner. +type DocVersion struct { + RevTreeID string + CV Version +} + +// String implements fmt.Stringer +func (v DocVersion) String() string { + return fmt.Sprintf("RevTreeID:%s,CV:%#v", v.RevTreeID, v.CV) +} + +// GoString implements fmt.GoStringer +func (v DocVersion) GoString() string { + return fmt.Sprintf("DocVersion{RevTreeID:%s,CV:%#v}", v.RevTreeID, v.CV) +} + +// DocVersionRevTreeEqual compares two DocVersions based on their RevTreeID. +func (v DocVersion) DocVersionRevTreeEqual(o DocVersion) bool { + if v.RevTreeID != o.RevTreeID { + return false + } + return true +} + +// GetRev returns the revision ID for the DocVersion. +func (v DocVersion) GetRev(useHLV bool) string { + if useHLV { + if v.CV.SourceID == "" { + return "" + } + return v.CV.String() + } else { + return v.RevTreeID + } +} + +// RevIDGeneration returns the Rev ID generation for the current version +func (v *DocVersion) RevIDGeneration() int { + if v == nil { + return 0 + } + gen, err := strconv.ParseInt(strings.Split(v.RevTreeID, "-")[0], 10, 64) + if err != nil { + base.AssertfCtx(context.TODO(), "Error parsing generation from rev ID %q: %v", v.RevTreeID, err) + return 0 + } + return int(gen) +} + +// RevIDDigest returns the Rev ID digest for the current version +func (v *DocVersion) RevIDDigest() string { + if v == nil { + return "" + } + return strings.Split(v.RevTreeID, "-")[1] +} diff --git a/db/utilities_hlv_testing.go b/db/utilities_hlv_testing.go index f90b82d87d..5397a4bb90 100644 --- a/db/utilities_hlv_testing.go +++ b/db/utilities_hlv_testing.go @@ -12,7 +12,6 @@ package db import ( "context" - "fmt" "strconv" "strings" "testing" @@ -22,59 +21,6 @@ import ( "github.com/stretchr/testify/require" ) -// DocVersion represents a specific version of a document in an revID/HLV agnostic manner. -type DocVersion struct { - RevTreeID string - CV Version -} - -func (v DocVersion) String() string { - return fmt.Sprintf("RevTreeID:%s,CV:%#v", v.RevTreeID, v.CV) -} - -func (v DocVersion) GoString() string { - return fmt.Sprintf("DocVersion{RevTreeID:%s,CV:%#v}", v.RevTreeID, v.CV) -} - -func (v DocVersion) DocVersionRevTreeEqual(o DocVersion) bool { - if v.RevTreeID != o.RevTreeID { - return false - } - return true -} - -func (v DocVersion) GetRev(useHLV bool) string { - if useHLV { - if v.CV.SourceID == "" { - return "" - } - return v.CV.String() - } else { - return v.RevTreeID - } -} - -// RevIDGeneration returns the Rev ID generation for the current version -func (v *DocVersion) RevIDGeneration() int { - if v == nil { - return 0 - } - gen, err := strconv.ParseInt(strings.Split(v.RevTreeID, "-")[0], 10, 64) - if err != nil { - base.AssertfCtx(context.TODO(), "Error parsing generation from rev ID %q: %v", v.RevTreeID, err) - return 0 - } - return int(gen) -} - -// RevIDDigest returns the Rev ID digest for the current version -func (v *DocVersion) RevIDDigest() string { - if v == nil { - return "" - } - return strings.Split(v.RevTreeID, "-")[1] -} - // HLVAgent performs HLV updates directly (not via SG) for simulating/testing interaction with non-SG HLV agents type HLVAgent struct { t *testing.T diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index bc1431cf8b..24ab42881c 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -13,13 +13,13 @@ DB-config-If-Match: schema: type: string description: 'If set to a configuration''s Etag value, enables optimistic concurrency control for the request. Returns HTTP 412 if another update happened underneath this one.' -If-Match: +Document-If-Match: name: If-Match in: header required: false schema: type: string - description: The revision ID to target. + description: An optimistic concurrency control (OCC) value used to prevent conflicts. Use the value returned in the ETag response header of the GET request for the resource being updated, or the latest known Revision Tree ID or Current Version of the document. Include-channels: name: channels in: query @@ -393,7 +393,7 @@ show_cv: required: false schema: type: boolean - description: Output the current version of the version vector in the response as property `_cv`. + description: Output the Current Version in the response as property `_cv`. startkey: name: startkey in: query diff --git a/docs/api/components/schemas.yaml b/docs/api/components/schemas.yaml index 194904f03d..39e968a9b6 100644 --- a/docs/api/components/schemas.yaml +++ b/docs/api/components/schemas.yaml @@ -494,7 +494,7 @@ Document: description: Prefix number for the latest revision. type: number ids: - description: 'Array of valid revision IDs, in reverse order (latest first).' + description: 'Array of valid Revision Tree IDs, in reverse order (latest first).' type: array items: type: string @@ -522,7 +522,10 @@ New-revision: description: Whether the request completed successfully. type: boolean rev: - description: The revision of the document. + description: The Revision Tree ID of the document. + type: string + cv: + description: The CV of the document. type: string required: - id @@ -558,9 +561,9 @@ Changes-feed: - type: object properties: cv: - description: The HLV Current Version associated with the change. This value requires the `version_type` preference set to `cv` and this document revision being written through SG 4.0+ + description: The Current Version associated with the change. This value requires the `version_type` preference set to `cv` and this document revision being written through SG 4.0+ type: string - title: HLV Current Version + title: Current Version uniqueItems: true uniqueItems: true last_seq: @@ -708,7 +711,7 @@ Replication: default: |- In priority order, this will cause - Deletes to always win (the delete with the longest revision history wins if both revisions are deletes) - - The revision with the longest revision history to win. This means the the revision with the most changes and therefore the highest revision ID will win. + - The revision with the longest revision history to win. This means the the revision with the most changes and therefore the highest Revision Tree ID will win. localWins: This will result in local revisions always being the winner in any conflict. remoteWins: This will result in remote revisions always being the winner in any conflict. custom: This will result in conflicts going through your own custom conflict resolver. You must provide this logic as a Javascript function in the `custom_conflict_resolver` parameter. diff --git a/docs/api/paths/admin/keyspace-_bulk_docs.yaml b/docs/api/paths/admin/keyspace-_bulk_docs.yaml index f209b6c039..dfa9834a3c 100644 --- a/docs/api/paths/admin/keyspace-_bulk_docs.yaml +++ b/docs/api/paths/admin/keyspace-_bulk_docs.yaml @@ -14,9 +14,9 @@ post: To create a new document, simply add the body in an object under `docs`. A doc ID will be generated by Sync Gateway unless `_id` is specified. - To update an existing document, provide the document ID (`_id`) and revision ID (`_rev`) as well as the new body values. + To update an existing document, provide the document ID (`_id`) and Revision Tree ID (`_rev`) as well as the new body values. - To delete an existing document, provide the document ID (`_id`), revision ID (`_rev`), and set the deletion flag (`_deleted`) to true. + To delete an existing document, provide the document ID (`_id`), Revision Tree ID (`_rev`), and set the deletion flag (`_deleted`) to true. Required Sync Gateway RBAC roles: diff --git a/docs/api/paths/admin/keyspace-_changes.yaml b/docs/api/paths/admin/keyspace-_changes.yaml index 9f328fb4ac..c19a89889f 100644 --- a/docs/api/paths/admin/keyspace-_changes.yaml +++ b/docs/api/paths/admin/keyspace-_changes.yaml @@ -113,7 +113,7 @@ get: - cv x-enumDescriptions: rev: 'Revision Tree IDs. For example: 1-293a80ce8f4874724732f27d35b3959a13cd96e0' - cv: 'HLV Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' + cv: 'Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' responses: '200': $ref: ../../components/responses.yaml#/changes-feed @@ -187,7 +187,7 @@ post: - cv x-enumDescriptions: rev: 'Revision Tree IDs. For example: 1-293a80ce8f4874724732f27d35b3959a13cd96e0' - cv: 'HLV Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' + cv: 'Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' responses: '200': $ref: ../../components/responses.yaml#/changes-feed diff --git a/docs/api/paths/admin/keyspace-_config-sync.yaml b/docs/api/paths/admin/keyspace-_config-sync.yaml index bb67438873..f3e8de8b79 100644 --- a/docs/api/paths/admin/keyspace-_config-sync.yaml +++ b/docs/api/paths/admin/keyspace-_config-sync.yaml @@ -87,7 +87,7 @@ delete: * Sync Gateway Architect parameters: - - $ref: ../../components/parameters.yaml#/If-Match + - $ref: ../../components/parameters.yaml#/Document-If-Match responses: '200': description: Successfully reset the sync function diff --git a/docs/api/paths/admin/keyspace-_local-docid.yaml b/docs/api/paths/admin/keyspace-_local-docid.yaml index 5866d75784..604e56eee8 100644 --- a/docs/api/paths/admin/keyspace-_local-docid.yaml +++ b/docs/api/paths/admin/keyspace-_local-docid.yaml @@ -37,7 +37,7 @@ get: put: summary: Upsert a local document description: |- - This request creates or updates a local document. Updating a local document requires that the revision ID be put in the body under `_rev`. + This request creates or updates a local document. Updating a local document requires that the Revision Tree ID be put in the body under `_rev`. Local document IDs are given a `_local/` prefix. Local documents are not replicated or indexed, don't support attachments, and don't save revision histories. In practice they are almost only used by the client's replicator, as a place to store replication checkpoint data. @@ -67,7 +67,7 @@ put: '404': $ref: ../../components/responses.yaml#/Not-found '409': - description: A revision ID conflict would result from updating this document revision. + description: A conflict would result from updating this document revision. tags: - Document operationId: put_keyspace-_local-docid @@ -84,7 +84,7 @@ delete: parameters: - name: rev in: query - description: The revision ID of the revision to delete. + description: The Revision Tree ID of the revision to delete. required: true schema: type: string @@ -96,7 +96,7 @@ delete: '404': $ref: ../../components/responses.yaml#/Not-found '409': - description: A revision ID conflict would result from deleting this document revision. + description: A conflict would result from deleting this document revision. tags: - Document operationId: delete_keyspace-_local-docid diff --git a/docs/api/paths/admin/keyspace-_raw-docid.yaml b/docs/api/paths/admin/keyspace-_raw-docid.yaml index 0e90774f31..e45d0ac2fb 100644 --- a/docs/api/paths/admin/keyspace-_raw-docid.yaml +++ b/docs/api/paths/admin/keyspace-_raw-docid.yaml @@ -62,7 +62,7 @@ get: nullable: true additionalProperties: true _vv: - description: Version vector for the document. + description: Version Vector for the document. type: object nullable: true additionalProperties: true diff --git a/docs/api/paths/admin/keyspace-_revs_diff.yaml b/docs/api/paths/admin/keyspace-_revs_diff.yaml index cd40e1c129..0b7418a4e7 100644 --- a/docs/api/paths/admin/keyspace-_revs_diff.yaml +++ b/docs/api/paths/admin/keyspace-_revs_diff.yaml @@ -10,7 +10,7 @@ parameters: post: summary: Compare revisions to what is in the database description: |- - Takes a set of document IDs, each with a set of revision IDs. For each document, an array of unknown revisions are returned with an array of known revisions that may be recent ancestors. + Takes a set of document IDs, each with a set of Revision Tree IDs. For each document, an array of unknown revisions are returned with an array of known revisions that may be recent ancestors. Required Sync Gateway RBAC roles: diff --git a/docs/api/paths/admin/keyspace-docid.yaml b/docs/api/paths/admin/keyspace-docid.yaml index 6c0743c461..90293933a7 100644 --- a/docs/api/paths/admin/keyspace-docid.yaml +++ b/docs/api/paths/admin/keyspace-docid.yaml @@ -43,7 +43,7 @@ get: description: The ID of the document. type: string _rev: - description: The revision ID of the document. + description: The Revision Tree ID of the document. type: string additionalProperties: true example: @@ -69,7 +69,7 @@ get: put: summary: Upsert a document description: |- - This will upsert a document meaning if it does not exist, then it will be created. Otherwise a new revision will be made for the existing document. A revision ID must be provided if targetting an existing document. + This will upsert a document, meaning if it does not exist then it will be created. Otherwise a new revision will be made for the existing document. A previous known version must be provided if targeting an existing document to prevent conflicts. A document ID must be specified for this endpoint. To let Sync Gateway generate the ID, use the `POST /{db}/` endpoint. @@ -85,7 +85,7 @@ put: - $ref: ../../components/parameters.yaml#/replicator2 - $ref: ../../components/parameters.yaml#/new_edits - $ref: ../../components/parameters.yaml#/rev - - $ref: ../../components/parameters.yaml#/If-Match + - $ref: ../../components/parameters.yaml#/Document-If-Match requestBody: content: application/json: @@ -126,7 +126,7 @@ delete: * Sync Gateway Application parameters: - $ref: ../../components/parameters.yaml#/rev - - $ref: ../../components/parameters.yaml#/If-Match + - $ref: ../../components/parameters.yaml#/Document-If-Match responses: '200': $ref: ../../components/responses.yaml#/New-revision diff --git a/docs/api/paths/public/keyspace-_bulk_docs.yaml b/docs/api/paths/public/keyspace-_bulk_docs.yaml index 32c836655d..c0f7d1df4f 100644 --- a/docs/api/paths/public/keyspace-_bulk_docs.yaml +++ b/docs/api/paths/public/keyspace-_bulk_docs.yaml @@ -14,9 +14,9 @@ post: To create a new document, simply add the body in an object under `docs`. A doc ID will be generated by Sync Gateway unless `_id` is specified. - To update an existing document, provide the document ID (`_id`) and revision ID (`_rev`) as well as the new body values. + To update an existing document, provide the document ID (`_id`) and Revision Tree ID (`_rev`) as well as the new body values. - To delete an existing document, provide the document ID (`_id`), revision ID (`_rev`), and set the deletion flag (`_deleted`) to true. + To delete an existing document, provide the document ID (`_id`), Revision Tree ID (`_rev`), and set the deletion flag (`_deleted`) to true. requestBody: content: application/json: diff --git a/docs/api/paths/public/keyspace-_changes.yaml b/docs/api/paths/public/keyspace-_changes.yaml index 019f52f7ba..6a1a348f9d 100644 --- a/docs/api/paths/public/keyspace-_changes.yaml +++ b/docs/api/paths/public/keyspace-_changes.yaml @@ -101,7 +101,7 @@ get: - cv x-enumDescriptions: rev: 'Revision Tree IDs. For example: 1-293a80ce8f4874724732f27d35b3959a13cd96e0' - cv: 'HLV Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' + cv: 'Current Version. For example: 1854e4e557cc0000@zTWkmBiYZgNQo7BHVZrB/Q' responses: '200': $ref: ../../components/responses.yaml#/changes-feed diff --git a/docs/api/paths/public/keyspace-_local-docid.yaml b/docs/api/paths/public/keyspace-_local-docid.yaml index 5917b03ac6..7cb1431458 100644 --- a/docs/api/paths/public/keyspace-_local-docid.yaml +++ b/docs/api/paths/public/keyspace-_local-docid.yaml @@ -57,7 +57,7 @@ put: '404': $ref: ../../components/responses.yaml#/Not-found '409': - description: A revision ID conflict would result from updating this document revision. + description: A conflict would result from updating this document revision. tags: - Document operationId: put_keyspace-_local-docid diff --git a/docs/api/paths/public/keyspace-docid.yaml b/docs/api/paths/public/keyspace-docid.yaml index fd5e4945ec..5e408fe38c 100644 --- a/docs/api/paths/public/keyspace-docid.yaml +++ b/docs/api/paths/public/keyspace-docid.yaml @@ -40,7 +40,7 @@ get: description: The revision ID of the document. type: string _cv: - description: The current version of version vector of the document. + description: The Current Version of the document. type: string additionalProperties: true example: @@ -78,7 +78,7 @@ put: - $ref: ../../components/parameters.yaml#/replicator2 - $ref: ../../components/parameters.yaml#/new_edits - $ref: ../../components/parameters.yaml#/rev - - $ref: ../../components/parameters.yaml#/If-Match + - $ref: ../../components/parameters.yaml#/Document-If-Match requestBody: content: application/json: @@ -115,7 +115,7 @@ delete: A revision ID either in the header or on the query parameters is required. parameters: - $ref: ../../components/parameters.yaml#/rev - - $ref: ../../components/parameters.yaml#/If-Match + - $ref: ../../components/parameters.yaml#/Document-If-Match responses: '200': $ref: ../../components/responses.yaml#/New-revision diff --git a/rest/api_test.go b/rest/api_test.go index c53a8986fc..73ce33da72 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -3244,3 +3244,45 @@ func TestHLVUpdateOnRevReplicatorPut(t *testing.T) { assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.Version) assert.Equal(t, base.HexCasToUint64(syncData.Cas), syncData.HLV.CurrentVersionCAS) } + +func TestDocCRUDWithCV(t *testing.T) { + rt := NewRestTester(t, nil) + defer rt.Close() + + const docID = "doc1" + createVersion := rt.PutDoc(docID, `{"create":true}`) + + getDocVersion, _ := rt.GetDoc(docID) + require.Equal(t, createVersion, getDocVersion) + + updateVersion := rt.UpdateDoc(docID, createVersion, `{"update":true}`) + require.NotEqual(t, createVersion, updateVersion) + assert.Greaterf(t, updateVersion.CV.Value, createVersion.CV.Value, "Expected CV Value to be bumped on update") + assert.Greaterf(t, updateVersion.RevIDGeneration(), createVersion.RevIDGeneration(), "Expected revision generation to be bumped on update") + + getDocVersion, _ = rt.GetDoc(docID) + require.Equal(t, updateVersion, getDocVersion) + + // fetch by CV (using the first create version to test cache retrieval) + resp := rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s?rev=%s", docID, createVersion.CV.String()), "") + RequireStatus(t, resp, http.StatusOK) + resp.DumpBody() + assert.NotContains(t, resp.BodyString(), `"update":true`) + assert.Contains(t, resp.BodyString(), `"create":true`) + assert.Contains(t, resp.BodyString(), `"_cv":"`+createVersion.CV.String()+`"`) + assert.Contains(t, resp.BodyString(), `"_rev":"`+createVersion.RevTreeID+`"`) + + // fetch by CV - updated version + resp = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s?rev=%s", docID, updateVersion.CV.String()), "") + RequireStatus(t, resp, http.StatusOK) + resp.DumpBody() + assert.NotContains(t, resp.BodyString(), `"create":true`) + assert.Contains(t, resp.BodyString(), `"update":true`) + assert.Contains(t, resp.BodyString(), `"_cv":"`+updateVersion.CV.String()+`"`) + assert.Contains(t, resp.BodyString(), `"_rev":"`+updateVersion.RevTreeID+`"`) + + deleteVersion := rt.DeleteDoc(docID, updateVersion) + require.NotEqual(t, updateVersion, deleteVersion) + assert.Greaterf(t, deleteVersion.CV.Value, updateVersion.CV.Value, "Expected CV Value to be bumped on delete") + assert.Greaterf(t, deleteVersion.RevIDGeneration(), updateVersion.RevIDGeneration(), "Expected revision generation to be bumped on delete") +} diff --git a/rest/bootstrap_test.go b/rest/bootstrap_test.go index 189b8a94cd..1f5fc40936 100644 --- a/rest/bootstrap_test.go +++ b/rest/bootstrap_test.go @@ -75,7 +75,8 @@ func TestBootstrapRESTAPISetup(t *testing.T) { resp = BootstrapAdminRequest(t, sc, http.MethodGet, "/db1/doc1", ``) resp.RequireStatus(http.StatusNotFound) resp = BootstrapAdminRequest(t, sc, http.MethodPut, "/db1/doc1", `{"foo":"bar"}`) - resp.RequireResponse(http.StatusCreated, `{"id":"doc1","ok":true,"rev":"1-cd809becc169215072fd567eebd8b8de"}`) + resp.RequireStatus(http.StatusCreated) + require.Contains(t, resp.Body, `"id":"doc1","ok":true`) resp = BootstrapAdminRequest(t, sc, http.MethodGet, "/db1/doc1", ``) resp.RequireStatus(http.StatusOK) assert.Contains(t, resp.Body, `"foo":"bar"`) diff --git a/rest/doc_api.go b/rest/doc_api.go index 0204a0fd81..6fc1986a49 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -331,6 +331,58 @@ func (h *handler) handleGetAttachment() error { } +// getOCCValue retrieves the optimistic concurrency control value for a document (Revision ID or CV) from several possible sources: +// - Query parameter "rev" +// - If-Match header +// - Body field "_rev" or "_cv" (if the body is provided in the request) +// It also validates that the provided OCC value exactly matches the corresponding body field if it exists in multiple places. +func (h *handler) getOCCValue(optionalBody db.Body) (occValue string, occValueType occVersionType, err error) { + // skipBodyMatchValidation can skip the validation of the occValue inside the body, since in some cases we're pulling that value out of the body anyway. + var skipBodyMatchValidation bool + + // occValue is the optimistic concurrency control value, which can be either the current rev ID or CV - used to prevent lost updates. + // we grab occValue from either query param, Etag header, or request body (in that order) + if revQuery := h.getQuery("rev"); revQuery != "" { + occValue = revQuery + occValueType = guessOCCVersionTypeFromValue(occValue) + } else if ifMatch, err := h.getEtag("If-Match"); err != nil { + return "", 0, err + } else if ifMatch != "" { + occValue = ifMatch + occValueType = guessOCCVersionTypeFromValue(occValue) + } else if bodyCV, ok := optionalBody[db.BodyCV]; ok { + if bodyCVStr, ok := bodyCV.(string); ok { + occValue = bodyCVStr + occValueType = VersionTypeCV + skipBodyMatchValidation = true + } + } else if bodyRev, ok := optionalBody[db.BodyRev]; ok { + if bodyRevStr, ok := bodyRev.(string); ok { + occValue = bodyRevStr + occValueType = VersionTypeRevTreeID + skipBodyMatchValidation = true + } + } + + // ensure the value provided matches exactly the one that may also be supplied in the body + if !skipBodyMatchValidation { + switch occValueType { + case VersionTypeRevTreeID: + if optionalBody[db.BodyRev] != nil && occValue != optionalBody[db.BodyRev] { + return "", 0, base.HTTPErrorf(http.StatusBadRequest, "Revision IDs provided do not match") + } + case VersionTypeCV: + if optionalBody[db.BodyCV] != nil && occValue != optionalBody[db.BodyCV] { + return "", 0, base.HTTPErrorf(http.StatusBadRequest, "CVs provided do not match") + } + default: + return "", 0, base.HTTPErrorf(http.StatusBadRequest, "Unknown version type provided: %q", occValue) + } + } + + return occValue, occValueType, nil +} + // HTTP handler for a PUT of an attachment func (h *handler) handlePutAttachment() error { @@ -344,30 +396,27 @@ func (h *handler) handlePutAttachment() error { if attachmentContentType == "" { attachmentContentType = "application/octet-stream" } - revid := h.getQuery("rev") - if revid == "" { - var err error - revid, err = h.getEtag("If-Match") - if err != nil { - return err - } + + occValue, occValueType, err := h.getOCCValue(nil) + if err != nil { + return err } + attachmentData, err := h.readBody() if err != nil { return err } - body, err := h.collection.Get1xRevBody(h.ctx(), docid, revid, false, nil) + body, err := h.collection.Get1xRevBody(h.ctx(), docid, occValue, false, nil) if err != nil { if base.IsDocNotFoundError(err) { - // couchdb creates empty body on attachment PUT - // for non-existent doc id - body = db.Body{db.BodyRev: revid} + // couchdb creates empty body on attachment PUT for non-existent doc id + body = db.Body{bodyKeyForOCCVersionType(occValueType): occValue} } else if err != nil { return err } } else if body != nil { - if revid == "" { + if occValue == "" { // If a revid is not specified and an active revision was found, // return a conflict now, rather than letting db.Put do it further down... return base.HTTPErrorf(http.StatusConflict, "Cannot modify attachments without a specific rev ID") @@ -389,13 +438,13 @@ func (h *handler) handlePutAttachment() error { attachments[attachmentName] = attachment body[db.BodyAttachments] = attachments - newRev, _, err := h.collection.Put(h.ctx(), docid, body) + newRev, doc, err := h.collection.Put(h.ctx(), docid, body) if err != nil { return err } h.setEtag(newRev) - h.writeRawJSONStatus(http.StatusCreated, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`"}`)) + h.writeRawJSONStatus(http.StatusCreated, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`","cv":"`+doc.HLV.GetCurrentVersionString()+`"}`)) return nil } @@ -406,16 +455,13 @@ func (h *handler) handleDeleteAttachment() error { docid := h.PathVar("docid") attachmentName := h.PathVar("attach") - revid := h.getQuery("rev") - if revid == "" { - var err error - revid, err = h.getEtag("If-Match") - if err != nil { - return err - } + + occValue, _, err := h.getOCCValue(nil) + if err != nil { + return err } - body, err := h.collection.Get1xRevBody(h.ctx(), docid, revid, false, nil) + body, err := h.collection.Get1xRevBody(h.ctx(), docid, occValue, false, nil) if err != nil { if base.IsDocNotFoundError(err) { // Check here if error is relating to incorrect revid, if so return 409 code else return 404 code @@ -428,7 +474,7 @@ func (h *handler) handleDeleteAttachment() error { return err } } else if body != nil { - if revid == "" { + if occValue == "" { // If a revid is not specified and an active revision was found, // return a conflict now, rather than letting db.Put do it further down... return base.HTTPErrorf(http.StatusConflict, "Cannot modify attachments without a specific rev ID") @@ -444,16 +490,44 @@ func (h *handler) handleDeleteAttachment() error { delete(attachments, attachmentName) body[db.BodyAttachments] = attachments - newRev, _, err := h.collection.Put(h.ctx(), docid, body) + newRev, doc, err := h.collection.Put(h.ctx(), docid, body) if err != nil { return err } h.setEtag(newRev) - h.writeRawJSONStatus(http.StatusOK, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`"}`)) + h.writeRawJSONStatus(http.StatusOK, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`","cv":"`+doc.HLV.GetCurrentVersionString()+`"}`)) return nil } +// occVersionType is a type used to represent the type of document version in optimistic concurrency control (OCC). Can be Revision Tree ID or a CV. +type occVersionType uint8 + +const ( + VersionTypeRevTreeID occVersionType = iota // Revision Tree ID (RevTreeID / RevID) + VersionTypeCV // HLV/Version Vector CV +) + +// guessOCCVersionTypeFromValue returns the type of document version based on the string value. Either a RevTree ID or a CV. +func guessOCCVersionTypeFromValue(s string) occVersionType { + if base.IsRevTreeID(s) { + return VersionTypeRevTreeID + } + // anything else we'll assume is a CV + // we _could_ check for a well-formatted CV, but given the usage for OCC, we only need an exact string match + return VersionTypeCV +} + +func bodyKeyForOCCVersionType(versionType occVersionType) string { + switch versionType { + case VersionTypeRevTreeID: + return db.BodyRev + case VersionTypeCV: + return db.BodyCV + } + return "" +} + // HTTP handler for a PUT of a document func (h *handler) handlePutDoc() error { @@ -500,23 +574,24 @@ func (h *handler) handlePutDoc() error { if h.getQuery("new_edits") != "false" { // Regular PUT: - bodyRev := body[db.BodyRev] - if oldRev := h.getQuery("rev"); oldRev != "" { - body[db.BodyRev] = oldRev - } else if ifMatch, _ := h.getEtag("If-Match"); ifMatch != "" { - body[db.BodyRev] = ifMatch - } - if bodyRev != nil && bodyRev != body[db.BodyRev] { - return base.HTTPErrorf(http.StatusBadRequest, "Revision IDs provided do not match") + + // occValue is the optimistic concurrency control value, which can be either the current rev ID or CV - used to prevent lost updates. + // we grab occValue from either query param, Etag header, or request body (in that order) + occValue, occValueType, err := h.getOCCValue(body) + if err != nil { + return err } + // set OCC version body value for Put + body[bodyKeyForOCCVersionType(occValueType)] = occValue + newRev, doc, err = h.collection.Put(h.ctx(), docid, body) if err != nil { return err } h.setEtag(newRev) } else { - // Replicator-style PUT with new_edits=false: + // Replicator-style PUT (allow new revisions/conflicts to be pushed) with new_edits=false: revisions := db.ParseRevisions(h.ctx(), body) if revisions == nil { return base.HTTPErrorf(http.StatusBadRequest, "Bad _revisions") @@ -527,13 +602,25 @@ func (h *handler) handlePutDoc() error { } } + respBody := []byte(`{"id":` + base.ConvertToJSONString(docid) + `,"ok":true,"rev":"` + newRev + `"}`) + + // if this was an idempotent update that didn't need to write an update, returned doc is nil, and we can't pull CV out of what we have available here. + // Accept this as an edge case that we don't need to support for CVs (this can only happen if the request is using new_edits=false, which is not the case for most clients, and definitely none that use CV) + if doc != nil { + respBody, err = base.InjectJSONProperties(respBody, base.KVPair{Key: "cv", Val: doc.HLV.GetCurrentVersionString()}) + if err != nil { + base.AssertfCtx(h.ctx(), "couldn't inject CV into response body: %v", err) + // safe to continue + } + } + if doc != nil && roundTrip { if err := h.collection.WaitForSequenceNotSkipped(h.ctx(), doc.Sequence); err != nil { return err } } - h.writeRawJSONStatus(http.StatusCreated, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`"}`)) + h.writeRawJSONStatus(http.StatusCreated, respBody) return nil } @@ -648,10 +735,24 @@ func (h *handler) handlePostDoc() error { h.setHeader("Location", docid) h.setEtag(newRev) - h.writeRawJSON([]byte(`{"id":"` + docid + `","ok":true,"rev":"` + newRev + `"}`)) + + h.writeRawJSONStatus(http.StatusOK, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`","cv":"`+doc.HLV.GetCurrentVersionString()+`"}`)) return nil } +func docVerisonFromOCCValue(occValue string, occValueType occVersionType) (docVersion db.DocVersion, err error) { + switch occValueType { + case VersionTypeRevTreeID: + docVersion.RevTreeID = occValue + case VersionTypeCV: + docVersion.CV, err = db.ParseVersion(occValue) + if err != nil { + return DocVersion{}, base.HTTPErrorf(http.StatusBadRequest, "Invalid CV: %v", err) + } + } + return docVersion, nil +} + // HTTP handler for a DELETE of a document func (h *handler) handleDeleteDoc() error { if h.db.DatabaseContext.Options.UnsupportedOptions != nil && h.db.DatabaseContext.Options.UnsupportedOptions.RejectWritesWithSkippedSequences { @@ -663,19 +764,35 @@ func (h *handler) handleDeleteDoc() error { } docid := h.PathVar("docid") - revid := h.getQuery("rev") - if revid == "" { - var err error - revid, err = h.getEtag("If-Match") + + // occValue is the optimistic concurrency control value, which can be either the current rev ID or CV - used to prevent lost updates. + // we grab occValue from either query param, Etag header, or request body (in that order) + occValue, occValueType, err := h.getOCCValue(nil) + if err != nil { + return fmt.Errorf("couldn't get OCC value from request: %w", err) + } + + docVersion, err := docVerisonFromOCCValue(occValue, occValueType) + if err != nil { + return fmt.Errorf("couldn't build document version from OCC value: %w", err) + } + + newRev, doc, err := h.collection.DeleteDoc(h.ctx(), docid, docVersion) + if err != nil { + return err + } + + respBody := []byte(`{"id":` + base.ConvertToJSONString(docid) + `,"ok":true,"rev":"` + newRev + `"}`) + if doc != nil { + respBody, err = base.InjectJSONProperties(respBody, base.KVPair{Key: "cv", Val: doc.HLV.GetCurrentVersionString()}) if err != nil { - return err + base.AssertfCtx(h.ctx(), "Failed to inject JSON properties: %v", err) + // safe to continue - we'll just have no deleted in response } } - newRev, _, err := h.collection.DeleteDoc(h.ctx(), docid, revid) - if err == nil { - h.writeRawJSONStatus(http.StatusOK, []byte(`{"id":`+base.ConvertToJSONString(docid)+`,"ok":true,"rev":"`+newRev+`"}`)) - } - return err + + h.writeRawJSONStatus(http.StatusOK, respBody) + return nil } // ////// LOCAL DOCS: diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index bc20086393..00f3e5c4d2 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -1138,6 +1138,17 @@ func (rt *RestTester) GetDocumentSequence(key string) (sequence uint64) { return rawResponse.Xattrs.Sync.Sequence } +// GetRawDoc returns the raw document response for a given document ID using the _raw endpoint. +func (rt *RestTester) GetRawDoc(key string) RawDocResponse { + response := rt.SendAdminRequest("GET", fmt.Sprintf("/{{.keyspace}}/_raw/%s", key), "") + require.Equal(rt.TB(), http.StatusOK, response.Code, "Error getting raw document %s", response.Body.String()) + response.DumpBody() + var rawResponse RawDocResponse + require.NoError(rt.TB(), base.JSONUnmarshal(response.BodyBytes(), &rawResponse)) + log.Printf("rawResponse: %#v", rawResponse) + return rawResponse +} + // ReplacePerBucketCredentials replaces buckets defined on StartupConfig.BucketCredentials then recreates the couchbase // cluster to pick up the changes func (rt *RestTester) ReplacePerBucketCredentials(config base.PerBucketCredentialsConfig) { @@ -2395,16 +2406,22 @@ func NewDocVersionFromFakeRev(fakeRev string) DocVersion { return DocVersion{RevTreeID: fakeRev} } -// DocVersionFromPutResponse returns a DocRevisionID from the given response to PUT /{, or fails the given test if a rev ID was not found. +// DocVersionFromPutResponse returns a DocVersion from the given response, or fails the given test if a version was not returned. func DocVersionFromPutResponse(t testing.TB, response *TestResponse) DocVersion { var r struct { DocID *string `json:"id"` RevID *string `json:"rev"` - } - require.NoError(t, json.Unmarshal(response.BodyBytes(), &r)) - require.NotNil(t, r.RevID, "expecting non-nil rev ID from response: %s", string(response.BodyBytes())) - require.NotEqual(t, "", *r.RevID, "expecting non-empty rev ID from response: %s", string(response.BodyBytes())) - return DocVersion{RevTreeID: *r.RevID} + CV *string `json:"cv"` + } + respBody := response.BodyString() + require.NoErrorf(t, json.Unmarshal(response.BodyBytes(), &r), "error unmarshalling response body: %s", respBody) + require.NotNilf(t, r.RevID, "expecting non-nil 'rev' from response: %s", respBody) + require.NotNilf(t, r.CV, "expecting non-nil 'cv' from response: %s", respBody) + require.NotEqualf(t, "", *r.RevID, "expecting non-empty 'rev' from response: %s", respBody) + require.NotEqualf(t, "", *r.CV, "expecting non-empty 'cv' from response: %s", respBody) + cv, err := db.ParseVersion(*r.CV) + require.NoErrorf(t, err, "error parsing CV %q: %v", *r.CV, err) + return DocVersion{RevTreeID: *r.RevID, CV: cv} } func MarshalConfig(t *testing.T, config db.ReplicationConfig) string { diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index aee3f630a1..f5669d22d2 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -15,6 +15,7 @@ import ( "net/http/httptest" "net/url" "slices" + "strings" "sync/atomic" "testing" "time" @@ -106,20 +107,42 @@ func (rt *RestTester) UpdateDocRev(docID, revID string, body string) string { // UpdateDoc updates a document at a specific version and returns the new version. func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) DocVersion { - resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevTreeID) - rawResponse := rt.SendAdminRequest(http.MethodPut, resource, body) - RequireStatus(rt.TB(), rawResponse, http.StatusCreated) - return DocVersionFromPutResponse(rt.TB(), rawResponse) + occValue := version.RevTreeID + if !version.CV.IsEmpty() { + occValue = version.CV.String() + } + resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, occValue) + resp := rt.SendAdminRequest(http.MethodPut, resource, body) + if isRespUseRevTreeIDInstead(resp) { + // trying to update a document in-conflict with a CV - try again with RevTreeID + // this is a pretty narrow edge-case and one that customers would deal with in the same way (get the document out of conflict using RevTreeID before using CV) + resp = rt.SendAdminRequest(http.MethodPut, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevTreeID), body) + } + RequireStatus(rt.TB(), resp, http.StatusCreated) + return DocVersionFromPutResponse(rt.TB(), resp) } // DeleteDoc deletes a document at a specific version. The test will fail if the revision does not exist. -func (rt *RestTester) DeleteDoc(docID string, docVersion DocVersion) DocVersion { - resp := rt.SendAdminRequest(http.MethodDelete, - fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, docVersion.RevTreeID), "") +func (rt *RestTester) DeleteDoc(docID string, version DocVersion) DocVersion { + occValue := version.RevTreeID + if !version.CV.IsEmpty() { + occValue = version.CV.String() + } + resp := rt.SendAdminRequest(http.MethodDelete, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, occValue), "") + if isRespUseRevTreeIDInstead(resp) { + // trying to update a document in-conflict with a CV - try again with RevTreeID + // this is a pretty narrow edge-case and one that customers would deal with in the same way (get the document out of conflict using RevTreeID before using CV) + resp = rt.SendAdminRequest(http.MethodDelete, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, version.RevTreeID), "") + } RequireStatus(rt.TB(), resp, http.StatusOK) return DocVersionFromPutResponse(rt.TB(), resp) } +// isRespUseRevTreeIDInstead returns true if the response indicates that a RevTree ID should be used instead of a CV for modifying a document in conflict. +func isRespUseRevTreeIDInstead(resp *TestResponse) bool { + return resp.Code == http.StatusBadRequest && strings.Contains(resp.BodyString(), "Cannot use CV to modify a document in conflict - resolve first with RevTree ID") +} + // DeleteDocRev removes a document at a specific revision. Deprecated for DeleteDoc. func (rt *RestTester) DeleteDocRev(docID, revID string) { rt.DeleteDoc(docID, DocVersion{RevTreeID: revID}) @@ -471,8 +494,7 @@ func (rt *RestTester) UpdateDocDirectly(docID string, version DocVersion, body d func (rt *RestTester) DeleteDocDirectly(docID string, version DocVersion) DocVersion { collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() - // TODO: CBG-4426 - DeleteDocDirectly does not support CV - rev, doc, err := collection.DeleteDoc(ctx, docID, version.RevTreeID) + rev, doc, err := collection.DeleteDoc(ctx, docID, version) require.NoError(rt.TB(), err) return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } diff --git a/topologytest/sync_gateway_peer_test.go b/topologytest/sync_gateway_peer_test.go index f2656636e6..b97208fd49 100644 --- a/topologytest/sync_gateway_peer_test.go +++ b/topologytest/sync_gateway_peer_test.go @@ -137,11 +137,9 @@ func (p *SyncGatewayPeer) WriteDocument(dsName sgbucket.DataStoreName, docID str func (p *SyncGatewayPeer) DeleteDocument(dsName sgbucket.DataStoreName, docID string) DocMetadata { collection, ctx := p.getCollection(dsName) doc, err := collection.GetDocument(ctx, docID, db.DocUnmarshalAll) - var revID string - if err == nil { - revID = doc.CurrentRev - } - _, doc, err = collection.DeleteDoc(ctx, docID, revID) + require.NoError(p.TB(), err) + require.NotNil(p.TB(), doc) + _, doc, err = collection.DeleteDoc(ctx, docID, doc.ExtractDocVersion()) require.NoError(p.TB(), err) docMeta := DocMetadataFromDocument(doc) p.TB().Logf("%s: Deleted document %s with %#+v", p, docID, docMeta) From d18431d6d963b0138d72a2753ec56221f8fa4e74 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Wed, 20 Aug 2025 14:58:30 +0100 Subject: [PATCH 02/14] Change moved code to just use int64 - almost certainly not needed for Rev generation but makes the code simpler --- db/document.go | 4 ++-- rest/utilities_testing_blip_client.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/db/document.go b/db/document.go index 694149e934..3d5d7c4255 100644 --- a/db/document.go +++ b/db/document.go @@ -1523,7 +1523,7 @@ func (v DocVersion) GetRev(useHLV bool) string { } // RevIDGeneration returns the Rev ID generation for the current version -func (v *DocVersion) RevIDGeneration() int { +func (v *DocVersion) RevIDGeneration() int64 { if v == nil { return 0 } @@ -1532,7 +1532,7 @@ func (v *DocVersion) RevIDGeneration() int { base.AssertfCtx(context.TODO(), "Error parsing generation from rev ID %q: %v", v.RevTreeID, err) return 0 } - return int(gen) + return gen } // RevIDDigest returns the Rev ID digest for the current version diff --git a/rest/utilities_testing_blip_client.go b/rest/utilities_testing_blip_client.go index 4bed1530b5..837ccfe0f5 100644 --- a/rest/utilities_testing_blip_client.go +++ b/rest/utilities_testing_blip_client.go @@ -1531,7 +1531,7 @@ func (btcc *BlipTesterCollectionClient) upsertDoc(docID string, parentVersion *D doc = newClientDocument(docID, 0, nil) } - newGen := 1 + var newGen int64 = 1 var hlv db.HybridLogicalVector if parentVersion != nil { // grab latest version for this doc and make sure we're doing an upsert on top of it to avoid branching revisions @@ -1613,7 +1613,7 @@ func (btc *BlipTesterClient) GetDocVersion(docID string) DocVersion { return DocVersion{RevTreeID: doc.CurrentRev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } -func (btcc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte, revGen int) (outputBody []byte) { +func (btcc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte, revGen int64) (outputBody []byte) { if !bytes.Contains(inputBody, []byte(db.BodyAttachments)) { return inputBody } From 8a1646ceb27b668c8f8e6cfabff702c12fc25b66 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Wed, 20 Aug 2025 17:28:42 +0100 Subject: [PATCH 03/14] Remove log.Printf in test util --- rest/utilities_testing.go | 1 - 1 file changed, 1 deletion(-) diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index 00f3e5c4d2..f2aeda662e 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -1145,7 +1145,6 @@ func (rt *RestTester) GetRawDoc(key string) RawDocResponse { response.DumpBody() var rawResponse RawDocResponse require.NoError(rt.TB(), base.JSONUnmarshal(response.BodyBytes(), &rawResponse)) - log.Printf("rawResponse: %#v", rawResponse) return rawResponse } From ad00f8359b0e598218b58b2bef6c0de0df0e7b63 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Wed, 20 Aug 2025 17:34:02 +0100 Subject: [PATCH 04/14] fix typo --- rest/doc_api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rest/doc_api.go b/rest/doc_api.go index 6fc1986a49..f590f5c131 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -740,7 +740,8 @@ func (h *handler) handlePostDoc() error { return nil } -func docVerisonFromOCCValue(occValue string, occValueType occVersionType) (docVersion db.DocVersion, err error) { +// docVersionFromOCCValue converts an OCC value and type into a DocVersion struct. +func docVersionFromOCCValue(occValue string, occValueType occVersionType) (docVersion db.DocVersion, err error) { switch occValueType { case VersionTypeRevTreeID: docVersion.RevTreeID = occValue @@ -772,7 +773,7 @@ func (h *handler) handleDeleteDoc() error { return fmt.Errorf("couldn't get OCC value from request: %w", err) } - docVersion, err := docVerisonFromOCCValue(occValue, occValueType) + docVersion, err := docVersionFromOCCValue(occValue, occValueType) if err != nil { return fmt.Errorf("couldn't build document version from OCC value: %w", err) } From 169f03481024e042bb26e0394fdc0a3394a30f8d Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 15:37:13 +0100 Subject: [PATCH 05/14] Cleanup of `DocVersion` --- db/crud.go | 8 ++--- db/document.go | 47 +++++++------------------- rest/api_test.go | 9 +++-- rest/attachment_test.go | 8 +++-- rest/replicatortest/replicator_test.go | 2 +- rest/utilities_testing_blip_client.go | 16 +++++---- 6 files changed, 37 insertions(+), 53 deletions(-) diff --git a/db/crud.go b/db/crud.go index 5fd73053c2..8fe8e50eac 100644 --- a/db/crud.go +++ b/db/crud.go @@ -2941,12 +2941,8 @@ func (db *DatabaseCollectionWithUser) Post(ctx context.Context, body Body) (doci // Deletes a document, by adding a new revision whose _deleted property is true. func (db *DatabaseCollectionWithUser) DeleteDoc(ctx context.Context, docid string, docVersion DocVersion) (string, *Document, error) { - body := Body{BodyDeleted: true} - if !docVersion.CV.IsEmpty() { - body[BodyCV] = docVersion.CV.String() - } else { - body[BodyRev] = docVersion.RevTreeID - } + versionKey, versionStr := docVersion.Body1xKVPair() + body := Body{BodyDeleted: true, versionKey: versionStr} newRevID, doc, err := db.Put(ctx, docid, body) return newRevID, doc, err } diff --git a/db/document.go b/db/document.go index 3d5d7c4255..ba348ea136 100644 --- a/db/document.go +++ b/db/document.go @@ -19,7 +19,6 @@ import ( "math" "net/http" "strconv" - "strings" "time" sgbucket "github.com/couchbase/sg-bucket" @@ -1487,6 +1486,7 @@ func (doc *Document) ExtractDocVersion() DocVersion { } // DocVersion represents a specific version of a document in an revID/HLV agnostic manner. +// Users should access the properties via the CV and RevTreeID methods, to ensure there's a check that the specific version type is not empty. type DocVersion struct { RevTreeID string CV Version @@ -1502,43 +1502,20 @@ func (v DocVersion) GoString() string { return fmt.Sprintf("DocVersion{RevTreeID:%s,CV:%#v}", v.RevTreeID, v.CV) } -// DocVersionRevTreeEqual compares two DocVersions based on their RevTreeID. -func (v DocVersion) DocVersionRevTreeEqual(o DocVersion) bool { - if v.RevTreeID != o.RevTreeID { - return false - } - return true -} - -// GetRev returns the revision ID for the DocVersion. -func (v DocVersion) GetRev(useHLV bool) string { - if useHLV { - if v.CV.SourceID == "" { - return "" - } - return v.CV.String() - } else { - return v.RevTreeID - } +// GetRevTreeID returns the Revision Tree ID of the document version, and a bool indicating whether it is present. +func (v DocVersion) GetRevTreeID() (string, bool) { + return v.RevTreeID, v.RevTreeID != "" } -// RevIDGeneration returns the Rev ID generation for the current version -func (v *DocVersion) RevIDGeneration() int64 { - if v == nil { - return 0 - } - gen, err := strconv.ParseInt(strings.Split(v.RevTreeID, "-")[0], 10, 64) - if err != nil { - base.AssertfCtx(context.TODO(), "Error parsing generation from rev ID %q: %v", v.RevTreeID, err) - return 0 - } - return gen +// GetCV returns the Current Version of the document, and a bool indicating whether it is present. +func (v DocVersion) GetCV() (Version, bool) { + return v.CV, !v.CV.IsEmpty() } -// RevIDDigest returns the Rev ID digest for the current version -func (v *DocVersion) RevIDDigest() string { - if v == nil { - return "" +// Body1xKVPair returns the key and value to use in a 1.x-style document body for the given DocVersion. +func (d DocVersion) Body1xKVPair() (bodyVersionKey, bodyVersionStr string) { + if cv, ok := d.GetCV(); ok { + return BodyCV, cv.String() } - return strings.Split(v.RevTreeID, "-")[1] + return BodyRev, d.RevTreeID } diff --git a/rest/api_test.go b/rest/api_test.go index 73ce33da72..bedf8c52d5 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -3255,10 +3255,15 @@ func TestDocCRUDWithCV(t *testing.T) { getDocVersion, _ := rt.GetDoc(docID) require.Equal(t, createVersion, getDocVersion) + revIDGen := func(v DocVersion) int { + gen, _ := db.ParseRevID(base.TestCtx(t), v.RevTreeID) + return gen + } + updateVersion := rt.UpdateDoc(docID, createVersion, `{"update":true}`) require.NotEqual(t, createVersion, updateVersion) assert.Greaterf(t, updateVersion.CV.Value, createVersion.CV.Value, "Expected CV Value to be bumped on update") - assert.Greaterf(t, updateVersion.RevIDGeneration(), createVersion.RevIDGeneration(), "Expected revision generation to be bumped on update") + assert.Greaterf(t, revIDGen(updateVersion), revIDGen(createVersion), "Expected revision generation to be bumped on update") getDocVersion, _ = rt.GetDoc(docID) require.Equal(t, updateVersion, getDocVersion) @@ -3284,5 +3289,5 @@ func TestDocCRUDWithCV(t *testing.T) { deleteVersion := rt.DeleteDoc(docID, updateVersion) require.NotEqual(t, updateVersion, deleteVersion) assert.Greaterf(t, deleteVersion.CV.Value, updateVersion.CV.Value, "Expected CV Value to be bumped on delete") - assert.Greaterf(t, deleteVersion.RevIDGeneration(), updateVersion.RevIDGeneration(), "Expected revision generation to be bumped on delete") + assert.Greaterf(t, revIDGen(deleteVersion), revIDGen(updateVersion), "Expected revision generation to be bumped on delete") } diff --git a/rest/attachment_test.go b/rest/attachment_test.go index 06e71829ac..3c9838dee4 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -727,7 +727,8 @@ func TestConflictWithInvalidAttachment(t *testing.T) { require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) // Modify Doc - parentRevList := [3]string{"foo3", "foo2", version.RevIDDigest()} + _, versionDigest := db.ParseRevID(base.TestCtx(t), version.RevTreeID) + parentRevList := [3]string{"foo3", "foo2", versionDigest} body["_rev"] = "3-foo3" body["rev"] = "3-foo3" body["_revisions"].(map[string]interface{})["ids"] = parentRevList @@ -794,16 +795,17 @@ func TestConflictingBranchAttachments(t *testing.T) { // Create a document version := rt.CreateTestDoc("doc1") + _, versionDigest := db.ParseRevID(base.TestCtx(t), version.RevTreeID) // //Create diverging tree - reqBodyRev2 := `{"_rev": "2-two", "_revisions": {"ids": ["two", "` + version.RevIDDigest() + `"], "start": 2}}` + reqBodyRev2 := `{"_rev": "2-two", "_revisions": {"ids": ["two", "` + versionDigest + `"], "start": 2}}` response := rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev2) RequireStatus(t, response, http.StatusCreated) docVersion2 := DocVersionFromPutResponse(t, response) - reqBodyRev2a := `{"_rev": "2-two", "_revisions": {"ids": ["twoa", "` + version.RevIDDigest() + `"], "start": 2}}` + reqBodyRev2a := `{"_rev": "2-two", "_revisions": {"ids": ["twoa", "` + versionDigest + `"], "start": 2}}` response = rt.SendAdminRequest("PUT", "/{{.keyspace}}/doc1?new_edits=false", reqBodyRev2a) RequireStatus(t, response, http.StatusCreated) docVersion2a := DocVersionFromPutResponse(t, response) diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index a24c896e27..0b9d50796f 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -4324,7 +4324,7 @@ func TestActiveReplicatorPushAndPullConflict(t *testing.T) { // Validate results on the remote (rt2) rt2Since := remoteDoc.Sequence - if test.expectedVersion.DocVersionRevTreeEqual(test.remoteVersion) { + if test.expectedVersion.RevTreeID == test.remoteVersion.RevTreeID { // no changes should have been pushed back up to rt2, because this rev won. rt2Since = 0 } diff --git a/rest/utilities_testing_blip_client.go b/rest/utilities_testing_blip_client.go index 837ccfe0f5..f1429f7a86 100644 --- a/rest/utilities_testing_blip_client.go +++ b/rest/utilities_testing_blip_client.go @@ -1531,7 +1531,7 @@ func (btcc *BlipTesterCollectionClient) upsertDoc(docID string, parentVersion *D doc = newClientDocument(docID, 0, nil) } - var newGen int64 = 1 + var newGen int = 1 var hlv db.HybridLogicalVector if parentVersion != nil { // grab latest version for this doc and make sure we're doing an upsert on top of it to avoid branching revisions @@ -1541,7 +1541,8 @@ func (btcc *BlipTesterCollectionClient) upsertDoc(docID string, parentVersion *D if btcc.UseHLV() { hlv = latestRev.HLV } else { - newGen = parentVersion.RevIDGeneration() + 1 + parentGen, _ := db.ParseRevID(btcc.ctx, parentVersion.RevTreeID) + newGen = parentGen + 1 } } @@ -1613,7 +1614,7 @@ func (btc *BlipTesterClient) GetDocVersion(docID string) DocVersion { return DocVersion{RevTreeID: doc.CurrentRev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} } -func (btcc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte, revGen int64) (outputBody []byte) { +func (btcc *BlipTesterCollectionClient) ProcessInlineAttachments(inputBody []byte, revGen int) (outputBody []byte) { if !bytes.Contains(inputBody, []byte(db.BodyAttachments)) { return inputBody } @@ -2083,11 +2084,14 @@ func (btcc *BlipTesterCollectionClient) getAllRevisions(docID string) []DocVersi return versions } -func (btc *BlipTesterClient) AssertDeltaSrcProperty(t *testing.T, msg *blip.Message, docVersion DocVersion) { +func (btc *BlipTesterClient) AssertDeltaSrcProperty(t *testing.T, msg *blip.Message, expectedVersion DocVersion) { subProtocol, err := db.ParseSubprotocolString(btc.SupportedBLIPProtocols[0]) require.NoError(t, err) - rev := docVersion.GetRev(subProtocol >= db.CBMobileReplicationV4) - assert.Equal(t, rev, msg.Properties[db.RevMessageDeltaSrc]) + expectedDeltaSrcRev := expectedVersion.RevTreeID + if subProtocol >= db.CBMobileReplicationV4 { + expectedDeltaSrcRev = expectedVersion.CV.String() + } + assert.Equal(t, expectedDeltaSrcRev, msg.Properties[db.RevMessageDeltaSrc]) } // getHLVFromRevMessage extracts the full HLV from a rev message. This will fail the test if the message does not contain a valid HLV. From abb7c1b19f352343e336bb9670f1a0b9bc2852b8 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 15:55:02 +0100 Subject: [PATCH 06/14] Add zero-value for occVersionType enum and more error handling --- rest/doc_api.go | 43 +++++++++++++++++++++------- rest/utilities_testing.go | 1 - rest/utilities_testing_resttester.go | 4 +-- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/rest/doc_api.go b/rest/doc_api.go index f590f5c131..c7185e051b 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -362,6 +362,14 @@ func (h *handler) getOCCValue(optionalBody db.Body) (occValue string, occValueTy occValueType = VersionTypeRevTreeID skipBodyMatchValidation = true } + } else { + // empty occValue - treat as a create operation without any parent + return "", VersionTypeRevTreeID, nil + } + + // defensive measure against falling out of above without a type set + if occValueType == VersionTypeUnknown { + return "", 0, base.HTTPErrorf(http.StatusBadRequest, "Invalid version type for OCC value: %q", occValue) } // ensure the value provided matches exactly the one that may also be supplied in the body @@ -410,8 +418,12 @@ func (h *handler) handlePutAttachment() error { body, err := h.collection.Get1xRevBody(h.ctx(), docid, occValue, false, nil) if err != nil { if base.IsDocNotFoundError(err) { + bodyKey, err := bodyKeyForOCCVersionType(occValueType) + if err != nil { + return base.HTTPErrorf(http.StatusBadRequest, "Invalid OCC version type: %v", err) + } // couchdb creates empty body on attachment PUT for non-existent doc id - body = db.Body{bodyKeyForOCCVersionType(occValueType): occValue} + body = db.Body{bodyKey: occValue} } else if err != nil { return err } @@ -504,8 +516,9 @@ func (h *handler) handleDeleteAttachment() error { type occVersionType uint8 const ( - VersionTypeRevTreeID occVersionType = iota // Revision Tree ID (RevTreeID / RevID) - VersionTypeCV // HLV/Version Vector CV + VersionTypeUnknown occVersionType = iota + VersionTypeRevTreeID // Revision Tree ID (RevTreeID / RevID) + VersionTypeCV // HLV/Version Vector CV ) // guessOCCVersionTypeFromValue returns the type of document version based on the string value. Either a RevTree ID or a CV. @@ -513,19 +526,21 @@ func guessOCCVersionTypeFromValue(s string) occVersionType { if base.IsRevTreeID(s) { return VersionTypeRevTreeID } - // anything else we'll assume is a CV - // we _could_ check for a well-formatted CV, but given the usage for OCC, we only need an exact string match - return VersionTypeCV + if _, err := db.ParseVersion(s); err == nil { + return VersionTypeCV + } + return VersionTypeUnknown } -func bodyKeyForOCCVersionType(versionType occVersionType) string { +func bodyKeyForOCCVersionType(versionType occVersionType) (string, error) { switch versionType { case VersionTypeRevTreeID: - return db.BodyRev + return db.BodyRev, nil case VersionTypeCV: - return db.BodyCV + return db.BodyCV, nil + default: + return "", fmt.Errorf("unknown occVersionType %d", versionType) } - return "" } // HTTP handler for a PUT of a document @@ -583,7 +598,11 @@ func (h *handler) handlePutDoc() error { } // set OCC version body value for Put - body[bodyKeyForOCCVersionType(occValueType)] = occValue + bodyKey, err := bodyKeyForOCCVersionType(occValueType) + if err != nil { + return base.HTTPErrorf(http.StatusBadRequest, "Invalid OCC version type: %v", err) + } + body[bodyKey] = occValue newRev, doc, err = h.collection.Put(h.ctx(), docid, body) if err != nil { @@ -750,6 +769,8 @@ func docVersionFromOCCValue(occValue string, occValueType occVersionType) (docVe if err != nil { return DocVersion{}, base.HTTPErrorf(http.StatusBadRequest, "Invalid CV: %v", err) } + default: + return DocVersion{}, base.HTTPErrorf(http.StatusBadRequest, "Unknown OCC version type: %d", occValueType) } return docVersion, nil } diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index f2aeda662e..af99175b8f 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -1142,7 +1142,6 @@ func (rt *RestTester) GetDocumentSequence(key string) (sequence uint64) { func (rt *RestTester) GetRawDoc(key string) RawDocResponse { response := rt.SendAdminRequest("GET", fmt.Sprintf("/{{.keyspace}}/_raw/%s", key), "") require.Equal(rt.TB(), http.StatusOK, response.Code, "Error getting raw document %s", response.Body.String()) - response.DumpBody() var rawResponse RawDocResponse require.NoError(rt.TB(), base.JSONUnmarshal(response.BodyBytes(), &rawResponse)) return rawResponse diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index f5669d22d2..9821087458 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -105,7 +105,7 @@ func (rt *RestTester) UpdateDocRev(docID, revID string, body string) string { return version.RevTreeID } -// UpdateDoc updates a document at a specific version and returns the new version. +// UpdateDoc updates a document at a specific version and returns the new version. Uses CV for REST API if present in DocVersion, otherwise fall back to RevTreeID. func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) DocVersion { occValue := version.RevTreeID if !version.CV.IsEmpty() { @@ -122,7 +122,7 @@ func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) D return DocVersionFromPutResponse(rt.TB(), resp) } -// DeleteDoc deletes a document at a specific version. The test will fail if the revision does not exist. +// DeleteDoc deletes a document at a specific version. The test will fail if the revision does not exist. Uses CV for REST API if present in DocVersion, otherwise fall back to RevTreeID. func (rt *RestTester) DeleteDoc(docID string, version DocVersion) DocVersion { occValue := version.RevTreeID if !version.CV.IsEmpty() { From 79edf1281acbf7e323293869ef078727761e7e87 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 18:19:05 +0100 Subject: [PATCH 07/14] Escape rev query param for CV in RestTester helper functions --- rest/utilities_testing_resttester.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 9821087458..f349e8b2a9 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -111,7 +111,7 @@ func (rt *RestTester) UpdateDoc(docID string, version DocVersion, body string) D if !version.CV.IsEmpty() { occValue = version.CV.String() } - resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, occValue) + resource := fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, url.QueryEscape(occValue)) resp := rt.SendAdminRequest(http.MethodPut, resource, body) if isRespUseRevTreeIDInstead(resp) { // trying to update a document in-conflict with a CV - try again with RevTreeID @@ -128,7 +128,7 @@ func (rt *RestTester) DeleteDoc(docID string, version DocVersion) DocVersion { if !version.CV.IsEmpty() { occValue = version.CV.String() } - resp := rt.SendAdminRequest(http.MethodDelete, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, occValue), "") + resp := rt.SendAdminRequest(http.MethodDelete, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, url.QueryEscape(occValue)), "") if isRespUseRevTreeIDInstead(resp) { // trying to update a document in-conflict with a CV - try again with RevTreeID // this is a pretty narrow edge-case and one that customers would deal with in the same way (get the document out of conflict using RevTreeID before using CV) From f17891cebe446f98f120084a86bf0ac7f2376881 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 18:54:16 +0100 Subject: [PATCH 08/14] Detect unescaped rev values - improve documentation and cover with test. --- docs/api/components/parameters.yaml | 6 ++++-- rest/doc_api.go | 6 ++++++ rest/doc_api_test.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index 24ab42881c..a64fe5fc4b 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -341,8 +341,10 @@ rev: required: false schema: type: string - example: 2-5145e1086bb8d1d71a531e9f6b543c58 - description: The document revision to target. + examples: + - 2-5145e1086bb8d1d71a531e9f6b543c58 + - 185dd4a2b4490000%404EZjEgl1AKEj8qh%2BvXS7OQ + description: The document revision to target. If this is a CV value, ensure the query parameter is URL encoded (`+`->`%2B`, `@`->`%40`, etc.) revs_from: name: revs_from in: query diff --git a/rest/doc_api.go b/rest/doc_api.go index c7185e051b..712c4f3248 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -344,6 +344,12 @@ func (h *handler) getOCCValue(optionalBody db.Body) (occValue string, occValueTy // we grab occValue from either query param, Etag header, or request body (in that order) if revQuery := h.getQuery("rev"); revQuery != "" { occValue = revQuery + // try to detect occ Values that are not URL Query escaped + // - `+` which can appear in base64 strings is converted to a space when not escaped properly + // other characters are difficult to correctly detect, since the value is already unescaped + if strings.ContainsAny(occValue, " ") { + return "", 0, base.HTTPErrorf(http.StatusBadRequest, "Bad rev query parameter: %q - ensure this query parameter value is URL Encoded", occValue) + } occValueType = guessOCCVersionTypeFromValue(occValue) } else if ifMatch, err := h.getEtag("If-Match"); err != nil { return "", 0, err diff --git a/rest/doc_api_test.go b/rest/doc_api_test.go index 77f1f034f3..7666188a6a 100644 --- a/rest/doc_api_test.go +++ b/rest/doc_api_test.go @@ -290,3 +290,31 @@ func readMultiPartBody(t *testing.T, response *TestResponse) []string { } return output } + +func TestCVUnescapedRevQueryParam(t *testing.T) { + tests := []struct { + revValue string + expectError bool + }{ + {revValue: "1-abc"}, // Normal Rev ID (doesn't need escaping) + {revValue: "185dd4a2b4490000%404EZjEgl1AKEj8qh%2BvXS7OQ"}, // CV escaped + {revValue: "185dd4a2b4490000@4EZjEgl1AKEj8qh+vXS7OQ", expectError: true}, // CV unescaped + } + + rt := NewRestTesterPersistentConfig(t) + defer rt.Close() + + for _, test := range tests { + t.Run(test.revValue, func(t *testing.T) { + resp := rt.SendAdminRequest(http.MethodPut, "/{{.keyspace}}/testdoc?rev="+test.revValue, `{"foo":"bar"}`) + if test.expectError { + RequireStatus(t, resp, http.StatusBadRequest) + assert.Contains(t, resp.BodyString(), "Bad rev query parameter") + } else { + // this is "successful" since there isn't a doc that exists with that rev but the request made it through + RequireStatus(t, resp, http.StatusConflict) + assert.Contains(t, resp.BodyString(), `Document revision conflict`) + } + }) + } +} From e5b70676f937f07b681182bfb10d0a7b837618bf Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 19:01:59 +0100 Subject: [PATCH 09/14] Remove multi-examples --- docs/api/components/parameters.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index a64fe5fc4b..6232e9fca8 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -341,9 +341,7 @@ rev: required: false schema: type: string - examples: - - 2-5145e1086bb8d1d71a531e9f6b543c58 - - 185dd4a2b4490000%404EZjEgl1AKEj8qh%2BvXS7OQ + example: 2-5145e1086bb8d1d71a531e9f6b543c58 description: The document revision to target. If this is a CV value, ensure the query parameter is URL encoded (`+`->`%2B`, `@`->`%40`, etc.) revs_from: name: revs_from From b7a3a5db2c62fcf6d24cf62ddee582af11dd8dde Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 00:52:45 +0100 Subject: [PATCH 10/14] Clean up fetch by CV REST API codepath. Wire up CV for use in RestTester.GetDocVersion for general test coverage. --- db/crud.go | 7 ++--- docs/api/components/parameters.yaml | 2 +- docs/api/paths/public/keyspace-docid.yaml | 2 +- rest/attachment_test.go | 6 +++++ rest/doc_api.go | 31 +++++++++++++++++------ rest/utilities_testing_resttester.go | 20 ++++++++++----- 6 files changed, 49 insertions(+), 19 deletions(-) diff --git a/db/crud.go b/db/crud.go index 8fe8e50eac..7922f3f042 100644 --- a/db/crud.go +++ b/db/crud.go @@ -295,13 +295,13 @@ func (c *DatabaseCollection) OnDemandImportForGet(ctx context.Context, docid str return docOut, nil } -// GetRev returns the revision for the given docID and revID, or the current active revision if revID is empty. -func (db *DatabaseCollectionWithUser) GetRev(ctx context.Context, docID, revID string, history bool, attachmentsSince []string) (DocumentRevision, error) { +// GetRev returns the revision for the given docID and revOrCV, or the current active revision if revOrCV is empty. +func (db *DatabaseCollectionWithUser) GetRev(ctx context.Context, docID, revOrCV string, history bool, attachmentsSince []string) (DocumentRevision, error) { maxHistory := 0 if history { maxHistory = math.MaxInt32 } - return db.getRev(ctx, docID, revID, maxHistory, nil) + return db.getRev(ctx, docID, revOrCV, maxHistory, nil) } // Returns the body of the current revision of a document @@ -2765,6 +2765,7 @@ func (db *DatabaseCollectionWithUser) postWriteUpdateHLV(ctx context.Context, do doc.HLV.CurrentVersionCAS = casOut } // backup new revision to the bucket now we have a doc assigned a CV (post macro expansion) for delta generation purposes + // we don't need to store revision body backups without delta sync in 4.0, since all clients know how to use the sendReplacementRevs feature backupRev := db.deltaSyncEnabled() && db.deltaSyncRevMaxAgeSeconds() != 0 if db.UseXattrs() && backupRev { var newBodyWithAtts = doc._rawBody diff --git a/docs/api/components/parameters.yaml b/docs/api/components/parameters.yaml index 6232e9fca8..4d8d8496a4 100644 --- a/docs/api/components/parameters.yaml +++ b/docs/api/components/parameters.yaml @@ -342,7 +342,7 @@ rev: schema: type: string example: 2-5145e1086bb8d1d71a531e9f6b543c58 - description: The document revision to target. If this is a CV value, ensure the query parameter is URL encoded (`+`->`%2B`, `@`->`%40`, etc.) + description: The document revision to target. This can be a RevTree ID or a CV (Current Version) ID. If this is a CV value, ensure the query parameter is URL encoded (`+`->`%2B`, `@`->`%40`, etc.) revs_from: name: revs_from in: query diff --git a/docs/api/paths/public/keyspace-docid.yaml b/docs/api/paths/public/keyspace-docid.yaml index 5e408fe38c..129c51eaf0 100644 --- a/docs/api/paths/public/keyspace-docid.yaml +++ b/docs/api/paths/public/keyspace-docid.yaml @@ -27,7 +27,7 @@ get: Etag: schema: type: string - description: The document revision ID if only returning 1 revision. + description: An optimistic concurrency control (OCC) value used to prevent conflicts in a subsequent update. This value can be a RevTree ID or a CV (Current Version) ID, depending on the version type requested. content: application/json: schema: diff --git a/rest/attachment_test.go b/rest/attachment_test.go index ae4280ba06..137b99dfc1 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -2135,6 +2135,9 @@ func TestAttachmentsMissing(t *testing.T) { rt.GetDatabase().FlushRevisionCacheForTest() + // strip CV from version - we don't store revision backups for old CVs like we do for RevIDs + version2.CV = db.Version{} + body := rt.GetDocVersion(docID, version2) require.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", body["_attachments"].(map[string]interface{})["hello.txt"].(map[string]interface{})["digest"]) } @@ -2158,6 +2161,9 @@ func TestAttachmentsMissingNoBody(t *testing.T) { rt.GetDatabase().FlushRevisionCacheForTest() + // strip CV from version - we don't store revision backups for old CVs like we do for RevIDs + version2.CV = db.Version{} + body := rt.GetDocVersion(docID, version2) require.Equal(t, "sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0=", body["_attachments"].(map[string]interface{})["hello.txt"].(map[string]interface{})["digest"]) } diff --git a/rest/doc_api.go b/rest/doc_api.go index 712c4f3248..94a741d1a3 100644 --- a/rest/doc_api.go +++ b/rest/doc_api.go @@ -27,15 +27,24 @@ import ( // HTTP handler for a GET of a document func (h *handler) handleGetDoc() error { docid := h.PathVar("docid") - revid := h.getQuery("rev") + rev := h.getQuery("rev") // Empty, RevTree ID, or CV openRevs := h.getQuery("open_revs") showExp := h.getBoolQuery("show_exp") const showCV = true // Post-beta 4.0 _always_ returns CV - negligible impact to revtree-only clients and promotes CV as the preferred OCC value if replicator2, _ := h.getOptBoolQuery("replicator2", false); replicator2 { - return h.handleGetDocReplicator2(docid, revid) + return h.handleGetDocReplicator2(docid, rev) + } + + // Extra validation of these options, since this combination isn't valid anyway. We want to prevent users from attempting to use CV with open_revs. + if openRevs != "" && rev != "" { + return base.HTTPErrorf(http.StatusBadRequest, "cannot specify both 'rev' and 'open_revs' query parameters") } + // We'll treat empty rev as a RevTree ID, which only affects what the ETag looks like. + // If the user specifically asked for a CV, they'll get a CV ETag - but everything else can stay as RevTree ID for compatibility. + isRevTreeID := rev == "" || base.IsRevTreeID(rev) + // Check whether the caller wants a revision history, or attachment bodies, or both: var revsLimit = 0 var revsFrom, attachmentsSince []string @@ -69,7 +78,7 @@ func (h *handler) handleGetDoc() error { if openRevs == "" { // Single-revision GET: - value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, revid, db.Get1xRevBodyOptions{ + value, err := h.collection.Get1xRevBodyWithHistory(h.ctx(), docid, rev, db.Get1xRevBodyOptions{ MaxHistory: revsLimit, HistoryFrom: revsFrom, AttachmentsSince: attachmentsSince, @@ -94,8 +103,14 @@ func (h *handler) handleGetDoc() error { } return kNotFoundError } - foundRev := value[db.BodyRev].(string) - h.setEtag(foundRev) + + var etagValue string + if isRevTreeID { + etagValue = value[db.BodyRev].(string) + } else { + etagValue = value[db.BodyCV].(string) + } + h.setEtag(etagValue) h.db.DbStats.Database().NumDocReadsRest.Add(1) hasBodies := attachmentsSince != nil && value[db.BodyAttachments] != nil @@ -110,7 +125,7 @@ func (h *handler) handleGetDoc() error { } base.Audit(h.ctx(), base.AuditIDDocumentRead, base.AuditFields{ base.AuditFieldDocID: docid, - base.AuditFieldDocVersion: foundRev, + base.AuditFieldDocVersion: etagValue, }) } else { var revids []string @@ -195,12 +210,12 @@ func (h *handler) handleGetDoc() error { return nil } -func (h *handler) handleGetDocReplicator2(docid, revid string) error { +func (h *handler) handleGetDocReplicator2(docid, revOrCV string) error { if !base.IsEnterpriseEdition() { return base.HTTPErrorf(http.StatusNotImplemented, "replicator2 endpoints are only supported in EE") } - rev, err := h.collection.GetRev(h.ctx(), docid, revid, true, nil) + rev, err := h.collection.GetRev(h.ctx(), docid, revOrCV, true, nil) if err != nil { return err } diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index f349e8b2a9..2d140bef82 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -78,7 +78,20 @@ func (rt *RestTester) TriggerOnDemandImport(docID string) { // GetDocVersion returns the doc body and version for the given docID and version. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocVersion(docID string, version DocVersion) db.Body { - rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+version.RevTreeID, "") + occValue := version.RevTreeID + if !version.CV.IsEmpty() { + occValue = version.CV.String() + } + rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+occValue, "") + RequireStatus(rt.TB(), rawResponse, http.StatusOK) + var body db.Body + require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) + return body +} + +// GetDocByRev returns the doc body for the given docID and Rev. If the document is not found, t.Fail will be called. +func (rt *RestTester) GetDocByRev(docID, revTreeID string) db.Body { + rawResponse := rt.SendAdminRequest("GET", fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, revTreeID), "") RequireStatus(rt.TB(), rawResponse, http.StatusOK) var body db.Body require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) @@ -143,11 +156,6 @@ func isRespUseRevTreeIDInstead(resp *TestResponse) bool { return resp.Code == http.StatusBadRequest && strings.Contains(resp.BodyString(), "Cannot use CV to modify a document in conflict - resolve first with RevTree ID") } -// DeleteDocRev removes a document at a specific revision. Deprecated for DeleteDoc. -func (rt *RestTester) DeleteDocRev(docID, revID string) { - rt.DeleteDoc(docID, DocVersion{RevTreeID: revID}) -} - func (rt *RestTester) GetDatabaseRoot(dbname string) DatabaseRoot { var dbroot DatabaseRoot resp := rt.SendAdminRequest("GET", "/"+dbname+"/", "") From 3e5eb2d3c2843e5874086f39ae17a024db963b5a Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 13:33:13 +0100 Subject: [PATCH 11/14] Remove usages of RestTester DocDirectly functions --- db/changes.go | 1 - rest/access_test.go | 2 +- rest/api_test.go | 2 +- rest/attachment_test.go | 11 ++-- rest/blip_api_collections_test.go | 6 +- rest/blip_api_crud_test.go | 59 +++++++++-------- rest/blip_api_delta_sync_test.go | 40 ++++++------ rest/blip_api_replication_test.go | 10 ++- rest/blip_channel_filter_test.go | 6 +- rest/blip_legacy_revid_test.go | 64 +++++++++---------- rest/changestest/changes_api_test.go | 6 +- rest/doc_api_test.go | 6 +- rest/importtest/import_test.go | 4 +- .../replicatortest/replicator_revtree_test.go | 16 ++--- rest/replicatortest/replicator_test.go | 46 ++++++------- rest/revocation_test.go | 18 +++--- rest/utilities_testing.go | 3 +- rest/utilities_testing_resttester.go | 57 +++++------------ 18 files changed, 160 insertions(+), 197 deletions(-) diff --git a/db/changes.go b/db/changes.go index a0ad2504e8..1efec86b3e 100644 --- a/db/changes.go +++ b/db/changes.go @@ -209,7 +209,6 @@ func (db *DatabaseCollectionWithUser) AddDocInstanceToChangeEntry(ctx context.Co } if options.IncludeDocs { var err error - // TODO: CBG-4776 - fetch by CV with sane APIs err = db.AddDocToChangeEntryUsingRevCache(ctx, entry, revID) if err != nil { base.WarnfCtx(ctx, "Changes feed: error getting revision body for %q (%s): %v", base.UD(entry.ID), revID, err) diff --git a/rest/access_test.go b/rest/access_test.go index d8a24803e9..4e961001e6 100644 --- a/rest/access_test.go +++ b/rest/access_test.go @@ -1144,7 +1144,7 @@ func TestAllDocsCV(t *testing.T) { defer rt.Close() const docID = "foo" - docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + docVersion := rt.PutDoc(docID, `{"foo": "bar"}`) testCases := []struct { name string diff --git a/rest/api_test.go b/rest/api_test.go index bedf8c52d5..3baa325a9c 100644 --- a/rest/api_test.go +++ b/rest/api_test.go @@ -2870,7 +2870,7 @@ func TestPvDeltaReadAndWrite(t *testing.T) { version1, _ := rt.GetDoc(docID) // update the above doc, this should push CV to PV and adds a new CV - version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"new": "update!"}) + version2 := rt.UpdateDoc(docID, version1, `{"new": "update!"}`) newDoc, _, err := collection.GetDocWithXattrs(ctx, existingHLVKey, db.DocUnmarshalAll) require.NoError(t, err) casV2 := newDoc.Cas diff --git a/rest/attachment_test.go b/rest/attachment_test.go index 137b99dfc1..f71e94f76b 100644 --- a/rest/attachment_test.go +++ b/rest/attachment_test.go @@ -2320,8 +2320,8 @@ func TestUpdateExistingAttachment(t *testing.T) { btcRunner.StartPull(btc.id) btcRunner.StartPush(btc.id) - doc1Version := rt.PutDocDirectly(doc1ID, db.Body{}) - doc2Version := rt.PutDocDirectly(doc2ID, db.Body{}) + doc1Version := rt.PutDoc(doc1ID, `{}`) + doc2Version := rt.PutDoc(doc2ID, `{}`) btcRunner.WaitForVersion(btc.id, doc1ID, doc1Version) btcRunner.WaitForVersion(btc.id, doc2ID, doc2Version) @@ -2375,7 +2375,7 @@ func TestPushUnknownAttachmentAsStub(t *testing.T) { btcRunner.StartPush(btc.id) // Add doc1 - doc1Version := rt.PutDocDirectly(doc1ID, db.Body{}) + doc1Version := rt.PutDoc(doc1ID, `{}`) btcRunner.WaitForVersion(btc.id, doc1ID, doc1Version) @@ -2587,9 +2587,8 @@ func TestCBLRevposHandling(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, nil) defer btc.Close() - startingBody := db.Body{"foo": "bar"} - doc1Version1 := rt.PutDocDirectly(doc1ID, startingBody) - doc2Version1 := rt.PutDocDirectly(doc2ID, startingBody) + doc1Version1 := rt.PutDoc(doc1ID, `{"foo": "bar"}`) + doc2Version1 := rt.PutDoc(doc2ID, `{"foo": "bar"}`) rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) diff --git a/rest/blip_api_collections_test.go b/rest/blip_api_collections_test.go index 1ea621633b..8f00964613 100644 --- a/rest/blip_api_collections_test.go +++ b/rest/blip_api_collections_test.go @@ -253,7 +253,7 @@ func TestCollectionsReplication(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, nil) defer btc.Close() - version := btc.rt.PutDocDirectly(docID, db.Body{}) + version := btc.rt.PutDoc(docID, `{}`) btc.rt.WaitForPendingChanges() btcRunner.StartOneshotPull(btc.id) @@ -279,7 +279,7 @@ func TestBlipReplicationMultipleCollections(t *testing.T) { body := `{"foo":"bar"}` versions := make([]DocVersion, 0, len(btc.rt.GetKeyspaces())) for _, collection := range btc.rt.GetDbCollections() { - docVersion := rt.PutDocDirectlyInCollection(collection, docName, db.Body{"foo": "bar"}) + docVersion := rt.PutDocInCollection(collection.Name, docName, `{"foo": "bar"}`) versions = append(versions, docVersion) } btc.rt.WaitForPendingChanges() @@ -326,7 +326,7 @@ func TestBlipReplicationMultipleCollectionsMismatchedDocSizes(t *testing.T) { blipName := btc.rt.getCollectionsForBLIP()[i] for j := 0; j < docCount; j++ { docName := fmt.Sprintf("doc%d", j) - version := rt.PutDocDirectlyInCollection(collection, docName, db.Body{"foo": "bar"}) + version := rt.PutDocInCollection(collection.Name, docName, `{"foo": "bar"}`) collectionVersions[blipName] = append(collectionVersions[blipName], version) collectionDocIDs[blipName] = append(collectionDocIDs[blipName], docName) diff --git a/rest/blip_api_crud_test.go b/rest/blip_api_crud_test.go index 7fbc4ce7d0..2fc612cb65 100644 --- a/rest/blip_api_crud_test.go +++ b/rest/blip_api_crud_test.go @@ -2010,7 +2010,7 @@ func TestSendReplacementRevision(t *testing.T) { rt.CreateUser(alice, userChannels) docID := test.name - version1 := rt.PutDocDirectly(docID, JsonToMap(t, fmt.Sprintf(`{"foo":"bar","channels":["%s"]}`, rev1Channel))) + version1 := rt.PutDoc(docID, fmt.Sprintf(`{"foo":"bar","channels":["%s"]}`, rev1Channel)) updatedVersion := make(chan DocVersion) collection, ctx := rt.GetSingleTestDatabaseCollection() @@ -2110,13 +2110,13 @@ func TestBlipPullRevMessageHistory(t *testing.T) { const docID = "doc1" // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version1 := rt.PutDocDirectly(docID, db.Body{"hello": "world!"}) + version1 := rt.PutDoc(docID, `{"hello": "world!"}`) data := btcRunner.WaitForVersion(client.id, docID, version1) assert.Equal(t, `{"hello":"world!"}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"hello": "alice"}) + version2 := rt.UpdateDoc(docID, version1, `{"hello": "alice"}`) data = btcRunner.WaitForVersion(client.id, docID, version2) assert.Equal(t, `{"hello":"alice"}`, string(data)) @@ -2171,7 +2171,7 @@ func TestPullReplicationUpdateOnOtherHLVAwarePeer(t *testing.T) { _ = btcRunner.WaitForVersion(client.id, docID, version1) // update the above doc - version2 := rt.UpdateDocDirectly(docID, version1, db.Body{"hello": "world!"}) + version2 := rt.UpdateDoc(docID, version1, `{"hello": "world!"}`) data := btcRunner.WaitForVersion(client.id, docID, version2) assert.Equal(t, `{"hello":"world!"}`, string(data)) @@ -2224,7 +2224,7 @@ func TestActiveOnlyContinuous(t *testing.T) { btc := btcRunner.NewBlipTesterClientOptsWithRT(rt, nil) defer btc.Close() - version := rt.PutDocDirectly(docID, db.Body{"test": true}) + version := rt.PutDoc(docID, `{"test": true}`) // start an initial pull btcRunner.StartPullSince(btc.id, BlipTesterPullOptions{Continuous: true, Since: "0", ActiveOnly: true}) @@ -2232,7 +2232,7 @@ func TestActiveOnlyContinuous(t *testing.T) { assert.Equal(t, `{"test":true}`, string(rev)) // delete the doc and make sure the client still gets the tombstone replicated - deletedVersion := rt.DeleteDocDirectly(docID, version) + deletedVersion := rt.DeleteDoc(docID, version) rev = btcRunner.WaitForVersion(btc.id, docID, deletedVersion) assert.Equal(t, `{}`, string(rev)) @@ -2312,7 +2312,7 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { defer btc.Close() const docID = "doc" - version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"A", "B"}}) + version := rt.PutDoc(docID, `{"channels": ["A", "B"]}`) changes := rt.WaitForChanges(1, "/{{.keyspace}}/_changes?since=0&revocations=true", "user", true) assert.Equal(t, "doc", changes.Results[0].ID) @@ -2321,7 +2321,7 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{"B"}}) + version = rt.UpdateDoc(docID, version, `{"channels": ["B"]}`) changes = rt.WaitForChanges(1, fmt.Sprintf("/{{.keyspace}}/_changes?since=%s&revocations=true", changes.Last_Seq), "user", true) assert.Equal(t, docID, changes.Results[0].ID) @@ -2330,9 +2330,9 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{}}) + version = rt.UpdateDoc(docID, version, `{"channels": []}`) const docMarker = "docmarker" - docMarkerVersion := rt.PutDocDirectly(docMarker, db.Body{"channels": []string{"!"}}) + docMarkerVersion := rt.PutDoc(docMarker, `{"channels": ["!"]}`) changes = rt.WaitForChanges(2, fmt.Sprintf("/{{.keyspace}}/_changes?since=%s&revocations=true", changes.Last_Seq), "user", true) assert.Equal(t, "doc", changes.Results[0].ID) @@ -2365,14 +2365,14 @@ func TestRemovedMessageWithAlternateAccess(t *testing.T) { } // TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication tests the following scenario: -// User has access to channel A and B -// Document rev 1 is in A and B -// Document rev 2 is in channel C -// Document rev 3 is in channel B -// User issues changes requests with since=0 for channel A -// Revocation should not be issued because the user currently has access to channel B, even though they didn't -// have access to the removal revision (rev 2). CBG-2277 - +// +// User has access to channel A and B +// Document rev 1 is in A and B +// Document rev 2 is in channel C +// Document rev 3 is in channel B +// User issues changes requests with since=0 for channel A +// Revocation should not be issued because the user currently has access to channel B, even though they didn't +// have access to the removal revision (rev 2). CBG-2277 func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testing.T) { defer db.SuspendSequenceBatching()() base.SetUpTestLogging(t, base.LevelDebug, base.KeyAll) @@ -2395,7 +2395,7 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi const ( docID = "doc" ) - version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"A", "B"}}) + version := rt.PutDoc(docID, `{"channels": ["A", "B"]}`) changes := rt.WaitForChanges(1, "/{{.keyspace}}/_changes?since=0&revocations=true", "user", true) assert.Equal(t, docID, changes.Results[0].ID) @@ -2404,7 +2404,7 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi btcRunner.StartOneshotPull(btc.id) _ = btcRunner.WaitForVersion(btc.id, docID, version) - version = rt.UpdateDocDirectly(docID, version, db.Body{"channels": []string{"C"}}) + version = rt.UpdateDoc(docID, version, `{"channels": ["C"]}`) rt.WaitForPendingChanges() // At this point changes should send revocation, as document isn't in any of the user's channels @@ -2417,7 +2417,7 @@ func TestRemovedMessageWithAlternateAccessAndChannelFilteredReplication(t *testi _ = rt.UpdateDoc(docID, version, `{"channels": ["B"]}`) markerID := "docmarker" - markerVersion := rt.PutDocDirectly(markerID, db.Body{"channels": []string{"A"}}) + markerVersion := rt.PutDoc(markerID, `{"channels": ["A"]}`) rt.WaitForPendingChanges() // Revocation should not be sent over blip, as document is now in user's channels - only marker document should be received @@ -2756,7 +2756,6 @@ func TestProcessRevIncrementsStat(t *testing.T) { require.EqualValues(t, 0, pullStats.HandlePutRevCount.Value()) const docID = "doc" - // need to have this return CV too, pending CBG-4751 version := remoteRT.CreateTestDoc(docID) assert.NoError(t, ar.Start(activeCtx)) @@ -2873,7 +2872,7 @@ func TestSendRevisionNoRevHandling(t *testing.T) { recievedNoRevs <- msg } - version := rt.PutDocDirectly(docName, db.Body{"foo": "bar"}) + version := rt.PutDoc(docName, `{"foo": "bar"}`) // Make the LeakyBucket return an error leakyDataStore.SetGetRawCallback(func(key string) error { @@ -2932,7 +2931,7 @@ func TestUnsubChanges(t *testing.T) { // Sub changes btcRunner.StartPull(btc.id) - doc1Version := rt.PutDocDirectly(doc1ID, db.Body{"key": "val1"}) + doc1Version := rt.PutDoc(doc1ID, `{"key": "val1"}`) _ = btcRunner.WaitForVersion(btc.id, doc1ID, doc1Version) activeReplStat := rt.GetDatabase().DbStats.CBLReplicationPull().NumPullReplActiveContinuous @@ -2943,7 +2942,7 @@ func TestUnsubChanges(t *testing.T) { base.RequireWaitForStat(t, activeReplStat.Value, 0) // Confirm no more changes are being sent - doc2Version := rt.PutDocDirectly(doc2ID, db.Body{"key": "val1"}) + doc2Version := rt.PutDoc(doc2ID, `{"key": "val1"}`) err := rt.WaitForConditionWithOptions(func() bool { _, found := btcRunner.GetVersion(btc.id, "doc2", doc2Version) return found @@ -3110,7 +3109,7 @@ func TestBlipRefreshUser(t *testing.T) { }) defer btc.Close() - version := rt.PutDocDirectly(docID, db.Body{"channels": []string{"chan1"}}) + version := rt.PutDoc(docID, `{"channels": ["chan1"]}`) // Start a regular one-shot pull btcRunner.StartPullSince(btc.id, BlipTesterPullOptions{Continuous: true, Since: "0"}) @@ -3164,8 +3163,8 @@ func TestImportInvalidSyncGetsNoRev(t *testing.T) { Username: bob, }) defer btc.Close() - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"some":"data", "channels":["ABC"]}`)) - version2 := rt.PutDocDirectly(docID2, JsonToMap(t, `{"some":"data", "channels":["ABC"]}`)) + version := rt.PutDoc(docID, `{"some":"data", "channels":["ABC"]}`) + version2 := rt.PutDoc(docID2, `{"some":"data", "channels":["ABC"]}`) rt.WaitForPendingChanges() // get changes resident in channel cache @@ -3336,7 +3335,7 @@ func TestBlipDatabaseClose(t *testing.T) { // put a doc, and make sure blip connection is established markerDoc := "markerDoc" - markerDocVersion := rt.PutDocDirectly(markerDoc, db.Body{"mark": "doc"}) + markerDocVersion := rt.PutDoc(markerDoc, `{"mark": "doc"}`) rt.WaitForPendingChanges() btcRunner.StartPull(btc.id) @@ -3553,7 +3552,7 @@ func TestBlipPullConflict(t *testing.T) { docID = "doc1" ) rt.CreateUser(alice, []string{"*"}) - sgVersion := rt.PutDocDirectly(docID, db.Body{"actor": "sg"}) + sgVersion := rt.PutDoc(docID, `{"actor": "sg"}`) opts := &BlipTesterClientOpts{ Username: alice, diff --git a/rest/blip_api_delta_sync_test.go b/rest/blip_api_delta_sync_test.go index c6c52a8dbc..ef4ad67b03 100644 --- a/rest/blip_api_delta_sync_test.go +++ b/rest/blip_api_delta_sync_test.go @@ -244,7 +244,7 @@ func TestBlipDeltaSyncPushPullNewAttachment(t *testing.T) { // Create doc1 rev 1-77d9041e49931ceef58a1eef5fd032e8 on SG with an attachment bodyText := `{"greetings":[{"hi": "alice"}],"_attachments":{"hello.txt":{"data":"aGVsbG8gd29ybGQ="}}}` // put doc directly needs to be here - version1 := rt.PutDocDirectly(docID, JsonToMap(t, bodyText)) + version1 := rt.PutDoc(docID, bodyText) data := btcRunner.WaitForVersion(btc.id, docID, version1) bodyTextExpected := `{"greetings":[{"hi":"alice"}],"_attachments":{"hello.txt":{"revpos":1,"length":11,"stub":true,"digest":"sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0="}}}` require.JSONEq(t, bodyTextExpected, string(data)) @@ -306,13 +306,13 @@ func TestBlipDeltaSyncNewAttachmentPull(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDocDirectly(doc1ID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + version := rt.PutDoc(doc1ID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) data := btcRunner.WaitForVersion(client.id, doc1ID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-10000d5ec533b29b117e60274b1e3653 on SG with the first attachment - version2 := rt.UpdateDocDirectly(doc1ID, version, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}], "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`)) + version2 := rt.UpdateDoc(doc1ID, version, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}], "_attachments": {"hello.txt": {"data":"aGVsbG8gd29ybGQ="}}}`) data = btcRunner.WaitForVersion(client.id, doc1ID, version2) require.Equal(t, db.AttachmentMap{ @@ -404,13 +404,13 @@ func TestBlipDeltaSyncPull(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + version := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version2 := rt.UpdateDocDirectly(docID, version, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`)) + version2 := rt.UpdateDoc(docID, version, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`) data = btcRunner.WaitForVersion(client.id, docID, version2) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(data)) @@ -472,7 +472,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { docID := "doc1" // create doc1 rev 1 - docVersion1 := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + docVersion1 := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) deltaSentCount := rt.GetDatabase().DbStats.DeltaSync().DeltasSent.Value() @@ -492,7 +492,7 @@ func TestBlipDeltaSyncPullResend(t *testing.T) { assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2 - docVersion2 := rt.UpdateDocDirectly(docID, docVersion1, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`)) + docVersion2 := rt.UpdateDoc(docID, docVersion1, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": 1234567890123}]}`) data = btcRunner.WaitForVersion(client.id, docID, docVersion2) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":1234567890123}]}`, string(data)) @@ -556,14 +556,14 @@ func TestBlipDeltaSyncPullRemoved(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-1513b53e2738671e634d9dd111f48de0 - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) + version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // create doc1 rev 2-ff91e11bc1fd12bbb4815a06571859a9 - version = rt.UpdateDocDirectly(docID, version, JsonToMap(t, `{"channels": ["private"], "greetings": [{"hello": "world!"}, {"hi": "bob"}]}`)) + version = rt.UpdateDoc(docID, version, `{"channels": ["private"], "greetings": [{"hello": "world!"}, {"hi": "bob"}]}`) data = btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"_removed":true}`, string(data)) @@ -631,13 +631,13 @@ func TestBlipDeltaSyncPullTombstoned(t *testing.T) { const docID = "doc1" // create doc1 rev 1-e89945d756a1d444fa212bffbbb31941 - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) + version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // tombstone doc1 at rev 2-2db70833630b396ef98a3ec75b3e90fc - version = rt.DeleteDocDirectly(docID, version) + version = rt.DeleteDoc(docID, version) data = btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{}`, string(data)) @@ -737,7 +737,7 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) { btcRunner.StartPull(client1.id) // create doc1 rev 1-e89945d756a1d444fa212bffbbb31941 - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`)) + version := rt.PutDoc(docID, `{"channels": ["public"], "greetings": [{"hello": "world!"}]}`) data := btcRunner.WaitForVersion(client1.id, docID, version) assert.Contains(t, string(data), `"channels":["public"]`) @@ -750,7 +750,7 @@ func TestBlipDeltaSyncPullTombstonedStarChan(t *testing.T) { assert.Contains(t, string(data), `"greetings":[{"hello":"world!"}]`) // tombstone doc1 at rev 2-2db70833630b396ef98a3ec75b3e90fc - version = rt.DeleteDocDirectly(docID, version) + version = rt.DeleteDoc(docID, version) data = btcRunner.WaitForVersion(client1.id, docID, version) assert.Equal(t, `{}`, string(data)) @@ -855,7 +855,7 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version1 := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + version1 := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) data := btcRunner.WaitForVersion(client.id, docID, version1) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) @@ -870,7 +870,7 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) // create doc1 rev 2-959f0e9ad32d84ff652fb91d8d0caa7e - version2 := rt.UpdateDocDirectly(docID, version1, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": "bob"}]}`)) + version2 := rt.UpdateDoc(docID, version1, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}, {"howdy": "bob"}]}`) data = btcRunner.WaitForVersion(client.id, docID, version2) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"},{"howdy":"bob"}]}`, string(data)) @@ -934,7 +934,7 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { // and checks that full body replication is still supported in CE. func TestBlipDeltaSyncPush(t *testing.T) { - t.Skip("TODO: CBG-4426 - DeleteDocDirectly does not support CV") + t.Skip("TODO: CBG-4832 - Failing because of broken legacy rev ID as deltaSrc logic in processRev") base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeySGTest, base.KeySyncMsg, base.KeySync) sgUseDeltas := base.IsEnterpriseEdition() rtConfig := RestTesterConfig{ @@ -962,7 +962,7 @@ func TestBlipDeltaSyncPush(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version := rt.PutDocDirectly(docID, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + version := rt.PutDoc(docID, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) data := btcRunner.WaitForVersion(client.id, docID, version) assert.Equal(t, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`, string(data)) @@ -1007,7 +1007,7 @@ func TestBlipDeltaSyncPush(t *testing.T) { assert.Equal(t, map[string]interface{}{"howdy": "bob"}, greetings[2]) // tombstone doc1 (gets rev 3-f3be6c85e0362153005dae6f08fc68bb) - deletedVersion := rt.DeleteDocDirectly(docID, newRev) + deletedVersion := rt.DeleteDoc(docID, newRev) data = btcRunner.WaitForVersion(client.id, docID, deletedVersion) assert.Equal(t, `{}`, string(data)) @@ -1092,10 +1092,8 @@ func TestBlipNonDeltaSyncPush(t *testing.T) { btcRunner.StartPush(client.id) rawBody := `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}` - var body db.Body - require.NoError(t, base.JSONUnmarshal([]byte(rawBody), &body)) // create doc1 rev 1-0335a345b6ffed05707ccc4cbc1b67f4 - version1 := rt.PutDocDirectly(docID, body) + version1 := rt.PutDoc(docID, rawBody) data := btcRunner.WaitForVersion(client.id, docID, version1) assert.Equal(t, rawBody, string(data)) diff --git a/rest/blip_api_replication_test.go b/rest/blip_api_replication_test.go index 06ed5e3150..4b5e836cd7 100644 --- a/rest/blip_api_replication_test.go +++ b/rest/blip_api_replication_test.go @@ -13,7 +13,6 @@ import ( "time" "github.com/couchbase/sync_gateway/base" - "github.com/couchbase/sync_gateway/db" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -50,7 +49,7 @@ func TestReplicationBroadcastTickerChange(t *testing.T) { btcRunner.StartPull(client.id) // create doc1 on SG and wait to replicate to client - versionDoc1 := rt.PutDocDirectly(docID, JsonToMap(t, `{"test": "value"}`)) + versionDoc1 := rt.PutDoc(docID, `{"test": "value"}`) btcRunner.WaitForVersion(client.id, docID, versionDoc1) // alter sync data of this doc to artificially create skipped sequences @@ -76,11 +75,11 @@ func TestReplicationBroadcastTickerChange(t *testing.T) { }, time.Second*10, time.Millisecond*100) // assert new change added still replicates to client - versionDoc2 := rt.PutDocDirectly(docID2, JsonToMap(t, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`)) + versionDoc2 := rt.PutDoc(docID2, `{"greetings": [{"hello": "world!"}, {"hi": "alice"}]}`) btcRunner.WaitForVersion(client.id, docID2, versionDoc2) // update doc1 that will trigger unused seq release to clear skipped and assert that update is received - versionDoc1 = rt.UpdateDocDirectly(docID, versionDoc1, JsonToMap(t, `{"test": "new value"}`)) + versionDoc1 = rt.UpdateDoc(docID, versionDoc1, `{"test": "new value"}`) btcRunner.WaitForVersion(client.id, docID, versionDoc1) // assert skipped is cleared and skipped sequence broadcast is not sent @@ -115,8 +114,7 @@ func TestBlipClientPushAndPullReplication(t *testing.T) { btcRunner.StartPush(client.id) // create doc1 on SG - docBody := db.Body{"greetings": []map[string]interface{}{{"hello": "world!"}, {"hi": "alice"}}} - version := rt.PutDocDirectly(docID, docBody) + version := rt.PutDoc(docID, `{"greetings":[{"hello":"world!"},{"hi":"alice"}]}`) // wait for doc on client data := btcRunner.WaitForVersion(client.id, docID, version) diff --git a/rest/blip_channel_filter_test.go b/rest/blip_channel_filter_test.go index c67654e9d8..902e2a6b16 100644 --- a/rest/blip_channel_filter_test.go +++ b/rest/blip_channel_filter_test.go @@ -52,7 +52,7 @@ func TestChannelFilterRemovalFromChannel(t *testing.T) { client := btcRunner.SingleCollection(btc.id) const docID = "doc1" - version1 := rt.PutDocDirectly("doc1", JsonToMap(t, `{"channels":["A"]}`)) + version1 := rt.PutDoc("doc1", `{"channels":["A"]}`) rt.WaitForPendingChanges() response := rt.SendUserRequest("GET", "/{{.keyspace}}/_changes?since=0&channels=A&include_docs=true", "", "alice") @@ -73,9 +73,9 @@ func TestChannelFilterRemovalFromChannel(t *testing.T) { btcRunner.WaitForVersion(btc.id, docID, version1) // remove channel A from doc1 - version2 := rt.UpdateDocDirectly(docID, version1, JsonToMap(t, `{"channels":["B"]}`)) + version2 := rt.UpdateDoc(docID, version1, `{"channels":["B"]}`) markerDocID := "marker" - markerDocVersion := rt.PutDocDirectly(markerDocID, JsonToMap(t, `{"channels":["A"]}`)) + markerDocVersion := rt.PutDoc(markerDocID, `{"channels":["A"]}`) rt.WaitForPendingChanges() // alice will see doc1 rev2 with body diff --git a/rest/blip_legacy_revid_test.go b/rest/blip_legacy_revid_test.go index 21e7dc4ea7..b85fc07905 100644 --- a/rest/blip_legacy_revid_test.go +++ b/rest/blip_legacy_revid_test.go @@ -256,7 +256,7 @@ func TestProcessLegacyRev(t *testing.T) { collection, _ := rt.GetSingleTestDatabaseCollection() // add doc to SGW - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID // Send another rev of same doc @@ -325,7 +325,7 @@ func TestProcessRevWithLegacyHistory(t *testing.T) { ) // 1. CBL sends rev=1010@CBL1, history=1-abc when SGW has current rev 1-abc (document underwent an update before being pushed to SGW) - docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + docVersion := rt.PutDoc(docID, `{"test": "doc"}`) rev1ID := docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -345,7 +345,7 @@ func TestProcessRevWithLegacyHistory(t *testing.T) { assert.NotNil(t, bucketDoc.History[rev1ID]) // 2. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 1-abc (document underwent multiple p2p updates before being pushed to SGW) - docVersion = rt.PutDocDirectly(docID2, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID2, `{"test": "doc"}`) rev1ID = docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -366,7 +366,7 @@ func TestProcessRevWithLegacyHistory(t *testing.T) { assert.NotNil(t, bucketDoc.History[rev1ID]) // 3. CBL sends rev=1010@CBL1, history=1000@CBL2,2-abc,1-abc when SGW has current rev 1-abc (document underwent multiple legacy and p2p updates before being pushed to SGW) - docVersion = rt.PutDocDirectly(docID3, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID3, `{"test": "doc"}`) rev1ID = docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -401,9 +401,9 @@ func TestProcessRevWithLegacyHistory(t *testing.T) { // 5. CBL sends rev=1010@CBL1, history=2-abc and SGW has 1000@CBL2, 2-abc // although HLV's are in conflict, this should pass conflict check as local current rev is parent of incoming rev - docVersion = rt.PutDocDirectly(docID5, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID5, `{"test": "doc"}`) - docVersion = rt.UpdateDocDirectly(docID5, docVersion, db.Body{"some": "update"}) + docVersion = rt.UpdateDoc(docID5, docVersion, `{"some": "update"}`) version := docVersion.CV.Value rev2ID := docVersion.RevTreeID pushedRev := db.Version{ @@ -429,7 +429,7 @@ func TestProcessRevWithLegacyHistory(t *testing.T) { // - a pre 4.0 client pulling this doc on one shot replication // - then this doc being updated a couple of times on client before client gets upgraded to 4.0 // - after the upgrade client updates it again and pushes to SGW - docVersion = rt.PutDocDirectly(docID6, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID6, `{"test": "doc"}`) rev1ID = docVersion.RevTreeID pushedRev = db.Version{ @@ -475,13 +475,13 @@ func TestProcessRevWithLegacyHistoryConflict(t *testing.T) { ) // 1. conflicting changes with legacy rev on both sides of communication (no upgrade of doc at all) - docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + docVersion := rt.PutDoc(docID, `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly(docID, docVersion, db.Body{"some": "update"}) + docVersion = rt.UpdateDoc(docID, docVersion, `{"some": "update"}`) rev2ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly(docID, docVersion, db.Body{"some": "update2"}) + docVersion = rt.UpdateDoc(docID, docVersion, `{"some": "update2"}`) // remove hlv here to simulate a legacy rev require.NoError(t, ds.RemoveXattrs(base.TestCtx(t), docID, []string{base.VvXattrName}, docVersion.CV.Value)) @@ -493,13 +493,13 @@ func TestProcessRevWithLegacyHistoryConflict(t *testing.T) { require.ErrorContains(t, err, "Document revision conflict") // 2. same as above but not having the rev be legacy on SGW side (don't remove the hlv) - docVersion = rt.PutDocDirectly(docID2, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID2, `{"test": "doc"}`) rev1ID = docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly(docID2, docVersion, db.Body{"some": "update"}) + docVersion = rt.UpdateDoc(docID2, docVersion, `{"some": "update"}`) rev2ID = docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly(docID2, docVersion, db.Body{"some": "update2"}) + docVersion = rt.UpdateDoc(docID2, docVersion, `{"some": "update2"}`) history = []string{rev2ID, rev1ID} sent, _, _, err = bt.SendRevWithHistory(docID2, "3-abc", history, []byte(`{"key": "val"}`), blip.Properties{}) @@ -507,10 +507,10 @@ func TestProcessRevWithLegacyHistoryConflict(t *testing.T) { require.ErrorContains(t, err, "Document revision conflict") // 3. CBL sends rev=1010@CBL1, history=1000@CBL2,1-abc when SGW has current rev 2-abc (document underwent multiple p2p updates before being pushed to SGW) - docVersion = rt.PutDocDirectly(docID3, db.Body{"test": "doc"}) + docVersion = rt.PutDoc(docID3, `{"test": "doc"}`) rev1ID = docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly(docID3, docVersion, db.Body{"some": "update"}) + docVersion = rt.UpdateDoc(docID3, docVersion, `{"some": "update"}`) // remove hlv here to simulate a legacy rev require.NoError(t, ds.RemoveXattrs(base.TestCtx(t), docID3, []string{base.VvXattrName}, docVersion.CV.Value)) @@ -539,10 +539,10 @@ func TestChangesResponseLegacyRev(t *testing.T) { defer bt.Close() rt := bt.restTester - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion2 := rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + docVersion2 := rt.UpdateDoc("doc1", docVersion, `{"test": "update"}`) // wait for pending change to avoid flakes where changes feed didn't pick up this change rt.WaitForPendingChanges() receivedChangesRequestWg := sync.WaitGroup{} @@ -640,7 +640,7 @@ func TestChangesResponseWithHLVInHistory(t *testing.T) { rt := bt.restTester collection, ctx := rt.GetSingleTestDatabaseCollection() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID newDoc, _, err := collection.GetDocWithXattrs(ctx, "doc1", db.DocUnmarshalAll) @@ -743,7 +743,7 @@ func TestCBLHasPreUpgradeMutationThatHasNotBeenReplicated(t *testing.T) { collection, ctx := rt.GetSingleTestDatabaseCollection() ds := rt.GetSingleDataStore() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -779,10 +779,10 @@ func TestCBLHasOfPreUpgradeMutationThatSGWAlreadyKnows(t *testing.T) { collection, ctx := rt.GetSingleTestDatabaseCollection() ds := rt.GetSingleDataStore() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update"}`) rev2ID := docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -817,10 +817,10 @@ func TestPushOfPostUpgradeMutationThatHasCommonAncestorToSGWVersion(t *testing.T collection, ctx := rt.GetSingleTestDatabaseCollection() ds := rt.GetSingleDataStore() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update"}`) rev2ID := docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -855,13 +855,13 @@ func TestPushDocConflictBetweenPreUpgradeCBLMutationAndPreUpgradeSGWMutation(t * collection, ctx := rt.GetSingleTestDatabaseCollection() ds := rt.GetSingleDataStore() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update"}`) rev2ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update1"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update1"}`) rev3ID := docVersion.RevTreeID // remove hlv here to simulate a legacy rev @@ -896,13 +896,13 @@ func TestPushDocConflictBetweenPreUpgradeCBLMutationAndPostUpgradeSGWMutation(t rt := bt.restTester collection, ctx := rt.GetSingleTestDatabaseCollection() - docVersion := rt.PutDocDirectly("doc1", db.Body{"test": "doc"}) + docVersion := rt.PutDoc("doc1", `{"test": "doc"}`) rev1ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update"}`) rev2ID := docVersion.RevTreeID - docVersion = rt.UpdateDocDirectly("doc1", docVersion, db.Body{"test": "update1"}) + docVersion = rt.UpdateDoc("doc1", docVersion, `{"test": "update1"}`) rev3ID := docVersion.RevTreeID // send rev 3-def @@ -939,7 +939,7 @@ func TestConflictBetweenPostUpgradeCBLMutationAndPostUpgradeSGWMutation(t *testi docID2 = "doc2" ) - docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + docVersion := rt.PutDoc(docID, `{"test": "doc"}`) rev1ID := docVersion.RevTreeID history := []string{rev1ID} @@ -955,7 +955,7 @@ func TestConflictBetweenPostUpgradeCBLMutationAndPostUpgradeSGWMutation(t *testi assert.Equal(t, docVersion.CV.Value, bucketDoc.HLV.PreviousVersions[docVersion.CV.SourceID]) // conflict rev - docVersion = rt.PutDocDirectly(docID2, db.Body{"some": "doc"}) + docVersion = rt.PutDoc(docID2, `{"some": "doc"}`) rev1ID = docVersion.RevTreeID history = []string{"1-abc"} @@ -982,7 +982,7 @@ func TestLegacyRevNotInConflict(t *testing.T) { collection, ctx := rt.GetSingleTestDatabaseCollection() const docID = "doc1" - docVersion := rt.PutDocDirectly(docID, db.Body{"test": "doc"}) + docVersion := rt.PutDoc(docID, `{"test": "doc"}`) rev1ID := docVersion.RevTreeID // have two history entries, 1 rev from a different CBL and 1 legacy rev, should generate conflict diff --git a/rest/changestest/changes_api_test.go b/rest/changestest/changes_api_test.go index 3f2c3bac16..0a29a2d581 100644 --- a/rest/changestest/changes_api_test.go +++ b/rest/changestest/changes_api_test.go @@ -1809,11 +1809,7 @@ func TestChangesIncludeDocs(t *testing.T) { revid = prunedRevId var cvs []string for i := 0; i < 5; i++ { - body := db.Body{ - "type": "pruned", - "channels": []string{"gamma"}, - } - docVersion := rt.UpdateDocDirectly("doc_pruned", db.DocVersion{RevTreeID: revid}, body) + docVersion := rt.UpdateDoc("doc_pruned", db.DocVersion{RevTreeID: revid}, `{"type": "pruned", "channels":["gamma"]}`) revid = docVersion.RevTreeID cvs = append(cvs, docVersion.CV.String()) } diff --git a/rest/doc_api_test.go b/rest/doc_api_test.go index 7666188a6a..87daebaa41 100644 --- a/rest/doc_api_test.go +++ b/rest/doc_api_test.go @@ -180,7 +180,7 @@ func TestGetDocWithCV(t *testing.T) { defer rt.Close() docID := "doc1" - docVersion := rt.PutDocDirectly(docID, db.Body{"foo": "bar"}) + docVersion := rt.PutDoc(docID, `{"foo": "bar"}`) testCases := []struct { name string url string @@ -231,8 +231,8 @@ func TestBulkGetWithCV(t *testing.T) { doc1ID := "doc1" doc2ID := "doc2" - doc1Version := rt.PutDocDirectly(doc1ID, db.Body{"foo": "bar"}) - doc2Version := rt.PutDocDirectly(doc2ID, db.Body{"foo": "baz"}) + doc1Version := rt.PutDoc(doc1ID, `{"foo": "bar"}`) + doc2Version := rt.PutDoc(doc2ID, `{"foo": "baz"}`) testCases := []struct { name string url string diff --git a/rest/importtest/import_test.go b/rest/importtest/import_test.go index 05a8c3cb4f..6175ba08c6 100644 --- a/rest/importtest/import_test.go +++ b/rest/importtest/import_test.go @@ -209,12 +209,12 @@ func TestXattrImportOldDocRevHistory(t *testing.T) { // 1. Create revision with history docID := t.Name() - version := rt.PutDocDirectly(docID, rest.JsonToMap(t, `{"val":-1}`)) + version := rt.PutDoc(docID, `{"val":-1}`) cv := version.CV.String() collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() for i := 0; i < 10; i++ { - version = rt.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"val":%d}`, i))) + version = rt.UpdateDoc(docID, version, fmt.Sprintf(`{"val":%d}`, i)) // Purge old revision JSON to simulate expiry, and to verify import doesn't attempt multiple retrievals // Revs are backed up by hash of CV now, switch to fetch by this till CBG-3748 (backwards compatibility for revID) cvHash := base.Crc32cHashString([]byte(cv)) diff --git a/rest/replicatortest/replicator_revtree_test.go b/rest/replicatortest/replicator_revtree_test.go index f0235283e6..e0dc81a3b4 100644 --- a/rest/replicatortest/replicator_revtree_test.go +++ b/rest/replicatortest/replicator_revtree_test.go @@ -61,11 +61,11 @@ func TestActiveReplicatorRevTreeReconciliation(t *testing.T) { docID := "doc1_" + tc.name var version rest.DocVersion if tc.replicationType == db.ActiveReplicatorTypePull { - version = rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","channels":["alice"]}`)) + version = rt2.PutDoc(docID, `{"source":"rt2","channels":["alice"]}`) docHistoryList = append(docHistoryList, version.RevTreeID) rt2.WaitForPendingChanges() } else { - version = rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + version = rt1.PutDoc(docID, `{"source":"rt1","channels":["alice"]}`) docHistoryList = append(docHistoryList, version.RevTreeID) rt1.WaitForPendingChanges() } @@ -110,13 +110,13 @@ func TestActiveReplicatorRevTreeReconciliation(t *testing.T) { if tc.replicationType == db.ActiveReplicatorTypePull { for i := 0; i < 10; i++ { - version = rt2.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"source":"rt2","channels":["alice"], "version": "%d"}`, i))) + version = rt2.UpdateDoc(docID, version, fmt.Sprintf(`{"source":"rt2","channels":["alice"], "version": "%d"}`, i)) docHistoryList = append(docHistoryList, version.RevTreeID) } rt2.WaitForPendingChanges() } else { for i := 0; i < 10; i++ { - version = rt1.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"source":"rt1","channels":["alice"], "version": "%d"}`, i))) + version = rt1.UpdateDoc(docID, version, fmt.Sprintf(`{"source":"rt1","channels":["alice"], "version": "%d"}`, i)) docHistoryList = append(docHistoryList, version.RevTreeID) } rt1.WaitForPendingChanges() @@ -204,10 +204,10 @@ func TestActiveReplicatorRevtreeLargeDiffInSize(t *testing.T) { docID := "doc1" var version rest.DocVersion if tc.replicationType == db.ActiveReplicatorTypePull { - version = rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + version = rt2.PutDoc(docID, `{"source":"rt1","channels":["alice"]}`) rt2.WaitForPendingChanges() } else { - version = rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","channels":["alice"]}`)) + version = rt1.PutDoc(docID, `{"source":"rt2","channels":["alice"]}`) rt1.WaitForPendingChanges() } @@ -254,12 +254,12 @@ func TestActiveReplicatorRevtreeLargeDiffInSize(t *testing.T) { // update doc hundreds of times to create a large diff in rev tree versions if tc.replicationType == db.ActiveReplicatorTypePull { for i := 0; i < 200; i++ { - version = rt2.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"source":"rt2","channels":["alice"], "version": "%d"}`, i))) + version = rt2.UpdateDoc(docID, version, fmt.Sprintf(`{"source":"rt2","channels":["alice"], "version": "%d"}`, i)) } rt2.WaitForPendingChanges() } else { for i := 0; i < 200; i++ { - version = rt1.UpdateDocDirectly(docID, version, rest.JsonToMap(t, fmt.Sprintf(`{"source":"rt1","channels":["alice"], "version": "%d"}`, i))) + version = rt1.UpdateDoc(docID, version, fmt.Sprintf(`{"source":"rt1","channels":["alice"], "version": "%d"}`, i)) } rt1.WaitForPendingChanges() } diff --git a/rest/replicatortest/replicator_test.go b/rest/replicatortest/replicator_test.go index 0b9d50796f..48d0f3e217 100644 --- a/rest/replicatortest/replicator_test.go +++ b/rest/replicatortest/replicator_test.go @@ -2072,7 +2072,7 @@ func TestActiveReplicatorPullBasic(t *testing.T) { rt2.CreateUser(username, []string{username}) docID := t.Name() + "rt2doc1" - version := rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","channels":["`+username+`"]}`)) + version := rt2.PutDoc(docID, `{"source":"rt2","channels":["`+username+`"]}`) rt2collection, rt2ctx := rt2.GetSingleTestDatabaseCollection() remoteDoc, err := rt2collection.GetDocument(rt2ctx, docID, db.DocUnmarshalAll) @@ -2186,7 +2186,7 @@ func TestActiveReplicatorPullSkippedSequence(t *testing.T) { docIDPrefix := t.Name() + "rt2doc" docID1 := docIDPrefix + "1" - doc1Version := rt2.PutDocDirectly(docID1, rest.JsonToMap(t, `{"source":"rt2","channels":["`+username+`"]}`)) + doc1Version := rt2.PutDoc(docID1, `{"source":"rt2","channels":["`+username+`"]}`) rt2.WaitForPendingChanges() // Start the replicator (implicit connect) @@ -2574,7 +2574,7 @@ func TestActiveReplicatorPullAttachments(t *testing.T) { attachment := `"_attachments":{"hi.txt":{"data":"aGk=","content_type":"text/plain"}}` docID := t.Name() + "rt2doc1" - version := rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","doc_num":1,`+attachment+`,"channels":["alice"]}`)) + version := rt2.PutDoc(docID, `{"source":"rt2","doc_num":1,`+attachment+`,"channels":["alice"]}`) // Active rt1 := rest.NewRestTester(t, nil) @@ -2617,7 +2617,7 @@ func TestActiveReplicatorPullAttachments(t *testing.T) { assert.Equal(t, int64(1), ar.Pull.GetStats().GetAttachment.Value()) docID = t.Name() + "rt2doc2" - version = rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","doc_num":2,`+attachment+`,"channels":["alice"]}`)) + version = rt2.PutDoc(docID, `{"source":"rt2","doc_num":2,`+attachment+`,"channels":["alice"]}`) // wait for the new document written to rt2 to arrive at rt1 changesResults = rt1.WaitForChanges(2, "/{{.keyspace}}/_changes?since=0", "", true) @@ -3205,7 +3205,7 @@ func TestActiveReplicatorPushBasic(t *testing.T) { ctx1 := rt1.Context() docID := t.Name() + "rt1doc1" - version := rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + version := rt1.PutDoc(docID, `{"source":"rt1","channels":["alice"]}`) rt1collection, rt1ctx := rt1.GetSingleTestDatabaseCollection() localDoc, err := rt1collection.GetDocument(rt1ctx, docID, db.DocUnmarshalAll) @@ -3274,7 +3274,7 @@ func TestActiveReplicatorPushAttachments(t *testing.T) { attachment := `"_attachments":{"hi.txt":{"data":"aGk=","content_type":"text/plain"}}` docID := t.Name() + "rt1doc1" - version := rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","doc_num":1,`+attachment+`,"channels":["alice"]}`)) + version := rt1.PutDoc(docID, `{"source":"rt1","doc_num":1,`+attachment+`,"channels":["alice"]}`) ar, err := db.NewActiveReplicator(ctx1, &db.ActiveReplicatorConfig{ ID: t.Name(), @@ -3313,7 +3313,7 @@ func TestActiveReplicatorPushAttachments(t *testing.T) { assert.Equal(t, int64(1), ar.Push.GetStats().HandleGetAttachment.Value()) docID = t.Name() + "rt1doc2" - version = rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","doc_num":2,`+attachment+`,"channels":["alice"]}`)) + version = rt1.PutDoc(docID, `{"source":"rt1","doc_num":2,`+attachment+`,"channels":["alice"]}`) // wait for the new document written to rt1 to arrive at rt2 changesResults = rt2.WaitForChanges(2, "/{{.keyspace}}/_changes?since=0", "", true) @@ -3691,7 +3691,7 @@ func TestActiveReplicatorPushOneshot(t *testing.T) { defer rt1.Close() docID := t.Name() + "rt1doc1" - version := rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + version := rt1.PutDoc(docID, `{"source":"rt1","channels":["alice"]}`) rt1collection, rt1ctx := rt1.GetSingleTestDatabaseCollection() localDoc, err := rt1collection.GetDocument(rt1ctx, docID, db.DocUnmarshalAll) @@ -3755,7 +3755,7 @@ func TestActiveReplicatorPullTombstone(t *testing.T) { rt2.CreateUser(username, []string{username}) docID := t.Name() + "rt2doc1" - version := rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","channels":["alice"]}`)) + version := rt2.PutDoc(docID, `{"source":"rt2","channels":["alice"]}`) // Active @@ -3799,7 +3799,7 @@ func TestActiveReplicatorPullTombstone(t *testing.T) { assert.Equal(t, "rt2", body["source"]) // Tombstone the doc in rt2 - deletedVersion := rt2.DeleteDocDirectly(docID, version) + deletedVersion := rt2.DeleteDoc(docID, version) // wait for the tombstone written to rt2 to arrive at rt1 changesResults = rt1.WaitForChanges(1, "/{{.keyspace}}/_changes?since="+strconv.FormatUint(doc.Sequence, 10), "", true) @@ -3837,7 +3837,7 @@ func TestActiveReplicatorPullPurgeOnRemoval(t *testing.T) { rt2.CreateUser(username, []string{username}) docID := t.Name() + "rt2doc1" - version := rt2.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt2","channels":["alice"]}`)) + version := rt2.PutDoc(docID, `{"source":"rt2","channels":["alice"]}`) // Active rt1 := rest.NewRestTester(t, nil) @@ -4384,7 +4384,7 @@ func TestActiveReplicatorPushBasicWithInsecureSkipVerifyEnabled(t *testing.T) { ctx1 := rt1.Context() docID := t.Name() + "rt1doc1" - version := rt1.PutDocDirectly(docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + version := rt1.PutDoc(docID, `{"source":"rt1","channels":["alice"]}`) // Make rt2 listen on an actual HTTP port, so it can receive the blipsync request from rt1. srv := httptest.NewTLSServer(rt2.TestPublicHandler()) @@ -5108,7 +5108,7 @@ func TestActiveReplicatorIgnoreNoConflicts(t *testing.T) { ctx1 := rt1.Context() rt1docID := t.Name() + "rt1doc1" - rt1Version := rt1.PutDocDirectly(rt1docID, rest.JsonToMap(t, `{"source":"rt1","channels":["alice"]}`)) + rt1Version := rt1.PutDoc(rt1docID, `{"source":"rt1","channels":["alice"]}`) // Make rt2 listen on an actual HTTP port, so it can receive the blipsync request from rt1. srv := httptest.NewServer(rt2.TestPublicHandler()) @@ -5160,7 +5160,7 @@ func TestActiveReplicatorIgnoreNoConflicts(t *testing.T) { // write a doc on rt2 ... rt2docID := t.Name() + "rt2doc1" - rt2Version := rt2.PutDocDirectly(rt2docID, rest.JsonToMap(t, `{"source":"rt2","channels":["alice"]}`)) + rt2Version := rt2.PutDoc(rt2docID, `{"source":"rt2","channels":["alice"]}`) // ... and wait to arrive at rt1 changesResults = rt1.WaitForChanges(2, "/{{.keyspace}}/_changes?since=0", "", true) @@ -6876,7 +6876,7 @@ func TestReplicatorDoNotSendDeltaWhenSrcIsTombstone(t *testing.T) { activeCtx := activeRT.Context() // Create a document // - version := activeRT.PutDocDirectly("test", rest.JsonToMap(t, `{"field1":"f1_1","field2":"f2_1"}`)) + version := activeRT.PutDoc("test", `{"field1":"f1_1","field2":"f2_1"}`) activeRT.WaitForVersion("test", version) // Set-up replicator // @@ -6901,14 +6901,14 @@ func TestReplicatorDoNotSendDeltaWhenSrcIsTombstone(t *testing.T) { passiveRT.WaitForVersion("test", version) // Delete active document - deletedVersion := activeRT.DeleteDocDirectly("test", version) + deletedVersion := activeRT.DeleteDoc("test", version) // Assert that the tombstone is replicated to passive // Get revision 2 on passive peer to assert it has been (a) replicated and (b) deleted passiveRT.WaitForTombstone("test", deletedVersion) // Resurrect tombstoned document - resurrectedVersion := activeRT.UpdateDocDirectly("test", deletedVersion, rest.JsonToMap(t, `{"field2":"f2_2"}`)) + resurrectedVersion := activeRT.UpdateDoc("test", deletedVersion, `{"field2":"f2_2"}`) // Replicate resurrection to passive passiveRT.WaitForVersion("test", resurrectedVersion) @@ -6958,7 +6958,7 @@ func TestUnprocessableDeltas(t *testing.T) { activeCtx := activeRT.Context() // Create a document // - version := activeRT.PutDocDirectly("test", rest.JsonToMap(t, `{"field1":"f1_1","field2":"f2_1"}`)) + version := activeRT.PutDoc("test", `{"field1":"f1_1","field2":"f2_1"}`) activeRT.WaitForVersion("test", version) ar, err := db.NewActiveReplicator(activeCtx, &db.ActiveReplicatorConfig{ @@ -6984,7 +6984,7 @@ func TestUnprocessableDeltas(t *testing.T) { assert.NoError(t, ar.Stop()) // Make 2nd revision - version2 := activeRT.UpdateDocDirectly("test", version, rest.JsonToMap(t, `{"field1":"f1_2","field2":"f2_2"}`)) + version2 := activeRT.UpdateDoc("test", version, `{"field1":"f1_2","field2":"f2_2"}`) activeRT.WaitForPendingChanges() passiveRTCollection, passiveRTCtx := passiveRT.GetSingleTestDatabaseCollection() @@ -7024,15 +7024,15 @@ func TestReplicatorIgnoreRemovalBodies(t *testing.T) { docID := t.Name() // Create the docs // // Doc rev 1 - version1 := activeRT.PutDocDirectly(docID, rest.JsonToMap(t, `{"key":"12","channels": ["rev1chan"]}`)) + version1 := activeRT.PutDoc(docID, `{"key":"12","channels": ["rev1chan"]}`) activeRT.WaitForVersion(docID, version1) // doc rev 2 - version2 := activeRT.UpdateDocDirectly(docID, version1, rest.JsonToMap(t, `{"key":"12","channels":["rev2+3chan"]}`)) + version2 := activeRT.UpdateDoc(docID, version1, `{"key":"12","channels":["rev2+3chan"]}`) activeRT.WaitForVersion(docID, version2) // Doc rev 3 - version3 := activeRT.UpdateDocDirectly(docID, version2, rest.JsonToMap(t, `{"key":"3","channels":["rev2+3chan"]}`)) + version3 := activeRT.UpdateDoc(docID, version2, `{"key":"3","channels":["rev2+3chan"]}`) activeRT.WaitForVersion(docID, version3) activeRT.GetDatabase().FlushRevisionCacheForTest() @@ -7278,7 +7278,7 @@ func TestReplicatorDeprecatedCredentials(t *testing.T) { require.NoError(t, err) docID := "test" - version := activeRT.PutDocDirectly(docID, rest.JsonToMap(t, `{"prop":true}`)) + version := activeRT.PutDoc(docID, `{"prop":true}`) replConfig := ` { diff --git a/rest/revocation_test.go b/rest/revocation_test.go index 108a68e862..9605f5e1dc 100644 --- a/rest/revocation_test.go +++ b/rest/revocation_test.go @@ -2222,7 +2222,7 @@ func TestRevocationMessage(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDocDirectly("doc", db.Body{"channels": "A"}) + version := rt.PutDoc("doc", `{"channels": "A"}`) // Start pull rt.WaitForPendingChanges() @@ -2235,10 +2235,10 @@ func TestRevocationMessage(t *testing.T) { revocationTester.removeRole("user", "foo") const doc1ID = "doc1" - version = rt.PutDocDirectly(doc1ID, db.Body{"channels": "!"}) + version = rt.PutDoc(doc1ID, `{"channels": "!"}`) revocationTester.fillToSeq(10) - version = rt.UpdateDocDirectly(doc1ID, version, db.Body{}) + version = rt.UpdateDoc(doc1ID, version, `{}`) // Start a pull since 5 to receive revocation and removal rt.WaitForPendingChanges() @@ -2331,7 +2331,7 @@ func TestRevocationNoRev(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDocDirectly(docID, db.Body{"channels": "A"}) + version := rt.PutDoc(docID, `{"channels": "A"}`) rt.WaitForPendingChanges() firstOneShotSinceSeq := rt.GetDocumentSequence("doc") @@ -2344,9 +2344,9 @@ func TestRevocationNoRev(t *testing.T) { // Remove role from user revocationTester.removeRole("user", "foo") - _ = rt.UpdateDocDirectly(docID, version, db.Body{"channels": "A", "val": "mutate"}) + _ = rt.UpdateDoc(docID, version, `{"channels": "A", "val": "mutate"}`) - waitMarkerVersion := rt.PutDocDirectly(waitMarkerID, db.Body{"channels": "!"}) + waitMarkerVersion := rt.PutDoc(waitMarkerID, `{"channels": "!"}`) rt.WaitForPendingChanges() lastSeqStr := strconv.FormatUint(firstOneShotSinceSeq, 10) @@ -2410,7 +2410,7 @@ func TestRevocationGetSyncDataError(t *testing.T) { // Skip to seq 4 and then create doc in channel A revocationTester.fillToSeq(4) - version := rt.PutDocDirectly(docID, db.Body{"channels": "A"}) + version := rt.PutDoc(docID, `{"channels": "A"}`) // OneShot pull to grab doc rt.WaitForPendingChanges() @@ -2424,9 +2424,9 @@ func TestRevocationGetSyncDataError(t *testing.T) { // Remove role from user revocationTester.removeRole("user", "foo") - _ = rt.UpdateDocDirectly(docID, version, db.Body{"channels": "A", "val": "mutate"}) + _ = rt.UpdateDoc(docID, version, `{"channels": "A", "val": "mutate"}`) - waitMarkerVersion := rt.PutDocDirectly(waitMarkerID, db.Body{"channels": "!"}) + waitMarkerVersion := rt.PutDoc(waitMarkerID, `{"channels": "!"}`) rt.WaitForPendingChanges() rt.WaitForPendingChanges() diff --git a/rest/utilities_testing.go b/rest/utilities_testing.go index af99175b8f..a2dd750cba 100644 --- a/rest/utilities_testing.go +++ b/rest/utilities_testing.go @@ -2384,8 +2384,7 @@ func RequireDocRevTreeEqual(t *testing.T, expected, actual DocVersion) { // RequireDocVersionNotEqual calls t.Fail if two document versions are equal. func RequireDocVersionNotEqual(t *testing.T, expected, actual DocVersion) { - // CBG-4751: should be able to uncomment this line once cv is included in write response - //require.NotEqual(t, expected.CV.String(), actual.CV.String(), "Versions mismatch. Expected: %v, Actual: %v", expected, actual) + require.NotEqual(t, expected.CV.String(), actual.CV.String(), "Versions mismatch. Expected: %v, Actual: %v", expected, actual) require.NotEqual(t, expected.RevTreeID, actual.RevTreeID, "Versions mismatch. Expected: %v, Actual: %v", expected.RevTreeID, actual.RevTreeID) } diff --git a/rest/utilities_testing_resttester.go b/rest/utilities_testing_resttester.go index 2d140bef82..98b602bd4b 100644 --- a/rest/utilities_testing_resttester.go +++ b/rest/utilities_testing_resttester.go @@ -82,7 +82,7 @@ func (rt *RestTester) GetDocVersion(docID string, version DocVersion) db.Body { if !version.CV.IsEmpty() { occValue = version.CV.String() } - rawResponse := rt.SendAdminRequest("GET", "/{{.keyspace}}/"+docID+"?rev="+occValue, "") + rawResponse := rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/"+docID+"?rev="+occValue, "") RequireStatus(rt.TB(), rawResponse, http.StatusOK) var body db.Body require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) @@ -91,7 +91,7 @@ func (rt *RestTester) GetDocVersion(docID string, version DocVersion) db.Body { // GetDocByRev returns the doc body for the given docID and Rev. If the document is not found, t.Fail will be called. func (rt *RestTester) GetDocByRev(docID, revTreeID string) db.Body { - rawResponse := rt.SendAdminRequest("GET", fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, revTreeID), "") + rawResponse := rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/%s/%s?rev=%s", rt.GetSingleKeyspace(), docID, revTreeID), "") RequireStatus(rt.TB(), rawResponse, http.StatusOK) var body db.Body require.NoError(rt.TB(), base.JSONUnmarshal(rawResponse.Body.Bytes(), &body)) @@ -100,20 +100,27 @@ func (rt *RestTester) GetDocByRev(docID, revTreeID string) db.Body { // CreateTestDoc creates a document with an arbitrary body. func (rt *RestTester) CreateTestDoc(docid string) DocVersion { - response := rt.SendAdminRequest("PUT", fmt.Sprintf("/%s/%s", rt.GetSingleKeyspace(), docid), `{"prop":true}`) + response := rt.SendAdminRequest(http.MethodPut, fmt.Sprintf("/%s/%s", rt.GetSingleKeyspace(), docid), `{"prop":true}`) RequireStatus(rt.TB(), response, 201) return DocVersionFromPutResponse(rt.TB(), response) } // PutDoc will upsert the document with a given contents. -func (rt *RestTester) PutDoc(docID string, body string) DocVersion { - rawResponse := rt.SendAdminRequest("PUT", fmt.Sprintf("/%s/%s", rt.GetSingleKeyspace(), docID), body) +func (rt *RestTester) PutDoc(docID, body string) DocVersion { + rawResponse := rt.SendAdminRequest(http.MethodPut, fmt.Sprintf("/%s/%s", rt.GetSingleKeyspace(), docID), body) + RequireStatus(rt.TB(), rawResponse, 201) + return DocVersionFromPutResponse(rt.TB(), rawResponse) +} + +// PutDocInCollection will upsert the document with a given contents in the given collection. +func (rt *RestTester) PutDocInCollection(collection, docID, body string) DocVersion { + rawResponse := rt.SendAdminRequest(http.MethodPut, fmt.Sprintf("/%s.%s/%s", rt.GetDatabase().Name, collection, docID), body) RequireStatus(rt.TB(), rawResponse, 201) return DocVersionFromPutResponse(rt.TB(), rawResponse) } // UpdateDocRev updates a document at a specific revision and returns the new version. Deprecated for UpdateDoc. -func (rt *RestTester) UpdateDocRev(docID, revID string, body string) string { +func (rt *RestTester) UpdateDocRev(docID, revID, body string) string { version := rt.UpdateDoc(docID, DocVersion{RevTreeID: revID}, body) return version.RevTreeID } @@ -483,40 +490,6 @@ func (rt *RestTester) RequireDbOnline() { require.Equal(rt.TB(), "Online", body["state"].(string)) } -// TEMPORARY HELPER METHODS FOR BLIP TEST CLIENT RUNNER -func (rt *RestTester) PutDocDirectly(docID string, body db.Body) DocVersion { - collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() - rev, doc, err := collection.Put(ctx, docID, body) - require.NoError(rt.TB(), err) - return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} -} - -func (rt *RestTester) UpdateDocDirectly(docID string, version DocVersion, body db.Body) DocVersion { - collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() - body[db.BodyId] = docID - body[db.BodyRev] = version.RevTreeID - rev, doc, err := collection.Put(ctx, docID, body) - require.NoError(rt.TB(), err) - return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} -} - -func (rt *RestTester) DeleteDocDirectly(docID string, version DocVersion) DocVersion { - collection, ctx := rt.GetSingleTestDatabaseCollectionWithUser() - rev, doc, err := collection.DeleteDoc(ctx, docID, version) - require.NoError(rt.TB(), err) - return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} -} - -func (rt *RestTester) PutDocDirectlyInCollection(collection *db.DatabaseCollection, docID string, body db.Body) DocVersion { - dbUser := &db.DatabaseCollectionWithUser{ - DatabaseCollection: collection, - } - ctx := base.UserLogCtx(collection.AddCollectionContext(rt.Context()), "gotest", base.UserDomainBuiltin, nil) - rev, doc, err := dbUser.Put(ctx, docID, body) - require.NoError(rt.TB(), err) - return DocVersion{RevTreeID: rev, CV: db.Version{SourceID: doc.HLV.SourceID, Value: doc.HLV.Version}} -} - // PutDocWithAttachment will upsert the document with a given contents and attachments. func (rt *RestTester) PutDocWithAttachment(docID string, body string, attachmentName, attachmentBody string) DocVersion { // create new body with a 1.x style inline attachment body like `{"_attachments": {"camera.txt": {"data": "Q2Fub24gRU9TIDVEIE1hcmsgSVY="}}}`. @@ -528,7 +501,9 @@ func (rt *RestTester) PutDocWithAttachment(docID string, body string, attachment rawBody[db.BodyAttachments] = map[string]any{ attachmentName: map[string]any{"data": attachmentBody}, } - return rt.PutDocDirectly(docID, rawBody) + newBody, err := base.JSONMarshal(rawBody) + require.NoError(rt.TB(), err) + return rt.PutDoc(docID, string(newBody)) } type RawDocResponse struct { From d62cb9e2a276a1a62c59835290a4f4decdd041d0 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Thu, 21 Aug 2025 17:52:12 +0100 Subject: [PATCH 12/14] Handle legacy rev IDs as deltaSrc even when running in HLV mode --- db/blip_handler.go | 4 ++-- rest/blip_api_delta_sync_test.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/db/blip_handler.go b/db/blip_handler.go index 117b78fa14..ea8917c21e 100644 --- a/db/blip_handler.go +++ b/db/blip_handler.go @@ -1141,10 +1141,10 @@ func (bh *blipHandler) processRev(rq *blip.Message, stats *processRevStats) (err // due to no-conflict write restriction, but we still need to enforce security here to prevent leaking data about previous // revisions to malicious actors (in the scenario where that user has write but not read access). var deltaSrcRev DocumentRevision - if bh.useHLV() { + if bh.useHLV() && !base.IsRevTreeID(deltaSrcRevID) { deltaSrcVersion, parseErr := ParseVersion(deltaSrcRevID) if parseErr != nil { - return base.HTTPErrorf(http.StatusUnprocessableEntity, "Unable to parse version for delta source for doc %s, error: %v", base.UD(docID), err) + return base.HTTPErrorf(http.StatusUnprocessableEntity, "Unable to parse version for delta source for doc %s, error: %v", base.UD(docID), parseErr) } deltaSrcRev, err = bh.collection.GetCV(bh.loggingCtx, docID, &deltaSrcVersion, false) } else { diff --git a/rest/blip_api_delta_sync_test.go b/rest/blip_api_delta_sync_test.go index ef4ad67b03..e4791f0f95 100644 --- a/rest/blip_api_delta_sync_test.go +++ b/rest/blip_api_delta_sync_test.go @@ -933,8 +933,6 @@ func TestBlipDeltaSyncPullRevCache(t *testing.T) { // TestBlipDeltaSyncPush tests that a simple push replication handles deltas in EE, // and checks that full body replication is still supported in CE. func TestBlipDeltaSyncPush(t *testing.T) { - - t.Skip("TODO: CBG-4832 - Failing because of broken legacy rev ID as deltaSrc logic in processRev") base.SetUpTestLogging(t, base.LevelDebug, base.KeyCRUD, base.KeySGTest, base.KeySyncMsg, base.KeySync) sgUseDeltas := base.IsEnterpriseEdition() rtConfig := RestTesterConfig{ From e48e44978fbbe9189b369354e269e8a0a36c6b54 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 21 Aug 2025 18:32:14 -0400 Subject: [PATCH 13/14] remove misplaced files --- integration-test/certs/ca.pem | 18 ---- integration-test/certs/chain.pem | 18 ---- .../certs/couchbase_cluster_spec.json | 1 - integration-test/certs/pkey.key | 27 ----- integration-test/certs/sg.key | 27 ----- integration-test/certs/sg.pem | 18 ---- integration-test/docker-compose.yml | 42 -------- integration-test/service-install-tests.sh | 42 -------- integration-test/service-test.sh | 70 ------------- integration-test/start_server.sh | 99 ------------------- 10 files changed, 362 deletions(-) delete mode 100644 integration-test/certs/ca.pem delete mode 100644 integration-test/certs/chain.pem delete mode 100644 integration-test/certs/couchbase_cluster_spec.json delete mode 100644 integration-test/certs/pkey.key delete mode 100644 integration-test/certs/sg.key delete mode 100644 integration-test/certs/sg.pem delete mode 100644 integration-test/docker-compose.yml delete mode 100755 integration-test/service-install-tests.sh delete mode 100755 integration-test/service-test.sh delete mode 100755 integration-test/start_server.sh diff --git a/integration-test/certs/ca.pem b/integration-test/certs/ca.pem deleted file mode 100644 index a3bfee7d81..0000000000 --- a/integration-test/certs/ca.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC+DCCAeCgAwIBAgIIGFXj4C4gXT0wDQYJKoZIhvcNAQELBQAwGjEYMBYGA1UE -AxMPU0cgVGVzdCBSb290IENBMB4XDTI1MDcyNTE5Mjk0MVoXDTM1MDcyNDE5Mjk0 -MVowGjEYMBYGA1UEAxMPU0cgVGVzdCBSb290IENBMIIBIjANBgkqhkiG9w0BAQEF -AAOCAQ8AMIIBCgKCAQEA4U33BEuhKvXh9+bRzg+eJp7idzqy5GhF9iQDdSHcnSmx -4avXZUgnJkNEsvLUSNQHqg2Wegzf7toGIucKuVkxO91jBnQdPjwUPd+56wEZbS6/ -w8Ac03ViDxBevp8YnH4KCoshBbjN6LbIv7mbSFrNoUaXXKETxxBQKYnSUhq+F7sJ -t5IlfVl0xna7dVBj1q1JbwAwJWkL1JJgcvHhr4y3WT/PL2ppRN2qQwcS450KHDTG -Qw8aPzYCCbg0/61u9ViE0pmZLk49xSrk4XeB05zboj5aGtRgGbcmq1DiaCNVSNF8 -SbOGIubMu5L8xhnlvymhwOdlDWwfJ9/s/7ts777DzwIDAQABo0IwQDAOBgNVHQ8B -Af8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUGw1VGQSZLsWZniYn -RuI7aewQHOIwDQYJKoZIhvcNAQELBQADggEBAIjFyqKhz5FZlL/30NHlUqBPJVf8 -2uB+pqkBSXhmPuKnMCIMmhCbLZmICWpN/8QGlyWN+wuwqrKO0RCscNPFN9jgisW3 -FbaXuYdBVVNb23/zEraSM86icbcdwx98/lVoPYboFy++q1xYfCFHFU6V61zUtcG6 -AMydnjxaJ9yeaaItbEOlIJWjOusORIy/ZPn/PVElwg3DbRCbm4JFM3jy0i74DuEo -2ADo7LVJGdg/ute7rpimwurZZMK8XmPB4mbd0omNCT2gm7Q21VAfVJOAlowKRmrk -zxYCOFDfw5fvlrNGVDaYbZl284VlJffzPtQoF9JscTsY2O1k8sjwO/RQIn0= ------END CERTIFICATE----- diff --git a/integration-test/certs/chain.pem b/integration-test/certs/chain.pem deleted file mode 100644 index d6b560863f..0000000000 --- a/integration-test/certs/chain.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC8jCCAdqgAwIBAgIIGFXj4DPOZ/EwDQYJKoZIhvcNAQELBQAwGjEYMBYGA1UE -AxMPU0cgVGVzdCBSb290IENBMB4XDTI1MDcyNTE5Mjk0MloXDTM1MDcyNDE5Mjk0 -MVowGTEXMBUGA1UEAxMOY2JzLW5vZGUubG9jYWwwggEiMA0GCSqGSIb3DQEBAQUA -A4IBDwAwggEKAoIBAQDTJ0aJw82YV6teaJHwT4fXEgPNPU1vvb5siQxQ2+ODfr8K -miNKx8gYWnMy37YkIkRQzWizUVn04pWkbhg2V/r60USf2Asccpfgi2D465+fxgbd -2KvLyK5rxVeWSY3v/134y25fDdnMOSKuIt3OSjoMCZ5pAV9ZXnNh5Ua+JumBkA4M -1dIHECRCUeCb4cjZtXP3vujxdTGRoPOCVlRl6Ex3B/LEE+7fMEdEYnQSwTEOzMG6 -aMFDh80+M2C5jb/Qk/1ynUNCSyAEkNszY9e32mfCRiCQtWPg66D//nVz5Jl+9Sp8 -RiOeQ30GfWogCJfahOokhVlD2qF8oCHu5MJIob4jAgMBAAGjPTA7MA4GA1UdDwEB -/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATAUBgNVHREEDTALgglsb2NhbGhv -c3QwDQYJKoZIhvcNAQELBQADggEBAEC6yJkfj0Du0gj4rMtNj6J1zBMBNxFOqKBq -Wq+3+OcLgxBhOraxLNifQ9wM0AZM7t9wwKtboxmQ+wZtv7iCN6IhwrRqCVrYIuFq -VnNtoC+qJ7ewZRAW13idXwxYjkRzZlR1Lz12ar8Ia7+h78nwTuB/4VTLnJ/O5U8r -owBTMNIgCHA1cJPtYtGg72PrQ6ibfEEPYoXGCswG3rLRG5HN6AZsgcekGZlkOt1f -EvDBmxGKucPbZnI5pZBNeaUpDZL6LsqwqDMEhyvCqIDwLTmwP2wO5jffjJ8jgzd3 -G8yfljsvmeG6pGqbEiryTGKiHReYG/LC5zcJ/jqMeN50Q0hfcfE= ------END CERTIFICATE----- diff --git a/integration-test/certs/couchbase_cluster_spec.json b/integration-test/certs/couchbase_cluster_spec.json deleted file mode 100644 index 6c96326ca3..0000000000 --- a/integration-test/certs/couchbase_cluster_spec.json +++ /dev/null @@ -1 +0,0 @@ -{"server":"couchbases://localhost","certpath":"/home/tcolvin/repos/sync_gateway/integration-test/certs/sg.pem","keypath":"/home/tcolvin/repos/sync_gateway/integration-test/certs/sg.key","cacertpath":"/home/tcolvin/repos/sync_gateway/integration-test/certs/ca.pem"} \ No newline at end of file diff --git a/integration-test/certs/pkey.key b/integration-test/certs/pkey.key deleted file mode 100644 index 737fc9dc3f..0000000000 --- a/integration-test/certs/pkey.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEogIBAAKCAQEA0ydGicPNmFerXmiR8E+H1xIDzT1Nb72+bIkMUNvjg36/Cpoj -SsfIGFpzMt+2JCJEUM1os1FZ9OKVpG4YNlf6+tFEn9gLHHKX4Itg+Oufn8YG3dir -y8iua8VXlkmN7/9d+MtuXw3ZzDkiriLdzko6DAmeaQFfWV5zYeVGvibpgZAODNXS -BxAkQlHgm+HI2bVz977o8XUxkaDzglZUZehMdwfyxBPu3zBHRGJ0EsExDszBumjB -Q4fNPjNguY2/0JP9cp1DQksgBJDbM2PXt9pnwkYgkLVj4Oug//51c+SZfvUqfEYj -nkN9Bn1qIAiX2oTqJIVZQ9qhfKAh7uTCSKG+IwIDAQABAoIBAD+TeUkco+glKWt1 -F8/f2lo2yd8/gHPPESlTwFoOQvwCKxpRm6O18HjorvvX4NsTWDduCYLIUUoK+Rx3 -q6GdLuvbG4r3PS01EaahwLJiG387XDDqvptOkrnPQtZ00iA7Zvt0oQhMvtGfOGJv -DBLDRaP/N2uNZrydVCdbJcg2JiOEhK9JMq/8l0QSbnVmz2Ub3fbKicQgxLO2452v -X0WycHrMDQk/z3uZCXUQrrVxp1wVJun14e4dE78JoGFDbA7ARJKomsJIFOtGPJVb -Nw5ykcxAziiLaD51df1Wt5bW9SLEZ3PokbxTgewJ+CtXOir8MUNksSbNKzY5+CjP -NhCm5u0CgYEA7ukpC4dfJhwJAJMvxF8F687vRRySOCjNULoW0+V3KWufO7sJ2d2T -paa/+4dxiK+OvOh3Y6ib3O6Q0LlzR67mmmkLJ3C07GRdGAGGULPAjQezSMbBqEbW -BbXXh8X57ylb5ghDCerHto+7lejK+xCrx4xNdKYJI8EgWXmzj2dBfQUCgYEA4kHV -WAvcNEyFhPtCABG9TWTQSAqxoAxPtC9rjdm/uMGEmOTYYkwXsonUKZKd9kFo1Nys -sRVVTGpDysCeuuq/6lOC0giEyDhJ7FAq6LsAA7OUXKpmTgKDGOiLCHZjrsS1vCSP -3iJxX646EkOcvZ1+V9zeSMqmkW12CPBlm17ydwcCgYAf+V5/538pd4kQ5aH38wu0 -0n4dTsSW9Yb87drOQyCej4PBF7gqy0fOXLHG9QqR04UT7TzFPrSVbew9swQlrNe6 -BKL0hVYBaTE4XEPgmx4DAevRqqASaGCOZRbSWgGoK23cLHDka+KMoVHmr0AzN7j1 -vOZE3U/N1DQDJZGNeLFADQKBgC2c0AG8AlyYwKIadSfGa79af5LGdSCq2raciLZE -G56HhM+98tF+PZjEqHzpDedDMHsZMcdRYazSD3CkfFt6T85Rn6HwDbS/hEebscrR -SCN25IX55D39y6gN2VmPZHErPuf7BvXlQ63iVdqwvryLL5lO8ZEDKalPw+fxbspv -zbmpAoGALyBKB1BDfvnYANPOHwY9zqGUghSDZcjVBssdR+mOmoRIkBz34eglnTLF -89LsO9OOfnJ/aP5h2BaEQXLA4sCjR82oR7G7nA+YNSWMtq4wlngDxRdBQAuXshkA -xVV5sYpY7dx2TcaYB3IcLab9LmZz+LzOc60xHIQYTPbr3UIVL3Y= ------END RSA PRIVATE KEY----- diff --git a/integration-test/certs/sg.key b/integration-test/certs/sg.key deleted file mode 100644 index 50709c191a..0000000000 --- a/integration-test/certs/sg.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzCS0RkU5/WnEeqicKuZAkkGpTpqp8w9wATq/M4e7TI+NNZ6w -lI0kwQUBiE25i15XRYCYBH42LaPMbwk3CvnWVIAIRqB4oTPmt1j2YW78bLDd+hfD -r7yAkQ5SmrKSHTD4k8KOsy+IUWFkGIWZd7RFESf1AM5j2Db8OfPbjI6hh1UJEavw -tiHqKHQWXZZ/0DOL4B3ehWluVB1ssTMRidmfxKtjh3KaOXZAmm47TV04zo+hFlAp -T9uVt5US401q5etSw3Omfln7lu6KpmqfBlCAmbwT5LmAT6AlMYeRI+yxtrMdSs0o -OI1QQtSJoePBuY7yIXJKBVImkcZbpw849Pjl2wIDAQABAoIBADAQPDmHNv4JFu9i -H1KiX3WP7BLLq1PEwLQpZrb1MA34hmCneh+fk5W2XgP/eL3telKs0h3MsWjRdeJ2 -ovT8mY/PjSNDyOL7W0izs16BSQE2Ky0kxzfrA8IjQyOVA33H996iIgLiIBA5A94a -JmXelZxScga8kRlo6L2kQn63XiSEXvA18x61gFMeaLP9ybshtqFHp8Qkqxzbt/Vt -cWF0Np1uYA70fu1Jw5DJavN/eG+cIFHKzKNTaqjVvgq2s72IEsEwX5AcCQXe9YZ1 -PICOAri6/DfZL939cAIZAgm0xtSfjKxRDdDaprwqWddISVGs2+cd40bNBSQPnRJ4 -HC2RKBECgYEA1NmHA78llxJN1Zi0kxmc89yjzKwj8M82sjbyJO5+afzF4cG7o9MW -9q4oMNwEnl3+otZmyswxVsaelj/N6x6PQxybJnltCitcpakY3tidGDkF/vK37+Ev -uhV2Bw3yiSoPbQn8rgom0sMa7GLjamWhqv0FIiXtjZ2XKhN6MLxe4g8CgYEA9YdV -0GJRyNbs4lN9uCMz2Qi3AgYxXHHGS8NUnvqg/a0AqIpGvhfrEiJ6Ex8MJx0rn3ue -70CrX9IQSbrQDsou5UHak/yi/phVNuSFCMjRlQ/RcPqLNVO0tQwO0PN0SvgcJrct -8vc1850bPGTCrcSTwnKKYfaMlTgx9k/WfVv8G3UCgYEAs4cKPxnBbevNZKSZYh1P -acynB+IFqn5MRwLbOFVEoMbIbQNH7gUEsGnykktxRdZICTbHmrOhxexfJKGKYI71 -DQkav9fZJaOvUDcROB0CW8T1DrXQeO65n72sQIT+Fb05J6It0unTFx/jHJDH+hzg -wGULKGNPO4w5TQ8CmAq6CPcCgYEAqxwjEPPEPWyTb/NxtSdLVeC98boxIlTkNh6t -1ZGjKscro0minY37tAq0+qhzhrrMkPvNOr3d8Qxrb4aywuvinME1PFcfnMC6+mNt -1z5k2TZJ5yukYoiwclAx7ysLi8e3jr+wVRg10E5YEdHC3ukVdLjwee8h8EhWgWsI -dxro6pUCgYBpSEWCQKN0yPUN/7SnA3mAK0ogTPT1dWGCKooKO3NcAXnRQuq81DCE -Q+1ZQ5YN+TxSkJ8qEi1a/oNITsjc9IjlgElaVLeKMxMnZRxSkBWkje9fC67drYfv -T6GL+SITgpAfIlXnDV4lHhZcwKe1GQYqfCn4+FePlhj0MBjTaFbwgg== ------END RSA PRIVATE KEY----- diff --git a/integration-test/certs/sg.pem b/integration-test/certs/sg.pem deleted file mode 100644 index a2de99e9ff..0000000000 --- a/integration-test/certs/sg.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC2zCCAcOgAwIBAgIIGFXj4DcH9z4wDQYJKoZIhvcNAQELBQAwGjEYMBYGA1UE -AxMPU0cgVGVzdCBSb290IENBMB4XDTI1MDcyNTE5Mjk0MloXDTI1MDcyNzE5Mjk0 -MlowGDEWMBQGA1UEAxMNQWRtaW5pc3RyYXRvcjCCASIwDQYJKoZIhvcNAQEBBQAD -ggEPADCCAQoCggEBAMwktEZFOf1pxHqonCrmQJJBqU6aqfMPcAE6vzOHu0yPjTWe -sJSNJMEFAYhNuYteV0WAmAR+Ni2jzG8JNwr51lSACEageKEz5rdY9mFu/Gyw3foX -w6+8gJEOUpqykh0w+JPCjrMviFFhZBiFmXe0RREn9QDOY9g2/Dnz24yOoYdVCRGr -8LYh6ih0Fl2Wf9Azi+Ad3oVpblQdbLEzEYnZn8SrY4dymjl2QJpuO01dOM6PoRZQ -KU/blbeVEuNNauXrUsNzpn5Z+5buiqZqnwZQgJm8E+S5gE+gJTGHkSPssbazHUrN -KDiNUELUiaHjwbmO8iFySgVSJpHGW6cPOPT45dsCAwEAAaMnMCUwDgYDVR0PAQH/ -BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMCMA0GCSqGSIb3DQEBCwUAA4IBAQCd -WrdJkTuVMR8pEXIgmf4d0Z+4BLVmaX4AUaE60ffrp8+e585vxg94j1V67CdSciLE -J/5p3Z9u4jmNcJMhNNEytyVZlmyct/QntdjCLoGLtX1HlOoTbKNaYVbyrGKVCZqN -CfTy/4KTNA4/uJUyA3KRD5ClOnQhWYBEwODr/s/2E44iEiFBgxpw83QtEhGdf/a3 -JuZEapf/oeVTJYW1X92a724s20P3PdvGt0CqUqV4XTPitiNSG9Kq2EwoinuPFM51 -xm3lze4cOWyvzADY/hP8d61PmMBvvl3YUAM4+gO7tiS1W4+wX7ynRke+8BXAluVG -UughJ9Jf6yHBtGi46xY8 ------END CERTIFICATE----- diff --git a/integration-test/docker-compose.yml b/integration-test/docker-compose.yml deleted file mode 100644 index dc59f90205..0000000000 --- a/integration-test/docker-compose.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2023-Present Couchbase, Inc. -# -# Use of this software is governed by the Business Source License included -# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -# in that file, in accordance with the Business Source License, use of this -# software will be governed by the Apache License, Version 2.0, included in -# the file licenses/APL2.txt. - -services: - couchbase: - container_name: couchbase - image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.6.5}" - ports: - - 8091:8091 - - 8092:8092 - - 8093:8093 - - 8094:8094 - - 8095:8095 - - 8096:8096 - - 8097:8097 - - 9102:9102 - - 9123:9123 - - 11207:11207 - - 11210:11210 - - 11211:11211 - - 18091:18091 - - 18092:18092 - - 18093:18093 - - 18094:18094 - - 18095:18095 - - 18096:18096 - - 18097:18097 - - 19102:19102 - volumes: - - "${DOCKER_CBS_ROOT_DIR:-.}/cbs:/root" - - "${WORKSPACE_ROOT:-.}:/workspace" - couchbase-replica1: - container_name: couchbase-replica1 - image: "couchbase/${COUCHBASE_DOCKER_IMAGE_NAME:-server:enterprise-7.6.5}" - couchbase-replica2: - container_name: couchbase-replica2 - image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.6.5}" diff --git a/integration-test/service-install-tests.sh b/integration-test/service-install-tests.sh deleted file mode 100755 index 1440cc373b..0000000000 --- a/integration-test/service-install-tests.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright 2024-Present Couchbase, Inc. -# -# Use of this software is governed by the Business Source License included -# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -# in that file, in accordance with the Business Source License, use of this -# software will be governed by the Apache License, Version 2.0, included in -# the file licenses/APL2.txt. - -# This file is used by github CI or locally and runs a subset of test scripts to validate the service installation done by the package managers. This is intended to be run from Linux or Mac. - -set -eux -o pipefail - -IMAGES=( - "almalinux:9" - "amazonlinux:2" - "amazonlinux:2023" - "debian:10" - "debian:11" - "debian:12" - "redhat/ubi8" - "redhat/ubi9" - "rockylinux:9" - "ubuntu:20.04" - "ubuntu:22.04" - "ubuntu:24.04" - - # not technically supported - "oraclelinux:9" -) - -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") -SYNC_GATEWAY_DIR=$(realpath ${SCRIPT_DIR}/..) - -if [ "$(uname)" == "Darwin" ]; then - sudo ${SYNC_GATEWAY_DIR}/integration-test/service-test.sh -fi - -for IMAGE in "${IMAGES[@]}"; do - echo "Running tests for ${IMAGE}" - docker run --mount src=${SYNC_GATEWAY_DIR},target=/sync_gateway,type=bind ${IMAGE} /bin/bash -c "/sync_gateway/integration-test/service-test.sh" -done diff --git a/integration-test/service-test.sh b/integration-test/service-test.sh deleted file mode 100755 index 3320595672..0000000000 --- a/integration-test/service-test.sh +++ /dev/null @@ -1,70 +0,0 @@ -#/bin/sh - -# Copyright 2024-Present Couchbase, Inc. -# -# Use of this software is governed by the Business Source License included -# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -# in that file, in accordance with the Business Source License, use of this -# software will be governed by the Apache License, Version 2.0, included in -# the file licenses/APL2.txt. - -# This code is intneded to be run from within a docker container of a specific platform and runs a subset of the service scripts. The full service can not be validated since systemd does not work in docker contains. This is intended to run in /bin/sh to test dash environments on debian systems. - -set -eux -o pipefail - -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") - -cd ${SCRIPT_DIR}/../service - -./sync_gateway_service_install.sh --servicecmd - -# /etc/os-release doesn't exist on Darwin -if [ -f /etc/os-release ]; then - . /etc/os-release - case ${ID} in - amzn) - yum install -y shadow-utils systemd - ;; - esac - - groupadd -r sync_gateway - useradd -g sync_gateway sync_gateway - - # bash would support export -f for a systemctl wrapper, but dash does not support exporting aliases or functions - - mkdir -p /tmp/systemctl_wrapper - - cat << 'EOF' > /tmp/systemctl_wrapper/systemctl -#!/bin/bash - -set -eu -o pipefail - -case ${1:-} in -start) - echo "No-op systemctl start in docker, since we're not running systemd" - ;; -stop) - echo "No-op systemctl stop in docker, since we're not running systemd" - ;; -*) - echo "Running systemctl $@" - command /usr/bin/systemctl "$@" - ;; -esac -EOF - - chmod +x /tmp/systemctl_wrapper/systemctl - - export PATH=/tmp/systemctl_wrapper:$PATH -fi - -./sync_gateway_service_install.sh -./sync_gateway_service_upgrade.sh -./sync_gateway_service_uninstall.sh - -# test again with runas option -./sync_gateway_service_install.sh --runas=root -./sync_gateway_service_upgrade.sh -./sync_gateway_service_uninstall.sh - -echo "Successful service test" diff --git a/integration-test/start_server.sh b/integration-test/start_server.sh deleted file mode 100755 index aba7b5a20f..0000000000 --- a/integration-test/start_server.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# Copyright 2023-Present Couchbase, Inc. -# -# Use of this software is governed by the Business Source License included -# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified -# in that file, in accordance with the Business Source License, use of this -# software will be governed by the Apache License, Version 2.0, included in -# the file licenses/APL2.txt. - -set -eux -o pipefail - -function usage() { - echo "Usage: $0 [-m] [-h] containername" -} - -if [ $# -gt 2 ]; then - echo "Expected maximally two arguments" - exit 1 -fi - -while [[ $# -gt 0 ]]; do - key="$1" - case $key in - -m | --multi-node) - MULTI_NODE=true - shift - ;; - -h | --help) - echo "Usage: $0 [-m] [-h] containername" - exit 1 - ;; - --non-dockerhub) - DOCKERHUB=false - shift - ;; - *) - COUCHBASE_DOCKER_IMAGE_NAME="$1" - shift - ;; - esac -done - -WORKSPACE_ROOT="$(pwd)" -DOCKER_CBS_ROOT_DIR="$(pwd)" -if [ "${CBS_ROOT_DIR:-}" != "" ]; then - DOCKER_CBS_ROOT_DIR="${CBS_ROOT_DIR}" -fi - -set +e -AMAZON_LINUX_2=$(grep 'Amazon Linux 2"' /etc/os-release) -set -e -if [[ -n "${AMAZON_LINUX_2}" ]]; then - DOCKER_COMPOSE="docker-compose" # use docker-compose v1 for Jenkins AWS Linux 2 -else - DOCKER_COMPOSE="docker compose" -fi -cd -- "${BASH_SOURCE%/*}/" -${DOCKER_COMPOSE} down || true -export SG_TEST_COUCHBASE_SERVER_DOCKER_NAME=couchbase -# Start CBS -docker stop ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true -docker rm ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true -# --volume: Makes and mounts a CBS folder for storing a CBCollect if needed - -# use dockerhub if no registry is specified, allows for pre-release images from alternative registries -if [[ ! "${COUCHBASE_DOCKER_IMAGE_NAME}" =~ ghcr.io/* && "${DOCKERHUB:-}" != "false" ]]; then - COUCHBASE_DOCKER_IMAGE_NAME="couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME}" -fi - -if [ "${MULTI_NODE:-}" == "true" ]; then - ${DOCKER_COMPOSE} up -d --force-recreate --renew-anon-volumes --remove-orphans -else - # single node - docker run --rm -d --name ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} --volume "${WORKSPACE_ROOT}:/workspace" -p 8091-8097:8091-8097 -p 9102:9102 -p 9123:9123 -p 11207:11207 -p 11210:11210 -p 11211:11211 -p 18091-18097:18091-18097 -p 19102:19102 "${COUCHBASE_DOCKER_IMAGE_NAME}" -fi - -# Test to see if Couchbase Server is up -# Each retry min wait 5s, max 10s. Retry 20 times with exponential backoff (delay 0), fail at 120s -curl --retry-all-errors --connect-timeout 5 --max-time 10 --retry 20 --retry-delay 0 --retry-max-time 120 'http://127.0.0.1:8091' - -# Set up CBS - -docker exec couchbase couchbase-cli cluster-init --cluster-username Administrator --cluster-password password --cluster-ramsize 3072 --cluster-index-ramsize 3072 --cluster-fts-ramsize 256 --services data,index,query -docker exec couchbase couchbase-cli setting-index --cluster couchbase://localhost --username Administrator --password password --index-threads 4 --index-log-level verbose --index-max-rollback-points 10 --index-storage-setting default --index-memory-snapshot-interval 150 --index-stable-snapshot-interval 40000 - -curl -u Administrator:password -v -X POST http://127.0.0.1:8091/node/controller/rename -d 'hostname=127.0.0.1' - -if [ "${MULTI_NODE:-}" == "true" ]; then - REPLICA1_NAME=couchbase-replica1 - REPLICA2_NAME=couchbase-replica2 - CLI_ARGS=(-c couchbase://couchbase -u Administrator -p password) - docker exec ${REPLICA1_NAME} couchbase-cli node-init "${CLI_ARGS[@]}" - docker exec ${REPLICA2_NAME} couchbase-cli node-init "${CLI_ARGS[@]}" - REPLICA1_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA1_NAME} | jq -r 'first(.[]) | .IPAddress') - REPLICA2_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA2_NAME} | jq -r 'first(.[]) | .IPAddress') - docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA2_IP" --server-add-username Administrator --server-add-password password --services data,index,query - docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA1_IP" --server-add-username Administrator --server-add-password password --services data,index,query - docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli rebalance "${CLI_ARGS[@]}" -fi From d4abe108a47bb20e556af64fd749c7fb80994537 Mon Sep 17 00:00:00 2001 From: Tor Colvin Date: Thu, 21 Aug 2025 18:32:41 -0400 Subject: [PATCH 14/14] fix back to main --- integration-test/docker-compose.yml | 42 ++++++++++ integration-test/service-install-tests.sh | 42 ++++++++++ integration-test/service-test.sh | 70 ++++++++++++++++ integration-test/start_server.sh | 99 +++++++++++++++++++++++ 4 files changed, 253 insertions(+) create mode 100644 integration-test/docker-compose.yml create mode 100755 integration-test/service-install-tests.sh create mode 100755 integration-test/service-test.sh create mode 100755 integration-test/start_server.sh diff --git a/integration-test/docker-compose.yml b/integration-test/docker-compose.yml new file mode 100644 index 0000000000..dc59f90205 --- /dev/null +++ b/integration-test/docker-compose.yml @@ -0,0 +1,42 @@ +# Copyright 2023-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +services: + couchbase: + container_name: couchbase + image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.6.5}" + ports: + - 8091:8091 + - 8092:8092 + - 8093:8093 + - 8094:8094 + - 8095:8095 + - 8096:8096 + - 8097:8097 + - 9102:9102 + - 9123:9123 + - 11207:11207 + - 11210:11210 + - 11211:11211 + - 18091:18091 + - 18092:18092 + - 18093:18093 + - 18094:18094 + - 18095:18095 + - 18096:18096 + - 18097:18097 + - 19102:19102 + volumes: + - "${DOCKER_CBS_ROOT_DIR:-.}/cbs:/root" + - "${WORKSPACE_ROOT:-.}:/workspace" + couchbase-replica1: + container_name: couchbase-replica1 + image: "couchbase/${COUCHBASE_DOCKER_IMAGE_NAME:-server:enterprise-7.6.5}" + couchbase-replica2: + container_name: couchbase-replica2 + image: "couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME:-enterprise-7.6.5}" diff --git a/integration-test/service-install-tests.sh b/integration-test/service-install-tests.sh new file mode 100755 index 0000000000..1440cc373b --- /dev/null +++ b/integration-test/service-install-tests.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# Copyright 2024-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +# This file is used by github CI or locally and runs a subset of test scripts to validate the service installation done by the package managers. This is intended to be run from Linux or Mac. + +set -eux -o pipefail + +IMAGES=( + "almalinux:9" + "amazonlinux:2" + "amazonlinux:2023" + "debian:10" + "debian:11" + "debian:12" + "redhat/ubi8" + "redhat/ubi9" + "rockylinux:9" + "ubuntu:20.04" + "ubuntu:22.04" + "ubuntu:24.04" + + # not technically supported + "oraclelinux:9" +) + +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") +SYNC_GATEWAY_DIR=$(realpath ${SCRIPT_DIR}/..) + +if [ "$(uname)" == "Darwin" ]; then + sudo ${SYNC_GATEWAY_DIR}/integration-test/service-test.sh +fi + +for IMAGE in "${IMAGES[@]}"; do + echo "Running tests for ${IMAGE}" + docker run --mount src=${SYNC_GATEWAY_DIR},target=/sync_gateway,type=bind ${IMAGE} /bin/bash -c "/sync_gateway/integration-test/service-test.sh" +done diff --git a/integration-test/service-test.sh b/integration-test/service-test.sh new file mode 100755 index 0000000000..3320595672 --- /dev/null +++ b/integration-test/service-test.sh @@ -0,0 +1,70 @@ +#/bin/sh + +# Copyright 2024-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +# This code is intneded to be run from within a docker container of a specific platform and runs a subset of the service scripts. The full service can not be validated since systemd does not work in docker contains. This is intended to run in /bin/sh to test dash environments on debian systems. + +set -eux -o pipefail + +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") + +cd ${SCRIPT_DIR}/../service + +./sync_gateway_service_install.sh --servicecmd + +# /etc/os-release doesn't exist on Darwin +if [ -f /etc/os-release ]; then + . /etc/os-release + case ${ID} in + amzn) + yum install -y shadow-utils systemd + ;; + esac + + groupadd -r sync_gateway + useradd -g sync_gateway sync_gateway + + # bash would support export -f for a systemctl wrapper, but dash does not support exporting aliases or functions + + mkdir -p /tmp/systemctl_wrapper + + cat << 'EOF' > /tmp/systemctl_wrapper/systemctl +#!/bin/bash + +set -eu -o pipefail + +case ${1:-} in +start) + echo "No-op systemctl start in docker, since we're not running systemd" + ;; +stop) + echo "No-op systemctl stop in docker, since we're not running systemd" + ;; +*) + echo "Running systemctl $@" + command /usr/bin/systemctl "$@" + ;; +esac +EOF + + chmod +x /tmp/systemctl_wrapper/systemctl + + export PATH=/tmp/systemctl_wrapper:$PATH +fi + +./sync_gateway_service_install.sh +./sync_gateway_service_upgrade.sh +./sync_gateway_service_uninstall.sh + +# test again with runas option +./sync_gateway_service_install.sh --runas=root +./sync_gateway_service_upgrade.sh +./sync_gateway_service_uninstall.sh + +echo "Successful service test" diff --git a/integration-test/start_server.sh b/integration-test/start_server.sh new file mode 100755 index 0000000000..aba7b5a20f --- /dev/null +++ b/integration-test/start_server.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# Copyright 2023-Present Couchbase, Inc. +# +# Use of this software is governed by the Business Source License included +# in the file licenses/BSL-Couchbase.txt. As of the Change Date specified +# in that file, in accordance with the Business Source License, use of this +# software will be governed by the Apache License, Version 2.0, included in +# the file licenses/APL2.txt. + +set -eux -o pipefail + +function usage() { + echo "Usage: $0 [-m] [-h] containername" +} + +if [ $# -gt 2 ]; then + echo "Expected maximally two arguments" + exit 1 +fi + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -m | --multi-node) + MULTI_NODE=true + shift + ;; + -h | --help) + echo "Usage: $0 [-m] [-h] containername" + exit 1 + ;; + --non-dockerhub) + DOCKERHUB=false + shift + ;; + *) + COUCHBASE_DOCKER_IMAGE_NAME="$1" + shift + ;; + esac +done + +WORKSPACE_ROOT="$(pwd)" +DOCKER_CBS_ROOT_DIR="$(pwd)" +if [ "${CBS_ROOT_DIR:-}" != "" ]; then + DOCKER_CBS_ROOT_DIR="${CBS_ROOT_DIR}" +fi + +set +e +AMAZON_LINUX_2=$(grep 'Amazon Linux 2"' /etc/os-release) +set -e +if [[ -n "${AMAZON_LINUX_2}" ]]; then + DOCKER_COMPOSE="docker-compose" # use docker-compose v1 for Jenkins AWS Linux 2 +else + DOCKER_COMPOSE="docker compose" +fi +cd -- "${BASH_SOURCE%/*}/" +${DOCKER_COMPOSE} down || true +export SG_TEST_COUCHBASE_SERVER_DOCKER_NAME=couchbase +# Start CBS +docker stop ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true +docker rm ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} || true +# --volume: Makes and mounts a CBS folder for storing a CBCollect if needed + +# use dockerhub if no registry is specified, allows for pre-release images from alternative registries +if [[ ! "${COUCHBASE_DOCKER_IMAGE_NAME}" =~ ghcr.io/* && "${DOCKERHUB:-}" != "false" ]]; then + COUCHBASE_DOCKER_IMAGE_NAME="couchbase/server:${COUCHBASE_DOCKER_IMAGE_NAME}" +fi + +if [ "${MULTI_NODE:-}" == "true" ]; then + ${DOCKER_COMPOSE} up -d --force-recreate --renew-anon-volumes --remove-orphans +else + # single node + docker run --rm -d --name ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} --volume "${WORKSPACE_ROOT}:/workspace" -p 8091-8097:8091-8097 -p 9102:9102 -p 9123:9123 -p 11207:11207 -p 11210:11210 -p 11211:11211 -p 18091-18097:18091-18097 -p 19102:19102 "${COUCHBASE_DOCKER_IMAGE_NAME}" +fi + +# Test to see if Couchbase Server is up +# Each retry min wait 5s, max 10s. Retry 20 times with exponential backoff (delay 0), fail at 120s +curl --retry-all-errors --connect-timeout 5 --max-time 10 --retry 20 --retry-delay 0 --retry-max-time 120 'http://127.0.0.1:8091' + +# Set up CBS + +docker exec couchbase couchbase-cli cluster-init --cluster-username Administrator --cluster-password password --cluster-ramsize 3072 --cluster-index-ramsize 3072 --cluster-fts-ramsize 256 --services data,index,query +docker exec couchbase couchbase-cli setting-index --cluster couchbase://localhost --username Administrator --password password --index-threads 4 --index-log-level verbose --index-max-rollback-points 10 --index-storage-setting default --index-memory-snapshot-interval 150 --index-stable-snapshot-interval 40000 + +curl -u Administrator:password -v -X POST http://127.0.0.1:8091/node/controller/rename -d 'hostname=127.0.0.1' + +if [ "${MULTI_NODE:-}" == "true" ]; then + REPLICA1_NAME=couchbase-replica1 + REPLICA2_NAME=couchbase-replica2 + CLI_ARGS=(-c couchbase://couchbase -u Administrator -p password) + docker exec ${REPLICA1_NAME} couchbase-cli node-init "${CLI_ARGS[@]}" + docker exec ${REPLICA2_NAME} couchbase-cli node-init "${CLI_ARGS[@]}" + REPLICA1_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA1_NAME} | jq -r 'first(.[]) | .IPAddress') + REPLICA2_IP=$(docker inspect --format '{{json .NetworkSettings.Networks}}' ${REPLICA2_NAME} | jq -r 'first(.[]) | .IPAddress') + docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA2_IP" --server-add-username Administrator --server-add-password password --services data,index,query + docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli server-add "${CLI_ARGS[@]}" --server-add "$REPLICA1_IP" --server-add-username Administrator --server-add-password password --services data,index,query + docker exec ${SG_TEST_COUCHBASE_SERVER_DOCKER_NAME} couchbase-cli rebalance "${CLI_ARGS[@]}" +fi