CBG-4752: Bulk APIs for CV (read and write)

docs/api/components/parameters.yaml

@@ -387,13 +387,6 @@ show_exp:
   schema:
     type: boolean
   description: Whether to show the expiry property (`_exp`) in the response.
-show_cv:
-  name: show_cv
-  in: query
-  required: false
-  schema:
-    type: boolean
-  description: Output the Current Version in the response as property `_cv`.
 startkey:
   name: startkey
   in: query

docs/api/paths/admin/keyspace-_all_docs.yaml

@@ -29,7 +29,6 @@ get:
     - $ref: ../../components/parameters.yaml#/startkey
     - $ref: ../../components/parameters.yaml#/endkey
     - $ref: ../../components/parameters.yaml#/limit-result-rows
-    - $ref: ../../components/parameters.yaml#/show_cv
   responses:
     '200':
       $ref: ../../components/responses.yaml#/all-docs

docs/api/paths/admin/keyspace-_bulk_get.yaml

@@ -45,7 +45,6 @@ post:
       description: If this header includes `gzip` then the the HTTP response will be compressed. This takes priority over `X-Accept-Part-Encoding`. Only part compression will be done if `X-Accept-Part-Encoding=gzip` and the `User-Agent` is below 1.2 due to clients not being able to handle full compression.
       schema:
         type: string
-    - $ref: ../../components/parameters.yaml#/show_cv
   requestBody:
     content:
       application/json:

docs/api/paths/public/keyspace-_all_docs.yaml

@@ -24,7 +24,6 @@ get:
     - $ref: ../../components/parameters.yaml#/startkey
     - $ref: ../../components/parameters.yaml#/endkey
     - $ref: ../../components/parameters.yaml#/limit-result-rows
-    - $ref: ../../components/parameters.yaml#/show_cv
   responses:
     '200':
       $ref: ../../components/responses.yaml#/all-docs

docs/api/paths/public/keyspace-_bulk_get.yaml

@@ -40,7 +40,6 @@ post:
       description: If this header includes `gzip` then the the HTTP response will be compressed. This takes priority over `X-Accept-Part-Encoding`. Only part compression will be done if `X-Accept-Part-Encoding=gzip` and the `User-Agent` is below 1.2 due to clients not being able to handle full compression.
       schema:
         type: string
-    - $ref: ../../components/parameters.yaml#/show_cv
   requestBody:
     content:
       application/json:

rest/access_test.go

@@ -496,11 +496,20 @@ func TestForceAPIForbiddenErrors(t *testing.T) {
 		})
 	}
 }
-func TestBulkDocsChangeToAccess(t *testing.T) {
 
+// TestBulkDocsChangeToAccess verifies that access() grants from one doc in a single bulk_docs request apply to subsequent docs in the same batch.
+func TestBulkDocsChangeToAccess(t *testing.T) {
 	base.SetUpTestLogging(t, base.LevelInfo, base.KeyAccess)
 
-	rtConfig := RestTesterConfig{SyncFn: `function(doc) {if(doc.type == "setaccess") {channel(doc.channel); access(doc.owner, doc.channel);} else { requireAccess(doc.channel)}}`}
+	rtConfig := RestTesterConfig{SyncFn: `
+		function(doc) {
+			if(doc.type == "setaccess") {
+				channel(doc.channel);
+				access(doc.owner, doc.channel);
+			} else {
+				requireAccess(doc.channel);
+			}
+		}`}
 	rt := NewRestTesterDefaultCollection(t, &rtConfig)
 	defer rt.Close()
 
@@ -518,17 +527,18 @@ func TestBulkDocsChangeToAccess(t *testing.T) {
 	assert.NoError(t, a.Save(user))
 
 	input := `{"docs": [{"_id": "bulk1", "type" : "setaccess", "owner":"user1" , "channel":"chan1"}, {"_id": "bulk2" , "channel":"chan1"}]}`
-
 	response := rt.SendUserRequest("POST", "/db/_bulk_docs", input, "user1")
 	RequireStatus(t, response, 201)
 
 	var docs []interface{}
 	require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	assert.Len(t, docs, 2)
-	assert.Equal(t, map[string]interface{}{"rev": "1-afbcffa8a4641a0f4dd94d3fc9593e74", "id": "bulk1"}, docs[0])
-
-	assert.Equal(t, map[string]interface{}{"rev": "1-4d79588b9fe9c38faae61f0c1b9471c0", "id": "bulk2"}, docs[1])
-
+	require.Len(t, docs, 2)
+	assert.NotContains(t, response.BodyString(), `missing channel access`)
+	assert.NotContains(t, response.BodyString(), `"status":403"`)
+	assert.Contains(t, response.BodyString(), `"id":"bulk1"`)
+	assert.Contains(t, response.BodyString(), `"rev":"1-afbcffa8a4641a0f4dd94d3fc9593e74"`)
+	assert.Contains(t, response.BodyString(), `"id":"bulk2"`)
+	assert.Contains(t, response.BodyString(), `"rev":"1-4d79588b9fe9c38faae61f0c1b9471c0"`)
 }
 
 // Test _all_docs API call under different security scenarios
@@ -1154,15 +1164,6 @@ func TestAllDocsCV(t *testing.T) {
 		{
 			name: "no query string",
 			url:  "/{{.keyspace}}/_all_docs",
-			output: fmt.Sprintf(`{
-				"total_rows": 1,
-				"update_seq": 1,
-				"rows": [{"key": "%s", "id": "%s", "value": {"rev": "%s"}}]
-			}`, docID, docID, docVersion.RevTreeID),
-		},
-		{
-			name: "cvs=true",
-			url:  "/{{.keyspace}}/_all_docs?show_cv=true",
 			output: fmt.Sprintf(`{
 				"total_rows": 1,
 				"update_seq": 1,

rest/adminapitest/admin_api_test.go

@@ -2229,8 +2229,15 @@ func TestHandleGetRevTree(t *testing.T) {
 
 	resp := rt.SendAdminRequest(http.MethodPost, "/{{.keyspace}}/_bulk_docs", reqBodyJson)
 	rest.RequireStatus(t, resp, http.StatusCreated)
-	respBodyExpected := `[{"id":"foo","rev":"1-123"},{"id":"foo","rev":"1-456"},{"id":"foo","rev":"1-789"}]`
-	assert.Equal(t, respBodyExpected, resp.Body.String())
+	// Response now includes unpredictable CV values - just validate ids and revs...
+	// It's likely this test will be removed in 4.0, or shortly after, since we're never allowing new conflicts to be created
+	var bulkDocsResp []map[string]interface{}
+	require.NoError(t, json.Unmarshal(resp.BodyBytes(), &bulkDocsResp))
+	require.Len(t, bulkDocsResp, 3)
+	for _, d := range bulkDocsResp {
+		assert.Equal(t, "foo", d["id"])
+		assert.Truef(t, strings.HasPrefix(d["rev"].(string), "1-"), "expected rev to start with '1-', got %s", d["rev"])
+	}
 
 	// Get the revision tree  of the user foo
 	resp = rt.SendAdminRequest(http.MethodGet, "/{{.keyspace}}/_revtree/foo", "")

rest/api_test.go

@@ -402,48 +402,129 @@ func assertGatewayStatus(t *testing.T, response *TestResponse, expected int) {
 }
 
 func TestBulkDocs(t *testing.T) {
-	rt := NewRestTester(t, nil)
-	defer rt.Close()
-
-	input := `{"docs": [{"_id": "bulk1", "n": 1}, {"_id": "bulk2", "n": 2}, {"_id": "_local/bulk3", "n": 3}]}`
-	response := rt.SendAdminRequest("POST", "/{{.keyspace}}/_bulk_docs", input)
-	RequireStatus(t, response, 201)
-
-	var docs []interface{}
-	assert.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	assert.Len(t, docs, 3)
-	assert.Equal(t, map[string]interface{}{"rev": "1-50133ddd8e49efad34ad9ecae4cb9907", "id": "bulk1"}, docs[0])
-
-	response = rt.SendAdminRequest("GET", "/{{.keyspace}}/bulk1", "")
-	RequireStatus(t, response, 200)
-	var respBody db.Body
-	assert.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &respBody))
-	assert.Equal(t, "bulk1", respBody[db.BodyId])
-	assert.Equal(t, "1-50133ddd8e49efad34ad9ecae4cb9907", respBody[db.BodyRev])
-	assert.Equal(t, float64(1), respBody["n"])
-	assert.Equal(t, map[string]interface{}{"rev": "1-035168c88bd4b80fb098a8da72f881ce", "id": "bulk2"}, docs[1])
-	assert.Equal(t, map[string]interface{}{"rev": "0-1", "id": "_local/bulk3"}, docs[2])
-
-	response = rt.SendAdminRequest("GET", "/{{.keyspace}}/_local/bulk3", "")
-	RequireStatus(t, response, 200)
-	assert.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &respBody))
-	assert.Equal(t, "_local/bulk3", respBody[db.BodyId])
-	assert.Equal(t, "0-1", respBody[db.BodyRev])
-	assert.Equal(t, float64(3), respBody["n"])
+	tests := []struct {
+		occKey string
+	}{
+		{occKey: db.BodyRev},
+		{occKey: db.BodyCV},
+	}
 
-	// update all documents
-	input = `{"docs": [{"_id": "bulk1", "_rev" : "1-50133ddd8e49efad34ad9ecae4cb9907", "n": 10}, {"_id": "bulk2", "_rev":"1-035168c88bd4b80fb098a8da72f881ce", "n": 20}, {"_id": "_local/bulk3","_rev":"0-1","n": 30}]}`
-	response = rt.SendAdminRequest("POST", "/{{.keyspace}}/_bulk_docs", input)
-	assert.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	assert.Len(t, docs, 3)
-	assert.Equal(t, map[string]interface{}{"rev": "2-7e384b16e63ee3218349ee568f156d6f", "id": "bulk1"}, docs[0])
+	for _, test := range tests {
+		t.Run(test.occKey, func(t *testing.T) {
+			rt := NewRestTester(t, nil)
+			defer rt.Close()
 
-	response = rt.SendAdminRequest("GET", "/{{.keyspace}}/_local/bulk3", "")
-	RequireStatus(t, response, 200)
-	assert.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &respBody))
-	assert.Equal(t, "_local/bulk3", respBody[db.BodyId])
-	assert.Equal(t, "0-2", respBody[db.BodyRev])
-	assert.Equal(t, float64(30), respBody["n"])
+			const bulk1DocID = "bulk1"
+			const bulk2DocID = "bulk2"
+			const bulk3LocalDocID = "_local/bulk3"
+
+			// insert all
+
+			input := fmt.Sprintf(
+				`{"docs": [{%q:%q, %q:%d}, {%q:%q, %q:%d}, {%q:%q, %q:%d}]}`,
+				db.BodyId, bulk1DocID, "n", 1,
+				db.BodyId, bulk2DocID, "n", 2,
+				db.BodyId, bulk3LocalDocID, "n", 3,
+			)
+			response := rt.SendAdminRequest(http.MethodPost, "/{{.keyspace}}/_bulk_docs", input)
+			RequireStatus(t, response, http.StatusCreated)
+
+			// get persisted versions
+			bulk1Version, _ := rt.GetDoc(bulk1DocID)
+			bulk2Version, _ := rt.GetDoc(bulk2DocID)
+			bulk3Version, _ := rt.GetDoc(bulk3LocalDocID)
+
+			var docs []interface{}
+			require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
+			require.Len(t, docs, 3)
+			assert.Equal(t, map[string]interface{}{"rev": bulk1Version.RevTreeID, "cv": bulk1Version.CV.String(), "id": bulk1DocID}, docs[0])
+			assert.Equal(t, map[string]interface{}{"rev": bulk2Version.RevTreeID, "cv": bulk2Version.CV.String(), "id": bulk2DocID}, docs[1])
+			assert.Equal(t, map[string]interface{}{"rev": bulk3Version.RevTreeID, "id": bulk3LocalDocID}, docs[2])
+
+			// check stored docs
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk1DocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk1DocID,
+				db.BodyRev, bulk1Version.RevTreeID,
+				db.BodyCV, bulk1Version.CV,
+				"n", 1,
+			), response.BodyString())
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk2DocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk2DocID,
+				db.BodyRev, bulk2Version.RevTreeID,
+				db.BodyCV, bulk2Version.CV,
+				"n", 2,
+			), response.BodyString())
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk3LocalDocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk3LocalDocID,
+				db.BodyRev, bulk3Version.RevTreeID,
+				"n", 3,
+			), response.BodyString())
+
+			// Update all documents
+			switch test.occKey {
+			case db.BodyCV:
+				input = fmt.Sprintf(
+					`{"docs": [{%q:%q, %q:%q, %q:%d}, {%q:%q, %q:%q, %q:%d}, {%q:%q, %q:%q, %q:%d}]}`,
+					db.BodyId, bulk1DocID, db.BodyCV, bulk1Version.CV, "n", 10,
+					db.BodyId, bulk2DocID, db.BodyCV, bulk2Version.CV, "n", 20,
+					db.BodyId, bulk3LocalDocID, db.BodyRev, bulk3Version.RevTreeID, "n", 30, //  local docs don't have a CV so we can only use the returned rev as the OCC value for update
+				)
+			case db.BodyRev:
+				input = fmt.Sprintf(
+					`{"docs": [{%q:%q, %q:%q, %q:%d}, {%q:%q, %q:%q, %q:%d}, {%q:%q, %q:%q, %q:%d}]}`,
+					db.BodyId, bulk1DocID, db.BodyRev, bulk1Version.RevTreeID, "n", 10,
+					db.BodyId, bulk2DocID, db.BodyRev, bulk2Version.RevTreeID, "n", 20,
+					db.BodyId, bulk3LocalDocID, db.BodyRev, bulk3Version.RevTreeID, "n", 30,
+				)
+			default:
+				t.Fatalf("Unexpected occKey: %q", test.occKey)
+			}
+			response = rt.SendAdminRequest(http.MethodPost, "/{{.keyspace}}/_bulk_docs", input)
+			RequireStatus(t, response, http.StatusCreated)
+
+			// refresh versions
+			bulk1Version, _ = rt.GetDoc(bulk1DocID)
+			bulk2Version, _ = rt.GetDoc(bulk2DocID)
+			bulk3Version, _ = rt.GetDoc(bulk3LocalDocID)
+
+			require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
+			require.Len(t, docs, 3)
+			assert.Equal(t, map[string]interface{}{"rev": bulk1Version.RevTreeID, "cv": bulk1Version.CV.String(), "id": bulk1DocID}, docs[0])
+			assert.Equal(t, map[string]interface{}{"rev": bulk2Version.RevTreeID, "cv": bulk2Version.CV.String(), "id": bulk2DocID}, docs[1])
+			assert.Equal(t, map[string]interface{}{"rev": bulk3Version.RevTreeID, "id": bulk3LocalDocID}, docs[2])
+
+			// check for stored updates
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk1DocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk1DocID,
+				db.BodyRev, bulk1Version.RevTreeID,
+				db.BodyCV, bulk1Version.CV,
+				"n", 10,
+			), response.BodyString())
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk2DocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk2DocID,
+				db.BodyRev, bulk2Version.RevTreeID,
+				db.BodyCV, bulk2Version.CV,
+				"n", 20,
+			), response.BodyString())
+			response = rt.SendAdminRequest(http.MethodGet, fmt.Sprintf("/{{.keyspace}}/%s", bulk3LocalDocID), "")
+			RequireStatus(t, response, http.StatusOK)
+			assert.JSONEq(t, fmt.Sprintf(`{%q:%q, %q:%q, %q:%d}`,
+				db.BodyId, bulk3LocalDocID,
+				db.BodyRev, bulk3Version.RevTreeID,
+				"n", 30,
+			), response.BodyString())
+		})
+	}
 }
 
 func TestBulkDocsIDGeneration(t *testing.T) {
@@ -455,7 +536,6 @@ func TestBulkDocsIDGeneration(t *testing.T) {
 	RequireStatus(t, response, 201)
 	var docs []map[string]string
 	require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	log.Printf("response: %s", response.Body.Bytes())
 	RequireStatus(t, response, 201)
 	assert.Len(t, docs, 2)
 	assert.Equal(t, "1-50133ddd8e49efad34ad9ecae4cb9907", docs[0]["rev"])
@@ -895,6 +975,8 @@ func TestBulkGetEmptyDocs(t *testing.T) {
 	RequireStatus(t, response, 400)
 }
 
+// TestBulkDocsNoEdits verifies that POSTing /_bulk_docs with new_edits=false stores
+// the provided revisions verbatim, and subsequent posts append the given history without generating new rev IDs.
 func TestBulkDocsNoEdits(t *testing.T) {
 	rt := NewRestTester(t, nil)
 	defer rt.Close()
@@ -909,10 +991,16 @@ func TestBulkDocsNoEdits(t *testing.T) {
 	RequireStatus(t, response, 201)
 	var docs []interface{}
 	require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	assert.Len(t, docs, 2)
-	assert.Equal(t, map[string]interface{}{"rev": "12-abc", "id": "bdne1"}, docs[0])
+	require.Len(t, docs, 2)
+	first, ok := docs[0].(map[string]interface{})
+	require.True(t, ok)
+	assert.Equal(t, "bdne1", first["id"])
+	assert.Equal(t, "12-abc", first["rev"])
 
-	assert.Equal(t, map[string]interface{}{"rev": "34-def", "id": "bdne2"}, docs[1])
+	second, ok := docs[1].(map[string]interface{})
+	require.True(t, ok)
+	assert.Equal(t, "bdne2", second["id"])
+	assert.Equal(t, "34-def", second["rev"])
 
 	// Now update the first doc with two new revisions:
 	input = `{"new_edits":false, "docs": [
@@ -922,8 +1010,11 @@ func TestBulkDocsNoEdits(t *testing.T) {
 	response = rt.SendAdminRequest("POST", "/{{.keyspace}}/_bulk_docs", input)
 	RequireStatus(t, response, 201)
 	require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &docs))
-	assert.Len(t, docs, 1)
-	assert.Equal(t, map[string]interface{}{"rev": "14-jkl", "id": "bdne1"}, docs[0])
+	require.Len(t, docs, 1)
+	updated, ok := docs[0].(map[string]interface{})
+	require.True(t, ok)
+	assert.Equal(t, "bdne1", updated["id"])
+	assert.Equal(t, "14-jkl", updated["rev"])
 
 }