MB-63831, MB-60971, MB-62182: Making persister and merger more resource

			friendly

- blevesearch/bleve#2100
- blevesearch/bleve#2134

Change-Id: I8e3e6e8f60d16094d3d4ddee44115c07a872fcfb
Reviewed-on: https://review.couchbase.org/c/cbft/+/225100
Reviewed-by: Abhi Dangeti <abhinav@couchbase.com>
Tested-by: <thejas.orkombu@couchbase.com>
Well-Formed: Build Bot <build@couchbase.com>

Author: Thejas-bhat

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
 	github.com/blevesearch/bleve-mapping-ui v0.6.0
-	github.com/blevesearch/bleve/v2 v2.4.4-0.20250320075922-5040e2d2092a
+	github.com/blevesearch/bleve/v2 v2.4.4-0.20250327105221-47f95bceb5e4
 	github.com/blevesearch/bleve_index_api v1.2.3
 	github.com/blevesearch/zapx/v11 v11.4.1
 	github.com/blevesearch/zapx/v12 v12.4.1

go.sum

@@ -93,8 +93,8 @@ github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZ
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/blevesearch/bleve-mapping-ui v0.6.0 h1:lTne3hGUNNDnHz/sR5cp9MpIQ8x5AfAsufkc6R83pu0=
 github.com/blevesearch/bleve-mapping-ui v0.6.0/go.mod h1:ATheobclG2Dnf5rHpmL/+sOeggsemKW9yEN6GoExA+A=
-github.com/blevesearch/bleve/v2 v2.4.4-0.20250320075922-5040e2d2092a h1:UZC9yTvfDW1+n6AAeXCO956l5zrE/mm3CvuXCD59zfE=
-github.com/blevesearch/bleve/v2 v2.4.4-0.20250320075922-5040e2d2092a/go.mod h1:7dcR9fviWC63sppS1iZfd2r8y3gZb0gED1mJ2HIH/r8=
+github.com/blevesearch/bleve/v2 v2.4.4-0.20250327105221-47f95bceb5e4 h1:2Agz3Q5sBs2wGMwrDCE1vnwVwZufXogWRsXUNfzMThE=
+github.com/blevesearch/bleve/v2 v2.4.4-0.20250327105221-47f95bceb5e4/go.mod h1:7dcR9fviWC63sppS1iZfd2r8y3gZb0gED1mJ2HIH/r8=
 github.com/blevesearch/bleve_index_api v1.2.3 h1:aJJPhTb30zc68qg2HbvCZLAF1Eh5Hhm296mDAUFaUeg=
 github.com/blevesearch/bleve_index_api v1.2.3/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=

pindex_bleve.go

@@ -702,6 +702,40 @@ func PrepareIndexDef(mgr *cbgt.Manager, indexDef *cbgt.IndexDef) (
 			return nil, cbgt.NewBadRequestError("PrepareIndex, err: cosine similarity metric for vectors " +
 				"not supported in this cluster")
 		}
+
+		// setting default values for the number of persister workers and how much
+		// data each worker can handle in memory before flushing to disk.
+		//
+		// these defaults can be overridden by the user in the index definition.
+		// MB-63831 for more details.
+		// For non-vector indexes, we don't need to be aggressive
+		// with the flushing and background merging since the
+		// payload is generally smaller.
+		var numPersisterWorkers = 4
+		var maxSizeInMemoryMergePerWorker = 20 * 1024 * 1024
+		if indexVectorPicture.fields != noVectorFields {
+			// Keeping things a bit more parallelized and
+			// making each worker do the
+			// job on a larger amount of data because
+			// vector payload is generally higher
+			numPersisterWorkers = 8
+			maxSizeInMemoryMergePerWorker = 200 * 1024 * 1024
+		}
+
+		if v, ok := bp.Store["scorchPersisterOptions"].(map[string]interface{}); ok {
+			if v["numPersisterWorkers"] == nil {
+				v["numPersisterWorkers"] = numPersisterWorkers
+			}
+			if v["maxSizeInMemoryMergePerWorker"] == nil {
+				v["maxSizeInMemoryMergePerWorker"] = maxSizeInMemoryMergePerWorker
+			}
+		} else {
+			bp.Store["scorchPersisterOptions"] = map[string]interface{}{
+				"numPersisterWorkers":           numPersisterWorkers,
+				"maxSizeInMemoryMergePerWorker": maxSizeInMemoryMergePerWorker,
+			}
+		}
+
 	}
 
 	segmentVersionSupported := cbgt.IsFeatureSupportedByCluster(