MB-63831: new stat to indicate an index's convergence

- introduce a new stat `index_bgthreads_active` which indicates whether
all the partitions belong to an index on a specific node are in the
process of reaching a steady state (due to background routines' activity).

- if the value is false, its safe to assume that every partition of that
index has reached a steady state (no merger/persister activity)

- blevesearch/bleve#2152

Change-Id: I9603164a7cac6737de2daf5004589d00f2ad8833
Reviewed-on: https://review.couchbase.org/c/cbft/+/223958
Well-Formed: Build Bot <build@couchbase.com>
Tested-by: <thejas.orkombu@couchbase.com>
Reviewed-by: Abhi Dangeti <abhinav@couchbase.com>

Author: Thejas-bhat

go.mod

@@ -6,14 +6,14 @@ require (
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.48.1
 	github.com/blevesearch/bleve-mapping-ui v0.6.0
-	github.com/blevesearch/bleve/v2 v2.4.4-0.20250224172532-de9c49c0186a
-	github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40
+	github.com/blevesearch/bleve/v2 v2.4.4-0.20250228153102-166dadb2eb06
+	github.com/blevesearch/bleve_index_api v1.2.2
 	github.com/blevesearch/zapx/v11 v11.4.1
 	github.com/blevesearch/zapx/v12 v12.4.1
 	github.com/blevesearch/zapx/v13 v13.4.1
 	github.com/blevesearch/zapx/v14 v14.4.1
 	github.com/blevesearch/zapx/v15 v15.4.1
-	github.com/blevesearch/zapx/v16 v16.2.1
+	github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241
 	github.com/buger/jsonparser v1.1.1
 	github.com/cloudfoundry/gosigar v1.3.4
 	github.com/couchbase/cbauth v0.1.13
@@ -58,7 +58,7 @@ require (
 	github.com/blevesearch/goleveldb v1.0.1 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
 	github.com/blevesearch/mmap-go v1.0.4 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.3.3 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.3.4 // indirect
 	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
 	github.com/blevesearch/stempel v0.2.0 // indirect

go.sum

@@ -93,10 +93,10 @@ github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZ
 github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
 github.com/blevesearch/bleve-mapping-ui v0.6.0 h1:lTne3hGUNNDnHz/sR5cp9MpIQ8x5AfAsufkc6R83pu0=
 github.com/blevesearch/bleve-mapping-ui v0.6.0/go.mod h1:ATheobclG2Dnf5rHpmL/+sOeggsemKW9yEN6GoExA+A=
-github.com/blevesearch/bleve/v2 v2.4.4-0.20250224172532-de9c49c0186a h1:S9/ZWSIwU4uQ+Mf+qc0b8AxyslO41YWuIXlD+TJl578=
-github.com/blevesearch/bleve/v2 v2.4.4-0.20250224172532-de9c49c0186a/go.mod h1:v3G+mDypdfANYVtfdWIOCJX2hUT1YJYTO44v8hjWXeo=
-github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40 h1:HmfirZlJf9gdQE8TCAF10cZ/2ubydgK7vCcLojCqmt0=
-github.com/blevesearch/bleve_index_api v1.2.2-0.20250220160913-17c09f7a4a40/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
+github.com/blevesearch/bleve/v2 v2.4.4-0.20250228153102-166dadb2eb06 h1:QvQSO0VTbuCKO1Yb7RARUaHsAGVE9WVYJlB5vgFwVwk=
+github.com/blevesearch/bleve/v2 v2.4.4-0.20250228153102-166dadb2eb06/go.mod h1:ryjo+3T9dgfOzyYU3rWVdWOdPJYOb70eWB3Wg3GtsP4=
+github.com/blevesearch/bleve_index_api v1.2.2 h1:KUa+Md44lL+bZ6N9CxxdI/KEQD6sQduVWyd0S+A2+bg=
+github.com/blevesearch/bleve_index_api v1.2.2/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
 github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
 github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
 github.com/blevesearch/go-faiss v1.0.24 h1:K79IvKjoKHdi7FdiXEsAhxpMuns0x4fM0BO93bW5jLI=
@@ -110,8 +110,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
 github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
 github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
 github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
-github.com/blevesearch/scorch_segment_api/v2 v2.3.3 h1:LtyQ1Wltja54bchqwgY20SvVe6HltUL4PsAPH3UNrQI=
-github.com/blevesearch/scorch_segment_api/v2 v2.3.3/go.mod h1:LXidEjeenMdbcLKP/UdZi1HJOny61FbhslAh5SgN5Ik=
+github.com/blevesearch/scorch_segment_api/v2 v2.3.4 h1:xiXqP168XL3M71uJ8s2t0EoTNTULrW4X6tTfaHs9aBk=
+github.com/blevesearch/scorch_segment_api/v2 v2.3.4/go.mod h1:w45Nio18Q5PLo+Muqz3x/zv3C+ykqimhzhbRYJlDx3U=
 github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
 github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
 github.com/blevesearch/snowballstem v0.9.0 h1:lMQ189YspGP6sXvZQ4WZ+MLawfV8wOmPoD/iWeNXm8s=
@@ -132,8 +132,8 @@ github.com/blevesearch/zapx/v14 v14.4.1 h1:G47kGCshknBZzZAtjcnIAMn3oNx8XBLxp8DMq
 github.com/blevesearch/zapx/v14 v14.4.1/go.mod h1:O7sDxiaL2r2PnCXbhh1Bvm7b4sP+jp4unE9DDPWGoms=
 github.com/blevesearch/zapx/v15 v15.4.1 h1:B5IoTMUCEzFdc9FSQbhVOxAY+BO17c05866fNruiI7g=
 github.com/blevesearch/zapx/v15 v15.4.1/go.mod h1:b/MreHjYeQoLjyY2+UaM0hGZZUajEbE0xhnr1A2/Q6Y=
-github.com/blevesearch/zapx/v16 v16.2.1 h1:M9cqgY39kYdvN8HJ3f6aj+9qsZ7Gybebme4rxKCExV8=
-github.com/blevesearch/zapx/v16 v16.2.1/go.mod h1:aXtMpbJKVQdLwwOz4IIgjTxxo4Ac2osKeLpQxVAueTI=
+github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241 h1:6S4ZN9QASFbvdV+u10+7eRqqNhFXaXDqeKVn6r/dA9Y=
+github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241/go.mod h1:prB/UsGsmPYGHH4BTTcTXMxGXv+RPk4Frj+I/n70asY=
 github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
 github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

ns_server.go

@@ -303,6 +303,7 @@ var statkeys = []string{
 	"total_term_searchers_finished", // per-index stat.
 	"total_knn_searches",            // per-index stat.
 	"total_synonym_searches",        // per-index stat.
+	"index_bgthreads_active",        //per-index stat
 
 	// "curr_batches_blocked_by_herder"   -- PROCESS-LEVEL stat.
 	// "total_queries_rejected_by_herder" -- PROCESS-LEVEL stat
@@ -1090,6 +1091,7 @@ var scorchStats = map[string]string{
 	"/num_bytes_used_disk_by_root_reclaimable": "num_bytes_used_disk_by_root_reclaimable",
 	"/num_bytes_read_at_query_time":            "num_bytes_read_at_query_time",
 	"/num_bytes_written_at_index_time":         "num_bytes_written_at_index_time",
+	"/index_bgthreads_active":                  "index_bgthreads_active",
 }
 
 func extractScorchStats(sstats, nsIndexStat map[string]interface{},
@@ -1100,6 +1102,26 @@ func extractScorchStats(sstats, nsIndexStat map[string]interface{},
 			updateStat(statname, float64(vuint64), nsIndexStat, aggrIdxStats)
 		} else if fuint64, ok := v.(float64); ok {
 			updateStat(statname, float64(fuint64), nsIndexStat, aggrIdxStats)
+		} else if b, ok := v.(bool); ok {
+			// `index_bgthreads_active` indicates that the partition is still in the process of
+			// reaching a steady state.
+			// at an index level (multiple partitions), `index_bgthreads_active` is an OR
+			// operation of the activity status of all partitions.
+			if statname == "index_bgthreads_active" {
+				oldValue, ok := nsIndexStat["index_bgthreads_active"]
+				if ok {
+					switch oldValue := oldValue.(type) {
+					case float64:
+						val := uint64(oldValue)
+						if b {
+							val = uint64(oldValue) | 1
+						}
+						nsIndexStat["index_bgthreads_active"] = val != 0
+					case bool:
+						nsIndexStat["index_bgthreads_active"] = oldValue || b
+					}
+				}
+			}
 		}
 	}