Skip to content

Commit 1dc7f86

Browse files
CascadingRadiumabhinavdangeti
CascadingRadium
and
abhinavdangeti
authored
MB-65554: Add a total_mutations_filtered stat to track mutations that qualify for a type mapping (#2157)
- This stat tracks the number of mutations processed by the Bleve index that are searchable by the user. It helps monitor the progress of the index-building process. - The stat is incremented for each mutation processed by the Bleve index that is associated with a valid type mapping. --------- Co-authored-by: Abhinav Dangeti <abhinav@couchbase.com>
1 parent af9e311 commit 1dc7f86

File tree

6 files changed

+25
-9
lines changed

6 files changed

+25
-9
lines changed

‎document/document.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ type Document struct {
3434
Fields []Field `json:"fields"`
3535
CompositeFields []*CompositeField
3636
StoredFieldsSize uint64
37+
indexed bool
3738
}
3839

3940
func (d *Document) StoredFieldsBytes() uint64 {
@@ -148,3 +149,11 @@ func (d *Document) VisitSynonymFields(visitor index.SynonymFieldVisitor) {
148149
}
149150
}
150151
}
152+
153+
func (d *Document) SetIndexed() {
154+
d.indexed = true
155+
}
156+
157+
func (d *Document) Indexed() bool {
158+
return d.indexed
159+
}

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ go 1.21
55
require (
66
github.com/RoaringBitmap/roaring/v2 v2.4.5
77
github.com/bits-and-blooms/bitset v1.12.0
8-
github.com/blevesearch/bleve_index_api v1.2.2
8+
github.com/blevesearch/bleve_index_api v1.2.3
99
github.com/blevesearch/geo v0.1.20
1010
github.com/blevesearch/go-faiss v1.0.24
1111
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
1212
github.com/blevesearch/go-porterstemmer v1.0.3
1313
github.com/blevesearch/goleveldb v1.0.1
1414
github.com/blevesearch/gtreap v0.1.1
15-
github.com/blevesearch/scorch_segment_api/v2 v2.3.4
15+
github.com/blevesearch/scorch_segment_api/v2 v2.3.5
1616
github.com/blevesearch/segment v0.9.1
1717
github.com/blevesearch/snowball v0.6.1
1818
github.com/blevesearch/snowballstem v0.9.0
@@ -24,7 +24,7 @@ require (
2424
github.com/blevesearch/zapx/v13 v13.4.1
2525
github.com/blevesearch/zapx/v14 v14.4.1
2626
github.com/blevesearch/zapx/v15 v15.4.1
27-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241
27+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9
2828
github.com/couchbase/moss v0.2.0
2929
github.com/golang/protobuf v1.3.2
3030
github.com/spf13/cobra v1.7.0

‎go.sum‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2
22
github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
33
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
44
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
5-
github.com/blevesearch/bleve_index_api v1.2.2 h1:KUa+Md44lL+bZ6N9CxxdI/KEQD6sQduVWyd0S+A2+bg=
6-
github.com/blevesearch/bleve_index_api v1.2.2/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
5+
github.com/blevesearch/bleve_index_api v1.2.3 h1:aJJPhTb30zc68qg2HbvCZLAF1Eh5Hhm296mDAUFaUeg=
6+
github.com/blevesearch/bleve_index_api v1.2.3/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
77
github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
88
github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
99
github.com/blevesearch/go-faiss v1.0.24 h1:K79IvKjoKHdi7FdiXEsAhxpMuns0x4fM0BO93bW5jLI=
@@ -19,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
1919
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
2020
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
2121
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
22-
github.com/blevesearch/scorch_segment_api/v2 v2.3.4 h1:xiXqP168XL3M71uJ8s2t0EoTNTULrW4X6tTfaHs9aBk=
23-
github.com/blevesearch/scorch_segment_api/v2 v2.3.4/go.mod h1:w45Nio18Q5PLo+Muqz3x/zv3C+ykqimhzhbRYJlDx3U=
22+
github.com/blevesearch/scorch_segment_api/v2 v2.3.5 h1:t6NVTdS8xp1xHMJoe5WjxdZWLKhsIIb6kB0U49rhAno=
23+
github.com/blevesearch/scorch_segment_api/v2 v2.3.5/go.mod h1:7gG1vsL3lu2L+7RuEtZ7PJhxGYE+AV9zjxTSHuXFMdM=
2424
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
2525
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
2626
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.4.1 h1:G47kGCshknBZzZAtjcnIAMn3oNx8XBLxp8DMq
4343
github.com/blevesearch/zapx/v14 v14.4.1/go.mod h1:O7sDxiaL2r2PnCXbhh1Bvm7b4sP+jp4unE9DDPWGoms=
4444
github.com/blevesearch/zapx/v15 v15.4.1 h1:B5IoTMUCEzFdc9FSQbhVOxAY+BO17c05866fNruiI7g=
4545
github.com/blevesearch/zapx/v15 v15.4.1/go.mod h1:b/MreHjYeQoLjyY2+UaM0hGZZUajEbE0xhnr1A2/Q6Y=
46-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241 h1:6S4ZN9QASFbvdV+u10+7eRqqNhFXaXDqeKVn6r/dA9Y=
47-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250225163414-355d2eb8f241/go.mod h1:prB/UsGsmPYGHH4BTTcTXMxGXv+RPk4Frj+I/n70asY=
46+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9 h1:a5GLOQkJqCnpD/lTfNiEVQnJ1e2CVOR8JFO6u1D7bLE=
47+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9/go.mod h1:v8B9VjBtELr2qRO0f3RoRG8RAZg3Jml/5SYuBT0hvGw=
4848
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
4949
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
5050
github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

‎index/scorch/scorch.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,10 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) {
424424
for itemsDeQueued < numUpdates {
425425
result := <-resultChan
426426
resultSize := result.Size()
427+
// check if the document is searchable by the index
428+
if result.Indexed() {
429+
atomic.AddUint64(&s.stats.TotMutationsFiltered, 1)
430+
}
427431
atomic.AddUint64(&s.iStats.analysisBytesAdded, uint64(resultSize))
428432
totalAnalysisSize += resultSize
429433
analysisResults[itemsDeQueued] = result
@@ -636,6 +640,7 @@ func (s *Scorch) StatsMap() map[string]interface{} {
636640
m["term_searchers_finished"] = m["TotTermSearchersFinished"]
637641
m["knn_searches"] = m["TotKNNSearches"]
638642
m["synonym_searches"] = m["TotSynonymSearches"]
643+
m["total_mutations_filtered"] = m["TotMutationsFiltered"]
639644

640645
m["num_bytes_read_at_query_time"] = m["TotBytesReadAtQueryTime"]
641646
m["num_plain_text_bytes_indexed"] = m["TotIndexedPlainTextBytes"]

‎index/scorch/stats.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ type Stats struct {
8181
TotPersistedItems uint64
8282
TotItemsToPersist uint64
8383
TotPersistedSegments uint64
84+
TotMutationsFiltered uint64
8485

8586
TotPersisterSlowMergerPause uint64
8687
TotPersisterSlowMergerResume uint64

‎mapping/index.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ func (im *IndexMappingImpl) MapDocument(doc *document.Document, data interface{}
366366
field := document.NewCompositeFieldWithIndexingOptions("_all", true, []string{}, walkContext.excludedFromAll, index.IndexField|index.IncludeTermVectors)
367367
doc.AddField(field)
368368
}
369+
doc.SetIndexed()
369370
}
370371

371372
return nil

0 commit comments

Comments
 (0)