Skip to content

Commit e24d4b4

Browse files
Likith101abhinavdangeti
Likith101
and
abhinavdangeti
authored
MB-59633: Removed encoded geo shapes from dictionaries (#2162)
Co-authored-by: Abhinav Dangeti <abhinav@couchbase.com>
1 parent 47f95bc commit e24d4b4

File tree

4 files changed

+34
-16
lines changed

4 files changed

+34
-16
lines changed

‎document/field_geoshape.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,6 @@ func (n *GeoShapeField) AnalyzedTokenFrequencies() index.TokenFrequencies {
8787
func (n *GeoShapeField) Analyze() {
8888
// compute the bytes representation for the coordinates
8989
tokens := make(analysis.TokenStream, 0)
90-
tokens = append(tokens, &analysis.Token{
91-
Start: 0,
92-
End: len(n.encodedValue),
93-
Term: n.encodedValue,
94-
Position: 1,
95-
Type: analysis.AlphaNumeric,
96-
})
9790

9891
rti := geo.GetSpatialAnalyzerPlugin("s2")
9992
terms := rti.GetIndexTokens(n.shape)
@@ -126,6 +119,10 @@ func (n *GeoShapeField) NumPlainTextBytes() uint64 {
126119
return n.numPlainTextBytes
127120
}
128121

122+
func (n *GeoShapeField) EncodedShape() []byte {
123+
return n.encodedValue
124+
}
125+
129126
func NewGeoShapeField(name string, arrayPositions []uint64,
130127
coordinates [][][][]float64, typ string) *GeoShapeField {
131128
return NewGeoShapeFieldWithIndexingOptions(name, arrayPositions,

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ toolchain go1.23.5
77
require (
88
github.com/RoaringBitmap/roaring/v2 v2.4.5
99
github.com/bits-and-blooms/bitset v1.12.0
10-
github.com/blevesearch/bleve_index_api v1.2.3
10+
github.com/blevesearch/bleve_index_api v1.2.4
1111
github.com/blevesearch/geo v0.1.20
1212
github.com/blevesearch/go-faiss v1.0.24
1313
github.com/blevesearch/go-metrics v0.0.0-20201227073835-cf1acfcdf475
1414
github.com/blevesearch/go-porterstemmer v1.0.3
1515
github.com/blevesearch/goleveldb v1.0.1
1616
github.com/blevesearch/gtreap v0.1.1
17-
github.com/blevesearch/scorch_segment_api/v2 v2.3.5
17+
github.com/blevesearch/scorch_segment_api/v2 v2.3.6
1818
github.com/blevesearch/segment v0.9.1
1919
github.com/blevesearch/snowball v0.6.1
2020
github.com/blevesearch/snowballstem v0.9.0
@@ -26,7 +26,7 @@ require (
2626
github.com/blevesearch/zapx/v13 v13.4.1
2727
github.com/blevesearch/zapx/v14 v14.4.1
2828
github.com/blevesearch/zapx/v15 v15.4.1
29-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9
29+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250324225331-76a4d38c3165
3030
github.com/couchbase/moss v0.2.0
3131
github.com/golang/protobuf v1.3.2
3232
github.com/spf13/cobra v1.8.1

‎go.sum‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ github.com/RoaringBitmap/roaring/v2 v2.4.5 h1:uGrrMreGjvAtTBobc0g5IrW1D5ldxDQYe2
22
github.com/RoaringBitmap/roaring/v2 v2.4.5/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0=
33
github.com/bits-and-blooms/bitset v1.12.0 h1:U/q1fAF7xXRhFCrhROzIfffYnu+dlS38vCZtmFVPHmA=
44
github.com/bits-and-blooms/bitset v1.12.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
5-
github.com/blevesearch/bleve_index_api v1.2.3 h1:aJJPhTb30zc68qg2HbvCZLAF1Eh5Hhm296mDAUFaUeg=
6-
github.com/blevesearch/bleve_index_api v1.2.3/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
5+
github.com/blevesearch/bleve_index_api v1.2.4 h1:XRk+wWtbNZSxKvW2bNg4fUpoc3XQDD63krBzB45fJDY=
6+
github.com/blevesearch/bleve_index_api v1.2.4/go.mod h1:rKQDl4u51uwafZxFrPD1R7xFOwKnzZW7s/LSeK4lgo0=
77
github.com/blevesearch/geo v0.1.20 h1:paaSpu2Ewh/tn5DKn/FB5SzvH0EWupxHEIwbCk/QPqM=
88
github.com/blevesearch/geo v0.1.20/go.mod h1:DVG2QjwHNMFmjo+ZgzrIq2sfCh6rIHzy9d9d0B59I6w=
99
github.com/blevesearch/go-faiss v1.0.24 h1:K79IvKjoKHdi7FdiXEsAhxpMuns0x4fM0BO93bW5jLI=
@@ -19,8 +19,8 @@ github.com/blevesearch/gtreap v0.1.1/go.mod h1:QaQyDRAT51sotthUWAH4Sj08awFSSWzgY
1919
github.com/blevesearch/mmap-go v1.0.2/go.mod h1:ol2qBqYaOUsGdm7aRMRrYGgPvnwLe6Y+7LMvAB5IbSA=
2020
github.com/blevesearch/mmap-go v1.0.4 h1:OVhDhT5B/M1HNPpYPBKIEJaD0F3Si+CrEKULGCDPWmc=
2121
github.com/blevesearch/mmap-go v1.0.4/go.mod h1:EWmEAOmdAS9z/pi/+Toxu99DnsbhG1TIxUoRmJw/pSs=
22-
github.com/blevesearch/scorch_segment_api/v2 v2.3.5 h1:t6NVTdS8xp1xHMJoe5WjxdZWLKhsIIb6kB0U49rhAno=
23-
github.com/blevesearch/scorch_segment_api/v2 v2.3.5/go.mod h1:7gG1vsL3lu2L+7RuEtZ7PJhxGYE+AV9zjxTSHuXFMdM=
22+
github.com/blevesearch/scorch_segment_api/v2 v2.3.6 h1:W9tnVyNqaowWrg8BgndTI4BXmwM+Pr6R/5+Xof7nJMk=
23+
github.com/blevesearch/scorch_segment_api/v2 v2.3.6/go.mod h1:JNx97hWfRl223SpaO9aq0C6WZI9SNdBovMp7X9K7XqU=
2424
github.com/blevesearch/segment v0.9.1 h1:+dThDy+Lvgj5JMxhmOVlgFfkUtZV2kw49xax4+jTfSU=
2525
github.com/blevesearch/segment v0.9.1/go.mod h1:zN21iLm7+GnBHWTao9I+Au/7MBiL8pPFtJBJTsk6kQw=
2626
github.com/blevesearch/snowball v0.6.1 h1:cDYjn/NCH+wwt2UdehaLpr2e4BwLIjN4V/TdLsL+B5A=
@@ -43,8 +43,8 @@ github.com/blevesearch/zapx/v14 v14.4.1 h1:G47kGCshknBZzZAtjcnIAMn3oNx8XBLxp8DMq
4343
github.com/blevesearch/zapx/v14 v14.4.1/go.mod h1:O7sDxiaL2r2PnCXbhh1Bvm7b4sP+jp4unE9DDPWGoms=
4444
github.com/blevesearch/zapx/v15 v15.4.1 h1:B5IoTMUCEzFdc9FSQbhVOxAY+BO17c05866fNruiI7g=
4545
github.com/blevesearch/zapx/v15 v15.4.1/go.mod h1:b/MreHjYeQoLjyY2+UaM0hGZZUajEbE0xhnr1A2/Q6Y=
46-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9 h1:a5GLOQkJqCnpD/lTfNiEVQnJ1e2CVOR8JFO6u1D7bLE=
47-
github.com/blevesearch/zapx/v16 v16.2.2-0.20250305220028-89edb0ef9aa9/go.mod h1:v8B9VjBtELr2qRO0f3RoRG8RAZg3Jml/5SYuBT0hvGw=
46+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250324225331-76a4d38c3165 h1:Z39zEtNDBP5JqDVJH1L9I2kFq/8aXAe7wAP6v7m9yzM=
47+
github.com/blevesearch/zapx/v16 v16.2.2-0.20250324225331-76a4d38c3165/go.mod h1:SM1MLOIXjtjymRYko2E40zKxezms7243PAwARy77FBg=
4848
github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps=
4949
github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k=
5050
github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o=

‎index/scorch/scorch.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -740,6 +740,27 @@ func analyze(d index.Document, fn customAnalyzerPluginInitFunc) {
740740
d.VisitComposite(func(cf index.CompositeField) {
741741
cf.Compose(field.Name(), field.AnalyzedLength(), field.AnalyzedTokenFrequencies())
742742
})
743+
// Since the encoded geoShape is only necessary within the doc values
744+
// of the geoShapeField, it has been removed from the field's term dictionary.
745+
// However, '_all' field uses its term dictionary as its docValues, so it
746+
// becomes necessary to add the geoShape into the '_all' field's term dictionary
747+
if f, ok := field.(index.GeoShapeField); ok {
748+
d.VisitComposite(func(cf index.CompositeField) {
749+
geoshape := f.EncodedShape()
750+
cf.Compose(field.Name(), 1, index.TokenFrequencies{
751+
string(geoshape): &index.TokenFreq{
752+
Term: geoshape,
753+
Locations: []*index.TokenLocation{
754+
{
755+
Start: 0,
756+
End: len(geoshape),
757+
Position: 1,
758+
},
759+
},
760+
},
761+
})
762+
})
763+
}
743764
}
744765
}
745766
})

0 commit comments

Comments
 (0)