diff --git a/builder.go b/builder.go index 30285a2e4..f170317ee 100644 --- a/builder.go +++ b/builder.go @@ -68,7 +68,7 @@ func newBuilder(path string, mapping mapping.IndexMapping, config map[string]int return nil, err } config["internal"] = map[string][]byte{ - string(mappingInternalKey): mappingBytes, + string(util.MappingInternalKey): mappingBytes, } // do not use real config, as these are options for the builder, diff --git a/docs/index_update.md b/docs/index_update.md new file mode 100644 index 000000000..f736dde73 --- /dev/null +++ b/docs/index_update.md @@ -0,0 +1,41 @@ +# Ability to reduce downtime during index mapping updates + +* *v2.5.4* (and after) will come with support to delete or modify any field mapping in the index mapping without requiring a full rebuild of the index +* We do this by storing which portions of the field has to be deleted within zap and then lazily executing the deletion during subsequent merging of the segments + +## Usage + +While opening an index, if an updated mapping is provided as a string under the key `updated_mapping` within the `runtimeConfig` parameter of `OpenUsing`, then we open the index and try to update it to use the new mapping provided. + +If the update fails, the index is unchanged and an error is returned explaining why the update was unsuccessful. + +## What can be deleted and what can't be deleted? +Fields can be partially deleted by changing their Index, Store, and DocValues parameters from true to false, or completely removed by deleting the field itself. + +Additionally, document mappings can be deleted either by fully removing them from the index mapping or by setting the Enabled value to false, which deletes all fields defined within that mapping. + +However, if any of the following conditions are met, the index is considered non-updatable. +* Any additional fields or enabled document mappings in the new index mapping +* Any changes to IncludeInAll, type, IncludeTermVectors and SkipFreqNorm +* Any document mapping having it's enabled value changing from false to true +* Text fields with a different analyser or date time fields with a different date time format +* Vector and VectorBase64 fields changing dims, similarity or vectorIndexOptimizedFor +* Any changes when field is part of `_all` +* Full field deletions when it is covered by any dynamic setting (Index, Store or DocValues Dynamic) +* Any changes to dynamic settings at the top level or any enabled document mapping +* If multiple fields sharing the same field name either from different type mappings or aliases are present, then any non compatible changes across all of these fields + +## How to enforce immediate deletion? +Since the deletion is only done during merging, a [force merge](https://github.com/blevesearch/bleve/blob/b82baf10b205511cf12da5cb24330abd9f5b1b74/index/scorch/merge.go#L164) may be used to completely remove the stale data. + +## Sample code to update an existing index +``` +newMapping := `` +config := map[string]interface{}{ + "updated_mapping": newMapping +} +index, err := OpenUsing("", config) +if err != nil { + return err +} +``` diff --git a/index.go b/index.go index 44fa6a00b..a9c8ada34 100644 --- a/index.go +++ b/index.go @@ -325,6 +325,8 @@ func Open(path string) (Index, error) { // The mapping used when it was created will be used for all Index/Search operations. // The provided runtimeConfig can override settings // persisted when the kvstore was created. +// If runtimeConfig has updated mapping, then an index update is attempted +// Throws an error without any changes to the index if an unupdatable mapping is provided func OpenUsing(path string, runtimeConfig map[string]interface{}) (Index, error) { return openIndexUsing(path, runtimeConfig) } diff --git a/index/scorch/optimize_knn.go b/index/scorch/optimize_knn.go index 07127dab3..affb4ff13 100644 --- a/index/scorch/optimize_knn.go +++ b/index/scorch/optimize_knn.go @@ -79,6 +79,12 @@ func (o *OptimizeVR) Finish() error { wg.Done() }() for field, vrs := range o.vrs { + // Early exit if the field is supposed to be completely deleted or + // if it's index data has been deleted + if info, ok := o.snapshot.updatedFields[field]; ok && (info.Deleted || info.Index) { + continue + } + vecIndex, err := segment.InterpretVectorIndex(field, o.requiresFiltering, origSeg.deleted) if err != nil { diff --git a/index/scorch/persister.go b/index/scorch/persister.go index cd958c358..d92c3a85b 100644 --- a/index/scorch/persister.go +++ b/index/scorch/persister.go @@ -608,7 +608,7 @@ func persistToDirectory(seg segment.UnpersistedSegment, d index.Directory, func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, segPlugin SegmentPlugin, exclude map[uint64]struct{}, d index.Directory) ( []string, map[uint64]string, error) { - snapshotsBucket, err := tx.CreateBucketIfNotExists(boltSnapshotsBucket) + snapshotsBucket, err := tx.CreateBucketIfNotExists(util.BoltSnapshotsBucket) if err != nil { return nil, nil, err } @@ -619,17 +619,17 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, } // persist meta values - metaBucket, err := snapshotBucket.CreateBucketIfNotExists(boltMetaDataKey) + metaBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltMetaDataKey) if err != nil { return nil, nil, err } - err = metaBucket.Put(boltMetaDataSegmentTypeKey, []byte(segPlugin.Type())) + err = metaBucket.Put(util.BoltMetaDataSegmentTypeKey, []byte(segPlugin.Type())) if err != nil { return nil, nil, err } buf := make([]byte, binary.MaxVarintLen32) binary.BigEndian.PutUint32(buf, segPlugin.Version()) - err = metaBucket.Put(boltMetaDataSegmentVersionKey, buf) + err = metaBucket.Put(util.BoltMetaDataSegmentVersionKey, buf) if err != nil { return nil, nil, err } @@ -643,13 +643,13 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, if err != nil { return nil, nil, err } - err = metaBucket.Put(boltMetaDataTimeStamp, timeStampBinary) + err = metaBucket.Put(util.BoltMetaDataTimeStamp, timeStampBinary) if err != nil { return nil, nil, err } // persist internal values - internalBucket, err := snapshotBucket.CreateBucketIfNotExists(boltInternalKey) + internalBucket, err := snapshotBucket.CreateBucketIfNotExists(util.BoltInternalKey) if err != nil { return nil, nil, err } @@ -665,7 +665,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, val := make([]byte, 8) bytesWritten := atomic.LoadUint64(&snapshot.parent.stats.TotBytesWrittenAtIndexTime) binary.LittleEndian.PutUint64(val, bytesWritten) - err = internalBucket.Put(TotBytesWrittenKey, val) + err = internalBucket.Put(util.TotBytesWrittenKey, val) if err != nil { return nil, nil, err } @@ -689,7 +689,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, return nil, nil, fmt.Errorf("segment: %s copy err: %v", segPath, err) } filename := filepath.Base(segPath) - err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + err = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename)) if err != nil { return nil, nil, err } @@ -705,7 +705,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, return nil, nil, fmt.Errorf("segment: %s persist err: %v", path, err) } newSegmentPaths[segmentSnapshot.id] = path - err = snapshotSegmentBucket.Put(boltPathKey, []byte(filename)) + err = snapshotSegmentBucket.Put(util.BoltPathKey, []byte(filename)) if err != nil { return nil, nil, err } @@ -721,7 +721,7 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, if err != nil { return nil, nil, fmt.Errorf("error persisting roaring bytes: %v", err) } - err = snapshotSegmentBucket.Put(boltDeletedKey, roaringBuf.Bytes()) + err = snapshotSegmentBucket.Put(util.BoltDeletedKey, roaringBuf.Bytes()) if err != nil { return nil, nil, err } @@ -733,7 +733,19 @@ func prepareBoltSnapshot(snapshot *IndexSnapshot, tx *bolt.Tx, path string, if err != nil { return nil, nil, err } - err = snapshotSegmentBucket.Put(boltStatsKey, b) + err = snapshotSegmentBucket.Put(util.BoltStatsKey, b) + if err != nil { + return nil, nil, err + } + } + + // store updated field info + if segmentSnapshot.updatedFields != nil { + b, err := json.Marshal(segmentSnapshot.updatedFields) + if err != nil { + return nil, nil, err + } + err = snapshotSegmentBucket.Put(util.BoltUpdatedFieldsKey, b) if err != nil { return nil, nil, err } @@ -832,22 +844,9 @@ func zapFileName(epoch uint64) string { // bolt snapshot code -var ( - boltSnapshotsBucket = []byte{'s'} - boltPathKey = []byte{'p'} - boltDeletedKey = []byte{'d'} - boltInternalKey = []byte{'i'} - boltMetaDataKey = []byte{'m'} - boltMetaDataSegmentTypeKey = []byte("type") - boltMetaDataSegmentVersionKey = []byte("version") - boltMetaDataTimeStamp = []byte("timeStamp") - boltStatsKey = []byte("stats") - TotBytesWrittenKey = []byte("TotBytesWritten") -) - func (s *Scorch) loadFromBolt() error { err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -912,7 +911,7 @@ func (s *Scorch) loadFromBolt() error { // NOTE: this is currently ONLY intended to be used by the command-line tool func (s *Scorch) LoadSnapshot(epoch uint64) (rv *IndexSnapshot, err error) { err = s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -940,14 +939,14 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { // first we look for the meta-data bucket, this will tell us // which segment type/version was used for this snapshot // all operations for this scorch will use this type/version - metaBucket := snapshot.Bucket(boltMetaDataKey) + metaBucket := snapshot.Bucket(util.BoltMetaDataKey) if metaBucket == nil { _ = rv.DecRef() return nil, fmt.Errorf("meta-data bucket missing") } - segmentType := string(metaBucket.Get(boltMetaDataSegmentTypeKey)) + segmentType := string(metaBucket.Get(util.BoltMetaDataSegmentTypeKey)) segmentVersion := binary.BigEndian.Uint32( - metaBucket.Get(boltMetaDataSegmentVersionKey)) + metaBucket.Get(util.BoltMetaDataSegmentVersionKey)) err := s.loadSegmentPlugin(segmentType, segmentVersion) if err != nil { _ = rv.DecRef() @@ -957,7 +956,7 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { var running uint64 c := snapshot.Cursor() for k, _ := c.First(); k != nil; k, _ = c.Next() { - if k[0] == boltInternalKey[0] { + if k[0] == util.BoltInternalKey[0] { internalBucket := snapshot.Bucket(k) if internalBucket == nil { _ = rv.DecRef() @@ -972,11 +971,11 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { _ = rv.DecRef() return nil, err } - } else if k[0] != boltMetaDataKey[0] { + } else if k[0] != util.BoltMetaDataKey[0] { segmentBucket := snapshot.Bucket(k) if segmentBucket == nil { _ = rv.DecRef() - return nil, fmt.Errorf("segment key, but bucket missing % x", k) + return nil, fmt.Errorf("segment key, but bucket missing %x", k) } segmentSnapshot, err := s.loadSegment(segmentBucket) if err != nil { @@ -990,6 +989,10 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { } rv.segment = append(rv.segment, segmentSnapshot) rv.offsets = append(rv.offsets, running) + // Merge all segment level updated field info for use during queries + if segmentSnapshot.updatedFields != nil { + rv.MergeUpdateFieldsInfo(segmentSnapshot.updatedFields) + } running += segmentSnapshot.segment.Count() } } @@ -997,46 +1000,59 @@ func (s *Scorch) loadSnapshot(snapshot *bolt.Bucket) (*IndexSnapshot, error) { } func (s *Scorch) loadSegment(segmentBucket *bolt.Bucket) (*SegmentSnapshot, error) { - pathBytes := segmentBucket.Get(boltPathKey) + pathBytes := segmentBucket.Get(util.BoltPathKey) if pathBytes == nil { return nil, fmt.Errorf("segment path missing") } segmentPath := s.path + string(os.PathSeparator) + string(pathBytes) - segment, err := s.segPlugin.Open(segmentPath) + seg, err := s.segPlugin.Open(segmentPath) if err != nil { return nil, fmt.Errorf("error opening bolt segment: %v", err) } rv := &SegmentSnapshot{ - segment: segment, + segment: seg, cachedDocs: &cachedDocs{cache: nil}, cachedMeta: &cachedMeta{meta: nil}, } - deletedBytes := segmentBucket.Get(boltDeletedKey) + deletedBytes := segmentBucket.Get(util.BoltDeletedKey) if deletedBytes != nil { deletedBitmap := roaring.NewBitmap() r := bytes.NewReader(deletedBytes) _, err := deletedBitmap.ReadFrom(r) if err != nil { - _ = segment.Close() + _ = seg.Close() return nil, fmt.Errorf("error reading deleted bytes: %v", err) } if !deletedBitmap.IsEmpty() { rv.deleted = deletedBitmap } } - statBytes := segmentBucket.Get(boltStatsKey) + statBytes := segmentBucket.Get(util.BoltStatsKey) if statBytes != nil { var statsMap map[string]map[string]uint64 err := json.Unmarshal(statBytes, &statsMap) stats := &fieldStats{statMap: statsMap} if err != nil { - _ = segment.Close() + _ = seg.Close() return nil, fmt.Errorf("error reading stat bytes: %v", err) } rv.stats = stats } + updatedFieldBytes := segmentBucket.Get(util.BoltUpdatedFieldsKey) + if updatedFieldBytes != nil { + var updatedFields map[string]*index.UpdateFieldInfo + + err := json.Unmarshal(updatedFieldBytes, &updatedFields) + if err != nil { + _ = seg.Close() + return nil, fmt.Errorf("error reading updated field bytes: %v", err) + } + rv.updatedFields = updatedFields + // Set the value within the segment base for use during merge + rv.UpdateFieldsInfo(rv.updatedFields) + } return rv, nil } @@ -1215,7 +1231,7 @@ func (s *Scorch) removeOldBoltSnapshots() (numRemoved int, err error) { } }() - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return 0, nil } @@ -1325,7 +1341,7 @@ func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) { expirationDuration := time.Duration(s.numSnapshotsToKeep-1) * s.rollbackSamplingInterval err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -1349,11 +1365,11 @@ func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) { if snapshot == nil { continue } - metaBucket := snapshot.Bucket(boltMetaDataKey) + metaBucket := snapshot.Bucket(util.BoltMetaDataKey) if metaBucket == nil { continue } - timeStampBytes := metaBucket.Get(boltMetaDataTimeStamp) + timeStampBytes := metaBucket.Get(util.BoltMetaDataTimeStamp) var timeStamp time.Time err = timeStamp.UnmarshalText(timeStampBytes) if err != nil { @@ -1390,7 +1406,7 @@ func (s *Scorch) rootBoltSnapshotMetaData() ([]*snapshotMetaData, error) { func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { var rv []uint64 err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -1411,7 +1427,7 @@ func (s *Scorch) RootBoltSnapshotEpochs() ([]uint64, error) { func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { rv := map[string]struct{}{} err := s.rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -1423,14 +1439,14 @@ func (s *Scorch) loadZapFileNames() (map[string]struct{}, error) { } segc := snapshot.Cursor() for segk, _ := segc.First(); segk != nil; segk, _ = segc.Next() { - if segk[0] == boltInternalKey[0] { + if segk[0] == util.BoltInternalKey[0] { continue } segmentBucket := snapshot.Bucket(segk) if segmentBucket == nil { continue } - pathBytes := segmentBucket.Get(boltPathKey) + pathBytes := segmentBucket.Get(util.BoltPathKey) if pathBytes == nil { continue } diff --git a/index/scorch/rollback.go b/index/scorch/rollback.go index 895f939dd..f047762fa 100644 --- a/index/scorch/rollback.go +++ b/index/scorch/rollback.go @@ -19,6 +19,7 @@ import ( "log" "os" + "github.com/blevesearch/bleve/v2/util" bolt "go.etcd.io/bbolt" ) @@ -61,7 +62,7 @@ func RollbackPoints(path string) ([]*RollbackPoint, error) { _ = rootBolt.Close() }() - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil, nil } @@ -87,7 +88,7 @@ func RollbackPoints(path string) ([]*RollbackPoint, error) { meta := map[string][]byte{} c2 := snapshot.Cursor() for j, _ := c2.First(); j != nil; j, _ = c2.Next() { - if j[0] == boltInternalKey[0] { + if j[0] == util.BoltInternalKey[0] { internalBucket := snapshot.Bucket(j) if internalBucket == nil { err = fmt.Errorf("internal bucket missing") @@ -151,7 +152,7 @@ func Rollback(path string, to *RollbackPoint) error { var found bool var eligibleEpochs []uint64 err = rootBolt.View(func(tx *bolt.Tx) error { - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } @@ -193,7 +194,7 @@ func Rollback(path string, to *RollbackPoint) error { } }() - snapshots := tx.Bucket(boltSnapshotsBucket) + snapshots := tx.Bucket(util.BoltSnapshotsBucket) if snapshots == nil { return nil } diff --git a/index/scorch/scorch.go b/index/scorch/scorch.go index 54dcb9274..83924978e 100644 --- a/index/scorch/scorch.go +++ b/index/scorch/scorch.go @@ -25,6 +25,7 @@ import ( "github.com/RoaringBitmap/roaring/v2" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" segment "github.com/blevesearch/scorch_segment_api/v2" bolt "go.etcd.io/bbolt" @@ -217,9 +218,11 @@ func (s *Scorch) fireAsyncError(err error) { } func (s *Scorch) Open() error { - err := s.openBolt() - if err != nil { - return err + if s.rootBolt == nil { + err := s.openBolt() + if err != nil { + return err + } } s.asyncTasks.Add(1) @@ -371,6 +374,7 @@ func (s *Scorch) Close() (err error) { } } s.root = nil + s.rootBolt = nil s.rootLock.Unlock() } @@ -940,3 +944,96 @@ func (s *Scorch) CopyReader() index.CopyReader { func (s *Scorch) FireIndexEvent() { s.fireEvent(EventKindIndexStart, 0) } + +// Updates bolt db with the given field info. Existing field info already in bolt +// will be merged before persisting. The index mapping is also overwritted both +// in bolt as well as the index snapshot +func (s *Scorch) UpdateFields(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error { + err := s.updateBolt(fieldInfo, mappingBytes) + if err != nil { + return err + } + // Pass the update field info to all snapshots and segment bases + s.root.UpdateFieldsInfo(fieldInfo) + return nil +} + +func (s *Scorch) OpenMeta() error { + if s.rootBolt == nil { + err := s.openBolt() + if err != nil { + return err + } + } + + return nil +} + +// Merge and update deleted field info and rewrite index mapping +func (s *Scorch) updateBolt(fieldInfo map[string]*index.UpdateFieldInfo, mappingBytes []byte) error { + return s.rootBolt.Update(func(tx *bolt.Tx) error { + snapshots := tx.Bucket(util.BoltSnapshotsBucket) + if snapshots == nil { + return nil + } + + c := snapshots.Cursor() + for k, _ := c.Last(); k != nil; k, _ = c.Prev() { + _, _, err := decodeUvarintAscending(k) + if err != nil { + fmt.Printf("unable to parse segment epoch %x, continuing", k) + continue + } + snapshot := snapshots.Bucket(k) + cc := snapshot.Cursor() + for kk, _ := cc.First(); kk != nil; kk, _ = cc.Next() { + if kk[0] == util.BoltInternalKey[0] { + internalBucket := snapshot.Bucket(kk) + if internalBucket == nil { + return fmt.Errorf("segment key, but bucket missing %x", kk) + } + err = internalBucket.Put(util.MappingInternalKey, mappingBytes) + if err != nil { + return err + } + } else if kk[0] != util.BoltMetaDataKey[0] { + segmentBucket := snapshot.Bucket(kk) + if segmentBucket == nil { + return fmt.Errorf("segment key, but bucket missing %x", kk) + } + var updatedFields map[string]*index.UpdateFieldInfo + updatedFieldBytes := segmentBucket.Get(util.BoltUpdatedFieldsKey) + if updatedFieldBytes != nil { + err := json.Unmarshal(updatedFieldBytes, &updatedFields) + if err != nil { + return fmt.Errorf("error reading updated field bytes: %v", err) + } + for field, info := range fieldInfo { + if val, ok := updatedFields[field]; ok { + updatedFields[field] = &index.UpdateFieldInfo{ + Deleted: info.Deleted || val.Deleted, + Store: info.Store || val.Store, + DocValues: info.DocValues || val.DocValues, + Index: info.Index || val.Index, + } + } else { + updatedFields[field] = info + } + } + } else { + updatedFields = fieldInfo + } + b, err := json.Marshal(updatedFields) + if err != nil { + return err + } + err = segmentBucket.Put(util.BoltUpdatedFieldsKey, b) + if err != nil { + return err + } + } + } + } + return nil + }) +} diff --git a/index/scorch/snapshot_index.go b/index/scorch/snapshot_index.go index 4f67a3c0b..c09a7db40 100644 --- a/index/scorch/snapshot_index.go +++ b/index/scorch/snapshot_index.go @@ -84,6 +84,13 @@ type IndexSnapshot struct { m3 sync.RWMutex // bm25 metrics specific - not to interfere with TFR creation fieldCardinality map[string]int + + // Stores information about zapx fields that have been + // fully deleted (indicated by UpdateFieldInfo.Deleted) or + // partially deleted index, store or docvalues (indicated by + // UpdateFieldInfo.Index or .Store or .DocValues). + // Used to short circuit queries trying to read stale data + updatedFields map[string]*index.UpdateFieldInfo } func (i *IndexSnapshot) Segments() []*SegmentSnapshot { @@ -509,6 +516,13 @@ func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) { // Keeping that TODO for now until we have a cleaner way. rvd.StoredFieldsSize += uint64(len(val)) + // Skip fields that have been completely deleted or had their + // store data deleted + if info, ok := is.updatedFields[name]; ok && + (info.Deleted || info.Store) { + return true + } + // copy value, array positions to preserve them beyond the scope of this callback value := append([]byte(nil), val...) arrayPos := append([]uint64(nil), pos...) @@ -634,10 +648,22 @@ func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field segBytesRead := s.segment.BytesRead() rv.incrementBytesRead(segBytesRead) } - dict, err := s.segment.Dictionary(field) + + var dict segment.TermDictionary + var err error + + // Skip fields that have been completely deleted or had their + // index data deleted + if info, ok := is.updatedFields[field]; ok && + (info.Index || info.Deleted) { + dict, err = s.segment.Dictionary("") + } else { + dict, err = s.segment.Dictionary(field) + } if err != nil { return nil, err } + if dictStats, ok := dict.(segment.DiskStatsReporter); ok { bytesRead := dictStats.BytesRead() rv.incrementBytesRead(bytesRead) @@ -783,6 +809,23 @@ func (is *IndexSnapshot) documentVisitFieldTermsOnSegment( } } + // Filter out fields that have been completely deleted or had their + // docvalues data deleted from both visitable fields and required fields + filterUpdatedFields := func(fields []string) []string { + filteredFields := make([]string, 0) + for _, field := range fields { + if info, ok := is.updatedFields[field]; ok && + (info.DocValues || info.Deleted) { + continue + } + filteredFields = append(filteredFields, field) + } + return filteredFields + } + + fieldsFiltered := filterUpdatedFields(fields) + vFieldsFiltered := filterUpdatedFields(vFields) + var errCh chan error // cFields represents the fields that we'll need from the @@ -790,7 +833,7 @@ func (is *IndexSnapshot) documentVisitFieldTermsOnSegment( // if the caller happens to know we're on the same segmentIndex // from a previous invocation if cFields == nil { - cFields = subtractStrings(fields, vFields) + cFields = subtractStrings(fieldsFiltered, vFieldsFiltered) if !ss.cachedDocs.hasFields(cFields) { errCh = make(chan error, 1) @@ -805,8 +848,8 @@ func (is *IndexSnapshot) documentVisitFieldTermsOnSegment( } } - if ssvOk && ssv != nil && len(vFields) > 0 { - dvs, err = ssv.VisitDocValues(localDocNum, fields, visitor, dvs) + if ssvOk && ssv != nil && len(vFieldsFiltered) > 0 { + dvs, err = ssv.VisitDocValues(localDocNum, fieldsFiltered, visitor, dvs) if err != nil { return nil, nil, err } @@ -1161,3 +1204,33 @@ func (is *IndexSnapshot) ThesaurusKeysRegexp(name string, func (is *IndexSnapshot) UpdateSynonymSearchCount(delta uint64) { atomic.AddUint64(&is.parent.stats.TotSynonymSearches, delta) } + +// Update current snapshot updated field data as well as pass it on to all segments and segment bases +func (is *IndexSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + is.m.Lock() + defer is.m.Unlock() + + is.MergeUpdateFieldsInfo(updatedFields) + + for _, segmentSnapshot := range is.segment { + segmentSnapshot.UpdateFieldsInfo(is.updatedFields) + } +} + +// Merge given updated field information with existing updated field information +func (is *IndexSnapshot) MergeUpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + if is.updatedFields == nil { + is.updatedFields = updatedFields + } else { + for fieldName, info := range updatedFields { + if val, ok := is.updatedFields[fieldName]; ok { + val.Deleted = val.Deleted || info.Deleted + val.Index = val.Index || info.Index + val.DocValues = val.DocValues || info.DocValues + val.Store = val.Store || info.Store + } else { + is.updatedFields[fieldName] = info + } + } + } +} diff --git a/index/scorch/snapshot_index_vr.go b/index/scorch/snapshot_index_vr.go index 7c6741125..3f2a43a12 100644 --- a/index/scorch/snapshot_index_vr.go +++ b/index/scorch/snapshot_index_vr.go @@ -83,6 +83,10 @@ func (i *IndexSnapshotVectorReader) Next(preAlloced *index.VectorDoc) ( } for i.segmentOffset < len(i.iterators) { + if i.iterators[i.segmentOffset] == nil { + i.segmentOffset++ + continue + } next, err := i.iterators[i.segmentOffset].Next() if err != nil { return nil, err diff --git a/index/scorch/snapshot_segment.go b/index/scorch/snapshot_segment.go index ec65bf800..c6f3584cc 100644 --- a/index/scorch/snapshot_segment.go +++ b/index/scorch/snapshot_segment.go @@ -35,12 +35,13 @@ type SegmentSnapshot struct { // segment was mmaped recently, in which case // we consider the loading cost of the metadata // as part of IO stats. - mmaped uint32 - id uint64 - segment segment.Segment - deleted *roaring.Bitmap - creator string - stats *fieldStats + mmaped uint32 + id uint64 + segment segment.Segment + deleted *roaring.Bitmap + creator string + stats *fieldStats + updatedFields map[string]*index.UpdateFieldInfo cachedMeta *cachedMeta @@ -146,6 +147,28 @@ func (s *SegmentSnapshot) Size() (rv int) { return } +// Merge given updated field information with existing and pass it on to the segment base +func (s *SegmentSnapshot) UpdateFieldsInfo(updatedFields map[string]*index.UpdateFieldInfo) { + if s.updatedFields == nil { + s.updatedFields = updatedFields + } else { + for fieldName, info := range updatedFields { + if val, ok := s.updatedFields[fieldName]; ok { + val.Deleted = val.Deleted || info.Deleted + val.Index = val.Index || info.Index + val.DocValues = val.DocValues || info.DocValues + val.Store = val.Store || info.Store + } else { + s.updatedFields[fieldName] = info + } + } + } + + if segment, ok := s.segment.(segment.UpdatableSegment); ok { + segment.SetUpdatedFields(s.updatedFields) + } +} + type cachedFieldDocs struct { m sync.Mutex readyCh chan struct{} // closed when the cachedFieldDocs.docs is ready to be used. diff --git a/index_impl.go b/index_impl.go index 5cc0c5899..303a0dcec 100644 --- a/index_impl.go +++ b/index_impl.go @@ -133,7 +133,7 @@ func newIndexUsing(path string, mapping mapping.IndexMapping, indexType string, if err != nil { return nil, err } - err = rv.i.SetInternal(mappingInternalKey, mappingBytes) + err = rv.i.SetInternal(util.MappingInternalKey, mappingBytes) if err != nil { return nil, err } @@ -163,6 +163,9 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde rv.meta.IndexType = upsidedown.Name } + var um *mapping.IndexMappingImpl + var umBytes []byte + storeConfig := rv.meta.Config if storeConfig == nil { storeConfig = map[string]interface{}{} @@ -173,6 +176,21 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde storeConfig["error_if_exists"] = false for rck, rcv := range runtimeConfig { storeConfig[rck] = rcv + if rck == "updated_mapping" { + if val, ok := rcv.(string); ok { + if len(val) == 0 { + return nil, fmt.Errorf("updated_mapping is empty") + } + umBytes = []byte(val) + + err = util.UnmarshalJSON(umBytes, &um) + if err != nil { + return nil, fmt.Errorf("error parsing updated_mapping into JSON: %v\nmapping contents:\n%v", err, rck) + } + } else { + return nil, fmt.Errorf("updated_mapping not of type string") + } + } } // open the index @@ -185,15 +203,32 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde if err != nil { return nil, err } - err = rv.i.Open() - if err != nil { - return nil, err - } - defer func(rv *indexImpl) { - if !rv.open { - rv.i.Close() + + var ui index.UpdateIndex + if um != nil { + var ok bool + ui, ok = rv.i.(index.UpdateIndex) + if !ok { + return nil, fmt.Errorf("updated mapping present for unupdatable index") + } + + // Load the meta data from bolt so that we can read the current index + // mapping to compare with + err = ui.OpenMeta() + if err != nil { + return nil, err + } + } else { + err = rv.i.Open() + if err != nil { + return nil, err } - }(rv) + defer func(rv *indexImpl) { + if !rv.open { + rv.i.Close() + } + }(rv) + } // now load the mapping indexReader, err := rv.i.Reader() @@ -206,7 +241,7 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde } }() - mappingBytes, err := indexReader.GetInternal(mappingInternalKey) + mappingBytes, err := indexReader.GetInternal(util.MappingInternalKey) if err != nil { return nil, err } @@ -217,19 +252,48 @@ func openIndexUsing(path string, runtimeConfig map[string]interface{}) (rv *inde return nil, fmt.Errorf("error parsing mapping JSON: %v\nmapping contents:\n%s", err, string(mappingBytes)) } - // mark the index as open - rv.mutex.Lock() - defer rv.mutex.Unlock() - rv.open = true - // validate the mapping err = im.Validate() if err != nil { - // note even if the mapping is invalid - // we still return an open usable index - return rv, err + // no longer return usable index on error because there + // is a chance the index is not open at this stage + return nil, err } + // Validate and update the index with the new mapping + if um != nil && ui != nil { + err = um.Validate() + if err != nil { + return nil, err + } + + fieldInfo, err := DeletedFields(im, um) + if err != nil { + return nil, err + } + + err = ui.UpdateFields(fieldInfo, umBytes) + if err != nil { + return nil, err + } + im = um + + err = rv.i.Open() + if err != nil { + return nil, err + } + defer func(rv *indexImpl) { + if !rv.open { + rv.i.Close() + } + }(rv) + } + + // mark the index as open + rv.mutex.Lock() + defer rv.mutex.Unlock() + rv.open = true + rv.m = im indexStats.Register(rv) return rv, err diff --git a/index_update.go b/index_update.go new file mode 100644 index 000000000..fa9789bb1 --- /dev/null +++ b/index_update.go @@ -0,0 +1,595 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "fmt" + "reflect" + + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" +) + +// Store all the fields that interact with the data +// from a document path +type pathInfo struct { + fieldMapInfo []*fieldMapInfo + dynamic bool + path string + analyser string + parentPath string +} + +// Store the field information with respect to the +// document paths +type fieldMapInfo struct { + fieldMapping *mapping.FieldMapping + analyzer string + datetimeParser string + rootName string + parent *pathInfo +} + +// Compare two index mappings to identify all of the updatable changes +func DeletedFields(ori, upd *mapping.IndexMappingImpl) (map[string]*index.UpdateFieldInfo, error) { + // Compare all of the top level fields in an index mapping + err := compareMappings(ori, upd) + if err != nil { + return nil, err + } + + // Check for new mappings present in the type mappings + // of the updated compared to the original + for name, updDMapping := range upd.TypeMapping { + err = checkUpdatedMapping(ori.TypeMapping[name], updDMapping) + if err != nil { + return nil, err + } + } + + // Check for new mappings present in the default mappings + // of the updated compared to the original + err = checkUpdatedMapping(ori.DefaultMapping, upd.DefaultMapping) + if err != nil { + return nil, err + } + + oriPaths := make(map[string]*pathInfo) + updPaths := make(map[string]*pathInfo) + + // Go through each mapping present in the original + // and consolidate according to the document paths + for name, oriDMapping := range ori.TypeMapping { + addPathInfo(oriPaths, "", oriDMapping, ori, nil, name) + } + addPathInfo(oriPaths, "", ori.DefaultMapping, ori, nil, "") + + // Go through each mapping present in the updated + // and consolidate according to the document paths + for name, updDMapping := range upd.TypeMapping { + addPathInfo(updPaths, "", updDMapping, upd, nil, name) + } + addPathInfo(updPaths, "", upd.DefaultMapping, upd, nil, "") + + // Compare all components of custom analysis currently in use + err = compareCustomComponents(oriPaths, updPaths, ori, upd) + if err != nil { + return nil, err + } + + // Compare both the mappings based on the document paths + // and create a list of index, docvalues, store differences + // for every single field possible + fieldInfo := make(map[string]*index.UpdateFieldInfo) + for path, info := range oriPaths { + err = addFieldInfo(fieldInfo, info, updPaths[path]) + if err != nil { + return nil, err + } + } + + // Remove entries from the list with no changes between the + // original and the updated mapping + for name, info := range fieldInfo { + if !info.Deleted && !info.Index && !info.DocValues && !info.Store { + delete(fieldInfo, name) + } + // A field cannot be completely deleted with any dynamic value turned on + if info.Deleted { + if upd.IndexDynamic { + return nil, fmt.Errorf("Mapping cannot be removed when index dynamic is true") + } + if upd.StoreDynamic { + return nil, fmt.Errorf("Mapping cannot be removed when store dynamic is true") + } + if upd.DocValuesDynamic { + return nil, fmt.Errorf("Mapping cannot be removed when docvalues dynamic is true") + } + } + } + return fieldInfo, nil +} + +// Ensures none of the top level index mapping fields have changed +func compareMappings(ori, upd *mapping.IndexMappingImpl) error { + if ori.TypeField != upd.TypeField && + (len(ori.TypeMapping) != 0 || len(upd.TypeMapping) != 0) { + return fmt.Errorf("type field cannot be changed when type mappings are present") + } + + if ori.DefaultType != upd.DefaultType { + return fmt.Errorf("default type cannot be changed") + } + + if ori.IndexDynamic != upd.IndexDynamic { + return fmt.Errorf("index dynamic cannot be changed") + } + + if ori.StoreDynamic != upd.StoreDynamic { + return fmt.Errorf("store dynamic cannot be changed") + } + + if ori.DocValuesDynamic != upd.DocValuesDynamic { + return fmt.Errorf("docvalues dynamic cannot be changed") + } + + if ori.DefaultAnalyzer != upd.DefaultAnalyzer && upd.IndexDynamic { + return fmt.Errorf("default analyser cannot be changed if index dynamic is true") + } + + if ori.DefaultDateTimeParser != upd.DefaultDateTimeParser && upd.IndexDynamic { + return fmt.Errorf("default datetime parser cannot be changed if index dynamic is true") + } + + // Scoring model changes between "", "tf-idf" and "bm25" require no index changes to be made + if ori.ScoringModel != upd.ScoringModel { + if ori.ScoringModel != "" && ori.ScoringModel != index.TFIDFScoring && ori.ScoringModel != index.BM25Scoring || + upd.ScoringModel != "" && upd.ScoringModel != index.TFIDFScoring && upd.ScoringModel != index.BM25Scoring { + return fmt.Errorf("scoring model can only be changed between \"\", %q and %q", index.TFIDFScoring, index.BM25Scoring) + } + } + + return nil +} + +// Ensures updated document mapping does not contain new +// field mappings or document mappings +func checkUpdatedMapping(ori, upd *mapping.DocumentMapping) error { + // Check to verify both original and updated are not nil + // and are enabled before proceeding + if ori == nil { + if upd == nil || !upd.Enabled { + return nil + } + return fmt.Errorf("updated index mapping contains new properties") + } + + if upd == nil || !upd.Enabled { + return nil + } + + var err error + // Recursively go through the child mappings + for name, updDMapping := range upd.Properties { + err = checkUpdatedMapping(ori.Properties[name], updDMapping) + if err != nil { + return err + } + } + + // Simple checks to ensure no new field mappings present + // in updated + for _, updFMapping := range upd.Fields { + var oriFMapping *mapping.FieldMapping + for _, fMapping := range ori.Fields { + if updFMapping.Name == fMapping.Name { + oriFMapping = fMapping + } + } + if oriFMapping == nil { + return fmt.Errorf("updated index mapping contains new fields") + } + } + + return nil +} + +// Adds all of the field mappings while maintaining a tree of the document structure +// to ensure traversal and verification is possible incase of multiple mappings defined +// for a single field or multiple document fields' data getting written to a single zapx field +func addPathInfo(paths map[string]*pathInfo, name string, mp *mapping.DocumentMapping, + im *mapping.IndexMappingImpl, parent *pathInfo, rootName string) { + // Early exit if mapping has been disabled + // Comparisions later on will be done with a nil object + if !mp.Enabled { + return + } + + // Consolidate path information like index dynamic across multiple + // mappings if path is the same + var pInfo *pathInfo + if val, ok := paths[name]; ok { + pInfo = val + } else { + pInfo = &pathInfo{ + fieldMapInfo: make([]*fieldMapInfo, 0), + } + pInfo.dynamic = mp.Dynamic && im.IndexDynamic + pInfo.analyser = im.AnalyzerNameForPath(name) + } + + pInfo.dynamic = (pInfo.dynamic || mp.Dynamic) && im.IndexDynamic + pInfo.path = name + if parent != nil { + pInfo.parentPath = parent.path + } + + // Recursively add path information for all child mappings + for cName, cMapping := range mp.Properties { + var pathName string + if name == "" { + pathName = cName + } else { + pathName = name + "." + cName + } + addPathInfo(paths, pathName, cMapping, im, pInfo, rootName) + } + + // Add field mapping information keeping the document structure intact + for _, fMap := range mp.Fields { + fieldMapInfo := &fieldMapInfo{ + fieldMapping: fMap, + rootName: rootName, + parent: pInfo, + } + pInfo.fieldMapInfo = append(pInfo.fieldMapInfo, fieldMapInfo) + } + + paths[name] = pInfo +} + +// Compares all of the custom analysis components in use +func compareCustomComponents(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + // Compare all analysers currently in use + err := compareAnalysers(oriPaths, updPaths, ori, upd) + if err != nil { + return err + } + + // Compare all datetime parsers currently in use + err = compareDateTimeParsers(oriPaths, updPaths, ori, upd) + if err != nil { + return err + } + + // Compare all synonum sources + err = compareSynonymSources(ori, upd) + if err != nil { + return err + } + + // Compare all char filters, tokenizers, token filters and token maps + err = compareAnalyserSubcomponents(ori, upd) + if err != nil { + return err + } + + return nil +} + +// Compares all analysers currently in use +// Standard analysers not in custom analysis are not compared +// Analysers in custom analysis but not in use are not compared +func compareAnalysers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + oriAnalyzers := make(map[string]interface{}) + updAnalyzers := make(map[string]interface{}) + + extractAnalyzers := func(paths map[string]*pathInfo, customAnalyzers map[string]map[string]interface{}, + analyzers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) { + for path, info := range paths { + for _, fInfo := range info.fieldMapInfo { + if fInfo.fieldMapping.Type == "text" { + analyzerName := indexMapping.AnalyzerNameForPath(path) + fInfo.analyzer = analyzerName + if val, ok := customAnalyzers[analyzerName]; ok { + analyzers[analyzerName] = val + } + } + } + } + } + + extractAnalyzers(oriPaths, ori.CustomAnalysis.Analyzers, oriAnalyzers, ori) + extractAnalyzers(updPaths, upd.CustomAnalysis.Analyzers, updAnalyzers, upd) + + for name, anUpd := range updAnalyzers { + if anOri, ok := oriAnalyzers[name]; ok { + if !reflect.DeepEqual(anUpd, anOri) { + return fmt.Errorf("analyser %s changed while being used by fields", name) + } + } else { + return fmt.Errorf("analyser %s newly added to an existing field", name) + } + } + + return nil +} + +// Compares all date time parsers currently in use +// Date time parsers in custom analysis but not in use are not compared +func compareDateTimeParsers(oriPaths, updPaths map[string]*pathInfo, ori, upd *mapping.IndexMappingImpl) error { + oriDateTimeParsers := make(map[string]interface{}) + updDateTimeParsers := make(map[string]interface{}) + + extractDateTimeParsers := func(paths map[string]*pathInfo, customParsers map[string]map[string]interface{}, + parsers map[string]interface{}, indexMapping *mapping.IndexMappingImpl) { + for _, info := range paths { + for _, fInfo := range info.fieldMapInfo { + if fInfo.fieldMapping.Type == "datetime" { + parserName := fInfo.fieldMapping.DateFormat + if parserName == "" { + parserName = indexMapping.DefaultDateTimeParser + } + fInfo.datetimeParser = parserName + if val, ok := customParsers[parserName]; ok { + parsers[parserName] = val + } + } + } + } + } + + extractDateTimeParsers(oriPaths, ori.CustomAnalysis.DateTimeParsers, oriDateTimeParsers, ori) + extractDateTimeParsers(updPaths, upd.CustomAnalysis.DateTimeParsers, updDateTimeParsers, upd) + + for name, dtUpd := range updDateTimeParsers { + if dtOri, ok := oriDateTimeParsers[name]; ok { + if !reflect.DeepEqual(dtUpd, dtOri) { + return fmt.Errorf("datetime parser %s changed while being used by fields", name) + } + } else { + return fmt.Errorf("datetime parser %s added to an existing field", name) + } + } + + return nil +} + +// Compares all synonym sources +// Synonym sources currently not in use are also compared +func compareSynonymSources(ori, upd *mapping.IndexMappingImpl) error { + if !reflect.DeepEqual(ori.CustomAnalysis.SynonymSources, upd.CustomAnalysis.SynonymSources) { + return fmt.Errorf("synonym sources cannot be changed") + } + + return nil +} + +// Compares all char filters, tokenizers, token filters and token maps +// Components not currently in use are also compared +func compareAnalyserSubcomponents(ori, upd *mapping.IndexMappingImpl) error { + if !reflect.DeepEqual(ori.CustomAnalysis.CharFilters, upd.CustomAnalysis.CharFilters) { + return fmt.Errorf("char filters cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.TokenFilters, upd.CustomAnalysis.TokenFilters) { + return fmt.Errorf("token filters cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.TokenMaps, upd.CustomAnalysis.TokenMaps) { + return fmt.Errorf("token maps cannot be changed") + } + + if !reflect.DeepEqual(ori.CustomAnalysis.Tokenizers, upd.CustomAnalysis.Tokenizers) { + return fmt.Errorf("tokenizers cannot be changed") + } + + return nil +} + +// Compare all of the fields at a particular document path and add its field information +func addFieldInfo(fInfo map[string]*index.UpdateFieldInfo, ori, upd *pathInfo) error { + var info *index.UpdateFieldInfo + var err error + + // Assume deleted or disabled mapping if upd is nil. Checks for ori being nil + // or upd having mappings not in orihave already been done before this stage + if upd == nil { + for _, oriFMapInfo := range ori.fieldMapInfo { + info, err = compareFieldMapping(oriFMapInfo.fieldMapping, nil) + if err != nil { + return err + } + err = validateFieldInfo(info, fInfo, ori, oriFMapInfo) + if err != nil { + return err + } + } + } else { + if upd.dynamic && ori.analyser != upd.analyser { + return fmt.Errorf("analyser has been changed for a dynamic mapping") + } + for _, oriFMapInfo := range ori.fieldMapInfo { + var updFMap *mapping.FieldMapping + var updAnalyser string + var updDatetimeParser string + + // For multiple fields at a single document path, compare + // only with the matching ones + for _, updFMapInfo := range upd.fieldMapInfo { + if oriFMapInfo.rootName == updFMapInfo.rootName && + oriFMapInfo.fieldMapping.Name == updFMapInfo.fieldMapping.Name { + updFMap = updFMapInfo.fieldMapping + if updFMap.Type == "text" { + updAnalyser = updFMapInfo.analyzer + } else if updFMap.Type == "datetime" { + updDatetimeParser = updFMapInfo.datetimeParser + } + } + } + // Compare analyser, datetime parser and synonym source before comparing + // the field mapping as it might not have this information + if updAnalyser != "" && oriFMapInfo.analyzer != updAnalyser { + return fmt.Errorf("analyser has been changed for a text field") + } + if updDatetimeParser != "" && oriFMapInfo.datetimeParser != updDatetimeParser { + return fmt.Errorf("datetime parser has been changed for a date time field") + } + info, err = compareFieldMapping(oriFMapInfo.fieldMapping, updFMap) + if err != nil { + return err + } + + // Validate to ensure change is possible + // Needed if multiple mappings are aliased to the same field + err = validateFieldInfo(info, fInfo, ori, oriFMapInfo) + if err != nil { + return err + } + } + } + if err != nil { + return err + } + + return nil +} + +// Compares two field mappings against each other, checking for changes in index, store, doc values +// and complete deletiion of the mapping while noting that the changes made are doable based on +// other values like includeInAll and dynamic +// first return argument gives an empty fieldInfo if no changes detected +// second return argument gives a flag indicating whether any changes, if detected, are doable or if +// update is impossible +// third argument is an error explaining exactly why the change is not possible +func compareFieldMapping(original, updated *mapping.FieldMapping) (*index.UpdateFieldInfo, error) { + rv := &index.UpdateFieldInfo{} + + if updated == nil { + if original != nil && !original.IncludeInAll { + rv.Deleted = true + return rv, nil + } else if original == nil { + return nil, fmt.Errorf("both field mappings cannot be nil") + } + return nil, fmt.Errorf("deleted field present in '_all' field") + } else if original == nil { + return nil, fmt.Errorf("matching field not found in original index mapping") + } + + if original.Type != updated.Type { + return nil, fmt.Errorf("field type cannot be updated") + } + if original.Type == "text" { + if original.Analyzer != updated.Analyzer { + return nil, fmt.Errorf("analyzer cannot be updated for text fields") + } + } + if original.Type == "datetime" { + if original.DateFormat != updated.DateFormat { + return nil, fmt.Errorf("dateFormat cannot be updated for datetime fields") + } + } + if original.Type == "vector" || original.Type == "vector_base64" { + if original.Dims != updated.Dims { + return nil, fmt.Errorf("dimensions cannot be updated for vector and vector_base64 fields") + } + if original.Similarity != updated.Similarity { + return nil, fmt.Errorf("similarity cannot be updated for vector and vector_base64 fields") + } + if original.VectorIndexOptimizedFor != updated.VectorIndexOptimizedFor { + return nil, fmt.Errorf("vectorIndexOptimizedFor cannot be updated for vector and vector_base64 fields") + } + } + if original.IncludeInAll != updated.IncludeInAll { + return nil, fmt.Errorf("includeInAll cannot be changed") + } + if original.IncludeTermVectors != updated.IncludeTermVectors { + return nil, fmt.Errorf("includeTermVectors cannot be changed") + } + if original.SkipFreqNorm != updated.SkipFreqNorm { + return nil, fmt.Errorf("skipFreqNorm cannot be changed") + } + + // Updating is not possible if store changes from true + // to false when the field is included in _all + if original.Store != updated.Store { + if updated.Store { + return nil, fmt.Errorf("store cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("store cannot be changed if field present in `_all' field") + } else { + rv.Store = true + } + } + + // Updating is not possible if index changes from true + // to false when the field is included in _all + if original.Index != updated.Index { + if updated.Index { + return nil, fmt.Errorf("index cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("index cannot be changed if field present in `_all' field") + } else { + rv.Index = true + rv.DocValues = true + } + } + + // Updating is not possible if docvalues changes from true + // to false when the field is included in _all + if original.DocValues != updated.DocValues { + if updated.DocValues { + return nil, fmt.Errorf("docvalues cannot be changed from false to true") + } else if updated.IncludeInAll { + return nil, fmt.Errorf("docvalues cannot be changed if field present in `_all' field") + } else { + rv.DocValues = true + } + } + + return rv, nil +} + +// After identifying changes, validate against the existing changes incase of duplicate fields. +// In such a situation, any conflicting changes found will abort the update process +func validateFieldInfo(newInfo *index.UpdateFieldInfo, fInfo map[string]*index.UpdateFieldInfo, + ori *pathInfo, oriFMapInfo *fieldMapInfo) error { + var name string + if oriFMapInfo.parent.parentPath == "" { + if oriFMapInfo.fieldMapping.Name == "" { + name = oriFMapInfo.parent.path + } else { + name = oriFMapInfo.fieldMapping.Name + } + } else { + if oriFMapInfo.fieldMapping.Name == "" { + name = oriFMapInfo.parent.parentPath + "." + oriFMapInfo.parent.path + } else { + name = oriFMapInfo.parent.parentPath + "." + oriFMapInfo.fieldMapping.Name + } + } + if (newInfo.Deleted || newInfo.Index || newInfo.DocValues || newInfo.Store) && ori.dynamic { + return fmt.Errorf("updated field is under a dynamic property") + } + if oldInfo, ok := fInfo[name]; ok { + if !reflect.DeepEqual(oldInfo, newInfo) { + return fmt.Errorf("updated field impossible to verify because multiple mappings point to the same field name") + } + } else { + fInfo[name] = newInfo + } + return nil +} diff --git a/index_update_test.go b/index_update_test.go new file mode 100644 index 000000000..5d6326576 --- /dev/null +++ b/index_update_test.go @@ -0,0 +1,3084 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bleve + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "reflect" + "testing" + + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/analyzer/simple" + "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" + "github.com/blevesearch/bleve/v2/analysis/datetime/percent" + "github.com/blevesearch/bleve/v2/analysis/datetime/sanitized" + "github.com/blevesearch/bleve/v2/analysis/lang/en" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/letter" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/whitespace" + "github.com/blevesearch/bleve/v2/index/scorch" + "github.com/blevesearch/bleve/v2/index/scorch/mergeplan" + "github.com/blevesearch/bleve/v2/mapping" + index "github.com/blevesearch/bleve_index_api" +) + +func TestCompareFieldMapping(t *testing.T) { + tests := []struct { + original *mapping.FieldMapping + updated *mapping.FieldMapping + indexFieldInfo *index.UpdateFieldInfo + err bool + }{ + { // both nil => error + original: nil, + updated: nil, + indexFieldInfo: nil, + err: true, + }, + { // updated nil => delete all + original: &mapping.FieldMapping{}, + updated: nil, + indexFieldInfo: &index.UpdateFieldInfo{ + Deleted: true, + }, + err: false, + }, + { // type changed => not updatable + original: &mapping.FieldMapping{ + Type: "text", + }, + updated: &mapping.FieldMapping{ + Type: "datetime", + }, + indexFieldInfo: nil, + err: true, + }, + { // synonym source changed for text => updatable + original: &mapping.FieldMapping{ + Type: "text", + SynonymSource: "a", + }, + updated: &mapping.FieldMapping{ + Type: "text", + SynonymSource: "b", + }, + indexFieldInfo: &index.UpdateFieldInfo{}, + err: false, + }, + { // analyser changed for text => not updatable + original: &mapping.FieldMapping{ + Type: "text", + Analyzer: "a", + }, + updated: &mapping.FieldMapping{ + Type: "text", + Analyzer: "b", + }, + indexFieldInfo: nil, + err: true, + }, + { // dims changed for vector => not updatable + original: &mapping.FieldMapping{ + Type: "vector", + Dims: 128, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector", + Dims: 1024, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "memory-efficient", + }, + indexFieldInfo: nil, + err: true, + }, + { // similarity changed for vectorbase64 => not updatable + original: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "l2_norm", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector_base64", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + indexFieldInfo: nil, + err: true, + }, + { // vectorindexoptimizedfor chagned for vector => not updatable + original: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "memory-efficient", + }, + updated: &mapping.FieldMapping{ + Type: "vector", + Similarity: "dot_product", + Dims: 128, + VectorIndexOptimizedFor: "latency", + }, + indexFieldInfo: nil, + err: true, + }, + { // includeinall changed => not updatable + original: &mapping.FieldMapping{ + Type: "numeric", + IncludeInAll: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + IncludeInAll: false, + }, + indexFieldInfo: nil, + err: true, + }, + { //includetermvectors changed => not updatable + original: &mapping.FieldMapping{ + Type: "numeric", + IncludeTermVectors: false, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + IncludeTermVectors: true, + }, + indexFieldInfo: nil, + err: true, + }, + { // store changed after all checks => updatable with store delete + original: &mapping.FieldMapping{ + Type: "numeric", + SkipFreqNorm: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + SkipFreqNorm: false, + }, + indexFieldInfo: nil, + err: true, + }, + { // index changed after all checks => updatable with index and docvalues delete + original: &mapping.FieldMapping{ + Type: "geopoint", + Index: true, + }, + updated: &mapping.FieldMapping{ + Type: "geopoint", + Index: false, + }, + indexFieldInfo: &index.UpdateFieldInfo{ + Index: true, + DocValues: true, + }, + err: false, + }, + { // docvalues changed after all checks => docvalues delete + original: &mapping.FieldMapping{ + Type: "numeric", + DocValues: true, + }, + updated: &mapping.FieldMapping{ + Type: "numeric", + DocValues: false, + }, + indexFieldInfo: &index.UpdateFieldInfo{ + DocValues: true, + }, + err: false, + }, + { // no relavent changes => continue but no op + original: &mapping.FieldMapping{ + Name: "", + Type: "datetime", + Analyzer: "a", + Store: true, + Index: false, + IncludeTermVectors: true, + IncludeInAll: false, + DateFormat: "a", + DocValues: false, + SkipFreqNorm: true, + Dims: 128, + Similarity: "dot_product", + VectorIndexOptimizedFor: "memory-efficient", + SynonymSource: "a", + }, + updated: &mapping.FieldMapping{ + Name: "", + Type: "datetime", + Analyzer: "b", + Store: true, + Index: false, + IncludeTermVectors: true, + IncludeInAll: false, + DateFormat: "a", + DocValues: false, + SkipFreqNorm: true, + Dims: 256, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + SynonymSource: "b", + }, + indexFieldInfo: &index.UpdateFieldInfo{}, + err: false, + }, + } + + for i, test := range tests { + rv, err := compareFieldMapping(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + if rv == nil && test.indexFieldInfo != nil || rv != nil && test.indexFieldInfo == nil || !reflect.DeepEqual(rv, test.indexFieldInfo) { + t.Errorf("Unexpected index field info value for test %d, expecting %+v, got %+v, err %v", i, test.indexFieldInfo, rv, err) + } + } +} + +func TestCompareMappings(t *testing.T) { + tests := []struct { + original *mapping.IndexMappingImpl + updated *mapping.IndexMappingImpl + err bool + }{ + { // changed type field when non empty mappings are present => error + original: &mapping.IndexMappingImpl{ + TypeField: "a", + TypeMapping: map[string]*mapping.DocumentMapping{ + "a": {}, + "b": {}, + }, + }, + updated: &mapping.IndexMappingImpl{ + TypeField: "b", + TypeMapping: map[string]*mapping.DocumentMapping{ + "a": {}, + "b": {}, + }, + }, + err: true, + }, + { // changed default type => error + original: &mapping.IndexMappingImpl{ + DefaultType: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultType: "b", + }, + err: true, + }, + { // changed default analyzer => analyser true + original: &mapping.IndexMappingImpl{ + DefaultAnalyzer: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultAnalyzer: "b", + }, + err: false, + }, + { // changed default datetimeparser => datetimeparser true + original: &mapping.IndexMappingImpl{ + DefaultDateTimeParser: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultDateTimeParser: "b", + }, + err: false, + }, + { // changed default synonym source => synonym source true + original: &mapping.IndexMappingImpl{ + DefaultSynonymSource: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultSynonymSource: "b", + }, + err: false, + }, + { // changed default field => false + original: &mapping.IndexMappingImpl{ + DefaultField: "a", + }, + updated: &mapping.IndexMappingImpl{ + DefaultField: "b", + }, + err: false, + }, + { // changed index dynamic => error + original: &mapping.IndexMappingImpl{ + IndexDynamic: true, + }, + updated: &mapping.IndexMappingImpl{ + IndexDynamic: false, + }, + err: true, + }, + { // changed store dynamic => error + original: &mapping.IndexMappingImpl{ + StoreDynamic: false, + }, + updated: &mapping.IndexMappingImpl{ + StoreDynamic: true, + }, + err: true, + }, + { // changed docvalues dynamic => error + original: &mapping.IndexMappingImpl{ + DocValuesDynamic: true, + }, + updated: &mapping.IndexMappingImpl{ + DocValuesDynamic: false, + }, + err: true, + }, + } + + for i, test := range tests { + err := compareMappings(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + } +} + +func TestCompareAnalysers(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + ori.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + ori.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + upd.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + upd.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + if err := ori.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + if err := upd.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + oriPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + updPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "text", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + // Test case has identical analysers for text fields + err := compareAnalysers(oriPaths, updPaths, ori, upd) + if err != nil { + t.Errorf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + ori2.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + ori2.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("c", NewTextFieldMapping()) + upd2.DefaultMapping.Properties["b"].DefaultAnalyzer = "3xbla" + upd2.DefaultMapping.Properties["c"].DefaultAnalyzer = simple.Name + + if err := ori2.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": whitespace.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + if err := upd2.AddCustomAnalyzer("3xbla", map[string]interface{}{ + "type": custom.Name, + "tokenizer": letter.Name, + "token_filters": []interface{}{lowercase.Name, "stop_en"}, + }); err != nil { + t.Fatal(err) + } + + // Test case has different custom analyser for field "b" + err = compareAnalysers(oriPaths, updPaths, ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestCompareDatetimeParsers(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + ori.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + ori.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + upd.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + upd.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + err := ori.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + err = upd.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + oriPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + DateFormat: "customDT", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + updPaths := map[string]*pathInfo{ + "a": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "a", + parentPath: "", + }, + "b": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + DateFormat: "customDT", + }, + }, + }, + dynamic: false, + path: "b", + parentPath: "", + }, + "c": { + fieldMapInfo: []*fieldMapInfo{ + { + fieldMapping: &mapping.FieldMapping{ + Type: "datetime", + }, + }, + }, + dynamic: false, + path: "c", + parentPath: "", + }, + } + + // Test case has identical datetime parsers for all fields + err = compareDateTimeParsers(oriPaths, updPaths, ori, upd) + if err != nil { + t.Fatalf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + ori2.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + ori2.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewDateTimeFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewDateTimeFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("c", NewDateTimeFieldMapping()) + upd2.DefaultMapping.Properties["b"].Fields[0].DateFormat = "customDT" + upd2.DefaultMapping.Properties["c"].Fields[0].DateFormat = percent.Name + + err = ori2.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02 3:04PM", + }, + }) + if err != nil { + t.Fatal(err) + } + + err = upd2.AddCustomDateTimeParser("customDT", map[string]interface{}{ + "type": sanitized.Name, + "layouts": []interface{}{ + "02/01/2006 15:04:05", + "2006/01/02", + }, + }) + if err != nil { + t.Fatal(err) + } + + // test case has different custom datetime parser for field "b" + err = compareDateTimeParsers(oriPaths, updPaths, ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestCompareSynonymSources(t *testing.T) { + + ori := mapping.NewIndexMapping() + ori.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori.DefaultMapping.DefaultSynonymSource = "syn1" + ori.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + upd := mapping.NewIndexMapping() + upd.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd.DefaultMapping.DefaultSynonymSource = "syn1" + upd.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + err := ori.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = ori.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + err = upd.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = upd.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + // Test case has identical synonym sources + err = compareSynonymSources(ori, upd) + if err != nil { + t.Errorf("Expected error to be nil, got %v", err) + } + + ori2 := mapping.NewIndexMapping() + ori2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + ori2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + ori2.DefaultMapping.DefaultSynonymSource = "syn1" + ori2.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + upd2 := mapping.NewIndexMapping() + upd2.DefaultMapping.AddFieldMappingsAt("a", NewTextFieldMapping()) + upd2.DefaultMapping.AddFieldMappingsAt("b", NewTextFieldMapping()) + upd2.DefaultMapping.DefaultSynonymSource = "syn1" + upd2.DefaultMapping.Properties["b"].Fields[0].SynonymSource = "syn2" + + err = ori2.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = ori2.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col2", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + err = upd2.AddSynonymSource("syn1", map[string]interface{}{ + "collection": "col1", + "analyzer": simple.Name, + }) + if err != nil { + t.Fatal(err) + } + err = upd2.AddSynonymSource("syn2", map[string]interface{}{ + "collection": "col3", + "analyzer": standard.Name, + }) + if err != nil { + t.Fatal(err) + } + + // test case has different synonym sources + err = compareSynonymSources(ori2, upd2) + if err == nil { + t.Errorf("Expected error, got nil") + } +} + +func TestDeletedFields(t *testing.T) { + tests := []struct { + original *mapping.IndexMappingImpl + updated *mapping.IndexMappingImpl + fieldInfo map[string]*index.UpdateFieldInfo + err bool + }{ + { + // changed default analyzer with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultAnalyzer: standard.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultAnalyzer: simple.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // changed default analyzer within a mapping with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: true, + DefaultAnalyzer: standard.Name, + }, + DefaultAnalyzer: "", + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: true, + DefaultAnalyzer: simple.Name, + }, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // changed default datetime parser with index dynamic + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultDateTimeParser: percent.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{}, + DefaultDateTimeParser: sanitized.Name, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // no change between original and updated having type and default mapping + // => empty fieldInfo with no error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{}, + err: false, + }, + { + // no changes in type mappings and default mapping disabled with changes + // => empty fieldInfo with no error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: false, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: false, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{}, + err: false, + }, + { + // new type mappings in updated => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // new mappings in default mapping => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{}, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // fully removed mapping in type with some dynamic => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: true, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // semi removed mapping in default with some dynamic + // proper fieldInfo with no errors + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "b": { + Index: true, + DocValues: true, + }, + }, + err: false, + }, + { + // two fields from diff paths with removed content matching + // => relavent fieldInfo + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + }, + err: false, + }, + { + // two fields from diff paths with removed content not matching + // => error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: nil, + err: true, + }, + { + // two fields from the same path => relavent fieldInfo + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Name: "a", + Type: "numeric", + Index: true, + Store: true, + }, + { + Name: "b", + Type: "numeric", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Name: "a", + Type: "numeric", + Index: false, + Store: true, + }, + { + Name: "b", + Type: "numeric", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + "b": { + Store: true, + }, + }, + err: false, + }, + { + // one store, one index, one dynamic and one all removed in type and default + // => relavent fieldInfo without error + original: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Store: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map3": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + DocValues: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: true, + Store: true, + DocValues: true, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + updated: &mapping.IndexMappingImpl{ + TypeMapping: map[string]*mapping.DocumentMapping{ + "map1": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Index: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map2": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + Store: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + "map3": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "numeric", + DocValues: false, + }, + }, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + }, + DefaultMapping: &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "", + DefaultSynonymSource: "", + }, + IndexDynamic: false, + StoreDynamic: false, + DocValuesDynamic: false, + CustomAnalysis: NewIndexMapping().CustomAnalysis, + }, + fieldInfo: map[string]*index.UpdateFieldInfo{ + "a": { + Index: true, + DocValues: true, + }, + "b": { + Store: true, + }, + "c": { + DocValues: true, + }, + "d": { + Deleted: true, + }, + }, + err: false, + }, + } + + for i, test := range tests { + info, err := DeletedFields(test.original, test.updated) + + if err == nil && test.err || err != nil && !test.err { + t.Errorf("Unexpected error value for test %d, expecting %t, got %v\n", i, test.err, err) + } + if info == nil && test.fieldInfo != nil || info != nil && test.fieldInfo == nil || !reflect.DeepEqual(info, test.fieldInfo) { + t.Errorf("Unexpected default info value for test %d, expecting %+v, got %+v, err %v", i, test.fieldInfo, info, err) + } + } +} + +func TestIndexUpdateText(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + doc1 := map[string]interface{}{"a": "xyz", "b": "abc", "c": "def", "d": "ghi"} + doc2 := map[string]interface{}{"a": "uvw", "b": "rst", "c": "klm", "d": "pqr"} + doc3 := map[string]interface{}{"a": "xyz", "b": "def", "c": "abc", "d": "mno"} + batch := index.NewBatch() + err = batch.Index("001", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("002", doc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc3) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: false, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + q1 := NewSearchRequest(NewQueryStringQuery("a:*")) + q1.Fields = append(q1.Fields, "a") + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 3 { + t.Errorf("Expected 3 hits, got %d\n", len(res1.Hits)) + } + if len(res1.Hits[0].Fields) != 1 { + t.Errorf("Expected 1 field, got %d\n", len(res1.Hits[0].Fields)) + } + q2 := NewSearchRequest(NewQueryStringQuery("b:*")) + q2.Fields = append(q2.Fields, "b") + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + q3 := NewSearchRequest(NewQueryStringQuery("c:*")) + q3.Fields = append(q3.Fields, "c") + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 3 { + t.Errorf("Expected 3 hits, got %d\n", len(res3.Hits)) + } + if len(res3.Hits[0].Fields) != 0 { + t.Errorf("Expected 0 fields, got %d\n", len(res3.Hits[0].Fields)) + } + q4 := NewSearchRequest(NewQueryStringQuery("d:*")) + q4.Fields = append(q4.Fields, "d") + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res4.Hits)) + } +} + +func TestIndexUpdateSynonym(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + synonymCollection := "collection1" + synonymSourceName := "english" + analyzer := en.AnalyzerName + synonymSourceConfig := map[string]interface{}{ + "collection": synonymCollection, + "analyzer": analyzer, + } + + a := mapping.NewTextFieldMapping() + a.Analyzer = analyzer + a.SynonymSource = synonymSourceName + a.IncludeInAll = false + + b := mapping.NewTextFieldMapping() + b.Analyzer = analyzer + b.SynonymSource = synonymSourceName + b.IncludeInAll = false + + c := mapping.NewTextFieldMapping() + c.Analyzer = analyzer + c.SynonymSource = synonymSourceName + c.IncludeInAll = false + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("a", a) + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("b", b) + indexMappingBefore.DefaultMapping.AddFieldMappingsAt("c", c) + err := indexMappingBefore.AddSynonymSource(synonymSourceName, synonymSourceConfig) + if err != nil { + t.Fatal(err) + } + + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + + doc1 := map[string]interface{}{ + "a": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + "b": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + "c": `The hardworking employee consistently strives to exceed expectations. + His industrious nature makes him a valuable asset to any team. + His conscientious attention to detail ensures that projects are completed efficiently and accurately. + He remains persistent even in the face of challenges.`, + } + doc2 := map[string]interface{}{ + "a": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + "b": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + "c": `The tranquil surroundings of the retreat provide a perfect escape from the hustle and bustle of city life. + Guests enjoy the peaceful atmosphere, which is perfect for relaxation and rejuvenation. + The calm environment offers the ideal place to meditate and connect with nature. + Even the most stressed individuals find themselves feeling relaxed and at ease.`, + } + synDoc1 := &SynonymDefinition{Synonyms: []string{"hardworking", "industrious", "conscientious", "persistent", "focused", "devoted"}} + synDoc2 := &SynonymDefinition{Synonyms: []string{"tranquil", "peaceful", "calm", "relaxed", "unruffled"}} + + batch := index.NewBatch() + err = batch.IndexSynonym("001", synonymCollection, synDoc1) + if err != nil { + t.Fatal(err) + } + err = batch.IndexSynonym("002", synonymCollection, synDoc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("004", doc2) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.DefaultMapping.AddFieldMappingsAt("a", a) + b.Index = false + indexMappingAfter.DefaultMapping.AddFieldMappingsAt("b", b) + err = indexMappingAfter.AddSynonymSource(synonymSourceName, synonymSourceConfig) + if err != nil { + t.Fatal(err) + } + + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + q1 := NewSearchRequest(NewQueryStringQuery("a:devoted")) + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 1 { + t.Errorf("Expected 1 hit, got %d\n", len(res1.Hits)) + } + + q2 := NewSearchRequest(NewQueryStringQuery("b:devoted")) + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + + q3 := NewSearchRequest(NewQueryStringQuery("c:unruffled")) + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res3.Hits)) + } +} + +func TestIndexUpdateMerge(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + + numDocsPerBatch := 1000 + numBatches := 10 + + var batch *Batch + doc := make(map[string]interface{}) + const letters = "abcdefghijklmnopqrstuvwxyz" + + randStr := func() string { + result := make([]byte, 3) + for i := 0; i < 3; i++ { + result[i] = letters[rand.Intn(len(letters))] + } + return string(result) + } + for i := 0; i < numBatches; i++ { + batch = index.NewBatch() + for j := 0; j < numDocsPerBatch; j++ { + doc["a"] = randStr() + doc["b"] = randStr() + doc["c"] = randStr() + doc["d"] = randStr() + err = batch.Index(fmt.Sprintf("%d", i*numDocsPerBatch+j), doc) + if err != nil { + t.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + } + + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: false, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + t.Fatal(err) + } + }() + + impl, ok := index.(*indexImpl) + if !ok { + t.Fatalf("Typecasting index to indexImpl failed") + } + sindex, ok := impl.i.(*scorch.Scorch) + if !ok { + t.Fatalf("Typecasting index to scorch index failed") + } + + err = sindex.ForceMerge(context.Background(), &mergeplan.SingleSegmentMergePlanOptions) + if err != nil { + t.Fatal(err) + } + + q1 := NewSearchRequest(NewQueryStringQuery("a:*")) + q1.Fields = append(q1.Fields, "a") + + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 10 { + t.Errorf("Expected 10 hits, got %d\n", len(res1.Hits)) + } + if len(res1.Hits[0].Fields) != 1 { + t.Errorf("Expected 1 field, got %d\n", len(res1.Hits[0].Fields)) + } + q2 := NewSearchRequest(NewQueryStringQuery("b:*")) + q2.Fields = append(q2.Fields, "b") + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + q3 := NewSearchRequest(NewQueryStringQuery("c:*")) + q3.Fields = append(q3.Fields, "c") + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 10 { + t.Errorf("Expected 10 hits, got %d\n", len(res3.Hits)) + } + if len(res3.Hits[0].Fields) != 0 { + t.Errorf("Expected 0 fields, got %d\n", len(res3.Hits[0].Fields)) + } + q4 := NewSearchRequest(NewQueryStringQuery("d:*")) + q4.Fields = append(q4.Fields, "d") + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Errorf("Expected 0 hits, got %d\n", len(res4.Hits)) + } +} + +func BenchmarkIndexUpdateText(b *testing.B) { + + tmpIndexPath := createTmpIndexPath(b) + defer cleanupTmpIndexPath(b, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: true, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + b.Fatal(err) + } + + numDocsPerBatch := 1000 + numBatches := 5 + + var batch *Batch + doc := make(map[string]interface{}) + const letters = "abcdefghijklmnopqrstuvwxyz" + + randStr := func() string { + result := make([]byte, 3) + for i := 0; i < 3; i++ { + result[i] = letters[rand.Intn(len(letters))] + } + return string(result) + } + for i := 0; i < numBatches; i++ { + batch = index.NewBatch() + for j := 0; j < numDocsPerBatch; j++ { + doc["a"] = randStr() + err = batch.Index(fmt.Sprintf("%d", i*numDocsPerBatch+j), doc) + if err != nil { + b.Fatal(err) + } + } + err = index.Batch(batch) + if err != nil { + b.Fatal(err) + } + } + + err = index.Close() + if err != nil { + b.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "text", + Index: true, + Store: false, + }, + }, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + }, + }, + Fields: []*mapping.FieldMapping{}, + DefaultAnalyzer: "standard", + DefaultSynonymSource: "", + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + b.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + b.Fatal(err) + } + defer func() { + err := index.Close() + if err != nil { + b.Fatal(err) + } + }() + + b.ResetTimer() + + for i := 0; i < b.N; i++ { + q := NewQueryStringQuery("a:*") + req := NewSearchRequest(q) + if _, err = index.Search(req); err != nil { + b.Fatal(err) + } + } +} diff --git a/search_knn_test.go b/search_knn_test.go index a2d207bfc..4dbe25744 100644 --- a/search_knn_test.go +++ b/search_knn_test.go @@ -1701,3 +1701,211 @@ func TestNumVecsStat(t *testing.T) { } } } + +func TestIndexUpdateVector(t *testing.T) { + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + + indexMappingBefore := mapping.NewIndexMapping() + indexMappingBefore.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingBefore.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "b": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + }, + Fields: []*mapping.FieldMapping{}, + } + indexMappingBefore.IndexDynamic = false + indexMappingBefore.StoreDynamic = false + indexMappingBefore.DocValuesDynamic = false + + index, err := New(tmpIndexPath, indexMappingBefore) + if err != nil { + t.Fatal(err) + } + doc1 := map[string]interface{}{"a": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, "b": []float32{0.32894259691238403, 0.6973215341567993, 0.6835201978683472, 0.38296082615852356}, "c": "L5MOPw7NID5SQMU9pHUoPw==", "d": "L5MOPw7NID5SQMU9pHUoPw=="} + doc2 := map[string]interface{}{"a": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, "b": []float32{0.0018692062003538013, 0.41076546907424927, 0.5675257444381714, 0.45832985639572144}, "c": "czloP94ZCD71ldY+GbAOPw==", "d": "czloP94ZCD71ldY+GbAOPw=="} + doc3 := map[string]interface{}{"a": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, "b": []float32{0.7853356599807739, 0.6904757618904114, 0.5643226504325867, 0.682637631893158}, "c": "Chh6P2lOqT47mjg/0odlPg==", "d": "Chh6P2lOqT47mjg/0odlPg=="} + batch := index.NewBatch() + err = batch.Index("001", doc1) + if err != nil { + t.Fatal(err) + } + err = batch.Index("002", doc2) + if err != nil { + t.Fatal(err) + } + err = batch.Index("003", doc3) + if err != nil { + t.Fatal(err) + } + err = index.Batch(batch) + if err != nil { + t.Fatal(err) + } + err = index.Close() + if err != nil { + t.Fatal(err) + } + + indexMappingAfter := mapping.NewIndexMapping() + indexMappingAfter.TypeMapping = map[string]*mapping.DocumentMapping{} + indexMappingAfter.DefaultMapping = &mapping.DocumentMapping{ + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{ + "a": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "c": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: true, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + "d": { + Enabled: true, + Dynamic: false, + Properties: map[string]*mapping.DocumentMapping{}, + Fields: []*mapping.FieldMapping{ + { + Type: "vector_base64", + Index: false, + Dims: 4, + Similarity: "l2_norm", + VectorIndexOptimizedFor: "latency", + }, + }, + }, + }, + Fields: []*mapping.FieldMapping{}, + } + indexMappingAfter.IndexDynamic = false + indexMappingAfter.StoreDynamic = false + indexMappingAfter.DocValuesDynamic = false + + mappingString, err := json.Marshal(indexMappingAfter) + if err != nil { + t.Fatal(err) + } + config := map[string]interface{}{ + "updated_mapping": string(mappingString), + } + + index, err = OpenUsing(tmpIndexPath, config) + if err != nil { + t.Fatal(err) + } + + q1 := NewSearchRequest(NewMatchNoneQuery()) + q1.AddKNN("a", []float32{1, 2, 3, 4}, 3, 1.0) + res1, err := index.Search(q1) + if err != nil { + t.Fatal(err) + } + if len(res1.Hits) != 3 { + t.Fatalf("Expected 3 hits, got %d\n", len(res1.Hits)) + } + q2 := NewSearchRequest(NewMatchNoneQuery()) + q2.AddKNN("b", []float32{1, 2, 3, 4}, 3, 1.0) + res2, err := index.Search(q2) + if err != nil { + t.Fatal(err) + } + if len(res2.Hits) != 0 { + t.Fatalf("Expected 0 hits, got %d\n", len(res2.Hits)) + } + q3 := NewSearchRequest(NewMatchNoneQuery()) + q3.AddKNN("c", []float32{1, 2, 3, 4}, 3, 1.0) + res3, err := index.Search(q3) + if err != nil { + t.Fatal(err) + } + if len(res3.Hits) != 3 { + t.Fatalf("Expected 3 hits, got %d\n", len(res3.Hits)) + } + q4 := NewSearchRequest(NewMatchNoneQuery()) + q4.AddKNN("d", []float32{1, 2, 3, 4}, 3, 1.0) + res4, err := index.Search(q4) + if err != nil { + t.Fatal(err) + } + if len(res4.Hits) != 0 { + t.Fatalf("Expected 0 hits, got %d\n", len(res4.Hits)) + } +} diff --git a/util/keys.go b/util/keys.go new file mode 100644 index 000000000..b71a7f48b --- /dev/null +++ b/util/keys.go @@ -0,0 +1,32 @@ +// Copyright (c) 2025 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +var ( + // Bolt keys + BoltSnapshotsBucket = []byte{'s'} + BoltPathKey = []byte{'p'} + BoltDeletedKey = []byte{'d'} + BoltInternalKey = []byte{'i'} + BoltMetaDataKey = []byte{'m'} + BoltMetaDataSegmentTypeKey = []byte("type") + BoltMetaDataSegmentVersionKey = []byte("version") + BoltMetaDataTimeStamp = []byte("timeStamp") + BoltStatsKey = []byte("stats") + BoltUpdatedFieldsKey = []byte("fields") + TotBytesWrittenKey = []byte("TotBytesWritten") + + MappingInternalKey = []byte("_mapping") +)