GT-176 Adjust Inverted Index changes (#433)

Author: jwierzbo

collection_indexes.go

@@ -208,26 +208,60 @@ type EnsureZKDIndexOptions struct {
 type InvertedIndexOptions struct {
 	// Name optional user defined name used for hints in AQL queries
 	Name string `json:"name"`
-	// InBackground if true will not hold an exclusive collection lock for the entire index creation period (rocksdb only).
+	// InBackground This attribute can be set to true to create the index in the background,
+	// not write-locking the underlying collection for as long as if the index is built in the foreground.
+	// The default value is false.
 	InBackground   bool `json:"inBackground,omitempty"`
 	IsNewlyCreated bool `json:"isNewlyCreated,omitempty"`
 
+	// The number of threads to use for indexing the fields. Default: 2
 	Parallelism int `json:"parallelism,omitempty"`
-	// PrimarySort describes how individual fields are sorted
+	// PrimarySort You can define a primary sort order to enable an AQL optimization.
+	// If a query iterates over all documents of a collection, wants to sort them by attribute values, and the (left-most) fields to sort by,
+	// as well as their sorting direction, match with the primarySort definition, then the SORT operation is optimized away.
 	PrimarySort InvertedIndexPrimarySort `json:"primarySort,omitempty"`
-	// StoredValues these values specifies how the index should track values.
+	// StoredValues The optional storedValues attribute can contain an array of paths to additional attributes to store in the index.
+	// These additional attributes cannot be used for index lookups or for sorting, but they can be used for projections.
+	// This allows an index to fully cover more queries and avoid extra document lookups.
 	StoredValues []StoredValue `json:"storedValues,omitempty"`
-	// Analyzer to be used for indexing
-	Analyzer ArangoSearchAnalyzerType `json:"analyzer,omitempty"`
+	// Analyzer  The name of an Analyzer to use by default. This Analyzer is applied to the values of the indexed fields for which you don’t define Analyzers explicitly.
+	Analyzer string `json:"analyzer,omitempty"`
 	// Features list of analyzer features, default []
 	Features []ArangoSearchAnalyzerFeature `json:"features,omitempty"`
 	// IncludeAllFields If set to true, all fields of this element will be indexed. Defaults to false.
 	IncludeAllFields bool `json:"includeAllFields,omitempty"`
 	// TrackListPositions If set to true, values in a listed are treated as separate values. Defaults to false.
 	TrackListPositions bool `json:"trackListPositions,omitempty"`
+	// This option only applies if you use the inverted index in a search-alias Views.
+	// You can set the option to true to get the same behavior as with arangosearch Views regarding the indexing of array values as the default.
+	// If enabled, both, array and primitive values (strings, numbers, etc.) are accepted. Every element of an array is indexed according to the trackListPositions option.
+	// If set to false, it depends on the attribute path. If it explicitly expand an array ([*]), then the elements are indexed separately.
+	// Otherwise, the array is indexed as a whole, but only geopoint and aql Analyzers accept array inputs.
+	// You cannot use an array expansion if searchField is enabled.
+	SearchField bool `json:"searchField,omitempty"`
 	// Fields contains the properties for individual fields of the element.
 	// The key of the map are field names.
 	Fields []InvertedIndexField `json:"fields,omitempty"`
+	// ConsolidationIntervalMsec Wait at least this many milliseconds between applying ‘consolidationPolicy’ to consolidate View data store
+	// and possibly release space on the filesystem (default: 1000, to disable use: 0).
+	ConsolidationIntervalMsec *int64 `json:"consolidationIntervalMsec,omitempty"`
+	// CommitIntervalMsec Wait at least this many milliseconds between committing View data store changes and making
+	// documents visible to queries (default: 1000, to disable use: 0).
+	CommitIntervalMsec *int64 `json:"commitIntervalMsec,omitempty"`
+	// CleanupIntervalStep Wait at least this many commits between removing unused files in the ArangoSearch data directory
+	// (default: 2, to disable use: 0).
+	CleanupIntervalStep *int64 `json:"cleanupIntervalStep,omitempty"`
+	// ConsolidationPolicy The consolidation policy to apply for selecting which segments should be merged (default: {}).
+	ConsolidationPolicy *ArangoSearchConsolidationPolicy `json:"consolidationPolicy,omitempty"`
+	// WriteBufferIdle Maximum number of writers (segments) cached in the pool (default: 64, use 0 to disable)
+	WriteBufferIdle *int64 `json:"writebufferIdle,omitempty"`
+	// WriteBufferActive Maximum number of concurrent active writers (segments) that perform a transaction.
+	// Other writers (segments) wait till current active writers (segments) finish (default: 0, use 0 to disable)
+	WriteBufferActive *int64 `json:"writebufferActive,omitempty"`
+	// WriteBufferSizeMax Maximum memory byte size per writer (segment) before a writer (segment) flush is triggered.
+	// 0 value turns off this limit for any writer (buffer) and data will be flushed periodically based on the value defined for the flush thread (ArangoDB server startup option).
+	// 0 value should be used carefully due to high potential memory consumption (default: 33554432, use 0 to disable)
+	WriteBufferSizeMax *int64 `json:"writebufferSizeMax,omitempty"`
 }
 
 // InvertedIndexPrimarySort defines compression and list of fields to be sorted.
@@ -239,17 +273,30 @@ type InvertedIndexPrimarySort struct {
 
 // InvertedIndexField contains configuration for indexing of the field
 type InvertedIndexField struct {
-	// Name of the field
+	// Name An attribute path. The . character denotes sub-attributes.
 	Name string `json:"name"`
-	// Analyzer optional
-	Analyzer ArangoSearchAnalyzerType `json:"analyzer,omitempty"`
-	// IncludeAllFields If set to true, all fields of this element will be indexed. Defaults to false.
+	// Analyzer indicating the name of an analyzer instance
+	// Default: the value defined by the top-level analyzer option, or if not set, the default identity Analyzer.
+	Analyzer string `json:"analyzer,omitempty"`
+	// IncludeAllFields This option only applies if you use the inverted index in a search-alias Views.
+	// If set to true, then all sub-attributes of this field are indexed, excluding any sub-attributes that are configured separately by other elements in the fields array (and their sub-attributes). The analyzer and features properties apply to the sub-attributes.
+	// If set to false, then sub-attributes are ignored. The default value is defined by the top-level includeAllFields option, or false if not set.
 	IncludeAllFields bool `json:"includeAllFields,omitempty"`
-	// TrackListPositions If set to true, values in a listed are treated as separate values. Defaults to false.
+	// SearchField This option only applies if you use the inverted index in a search-alias Views.
+	// You can set the option to true to get the same behavior as with arangosearch Views regarding the indexing of array values for this field. If enabled, both, array and primitive values (strings, numbers, etc.) are accepted. Every element of an array is indexed according to the trackListPositions option.
+	// If set to false, it depends on the attribute path. If it explicitly expand an array ([*]), then the elements are indexed separately. Otherwise, the array is indexed as a whole, but only geopoint and aql Analyzers accept array inputs. You cannot use an array expansion if searchField is enabled.
+	// Default: the value defined by the top-level searchField option, or false if not set.
+	SearchField bool `json:"searchField,omitempty"`
+	// TrackListPositions This option only applies if you use the inverted index in a search-alias Views.
+	// If set to true, then track the value position in arrays for array values. For example, when querying a document like { attr: [ "valueX", "valueY", "valueZ" ] }, you need to specify the array element, e.g. doc.attr[1] == "valueY".
+	// If set to false, all values in an array are treated as equal alternatives. You don’t specify an array element in queries, e.g. doc.attr == "valueY", and all elements are searched for a match.
+	// Default: the value defined by the top-level trackListPositions option, or false if not set.
 	TrackListPositions bool `json:"trackListPositions,omitempty"`
-	// Features list of analyzer features, default [].
+	// A list of Analyzer features to use for this field. They define what features are enabled for the analyzer
 	Features []ArangoSearchAnalyzerFeature `json:"features,omitempty"`
-	// Nested
+	// Nested - Index the specified sub-objects that are stored in an array.
+	// Other than with the fields property, the values get indexed in a way that lets you query for co-occurring values.
+	// For example, you can search the sub-objects and all the conditions need to be met by a single sub-object instead of across all of them.
 	// Enterprise-only feature
 	Nested []InvertedIndexField `json:"nested,omitempty"`
 }

test/index_ensure_test.go

@@ -528,8 +528,6 @@ func TestEnsureInvertedIndex(t *testing.T) {
 					},
 					Compression: driver.PrimarySortCompressionLz4,
 				},
-				Features:     []driver.ArangoSearchAnalyzerFeature{},
-				StoredValues: []driver.StoredValue{},
 				Fields: []driver.InvertedIndexField{
 					{Name: "field1", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeatureFrequency}, Nested: nil},
 					{Name: "field2", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeaturePosition}, TrackListPositions: false, Nested: nil},
@@ -547,8 +545,6 @@ func TestEnsureInvertedIndex(t *testing.T) {
 					},
 					Compression: driver.PrimarySortCompressionLz4,
 				},
-				Features:     []driver.ArangoSearchAnalyzerFeature{},
-				StoredValues: []driver.StoredValue{},
 				Fields: []driver.InvertedIndexField{
 					{Name: "field1", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeatureFrequency}, Nested: nil},
 					{Name: "field2", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeaturePosition}, TrackListPositions: false,
@@ -584,12 +580,13 @@ func TestEnsureInvertedIndex(t *testing.T) {
 			require.True(t, created)
 
 			tc.Options.IsNewlyCreated = true
-			tc.Options.Analyzer = driver.ArangoSearchAnalyzerTypeIdentity // default value for analyzer
+			tc.Options.Analyzer = string(driver.ArangoSearchAnalyzerTypeIdentity) // default value for analyzer
 
 			requireIdxEquality := func(invertedIdx driver.Index) {
 				require.Equal(t, driver.InvertedIndex, idx.Type())
 				require.Equal(t, tc.Options.Name, idx.UserName())
-				require.Equal(t, tc.Options, idx.InvertedIndexOptions())
+				require.Equal(t, tc.Options.PrimarySort, idx.InvertedIndexOptions().PrimarySort)
+				require.Equal(t, tc.Options.Fields, idx.InvertedIndexOptions().Fields)
 			}
 			requireIdxEquality(idx)
 

view_arangosearch.go

@@ -319,7 +319,7 @@ type ArangoSearchViewProperties struct {
 	// CommitInterval ArangoSearch waits at least this many milliseconds between committing view data store changes and making documents visible to queries
 	CommitInterval *int64 `json:"commitIntervalMsec,omitempty"`
 
-	// WriteBufferIdel specifies the maximum number of writers (segments) cached in the pool.
+	// WriteBufferIdle specifies the maximum number of writers (segments) cached in the pool.
 	// 0 value turns off caching, default value is 64.
 	WriteBufferIdel *int64 `json:"writebufferIdle,omitempty"`