feat: Adds type and fields attributes in resource and data sources for search_index (#1605) INTMDB-1260

* add type field

* add fields field

* log when using existing cluster

* remove skip

* change random prefixes

* fix tests

* const collectionName

* const searchAnalyzer

* const resourceName and datasourceName

* reduce cluster name size

* don't check dangling indices for existing clusters as it takes some time for them to be deleted

* failing test for vector search index

* create vector index in test

* change data source

* continue if it can't unmarshal

* add fields to search-indexes data source

* initial doc

* small fix in doc

* fix example

* Update search_index.html.markdown

formatting

* Update search_index.html.markdown

formatting

* adapt doc and example for Fields

* detect format errors in fields and mappings_fields

* detect format errors in analyzers

* test stringified JSON attributes for fields

* generic json diff check

* test with explicit search type

---------

Co-authored-by: Zuhair Ahmed <zuhair.ahmed@mongodb.com>

Author: lantoli

mongodbatlas/data_source_mongodbatlas_search_index.go

@@ -87,6 +87,15 @@ func returnSearchIndexDSSchema() map[string]*schema.Schema {
 			Optional: true,
 			Computed: true,
 		},
+		"type": {
+			Type:     schema.TypeString,
+			Optional: true,
+		},
+		"fields": {
+			Type:             schema.TypeString,
+			Optional:         true,
+			DiffSuppressFunc: validateSearchIndexMappingDiff,
+		},
 	}
 }
 
@@ -105,6 +114,10 @@ func dataSourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resour
 		return diag.Errorf("error getting search index information: %s", err)
 	}
 
+	if err := d.Set("type", searchIndex.Type); err != nil {
+		return diag.Errorf("error setting `type` for search index (%s): %s", d.Id(), err)
+	}
+
 	if err := d.Set("index_id", indexID); err != nil {
 		return diag.Errorf("error setting `index_id` for search index (%s): %s", d.Id(), err)
 	}
@@ -140,21 +153,34 @@ func dataSourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resour
 		return diag.Errorf("error setting `searchAnalyzer` for search index (%s): %s", d.Id(), err)
 	}
 
-	if err := d.Set("mappings_dynamic", searchIndex.Mappings.Dynamic); err != nil {
-		return diag.Errorf("error setting `mappings_dynamic` for search index (%s): %s", d.Id(), err)
-	}
-
 	if err := d.Set("synonyms", flattenSearchIndexSynonyms(searchIndex.Synonyms)); err != nil {
 		return diag.Errorf("error setting `synonyms` for search index (%s): %s", d.Id(), err)
 	}
 
-	if len(searchIndex.Mappings.Fields) > 0 {
-		searchIndexMappingFields, err := marshalSearchIndex(searchIndex.Mappings.Fields)
+	if searchIndex.Mappings != nil {
+		if err := d.Set("mappings_dynamic", searchIndex.Mappings.Dynamic); err != nil {
+			return diag.Errorf("error setting `mappings_dynamic` for search index (%s): %s", d.Id(), err)
+		}
+
+		if len(searchIndex.Mappings.Fields) > 0 {
+			searchIndexMappingFields, err := marshalSearchIndex(searchIndex.Mappings.Fields)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+			if err := d.Set("mappings_fields", searchIndexMappingFields); err != nil {
+				return diag.Errorf("error setting `mappings_fields` for for search index (%s): %s", d.Id(), err)
+			}
+		}
+	}
+
+	if len(searchIndex.Fields) > 0 {
+		fields, err := marshalSearchIndex(searchIndex.Fields)
 		if err != nil {
 			return diag.FromErr(err)
 		}
-		if err := d.Set("mappings_fields", searchIndexMappingFields); err != nil {
-			return diag.Errorf("error setting `mappings_fields` for for search index (%s): %s", d.Id(), err)
+
+		if err := d.Set("fields", fields); err != nil {
+			return diag.Errorf("error setting `fields` for for search index (%s): %s", d.Id(), err)
 		}
 	}
 

mongodbatlas/data_source_mongodbatlas_search_indexes.go

@@ -100,25 +100,29 @@ func flattenSearchIndexes(searchIndexes []admin.ClusterSearchIndex, projectID, c
 
 	for i := range searchIndexes {
 		searchIndexesMap[i] = map[string]any{
-			"project_id":       projectID,
-			"cluster_name":     clusterName,
-			"analyzer":         searchIndexes[i].Analyzer,
-			"collection_name":  searchIndexes[i].CollectionName,
-			"database":         searchIndexes[i].Database,
-			"index_id":         searchIndexes[i].IndexID,
-			"mappings_dynamic": searchIndexes[i].Mappings.Dynamic,
-			"name":             searchIndexes[i].Name,
-			"search_analyzer":  searchIndexes[i].SearchAnalyzer,
-			"status":           searchIndexes[i].Status,
-			"synonyms":         flattenSearchIndexSynonyms(searchIndexes[i].Synonyms),
+			"project_id":      projectID,
+			"cluster_name":    clusterName,
+			"analyzer":        searchIndexes[i].Analyzer,
+			"collection_name": searchIndexes[i].CollectionName,
+			"database":        searchIndexes[i].Database,
+			"index_id":        searchIndexes[i].IndexID,
+			"name":            searchIndexes[i].Name,
+			"search_analyzer": searchIndexes[i].SearchAnalyzer,
+			"status":          searchIndexes[i].Status,
+			"synonyms":        flattenSearchIndexSynonyms(searchIndexes[i].Synonyms),
+			"type":            searchIndexes[i].Type,
 		}
 
-		if len(searchIndexes[i].Mappings.Fields) > 0 {
-			searchIndexMappingFields, err := marshalSearchIndex(searchIndexes[i].Mappings.Fields)
-			if err != nil {
-				return nil, err
+		if searchIndexes[i].Mappings != nil {
+			searchIndexesMap[i]["mappings_dynamic"] = searchIndexes[i].Mappings.Dynamic
+
+			if len(searchIndexes[i].Mappings.Fields) > 0 {
+				searchIndexMappingFields, err := marshalSearchIndex(searchIndexes[i].Mappings.Fields)
+				if err != nil {
+					return nil, err
+				}
+				searchIndexesMap[i]["mappings_fields"] = searchIndexMappingFields
 			}
-			searchIndexesMap[i]["mappings_fields"] = searchIndexMappingFields
 		}
 
 		if len(searchIndexes[i].Analyzers) > 0 {
@@ -128,6 +132,14 @@ func flattenSearchIndexes(searchIndexes []admin.ClusterSearchIndex, projectID, c
 			}
 			searchIndexesMap[i]["analyzers"] = searchIndexAnalyzers
 		}
+
+		if len(searchIndexes[i].Fields) > 0 {
+			fields, err := marshalSearchIndex(searchIndexes[i].Fields)
+			if err != nil {
+				return nil, err
+			}
+			searchIndexesMap[i]["fields"] = fields
+		}
 	}
 
 	return searchIndexesMap, nil

mongodbatlas/resource_mongodbatlas_search_index.go

@@ -17,6 +17,10 @@ import (
 	"go.mongodb.org/atlas-sdk/v20231115001/admin"
 )
 
+const (
+	vectorSearch = "vectorSearch"
+)
+
 func resourceMongoDBAtlasSearchIndex() *schema.Resource {
 	return &schema.Resource{
 		CreateWithoutTimeout: resourceMongoDBAtlasSearchIndexCreate,
@@ -115,6 +119,15 @@ func returnSearchIndexSchema() map[string]*schema.Schema {
 			Type:     schema.TypeBool,
 			Optional: true,
 		},
+		"type": {
+			Type:     schema.TypeString,
+			Optional: true,
+		},
+		"fields": {
+			Type:             schema.TypeString,
+			Optional:         true,
+			DiffSuppressFunc: validateSearchIndexMappingDiff,
+		},
 	}
 }
 
@@ -181,12 +194,12 @@ func resourceMongoDBAtlasSearchIndexUpdate(ctx context.Context, d *schema.Resour
 		return diag.Errorf("error getting search index information: %s", err)
 	}
 
-	if d.HasChange("analyzer") {
-		searchIndex.Analyzer = stringPtr(d.Get("analyzer").(string))
+	if d.HasChange("type") {
+		searchIndex.Type = stringPtr(d.Get("type").(string))
 	}
 
-	if d.HasChange("analyzers") {
-		searchIndex.Analyzers = unmarshalSearchIndexAnalyzersFields(d.Get("analyzers").(string))
+	if d.HasChange("analyzer") {
+		searchIndex.Analyzer = stringPtr(d.Get("analyzer").(string))
 	}
 
 	if d.HasChange("collection_name") {
@@ -210,8 +223,28 @@ func resourceMongoDBAtlasSearchIndexUpdate(ctx context.Context, d *schema.Resour
 		searchIndex.Mappings.Dynamic = &dynamic
 	}
 
+	if d.HasChange("analyzers") {
+		analyzers, err := unmarshalSearchIndexAnalyzersFields(d.Get("analyzers").(string))
+		if err != nil {
+			return err
+		}
+		searchIndex.Analyzers = analyzers
+	}
+
 	if d.HasChange("mappings_fields") {
-		searchIndex.Mappings.Fields = unmarshalSearchIndexMappingFields(d.Get("mappings_fields").(string))
+		mappingsFields, err := unmarshalSearchIndexMappingFields(d.Get("mappings_fields").(string))
+		if err != nil {
+			return err
+		}
+		searchIndex.Mappings.Fields = mappingsFields
+	}
+
+	if d.HasChange("fields") {
+		fields, err := unmarshalSearchIndexFields(d.Get("fields").(string))
+		if err != nil {
+			return err
+		}
+		searchIndex.Fields = fields
 	}
 
 	if d.HasChange("synonyms") {
@@ -272,6 +305,10 @@ func resourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resource
 		return diag.Errorf("error setting `index_id` for search index (%s): %s", d.Id(), err)
 	}
 
+	if err := d.Set("type", searchIndex.Type); err != nil {
+		return diag.Errorf("error setting `type` for search index (%s): %s", d.Id(), err)
+	}
+
 	if err := d.Set("analyzer", searchIndex.Analyzer); err != nil {
 		return diag.Errorf("error setting `analyzer` for search index (%s): %s", d.Id(), err)
 	}
@@ -283,7 +320,7 @@ func resourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resource
 		}
 
 		if err := d.Set("analyzers", searchIndexMappingFields); err != nil {
-			return diag.Errorf("error setting `analyzer` for search index (%s): %s", d.Id(), err)
+			return diag.Errorf("error setting `analyzers` for search index (%s): %s", d.Id(), err)
 		}
 	}
 
@@ -303,22 +340,35 @@ func resourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resource
 		return diag.Errorf("error setting `searchAnalyzer` for search index (%s): %s", d.Id(), err)
 	}
 
-	if err := d.Set("mappings_dynamic", searchIndex.Mappings.Dynamic); err != nil {
-		return diag.Errorf("error setting `mappings_dynamic` for search index (%s): %s", d.Id(), err)
-	}
-
 	if err := d.Set("synonyms", flattenSearchIndexSynonyms(searchIndex.Synonyms)); err != nil {
 		return diag.Errorf("error setting `synonyms` for search index (%s): %s", d.Id(), err)
 	}
 
-	if len(searchIndex.Mappings.Fields) > 0 {
-		searchIndexMappingFields, err := marshalSearchIndex(searchIndex.Mappings.Fields)
+	if searchIndex.Mappings != nil {
+		if err := d.Set("mappings_dynamic", searchIndex.Mappings.Dynamic); err != nil {
+			return diag.Errorf("error setting `mappings_dynamic` for search index (%s): %s", d.Id(), err)
+		}
+
+		if len(searchIndex.Mappings.Fields) > 0 {
+			searchIndexMappingFields, err := marshalSearchIndex(searchIndex.Mappings.Fields)
+			if err != nil {
+				return diag.FromErr(err)
+			}
+
+			if err := d.Set("mappings_fields", searchIndexMappingFields); err != nil {
+				return diag.Errorf("error setting `mappings_fields` for for search index (%s): %s", d.Id(), err)
+			}
+		}
+	}
+
+	if len(searchIndex.Fields) > 0 {
+		fields, err := marshalSearchIndex(searchIndex.Fields)
 		if err != nil {
 			return diag.FromErr(err)
 		}
 
-		if err := d.Set("mappings_fields", searchIndexMappingFields); err != nil {
-			return diag.Errorf("error setting `mappings_fields` for for search index (%s): %s", d.Id(), err)
+		if err := d.Set("fields", fields); err != nil {
+			return diag.Errorf("error setting `fields` for for search index (%s): %s", d.Id(), err)
 		}
 	}
 
@@ -346,21 +396,42 @@ func resourceMongoDBAtlasSearchIndexCreate(ctx context.Context, d *schema.Resour
 	connV2 := meta.(*MongoDBClient).AtlasV2
 	projectID := d.Get("project_id").(string)
 	clusterName := d.Get("cluster_name").(string)
-	dynamic := d.Get("mappings_dynamic").(bool)
+	indexType := d.Get("type").(string)
+
 	searchIndexRequest := &admin.ClusterSearchIndex{
+		Type:           stringPtr(indexType),
 		Analyzer:       stringPtr(d.Get("analyzer").(string)),
-		Analyzers:      unmarshalSearchIndexAnalyzersFields(d.Get("analyzers").(string)),
 		CollectionName: d.Get("collection_name").(string),
 		Database:       d.Get("database").(string),
-		Mappings: &admin.ApiAtlasFTSMappings{
-			Dynamic: &dynamic,
-			Fields:  unmarshalSearchIndexMappingFields(d.Get("mappings_fields").(string)),
-		},
 		Name:           d.Get("name").(string),
 		SearchAnalyzer: stringPtr(d.Get("search_analyzer").(string)),
 		Status:         stringPtr(d.Get("status").(string)),
 		Synonyms:       expandSearchIndexSynonyms(d),
 	}
+
+	if indexType == vectorSearch {
+		fields, err := unmarshalSearchIndexFields(d.Get("fields").(string))
+		if err != nil {
+			return err
+		}
+		searchIndexRequest.Fields = fields
+	} else {
+		analyzers, err := unmarshalSearchIndexAnalyzersFields(d.Get("analyzers").(string))
+		if err != nil {
+			return err
+		}
+		searchIndexRequest.Analyzers = analyzers
+		mappingsFields, err := unmarshalSearchIndexMappingFields(d.Get("mappings_fields").(string))
+		if err != nil {
+			return err
+		}
+		dynamic := d.Get("mappings_dynamic").(bool)
+		searchIndexRequest.Mappings = &admin.ApiAtlasFTSMappings{
+			Dynamic: &dynamic,
+			Fields:  mappingsFields,
+		}
+	}
+
 	dbSearchIndexRes, _, err := connV2.AtlasSearchApi.CreateAtlasSearchIndex(ctx, projectID, clusterName, searchIndexRequest).Execute()
 	if err != nil {
 		return diag.Errorf("error creating index: %s", err)
@@ -467,28 +538,38 @@ func validateSearchAnalyzersDiff(k, old, newStr string, d *schema.ResourceData)
 	return true
 }
 
-func unmarshalSearchIndexMappingFields(mappingString string) map[string]any {
+func unmarshalSearchIndexMappingFields(mappingString string) (map[string]any, diag.Diagnostics) {
 	if mappingString == "" {
-		return nil
+		return nil, nil
 	}
 	var fields map[string]any
 	if err := json.Unmarshal([]byte(mappingString), &fields); err != nil {
-		log.Printf("[ERROR] cannot unmarshal search index mapping fields: %v", err)
-		return nil
+		return nil, diag.Errorf("cannot unmarshal search index attribute `mappings_fields` because it has an incorrect format")
 	}
-	return fields
+	return fields, nil
+}
+
+func unmarshalSearchIndexFields(fieldsStr string) ([]map[string]any, diag.Diagnostics) {
+	if fieldsStr == "" {
+		return nil, nil
+	}
+	var fields []map[string]any
+	if err := json.Unmarshal([]byte(fieldsStr), &fields); err != nil {
+		return nil, diag.Errorf("cannot unmarshal search index attribute `fields` because it has an incorrect format")
+	}
+
+	return fields, nil
 }
 
-func unmarshalSearchIndexAnalyzersFields(mappingString string) []admin.ApiAtlasFTSAnalyzers {
+func unmarshalSearchIndexAnalyzersFields(mappingString string) ([]admin.ApiAtlasFTSAnalyzers, diag.Diagnostics) {
 	if mappingString == "" {
-		return nil
+		return nil, nil
 	}
 	var fields []admin.ApiAtlasFTSAnalyzers
 	if err := json.Unmarshal([]byte(mappingString), &fields); err != nil {
-		log.Printf("[ERROR] cannot unmarshal search index mapping fields: %v", err)
-		return nil
+		return nil, diag.Errorf("cannot unmarshal search index attribute `analyzers` because it has an incorrect format")
 	}
-	return fields
+	return fields, nil
 }
 
 func resourceSearchIndexRefreshFunc(ctx context.Context, clusterName, projectID, indexID string, connV2 *admin.APIClient) retry.StateRefreshFunc {