feat: Add Configurable Dynamic Indexing for Atlas Search (#3867)

* Add type_sets field to search_index resource

* Add type_sets and mappings_dynamic_config fields to singular and plural data source

* Refactor function into accepting multiple fields

* Add cannonical function for JSON

* Ensure mappings_dynamic_config is not set when mappings_dynamic is being used

* Refactor acceptance tests for less verbosity

* Add changelog entry

* Add documentation

* Extract hashtype implementation for better readability

* Address PR comments - documentation

* Add acceptance test step for omitted type_set

* Rephrase documentation for mappings_dynamic_config

* Address docs PR comment suggestions

* Update changelog sentences to be in 3rd person

* Refactor read operation for resource and single data source on mappings field

* Refactor canonical JSON function for simplicity purposes

* Remove unnecessary checks for resource read on mappings_dynamic and mappings_dynamic_config

Author: marcabreracast

.changelog/3867.txt

@@ -0,0 +1,15 @@
+```release-note:enhancement
+resource/mongodbatlas_search_index: Adds `type_sets` field to support Atlas Search configurable dynamic type sets
+```
+
+```release-note:enhancement
+resource/mongodbatlas_search_index: Adds `mappings_dynamic_config` (mutually exclusive with `mappings_dynamic`) to support object form of `mappings.dynamic`
+```
+
+```release-note:enhancement
+data-source/mongodbatlas_search_index: Exposes `mappings_dynamic_config` and `type_sets`
+```
+
+```release-note:enhancement
+data-source/mongodbatlas_search_indexes: Exposes `mappings_dynamic_config` and `type_sets` per result
+```

docs/data-sources/search_index.md

@@ -35,12 +35,16 @@ data "mongodbatlas_search_index" "test" {
 * `collection_name` - Name of the collection the index is on.
 * `database` - Name of the database the collection is in.
 * `mappings_dynamic` - Flag indicating whether the index uses dynamic or static mappings.
+* `mappings_dynamic_config` - JSON object for `mappings.dynamic` when Atlas returns an object (Please see the documentation for [dynamic and static mappings](https://www.mongodb.com/docs/atlas/atlas-search/index-definitions/#field-mapping-examples)). Mutually exclusive with `mappings_dynamic`.
 * `mappings_fields` - Object containing one or more field specifications.
 * `search_analyzer` - [Analyzer](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/#std-label-analyzers-ref) to use when searching the index.
 * `synonyms` - 	Synonyms mapping definition to use in this index.
 * `synonyms.#.name` - Name of the [synonym mapping definition](https://docs.atlas.mongodb.com/reference/atlas-search/synonyms/#std-label-synonyms-ref).
 * `synonyms.#.source_collection` - Name of the source MongoDB collection for the synonyms.
 * `synonyms.#.analyzer` - Name of the [analyzer](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/#std-label-analyzers-ref) to use with this synonym mapping. 
 * `stored_source` - String that can be "true" (store all fields), "false" (default, don't store any field), or a JSON string that contains the list of fields to store (include) or not store (exclude) on Atlas Search. To learn more, see [Stored Source Fields](https://www.mongodb.com/docs/atlas/atlas-search/stored-source-definition/).
+* `type_sets` - Set of type set definitions (when present). Each item includes:
+  * `name` - Type set name.
+  * `types` - JSON array string describing the types for the set.
 
 For more information see: [MongoDB Atlas API Reference.](https://docs.atlas.mongodb.com/atlas-search/) - [and MongoDB Atlas API - Search](https://docs.atlas.mongodb.com/reference/api/atlas-search/) Documentation for more information.

docs/data-sources/search_indexes.md

@@ -40,13 +40,17 @@ data "mongodbatlas_search_indexes" "test" {
 * `analyzers` - [Custom analyzers](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/custom/#std-label-custom-analyzers) to use in this index (this is an array of objects).
 * `collection_name` - (Required) Name of the collection the index is on.
 * `database` - (Required) Name of the database the collection is in.
-* `mappings_dynamic` - Flag indicating whether the index uses dynamic or static mappings.
+* `mappings_dynamic` - Flag indicating whether the index uses dynamic or static mappings. Mutually exclusive with `mappings_dynamic_config`.
+* `mappings_dynamic_config` - JSON object for `mappings.dynamic` when Atlas returns an object (Please see the documentation for [dynamic and static mappings](https://www.mongodb.com/docs/atlas/atlas-search/index-definitions/#field-mapping-examples)). Mutually exclusive with `mappings_dynamic`.
 * `mappings_fields` - Object containing one or more field specifications.
 * `search_analyzer` - [Analyzer](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/#std-label-analyzers-ref) to use when searching the index.
 * `synonyms` - 	Synonyms mapping definition to use in this index.
 * `synonyms.#.name` - Name of the [synonym mapping definition](https://docs.atlas.mongodb.com/reference/atlas-search/synonyms/#std-label-synonyms-ref).
 * `synonyms.#.source_collection` - Name of the source MongoDB collection for the synonyms.
 * `synonyms.#.analyzer` - Name of the [analyzer](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/#std-label-analyzers-ref) to use with this synonym mapping.
 * `stored_source` - String that can be "true" (store all fields), "false" (default, don't store any field), or a JSON string that contains the list of fields to store (include) or not store (exclude) on Atlas Search. To learn more, see [Stored Source Fields](https://www.mongodb.com/docs/atlas/atlas-search/stored-source-definition/).
+* `type_sets` - Set of type set definitions (when present). Each item includes:
+  * `name` - Type set name.
+  * `types` - JSON array string describing the types for the set.
 
 For more information see: [MongoDB Atlas API Reference.](https://docs.atlas.mongodb.com/atlas-search/) - [and MongoDB Atlas API - Search](https://docs.atlas.mongodb.com/reference/api/atlas-search/) Documentation for more information.

docs/resources/search_index.md

@@ -113,6 +113,33 @@ EOF
 }
 ```
 
+### Configurable dynamic (typeSets + dynamic object)
+```terraform
+resource "mongodbatlas_search_index" "conf-dynamic" {
+  project_id      = "<PROJECT_ID>"
+  cluster_name    = "<CLUSTER_NAME>"
+  collection_name = "collection_test"
+  database        = "database_test"
+  name            = "conf-dynamic"
+  type            = "search"
+
+  # mappings.dynamic as an object referencing a type set
+  mappings_dynamic_config = <<-EOF
+  { "typeSet": "type_set_name" }
+  EOF
+
+  # Define the referenced type set
+  type_sets {
+    name  = "type_set_name"
+    types = <<-EOF
+    [
+      { "type": "string" }
+    ]
+    EOF
+  }
+}
+```
+
 ## Argument Reference
 
 * `type` - (Optional) Type of index: `search` or `vectorSearch`. Default type is `search`.
@@ -152,7 +179,7 @@ EOF
 
 * `database` - (Required) Name of the database the collection is in.
 
-* `mappings_dynamic` - Indicates whether the search index uses dynamic or static mapping. For dynamic mapping, set the value to `true`. For static mapping, specify the fields to index using `mappings_fields`
+* `mappings_dynamic` - Indicates whether the search index uses dynamic or static mapping. For default dynamic mapping, set the value to `true`. For static mapping, specify the fields to index using `mappings_fields`. Mutually exclusive with `mappings_dynamic_config`.
 
 * `mappings_fields` - attribute is required in search indexes when `mappings_dynamic` is false. This field needs to be a JSON string in order to be decoded correctly.
   ```terraform
@@ -190,6 +217,12 @@ EOF
     EOF
   ```
 
+* `mappings_dynamic_config` - (Optional) JSON object for `mappings.dynamic` when using configurable dynamic. See the MongoDB documentation for further information on [Static and Dynamic Mapping](https://www.mongodb.com/docs/atlas/atlas-search/define-field-mappings/#std-label-fts-field-mappings). Mutually exclusive with `mappings_dynamic`.
+
+* `type_sets` - (Optional) One or more blocks defining configurable dynamic type sets. Atlas only persists/returns `typeSets` when `mappings.dynamic` is an object referencing a `typeSet` name.
+  * `name` - (Required) Name of the type set.
+  * `types` - (Optional) JSON array describing the types.
+
 * `search_analyzer` - [Analyzer](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/#std-label-analyzers-ref) to use when searching the index. Defaults to [lucene.standard](https://docs.atlas.mongodb.com/reference/atlas-search/analyzers/standard/#std-label-ref-standard-analyzer)
 * `synonyms` - Synonyms mapping definition to use in this index.
 

internal/service/searchindex/data_source_search_index.go

@@ -58,6 +58,10 @@ func returnSearchIndexDSSchema() map[string]*schema.Schema {
 			Type:     schema.TypeBool,
 			Computed: true,
 		},
+		"mappings_dynamic_config": {
+			Type:     schema.TypeString,
+			Computed: true,
+		},
 		"mappings_fields": {
 			Type:     schema.TypeString,
 			Computed: true,
@@ -98,6 +102,22 @@ func returnSearchIndexDSSchema() map[string]*schema.Schema {
 			Type:     schema.TypeString,
 			Computed: true,
 		},
+		"type_sets": {
+			Type:     schema.TypeSet,
+			Computed: true,
+			Elem: &schema.Resource{
+				Schema: map[string]*schema.Schema{
+					"name": {
+						Type:     schema.TypeString,
+						Computed: true,
+					},
+					"types": {
+						Type:     schema.TypeString,
+						Computed: true,
+					},
+				},
+			},
+		},
 	}
 }
 
@@ -160,18 +180,8 @@ func dataSourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resour
 	}
 
 	if searchIndex.LatestDefinition.Mappings != nil {
-		if err := d.Set("mappings_dynamic", searchIndex.LatestDefinition.Mappings.Dynamic); err != nil {
-			return diag.Errorf("error setting `mappings_dynamic` for search index (%s): %s", d.Id(), err)
-		}
-
-		if fields := searchIndex.LatestDefinition.Mappings.Fields; fields != nil && conversion.HasElementsSliceOrMap(*fields) {
-			searchIndexMappingFields, err := marshalSearchIndex(*fields)
-			if err != nil {
-				return diag.FromErr(err)
-			}
-			if err := d.Set("mappings_fields", searchIndexMappingFields); err != nil {
-				return diag.Errorf("error setting `mappings_fields` for for search index (%s): %s", d.Id(), err)
-			}
+		if diags := setMappingsAttributesFromDefinition(d, searchIndex.LatestDefinition.Mappings); diags != nil {
+			return diags
 		}
 	}
 
@@ -186,6 +196,24 @@ func dataSourceMongoDBAtlasSearchIndexRead(ctx context.Context, d *schema.Resour
 		}
 	}
 
+	if typeSets := searchIndex.LatestDefinition.GetTypeSets(); len(typeSets) > 0 {
+		var flattened []map[string]any
+		for _, t := range typeSets {
+			entry := map[string]any{"name": t.Name}
+			if types := t.GetTypes(); len(types) > 0 {
+				j, err := marshalSearchIndex(types)
+				if err != nil {
+					return diag.FromErr(err)
+				}
+				entry["types"] = j
+			}
+			flattened = append(flattened, entry)
+		}
+		if err := d.Set("type_sets", flattened); err != nil {
+			return diag.Errorf("error setting `type_sets` for search index (%s): %s", d.Id(), err)
+		}
+	}
+
 	storedSource := searchIndex.LatestDefinition.GetStoredSource()
 	strStoredSource, errStoredSource := MarshalStoredSource(storedSource)
 	if errStoredSource != nil {

internal/service/searchindex/data_source_search_indexes.go

@@ -105,7 +105,17 @@ func flattenSearchIndexes(searchIndexes []admin.SearchIndexResponse, projectID,
 		}
 
 		if searchIndexes[i].LatestDefinition.Mappings != nil {
-			searchIndexesMap[i]["mappings_dynamic"] = searchIndexes[i].LatestDefinition.Mappings.Dynamic
+			switch v := searchIndexes[i].LatestDefinition.Mappings.GetDynamic().(type) {
+			case bool:
+				searchIndexesMap[i]["mappings_dynamic"] = v
+			case map[string]any:
+				j, err := marshalSearchIndex(v)
+				if err != nil {
+					return nil, err
+				}
+				searchIndexesMap[i]["mappings_dynamic_config"] = j
+			default:
+			}
 
 			if conversion.HasElementsSliceOrMap(searchIndexes[i].LatestDefinition.Mappings.Fields) {
 				searchIndexMappingFields, err := marshalSearchIndex(searchIndexes[i].LatestDefinition.Mappings.Fields)
@@ -116,6 +126,22 @@ func flattenSearchIndexes(searchIndexes []admin.SearchIndexResponse, projectID,
 			}
 		}
 
+		if typeSets := searchIndexes[i].LatestDefinition.GetTypeSets(); len(typeSets) > 0 {
+			var flattened []map[string]any
+			for _, ts := range typeSets {
+				entry := map[string]any{"name": ts.Name}
+				if types := ts.GetTypes(); len(types) > 0 {
+					j, err := marshalSearchIndex(types)
+					if err != nil {
+						return nil, err
+					}
+					entry["types"] = j
+				}
+				flattened = append(flattened, entry)
+			}
+			searchIndexesMap[i]["type_sets"] = flattened
+		}
+
 		if analyzers := searchIndexes[i].LatestDefinition.GetAnalyzers(); len(analyzers) > 0 {
 			searchIndexAnalyzers, err := marshalSearchIndex(analyzers)
 			if err != nil {

internal/service/searchindex/model_search_index.go

@@ -60,16 +60,19 @@ func unmarshalSearchIndexMappingFields(str string) (map[string]any, diag.Diagnos
 	return fields, nil
 }
 
-func unmarshalSearchIndexFields(str string) ([]map[string]any, diag.Diagnostics) {
-	fields := []map[string]any{}
+func unmarshalJSONArrayForAttr(str, attr string) ([]map[string]any, diag.Diagnostics) {
+	arr := []map[string]any{}
 	if str == "" {
-		return fields, nil
+		return arr, nil
 	}
-	if err := json.Unmarshal([]byte(str), &fields); err != nil {
-		return nil, diag.Errorf("cannot unmarshal search index attribute `fields` because it has an incorrect format")
+	if err := json.Unmarshal([]byte(str), &arr); err != nil {
+		return nil, diag.Errorf("cannot unmarshal search index attribute `%s` because it has an incorrect format", attr)
 	}
+	return arr, nil
+}
 
-	return fields, nil
+func unmarshalSearchIndexFields(str string) ([]map[string]any, diag.Diagnostics) {
+	return unmarshalJSONArrayForAttr(str, "fields")
 }
 
 func UnmarshalSearchIndexAnalyzersFields(str string) ([]admin.AtlasSearchAnalyzer, diag.Diagnostics) {
@@ -85,6 +88,35 @@ func UnmarshalSearchIndexAnalyzersFields(str string) ([]admin.AtlasSearchAnalyze
 	return fields, nil
 }
 
+func expandSearchIndexTypeSets(d *schema.ResourceData) ([]admin.SearchTypeSets, diag.Diagnostics) {
+	var result []admin.SearchTypeSets
+
+	v, ok := d.GetOk("type_sets")
+	if !ok {
+		return result, nil
+	}
+
+	for _, raw := range v.(*schema.Set).List() {
+		item := raw.(map[string]any)
+
+		ts := admin.SearchTypeSets{
+			Name: item["name"].(string),
+		}
+
+		if s, ok := item["types"].(string); ok && s != "" {
+			arr, diags := unmarshalJSONArrayForAttr(s, "type_sets.types")
+			if diags != nil {
+				return nil, diags
+			}
+			ts.Types = conversion.ToAnySlicePointer(&arr)
+		}
+
+		result = append(result, ts)
+	}
+
+	return result, nil
+}
+
 func MarshalStoredSource(obj any) (string, error) {
 	if obj == nil {
 		return "", nil
@@ -127,3 +159,29 @@ func resourceSearchIndexRefreshFunc(ctx context.Context, clusterName, projectID,
 		return searchIndex, status, nil
 	}
 }
+
+func canonicalizeJSONString(s string) string {
+	if s == "" {
+		return ""
+	}
+	var v any
+	if err := json.Unmarshal([]byte(s), &v); err != nil {
+		return s
+	}
+	by, err := json.Marshal(v)
+	if err != nil {
+		return s
+	}
+	return string(by)
+}
+
+func hashTypeSetElement(v interface{}) int {
+	m := v.(map[string]interface{})
+	name := ""
+	if nv, ok := m["name"].(string); ok {
+		name = nv
+	}
+	typesStr, _ := m["types"].(string)
+	canon := canonicalizeJSONString(typesStr)
+	return schema.HashString(name + "|" + canon)
+}