fix docs; add include_defaults unit and yaml test

Author: markjhoy

docs/reference/elasticsearch/mapping-reference/sparse-vector.md

@@ -24,7 +24,11 @@ PUT my-index
 }
 ```
 
-{applies_to}`stack: preview 9.1`
+## Token pruning
+```{applies_to}
+stack: preview 9.1
+```
+
 With any new indices created, token pruning will be turned on by default with appropriate defaults. You can control this behaviour using the optional `index_options` parameters for the field:
 
 ```console

server/src/main/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapper.java

@@ -146,7 +146,11 @@ public SparseVectorFieldMapper build(MapperBuilderContext context) {
         }
 
         private boolean indexOptionsSerializerCheck(boolean includeDefaults, boolean isConfigured, IndexOptions value) {
-            return includeDefaults || (value == null || IndexOptions.isDefaultOptions(value, indexVersion)) == false;
+            return (
+                indexVersionSupportsDefaultPruningConfig(indexVersion) && (
+                    includeDefaults || (value == null || IndexOptions.isDefaultOptions(value, indexVersion)) == false
+                )
+            );
         }
     }
 

server/src/test/java/org/elasticsearch/index/mapper/vectors/SparseVectorFieldMapperTests.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.test.index.IndexVersionUtils;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParseException;
+import org.elasticsearch.xcontent.json.JsonXContent;
 import org.hamcrest.Matchers;
 import org.junit.AssumptionViolatedException;
 
@@ -80,6 +81,21 @@ protected void minimalMapping(XContentBuilder b) throws IOException {
         b.field("type", "sparse_vector");
     }
 
+    protected void minimalMappingWithExplicitDefaults(XContentBuilder b) throws IOException {
+        b.field("type", "sparse_vector");
+        b.startObject("index_options");
+        {
+            b.field("prune", true);
+            b.startObject("pruning_config");
+            {
+                b.field("tokens_freq_ratio_threshold", TokenPruningConfig.DEFAULT_TOKENS_FREQ_RATIO_THRESHOLD);
+                b.field("tokens_weight_threshold", TokenPruningConfig.DEFAULT_TOKENS_WEIGHT_THRESHOLD);
+            }
+            b.endObject();
+        }
+        b.endObject();
+    }
+
     protected void minimalMappingWithExplicitIndexOptions(XContentBuilder b) throws IOException {
         b.field("type", "sparse_vector");
         b.startObject("index_options");
@@ -195,6 +211,19 @@ public void testDefaults() throws Exception {
         assertTrue(freq1 < freq2);
     }
 
+    public void testDefaultsWithIncludeDefaults() throws Exception {
+        XContentBuilder orig = JsonXContent.contentBuilder().startObject();
+        createMapperService(fieldMapping(this::minimalMapping)).documentMapper().mapping().toXContent(orig, INCLUDE_DEFAULTS);
+        orig.endObject();
+
+        XContentBuilder withDefaults = JsonXContent.contentBuilder().startObject();
+        createMapperService(fieldMapping(this::minimalMappingWithExplicitDefaults)).documentMapper().mapping()
+            .toXContent(withDefaults, INCLUDE_DEFAULTS);
+        withDefaults.endObject();
+
+        assertEquals(Strings.toString(withDefaults), Strings.toString(orig));
+    }
+
     public void testMappingWithoutIndexOptionsUsesDefaults() throws Exception {
         DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping));
         assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString());

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/ml/sparse_vector_search.yml

@@ -595,13 +595,15 @@ teardown:
   - do:
       headers:
         Content-Type: application/json
-      indices.get_mapping:
+      indices.get_field_mapping:
         index: sparse_vector_pruning_test
+        fields: ml.tokens
+        include_defaults: true
 
   # the index_options with pruning defaults will be serialized here explicitly
-  - match: { sparse_vector_pruning_test.mappings.properties.ml.properties.tokens.index_options.prune: true }
-  - match: { sparse_vector_pruning_test.mappings.properties.ml.properties.tokens.index_options.pruning_config.tokens_freq_ratio_threshold: 5.0 }
-  - match: { sparse_vector_pruning_test.mappings.properties.ml.properties.tokens.index_options.pruning_config.tokens_weight_threshold: 0.4 }
+  - match: { sparse_vector_pruning_test.mappings.ml.tokens.mapping.tokens.index_options.prune: true }
+  - match: { sparse_vector_pruning_test.mappings.ml.tokens.mapping.tokens.index_options.pruning_config.tokens_freq_ratio_threshold: 5.0 }
+  - match: { sparse_vector_pruning_test.mappings.ml.tokens.mapping.tokens.index_options.pruning_config.tokens_weight_threshold: 0.4 }
 
 
 ---