Introduce an ignore_above index-level setting

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesSourceReaderBenchmark.java

@@ -43,7 +43,6 @@
 import org.elasticsearch.compute.operator.topn.TopNOperator;
 import org.elasticsearch.core.IOUtils;
 import org.elasticsearch.index.IndexSettings;
-import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
@@ -183,7 +182,7 @@ private static BlockLoader blockLoader(String name) {
                 Lucene.KEYWORD_ANALYZER,
                 Lucene.KEYWORD_ANALYZER,
                 Lucene.KEYWORD_ANALYZER,
-                new KeywordFieldMapper.Builder(name, IndexVersion.current()).docValues(ft.docValuesType() != DocValuesType.NONE),
+                KeywordFieldMapper.buildForTest(name, ft.docValuesType() != DocValuesType.NONE),
                 syntheticSource
             ).blockLoader(new MappedFieldType.BlockLoaderContext() {
                 @Override

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml

@@ -0,0 +1,214 @@
+---
+ignore_above mapping level setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+  - match: { hits.hits.0.fields.flattened: null }
+
+---
+ignore_above mapping level setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": ["foo bar", "the quick brown fox"], "flattened": { "value": ["the quick brown fox", "jumps over"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] }
+  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+  - match: { hits.hits.0.fields.flattened.0.value: "jumps over" }
+
+---
+ignore_above mapping overrides setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar baz foo bar baz", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0.fields.flattened.0.value: "the quick brown fox" }
+
+---
+ignore_above mapping overrides setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index: test
+        refresh: true
+        id: "1"
+        body: { "keyword": ["foo bar baz foo bar baz", "the quick brown fox jumps over"], "flattened": { "value": ["the quick brown fox", "jumps over the lazy dog"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over the lazy dog"] }
+  - match: { hits.hits.0.fields.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0.fields.flattened.0.value: ["the quick brown fox", "jumps over the lazy dog"] }
+
+---
+date ignore_above index level setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              date:
+                type: date
+                format: "yyyy-MM-dd'T'HH:mm:ss"
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"], "date": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - date
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+  - match: { hits.hits.0._source.date: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+  - match: { hits.hits.0.fields.keyword: null }
+  - match: { hits.hits.0.fields.date: ["2023-09-17T15:30:00","2023-09-17T15:31:00"] }