Introduce an ignore_above index-level setting

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/530_ignore_above_stored_source.yml

@@ -0,0 +1,214 @@
+---
+ignore_above mapping level setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+  - match: { hits.hits.0.fields.flattened: null }
+
+---
+ignore_above mapping level setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": ["foo bar", "the quick brown fox"], "flattened": { "value": ["the quick brown fox", "jumps over"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] }
+  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+  - match: { hits.hits.0.fields.flattened.0.value: "jumps over" }
+
+---
+ignore_above mapping overrides setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar baz foo bar baz", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0.fields.flattened.0.value: "the quick brown fox" }
+
+---
+ignore_above mapping overrides setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index: test
+        refresh: true
+        id: "1"
+        body: { "keyword": ["foo bar baz foo bar baz", "the quick brown fox jumps over"], "flattened": { "value": ["the quick brown fox", "jumps over the lazy dog"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over the lazy dog"] }
+  - match: { hits.hits.0.fields.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0.fields.flattened.0.value: ["the quick brown fox", "jumps over the lazy dog"] }
+
+---
+date ignore_above index level setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            properties:
+              keyword:
+                type: keyword
+              date:
+                type: date
+                format: "yyyy-MM-dd'T'HH:mm:ss"
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"], "date": ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - date
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+  - match: { hits.hits.0._source.date: ["2023-09-17T15:30:00", "2023-09-17T15:31:00"] }
+  - match: { hits.hits.0.fields.keyword: null }
+  - match: { hits.hits.0.fields.date: ["2023-09-17T15:30:00","2023-09-17T15:31:00"] }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml

@@ -0,0 +1,179 @@
+---
+ignore_above mapping level setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+
+---
+ignore_above mapping level setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+              flattened:
+                type: flattened
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": ["foo bar", "the quick brown fox"], "flattened": { "value": ["the quick brown fox", "jumps over"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: {}
+
+  - length: { hits.hits: 1 }
+  #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here)
+  #- match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] }
+  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar" }
+  - match: { hits.hits.0.fields.flattened.0.value: "jumps over" }
+
+---
+ignore_above mapping overrides setting:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index:  test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index:  test
+        refresh: true
+        id: "1"
+        body:   { "keyword": "foo bar baz foo bar baz", "flattened": { "value": "the quick brown fox" } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0._source.flattened.value: "the quick brown fox" }
+  - match: { hits.hits.0.fields.keyword.0: "foo bar baz foo bar baz" }
+  - match: { hits.hits.0.fields.flattened.0.value: "the quick brown fox" }
+
+---
+ignore_above mapping overrides setting on arrays:
+  - requires:
+      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
+      reason: introduce ignore_above index level setting
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                ignore_above: 10
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              keyword:
+                type: keyword
+                ignore_above: 100
+              flattened:
+                type: flattened
+                ignore_above: 100
+
+  - do:
+      index:
+        index: test
+        refresh: true
+        id: "1"
+        body: { "keyword": ["foo bar baz foo bar baz", "the quick brown fox jumps over"], "flattened": { "value": ["the quick brown fox", "jumps over the lazy dog"] } }
+
+  - do:
+      search:
+        body:
+          fields:
+            - keyword
+            - flattened
+          query:
+            match_all: { }
+
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._source.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0._source.flattened.value: ["jumps over the lazy dog", "the quick brown fox"] }
+  - match: { hits.hits.0.fields.keyword: ["foo bar baz foo bar baz", "the quick brown fox jumps over"] }
+  - match: { hits.hits.0.fields.flattened.0.value: ["jumps over the lazy dog", "the quick brown fox"] }