Merge remote-tracking branch 'origin/main' into lucene_snapshot

Author: elasticsearchmachine

docs/changelog/112645.yaml

@@ -0,0 +1,6 @@
+pr: 112645
+summary: Add support for multi-value dimensions
+area: Mapping
+type: enhancement
+issues:
+ - 110387

docs/reference/mapping/types/keyword.asciidoc

@@ -163,7 +163,6 @@ index setting limits the number of dimensions in an index.
 Dimension fields have the following constraints:
 
 * The `doc_values` and `index` mapping parameters must be `true`.
-* Field values cannot be an <<array,array or multi-value>>.
 // end::dimension[]
 * Dimension values are used to identify a document’s time series. If dimension values are altered in any way during indexing, the document will be stored as belonging to different from intended time series. As a result there are additional constraints:
 ** The field cannot use a <<normalizer,`normalizer`>>.

docs/reference/release-notes/8.15.0.asciidoc

@@ -44,6 +44,14 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
+
 [[breaking-8.15.0]]
 [float]
 === Breaking changes

docs/reference/release-notes/8.15.1.asciidoc

@@ -24,13 +24,20 @@ To work around this issue, you have a number of options:
 <<esql-kibana-enable,disable ES|QL queries in {kib}>>
 ** Change the default data view in Discover to a smaller set of indices and/or one with fewer mapping conflicts.
 
-* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field 
+* Index Stats, Node Stats and Cluster Stats API can return a null pointer exception if an index contains a `dense_vector` field
 but there is an index segment that does not contain any documents with a dense vector field ({es-pull}112720[#112720]). Workarounds:
 ** If the affected index already contains documents with a dense vector field, force merge the index to a single segment.
 ** If the affected index does not already contain documents with a dense vector field, index a document with a dense vector field
 and then force merge to a single segment.
 ** If the affected index's `dense_vector` fields are unused, reindex without the `dense_vector` fields.
 
+* Synthetic source bug. Synthetic source may fail generating the _source at runtime, causing failures in get APIs or
+partial failures in the search APIs. The result is that for the affected documents the _source can't be retrieved.
+There is no workaround and the only option to is to upgrade to 8.15.2 when released.
++
+If you use synthetic source then you may be affected by this bug if the following is true:
+** If you have more fields then the `index.mapping.total_fields.limit` setting allows.
+** If you use dynamic mappings and the `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting is enabled.
 
 [[bug-8.15.1]]
 [float]

docs/reference/rest-api/security/get-service-accounts.asciidoc

@@ -250,7 +250,8 @@ GET /_security/service/elastic/fleet-server
             "monitor",
             "create_index",
             "auto_configure",
-            "maintenance"
+            "maintenance",
+            "view_index_metadata"
           ],
           "allow_restricted_indices": false
         },
@@ -265,7 +266,8 @@ GET /_security/service/elastic/fleet-server
             "monitor",
             "create_index",
             "auto_configure",
-            "maintenance"
+            "maintenance",
+            "view_index_metadata"
           ],
           "allow_restricted_indices": false
         }

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -1230,3 +1230,83 @@ non string dimension fields:
   - match: { .$idx0name.mappings.properties.attributes.properties.double.time_series_dimension: true }
   - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.type: 'ip' }
   - match: { .$idx0name.mappings.properties.attributes.properties.host\.ip.time_series_dimension: true }
+
+---
+multi value dimensions:
+  - requires:
+      cluster_features: ["routing.multi_value_routing_path"]
+      reason: support for multi-value dimensions
+
+  - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
+      indices.put_index_template:
+        name: my-dynamic-template
+        body:
+          index_patterns: [k9s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_shards: 1
+                mode: time_series
+                time_series:
+                  start_time: 2023-08-31T13:03:08.138Z
+
+            mappings:
+              properties:
+                attributes:
+                  type: passthrough
+                  dynamic: true
+                  time_series_dimension: true
+                  priority: 1
+              dynamic_templates:
+                - counter_metric:
+                    mapping:
+                      type: integer
+                      time_series_metric: counter
+
+  - do:
+      bulk:
+        index: k9s
+        refresh: true
+        body:
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "10", "attributes": { "dim1": ["a" , "b"], "dim2": [1, 2] } }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["b" , "a"], "dim2": [1, 2] } }'
+          - '{ "create": { "dynamic_templates": { "data": "counter_metric" } } }'
+          - '{ "@timestamp": "2023-09-01T13:03:08.138Z","data": "20", "attributes": { "dim1": ["c" , "b"], "dim2": [1, 2] } }'
+  - is_false: errors
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            tsids:
+              terms:
+                field: _tsid
+
+  - length: { aggregations.tsids.buckets: 3 } # only the order of the dim1 attribute is different, yet we expect to have two distinct time series
+
+  - do:
+      search:
+        index: k9s
+        body:
+          size: 0
+          aggs:
+            dims:
+              terms:
+                field: dim1
+                order:
+                  _key: asc
+
+  - length: { aggregations.dims.buckets: 3 }
+  - match: { aggregations.dims.buckets.0.key: a }
+  - match: { aggregations.dims.buckets.0.doc_count: 2 }
+  - match: { aggregations.dims.buckets.1.key: b }
+  - match: { aggregations.dims.buckets.1.doc_count: 3 }
+  - match: { aggregations.dims.buckets.2.key: c }
+  - match: { aggregations.dims.buckets.2.doc_count: 1 }

muted-tests.yml

@@ -330,6 +330,12 @@ tests:
 - class: org.elasticsearch.integration.KibanaUserRoleIntegTests
   method: testSearchAndMSearch
   issue: https://github.com/elastic/elasticsearch/issues/113345
+- class: org.elasticsearch.action.bulk.IncrementalBulkIT
+  method: testBulkLevelBulkFailureAfterFirstIncrementalRequest
+  issue: https://github.com/elastic/elasticsearch/issues/113365
+- class: org.elasticsearch.xpack.ml.integration.MlJobIT
+  method: testDeleteJob_TimingStatsDocumentIsDeleted
+  issue: https://github.com/elastic/elasticsearch/issues/113370
 
 # Examples:
 #

rest-api-spec/build.gradle

@@ -53,3 +53,7 @@ tasks.register('enforceYamlTestConvention').configure {
 tasks.named("precommit").configure {
   dependsOn 'enforceYamlTestConvention'
 }
+
+tasks.named("yamlRestCompatTestTransform").configure({ task ->
+  task.skipTest("tsdb/140_routing_path/multi-value routing path field", "Multi-value routing paths are allowed now. See #112645")
+})

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/tsdb/140_routing_path.yml

@@ -119,11 +119,11 @@ missing dimension on routing path field:
                 type: keyword
 
 ---
-multi-value routing path field:
+multi-value routing path field succeeds:
   - requires:
       test_runner_features: close_to
-      cluster_features: ["gte_v8.13.0"]
-      reason: _tsid hashing introduced in 8.13
+      cluster_features: ["routing.multi_value_routing_path"]
+      reason: support for multi-value dimensions
 
   - do:
       indices.create:
@@ -172,12 +172,7 @@ multi-value routing path field:
           - '{"index": {}}'
           - '{"@timestamp": "2021-04-28T18:35:54.467Z", "uid": "df3145b3-0563-4d3b-a0f7-897eb2876ea9", "voltage": 6.8, "unmapped_field": 40, "tag": [ "one", "three" ] }'
 
-  - is_true: errors
-
-  - match: {items.1.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" }
-  - match: {items.3.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" }
-  - match: {items.4.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" }
-  - match: {items.7.index.error.reason: "Error extracting routing: Routing values must be strings but found [START_ARRAY]" }
+  - is_false: errors
 
   - do:
       search:
@@ -195,13 +190,21 @@ multi-value routing path field:
                   avg:
                     field: voltage
 
-  - match: {hits.total.value: 4}
-  - length: {aggregations.tsids.buckets: 2}
+  - match: {hits.total.value: 8}
+  - length: {aggregations.tsids.buckets: 4}
 
-  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.0.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSOYoRw_LI4DD7DFEGEJ3NR3eQkMY" }
   - match: {aggregations.tsids.buckets.0.doc_count: 2 }
   - close_to: {aggregations.tsids.buckets.0.voltage.value: { value: 6.70, error: 0.01 }}
 
-  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: { aggregations.tsids.buckets.1.key: "KDODRmbj7vu4rLWvjrJbpUtt0uPSddqA4WYKglGPR_C0cJe8QGaiC2c" }
   - match: {aggregations.tsids.buckets.1.doc_count: 2 }
-  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.30, error: 0.01 }}
+  - close_to: {aggregations.tsids.buckets.1.voltage.value: { value: 7.15, error: 0.01 }}
+
+  - match: { aggregations.tsids.buckets.2.key: "KDODRmbj7vu4rLWvjrJbpUuaET_vOYoRw6ImzKEcF4sEaGKnXSaKfM0" }
+  - match: {aggregations.tsids.buckets.2.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.2.voltage.value: { value: 6.70, error: 0.01 }}
+
+  - match: { aggregations.tsids.buckets.3.key: "KDODRmbj7vu4rLWvjrJbpUvcUWJEddqA4Seo8jbBBBFxwC0lrefCb6A" }
+  - match: {aggregations.tsids.buckets.3.doc_count: 2 }
+  - close_to: {aggregations.tsids.buckets.3.voltage.value: { value: 7.30, error: 0.01 }}

server/src/internalClusterTest/java/org/elasticsearch/snapshots/ConcurrentSnapshotsIT.java

@@ -18,7 +18,6 @@
 import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus;
 import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
-import org.elasticsearch.action.admin.indices.delete.DeleteIndexClusterStateUpdateRequest;
 import org.elasticsearch.action.support.ActionTestUtils;
 import org.elasticsearch.action.support.GroupedActionListener;
 import org.elasticsearch.action.support.PlainActionFuture;
@@ -33,9 +32,7 @@
 import org.elasticsearch.common.util.concurrent.ListenableFuture;
 import org.elasticsearch.common.util.concurrent.UncategorizedExecutionException;
 import org.elasticsearch.core.PathUtils;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.discovery.AbstractDisruptionTestCase;
-import org.elasticsearch.index.Index;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.repositories.IndexId;
 import org.elasticsearch.repositories.RepositoryConflictException;
@@ -59,6 +56,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
@@ -2214,12 +2212,17 @@ public void testDeleteIndexWithOutOfOrderFinalization() {
                     .anyMatch(e -> e.snapshot().getSnapshotId().getName().equals("snapshot-with-index-1") && e.state().completed())
             )
             // execute the index deletion _directly on the master_ so it happens before the snapshot finalization executes
-            .andThen(l -> masterDeleteIndexService.deleteIndices(new DeleteIndexClusterStateUpdateRequest(l.map(r -> {
-                assertTrue(r.isAcknowledged());
-                return null;
-            })).indices(new Index[] { internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex() })
-                .ackTimeout(TimeValue.timeValueSeconds(10))
-                .masterNodeTimeout(TimeValue.timeValueSeconds(10))))
+            .andThen(
+                l -> masterDeleteIndexService.deleteIndices(
+                    TEST_REQUEST_TIMEOUT,
+                    TEST_REQUEST_TIMEOUT,
+                    Set.of(internalCluster().clusterService().state().metadata().index(indexToDelete).getIndex()),
+                    l.map(r -> {
+                        assertTrue(r.isAcknowledged());
+                        return null;
+                    })
+                )
+            )
             // ultimately create the index again so that taking a full snapshot will pick up any missing shard gen blob, and deleting that
             // full snapshot will clean up all dangling shard-level blobs
             .andThen((l, ignored) -> prepareCreate(indexToDelete, indexSettingsNoReplicas(1)).execute(l.map(r -> {