Skip to content

Commit ebca2bb

Browse files
drempapisdrempapis
authored
Merge branch 'main' into fixup-search-error-trace-it
2 parents 2ac4e98 + 1bb9d2b commit ebca2bb

File tree

46 files changed

+1020
-392
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1020
-392
lines changed

‎distribution/docker/src/docker/iron_bank/hardening_manifest.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ tags:
1111
# Build args passed to Dockerfile ARGs
1212
args:
1313
BASE_IMAGE: "redhat/ubi/ubi9"
14-
BASE_TAG: "9.5"
14+
BASE_TAG: "9.6"
1515
# Docker image labels
1616
labels:
1717
org.opencontainers.image.title: "elasticsearch"

‎docs/changelog/129662.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 129662
2+
summary: "[Security] Add entity store and asset criticality index privileges to built\
3+
\ in Editor, Viewer and Kibana System roles"
4+
area: Authorization
5+
type: enhancement
6+
issues: []

‎docs/changelog/131261.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pr: 131261
2+
summary: Enable Failure Store for new logs-*-* data streams
3+
area: Data streams
4+
type: feature
5+
issues:
6+
- 131105
7+
highlight:
8+
title: Enable Failure Store for new logs data streams
9+
body: |-
10+
The [Failure Store](docs-content://manage-data/data-store/data-streams/failure-store.md) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a
11+
dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](docs-content://manage-data/data-store/data-streams/failure-store.md#set-up-failure-store-existing) for existing data streams.
12+
Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`.
13+
notable: true

‎docs/changelog/131937.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 131937
2+
summary: Fix race condition in `RemoteClusterService.collectNodes()`
3+
area: Distributed
4+
type: bug
5+
issues: []

‎docs/changelog/132101.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 132101
2+
summary: Simulate ingest API uses existing index mapping when `mapping_addition` is
3+
given
4+
area: Ingest Node
5+
type: bug
6+
issues: []

‎gradle/verification-metadata.xml‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2150,24 +2150,24 @@
21502150
<sha256 value="5c8551990307a032336d98ddaed549a39a689f07d4d4c6b950601bf22b3d6a1b" origin="Generated by Gradle"/>
21512151
</artifact>
21522152
</component>
2153-
<component group="org.apache.arrow" name="arrow-format" version="16.1.0">
2154-
<artifact name="arrow-format-16.1.0.jar">
2155-
<sha256 value="ad97e0fc72e193b1de3cbce4818d1ff16e81673fd523d001e8d2774bde40ee6c" origin="Generated by Gradle"/>
2153+
<component group="org.apache.arrow" name="arrow-format" version="18.3.0">
2154+
<artifact name="arrow-format-18.3.0.jar">
2155+
<sha256 value="728297d7757d192053ce071cc59e76c8a03ff4f5e430177fa97e96b29dd2de1b" origin="Generated by Gradle"/>
21562156
</artifact>
21572157
</component>
2158-
<component group="org.apache.arrow" name="arrow-memory-core" version="16.1.0">
2159-
<artifact name="arrow-memory-core-16.1.0.jar">
2160-
<sha256 value="da7af1a1a899bd5a1b6c71284243b9f3c0e1098f0cb10cd7be4b8b455ced79dd" origin="Generated by Gradle"/>
2158+
<component group="org.apache.arrow" name="arrow-memory-core" version="18.3.0">
2159+
<artifact name="arrow-memory-core-18.3.0.jar">
2160+
<sha256 value="32ed0719bf2ba42becc3a88c95722851cf1bb88b37d06c95654a1152bed6ef2e" origin="Generated by Gradle"/>
21612161
</artifact>
21622162
</component>
2163-
<component group="org.apache.arrow" name="arrow-memory-unsafe" version="16.1.0">
2164-
<artifact name="arrow-memory-unsafe-16.1.0.jar">
2165-
<sha256 value="6534eded25f2c30593416a294c1047f0b017baa9906d98f6f3270737b076c745" origin="Generated by Gradle"/>
2163+
<component group="org.apache.arrow" name="arrow-memory-unsafe" version="18.3.0">
2164+
<artifact name="arrow-memory-unsafe-18.3.0.jar">
2165+
<sha256 value="e9652f322d7be306c400ea0d775b3114a11b48d2afe23e8b8566f13c469d5483" origin="Generated by Gradle"/>
21662166
</artifact>
21672167
</component>
2168-
<component group="org.apache.arrow" name="arrow-vector" version="16.1.0">
2169-
<artifact name="arrow-vector-16.1.0.jar">
2170-
<sha256 value="c5837b3aa24dfd93759f57bc5759b9a8fbb5bf3912d55994d70cabb904436aab" origin="Generated by Gradle"/>
2168+
<component group="org.apache.arrow" name="arrow-vector" version="18.3.0">
2169+
<artifact name="arrow-vector-18.3.0.jar">
2170+
<sha256 value="b37eda92daccaffc12abf5ed425db1d0bdb3edea150a6ca856f1cf4292442299" origin="Generated by Gradle"/>
21712171
</artifact>
21722172
</component>
21732173
<component group="org.apache.avro" name="avro" version="1.7.4">

‎modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import static org.hamcrest.Matchers.contains;
1818
import static org.hamcrest.Matchers.containsInAnyOrder;
19+
import static org.hamcrest.Matchers.containsString;
1920
import static org.hamcrest.Matchers.empty;
2021
import static org.hamcrest.Matchers.equalTo;
2122
import static org.hamcrest.Matchers.is;
@@ -740,6 +741,73 @@ public void testIgnoreDynamicBeyondLimit() throws Exception {
740741
assertThat(ignored.stream().filter(i -> i.startsWith("field") == false).toList(), empty());
741742
}
742743

744+
@SuppressWarnings("unchecked")
745+
public void testFailureStoreWithInvalidFieldType() throws Exception {
746+
String dataStreamName = "logs-app-with-failure-store";
747+
createDataStream(client, dataStreamName);
748+
749+
indexDoc(client, dataStreamName, """
750+
{
751+
"@timestamp": "2023-11-30T12:00:00Z",
752+
"message": "This is a valid message"
753+
}
754+
""");
755+
756+
// invalid document (message as an object instead of string)
757+
indexDoc(client, dataStreamName, """
758+
{
759+
"@timestamp": "2023-11-30T12:01:00Z",
760+
"message": {
761+
"nested": "This should fail because message should be a string"
762+
}
763+
}
764+
""");
765+
766+
refreshAllIndices();
767+
768+
Request dsInfoRequest = new Request("GET", "/_data_stream/" + dataStreamName);
769+
Map<String, Object> dsInfoResponse = entityAsMap(client.performRequest(dsInfoRequest));
770+
List<Map<String, Object>> dataStreams = (List<Map<String, Object>>) dsInfoResponse.get("data_streams");
771+
Map<String, Object> dataStream = dataStreams.getFirst();
772+
Map<String, Object> failureStoreInfo = (Map<String, Object>) dataStream.get("failure_store");
773+
assertNotNull(failureStoreInfo);
774+
assertThat(failureStoreInfo.get("enabled"), is(true));
775+
List<Map<String, Object>> failureIndices = (List<Map<String, Object>>) failureStoreInfo.get("indices");
776+
777+
assertThat(failureIndices, not(empty()));
778+
String failureIndex = (String) failureIndices.getFirst().get("index_name");
779+
assertThat(failureIndex, matchesRegex("\\.fs-" + dataStreamName + "-.*"));
780+
781+
// query the failure store index
782+
Request failureStoreQuery = new Request("GET", "/" + failureIndex + "/_search");
783+
failureStoreQuery.setJsonEntity("""
784+
{
785+
"query": {
786+
"match_all": {}
787+
}
788+
}
789+
""");
790+
Map<String, Object> failureStoreResponse = entityAsMap(client.performRequest(failureStoreQuery));
791+
Map<String, Object> hits = (Map<String, Object>) failureStoreResponse.get("hits");
792+
List<Map<String, Object>> hitsList = (List<Map<String, Object>>) hits.get("hits");
793+
794+
// Verify the failed document is in the failure store
795+
assertThat(hitsList.size(), is(1));
796+
Map<String, Object> failedDoc = (Map<String, Object>) hitsList.getFirst().get("_source");
797+
Map<String, Object> document = (Map<String, Object>) failedDoc.get("document");
798+
assertNotNull(document);
799+
Map<String, Object> source = (Map<String, Object>) document.get("source");
800+
assertNotNull(source);
801+
Map<String, Object> message = (Map<String, Object>) source.get("message");
802+
assertNotNull(message);
803+
assertThat(message.get("nested"), equalTo("This should fail because message should be a string"));
804+
Map<String, Object> error = (Map<String, Object>) failedDoc.get("error");
805+
assertNotNull(error);
806+
assertEquals("document_parsing_exception", error.get("type"));
807+
String errorMessage = (String) error.get("message");
808+
assertThat(errorMessage, containsString("failed to parse field [message] of type [match_only_text] in document with id"));
809+
}
810+
743811
@Override
744812
protected String indexTemplateName() {
745813
return "logs";

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,20 @@ public class DataStreamFeatures implements FeatureSpecification {
2828

2929
public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
3030

31+
public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled");
32+
3133
@Override
3234
public Set<NodeFeature> getFeatures() {
3335
return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE);
3436
}
3537

3638
@Override
3739
public Set<NodeFeature> getTestFeatures() {
38-
return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
40+
return Set.of(
41+
DATA_STREAM_FAILURE_STORE_TSDB_FIX,
42+
DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX,
43+
LOGS_STREAM_FEATURE,
44+
FAILURE_STORE_IN_LOG_DATA_STREAMS
45+
);
3946
}
4047
}

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
---
2+
setup:
3+
- requires:
4+
cluster_features: [ "logs_data_streams.failure_store.enabled" ]
5+
reason: "failure store became enabled by default for log data streams in 9.2.0"
6+
7+
- do:
8+
indices.create_data_stream:
9+
name: logs-app-default
10+
---
11+
teardown:
12+
- do:
13+
indices.delete_data_stream:
14+
name: logs-app-default
15+
ignore: 404
16+
17+
---
18+
"Test logs-*-* data streams have failure store enabled by default":
19+
# index a valid document (string message)
20+
- do:
21+
index:
22+
index: logs-app-default
23+
refresh: true
24+
body:
25+
'@timestamp': '2023-01-01T12:00:00Z'
26+
host:
27+
name: 'server-01'
28+
severity: 'INFO'
29+
message: "Application started successfully"
30+
- match: { result: created }
31+
32+
- do:
33+
indices.get_data_stream:
34+
name: logs-app-default
35+
- match: { data_streams.0.name: logs-app-default }
36+
- length: { data_streams.0.indices: 1 }
37+
- match: { data_streams.0.failure_store.enabled: true }
38+
- length: { data_streams.0.failure_store.indices: 0 }
39+
40+
# index a document with (object message, causing a mapping conflict)
41+
- do:
42+
index:
43+
index: logs-app-default
44+
refresh: true
45+
body:
46+
'@timestamp': '2023-01-01T12:01:00Z'
47+
host:
48+
name: 'server-02'
49+
severity: 'ERROR'
50+
message:
51+
struct:
52+
value: 42
53+
- match: { result: 'created' }
54+
- match: { failure_store: used}
55+
56+
- do:
57+
indices.get_data_stream:
58+
name: logs-app-default
59+
- length: { data_streams.0.failure_store.indices: 1 }
60+
61+
- do:
62+
search:
63+
index: logs-app-default::data
64+
body:
65+
query:
66+
match_all: {}
67+
- length: { hits.hits: 1 }
68+
- match: { hits.hits.0._source.severity: "INFO" }
69+
- match: { hits.hits.0._source.message: "Application started successfully" }
70+
71+
- do:
72+
search:
73+
index: logs-app-default::failures
74+
body:
75+
query:
76+
match_all: {}
77+
- length: { hits.hits: 1 }
78+
- match: { hits.hits.0._source.document.source.message.struct.value: 42 }
79+
- match: { hits.hits.0._source.error.type: "document_parsing_exception" }

‎muted-tests.yml‎

Lines changed: 39 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -566,15 +566,6 @@ tests:
566566
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
567567
method: testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_disk}
568568
issue: https://github.com/elastic/elasticsearch/issues/132134
569-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
570-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=1}
571-
issue: https://github.com/elastic/elasticsearch/issues/132135
572-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
573-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=2}
574-
issue: https://github.com/elastic/elasticsearch/issues/132136
575-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
576-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=3}
577-
issue: https://github.com/elastic/elasticsearch/issues/132137
578569
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
579570
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=int4_hnsw}
580571
issue: https://github.com/elastic/elasticsearch/issues/132140
@@ -614,7 +605,45 @@ tests:
614605
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
615606
method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
616607
issue: https://github.com/elastic/elasticsearch/issues/132189
617-
608+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
609+
method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_hnsw}
610+
issue: https://github.com/elastic/elasticsearch/issues/132211
611+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
612+
method: "testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk #2}"
613+
issue: https://github.com/elastic/elasticsearch/issues/132213
614+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
615+
method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk}
616+
issue: https://github.com/elastic/elasticsearch/issues/132214
617+
- class: org.elasticsearch.xpack.security.authz.store.CompositeRolesStoreTests
618+
method: testCacheCleanupOnProjectDeletion
619+
issue: https://github.com/elastic/elasticsearch/issues/132218
620+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
621+
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_hnsw}
622+
issue: https://github.com/elastic/elasticsearch/issues/132219
623+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
624+
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=bbq_disk}
625+
issue: https://github.com/elastic/elasticsearch/issues/132220
626+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
627+
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_disk}
628+
issue: https://github.com/elastic/elasticsearch/issues/132221
629+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
630+
method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk}
631+
issue: https://github.com/elastic/elasticsearch/issues/132222
632+
- class: org.elasticsearch.test.rest.yaml.MDPYamlTestSuiteIT
633+
method: test {yaml=mdp/10_basic/Index using shared data path}
634+
issue: https://github.com/elastic/elasticsearch/issues/132223
635+
- class: org.elasticsearch.xpack.logsdb.qa.StandardVersusStandardReindexedIntoLogsDbChallengeRestIT
636+
method: testTermsQuery
637+
issue: https://github.com/elastic/elasticsearch/issues/132225
638+
- class: org.elasticsearch.xpack.logsdb.qa.StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT
639+
method: testTermsQuery
640+
issue: https://github.com/elastic/elasticsearch/issues/132226
641+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
642+
method: "testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_disk #2}"
643+
issue: https://github.com/elastic/elasticsearch/issues/132228
644+
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
645+
method: "testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_disk #2}"
646+
issue: https://github.com/elastic/elasticsearch/issues/132234
618647

619648
# Examples:
620649
#

0 commit comments

Comments
 (0)