Skip to content

Commit 6e9bbd0

Browse files
tvernumtvernum
authored
Merge branch 'main' into ssl/profile/start
2 parents 2c44338 + fe4a523 commit 6e9bbd0

File tree

11 files changed

+244
-193
lines changed

11 files changed

+244
-193
lines changed

‎docs/changelog/131261.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pr: 131261
2+
summary: Enable Failure Store for new logs-*-* data streams
3+
area: Data streams
4+
type: feature
5+
issues:
6+
- 131105
7+
highlight:
8+
title: Enable Failure Store for new logs data streams
9+
body: |-
10+
The [Failure Store](docs-content://manage-data/data-store/data-streams/failure-store.md) is now enabled by default for new logs data streams matching the pattern `logs-*-*`. This means that such data streams will now store invalid documents in a
11+
dedicated failure index instead of rejecting them, allowing better visibility and control over data quality issues without loosing data. This can be [enabled manually](docs-content://manage-data/data-store/data-streams/failure-store.md#set-up-failure-store-existing) for existing data streams.
12+
Note: With the failure store enabled, the http response code clients receive when indexing invalid documents will change from `400 Bad Request` to `201 Created`, with an additional response attribute `"failure_store" : "used"`.
13+
notable: true

‎modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
import static org.hamcrest.Matchers.contains;
1818
import static org.hamcrest.Matchers.containsInAnyOrder;
19+
import static org.hamcrest.Matchers.containsString;
1920
import static org.hamcrest.Matchers.empty;
2021
import static org.hamcrest.Matchers.equalTo;
2122
import static org.hamcrest.Matchers.is;
@@ -740,6 +741,73 @@ public void testIgnoreDynamicBeyondLimit() throws Exception {
740741
assertThat(ignored.stream().filter(i -> i.startsWith("field") == false).toList(), empty());
741742
}
742743

744+
@SuppressWarnings("unchecked")
745+
public void testFailureStoreWithInvalidFieldType() throws Exception {
746+
String dataStreamName = "logs-app-with-failure-store";
747+
createDataStream(client, dataStreamName);
748+
749+
indexDoc(client, dataStreamName, """
750+
{
751+
"@timestamp": "2023-11-30T12:00:00Z",
752+
"message": "This is a valid message"
753+
}
754+
""");
755+
756+
// invalid document (message as an object instead of string)
757+
indexDoc(client, dataStreamName, """
758+
{
759+
"@timestamp": "2023-11-30T12:01:00Z",
760+
"message": {
761+
"nested": "This should fail because message should be a string"
762+
}
763+
}
764+
""");
765+
766+
refreshAllIndices();
767+
768+
Request dsInfoRequest = new Request("GET", "/_data_stream/" + dataStreamName);
769+
Map<String, Object> dsInfoResponse = entityAsMap(client.performRequest(dsInfoRequest));
770+
List<Map<String, Object>> dataStreams = (List<Map<String, Object>>) dsInfoResponse.get("data_streams");
771+
Map<String, Object> dataStream = dataStreams.getFirst();
772+
Map<String, Object> failureStoreInfo = (Map<String, Object>) dataStream.get("failure_store");
773+
assertNotNull(failureStoreInfo);
774+
assertThat(failureStoreInfo.get("enabled"), is(true));
775+
List<Map<String, Object>> failureIndices = (List<Map<String, Object>>) failureStoreInfo.get("indices");
776+
777+
assertThat(failureIndices, not(empty()));
778+
String failureIndex = (String) failureIndices.getFirst().get("index_name");
779+
assertThat(failureIndex, matchesRegex("\\.fs-" + dataStreamName + "-.*"));
780+
781+
// query the failure store index
782+
Request failureStoreQuery = new Request("GET", "/" + failureIndex + "/_search");
783+
failureStoreQuery.setJsonEntity("""
784+
{
785+
"query": {
786+
"match_all": {}
787+
}
788+
}
789+
""");
790+
Map<String, Object> failureStoreResponse = entityAsMap(client.performRequest(failureStoreQuery));
791+
Map<String, Object> hits = (Map<String, Object>) failureStoreResponse.get("hits");
792+
List<Map<String, Object>> hitsList = (List<Map<String, Object>>) hits.get("hits");
793+
794+
// Verify the failed document is in the failure store
795+
assertThat(hitsList.size(), is(1));
796+
Map<String, Object> failedDoc = (Map<String, Object>) hitsList.getFirst().get("_source");
797+
Map<String, Object> document = (Map<String, Object>) failedDoc.get("document");
798+
assertNotNull(document);
799+
Map<String, Object> source = (Map<String, Object>) document.get("source");
800+
assertNotNull(source);
801+
Map<String, Object> message = (Map<String, Object>) source.get("message");
802+
assertNotNull(message);
803+
assertThat(message.get("nested"), equalTo("This should fail because message should be a string"));
804+
Map<String, Object> error = (Map<String, Object>) failedDoc.get("error");
805+
assertNotNull(error);
806+
assertEquals("document_parsing_exception", error.get("type"));
807+
String errorMessage = (String) error.get("message");
808+
assertThat(errorMessage, containsString("failed to parse field [message] of type [match_only_text] in document with id"));
809+
}
810+
743811
@Override
744812
protected String indexTemplateName() {
745813
return "logs";

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/DataStreamFeatures.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,20 @@ public class DataStreamFeatures implements FeatureSpecification {
2828

2929
public static final NodeFeature LOGS_STREAM_FEATURE = new NodeFeature("logs_stream");
3030

31+
public static final NodeFeature FAILURE_STORE_IN_LOG_DATA_STREAMS = new NodeFeature("logs_data_streams.failure_store.enabled");
32+
3133
@Override
3234
public Set<NodeFeature> getFeatures() {
3335
return Set.of(DataStream.DATA_STREAM_FAILURE_STORE_FEATURE);
3436
}
3537

3638
@Override
3739
public Set<NodeFeature> getTestFeatures() {
38-
return Set.of(DATA_STREAM_FAILURE_STORE_TSDB_FIX, DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX, LOGS_STREAM_FEATURE);
40+
return Set.of(
41+
DATA_STREAM_FAILURE_STORE_TSDB_FIX,
42+
DOWNSAMPLE_AGGREGATE_DEFAULT_METRIC_FIX,
43+
LOGS_STREAM_FEATURE,
44+
FAILURE_STORE_IN_LOG_DATA_STREAMS
45+
);
3946
}
4047
}

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
---
2+
setup:
3+
- requires:
4+
cluster_features: [ "logs_data_streams.failure_store.enabled" ]
5+
reason: "failure store became enabled by default for log data streams in 9.2.0"
6+
7+
- do:
8+
indices.create_data_stream:
9+
name: logs-app-default
10+
---
11+
teardown:
12+
- do:
13+
indices.delete_data_stream:
14+
name: logs-app-default
15+
ignore: 404
16+
17+
---
18+
"Test logs-*-* data streams have failure store enabled by default":
19+
# index a valid document (string message)
20+
- do:
21+
index:
22+
index: logs-app-default
23+
refresh: true
24+
body:
25+
'@timestamp': '2023-01-01T12:00:00Z'
26+
host:
27+
name: 'server-01'
28+
severity: 'INFO'
29+
message: "Application started successfully"
30+
- match: { result: created }
31+
32+
- do:
33+
indices.get_data_stream:
34+
name: logs-app-default
35+
- match: { data_streams.0.name: logs-app-default }
36+
- length: { data_streams.0.indices: 1 }
37+
- match: { data_streams.0.failure_store.enabled: true }
38+
- length: { data_streams.0.failure_store.indices: 0 }
39+
40+
# index a document with (object message, causing a mapping conflict)
41+
- do:
42+
index:
43+
index: logs-app-default
44+
refresh: true
45+
body:
46+
'@timestamp': '2023-01-01T12:01:00Z'
47+
host:
48+
name: 'server-02'
49+
severity: 'ERROR'
50+
message:
51+
struct:
52+
value: 42
53+
- match: { result: 'created' }
54+
- match: { failure_store: used}
55+
56+
- do:
57+
indices.get_data_stream:
58+
name: logs-app-default
59+
- length: { data_streams.0.failure_store.indices: 1 }
60+
61+
- do:
62+
search:
63+
index: logs-app-default::data
64+
body:
65+
query:
66+
match_all: {}
67+
- length: { hits.hits: 1 }
68+
- match: { hits.hits.0._source.severity: "INFO" }
69+
- match: { hits.hits.0._source.message: "Application started successfully" }
70+
71+
- do:
72+
search:
73+
index: logs-app-default::failures
74+
body:
75+
query:
76+
match_all: {}
77+
- length: { hits.hits: 1 }
78+
- match: { hits.hits.0._source.document.source.message.struct.value: 42 }
79+
- match: { hits.hits.0._source.error.type: "document_parsing_exception" }

‎muted-tests.yml‎

Lines changed: 0 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -512,132 +512,15 @@ tests:
512512
- class: org.elasticsearch.xpack.test.rest.XPackRestIT
513513
method: test {p0=transform/transforms_crud/Test transform where source query is invalid}
514514
issue: https://github.com/elastic/elasticsearch/issues/132111
515-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
516-
method: testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_hnsw}
517-
issue: https://github.com/elastic/elasticsearch/issues/132112
518-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
519-
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=int4_hnsw}
520-
issue: https://github.com/elastic/elasticsearch/issues/132113
521-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
522-
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=bbq_hnsw}
523-
issue: https://github.com/elastic/elasticsearch/issues/132115
524-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
525-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=int8_hnsw}
526-
issue: https://github.com/elastic/elasticsearch/issues/132116
527-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
528-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=int8_hnsw}
529-
issue: https://github.com/elastic/elasticsearch/issues/132117
530-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
531-
method: testDenseVectorMappingUpdate {initialType=flat updateType=hnsw}
532-
issue: https://github.com/elastic/elasticsearch/issues/132119
533-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
534-
method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_flat}
535-
issue: https://github.com/elastic/elasticsearch/issues/132120
536-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
537-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk}
538-
issue: https://github.com/elastic/elasticsearch/issues/132122
539-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
540-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=int4_hnsw}
541-
issue: https://github.com/elastic/elasticsearch/issues/132123
542-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
543-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=hnsw}
544-
issue: https://github.com/elastic/elasticsearch/issues/132124
545-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
546-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=int4_flat}
547-
issue: https://github.com/elastic/elasticsearch/issues/132125
548-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
549-
method: testDenseVectorMappingUpdate {initialType=flat updateType=int8_hnsw}
550-
issue: https://github.com/elastic/elasticsearch/issues/132126
551-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
552-
method: testDenseVectorMappingUpdate {initialType=bbq_disk updateType=bbq_disk}
553-
issue: https://github.com/elastic/elasticsearch/issues/132127
554-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
555-
method: testDenseVectorMappingUpdate {initialType=flat updateType=int8_flat}
556-
issue: https://github.com/elastic/elasticsearch/issues/132129
557-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
558-
method: "testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk #2}"
559-
issue: https://github.com/elastic/elasticsearch/issues/132130
560-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
561-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_flat}
562-
issue: https://github.com/elastic/elasticsearch/issues/132132
563-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
564-
method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=int4_hnsw}
565-
issue: https://github.com/elastic/elasticsearch/issues/132133
566-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
567-
method: testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_disk}
568-
issue: https://github.com/elastic/elasticsearch/issues/132134
569-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
570-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=1}
571-
issue: https://github.com/elastic/elasticsearch/issues/132135
572-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
573-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=2}
574-
issue: https://github.com/elastic/elasticsearch/issues/132136
575-
- class: org.elasticsearch.upgrades.RunningSnapshotIT
576-
method: testRunningSnapshotCompleteAfterUpgrade {upgradedNodes=3}
577-
issue: https://github.com/elastic/elasticsearch/issues/132137
578-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
579-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=int4_hnsw}
580-
issue: https://github.com/elastic/elasticsearch/issues/132140
581-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
582-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_hnsw}
583-
issue: https://github.com/elastic/elasticsearch/issues/132141
584515
- class: org.elasticsearch.index.engine.MergeWithLowDiskSpaceIT
585516
method: testRelocationWhileForceMerging
586517
issue: https://github.com/elastic/elasticsearch/issues/131789
587-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
588-
method: testDenseVectorMappingUpdate {initialType=flat updateType=int4_hnsw}
589-
issue: https://github.com/elastic/elasticsearch/issues/132149
590-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
591-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=hnsw}
592-
issue: https://github.com/elastic/elasticsearch/issues/132150
593-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
594-
method: testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_flat}
595-
issue: https://github.com/elastic/elasticsearch/issues/132151
596-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
597-
method: "testDenseVectorMappingUpdate {initialType=bbq_hnsw updateType=bbq_disk #2}"
598-
issue: https://github.com/elastic/elasticsearch/issues/132152
599-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
600-
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=int8_hnsw}
601-
issue: https://github.com/elastic/elasticsearch/issues/132164
602-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
603-
method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_disk}
604-
issue: https://github.com/elastic/elasticsearch/issues/132165
605518
- class: org.elasticsearch.indices.cluster.FieldCapsForceConnectTimeoutIT
606519
method: testTimeoutSetting
607520
issue: https://github.com/elastic/elasticsearch/issues/132179
608-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
609-
method: "testDenseVectorMappingUpdate {initialType=bbq_flat updateType=bbq_disk #2}"
610-
issue: https://github.com/elastic/elasticsearch/issues/132184
611-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
612-
method: testDenseVectorMappingUpdate {initialType=bbq_hnsw updateType=bbq_disk}
613-
issue: https://github.com/elastic/elasticsearch/issues/132188
614-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
615-
method: "testDenseVectorMappingUpdate {initialType=int8_flat updateType=bbq_disk #2}"
616-
issue: https://github.com/elastic/elasticsearch/issues/132189
617-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
618-
method: testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_hnsw}
619-
issue: https://github.com/elastic/elasticsearch/issues/132211
620-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
621-
method: "testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk #2}"
622-
issue: https://github.com/elastic/elasticsearch/issues/132213
623-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
624-
method: testDenseVectorMappingUpdate {initialType=int8_hnsw updateType=bbq_disk}
625-
issue: https://github.com/elastic/elasticsearch/issues/132214
626521
- class: org.elasticsearch.xpack.security.authz.store.CompositeRolesStoreTests
627522
method: testCacheCleanupOnProjectDeletion
628523
issue: https://github.com/elastic/elasticsearch/issues/132218
629-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
630-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_hnsw}
631-
issue: https://github.com/elastic/elasticsearch/issues/132219
632-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
633-
method: testDenseVectorMappingUpdate {initialType=hnsw updateType=bbq_disk}
634-
issue: https://github.com/elastic/elasticsearch/issues/132220
635-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
636-
method: testDenseVectorMappingUpdate {initialType=int4_flat updateType=bbq_disk}
637-
issue: https://github.com/elastic/elasticsearch/issues/132221
638-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
639-
method: testDenseVectorMappingUpdate {initialType=flat updateType=bbq_disk}
640-
issue: https://github.com/elastic/elasticsearch/issues/132222
641524
- class: org.elasticsearch.test.rest.yaml.MDPYamlTestSuiteIT
642525
method: test {yaml=mdp/10_basic/Index using shared data path}
643526
issue: https://github.com/elastic/elasticsearch/issues/132223
@@ -647,10 +530,6 @@ tests:
647530
- class: org.elasticsearch.xpack.logsdb.qa.StoredSourceLogsDbVersusReindexedLogsDbChallengeRestIT
648531
method: testTermsQuery
649532
issue: https://github.com/elastic/elasticsearch/issues/132226
650-
- class: org.elasticsearch.index.mapper.vectors.DenseVectorFieldIndexTypeUpdateIT
651-
method: "testDenseVectorMappingUpdate {initialType=int4_hnsw updateType=bbq_disk #2}"
652-
issue: https://github.com/elastic/elasticsearch/issues/132228
653-
654533
# Examples:
655534
#
656535
# Mute a single test case in a YAML test suite:

0 commit comments

Comments
 (0)