Skip to content

Commit 37f03dc

Browse files
lukewhitinglukewhiting
authored
#111893 Add Warnings For Missing Index Templates (#114589)
* Add data stream template validation to snapshot restore * Add data stream template validation to data stream promotion endpoint * Add new assertion for response headers Add a new assertion to synchronously execute a request and check the response contains a specific warning header * Test for warning header on snapshot restore When missing templates * Test for promotion warnings * Add documentation for the potential error states * PR changes * Spotless reformatting * Add logic to look in snapshot global metadata This checks if the snapshot contains a matching template for the DS * Comment on test cleanup to explain it was copied * Removed cluster service field
1 parent 39168e1 commit 37f03dc

File tree

7 files changed

+407
-2
lines changed

7 files changed

+407
-2
lines changed

‎docs/reference/data-streams/promote-data-stream-api.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,10 @@ available, the data stream in the local cluster can be promoted
1818
to a regular data stream, which allows these data streams to
1919
be rolled over in the local cluster.
2020

21+
NOTE: When promoting a data stream, ensure the local cluster has a data stream enabled index template that matches the data stream.
22+
If this is missing, the data stream will not be able to roll over until a matching index template is created.
23+
This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
24+
2125
[source,console]
2226
----
2327
POST /_data_stream/_promote/my-data-stream

‎docs/reference/snapshot-restore/index.asciidoc‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,9 @@ Snapshots don't contain or back up:
4848
* Node configuration files
4949
* <<security-files,Security configuration files>>
5050

51+
NOTE: When restoring a data stream, if the target cluster does not have an index template that matches the data stream, the data stream will not be able to roll over until a matching index template is created.
52+
This will affect the lifecycle management of the data stream and interfere with the data stream size and retention.
53+
5154
[discrete]
5255
[[feature-state]]
5356
=== Feature states

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java‎

Lines changed: 153 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest;
1616
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
1717
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
18+
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
1819
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1920
import org.elasticsearch.action.admin.indices.alias.Alias;
2021
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest;
@@ -45,6 +46,7 @@
4546
import org.elasticsearch.search.SearchHit;
4647
import org.elasticsearch.snapshots.AbstractSnapshotIntegTestCase;
4748
import org.elasticsearch.snapshots.RestoreInfo;
49+
import org.elasticsearch.snapshots.SnapshotId;
4850
import org.elasticsearch.snapshots.SnapshotInProgressException;
4951
import org.elasticsearch.snapshots.SnapshotInfo;
5052
import org.elasticsearch.snapshots.SnapshotRestoreException;
@@ -62,7 +64,9 @@
6264
import java.util.stream.Collectors;
6365

6466
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
67+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoWarningHeaderOnResponse;
6568
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse;
69+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertWarningHeaderOnResponse;
6670
import static org.hamcrest.Matchers.anEmptyMap;
6771
import static org.hamcrest.Matchers.contains;
6872
import static org.hamcrest.Matchers.containsInAnyOrder;
@@ -80,6 +84,8 @@ public class DataStreamsSnapshotsIT extends AbstractSnapshotIntegTestCase {
8084
private static final Map<String, Integer> DOCUMENT_SOURCE = Collections.singletonMap("@timestamp", 123);
8185
public static final String REPO = "repo";
8286
public static final String SNAPSHOT = "snap";
87+
public static final String TEMPLATE_1_ID = "t1";
88+
public static final String TEMPLATE_2_ID = "t2";
8389
private Client client;
8490

8591
private String dsBackingIndexName;
@@ -103,8 +109,8 @@ public void setup() throws Exception {
103109
Path location = randomRepoPath();
104110
createRepository(REPO, "fs", location);
105111

106-
DataStreamIT.putComposableIndexTemplate("t1", List.of("ds", "other-ds"));
107-
DataStreamIT.putComposableIndexTemplate("t2", """
112+
DataStreamIT.putComposableIndexTemplate(TEMPLATE_1_ID, List.of("ds", "other-ds"));
113+
DataStreamIT.putComposableIndexTemplate(TEMPLATE_2_ID, """
108114
{
109115
"properties": {
110116
"@timestamp": {
@@ -1335,4 +1341,149 @@ public void testRestoreDataStreamAliasWithConflictingIndicesAlias() throws Excep
13351341
);
13361342
assertThat(e.getMessage(), containsString("data stream alias and indices alias have the same name (my-alias)"));
13371343
}
1344+
1345+
public void testWarningHeaderOnRestoreWithoutTemplates() throws Exception {
1346+
String datastreamName = "ds";
1347+
1348+
CreateSnapshotResponse createSnapshotResponse = client.admin()
1349+
.cluster()
1350+
.prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1351+
.setWaitForCompletion(true)
1352+
.setIndices(datastreamName)
1353+
.setIncludeGlobalState(false)
1354+
.get();
1355+
1356+
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
1357+
SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
1358+
assertEquals(RestStatus.OK, status);
1359+
1360+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
1361+
1362+
assertAcked(
1363+
client.execute(
1364+
DeleteDataStreamAction.INSTANCE,
1365+
new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
1366+
)
1367+
);
1368+
1369+
assertAcked(
1370+
client.execute(
1371+
TransportDeleteComposableIndexTemplateAction.TYPE,
1372+
new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
1373+
).get()
1374+
);
1375+
1376+
RestoreSnapshotRequestBuilder request = client.admin()
1377+
.cluster()
1378+
.prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1379+
.setWaitForCompletion(true)
1380+
.setIndices(datastreamName);
1381+
1382+
assertWarningHeaderOnResponse(
1383+
client,
1384+
request,
1385+
"Snapshot ["
1386+
+ snapshotId
1387+
+ "] contains data stream ["
1388+
+ datastreamName
1389+
+ "] but custer does not have a matching index "
1390+
+ "template. This will cause rollover to fail until a matching index template is created"
1391+
);
1392+
1393+
}
1394+
1395+
public void testWarningHeaderAbsentOnRestoreWithTemplates() throws Exception {
1396+
String datastreamName = "ds";
1397+
1398+
CreateSnapshotResponse createSnapshotResponse = client.admin()
1399+
.cluster()
1400+
.prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1401+
.setWaitForCompletion(true)
1402+
.setIndices(datastreamName)
1403+
.setIncludeGlobalState(false)
1404+
.get();
1405+
1406+
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
1407+
SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
1408+
assertEquals(RestStatus.OK, status);
1409+
1410+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
1411+
1412+
assertAcked(
1413+
client.execute(
1414+
DeleteDataStreamAction.INSTANCE,
1415+
new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds", "with-fs")
1416+
)
1417+
);
1418+
1419+
RestoreSnapshotRequestBuilder request = client.admin()
1420+
.cluster()
1421+
.prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1422+
.setWaitForCompletion(true)
1423+
.setIndices(datastreamName);
1424+
1425+
assertNoWarningHeaderOnResponse(
1426+
client,
1427+
request,
1428+
"but custer does not have a matching index template. This will cause rollover to fail until a matching index "
1429+
+ "template is created"
1430+
);
1431+
1432+
}
1433+
1434+
/**
1435+
* This test is muted as it's awaiting the same fix as {@link #testPartialRestoreSnapshotThatIncludesDataStreamWithGlobalState()}
1436+
*/
1437+
@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/107515")
1438+
public void testWarningHeaderOnRestoreTemplateFromSnapshot() throws Exception {
1439+
String datastreamName = "ds";
1440+
1441+
CreateSnapshotResponse createSnapshotResponse = client.admin()
1442+
.cluster()
1443+
.prepareCreateSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1444+
.setWaitForCompletion(true)
1445+
.setIndices(datastreamName)
1446+
.setIncludeGlobalState(true)
1447+
.get();
1448+
1449+
RestStatus status = createSnapshotResponse.getSnapshotInfo().status();
1450+
SnapshotId snapshotId = createSnapshotResponse.getSnapshotInfo().snapshotId();
1451+
assertEquals(RestStatus.OK, status);
1452+
1453+
assertEquals(Collections.singletonList(dsBackingIndexName), getSnapshot(REPO, SNAPSHOT).indices());
1454+
1455+
assertAcked(
1456+
client.execute(
1457+
DeleteDataStreamAction.INSTANCE,
1458+
new DeleteDataStreamAction.Request(TEST_REQUEST_TIMEOUT, datastreamName, "other-ds")
1459+
)
1460+
);
1461+
1462+
assertAcked(
1463+
client.execute(
1464+
TransportDeleteComposableIndexTemplateAction.TYPE,
1465+
new TransportDeleteComposableIndexTemplateAction.Request(TEMPLATE_1_ID)
1466+
).get()
1467+
);
1468+
1469+
RestoreSnapshotRequestBuilder request = client.admin()
1470+
.cluster()
1471+
.prepareRestoreSnapshot(TEST_REQUEST_TIMEOUT, REPO, SNAPSHOT)
1472+
.setWaitForCompletion(true)
1473+
.setRestoreGlobalState(true)
1474+
.setIndices(datastreamName);
1475+
1476+
assertNoWarningHeaderOnResponse(
1477+
client,
1478+
request,
1479+
"Snapshot ["
1480+
+ snapshotId
1481+
+ "] contains data stream ["
1482+
+ datastreamName
1483+
+ "] but custer does not have a matching index "
1484+
+ "template. This will cause rollover to fail until a matching index template is created"
1485+
);
1486+
1487+
}
1488+
13381489
}

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/PromoteDataStreamTransportAction.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
*/
99
package org.elasticsearch.datastreams.action;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
1113
import org.elasticsearch.ResourceNotFoundException;
1214
import org.elasticsearch.action.ActionListener;
1315
import org.elasticsearch.action.datastreams.PromoteDataStreamAction;
@@ -23,6 +25,8 @@
2325
import org.elasticsearch.cluster.metadata.Metadata;
2426
import org.elasticsearch.cluster.service.ClusterService;
2527
import org.elasticsearch.common.Priority;
28+
import org.elasticsearch.common.logging.HeaderWarning;
29+
import org.elasticsearch.common.regex.Regex;
2630
import org.elasticsearch.common.util.concurrent.EsExecutors;
2731
import org.elasticsearch.core.SuppressForbidden;
2832
import org.elasticsearch.indices.SystemIndices;
@@ -31,8 +35,12 @@
3135
import org.elasticsearch.threadpool.ThreadPool;
3236
import org.elasticsearch.transport.TransportService;
3337

38+
import static org.elasticsearch.core.Strings.format;
39+
3440
public class PromoteDataStreamTransportAction extends AcknowledgedTransportMasterNodeAction<PromoteDataStreamAction.Request> {
3541

42+
private static final Logger logger = LogManager.getLogger(PromoteDataStreamTransportAction.class);
43+
3644
private final SystemIndices systemIndices;
3745

3846
@Inject
@@ -94,16 +102,41 @@ private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String
94102

95103
static ClusterState promoteDataStream(ClusterState currentState, PromoteDataStreamAction.Request request) {
96104
DataStream dataStream = currentState.getMetadata().dataStreams().get(request.getName());
105+
97106
if (dataStream == null) {
98107
throw new ResourceNotFoundException("data stream [" + request.getName() + "] does not exist");
99108
}
100109

110+
warnIfTemplateMissingForDatastream(dataStream, currentState);
111+
101112
DataStream promotedDataStream = dataStream.promoteDataStream();
102113
Metadata.Builder metadata = Metadata.builder(currentState.metadata());
103114
metadata.put(promotedDataStream);
104115
return ClusterState.builder(currentState).metadata(metadata).build();
105116
}
106117

118+
private static void warnIfTemplateMissingForDatastream(DataStream dataStream, ClusterState currentState) {
119+
var datastreamName = dataStream.getName();
120+
121+
var matchingIndex = currentState.metadata()
122+
.templatesV2()
123+
.values()
124+
.stream()
125+
.filter(cit -> cit.getDataStreamTemplate() != null)
126+
.flatMap(cit -> cit.indexPatterns().stream())
127+
.anyMatch(pattern -> Regex.simpleMatch(pattern, datastreamName));
128+
129+
if (matchingIndex == false) {
130+
String warningMessage = format(
131+
"Data stream [%s] does not have a matching index template. This will cause rollover to fail until a matching index "
132+
+ "template is created",
133+
datastreamName
134+
);
135+
logger.warn(() -> warningMessage);
136+
HeaderWarning.addWarning(warningMessage);
137+
}
138+
}
139+
107140
@Override
108141
protected ClusterBlockException checkBlock(PromoteDataStreamAction.Request request, ClusterState state) {
109142
return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE);

‎server/src/main/java/org/elasticsearch/snapshots/RestoreService.java‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
2727
import org.elasticsearch.cluster.block.ClusterBlocks;
2828
import org.elasticsearch.cluster.metadata.AliasMetadata;
29+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
2930
import org.elasticsearch.cluster.metadata.DataStream;
3031
import org.elasticsearch.cluster.metadata.DataStreamAlias;
3132
import org.elasticsearch.cluster.metadata.IndexMetadata;
@@ -52,6 +53,7 @@
5253
import org.elasticsearch.common.Strings;
5354
import org.elasticsearch.common.UUIDs;
5455
import org.elasticsearch.common.collect.ImmutableOpenMap;
56+
import org.elasticsearch.common.logging.HeaderWarning;
5557
import org.elasticsearch.common.lucene.Lucene;
5658
import org.elasticsearch.common.regex.Regex;
5759
import org.elasticsearch.common.settings.ClusterSettings;
@@ -398,6 +400,8 @@ private void startRestore(
398400
Map<String, DataStream> dataStreamsToRestore = result.v1();
399401
Map<String, DataStreamAlias> dataStreamAliasesToRestore = result.v2();
400402

403+
validateDataStreamTemplatesExistAndWarnIfMissing(dataStreamsToRestore, snapshotInfo, globalMetadata);
404+
401405
// Remove the data streams from the list of requested indices
402406
requestIndices.removeAll(dataStreamsToRestore.keySet());
403407

@@ -510,6 +514,35 @@ private void startRestore(
510514
);
511515
}
512516

517+
private void validateDataStreamTemplatesExistAndWarnIfMissing(
518+
Map<String, DataStream> dataStreamsToRestore,
519+
SnapshotInfo snapshotInfo,
520+
Metadata globalMetadata
521+
) {
522+
523+
Stream<ComposableIndexTemplate> streams = Stream.concat(
524+
clusterService.state().metadata().templatesV2().values().stream(),
525+
globalMetadata == null ? Stream.empty() : globalMetadata.templatesV2().values().stream()
526+
);
527+
528+
Set<String> templatePatterns = streams.filter(cit -> cit.getDataStreamTemplate() != null)
529+
.flatMap(cit -> cit.indexPatterns().stream())
530+
.collect(Collectors.toSet());
531+
532+
for (String name : dataStreamsToRestore.keySet()) {
533+
if (templatePatterns.stream().noneMatch(pattern -> Regex.simpleMatch(pattern, name))) {
534+
String warningMessage = format(
535+
"Snapshot [%s] contains data stream [%s] but custer does not have a matching index template. This will cause"
536+
+ " rollover to fail until a matching index template is created",
537+
snapshotInfo.snapshotId(),
538+
name
539+
);
540+
logger.warn(() -> warningMessage);
541+
HeaderWarning.addWarning(warningMessage);
542+
}
543+
}
544+
}
545+
513546
@SuppressForbidden(reason = "legacy usage of unbatched task") // TODO add support for batching here
514547
private void submitUnbatchedTask(@SuppressWarnings("SameParameterValue") String source, ClusterStateUpdateTask task) {
515548
clusterService.submitUnbatchedStateUpdateTask(source, task);

0 commit comments

Comments
 (0)