diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 7b8709c908357..ef77154c76b5d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -2057,12 +2057,16 @@ protected Map getIndexMappingAsMap(String index) throws IOExcept } protected static boolean indexExists(String index) throws IOException { + return indexExists(client(), index); + } + + protected static boolean indexExists(RestClient client, String index) throws IOException { // We use the /_cluster/health/{index} API to ensure the index exists on the master node - which means all nodes see the index. Request request = new Request("GET", "/_cluster/health/" + index); request.addParameter("timeout", "0"); request.addParameter("level", "indices"); try { - final var response = client().performRequest(request); + final var response = client.performRequest(request); @SuppressWarnings("unchecked") final var indices = (Map) entityAsMap(response).get("indices"); return indices.containsKey(index); @@ -2122,9 +2126,16 @@ protected static boolean aliasExists(String index, String alias) throws IOExcept /** * Returns a list of the data stream's backing index names. */ - @SuppressWarnings("unchecked") protected static List getDataStreamBackingIndexNames(String dataStreamName) throws IOException { - Map response = getAsMap(client(), "/_data_stream/" + dataStreamName); + return getDataStreamBackingIndexNames(client(), dataStreamName); + } + + /** + * Returns a list of the data stream's backing index names. + */ + @SuppressWarnings("unchecked") + protected static List getDataStreamBackingIndexNames(RestClient client, String dataStreamName) throws IOException { + Map response = getAsMap(client, "/_data_stream/" + dataStreamName); List dataStreams = (List) response.get("data_streams"); assertThat(dataStreams.size(), equalTo(1)); Map dataStream = (Map) dataStreams.getFirst(); diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java index 81f84c6b0cf0a..7360953a135f0 100644 --- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AbstractCCRRestTestCase.java @@ -16,13 +16,11 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.util.LazyInitializable; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.support.XContentMapValues; -import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.rest.ESRestTestCase; import org.elasticsearch.xcontent.ToXContent; @@ -31,7 +29,6 @@ import org.junit.Before; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; @@ -46,7 +43,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.hasSize; @TestCaseOrdering(AbstractCCRRestTestCase.TargetClusterTestOrdering.class) public abstract class AbstractCCRRestTestCase extends ESRestTestCase { @@ -354,33 +350,16 @@ protected Set getCcrNodeTasks() throws IOException { protected record CcrNodeTask(String remoteCluster, String leaderIndex, String followerIndex, int shardId) {} - protected static boolean indexExists(String index) throws IOException { - Response response = adminClient().performRequest(new Request("HEAD", "/" + index)); - return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); - } - - protected static List verifyDataStream(final RestClient client, final String name, final String... expectedBackingIndices) + /** + * Verify that the specified data stream has the expected backing index generations. + */ + protected static List verifyDataStream(final RestClient client, final String name, final int... expectedBackingIndices) throws IOException { - Request request = new Request("GET", "/_data_stream/" + name); - Map response = toMap(client.performRequest(request)); - List retrievedDataStreams = (List) response.get("data_streams"); - assertThat(retrievedDataStreams, hasSize(1)); - List actualBackingIndexItems = (List) ((Map) retrievedDataStreams.get(0)).get("indices"); - assertThat(actualBackingIndexItems, hasSize(expectedBackingIndices.length)); - final List actualBackingIndices = new ArrayList<>(); + final List actualBackingIndices = getDataStreamBackingIndexNames(client, name); for (int i = 0; i < expectedBackingIndices.length; i++) { - Map actualBackingIndexItem = (Map) actualBackingIndexItems.get(i); - String actualBackingIndex = (String) actualBackingIndexItem.get("index_name"); - String expectedBackingIndex = expectedBackingIndices[i]; - - String actualDataStreamName = actualBackingIndex.substring(5, actualBackingIndex.indexOf('-', 5)); - String expectedDataStreamName = expectedBackingIndex.substring(5, expectedBackingIndex.indexOf('-', 5)); - assertThat(actualDataStreamName, equalTo(expectedDataStreamName)); - - int actualGeneration = Integer.parseInt(actualBackingIndex.substring(actualBackingIndex.lastIndexOf('-'))); - int expectedGeneration = Integer.parseInt(expectedBackingIndex.substring(expectedBackingIndex.lastIndexOf('-'))); - assertThat(actualGeneration, equalTo(expectedGeneration)); - actualBackingIndices.add(actualBackingIndex); + String actualBackingIndex = actualBackingIndices.get(i); + int expectedBackingIndexGeneration = expectedBackingIndices[i]; + assertThat(actualBackingIndex, DataStreamTestHelper.backingIndexEqualTo(name, expectedBackingIndexGeneration)); } return List.copyOf(actualBackingIndices); } @@ -408,17 +387,6 @@ protected static void createAutoFollowPattern( assertOK(client.performRequest(request)); } - /** - * Fix point in time when data stream backing index is first time queried. - * This is required to avoid failures when running test at midnight. - * (index is created for day0, but assertions are executed for day1 assuming different time based index name that does not exist) - */ - private final LazyInitializable time = new LazyInitializable<>(System::currentTimeMillis); - - protected String backingIndexName(String dataStreamName, int generation) { - return DataStream.getDefaultBackingIndexName(dataStreamName, generation, time.getOrCompute()); - } - protected RestClient buildLeaderClient() throws IOException { assert targetCluster != TargetCluster.LEADER; return buildClient(getLeaderCluster().getHttpAddresses()); diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java index 533a77c84e22d..5679bbce59fd8 100644 --- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/AutoFollowIT.java @@ -18,7 +18,7 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.WarningFailureException; -import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -250,7 +250,7 @@ public void testAutoFollowPatterns() throws Exception { } else { assertThat(getIndexSettingsAsMap("metrics-20210101"), hasEntry("index.number_of_replicas", "1")); } - assertThat(indexExists(excludedIndex), is(false)); + assertThat(indexExists(adminClient(), excludedIndex), is(false)); }); assertLongBusy(() -> verifyCcrMonitoring("metrics-20210101", "metrics-20210101")); @@ -324,12 +324,12 @@ public void testDataStreams() throws Exception { indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, numDocs); } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(client(), dataStreamName, 1); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs); }); @@ -338,7 +338,7 @@ public void testDataStreams() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(leaderClient, dataStreamName, 1, 2); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -348,7 +348,7 @@ public void testDataStreams() throws Exception { } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 1, 2); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs + 1); }); @@ -357,13 +357,7 @@ public void testDataStreams() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream( - leaderClient, - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); + verifyDataStream(leaderClient, dataStreamName, 1, 2, 3); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -373,28 +367,14 @@ public void testDataStreams() throws Exception { } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 3)); - verifyDataStream( - client(), - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); + verifyDataStream(client(), dataStreamName, 1, 2, 3); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs + 2); }); } finally { - cleanUpFollower( - List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2), backingIndexName(dataStreamName, 3)), - List.of(dataStreamName), - List.of(autoFollowPatternName) - ); - cleanUpLeader( - List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2), backingIndexName(dataStreamName, 3)), - List.of(dataStreamName), - List.of() - ); + cleanUpFollower(List.of(), List.of(dataStreamName), List.of(autoFollowPatternName)); + cleanUpLeader(List.of(), List.of(dataStreamName), List.of()); } } @@ -421,19 +401,13 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, numDocs); } - logger.info( - "--> checking {} with index {} has been auto followed to {} with backing index {}", - dataStreamName, - backingIndexName(dataStreamName, 1), - dataStreamNameFollower, - backingIndexName(dataStreamNameFollower, 1) - ); + logger.info("--> checking {} has been auto followed to {}", dataStreamName, dataStreamNameFollower); assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamNameFollower, backingIndexName(dataStreamNameFollower, 1)); + verifyDataStream(client(), dataStreamNameFollower, 1); ensureYellow(dataStreamNameFollower); verifyDocuments(client(), dataStreamNameFollower, numDocs); }); @@ -443,7 +417,7 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(leaderClient, dataStreamName, 1, 2); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -453,12 +427,7 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2)); - verifyDataStream( - client(), - dataStreamNameFollower, - backingIndexName(dataStreamNameFollower, 1), - backingIndexName(dataStreamNameFollower, 2) - ); + verifyDataStream(client(), dataStreamNameFollower, 1, 2); ensureYellow(dataStreamNameFollower); verifyDocuments(client(), dataStreamNameFollower, numDocs + 1); }); @@ -468,13 +437,7 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream( - leaderClient, - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); + verifyDataStream(leaderClient, dataStreamName, 1, 2, 3); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -484,32 +447,14 @@ public void testDataStreamsRenameFollowDataStream() throws Exception { } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 3)); - verifyDataStream( - client(), - dataStreamNameFollower, - backingIndexName(dataStreamNameFollower, 1), - backingIndexName(dataStreamNameFollower, 2), - backingIndexName(dataStreamNameFollower, 3) - ); + verifyDataStream(client(), dataStreamNameFollower, 1, 2, 3); ensureYellow(dataStreamNameFollower); verifyDocuments(client(), dataStreamNameFollower, numDocs + 2); }); } finally { - cleanUpFollower( - List.of( - backingIndexName(dataStreamNameFollower, 1), - backingIndexName(dataStreamNameFollower, 2), - backingIndexName(dataStreamNameFollower, 3) - ), - List.of(dataStreamNameFollower), - List.of(autoFollowPatternName) - ); - cleanUpLeader( - List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2), backingIndexName(dataStreamName, 3)), - List.of(dataStreamName), - List.of() - ); + cleanUpFollower(List.of(), List.of(dataStreamNameFollower), List.of(autoFollowPatternName)); + cleanUpLeader(List.of(), List.of(dataStreamName), List.of()); } } @@ -541,7 +486,7 @@ public void testDataStreams_autoFollowAfterDataStreamCreated() throws Exception indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, initialNumDocs); } @@ -549,10 +494,11 @@ public void testDataStreams_autoFollowAfterDataStreamCreated() throws Exception createAutoFollowPattern(client(), autoFollowPatternName, dataStreamName + "*", "leader_cluster", null); // Rollover and ensure only second backing index is replicated: + final List backingIndexNames; try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + backingIndexNames = verifyDataStream(leaderClient, dataStreamName, 1, 2); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -562,31 +508,23 @@ public void testDataStreams_autoFollowAfterDataStreamCreated() throws Exception } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 2); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, 1); }); // Explicitly follow the first backing index and check that the data stream in follow cluster is updated correctly: - followIndex(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 1)); + followIndex(backingIndexNames.getFirst(), backingIndexNames.getFirst()); assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 1, 2); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, initialNumDocs + 1); }); } finally { - cleanUpFollower( - List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)), - List.of(dataStreamName), - List.of(autoFollowPatternName) - ); - cleanUpLeader( - List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)), - List.of(dataStreamName), - List.of() - ); + cleanUpFollower(List.of(), List.of(dataStreamName), List.of(autoFollowPatternName)); + cleanUpLeader(List.of(), List.of(dataStreamName), List.of()); } } @@ -614,7 +552,7 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, initialNumDocs); } @@ -622,10 +560,11 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { createAutoFollowPattern(client(), autoFollowPatternName, dataStreamName + "*", "leader_cluster", null); // Rollover and ensure only second backing index is replicated: + final List backingIndexNames; try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(leaderClient, dataStreamName, 1, 2); Request indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -635,22 +574,17 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { assertOK(leaderClient.performRequest(rolloverRequest)); assertOK(leaderClient.performRequest(indexRequest)); - verifyDataStream( - leaderClient, - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); + backingIndexNames = verifyDataStream(leaderClient, dataStreamName, 1, 2, 3); } - assertBusy(() -> assertThat(indexExists(backingIndexName(dataStreamName, 2)), is(true))); - assertBusy(() -> assertThat(indexExists(backingIndexName(dataStreamName, 3)), is(true))); + awaitIndexExists(backingIndexNames.get(1)); + awaitIndexExists(backingIndexNames.get(2)); // Replace a backing index in the follower data stream with one that has a prefix (simulating a shrink) - String shrunkIndexName = SHRUNKEN_INDEX_PREFIX + DataStream.getDefaultBackingIndexName(dataStreamName, 2); + final String secondBackingIndex = backingIndexNames.get(1); + String shrunkIndexName = SHRUNKEN_INDEX_PREFIX + secondBackingIndex; Request indexRequest = new Request("POST", "/" + shrunkIndexName + "/_doc"); indexRequest.addParameter("refresh", "true"); indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); @@ -666,7 +600,7 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { + dataStreamName + "\",\n" + " \"index\": \"" - + DataStream.getDefaultBackingIndexName(dataStreamName, 2) + + secondBackingIndex + "\"\n" + " }\n" + " },\n" @@ -689,14 +623,7 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream( - leaderClient, - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3), - backingIndexName(dataStreamName, 4) - ); + verifyDataStream(leaderClient, dataStreamName, 1, 2, 3, 4); indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -705,14 +632,11 @@ public void testDataStreamsBackingIndicesOrdering() throws Exception { } assertBusy(() -> { - Request request = new Request("GET", "/_data_stream/" + dataStreamName); - Map response = toMap(client().performRequest(request)); - List retrievedDataStreams = (List) response.get("data_streams"); - List actualBackingIndexItems = (List) ((Map) retrievedDataStreams.get(0)).get("indices"); + List actualBackingIndexItems = getDataStreamBackingIndexNames(dataStreamName); assertThat(actualBackingIndexItems.size(), is(3)); - Map writeIndexMap = (Map) actualBackingIndexItems.get(2); - assertThat(writeIndexMap.get("index_name"), not(shrunkIndexName)); - assertThat(writeIndexMap.get("index_name"), is(backingIndexName(dataStreamName, 4))); + String writeIndex = actualBackingIndexItems.get(2); + assertThat(writeIndex, not(shrunkIndexName)); + assertThat(writeIndex, DataStreamTestHelper.backingIndexEqualTo(dataStreamName, 4)); assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 3)); }); } @@ -740,12 +664,12 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, numDocs); } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(client(), dataStreamName, 1); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs); }); @@ -754,7 +678,7 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { try (var leaderClient = buildLeaderClient()) { var rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(leaderClient, dataStreamName, 1, 2); var indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -764,7 +688,7 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 1, 2); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs + 1); }); @@ -779,12 +703,13 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { "data stream [" + dataStreamName + "] cannot be rolled over, " + "because it is a replicated data stream" ) ); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + backingIndexNames = verifyDataStream(client(), dataStreamName, 1, 2); // Unfollow .ds-logs-tomcat-prod-000001 - pauseFollow(backingIndexName(dataStreamName, 1)); - closeIndex(backingIndexName(dataStreamName, 1)); - unfollow(backingIndexName(dataStreamName, 1)); + final String writeIndex = backingIndexNames.getFirst(); + pauseFollow(writeIndex); + closeIndex(writeIndex); + unfollow(writeIndex); // Try again var rolloverRequest2 = new Request("POST", "/" + dataStreamName + "/_rollover"); @@ -795,7 +720,7 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { "data stream [" + dataStreamName + "] cannot be rolled over, " + "because it is a replicated data stream" ) ); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 1, 2); // Promote local data stream var promoteRequest = new Request("POST", "/_data_stream/_promote/" + dataStreamName); @@ -804,13 +729,7 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { // Try again and now the rollover should be successful because local data stream is now : var rolloverRequest3 = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(client().performRequest(rolloverRequest3)); - backingIndexNames = verifyDataStream( - client(), - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); + backingIndexNames = verifyDataStream(client(), dataStreamName, 1, 2, 3); // TODO: verify that following a backing index for logs-tomcat-prod data stream in remote cluster fails, // because local data stream isn't a replicated data stream anymore. @@ -826,17 +745,8 @@ public void testRolloverDataStreamInFollowClusterForbidden() throws Exception { } } finally { - if (backingIndexNames == null) { - // we failed to compute the actual backing index names in the test because we failed earlier on, guessing them on a - // best-effort basis - backingIndexNames = List.of( - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2), - backingIndexName(dataStreamName, 3) - ); - } - cleanUpFollower(backingIndexNames, List.of(dataStreamName), List.of(autoFollowPatternName)); - cleanUpLeader(backingIndexNames.subList(0, 2), List.of(dataStreamName), List.of()); + cleanUpFollower(List.of(), List.of(dataStreamName), List.of(autoFollowPatternName)); + cleanUpLeader(List.of(), List.of(dataStreamName), List.of()); } } @@ -1000,12 +910,12 @@ public void testDataStreamsBiDirectionalReplication() throws Exception { indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, leaderDataStreamName, backingIndexName(leaderDataStreamName, 1)); + verifyDataStream(leaderClient, leaderDataStreamName, 1); verifyDocuments(leaderClient, leaderDataStreamName, numDocs); } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndicesInFollowCluster + 1)); - verifyDataStream(client(), leaderDataStreamName, backingIndexName(leaderDataStreamName, 1)); + verifyDataStream(client(), leaderDataStreamName, 1); ensureYellow(leaderDataStreamName); verifyDocuments(client(), leaderDataStreamName, numDocs); }); @@ -1044,7 +954,7 @@ public void testDataStreamsBiDirectionalReplication() throws Exception { getNumberOfSuccessfulFollowedIndices(leaderClient), equalTo(initialNumberOfSuccessfulFollowedIndicesInLeaderCluster + 1) ); - verifyDataStream(leaderClient, followerDataStreamName, backingIndexName(followerDataStreamName, 1)); + verifyDataStream(leaderClient, followerDataStreamName, 1); ensureYellow(followerDataStreamName); verifyDocuments(leaderClient, followerDataStreamName, numDocs); }); @@ -1096,16 +1006,8 @@ public void testDataStreamsBiDirectionalReplication() throws Exception { verifyDocuments(leaderClient, aliasName, (numDocs + moreDocs) * 2); } } finally { - cleanUpFollower( - List.of(backingIndexName(followerDataStreamName, 1), backingIndexName(leaderDataStreamName, 1)), - List.of(followerDataStreamName, leaderDataStreamName), - List.of("id1") - ); - cleanUpLeader( - List.of(backingIndexName(leaderDataStreamName, 1), backingIndexName(followerDataStreamName, 1)), - List.of(leaderDataStreamName, followerDataStreamName), - List.of("id2") - ); + cleanUpFollower(List.of(), List.of(followerDataStreamName, leaderDataStreamName), List.of("id1")); + cleanUpLeader(List.of(), List.of(leaderDataStreamName, followerDataStreamName), List.of("id2")); } } @@ -1216,7 +1118,6 @@ private void testDataStreamPromotionWarnings(Boolean createFollowerTemplate) thr final String autoFollowPatternName = getTestName().toLowerCase(Locale.ROOT); int initialNumberOfSuccessfulFollowedIndices = getNumberOfSuccessfulFollowedIndices(); - List backingIndexNames = null; try { // Create index template Request putComposableIndexTemplateRequest = new Request("POST", "/_index_template/" + getTestName().toLowerCase(Locale.ROOT)); @@ -1239,12 +1140,12 @@ private void testDataStreamPromotionWarnings(Boolean createFollowerTemplate) thr indexRequest.setJsonEntity("{\"@timestamp\": \"" + DATE_FORMAT.format(new Date()) + "\",\"message\":\"abc\"}"); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, numDocs); } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 1)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(client(), dataStreamName, 1); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs); }); @@ -1253,7 +1154,7 @@ private void testDataStreamPromotionWarnings(Boolean createFollowerTemplate) thr try (var leaderClient = buildLeaderClient()) { var rolloverRequest = new Request("POST", "/" + dataStreamName + "/_rollover"); assertOK(leaderClient.performRequest(rolloverRequest)); - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(leaderClient, dataStreamName, 1, 2); var indexRequest = new Request("POST", "/" + dataStreamName + "/_doc"); indexRequest.addParameter("refresh", "true"); @@ -1263,33 +1164,22 @@ private void testDataStreamPromotionWarnings(Boolean createFollowerTemplate) thr } assertBusy(() -> { assertThat(getNumberOfSuccessfulFollowedIndices(), equalTo(initialNumberOfSuccessfulFollowedIndices + 2)); - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); + verifyDataStream(client(), dataStreamName, 1, 2); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs + 1); }); - backingIndexNames = verifyDataStream( - client(), - dataStreamName, - backingIndexName(dataStreamName, 1), - backingIndexName(dataStreamName, 2) - ); + verifyDataStream(client(), dataStreamName, 1, 2); // Promote local data stream var promoteRequest = new Request("POST", "/_data_stream/_promote/" + dataStreamName); Response response = client().performRequest(promoteRequest); assertOK(response); } finally { - if (backingIndexNames == null) { - // we failed to compute the actual backing index names in the test because we failed earlier on, guessing them on a - // best-effort basis - backingIndexNames = List.of(backingIndexName(dataStreamName, 1), backingIndexName(dataStreamName, 2)); - } - // These cleanup methods are copied from the finally block of other Data Stream tests in this class however // they may no longer be required but have been included for completeness - cleanUpFollower(backingIndexNames, List.of(dataStreamName), List.of(autoFollowPatternName)); - cleanUpLeader(backingIndexNames.subList(0, 1), List.of(dataStreamName), List.of()); + cleanUpFollower(List.of(), List.of(dataStreamName), List.of(autoFollowPatternName)); + cleanUpLeader(List.of(), List.of(dataStreamName), List.of()); Request deleteTemplateRequest = new Request("DELETE", "/_index_template/" + getTestName().toLowerCase(Locale.ROOT)); if (createFollowerTemplate) { assertOK(client().performRequest(deleteTemplateRequest)); diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java index 930f546e4f681..76dfdccb60106 100644 --- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/DowngradeLicenseFollowIndexIT.java @@ -127,7 +127,7 @@ public void testDowngradeRemoteClusterToBasic() throws Exception { // Index2 was created in leader cluster after the downgrade and therefor the auto follow coordinator in // follow cluster should not pick that index up: - assertThat(indexExists(index2), is(false)); + assertThat(indexExists(adminClient(), index2), is(false)); // parse the logs and ensure that the auto-coordinator skipped coordination on the leader cluster assertBusy(() -> { diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 1f89d316a4e3d..46c4d64d5b9ad 100644 --- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -15,7 +15,6 @@ import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; -import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.settings.Settings; @@ -231,7 +230,7 @@ public void testFollowDataStreamFails() throws Exception { try (RestClient leaderClient = buildLeaderClient()) { Request request = new Request("PUT", "/_data_stream/" + dataStreamName); assertOK(leaderClient.performRequest(request)); - verifyDataStream(leaderClient, dataStreamName, DataStream.getDefaultBackingIndexName("logs-syslog-prod", 1)); + verifyDataStream(leaderClient, dataStreamName, 1); } ResponseException failure = expectThrows(ResponseException.class, () -> followIndex(dataStreamName, dataStreamName)); diff --git a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java index 0adf0b31b4ebd..13e31785f430b 100644 --- a/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java +++ b/x-pack/plugin/ccr/src/javaRestTest/java/org/elasticsearch/xpack/ccr/FollowIndexSecurityIT.java @@ -161,7 +161,7 @@ public void testFollowIndex() throws Exception { e = expectThrows(ResponseException.class, () -> followIndex(client(), "leader_cluster", unallowedIndex, unallowedIndex)); assertThat(e.getMessage(), containsString("action [indices:admin/xpack/ccr/put_follow] is unauthorized for user [test_ccr]")); // Verify that the follow index has not been created and no node tasks are running - assertThat(indexExists(unallowedIndex), is(false)); + assertThat(indexExists(adminClient(), unallowedIndex), is(false)); assertBusy(() -> assertThat(getCcrNodeTasks(), empty())); // User does have manage_follow_index index privilege on 'allowed' index, @@ -176,7 +176,7 @@ public void testFollowIndex() throws Exception { ) ); // Verify that the follow index has not been created and no node tasks are running - assertThat(indexExists(unallowedIndex), is(false)); + assertThat(indexExists(adminClient(), unallowedIndex), is(false)); assertBusy(() -> assertThat(getCcrNodeTasks(), empty())); followIndex(adminClient(), "leader_cluster", unallowedIndex, unallowedIndex); @@ -242,7 +242,7 @@ public void testAutoFollowPatterns() throws Exception { try { assertBusy(() -> ensureYellow(allowedIndex), 30, TimeUnit.SECONDS); assertBusy(() -> verifyDocuments(allowedIndex, 5, "*:*"), 30, TimeUnit.SECONDS); - assertThat(indexExists(disallowedIndex), is(false)); + assertThat(indexExists(adminClient(), disallowedIndex), is(false)); withMonitoring(logger, () -> { assertBusy(() -> verifyCcrMonitoring(allowedIndex, allowedIndex), 120L, TimeUnit.SECONDS); assertBusy(AbstractCCRRestTestCase::verifyAutoFollowMonitoring, 120L, TimeUnit.SECONDS); @@ -350,11 +350,11 @@ public void testUnPromoteAndFollowDataStream() throws Exception { """, dateFormat.format(new Date()))); assertOK(leaderClient.performRequest(indexRequest)); } - verifyDataStream(leaderClient, dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(leaderClient, dataStreamName, 1); verifyDocuments(leaderClient, dataStreamName, numDocs); } assertBusy(() -> { - verifyDataStream(client(), dataStreamName, backingIndexName(dataStreamName, 1)); + verifyDataStream(client(), dataStreamName, 1); ensureYellow(dataStreamName); verifyDocuments(client(), dataStreamName, numDocs); }); @@ -366,9 +366,10 @@ public void testUnPromoteAndFollowDataStream() throws Exception { assertOK(client().performRequest(new Request("POST", "/" + dataStreamName + "/_rollover"))); // Unfollow .ds-logs-eu-monitor1-000001, // which is now possible because this index can now be closed as it is no longer the write index. - pauseFollow(backingIndexName(dataStreamName, 1)); - closeIndex(backingIndexName(dataStreamName, 1)); - unfollow(backingIndexName(dataStreamName, 1)); + final String writeIndex = getDataStreamBackingIndexNames(dataStreamName).getFirst(); + pauseFollow(writeIndex); + closeIndex(writeIndex); + unfollow(writeIndex); } }