Skip to content

Fix Flaky Semantic Search CCS Integration Tests #135629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 29, 2025
Merged

Fix Flaky Semantic Search CCS Integration Tests #135629

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
060a3a8
Wait until remote clusters are connected
Mikep86 Sep 29, 2025
9677a19
Wait for index green status
Mikep86 Sep 29, 2025
652e89e
Unmute tests
Mikep86 Sep 29, 2025
0b9a1aa
Rename variable
Mikep86 Sep 29, 2025
f07145f
Simplified waitUntilRemoteClusterConnected
Mikep86 Sep 29, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions muted-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -597,18 +597,12 @@ tests:
- class: org.elasticsearch.action.admin.cluster.stats.SearchUsageStatsTests
method: testToXContent
issue: https://github.com/elastic/elasticsearch/issues/135558
- class: org.elasticsearch.search.ccs.SparseVectorQueryBuilderCrossClusterSearchIT
method: testSparseVectorQueryWithCcsMinimizeRoundTripsFalse
issue: https://github.com/elastic/elasticsearch/issues/135559
- class: org.elasticsearch.cluster.routing.allocation.decider.RestoreInProgressAllocationDeciderTests
method: testCanAllocatePrimaryExistingInRestoreInProgress
issue: https://github.com/elastic/elasticsearch/issues/135566
- class: org.elasticsearch.xpack.esql.inference.textembedding.TextEmbeddingOperatorTests
method: testSimpleCircuitBreaking
issue: https://github.com/elastic/elasticsearch/issues/135569
- class: org.elasticsearch.search.ccs.KnnVectorQueryBuilderCrossClusterSearchIT
method: testKnnQueryWithCcsMinimizeRoundTripsFalse
issue: https://github.com/elastic/elasticsearch/issues/135573
- class: org.elasticsearch.multiproject.test.XpackWithMultipleProjectsClientYamlTestSuiteIT
method: test {yaml=esql/60_usage/Basic ESQL usage output (telemetry) snapshot version}
issue: https://github.com/elastic/elasticsearch/issues/135579
Expand Down
36 changes: 33 additions & 3 deletions ...terTest/java/org/elasticsearch/search/ccs/AbstractSemanticCrossClusterSearchTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
package org.elasticsearch.search.ccs;

import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.admin.cluster.remote.RemoteInfoRequest;
import org.elasticsearch.action.admin.cluster.remote.RemoteInfoResponse;
import org.elasticsearch.action.admin.cluster.remote.TransportRemoteInfoAction;
import org.elasticsearch.action.search.OpenPointInTimeRequest;
import org.elasticsearch.action.search.OpenPointInTimeResponse;
import org.elasticsearch.action.search.SearchRequest;
Expand Down Expand Up @@ -35,6 +38,7 @@
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.AbstractMultiClustersTestCase;
import org.elasticsearch.transport.RemoteConnectionInfo;
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentFactory;
import org.elasticsearch.xcontent.XContentType;
Expand All @@ -50,6 +54,7 @@
import org.elasticsearch.xpack.ml.action.TransportCoordinatedInferenceAction;

import java.io.IOException;
import java.time.Duration;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
Expand Down Expand Up @@ -92,14 +97,16 @@ protected Collection<Class<? extends Plugin>> nodePlugins(String clusterAlias) {
return List.of(LocalStateInferencePlugin.class, TestInferenceServicePlugin.class, FakeMlPlugin.class);
}

protected void setupTwoClusters(TestIndexInfo localIndexInfo, TestIndexInfo remoteIndexInfo) throws IOException {
protected void setupTwoClusters(TestIndexInfo localIndexInfo, TestIndexInfo remoteIndexInfo) throws Exception {
setupCluster(LOCAL_CLUSTER, localIndexInfo);
setupCluster(REMOTE_CLUSTER, remoteIndexInfo);
waitUntilRemoteClusterConnected(REMOTE_CLUSTER);
}

protected void setupCluster(String clusterAlias, TestIndexInfo indexInfo) throws IOException {
final Client client = client(clusterAlias);
final String indexName = indexInfo.name();
final int dataNodeCount = cluster(clusterAlias).numDataNodes();

for (var entry : indexInfo.inferenceEndpoints().entrySet()) {
String inferenceId = entry.getKey();
Expand All @@ -117,13 +124,13 @@ protected void setupCluster(String clusterAlias, TestIndexInfo indexInfo) throws
createInferenceEndpoint(client, minimalServiceSettings.taskType(), inferenceId, serviceSettings);
}

Settings indexSettings = indexSettings(randomIntBetween(2, 5), randomIntBetween(0, 1)).build();
Settings indexSettings = indexSettings(randomIntBetween(1, dataNodeCount), 0).build();
assertAcked(client.admin().indices().prepareCreate(indexName).setSettings(indexSettings).setMapping(indexInfo.mappings()));
assertFalse(
client.admin()
.cluster()
.prepareHealth(TEST_REQUEST_TIMEOUT, indexName)
.setWaitForYellowStatus()
.setWaitForGreenStatus()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it should never be yellow if there are 0 replicas, only green, so maybe you could keep the wait for yellow status? But I'm OK leaving this as is as long as we don't think there's a testing hole

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, but if it should never be yellow then this check should be fine and IMO at the very least it clarifies the state we want the clusters to be in before starting tests.

.setTimeout(TimeValue.timeValueSeconds(10))
.get()
.isTimedOut()
Expand All @@ -140,6 +147,29 @@ protected void setupCluster(String clusterAlias, TestIndexInfo indexInfo) throws
assertThat(refreshResponse.getStatus(), is(RestStatus.OK));
}

protected void waitUntilRemoteClusterConnected(String clusterAlias) throws InterruptedException {
RemoteInfoRequest request = new RemoteInfoRequest();
boolean connected;
int attempts = 0;
int delay = 0;
do {
if (delay > 0) {
// Delay between retries so that we don't use up all our attempts in a tight loop
Thread.sleep(Duration.ofSeconds(delay));
}
RemoteInfoResponse response = client().execute(TransportRemoteInfoAction.TYPE, request).actionGet(TEST_REQUEST_TIMEOUT);
connected = response.getInfos()
.stream()
.filter(i -> i.getClusterAlias().equals(clusterAlias))
.anyMatch(RemoteConnectionInfo::isConnected);
delay += 5;
} while (connected == false && attempts++ < 5);

if (connected == false) {
throw new AssertionError("Cannot connect to remote cluster [" + clusterAlias + "]");
}
}

protected BytesReference openPointInTime(String[] indices, TimeValue keepAlive) {
OpenPointInTimeRequest request = new OpenPointInTimeRequest(indices).keepAlive(keepAlive);
final OpenPointInTimeResponse response = client().execute(TransportOpenPointInTimeAction.TYPE, request).actionGet();
Expand Down
3 changes: 1 addition & 2 deletions ...lClusterTest/java/org/elasticsearch/search/ccs/MatchQueryBuilderCrossClusterSearchIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.junit.Before;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Set;
Expand Down Expand Up @@ -151,7 +150,7 @@ public void testMatchQueryWithCcsMinimizeRoundTripsFalse() throws Exception {
);
}

private void configureClusters() throws IOException {
private void configureClusters() throws Exception {
final String commonInferenceId = "common-inference-id";
final String localInferenceId = "local-inference-id";
final String remoteInferenceId = "remote-inference-id";
Expand Down