From 52f36f2cfd2d07379c88e73f8e5e1d02b9bff9f5 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 2 Oct 2025 10:01:32 -0700 Subject: [PATCH 1/2] Skip spec tests when test cluster broken --- .../xpack/esql/qa/single_node/EsqlSpecIT.java | 1 + .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 38 +++++++++---------- 2 files changed, 20 insertions(+), 19 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index a21c416ba8741..dacc34167955a 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -53,6 +53,7 @@ protected boolean supportsSourceFieldMapping() { @Before public void configureChunks() throws IOException { + assumeTrue("test clusters were broken", testClustersOk); boolean smallChunks = randomBoolean(); Request request = new Request("PUT", "/_cluster/settings"); XContentBuilder builder = JsonXContent.contentBuilder().startObject().startObject("persistent"); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 33ff75519348c..0296b312c74ac 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -13,7 +13,6 @@ import org.apache.lucene.tests.util.TimeUnits; import org.elasticsearch.Version; import org.elasticsearch.client.Request; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.features.NodeFeature; @@ -36,7 +35,6 @@ import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.RequestObjectBuilder; import org.elasticsearch.xpack.esql.telemetry.TookMetrics; import org.junit.After; -import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; @@ -69,7 +67,6 @@ import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createInferenceEndpoints; -import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEndpoints; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs; import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.COMPLETION; @@ -122,9 +119,11 @@ protected EsqlSpecTestCase( } private static boolean dataLoaded = false; + protected static boolean testClustersOk = true; @Before public void setup() throws IOException { + assumeTrue("test clusters were broken", testClustersOk); boolean supportsLookup = supportsIndexModeLookup(); boolean supportsSourceMapping = supportsSourceFieldMapping(); boolean supportsInferenceTestService = supportsInferenceTestService(); @@ -138,21 +137,6 @@ public void setup() throws IOException { } } - @AfterClass - public static void wipeTestData() throws IOException { - try { - dataLoaded = false; - adminClient().performRequest(new Request("DELETE", "/*")); - } catch (ResponseException e) { - // 404 here just means we had no indexes - if (e.getResponse().getStatusLine().getStatusCode() != 404) { - throw e; - } - } - - deleteInferenceEndpoints(adminClient()); - } - public boolean logResults() { return false; } @@ -162,11 +146,25 @@ public final void test() throws Throwable { shouldSkipTest(testName); doTest(); } catch (Exception e) { + ensureTestClustersAreOk(e); throw reworkException(e); } } + protected void ensureTestClustersAreOk(Exception failure) { + try { + ensureHealth(client(), "", (request) -> { + request.addParameter("wait_for_status", "yellow"); + request.addParameter("level", "shards"); + }); + } catch (Exception inner) { + testClustersOk = false; + failure.addSuppressed(inner); + } + } + protected void shouldSkipTest(String testName) throws IOException { + assumeTrue("test clusters were broken", testClustersOk); if (requiresInferenceEndpoint()) { assumeTrue("Inference test service needs to be supported", supportsInferenceTestService()); } @@ -374,7 +372,9 @@ protected boolean preserveClusterUponCompletion() { @After public void assertRequestBreakerEmptyAfterTests() throws Exception { - assertRequestBreakerEmpty(); + if (testClustersOk) { + assertRequestBreakerEmpty(); + } } public static void assertRequestBreakerEmpty() throws Exception { From 9adf2ee50ea83f63dc6be082646740dbe9e07ec5 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Thu, 2 Oct 2025 10:50:43 -0700 Subject: [PATCH 2/2] reset data in shared clusters --- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 0296b312c74ac..fed250051d84d 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -13,6 +13,7 @@ import org.apache.lucene.tests.util.TimeUnits; import org.elasticsearch.Version; import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.features.NodeFeature; @@ -35,6 +36,7 @@ import org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.RequestObjectBuilder; import org.elasticsearch.xpack.esql.telemetry.TookMetrics; import org.junit.After; +import org.junit.AfterClass; import org.junit.Before; import org.junit.Rule; @@ -67,6 +69,7 @@ import static org.elasticsearch.xpack.esql.CsvTestUtils.isEnabled; import static org.elasticsearch.xpack.esql.CsvTestUtils.loadCsvSpecValues; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.createInferenceEndpoints; +import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.deleteInferenceEndpoints; import static org.elasticsearch.xpack.esql.CsvTestsDataLoader.loadDataSetIntoEs; import static org.elasticsearch.xpack.esql.EsqlTestUtils.classpathResources; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.COMPLETION; @@ -137,6 +140,24 @@ public void setup() throws IOException { } } + @AfterClass + public static void wipeTestData() throws IOException { + if (testClustersOk == false) { + return; + } + try { + dataLoaded = false; + adminClient().performRequest(new Request("DELETE", "/*")); + } catch (ResponseException e) { + // 404 here just means we had no indexes + if (e.getResponse().getStatusLine().getStatusCode() != 404) { + throw e; + } + } + + deleteInferenceEndpoints(adminClient()); + } + public boolean logResults() { return false; }