[ML] Adding retries to model download tests

muted-tests.yml

@@ -73,15 +73,6 @@ tests:
 - class: org.elasticsearch.xpack.apmdata.APMYamlTestSuiteIT
   method: test {yaml=/10_apm/Test template reinstallation}
   issue: https://github.com/elastic/elasticsearch/issues/116445
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/40_semantic_text_query/Query a field that uses the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/117027
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/30_semantic_text_inference/Calculates embeddings using the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/117349
-- class: org.elasticsearch.xpack.inference.InferenceRestIT
-  method: test {p0=inference/30_semantic_text_inference_bwc/Calculates embeddings using the default ELSER 2 endpoint}
-  issue: https://github.com/elastic/elasticsearch/issues/117349
 - class: org.elasticsearch.xpack.test.rest.XPackRestIT
   method: test {p0=transform/transforms_reset/Test reset running transform}
   issue: https://github.com/elastic/elasticsearch/issues/117473

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/RetryRule.java → test/framework/src/main/java/org/elasticsearch/test/RetryRule.java

@@ -1,11 +1,13 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
- * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.xpack.inference;
+package org.elasticsearch.test;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -25,7 +27,6 @@
  * <br/>
  * <code>public RetryRule retry = new RetryRule(3, TimeValue.timeValueSeconds(1));</code>
  * <br/>
- * See {@link InferenceGetServicesIT#retry} for an example.
  */
 public class RetryRule implements TestRule {
     private static final Logger logger = LogManager.getLogger(RetryRule.class);

x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/BaseMockEISAuthServerTest.java

@@ -13,6 +13,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.ThreadContext;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.RetryRule;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.ESRestTestCase;
@@ -54,7 +55,7 @@ public class BaseMockEISAuthServerTest extends ESRestTestCase {
 
     // The reason we're retrying is there's a race condition between the node retrieving the
     // authorization response and running the test. Retrieving the authorization should be very fast since
-    // we're hosting a local mock server but it's possible it could respond slower. So in the even of a test failure
+    // we're hosting a local mock server but it's possible it could respond slower. So in the event of a test failure
     // we'll automatically retry after waiting a second.
     // Note: @Rule is executed for each test
     @Rule

x-pack/plugin/inference/src/yamlRestTest/java/org/elasticsearch/xpack/inference/InferenceRestIT.java

@@ -12,13 +12,16 @@
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.test.RetryRule;
 import org.elasticsearch.test.cluster.ElasticsearchCluster;
 import org.elasticsearch.test.cluster.FeatureFlag;
 import org.elasticsearch.test.cluster.local.distribution.DistributionType;
 import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
 import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
 import org.junit.After;
 import org.junit.ClassRule;
+import org.junit.Rule;
 
 import java.io.IOException;
 import java.util.List;
@@ -37,6 +40,14 @@ public class InferenceRestIT extends ESClientYamlSuiteTestCase {
         .distribution(DistributionType.DEFAULT)
         .build();
 
+    /**
+     * This will retry a failed test up to 3 times with a 1 second wait between retries. We've observed transient network
+     * failures when trying to download the elser model during the test. These network failure cause the tests to fail intermittently.
+     * The proper way to fix this would be to add retry logic to the download code but that is a larger fix.
+     */
+    @Rule
+    public RetryRule retryRule = new RetryRule(3, TimeValue.timeValueSeconds(1));
+
     public InferenceRestIT(final ClientYamlTestCandidate testCandidate) {
         super(testCandidate);
     }