Improve resiliency of TransportVersionUtils.getNextVersion (elastic#141285)

Author: mark-vieira

muted-tests.yml

@@ -282,12 +282,6 @@ tests:
 - class: org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT
   method: test {csv-spec:spatial.ConvertFromStringParseError}
   issue: https://github.com/elastic/elasticsearch/issues/139213
-- class: org.elasticsearch.persistent.ClusterPersistentTasksCustomMetadataTests
-  method: testMinVersionSerialization
-  issue: https://github.com/elastic/elasticsearch/issues/140804
-- class: org.elasticsearch.persistent.PersistentTasksCustomMetadataTests
-  method: testMinVersionSerialization
-  issue: https://github.com/elastic/elasticsearch/issues/140805
 - class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
   method: test {csv-spec:string.Url_encode_component tests with table reads}
   issue: https://github.com/elastic/elasticsearch/issues/140809

server/src/test/java/org/elasticsearch/persistent/BasePersistentTasksCustomMetadataTests.java

@@ -210,7 +210,7 @@ public void testMinVersionSerialization() throws IOException {
             TestPersistentTasksPlugin.TestPersistentTasksExecutor.NAME,
             new TestPersistentTasksPlugin.TestParams(
                 null,
-                getNextVersion(streamVersion),
+                getNextVersion(streamVersion, true),
                 randomBoolean() ? Optional.empty() : Optional.of("test")
             ),
             randomAssignment()

test/framework/src/main/java/org/elasticsearch/test/TransportVersionUtils.java

@@ -90,9 +90,9 @@ public static TransportVersion getNextVersion(TransportVersion version) {
     public static TransportVersion getNextVersion(TransportVersion version, boolean createIfNecessary) {
         TransportVersion higher = (isPatchVersion(version) ? RELEASED_VERSIONS : NON_PATCH_VERSIONS).higher(version);
         if (higher != null && isPatchVersion(version) && isPatchVersion(higher) == false) {
-            throw new IllegalStateException(
-                "couldn't determine next version as version [" + version + "] is  patch, and is the newest patch"
-            );
+            // The provided version is a patch, and the latest patch for that minor. We don't want to just return the next "higher" version
+            // as it might not be "newer" and may result in incorrect semantics. Instead, we should delegate to "createIfNecessary" here.
+            higher = null;
         }
 
         if (higher == null) {