Merge remote-tracking branch 'jverwolf/new-tv-logic' into new-tv-logic

Author: jdconrad

server/src/main/java/org/elasticsearch/TransportVersionSet.java

@@ -12,6 +12,9 @@
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -48,10 +51,23 @@ public class TransportVersionSet {
         PARSER.declareIntArray(ConstructingObjectParser.constructorArg(), VERSIONS);
     }
 
-    private static final Map<String, TransportVersionSet> TRANSPORT_VERSION_SETS;
+    private static final Map<String, TransportVersionSet> TRANSPORT_VERSION_SETS = loadTransportVersionSets();
+    private static final List<TransportVersion> TRANSPORT_VERSIONS = collectTranportVersions();
 
-    static {
+    private static final String MANIFEST_LOCATION = "META-INF/transport-versions-files-manifest.txt";
+    private static final String METADATA_LOCATION = "org/elasticsearch/transport/";
+    private static final String LATEST_SUFFIX = "-LATEST.json";
+
+    private static Map<String, TransportVersionSet> loadTransportVersionSets() {
         Map<String, TransportVersionSet> transportVersionSets = new HashMap<>();
+
+        String latestResourceLocation = Version.CURRENT.major + "-" + Version.CURRENT.minor + LATEST_SUFFIX;
+        try (InputStream inputStream = TransportVersionSet.class.getResourceAsStream(latestResourceLocation)) {
+
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
+
         // TODO: load
         // TODO: TEST ONLY
         transportVersionSets.put("ml-inference-custom-service-embedding-type", new TransportVersionSet(
@@ -79,7 +95,18 @@ public class TransportVersionSet {
                 List.of(new TransportVersion(9114000))
         ));
         // TODO: END TEST ONLY
-        TRANSPORT_VERSION_SETS = Collections.unmodifiableMap(transportVersionSets);
+        return Collections.unmodifiableMap(transportVersionSets);
+    }
+
+    private static List<TransportVersion> collectTranportVersions() {
+        List<TransportVersion> tranportVersions = new ArrayList<>();
+        for (TransportVersionSet transportVersionSet : TRANSPORT_VERSION_SETS.values()) {
+            for (TransportVersion transportVersion : transportVersionSet.versions) {
+                tranportVersions.add(transportVersion);
+            }
+        }
+        tranportVersions.sort(TransportVersion::compareTo);
+        return tranportVersions;
     }
 
     public static TransportVersionSet get(String name) {
@@ -126,4 +153,75 @@ public boolean isCompatible(TransportVersion version) {
     public String toString() {
         return "TransportVersionSet{" + "name='" + name + '\'' + ", versions=" + versions + '}';
     }
+
+//    private static TransportVersionSet fromJSONInputStream(InputStream inputStream) throws IOException {
+//        XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, inputStream);
+//        return TransportVersionSet.fromXContent(parser);
+//    }
+//
+//
+//
+//
+//        static {
+//            var locallyDefinedVersionSets = loadTransportVersionSets();
+//            var extendedVersionSets = new ArrayList<TransportVersionSet>(); // TODO add SPI for serverless
+//
+//            var allVersionSets = Stream.concat(
+//                locallyDefinedVersionSets.stream(),
+//                extendedVersionSets.stream()
+//            ).toList();
+//
+//            ALL_VERSIONS_SORTED = allVersionSets.stream().flatMap(tvSet -> tvSet.versions.stream()).sorted().toList();
+//
+//            ALL_VERSIONS_MAP = ALL_VERSIONS_SORTED.stream()
+//                .collect(Collectors.toUnmodifiableMap(TransportVersion::id, Function.identity()));
+//
+//            ALL_VERSIONS_SETS_MAP = allVersionSets.stream()
+//                .collect(Collectors.toUnmodifiableMap(TransportVersionSet::name, Function.identity()));
+//
+//            LATEST = getLatestTVSet();
+//        }
+//
+//        private static TransportVersionSet getLatestTVSet() {
+//
+//            var major = Version.CURRENT.major;
+//            var minor = Version.CURRENT.minor;
+//            var fileName =  major + "." + minor + "-" + LATEST_SUFFIX;
+//
+//            var path = METADATA_LOCATION + fileName;
+//            return loadTransportVersionSet(path); // todo, use a different format?
+//        }
+//
+//        private static List<TransportVersionSet> loadTransportVersionSets() {
+//            var transportVersionSets = new ArrayList<TransportVersionSet>();
+//            for (var fileName : loadTransportVersionSetFileNames()) {
+//                var path = METADATA_LOCATION + fileName;
+//                transportVersionSets.add(loadTransportVersionSet(path));
+//            }
+//            return transportVersionSets;
+//        }
+//
+//        // TODO will getResourceAsStream work for IntelliJ?
+//        private static List<String> loadTransportVersionSetFileNames() {
+//            try (InputStream transportVersionManifest = TransportVersionSet.class.getResourceAsStream(MANIFEST_LOCATION)) {
+//                BufferedReader reader = new BufferedReader(new InputStreamReader(transportVersionManifest));
+//                return reader.lines().filter(line -> line.isBlank() == false).toList();
+//            } catch (IOException e) {
+//                throw new RuntimeException("Transport version metadata manifest file not found at " + MANIFEST_LOCATION, e);
+//            }
+//        }
+//
+//        private static TransportVersionSet loadTransportVersionSet(String path) {
+//            try (var fileStream = VersionsHolder.class.getResourceAsStream(path)) {
+//                if (fileStream != null) {
+//                    return TransportVersionSet.fromJSONInputStream(fileStream);
+//                } else {
+//                    throw new RuntimeException("Input stream is null");
+//                }
+//            } catch (IOException e) {
+//                throw new RuntimeException("Failed to load TransportVersionSet at path: " + path +
+//                    " specified in the manifest file: " + MANIFEST_LOCATION, e);
+//            }
+//        }
+//    }
 }

server/src/main/java/org/elasticsearch/Version.java

@@ -218,7 +218,10 @@ public class Version implements VersionId<Version>, ToXContentFragment {
     public static final Version V_9_0_4 = new Version(9_00_04_99);
     public static final Version V_9_1_0 = new Version(9_01_00_99);
     public static final Version V_9_2_0 = new Version(9_02_00_99);
-    public static final Version CURRENT = V_9_2_0;
+    public static final Version CURRENT = V_9_2_0; // here
+    /*
+    - need to associate current with the appropriate TV
+     */
 
     private static final NavigableMap<Integer, Version> VERSION_IDS;
     private static final Map<String, Version> VERSION_STRINGS;

server/src/main/resources/org/elasticsearch/transport/9.2-LATEST.json

@@ -1,4 +1,6 @@
 {
   "name": "ml-inference-custom-service-embedding-type",
-  "latest": 9118000
+  "ids": [
+    9118000
+  ]
 }