diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle index c04ba9b90d5e7..7b344051bb8e7 100644 --- a/build-tools-internal/build.gradle +++ b/build-tools-internal/build.gradle @@ -220,6 +220,30 @@ gradlePlugin { id = 'elasticsearch.internal-yaml-rest-test' implementationClass = 'org.elasticsearch.gradle.internal.test.rest.InternalYamlRestTestPlugin' } + locateTransportVersionsPlugin { // TODO ask mark what the heck is going on with this missing function + id = 'elasticsearch.locate-transport-versions' + implementationClass = 'org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin' + } + generateTransportVersionManifestPlugin { // TODO ask mark what the heck is going on with this missing function + id = 'elasticsearch.generate-transport-version-manifest' + implementationClass = 'org.elasticsearch.gradle.internal.transport.GenerateTransportVersionManifestPlugin' + } + generateTransportVersionDataPlugin { + id = 'elasticsearch.generate-transport-version-data' + implementationClass = 'org.elasticsearch.gradle.internal.transport.GenerateTransportVersionDataPlugin' + } + validateTransportVersionsPlugin { + id = 'elasticsearch.validate-transport-versions' + implementationClass = 'org.elasticsearch.gradle.internal.transport.ValidateTransportVersionsPlugin' + } + aggregateTransportVersionDeclarationsPlugin { + id = 'elasticsearch.aggregate-transport-versionDeclarations-plugin' + implementationClass = 'org.elasticsearch.gradle.internal.transport.AggregateTransportVersionDeclarationsPlugin' + } + validateTransportVersionDataFilesPlugin { + id = 'elasticsearch.validate-transport-version-data-files' + implementationClass = 'org.elasticsearch.gradle.internal.transport.ValidateTransportVersionDataFilesPlugin' + } } } diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java index b6f4c99e3d0e6..6f41c5bd02e16 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/BaseInternalPluginBuildPlugin.java @@ -15,6 +15,7 @@ import org.elasticsearch.gradle.internal.info.BuildParameterExtension; import org.elasticsearch.gradle.internal.precommit.JarHellPrecommitPlugin; import org.elasticsearch.gradle.internal.test.ClusterFeaturesMetadataPlugin; +import org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin; import org.elasticsearch.gradle.plugin.PluginBuildPlugin; import org.elasticsearch.gradle.plugin.PluginPropertiesExtension; import org.elasticsearch.gradle.util.GradleUtils; @@ -36,6 +37,16 @@ public void apply(Project project) { project.getPluginManager().apply(JarHellPrecommitPlugin.class); project.getPluginManager().apply(ElasticsearchJavaPlugin.class); project.getPluginManager().apply(ClusterFeaturesMetadataPlugin.class); + // We need to wire this up + /* + Old way is to set up a config here, then depend on it in the other plugin. + The new way is called variant aware artifacts or something + Basically attach some attributes to the artifacts + Then those attributes are how we are tying this together. + Now instead of saying I want this specific config, we now say I want the artifact from this project that has these attributes + + */ + project.getPluginManager().apply(LocateTransportVersionsPlugin.class); boolean isCi = project.getRootProject().getExtensions().getByType(BuildParameterExtension.class).getCi(); // Clear default dependencies added by public PluginBuildPlugin as we add our // own project dependencies for internal builds diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java index bc2f4bd6b31b1..a84415e69ddf2 100644 --- a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/EmbeddedProviderExtension.java @@ -33,6 +33,7 @@ public EmbeddedProviderExtension(Project project, TaskProvider metaTask) { this.metaTask = metaTask; } + // here, this creates the provider manifest void impl(String implName, Project implProject) { String projectName = implProject.getName(); String capitalName = capitalize(projectName); diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsPlugin.java new file mode 100644 index 0000000000000..d9af559ef418f --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsPlugin.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +import java.util.Map; + +public class AggregateTransportVersionDeclarationsPlugin implements Plugin { + public static final String ALL_TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/all-transport-version-names.txt"; + + @Override + public void apply(Project project) { + // need to have this task depend on all the tasks with BaseInternalPluginBuildPlugin registered + // need to get the output of all those tasks as input to this task + System.out.println("Potato: AggregateTransportVersionDeclarationsPlugin"); + // First thing is to create a configuration (holder for dependency information. Configurations are how the dep graph is modeled). + var configuration = project.getConfigurations().create("aggregateTransportVersionDeclarations"); + var deps = project.getDependencies(); + + project.getRootProject() + .getSubprojects() + .stream() + .filter( + p -> p.getParent().getPath().equals(":modules") + || p.getParent().getPath().equals(":plugins") + || p.getParent().getPath().equals(":x-pack:plugin") + ) + .forEach(p -> { + deps.add( + configuration.getName(), + deps.project(Map.of("path", p.getPath(), "configuration", "locateTransportVersionsConfig")) + ); // adding a dep to the config we created + }); + + var aggregationTask = project.getTasks() + .register("aggregateTransportVersionDeclarations", AggregateTransportVersionDeclarationsTask.class, t -> { + t.dependsOn(configuration); // this task can only run after this config is resolved + t.getTransportVersionNameDeclarationsFiles().setFrom(configuration); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(ALL_TRANSPORT_VERSION_NAMES_FILE)); + }); + + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsTask.java new file mode 100644 index 0000000000000..18113b7a84029 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/AggregateTransportVersionDeclarationsTask.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.ConfigurableFileCollection; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +public abstract class AggregateTransportVersionDeclarationsTask extends DefaultTask { + + @InputFiles + public abstract ConfigurableFileCollection getTransportVersionNameDeclarationsFiles(); + + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @TaskAction + public void aggregateTransportVersionDeclarations() { + final var files = getTransportVersionNameDeclarationsFiles().getFiles(); + var allTVNames = files.stream().flatMap(file -> { + try { + return Files.lines(Path.of(file.getPath())); + } catch (IOException e) { + throw new RuntimeException("Cannot read Transport Versions name declarations file", e); + } + }).toList(); + + File file = new File(getOutputFile().get().getAsFile().getAbsolutePath()); + try (FileWriter writer = new FileWriter(file)) { + for (String tvName : allTVNames) { + writer.write(tvName + "\n"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataPlugin.java new file mode 100644 index 0000000000000..8a5779ccd4cd3 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataPlugin.java @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class GenerateTransportVersionDataPlugin implements Plugin { + + @Override + public void apply(Project project) { + project.getTasks().register("generateTransportVersionData", GenerateTransportVersionDataTask.class, t -> { + t.setDescription("Generate transport version data"); // todo update this to be more descriptive + t.setGroup("Transport Versions"); // todo + t.getDataFileDirectory().set(project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/")); + t.getTVSetName().set("FOO"); // todo + t.getReleaseVersionForTV().set("9.2"); // todo + }); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java new file mode 100644 index 0000000000000..5a6b116c07a79 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionDataTask.java @@ -0,0 +1,247 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import com.google.common.collect.Streams; + +import org.elasticsearch.gradle.Version; +import org.elasticsearch.gradle.VersionProperties; +import org.elasticsearch.gradle.internal.transport.TransportVersionUtils.TransportVersionSetData; +import org.gradle.api.DefaultTask; +import org.gradle.api.GradleException; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.Input; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.TaskAction; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.LATEST_SUFFIX; +import static org.elasticsearch.gradle.internal.transport.TransportVersionUtils.getTVSetDataFilePath; + +/** + * This task generates TransportVersionSetData data files that contain information about transport versions. These files + * are added to the server project's resource directory at `server/src/main/resources/org/elasticsearch/transport/`. + * They have the following format: + * 
+ * Filename: my-transport-version-set.json  // Must be the same as the name of the transport version set.
+ * {
+ *   "name": "my-transport-version-set", // The name of the transport version set used for reference in the code.
+ *   "ids": [
+ *     9109000,  // The transport version introduced to the main branch.
+ *     8841059   // The transport version backported to a previous release branch.
+ *   ]
+ * }
+ *
+ */ +public abstract class GenerateTransportVersionDataTask extends DefaultTask { + + /** + * Specifies the directory in which contains all TransportVersionSet data files. + * + * @return + */ + @InputDirectory + public abstract RegularFileProperty getDataFileDirectory(); + + /** + * Used to set the name of the TransportVersionSet for which a data file will be generated. + */ + @Input + public abstract Property getTVSetName(); + + /** + * Used to set the `major.minor` release version for which the specific TransportVersion ID will be generated. + * E.g.: "9.2", "8.18", etc. + */ + @Input + public abstract Property getReleaseVersionForTV(); + + @TaskAction + public void generateTransportVersionData() { + final var tvDataDir = Objects.requireNonNull(getDataFileDirectory().getAsFile().get()); + final var tvSetName = Objects.requireNonNull(getTVSetName().get()); + final var tvReleaseVersion = ReleaseVersion.fromString(Objects.requireNonNull(getReleaseVersionForTV().get())); + + // Get the latest transport version data for the local version. + final var latestTVSetData = TransportVersionUtils.getLatestTVSetData(tvDataDir, tvReleaseVersion.toString()); + + // Get the latest transport version data for the prior release version. + final var priorReleaseVersion = getPriorReleaseVersion(tvDataDir, tvReleaseVersion); + var priorLatestTVSetData = TransportVersionUtils.getLatestTVSetData(tvDataDir, priorReleaseVersion.toString()); + if (priorLatestTVSetData == null) { + throw new GradleException( + "The latest Transport Version ID for the prior release was not found at: " + + tvDataDir.getAbsolutePath() + + formatLatestTVSetFilename(priorReleaseVersion) + + " This is required." + ); + } + + // Determine if it's a (major) version bump + final var isVersionBump = latestTVSetData == null; + final var isMajorVersionBump = isVersionBump && (tvReleaseVersion.major - priorReleaseVersion.major > 0); + + // Create the new version + final var mainReleaseVersion = ReleaseVersion.of(VersionProperties.getElasticsearchVersion()); + final var isTVReleaseVersionMain = tvReleaseVersion.equals(mainReleaseVersion); + final var tvIDToBump = isVersionBump ? priorLatestTVSetData.ids().getFirst() : latestTVSetData.ids().getFirst(); + int newVersion = bumpVersionNumber(tvIDToBump, tvReleaseVersion, isMajorVersionBump, isTVReleaseVersionMain); + + // Load the tvSetData for the specified name, if it exists + final var tvSetDataFromFile = TransportVersionUtils.getTVSetData(tvDataDir, tvSetName); + final var tvSetFileExists = tvSetDataFromFile != null; + + // Create/update the data file + if (tvSetFileExists) { + // This is not a new TVSet. We are creating a backport version for an existing TVSet. + // Check to ensure that there isn't already a TV id for this release version (e.g., if this task has been run twice). + var existingIDsForReleaseVersion = tvSetDataFromFile.ids().stream().filter(id -> { + var priorLatestID = priorLatestTVSetData.ids().getFirst(); + return priorLatestID < id && id <= newVersion; + }).toList(); + if (existingIDsForReleaseVersion.isEmpty() == false) { + throw new GradleException( + "A transport version could not be created because a preexisting one was found for this name & release." + + " This could be due to another pre-existing TV with the same name, or a result of running this" + + " task twice:" + + " Release version: " + + tvReleaseVersion + + " TransportVersion Id: " + + existingIDsForReleaseVersion.getFirst() + + " File: " + + getTVSetDataFilePath(tvDataDir, tvSetName) + ); + } + + // Update the existing data file for the backport. + new TransportVersionSetData( + tvSetName, + Streams.concat(tvSetDataFromFile.ids().stream(), Stream.of(newVersion)).sorted().toList().reversed() + ).writeToDataDir(tvDataDir); + } else { + // Create a new data file for the case where this is a new TV + new TransportVersionSetData(tvSetName, List.of(newVersion)).writeToDataDir(tvDataDir); + } + + // Update the LATEST file. + TransportVersionUtils.writeTVSetData( + tvDataDir, + formatLatestTVSetFilename(tvReleaseVersion), + new TransportVersionSetData(tvSetName, List.of(newVersion)) + ); + } + + // TODO Do I need to remove the patch when updating the server portion? NO, but probably need some additional checks + private static int bumpVersionNumber( + int tvIDToBump, + ReleaseVersion releaseVersion, + boolean majorVersionBump, + boolean isTVReleaseVersionMain + ) { + + /* The TV format: + * + * MM_NNN_S_PP + * + * M - The major version of Elasticsearch + * NNN - The server version part + * S - The subsidiary version part. It should always be 0 here, it is only used in subsidiary repositories. + * PP - The patch version part + */ + if (isTVReleaseVersionMain) { + if (majorVersionBump) { + // Bump the major version part, set all other parts to zero. + return releaseVersion.major * 1_000_000; // TODO add check that this doesn't cause overflow out of server versions + } else { + // Bump the server version part if not a major bump. + // TODO add check that this doesn't cause overflow out of server versions + // TODO Do we need to assert on the shape of the number? e.g. no patch version. + return tvIDToBump + 1000; + } + } else { + // bump the patch version part + return tvIDToBump + 1; // TODO add check that this doesn't cause overflow out of patch versions + } + } + + /** + * TODO update this + * Accepts a major.minor version string (e.g. "9.0") and returns the LATEST.json file of the + * previous release string (e.g. "8.19"). + */ + private static ReleaseVersion getPriorReleaseVersion(File tvDataDir, ReleaseVersion releaseVersion) { + assert tvDataDir != null; + assert tvDataDir.isDirectory(); + + if (releaseVersion.minor > 0) { + return new ReleaseVersion(releaseVersion.major, releaseVersion.minor - 1); + } + + // If the minor is 0, we need to find the largest minor on the previous major + var pattern = Pattern.compile("^(\\d+)\\.(\\d+)-LATEST\\.json$"); + var highestMinorOfPrevMajor = Arrays.stream(Objects.requireNonNull(tvDataDir.listFiles())) + .filter(tvDataFile -> tvDataFile.getName().endsWith("-LATEST.json")) + .flatMap(tvDataFile -> { + var matcher = pattern.matcher(tvDataFile.getName()); + var fileMajor = Integer.parseInt(matcher.group(1)); + var fileMinor = Integer.parseInt(matcher.group(2)); + return fileMajor == releaseVersion.major - 1 ? Stream.of(fileMinor) : Stream.empty(); + }) + .sorted() + .toList() + .getLast(); + return new ReleaseVersion(releaseVersion.major - 1, highestMinorOfPrevMajor); + } + + private static String formatLatestTVSetFilename(ReleaseVersion releaseVersion) { + return releaseVersion.toString() + LATEST_SUFFIX; + } + + private record ReleaseVersion(int major, int minor) { + public static ReleaseVersion fromString(String string) { + String[] versionParts = string.split("\\."); + assert versionParts.length == 2; + return new ReleaseVersion(Integer.parseInt(versionParts[0]), Integer.parseInt(versionParts[1])); + } + + public static ReleaseVersion of(Version version) { + return new ReleaseVersion(version.getMajor(), version.getMinor()); + } + + @Override + public @NotNull String toString() { + return major + "." + minor; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj instanceof ReleaseVersion) { + ReleaseVersion that = (ReleaseVersion) obj; + return major == that.major && minor == that.minor; + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(major, minor); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestPlugin.java new file mode 100644 index 0000000000000..55b7bf5c5b4d6 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestPlugin.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.language.jvm.tasks.ProcessResources; + +public class GenerateTransportVersionManifestPlugin implements Plugin { + @Override + public void apply(Project project) { + var transportVersionManifestTask = project.getTasks() + .register("generateTransportVersionManifest", GenerateTransportVersionManifestTask.class, t -> { + var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); + t.getManifestDirectory().set(dir); + t.getManifestFile() + .set(project.getLayout().getBuildDirectory().file("generated-transport-info/transport-versions-files-manifest.txt")); + }); + + // Add the manifest file to the jar + project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> { + task.into("META-INF", copy -> copy.from(transportVersionManifestTask)); + }); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java new file mode 100644 index 0000000000000..dba0885743bcb --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/GenerateTransportVersionManifestTask.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.FileWriter; +import java.io.IOException; + +public abstract class GenerateTransportVersionManifestTask extends DefaultTask { + + @InputDirectory + public abstract RegularFileProperty getManifestDirectory(); + + @OutputFile + public abstract RegularFileProperty getManifestFile(); + + @TaskAction + public void generateTransportVersionManifest() { + // 1. Go through the directory returned by getManifestDirectory(); + // 2. Pick out the file names + // 3. Write a file to the file provided by getManifestFile() containing all the filenames + System.out.println("Potato: Generating transport version manifest..."); + var manifestDirectory = getManifestDirectory(); + var dir = manifestDirectory.getAsFile().get(); + System.out.println("Potato: manifest directory location" + dir.getAbsolutePath()); + var manifestFile = getManifestFile(); + System.out.println("Potato: manifest file location: " + manifestFile.getAsFile().get().getAbsolutePath()); + try (FileWriter writer = new FileWriter(manifestFile.getAsFile().get())) { + for (var file : dir.listFiles()) { + System.out.println("Potato GenerateTransportVersionManifestTask: " + file.getAbsolutePath()); + var fileName = file.getName(); + if (fileName.endsWith("-LATEST.json") == false) { + writer.write(fileName + "\n"); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + System.out.println("Potato: Finished generating transport version manifest..."); + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java new file mode 100644 index 0000000000000..cb6c4c582bcb8 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsPlugin.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.elasticsearch.gradle.util.GradleUtils; +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.file.FileCollection; +import org.gradle.api.tasks.SourceSet; + +public class LocateTransportVersionsPlugin implements Plugin { + public static final String TRANSPORT_VERSION_NAMES_FILE = "generated-transport-info/transport-version-set-names.txt"; + + @Override + public void apply(Project project) { + + final var checkTransportVersion = project.getTasks().register("locateTransportVersions", LocateTransportVersionsTask.class, t -> { + t.setGroup("Transport Versions"); + t.setDescription("Collects all TransportVersion names used throughout the project"); + SourceSet mainSourceSet = GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME); + FileCollection clasDirs = mainSourceSet.getRuntimeClasspath(); + t.getClassDirs().set(clasDirs); + t.getOutputFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); + }); + + var config = project.getConfigurations().create("locateTransportVersionsConfig"); + project.getArtifacts().add(config.getName(), checkTransportVersion); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java new file mode 100644 index 0000000000000..47bc4c6ca916a --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/LocateTransportVersionsTask.java @@ -0,0 +1,152 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.FileCollection; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.provider.Property; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.OutputFile; +import org.gradle.api.tasks.TaskAction; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.MethodVisitor; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.tree.LdcInsnNode; +import org.objectweb.asm.tree.MethodNode; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * This task locates all method invocations of org.elasticsearch.TransportVersionSet#get(java.lang.String) in the + * provided directory, and then records the value of string literals passed as arguments. It then records each + * String on a newline in the provided output file. + */ +public abstract class LocateTransportVersionsTask extends DefaultTask { + public static final String TRANSPORT_VERSION_SET_CLASS = "org/elasticsearch/TransportVersionSet"; + public static final String TRANSPORT_VERSION_SET_METHOD_NAME = "get"; + + /** + * The directory to scan for TransportVersionSet#get invocations. + */ + @InputFiles + public abstract Property getClassDirs(); + + /** + * The output file, with each newline containing the string literal argument of each TransportVersionSet#get + * invocation. + */ + @OutputFile + public abstract RegularFileProperty getOutputFile(); + + @TaskAction + public void checkTransportVersion() { + var tvNames = getTVDeclarationNames(getClassDirs().get().getFiles()); + File file = new File(getOutputFile().get().getAsFile().getAbsolutePath()); + try (FileWriter writer = new FileWriter(file)) { + for (String tvName : tvNames) { + writer.write(tvName + "\n"); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static Set getTVDeclarationNames(Collection classDirs) { + + var javaFiles = findJavaFiles(classDirs); + + var results = new HashSet(); + for (File javaFile : javaFiles) { + // Print the path of each Java file found + // System.out.println("Found Java file: " + javaFile.getAbsolutePath()); + try (InputStream inputStream = new FileInputStream(javaFile)) { + + ClassVisitor classVisitor = new ClassVisitor(Opcodes.ASM9) { + @Override + public MethodVisitor visitMethod(int access, String name, String descriptor, String signature, String[] exceptions) { + // System.out.println("name: " + name + " descriptor: " + descriptor + " signature: " + signature); + + return new MethodNode(Opcodes.ASM9, access, name, descriptor, signature, exceptions) { + + @Override + public void visitMethodInsn(int opcode, String owner, String name, String descriptor, boolean isInterface) { + if (owner.equals(TRANSPORT_VERSION_SET_CLASS) && name.equals(TRANSPORT_VERSION_SET_METHOD_NAME)) { + // System.out.println("Potato: opcode: " + opcode + " owner: " + owner + " name: " + name); + var abstractInstruction = this.instructions.getLast(); + if (abstractInstruction instanceof LdcInsnNode ldcInsnNode) { + if (ldcInsnNode.cst instanceof String tvName && tvName.isEmpty() == false) { + System.out.println("Potato: constant: " + tvName); + results.add(tvName); + } else { + System.out.println( + "Transport Versions must be declared with a constant string. " + + "file: " + + javaFile.getPath() + ); + } + } + System.out.println( + "visitMethodInsn: opcode: " + + opcode + + " owner: " + + owner + + " name: " + + name + + " descriptor: " + + descriptor + + " file: " + + javaFile.getPath() + ); + } + super.visitMethodInsn(opcode, owner, name, descriptor, isInterface); + } + }; + } + }; + ClassReader classReader = new ClassReader(inputStream); + classReader.accept(classVisitor, 0); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return results; + } + + private static List findJavaFiles(Collection files) { + final String JAVA_EXTENSION = ".class"; + final String MODULE_INFO = "module-info.class"; + List javaFiles = new ArrayList<>(); + + for (File file : files) { + if (file.isDirectory()) { + File[] subFiles = file.listFiles(); + if (subFiles != null) { + javaFiles.addAll(findJavaFiles(Arrays.asList(subFiles))); + } + } else if (file.getName().endsWith(JAVA_EXTENSION) && file.getName().endsWith(MODULE_INFO) == false) { + javaFiles.add(file); + } + } + return javaFiles; + } + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java new file mode 100644 index 0000000000000..9f6b1a831bb30 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/TransportVersionUtils.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.File; +import java.io.Serializable; +import java.nio.file.Path; +import java.util.List; + +public class TransportVersionUtils { + public static final String LATEST_SUFFIX = "-LATEST.json"; + public static final String JSON_SUFFIX = ".json"; + + public record TransportVersionSetData(String name, List ids) implements Serializable { + public TransportVersionSetData(@JsonProperty("name") String name, @JsonProperty("ids") List ids) { + this.name = name; + this.ids = ids; + } + + public void writeToDataDir(File tvDataDir) { + TransportVersionUtils.writeTVSetData(tvDataDir, name + JSON_SUFFIX, this); + } + } + + public static void writeTVSetData(File tvDataDir, String filename, TransportVersionSetData versionSetData) { + ObjectMapper mapper = new ObjectMapper(); + try { + File tvSetFile = tvDataDir.toPath().resolve(filename).toFile(); + mapper.writerWithDefaultPrettyPrinter().writeValue(tvSetFile, versionSetData); + } catch (Exception e) { + throw new RuntimeException("Failed to write the TransportVersionSet data file: " + tvDataDir.getAbsolutePath(), e); + } + } + + public static TransportVersionSetData getLatestTVSetData(File tvDataDir, String majorMinor) { + return getTVSetData(tvDataDir.toPath().resolve(majorMinor + LATEST_SUFFIX)); + + } + + public static Path getTVSetDataFilePath(File tvDataDir, String tvSetNameField) { + return tvDataDir.toPath().resolve(tvSetNameField + JSON_SUFFIX); + } + + public static TransportVersionSetData getTVSetData(File tvDataDir, String tvSetNameField) { + return getTVSetData(getTVSetDataFilePath(tvDataDir, tvSetNameField)); + } + + /** + * Returns the TransportVersionSetData read from the file at the specified path, null if no file exists. + */ + public static TransportVersionSetData getTVSetData(Path path) { + File tvSetDataFile = path.toFile(); + if (tvSetDataFile.exists() == false) { + return null; + } + + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readValue(tvSetDataFile, TransportVersionSetData.class); + } catch (Exception e) { + throw new RuntimeException("Failed to read the TransportVersionSet data file: " + tvSetDataFile.getAbsolutePath(), e); + } + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesPlugin.java new file mode 100644 index 0000000000000..a8cb55c7d2cb5 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesPlugin.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +public class ValidateTransportVersionDataFilesPlugin implements Plugin { + @Override + public void apply(Project project) { + final var task = project.getTasks().register("validateTransportVersionDataFiles", ValidateTransportVersionsTask.class, t -> { + t.getTransportVersionSetNamesFile() + .set( + project.getLayout() + .getBuildDirectory() + .file(AggregateTransportVersionDeclarationsPlugin.ALL_TRANSPORT_VERSION_NAMES_FILE) + ); + t.getDataFileDirectory().set(project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/")); + }); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesTask.java new file mode 100644 index 0000000000000..f207afd805d7d --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionDataFilesTask.java @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.DefaultTask; + +public abstract class ValidateTransportVersionDataFilesTask extends DefaultTask { + // TODO + +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsPlugin.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsPlugin.java new file mode 100644 index 0000000000000..69106c76e3740 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsPlugin.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; + +import static org.elasticsearch.gradle.internal.transport.LocateTransportVersionsPlugin.TRANSPORT_VERSION_NAMES_FILE; + +public class ValidateTransportVersionsPlugin implements Plugin { + + @Override + public void apply(Project project) { + project.getTasks().register("validateTransportVersions", ValidateTransportVersionsTask.class, t -> { + var dir = project.getLayout().getProjectDirectory().file("src/main/resources/org/elasticsearch/transport/"); + t.getDataFileDirectory().set(dir); + t.getTransportVersionSetNamesFile().set(project.getLayout().getBuildDirectory().file(TRANSPORT_VERSION_NAMES_FILE)); + // TODO is this correct? Needs to have both global/per-plugin versions and dependencies + t.dependsOn(project.getTasks().withType(LocateTransportVersionsTask.class)); + t.setGroup("Transport Versions"); // TODO + t.setDescription("Validates that the transport versions used in the project are correct and up to date."); // TODO + }); + } +} diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsTask.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsTask.java new file mode 100644 index 0000000000000..aa3653e556019 --- /dev/null +++ b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/transport/ValidateTransportVersionsTask.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.gradle.internal.transport; + +import groovy.json.JsonSlurper; + +import org.gradle.api.DefaultTask; +import org.gradle.api.file.RegularFileProperty; +import org.gradle.api.tasks.InputDirectory; +import org.gradle.api.tasks.InputFile; +import org.gradle.api.tasks.TaskAction; + +import java.io.BufferedReader; +import java.io.FileReader; +import java.io.IOException; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Validates that each transport version declaration has an associated metadata file. + * TODO: + * - make this depend on the LocateTransportVersions task/plugin + * - make this both a local (per module/plugin) and global task + * - Also validate that there are no duplicate tvs (ids, names?) + */ +public abstract class ValidateTransportVersionsTask extends DefaultTask { + + @InputDirectory + public abstract RegularFileProperty getDataFileDirectory(); + + @InputFile + public abstract RegularFileProperty getTransportVersionSetNamesFile(); + + @TaskAction + public void validateTransportVersions() throws IOException { + var dataFileDirectory = getDataFileDirectory(); + var tvDataDir = dataFileDirectory.getAsFile().get(); + + Set tvSetNamesInDataFiles = new HashSet<>(); + for (var tvDataFile : Objects.requireNonNull(tvDataDir.listFiles())) { + if (tvDataFile.getName().endsWith("-LATEST.json") == false) { + var slurper = new JsonSlurper(); + if (slurper.parse(tvDataFile) instanceof Map json) { + System.out.println(tvDataFile.getAbsolutePath()); + String name = json.get("name").toString(); + tvSetNamesInDataFiles.add(name); + } else { + throw new RuntimeException(tvDataFile + " is not a json file"); + } + } + } + + var tvSetDeclaredNamesFile = getTransportVersionSetNamesFile().get().getAsFile(); + try (var reader = new BufferedReader(new FileReader(tvSetDeclaredNamesFile))) { + reader.lines().forEach(declaredName -> { + if (tvSetNamesInDataFiles.contains(declaredName) == false) { + throw new RuntimeException( + "TransportVersionSetData.get(\"" + + declaredName + + "\") was used, but lacks a" + + " data file with a corresponding transport version. This can be generated with the task" + ); // TODO + } + }); + } + } +} diff --git a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy index 72c67514f3a5c..2769552a404c2 100644 --- a/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy +++ b/build-tools/src/integTest/groovy/org/elasticsearch/gradle/plugin/PluginBuildPluginFuncTest.groovy @@ -6,7 +6,12 @@ * your election, the "Elastic License 2.0", the "GNU Affero General Public * License v3.0 only", or the "Server Side Public License, v 1". */ - +// look here +/* +This sets up a dummy plugin +applies the es plugin +actually runs the build + */ package org.elasticsearch.gradle.plugin import org.elasticsearch.gradle.VersionProperties diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java index e00a4b45f8a61..c4e85f0d0d77e 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/plugin/BasePluginBuildPlugin.java @@ -193,6 +193,10 @@ private static CopySpec createBundleSpec( : project.getTasks().named("jar") ); bundleSpec.from( + // TODO look at this + /* + Take the runtime classpath, and remove the things that are compileonly, and all the libs are transitively compileonly + */ project.getConfigurations() .getByName("runtimeClasspath") .minus(project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME)) diff --git a/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java index ed20d40582f57..52bdd1cf1b7cb 100644 --- a/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java +++ b/build-tools/src/main/java/org/elasticsearch/gradle/test/TestBuildInfoPlugin.java @@ -53,6 +53,7 @@ public void apply(Project project) { task.getCodeLocations().set(codeLocations); }); + // TODO look at this, it copies files to a different place so we can find them later project.getTasks().withType(ProcessResources.class).named("processResources").configure(task -> { task.into("META-INF", copy -> copy.from(testBuildInfoTask)); }); diff --git a/distribution/build.gradle b/distribution/build.gradle index fa6223d30e63b..6584452b5a070 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -35,9 +35,11 @@ configurations { attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, DependenciesInfoPlugin.USAGE_ATTRIBUTE)) } } - featuresMetadata { + featuresMetadata { // here, this is how we want to do this. Can't depend on explicit configs in serverless attributes { - attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) + attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, ClusterFeaturesMetadataPlugin.FEATURES_METADATA_TYPE) // disambiguates what artifact I need + // two configs in this dance, one that produces the artifact, and one that consumes it. + // - on the consumer side, I'm going to resolve the config } } } @@ -210,7 +212,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { } distro.copyModule(processDefaultOutputsTaskProvider, module) - dependencies.add('featuresMetadata', module) + dependencies.add('featuresMetadata', module) // TODO look at what we do here if (module.name.startsWith('transport-') || (buildParams.snapshotBuild == false && module.name == 'apm')) { distro.copyModule(processIntegTestOutputsTaskProvider, module) } diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 7b73575f76ef3..142c75706db65 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -26,6 +26,11 @@ configurations { } } +// Check this out( +/* +declaring a config called rest spec +contents of that config are just this folder + */ artifacts { basicRestSpecs(new File(projectDir, "src/main/resources")) restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api")) diff --git a/server/build.gradle b/server/build.gradle index be2b43745d0b2..7e289aa8fb12c 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -12,6 +12,12 @@ apply plugin: 'elasticsearch.publish' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.internal-test-artifact' apply plugin: 'elasticsearch.test-build-info' +apply plugin: 'elasticsearch.locate-transport-versions' +apply plugin: 'elasticsearch.generate-transport-version-manifest' +apply plugin: 'elasticsearch.generate-transport-version-data' +apply plugin: 'elasticsearch.validate-transport-versions' +apply plugin: 'elasticsearch.aggregate-transport-versionDeclarations-plugin' +apply plugin: 'elasticsearch.validate-transport-version-data-files' publishing { publications { @@ -116,6 +122,7 @@ def generateModulesList = tasks.register("generateModulesList") { } } +// here, check this out for how to aggregate def generatePluginsList = tasks.register("generatePluginsList") { Set plugins = new TreeSet<>(project(':plugins').childProjects.keySet()) plugins.remove('example') diff --git a/server/src/main/java/org/elasticsearch/TransportVersion.java b/server/src/main/java/org/elasticsearch/TransportVersion.java index f1e20ac4bb77e..d7678fd3e61d0 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersion.java +++ b/server/src/main/java/org/elasticsearch/TransportVersion.java @@ -16,12 +16,14 @@ import org.elasticsearch.plugins.ExtensionLoader; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.ServiceLoader; import java.util.function.Function; +import java.util.function.IntFunction; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -176,7 +178,7 @@ public boolean isPatchFrom(TransportVersion version) { * if applicable for this deployment, otherwise the raw version number. */ public String toReleaseVersion() { - return TransportVersions.VERSION_LOOKUP.apply(id); + return VersionsHolder.VERSION_LOOKUP.apply(id); } @Override @@ -185,6 +187,7 @@ public String toString() { } private static class VersionsHolder { + private static final IntFunction VERSION_LOOKUP; private static final List ALL_VERSIONS; private static final Map ALL_VERSIONS_MAP; private static final TransportVersion CURRENT; @@ -195,14 +198,15 @@ private static class VersionsHolder { .orElse(Collections.emptyList()); if (extendedVersions.isEmpty()) { - ALL_VERSIONS = TransportVersions.DEFINED_VERSIONS; + ALL_VERSIONS = new ArrayList<>(TransportVersions.DEFINED_VERSIONS); + ALL_VERSIONS.addAll(TransportVersionSet.TRANSPORT_VERSIONS); } else { ALL_VERSIONS = Stream.concat(TransportVersions.DEFINED_VERSIONS.stream(), extendedVersions.stream()).sorted().toList(); } ALL_VERSIONS_MAP = ALL_VERSIONS.stream().collect(Collectors.toUnmodifiableMap(TransportVersion::id, Function.identity())); - - CURRENT = ALL_VERSIONS.getLast(); + CURRENT = ALL_VERSIONS.stream().max(TransportVersion::compareTo).get(); + VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, CURRENT.id()); } } } diff --git a/server/src/main/java/org/elasticsearch/TransportVersionSet.java b/server/src/main/java/org/elasticsearch/TransportVersionSet.java new file mode 100644 index 0000000000000..0f1ebb70c9f4d --- /dev/null +++ b/server/src/main/java/org/elasticsearch/TransportVersionSet.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch; + +import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.ParseField; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.json.JsonXContent; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TransportVersionSet { + + private static final ParseField NAME = new ParseField("name"); + private static final ParseField IDS = new ParseField("ids"); + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + TransportVersionSet.class.getCanonicalName(), + false, + (args, latestTransportId) -> { + String name = (String) args[0]; + @SuppressWarnings("unchecked") + List ids = (List) args[1]; + List versions = new ArrayList<>(ids.size()); + for (int id = 0; id < ids.size(); ++id) { + if (id <= latestTransportId) { + versions.add(new TransportVersion(ids.get(id))); + } + } + if (versions.isEmpty()) { + // TODO: throw + } + return new TransportVersionSet(name, Collections.unmodifiableList(versions)); + } + ); + + static { + PARSER.declareString(ConstructingObjectParser.constructorArg(), NAME); + PARSER.declareIntArray(ConstructingObjectParser.constructorArg(), IDS); + } + + private static final Map TRANSPORT_VERSION_SETS = loadTransportVersionSets(); + public static final List TRANSPORT_VERSIONS = collectTransportVersions(); + + private static Map loadTransportVersionSets() { + Map transportVersionSets = new HashMap<>(); + + String latestLocation = "transport/" + Version.CURRENT.major + "." + Version.CURRENT.minor + "-LATEST.json"; + int latestId; + try (InputStream inputStream = TransportVersionSet.class.getResourceAsStream(latestLocation)) { + TransportVersionSet latest = fromXContent(inputStream, Integer.MAX_VALUE); + // TODO: validation of latest tranport version set + latestId = latest.versions.get(0).id(); + } catch (IOException ioe) { + throw new UncheckedIOException("latest transport version file not found at [" + latestLocation + "]", ioe); + } + + String manifestLocation = "META-INF/transport-versions-files-manifest.txt"; + List versionNames; + try (InputStream transportVersionManifest = TransportVersionSet.class.getClassLoader().getResourceAsStream(manifestLocation)) { + BufferedReader reader = new BufferedReader(new InputStreamReader(transportVersionManifest, StandardCharsets.UTF_8)); + versionNames = reader.lines().filter(line -> line.isBlank() == false).toList(); + } catch (IOException ioe) { + throw new UncheckedIOException("transport version metadata manifest file not found at [" + manifestLocation + "]", ioe); + } + + for (String name : versionNames) { + String versionLocation = "transport/" + name; + try (InputStream inputStream = TransportVersionSet.class.getResourceAsStream(versionLocation)) { + TransportVersionSet transportVersionSet = TransportVersionSet.fromXContent(inputStream, latestId); + transportVersionSets.put(transportVersionSet.name, transportVersionSet); + } catch (IOException ioe) { + throw new UncheckedIOException("transport version set file not found at [ " + versionLocation + "]", ioe); + } + } + + return Collections.unmodifiableMap(transportVersionSets); + } + + private static List collectTransportVersions() { + List transportVersions = new ArrayList<>(); + for (TransportVersionSet transportVersionSet : TRANSPORT_VERSION_SETS.values()) { + transportVersions.addAll(transportVersionSet.versions); + } + transportVersions.sort(TransportVersion::compareTo); + return transportVersions; + } + + private static TransportVersionSet fromXContent(InputStream inputStream, int maxTransportId) throws IOException { + XContentParser parser = JsonXContent.jsonXContent.createParser(XContentParserConfiguration.EMPTY, inputStream); + return PARSER.parse(parser, maxTransportId); + } + + public static TransportVersionSet get(String name) { + TransportVersionSet transportVersionSet = TRANSPORT_VERSION_SETS.get(name); + if (transportVersionSet == null) { + // TODO: throw + } + return transportVersionSet; + } + + public static TransportVersion local(String name) { + return get(name).local(); + } + + public static TransportVersion oldest(String name) { + return get(name).oldest(); + } + + public static boolean isCompatible(String name, TransportVersion version) { + return get(name).isCompatible(version); + } + + private final String name; + private final List versions; + + private TransportVersionSet(String name, List versions) { + this.name = name; + this.versions = versions; + } + + public String name() { + return name; + } + + public TransportVersion local() { + return versions.get(0); + } + + public TransportVersion oldest() { + return versions.get(versions.size() - 1); + } + + public boolean isCompatible(TransportVersion version) { + boolean compatible = version.onOrAfter(local()); + for (int v = 1; v < versions.size(); ++v) { + compatible |= version.isPatchFrom(versions.get(v)); + } + return compatible; + } + + public boolean isNotCompatible(TransportVersion version) { + boolean compatible = version.before(local()); + for (int v = 1; v < versions.size(); ++v) { + compatible &= version.isPatchFrom(versions.get(v)) == false; + } + return compatible; + } + + @Override + public String toString() { + return "TransportVersionSet{" + "name='" + name + '\'' + ", versions=" + versions + '}'; + } +} diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index ae0ccecf15ed7..2ef3b62dea9c5 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -19,7 +19,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.function.IntFunction; /** *

Transport version is used to coordinate compatible wire protocol communication between nodes, at a fine-grained level. This replaces @@ -200,18 +199,8 @@ static TransportVersion def(int id) { public static final TransportVersion IDP_CUSTOM_SAML_ATTRIBUTES_ALLOW_LIST_8_19 = def(8_841_0_50); public static final TransportVersion SETTINGS_IN_DATA_STREAMS_8_19 = def(8_841_0_51); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_REMOVE_ERROR_PARSING_8_19 = def(8_841_0_52); - public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE_8_19 = def(8_841_0_53); - public static final TransportVersion STREAMS_LOGS_SUPPORT_8_19 = def(8_841_0_54); - public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE_8_19 = def(8_841_0_55); public static final TransportVersion RANDOM_SAMPLER_QUERY_BUILDER_8_19 = def(8_841_0_56); - public static final TransportVersion ML_INFERENCE_SAGEMAKER_ELASTIC_8_19 = def(8_841_0_57); - public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS_8_19 = def(8_841_0_58); - public static final TransportVersion ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED_8_19 = def(8_841_0_59); - public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION_8_19 = def(8_841_0_60); public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61); - public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN_8_19 = def(8_841_0_62); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_8_19 = def(8_841_0_63); - public static final TransportVersion ESQL_FIXED_INDEX_LIKE_8_19 = def(8_841_0_64); public static final TransportVersion V_9_0_0 = def(9_000_0_09); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10); public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11); @@ -318,28 +307,8 @@ static TransportVersion def(int id) { public static final TransportVersion STATE_PARAM_GET_SNAPSHOT = def(9_100_0_00); public static final TransportVersion PROJECT_ID_IN_SNAPSHOTS_DELETIONS_AND_REPO_CLEANUP = def(9_101_0_00); public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_REMOVE_ERROR_PARSING = def(9_102_0_00); - public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE = def(9_103_0_00); - public static final TransportVersion STREAMS_LOGS_SUPPORT = def(9_104_0_00); - public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE = def(9_105_0_00); - public static final TransportVersion ML_INFERENCE_SAGEMAKER_ELASTIC = def(9_106_0_00); - public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS = def(9_107_0_00); - public static final TransportVersion CLUSTER_STATE_PROJECTS_SETTINGS = def(9_108_0_00); - public static final TransportVersion ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED = def(9_109_00_0); - public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION = def(9_110_0_00); - public static final TransportVersion ESQL_PROFILE_INCLUDE_PLAN = def(9_111_0_00); - public static final TransportVersion MAPPINGS_IN_DATA_STREAMS = def(9_112_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES_9_1 = def(9_112_0_01); - public static final TransportVersion ESQL_FIXED_INDEX_LIKE_9_1 = def(9_112_0_02); - // Below is the first version in 9.2 and NOT in 9.1. - public static final TransportVersion PROJECT_STATE_REGISTRY_RECORDS_DELETIONS = def(9_113_0_00); - public static final TransportVersion ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = def(9_114_0_00); - public static final TransportVersion ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = def(9_115_0_00); - public static final TransportVersion ESQL_SPLIT_ON_BIG_VALUES = def(9_116_0_00); - public static final TransportVersion ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = def(9_117_0_00); - public static final TransportVersion ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = def(9_118_0_00); - public static final TransportVersion ESQL_FIXED_INDEX_LIKE = def(9_119_0_00); - public static final TransportVersion LOOKUP_JOIN_CCS = def(9_120_0_00); - public static final TransportVersion NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = def(9_121_0_00); + + public static final TransportVersionSet FOO = TransportVersionSet.get("FOO"); /* * STOP! READ THIS FIRST! No, really, @@ -413,16 +382,6 @@ static TransportVersion def(int id) { */ static final List DEFINED_VERSIONS = collectAllVersionIdsDefinedInClass(TransportVersions.class); - // the highest transport version constant defined - static final TransportVersion LATEST_DEFINED; - static { - LATEST_DEFINED = DEFINED_VERSIONS.getLast(); - - // see comment on IDS field - // now we're registered all the transport versions, we can clear the map - IDS = null; - } - public static List collectAllVersionIdsDefinedInClass(Class cls) { Map versionIdFields = new HashMap<>(); List definedTransportVersions = new ArrayList<>(); @@ -464,8 +423,6 @@ public static List collectAllVersionIdsDefinedInClass(Class return List.copyOf(definedTransportVersions); } - static final IntFunction VERSION_LOOKUP = ReleaseVersions.generateVersionsLookup(TransportVersions.class, LATEST_DEFINED.id()); - // no instance private TransportVersions() {} } diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 6b8959f0c7756..034eba46b7e99 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -218,7 +218,10 @@ public class Version implements VersionId, ToXContentFragment { public static final Version V_9_0_4 = new Version(9_00_04_99); public static final Version V_9_1_0 = new Version(9_01_00_99); public static final Version V_9_2_0 = new Version(9_02_00_99); - public static final Version CURRENT = V_9_2_0; + public static final Version CURRENT = V_9_2_0; // here + /* + - need to associate current with the appropriate TV + */ private static final NavigableMap VERSION_IDS; private static final Map VERSION_STRINGS; diff --git a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java index 6d11700500c24..307d52eb45c75 100644 --- a/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java +++ b/server/src/main/java/org/elasticsearch/cluster/ClusterInfo.java @@ -10,6 +10,7 @@ package org.elasticsearch.cluster; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.routing.RecoverySource; import org.elasticsearch.cluster.routing.ShardRouting; @@ -49,6 +50,10 @@ public class ClusterInfo implements ChunkedToXContent, Writeable { public static final ClusterInfo EMPTY = new ClusterInfo(); + public static final TransportVersionSet NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO = TransportVersionSet.get( + "node-usage-stats-for-thread-pools-in-cluster-info" + ); + public static final TransportVersion DATA_PATH_NEW_KEY_VERSION = TransportVersions.V_8_6_0; private final Map leastAvailableSpaceUsage; @@ -111,7 +116,7 @@ public ClusterInfo(StreamInput in) throws IOException { } else { this.estimatedHeapUsages = Map.of(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO)) { + if (NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO.isCompatible(in.getTransportVersion())) { this.nodeUsageStatsForThreadPools = in.readImmutableMap(NodeUsageStatsForThreadPools::new); } else { this.nodeUsageStatsForThreadPools = Map.of(); @@ -133,7 +138,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.HEAP_USAGE_IN_CLUSTER_INFO)) { out.writeMap(this.estimatedHeapUsages, StreamOutput::writeWriteable); } - if (out.getTransportVersion().onOrAfter(TransportVersions.NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO)) { + if (NODE_USAGE_STATS_FOR_THREAD_POOLS_IN_CLUSTER_INFO.isCompatible(out.getTransportVersion())) { out.writeMap(this.nodeUsageStatsForThreadPools, StreamOutput::writeWriteable); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index c318159c9ca4c..7e7c4c8951016 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -18,6 +18,7 @@ import org.apache.lucene.index.PointValues; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.admin.indices.rollover.RolloverConfiguration; @@ -92,6 +93,7 @@ public final class DataStream implements SimpleDiffable, ToXContentO public static final TransportVersion ADDED_FAILURE_STORE_TRANSPORT_VERSION = TransportVersions.V_8_12_0; public static final TransportVersion ADDED_AUTO_SHARDING_EVENT_VERSION = TransportVersions.V_8_14_0; public static final TransportVersion ADD_DATA_STREAM_OPTIONS_VERSION = TransportVersions.V_8_16_0; + public static final TransportVersionSet MAPPINGS_IN_DATA_STREAMS = TransportVersionSet.get("mappings-in-data-streams"); public static final String BACKING_INDEX_PREFIX = ".ds-"; public static final String FAILURE_STORE_PREFIX = ".fs-"; @@ -302,7 +304,7 @@ public static DataStream read(StreamInput in) throws IOException { settings = Settings.EMPTY; } CompressedXContent mappings; - if (in.getTransportVersion().onOrAfter(TransportVersions.MAPPINGS_IN_DATA_STREAMS)) { + if (MAPPINGS_IN_DATA_STREAMS.isCompatible(in.getTransportVersion())) { mappings = CompressedXContent.readCompressedString(in); } else { mappings = EMPTY_MAPPINGS; @@ -1437,7 +1439,7 @@ public void writeTo(StreamOutput out) throws IOException { || out.getTransportVersion().isPatchFrom(TransportVersions.SETTINGS_IN_DATA_STREAMS_8_19)) { settings.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.MAPPINGS_IN_DATA_STREAMS)) { + if (MAPPINGS_IN_DATA_STREAMS.isCompatible(out.getTransportVersion())) { mappings.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 4b574b8313c28..cf8162cb55813 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -78,6 +78,7 @@ import static org.elasticsearch.cluster.metadata.LifecycleExecutionState.ILM_CUSTOM_METADATA_KEY; import static org.elasticsearch.cluster.metadata.Metadata.ALL; +import static org.elasticsearch.cluster.project.ProjectStateRegistry.CLUSTER_STATE_PROJECTS_SETTINGS; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; public class ProjectMetadata implements Iterable, Diffable, ChunkedToXContent { @@ -2204,8 +2205,7 @@ public static ProjectMetadata readFrom(StreamInput in) throws IOException { builder.put(ReservedStateMetadata.readFrom(in)); } - if (in.getTransportVersion() - .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { + if (in.getTransportVersion().between(TransportVersions.PROJECT_METADATA_SETTINGS, CLUSTER_STATE_PROJECTS_SETTINGS.local())) { Settings.readSettingsFromStream(in); } @@ -2240,8 +2240,7 @@ public void writeTo(StreamOutput out) throws IOException { VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); out.writeCollection(reservedStateMetadata.values()); - if (out.getTransportVersion() - .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { + if (out.getTransportVersion().between(TransportVersions.PROJECT_METADATA_SETTINGS, CLUSTER_STATE_PROJECTS_SETTINGS.local())) { Settings.EMPTY.writeTo(out); } } @@ -2308,8 +2307,7 @@ private ProjectMetadataDiff(ProjectMetadata before, ProjectMetadata after) { DiffableUtils.getStringKeySerializer(), RESERVED_DIFF_VALUE_READER ); - if (in.getTransportVersion() - .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { + if (in.getTransportVersion().between(TransportVersions.PROJECT_METADATA_SETTINGS, CLUSTER_STATE_PROJECTS_SETTINGS.local())) { Settings.readSettingsDiffFromStream(in); } } @@ -2336,8 +2334,7 @@ public void writeTo(StreamOutput out) throws IOException { templates.writeTo(out); customs.writeTo(out); reservedStateMetadata.writeTo(out); - if (out.getTransportVersion() - .between(TransportVersions.PROJECT_METADATA_SETTINGS, TransportVersions.CLUSTER_STATE_PROJECTS_SETTINGS)) { + if (out.getTransportVersion().between(TransportVersions.PROJECT_METADATA_SETTINGS, CLUSTER_STATE_PROJECTS_SETTINGS.local())) { Settings.EMPTY_DIFF.writeTo(out); } } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java index 99758350559d3..575caa7d589d5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/StreamsMetadata.java @@ -10,7 +10,7 @@ package org.elasticsearch.cluster.metadata; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.NamedDiff; import org.elasticsearch.common.collect.Iterators; @@ -33,6 +33,8 @@ public class StreamsMetadata extends AbstractNamedDiffable readDiffFrom(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java index 2876ebc13c70c..7c0246b21f25d 100644 --- a/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java +++ b/server/src/main/java/org/elasticsearch/cluster/project/ProjectStateRegistry.java @@ -10,7 +10,7 @@ package org.elasticsearch.cluster.project; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.cluster.AbstractNamedDiffable; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.NamedDiff; @@ -38,6 +38,12 @@ public class ProjectStateRegistry extends AbstractNamedDiffable projectsSettings; // Projects that have been marked for deletion based on their file-based setting private final Set projectsMarkedForDeletion; @@ -46,7 +52,7 @@ public class ProjectStateRegistry extends AbstractNamedDiffable is not + // executed prior to logging initialization. + private TransportVersion version = null; /** * The transport version the data is serialized as. */ public TransportVersion getTransportVersion() { + if (this.version == null) { + version = TransportVersion.current(); + } return this.version; } @@ -89,7 +95,7 @@ public TransportVersion getTransportVersion() { * Set the transport version of the data in this stream. */ public void setTransportVersion(TransportVersion version) { - this.version = version; + this.version = Objects.requireNonNull(version); } /** diff --git a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java index a1eadb9ae7a5a..e19cf7a7782b6 100644 --- a/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/query/WildcardQueryBuilder.java @@ -14,6 +14,7 @@ import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; @@ -46,6 +47,8 @@ public class WildcardQueryBuilder extends AbstractQueryBuilder queryVectors; @@ -129,8 +131,7 @@ public SparseVectorQueryBuilder(StreamInput in) throws IOException { super(in); this.fieldName = in.readString(); - if (in.getTransportVersion().isPatchFrom(SPARSE_VECTOR_FIELD_PRUNING_OPTIONS_8_19) - || in.getTransportVersion().onOrAfter(SPARSE_VECTOR_FIELD_PRUNING_OPTIONS)) { + if (SPARSE_VECTOR_FIELD_PRUNING_OPTIONS.isCompatible(in.getTransportVersion())) { this.shouldPruneTokens = in.readOptionalBoolean(); } else { this.shouldPruneTokens = in.readBoolean(); @@ -185,8 +186,7 @@ protected void doWriteTo(StreamOutput out) throws IOException { out.writeString(fieldName); - if (out.getTransportVersion().isPatchFrom(SPARSE_VECTOR_FIELD_PRUNING_OPTIONS_8_19) - || out.getTransportVersion().onOrAfter(SPARSE_VECTOR_FIELD_PRUNING_OPTIONS)) { + if (SPARSE_VECTOR_FIELD_PRUNING_OPTIONS.isCompatible(out.getTransportVersion())) { out.writeOptionalBoolean(shouldPruneTokens); } else { out.writeBoolean(shouldPruneTokens); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index daf26a7b5348f..dd654d9f9de33 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -26,6 +27,10 @@ */ public class EsField implements Writeable { + public static final TransportVersionSet ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE = TransportVersionSet.get( + "esql-serialize-timeseries-field-type" + ); + /** * Fields in a TSDB can be either dimensions or metrics. This enum provides a way to store, serialize, and operate on those field * roles within the ESQL query processing pipeline. @@ -169,13 +174,13 @@ public void writeContent(StreamOutput out) throws IOException { } protected void writeTimeSeriesFieldType(StreamOutput out) throws IOException { - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE)) { + if (ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE.isCompatible(out.getTransportVersion())) { this.timeSeriesFieldType.writeTo(out); } } protected static TimeSeriesFieldType readTimeSeriesFieldType(StreamInput in) throws IOException { - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE)) { + if (ESQL_SERIALIZE_TIMESERIES_FIELD_TYPE.isCompatible(in.getTransportVersion())) { return TimeSeriesFieldType.readFromStream(in); } else { return TimeSeriesFieldType.UNKNOWN; diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java index 4a8fcda81f82a..50e0bf7347d65 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/read/ValuesSourceReaderOperatorStatus.java @@ -8,6 +8,7 @@ package org.elasticsearch.compute.lucene.read; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -23,9 +24,6 @@ import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED; import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_8_19; -import static org.elasticsearch.TransportVersions.ESQL_SPLIT_ON_BIG_VALUES_9_1; public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToIteratorOperator.Status { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( @@ -34,6 +32,9 @@ public class ValuesSourceReaderOperatorStatus extends AbstractPageMappingToItera ValuesSourceReaderOperatorStatus::readFrom ); + // Will be TransportVersion.fromName("esql-split-on-big-values"); + public static final TransportVersionSet ESQL_SPLIT_ON_BIG_VALUES = TransportVersionSet.get("esql-split-on-big-values"); + private final Map readersBuilt; private final long valuesLoaded; @@ -103,9 +104,7 @@ public void writeTo(StreamOutput out) throws IOException { } private static boolean supportsSplitOnBigValues(TransportVersion version) { - return version.onOrAfter(ESQL_SPLIT_ON_BIG_VALUES) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_9_1) - || version.isPatchFrom(ESQL_SPLIT_ON_BIG_VALUES_8_19); + return ESQL_SPLIT_ON_BIG_VALUES.isCompatible(version); } private static boolean supportsValuesLoaded(TransportVersion version) { diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverCompletionInfo.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverCompletionInfo.java index 8f88f9d73b0d4..808cecbeb9c51 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverCompletionInfo.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/DriverCompletionInfo.java @@ -7,7 +7,7 @@ package org.elasticsearch.compute.operator; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -36,6 +36,8 @@ public record DriverCompletionInfo( List planProfiles ) implements Writeable { + public static final TransportVersionSet ESQL_PROFILE_INCLUDE_PLAN = TransportVersionSet.get("esql-profile-include-plan"); + /** * Completion info we use when we didn't properly complete any drivers. * Usually this is returned with an error, but it's also used when receiving @@ -94,10 +96,9 @@ public static DriverCompletionInfo readFrom(StreamInput in) throws IOException { in.readVLong(), in.readVLong(), in.readCollectionAsImmutableList(DriverProfile::readFrom), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN) - || in.getTransportVersion().isPatchFrom(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN_8_19) - ? in.readCollectionAsImmutableList(PlanProfile::readFrom) - : List.of() + ESQL_PROFILE_INCLUDE_PLAN.isCompatible(in.getTransportVersion()) + ? in.readCollectionAsImmutableList(PlanProfile::readFrom) + : List.of() ); } @@ -106,8 +107,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(documentsFound); out.writeVLong(valuesLoaded); out.writeCollection(driverProfiles); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN) - || out.getTransportVersion().isPatchFrom(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN_8_19)) { + if (ESQL_PROFILE_INCLUDE_PLAN.isCompatible(out.getTransportVersion())) { out.writeCollection(planProfiles); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java index 4afb1418b2585..57928f553c318 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlQueryResponse.java @@ -39,6 +39,7 @@ import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED; import static org.elasticsearch.TransportVersions.ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19; +import static org.elasticsearch.compute.operator.DriverCompletionInfo.ESQL_PROFILE_INCLUDE_PLAN; public class EsqlQueryResponse extends org.elasticsearch.xpack.core.esql.action.EsqlQueryResponse implements @@ -401,18 +402,16 @@ public record Profile(List drivers, List plans) impl public static Profile readFrom(StreamInput in) throws IOException { return new Profile( in.readCollectionAsImmutableList(DriverProfile::readFrom), - in.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN) - || in.getTransportVersion().isPatchFrom(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN_8_19) - ? in.readCollectionAsImmutableList(PlanProfile::readFrom) - : List.of() + ESQL_PROFILE_INCLUDE_PLAN.isCompatible(in.getTransportVersion()) + ? in.readCollectionAsImmutableList(PlanProfile::readFrom) + : List.of() ); } @Override public void writeTo(StreamOutput out) throws IOException { out.writeCollection(drivers); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN) - || out.getTransportVersion().isPatchFrom(TransportVersions.ESQL_PROFILE_INCLUDE_PLAN_8_19)) { + if (ESQL_PROFILE_INCLUDE_PLAN.isCompatible(out.getTransportVersion())) { out.writeCollection(plans); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java index 9cdf193c56fef..9fb6c4355b2d0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamWrapperQueryBuilder.java @@ -9,7 +9,6 @@ import org.apache.lucene.search.Query; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -20,6 +19,7 @@ import java.io.IOException; +import static org.elasticsearch.index.query.WildcardQueryBuilder.ESQL_FIXED_INDEX_LIKE; import static org.elasticsearch.index.query.WildcardQueryBuilder.expressionTransportSupported; /** @@ -55,7 +55,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ESQL_FIXED_INDEX_LIKE; + return ESQL_FIXED_INDEX_LIKE.local(); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java index 7a83fd800ab8e..f704b495387d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalRelation.java @@ -6,7 +6,7 @@ */ package org.elasticsearch.xpack.esql.plan.logical.local; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -29,6 +29,10 @@ public class LocalRelation extends LeafPlan { LocalRelation::new ); + public static final TransportVersionSet ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS = TransportVersionSet.get( + "esql-local-relation-with-new-blocks" + ); + private final List output; private final LocalSupplier supplier; @@ -41,7 +45,7 @@ public LocalRelation(Source source, List output, LocalSupplier suppli public LocalRelation(StreamInput in) throws IOException { super(Source.readFrom((PlanStreamInput) in)); this.output = in.readNamedWriteableCollectionAsList(Attribute.class); - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(in.getTransportVersion())) { this.supplier = in.readNamedWriteable(LocalSupplier.class); } else { this.supplier = LocalSourceExec.readLegacyLocalSupplierFrom((PlanStreamInput) in); @@ -52,7 +56,7 @@ public LocalRelation(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteableCollection(output); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(out.getTransportVersion())) { out.writeNamedWriteable(supplier); } else { if (supplier == EmptyLocalSupplier.EMPTY) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java index 5994ce813c851..2f09e588a3248 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/physical/LocalSourceExec.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.plan.physical; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -24,6 +23,8 @@ import java.util.List; import java.util.Objects; +import static org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS; + public class LocalSourceExec extends LeafExec { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry( PhysicalPlan.class, @@ -43,7 +44,7 @@ public LocalSourceExec(Source source, List output, LocalSupplier supp public LocalSourceExec(StreamInput in) throws IOException { super(Source.readFrom((PlanStreamInput) in)); this.output = in.readNamedWriteableCollectionAsList(Attribute.class); - if (in.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(in.getTransportVersion())) { this.supplier = in.readNamedWriteable(LocalSupplier.class); } else { this.supplier = readLegacyLocalSupplierFrom((PlanStreamInput) in); @@ -67,7 +68,7 @@ public static LocalSupplier readLegacyLocalSupplierFrom(PlanStreamInput in) thro public void writeTo(StreamOutput out) throws IOException { source().writeTo(out); out.writeNamedWriteableCollection(output); - if (out.getTransportVersion().onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(out.getTransportVersion())) { out.writeNamedWriteable(supplier); } else { if (supplier == EmptyLocalSupplier.EMPTY) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index e8a13848c7576..7201c7f6bc2e1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -9,7 +9,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.OriginalIndices; import org.elasticsearch.action.fieldcaps.FieldCapabilitiesFailure; @@ -131,6 +131,8 @@ public class EsqlSession { private static final Logger LOGGER = LogManager.getLogger(EsqlSession.class); + public static final TransportVersionSet LOOKUP_JOIN_CCS = TransportVersionSet.get("lookup-join-ccs"); + /** * Interface for running the underlying plan. * Abstracts away the underlying execution engine. @@ -650,7 +652,7 @@ private void validateRemoteVersions(EsqlExecutionInfo executionInfo) { if (clusterAlias.equals(RemoteClusterAware.LOCAL_CLUSTER_GROUP_KEY) == false) { // No need to check local, obviously var connection = remoteClusterService.getConnection(clusterAlias); - if (connection != null && connection.getTransportVersion().before(TransportVersions.LOOKUP_JOIN_CCS)) { + if (connection != null && connection.getTransportVersion().before(LOOKUP_JOIN_CCS.local())) { skipClusterOrError( clusterAlias, executionInfo, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/CopyingLocalSupplierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/CopyingLocalSupplierTests.java index b5503c5c6cc27..e5db17c080992 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/CopyingLocalSupplierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/CopyingLocalSupplierTests.java @@ -8,9 +8,9 @@ package org.elasticsearch.xpack.esql.plan.logical.local; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.compute.data.Block; +import static org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS; import static org.hamcrest.Matchers.equalTo; public class CopyingLocalSupplierTests extends LocalSupplierTests { @@ -23,7 +23,7 @@ protected LocalSupplier createTestInstance() { protected void assertOnBWCObject(LocalSupplier testInstance, LocalSupplier bwcDeserializedObject, TransportVersion version) { assertNotSame(version.toString(), bwcDeserializedObject, testInstance); - if (version.onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(version)) { assertThat(testInstance, equalTo(bwcDeserializedObject)); } else { assertTrue(version.toString(), bwcDeserializedObject instanceof ImmediateLocalSupplier); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EmptyLocalSupplierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EmptyLocalSupplierTests.java index c1a12e50417df..57b6b7ab92c9a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EmptyLocalSupplierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/EmptyLocalSupplierTests.java @@ -8,12 +8,12 @@ package org.elasticsearch.xpack.esql.plan.logical.local; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; import java.io.IOException; +import static org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS; import static org.hamcrest.Matchers.equalTo; public class EmptyLocalSupplierTests extends LocalSupplierTests { @@ -31,7 +31,7 @@ protected void assertOnBWCObject(LocalSupplier testInstance, LocalSupplier bwcDe @Override protected void writeTo(BytesStreamOutput output, LocalSupplier instance, TransportVersion version) throws IOException { - if (version.onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(version)) { new PlanStreamOutput(output, null).writeNamedWriteable(instance); } else { output.writeVInt(0); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalSupplierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalSupplierTests.java index 1d144b995711f..499b82ddf925c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalSupplierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/local/LocalSupplierTests.java @@ -28,6 +28,8 @@ import java.util.Arrays; import java.util.NavigableSet; +import static org.elasticsearch.xpack.esql.plan.logical.local.LocalRelation.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS; + public abstract class LocalSupplierTests extends AbstractWireTestCase { private static final NavigableSet DEFAULT_BWC_VERSIONS = getAllBWCVersions(); @@ -70,7 +72,7 @@ protected LocalSupplier copyInstance(LocalSupplier instance, TransportVersion ve } protected void writeTo(BytesStreamOutput output, LocalSupplier instance, TransportVersion version) throws IOException { - if (version.onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(version)) { new PlanStreamOutput(output, null).writeNamedWriteable(instance); } else { instance.writeTo(new PlanStreamOutput(output, null)); @@ -78,7 +80,7 @@ protected void writeTo(BytesStreamOutput output, LocalSupplier instance, Transpo } protected LocalSupplier readFrom(StreamInput input, TransportVersion version) throws IOException { - if (version.onOrAfter(TransportVersions.ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS)) { + if (ESQL_LOCAL_RELATION_WITH_NEW_BLOCKS.isCompatible(version)) { return new PlanStreamInput(input, getNamedWriteableRegistry(), null).readNamedWriteable(LocalSupplier.class); } else { return LocalSourceExec.readLegacyLocalSupplierFrom(new PlanStreamInput(input, getNamedWriteableRegistry(), null)); diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java index 07b81c22c8083..b38fff5b376ea 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettings.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; @@ -49,6 +50,8 @@ public class CohereServiceSettings extends FilteredXContentObject implements Ser public static final String API_VERSION = "api_version"; public static final String MODEL_REQUIRED_FOR_V2_API = "The [service_settings.model_id] field is required for the Cohere V2 API."; + public static final TransportVersionSet ML_INFERENCE_COHERE_API_VERSION = TransportVersionSet.get("ml-inference-cohere-api-version"); + public enum CohereApiVersion { V1, V2; @@ -182,8 +185,7 @@ public CohereServiceSettings(StreamInput in) throws IOException { } else { rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS; } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || in.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(in.getTransportVersion())) { this.apiVersion = in.readEnum(CohereServiceSettings.CohereApiVersion.class); } else { this.apiVersion = CohereServiceSettings.CohereApiVersion.V1; @@ -285,8 +287,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { rateLimitSettings.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || out.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(out.getTransportVersion())) { out.writeEnum(apiVersion); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java index 7f8ef305e5db6..497e9e05d087e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettings.java @@ -34,6 +34,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.createOptionalUri; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalString; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.API_VERSION; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.ML_INFERENCE_COHERE_API_VERSION; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.MODEL_REQUIRED_FOR_V2_API; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.apiVersionFromMap; @@ -102,8 +103,7 @@ public CohereCompletionServiceSettings(StreamInput in) throws IOException { uri = createOptionalUri(in.readOptionalString()); modelId = in.readOptionalString(); rateLimitSettings = new RateLimitSettings(in); - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || in.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(in.getTransportVersion())) { this.apiVersion = in.readEnum(CohereServiceSettings.CohereApiVersion.class); } else { this.apiVersion = CohereServiceSettings.CohereApiVersion.V1; @@ -155,8 +155,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(uriToWrite); out.writeOptionalString(modelId); rateLimitSettings.writeTo(out); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || out.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(out.getTransportVersion())) { out.writeEnum(apiVersion); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java index 651b8758c37f3..722602df9e149 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettings.java @@ -40,6 +40,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.removeAsType; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.API_VERSION; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.DEFAULT_RATE_LIMIT_SETTINGS; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.ML_INFERENCE_COHERE_API_VERSION; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.MODEL_REQUIRED_FOR_V2_API; import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.apiVersionFromMap; @@ -124,8 +125,7 @@ public CohereRerankServiceSettings(StreamInput in) throws IOException { this.rateLimitSettings = DEFAULT_RATE_LIMIT_SETTINGS; } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || in.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(in.getTransportVersion())) { this.apiVersion = in.readEnum(CohereServiceSettings.CohereApiVersion.class); } else { this.apiVersion = CohereServiceSettings.CohereApiVersion.V1; @@ -206,8 +206,7 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getTransportVersion().onOrAfter(TransportVersions.V_8_15_0)) { rateLimitSettings.writeTo(out); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - || out.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19)) { + if (ML_INFERENCE_COHERE_API_VERSION.isCompatible(out.getTransportVersion())) { out.writeEnum(apiVersion); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/CustomServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/CustomServiceSettings.java index 8b799e472d512..c8918415eb089 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/CustomServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/CustomServiceSettings.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.inference.services.custom; import org.elasticsearch.TransportVersion; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; @@ -66,6 +67,16 @@ public class CustomServiceSettings extends FilteredXContentObject implements Ser private static final String RESPONSE_SCOPE = String.join(".", ModelConfigurations.SERVICE_SETTINGS, RESPONSE); private static final int DEFAULT_EMBEDDING_BATCH_SIZE = 10; + public static final TransportVersionSet ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE = TransportVersionSet.get( + "ml-inference-custom-service-input-type" + ); + public static final TransportVersionSet ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE = TransportVersionSet.get( + "ml-inference-custom-service-embedding-type" + ); + public static final TransportVersionSet ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE = TransportVersionSet.get( + "ml-inference-custom-service-embedding-batch-size" + ); + public static CustomServiceSettings fromMap(Map map, ConfigurationParseContext context, TaskType taskType) { ValidationException validationException = new ValidationException(); @@ -169,7 +180,7 @@ public TextEmbeddingSettings(StreamInput in) throws IOException { this.dimensions = in.readOptionalVInt(); this.maxInputTokens = in.readOptionalVInt(); - if (in.getTransportVersion().before(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE)) { + if (in.getTransportVersion().before(ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE.local())) { in.readOptionalEnum(DenseVectorFieldMapper.ElementType.class); } } @@ -180,7 +191,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(dimensions); out.writeOptionalVInt(maxInputTokens); - if (out.getTransportVersion().before(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE)) { + if (out.getTransportVersion().before(ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE.local())) { out.writeOptionalEnum(null); } } @@ -285,15 +296,13 @@ public CustomServiceSettings(StreamInput in) throws IOException { in.readString(); } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE) - || in.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE_8_19)) { + if (ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE.isCompatible(in.getTransportVersion())) { batchSize = in.readVInt(); } else { batchSize = DEFAULT_EMBEDDING_BATCH_SIZE; } - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE) - || in.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE_8_19)) { + if (ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE.isCompatible(in.getTransportVersion())) { inputTypeTranslator = new InputTypeTranslator(in); } else { inputTypeTranslator = InputTypeTranslator.EMPTY_TRANSLATOR; @@ -444,13 +453,11 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(""); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE) - || out.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE_8_19)) { + if (ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_BATCH_SIZE.isCompatible(out.getTransportVersion())) { out.writeVInt(batchSize); } - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE) - || out.getTransportVersion().isPatchFrom(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE_8_19)) { + if (ML_INFERENCE_CUSTOM_SERVICE_INPUT_TYPE.isCompatible(out.getTransportVersion())) { inputTypeTranslator.writeTo(out); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParser.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParser.java index 51f7ef29be666..5ab4af5d0899c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParser.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParser.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.services.custom.response; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; @@ -30,6 +29,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalEnum; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredString; import static org.elasticsearch.xpack.inference.services.custom.CustomServiceSettings.JSON_PARSER; +import static org.elasticsearch.xpack.inference.services.custom.CustomServiceSettings.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE; public class TextEmbeddingResponseParser extends BaseCustomResponseParser { @@ -79,7 +79,7 @@ public TextEmbeddingResponseParser(String textEmbeddingsPath, CustomServiceEmbed public TextEmbeddingResponseParser(StreamInput in) throws IOException { this.textEmbeddingsPath = in.readString(); - if (in.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE)) { + if (ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE.isCompatible(in.getTransportVersion())) { this.embeddingType = in.readEnum(CustomServiceEmbeddingType.class); } else { this.embeddingType = CustomServiceEmbeddingType.FLOAT; @@ -89,7 +89,7 @@ public TextEmbeddingResponseParser(StreamInput in) throws IOException { public void writeTo(StreamOutput out) throws IOException { out.writeString(textEmbeddingsPath); - if (out.getTransportVersion().onOrAfter(TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE)) { + if (ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE.isCompatible(out.getTransportVersion())) { out.writeEnum(embeddingType); } } diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/densetextembeddings/ElasticInferenceServiceDenseTextEmbeddingsServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/densetextembeddings/ElasticInferenceServiceDenseTextEmbeddingsServiceSettings.java index e8eeee5a34dd4..9c6b84df604ff 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/densetextembeddings/ElasticInferenceServiceDenseTextEmbeddingsServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/elastic/densetextembeddings/ElasticInferenceServiceDenseTextEmbeddingsServiceSettings.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.services.elastic.densetextembeddings; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -45,6 +45,10 @@ public class ElasticInferenceServiceDenseTextEmbeddingsServiceSettings extends F public static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(10_000); + public static final TransportVersionSet ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED = TransportVersionSet.get( + "ml-inference-elastic-dense-text-embeddings-added" + ); + private final String modelId; private final SimilarityMeasure similarity; private final Integer dimensions; @@ -206,13 +210,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public TransportVersion getMinimalSupportedVersion() { assert false : "should never be called when supportsVersion is used"; - return TransportVersions.ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED; + return ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED.local(); } @Override public boolean supportsVersion(TransportVersion version) { - return version.onOrAfter(TransportVersions.ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED) - || version.isPatchFrom(TransportVersions.ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED_8_19); + return ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED.isCompatible(version); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/completion/IbmWatsonxChatCompletionServiceSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/completion/IbmWatsonxChatCompletionServiceSettings.java index 0492a626787cf..00ebd7d8b8e4c 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/completion/IbmWatsonxChatCompletionServiceSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/ibmwatsonx/completion/IbmWatsonxChatCompletionServiceSettings.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.services.ibmwatsonx.completion; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -48,6 +48,10 @@ public class IbmWatsonxChatCompletionServiceSettings extends FilteredXContentObj */ private static final RateLimitSettings DEFAULT_RATE_LIMIT_SETTINGS = new RateLimitSettings(120); + public static final TransportVersionSet ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED = TransportVersionSet.get( + "ml-inference-ibm-watsonx-completion-added" + ); + public static IbmWatsonxChatCompletionServiceSettings fromMap(Map map, ConfigurationParseContext context) { ValidationException validationException = new ValidationException(); @@ -160,7 +164,7 @@ protected XContentBuilder toXContentFragmentOfExposedFields(XContentBuilder buil @Override public TransportVersion getMinimalSupportedVersion() { - return TransportVersions.ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED; + return ML_INFERENCE_IBM_WATSONX_COMPLETION_ADDED.local(); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java index 6e1407beab1d8..4dda4f14d5af8 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/ElasticTextEmbeddingPayload.java @@ -11,7 +11,6 @@ import software.amazon.awssdk.services.sagemakerruntime.model.InvokeEndpointResponse; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -45,6 +44,7 @@ import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractOptionalPositiveInteger; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractRequiredEnum; import static org.elasticsearch.xpack.inference.services.ServiceUtils.extractSimilarity; +import static org.elasticsearch.xpack.inference.services.sagemaker.schema.elastic.SageMakerElasticTaskSettings.ML_INFERENCE_SAGEMAKER_ELASTIC; /** * TextEmbedding needs to differentiate between Bit, Byte, and Float types. Users must specify the @@ -251,13 +251,12 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { assert false : "should never be called when supportsVersion is used"; - return TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC; + return ML_INFERENCE_SAGEMAKER_ELASTIC.local(); } @Override public boolean supportsVersion(TransportVersion version) { - return version.onOrAfter(TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC) - || version.isPatchFrom(TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC_8_19); + return ML_INFERENCE_SAGEMAKER_ELASTIC.isCompatible(version); } @Override diff --git a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/SageMakerElasticTaskSettings.java b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/SageMakerElasticTaskSettings.java index 088de2068741c..e268f8424436e 100644 --- a/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/SageMakerElasticTaskSettings.java +++ b/x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/services/sagemaker/schema/elastic/SageMakerElasticTaskSettings.java @@ -8,7 +8,7 @@ package org.elasticsearch.xpack.inference.services.sagemaker.schema.elastic; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; +import org.elasticsearch.TransportVersionSet; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Nullable; @@ -25,6 +25,8 @@ record SageMakerElasticTaskSettings(@Nullable Map passthroughSettings) implements SageMakerStoredTaskSchema { static final String NAME = "sagemaker_elastic_task_settings"; + public static final TransportVersionSet ML_INFERENCE_SAGEMAKER_ELASTIC = TransportVersionSet.get("ml_inference_sagemaker_elastic"); + static SageMakerElasticTaskSettings empty() { return new SageMakerElasticTaskSettings(Map.of()); } @@ -51,13 +53,12 @@ public String getWriteableName() { @Override public TransportVersion getMinimalSupportedVersion() { assert false : "should never be called when supportsVersion is used"; - return TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC; + return ML_INFERENCE_SAGEMAKER_ELASTIC.local(); } @Override public boolean supportsVersion(TransportVersion version) { - return version.onOrAfter(TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC) - || version.isPatchFrom(TransportVersions.ML_INFERENCE_SAGEMAKER_ELASTIC_8_19); + return ML_INFERENCE_SAGEMAKER_ELASTIC.isCompatible(version); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettingsTests.java index cac416fd454a8..96c78ef771325 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/CohereServiceSettingsTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.inference.services.cohere; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.io.stream.Writeable; @@ -29,6 +28,7 @@ import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.ML_INFERENCE_COHERE_API_VERSION; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.is; @@ -364,8 +364,7 @@ public static Map getServiceSettingsMap(@Nullable String url, @N @Override protected CohereServiceSettings mutateInstanceForVersion(CohereServiceSettings instance, TransportVersion version) { - if (version.before(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - && (version.isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19) == false)) { + if (ML_INFERENCE_COHERE_API_VERSION.isNotCompatible(version)) { return new CohereServiceSettings( instance.uri(), instance.similarity(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java index 92ebb3fdc0a00..a7a01b3f973d3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/completion/CohereCompletionServiceSettingsTests.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.inference.services.cohere.completion; import org.elasticsearch.TransportVersion; -import org.elasticsearch.TransportVersions; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xcontent.XContentBuilder; @@ -25,6 +24,7 @@ import java.util.HashMap; import java.util.Map; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.ML_INFERENCE_COHERE_API_VERSION; import static org.hamcrest.Matchers.is; public class CohereCompletionServiceSettingsTests extends AbstractBWCWireSerializationTestCase { @@ -115,8 +115,7 @@ protected CohereCompletionServiceSettings mutateInstance(CohereCompletionService @Override protected CohereCompletionServiceSettings mutateInstanceForVersion(CohereCompletionServiceSettings instance, TransportVersion version) { - if (version.before(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - && (version.isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19) == false)) { + if (ML_INFERENCE_COHERE_API_VERSION.isNotCompatible(version)) { return new CohereCompletionServiceSettings( instance.uri(), instance.modelId(), diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java index 773ccc5933aa7..471b053473a07 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/cohere/rerank/CohereRerankServiceSettingsTests.java @@ -26,6 +26,7 @@ import java.util.Map; import static org.elasticsearch.xpack.inference.MatchersUtils.equalToIgnoringWhitespaceInJsonString; +import static org.elasticsearch.xpack.inference.services.cohere.CohereServiceSettings.ML_INFERENCE_COHERE_API_VERSION; public class CohereRerankServiceSettingsTests extends AbstractBWCWireSerializationTestCase { public static CohereRerankServiceSettings createRandom() { @@ -88,15 +89,14 @@ protected CohereRerankServiceSettings mutateInstanceForVersion(CohereRerankServi CohereServiceSettings.DEFAULT_RATE_LIMIT_SETTINGS, CohereServiceSettings.CohereApiVersion.V1 ); - } else if (version.before(TransportVersions.ML_INFERENCE_COHERE_API_VERSION) - && version.isPatchFrom(TransportVersions.ML_INFERENCE_COHERE_API_VERSION_8_19) == false) { - return new CohereRerankServiceSettings( - instance.uri(), - instance.modelId(), - instance.rateLimitSettings(), - CohereServiceSettings.CohereApiVersion.V1 - ); - } + } else if (ML_INFERENCE_COHERE_API_VERSION.isNotCompatible(version)) { + return new CohereRerankServiceSettings( + instance.uri(), + instance.modelId(), + instance.rateLimitSettings(), + CohereServiceSettings.CohereApiVersion.V1 + ); + } return instance; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParserTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParserTests.java index 6bb6da009e27c..aca942e9d82d7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParserTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/services/custom/response/TextEmbeddingResponseParserTests.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.TransportVersions.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE; +import static org.elasticsearch.xpack.inference.services.custom.CustomServiceSettings.ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE; import static org.elasticsearch.xpack.inference.services.custom.response.TextEmbeddingResponseParser.EMBEDDING_TYPE; import static org.elasticsearch.xpack.inference.services.custom.response.TextEmbeddingResponseParser.TEXT_EMBEDDING_PARSER_EMBEDDINGS; import static org.hamcrest.Matchers.is; @@ -317,7 +317,7 @@ public void testParse_ThrowsException_WhenExtractedField_IsNotAList() { @Override protected TextEmbeddingResponseParser mutateInstanceForVersion(TextEmbeddingResponseParser instance, TransportVersion version) { - if (version.before(ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE)) { + if (version.before(ML_INFERENCE_CUSTOM_SERVICE_EMBEDDING_TYPE.local())) { return new TextEmbeddingResponseParser(instance.getTextEmbeddingsPath(), CustomServiceEmbeddingType.FLOAT); } return instance;