Merge branch 'main' into distribution-cc-fixes

Author: breskeby

docs/changelog/121106.yaml

@@ -0,0 +1,5 @@
+pr: 121106
+summary: Add `ModelRegistryMetadata` to Cluster State
+area: Machine Learning
+type: enhancement
+issues: []

libs/entitlement/src/main/java/org/elasticsearch/entitlement/runtime/policy/PolicyUtils.java

@@ -49,29 +49,34 @@ public record PluginData(Path pluginPath, boolean isModular, boolean isExternalP
 
     private static final String POLICY_FILE_NAME = "entitlement-policy.yaml";
 
-    public static Map<String, Policy> createPluginPolicies(Collection<PluginData> pluginData, Map<String, String> overrides, String version)
-        throws IOException {
+    public static Map<String, Policy> createPluginPolicies(
+        Collection<PluginData> pluginData,
+        Map<String, String> pluginPolicyPatches,
+        String version
+    ) throws IOException {
         Map<String, Policy> pluginPolicies = new HashMap<>(pluginData.size());
         for (var entry : pluginData) {
             Path pluginRoot = entry.pluginPath();
+            Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
             String pluginName = pluginRoot.getFileName().toString();
             final Set<String> moduleNames = getModuleNames(pluginRoot, entry.isModular());
 
-            var overriddenPolicy = parseEncodedPolicyIfExists(
-                overrides.get(pluginName),
+            var pluginPolicyPatch = parseEncodedPolicyIfExists(
+                pluginPolicyPatches.get(pluginName),
                 version,
                 entry.isExternalPlugin(),
                 pluginName,
                 moduleNames
             );
-            if (overriddenPolicy != null) {
-                pluginPolicies.put(pluginName, overriddenPolicy);
-            } else {
-                Path policyFile = pluginRoot.resolve(POLICY_FILE_NAME);
-                var policy = parsePolicyIfExists(pluginName, policyFile, entry.isExternalPlugin());
-                validatePolicyScopes(pluginName, policy, moduleNames, policyFile.toString());
-                pluginPolicies.put(pluginName, policy);
-            }
+            var pluginPolicy = parsePolicyIfExists(pluginName, policyFile, entry.isExternalPlugin());
+            validatePolicyScopes(pluginName, pluginPolicy, moduleNames, policyFile.toString());
+
+            pluginPolicies.put(
+                pluginName,
+                pluginPolicyPatch == null
+                    ? pluginPolicy
+                    : new Policy(pluginPolicy.name(), PolicyUtils.mergeScopes(pluginPolicy.scopes(), pluginPolicyPatch.scopes()))
+            );
         }
         return pluginPolicies;
     }

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -187,6 +187,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion ML_INFERENCE_DEEPSEEK = def(9_029_00_0);
     public static final TransportVersion ESQL_FAILURE_FROM_REMOTE = def(9_030_00_0);
     public static final TransportVersion INDEX_RESHARDING_METADATA = def(9_031_0_00);
+    public static final TransportVersion INFERENCE_MODEL_REGISTRY_METADATA = def(9_032_0_00);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -86,8 +86,8 @@
  */
 class Elasticsearch {
 
-    private static final String PLUGIN_POLICY_OVERRIDE_PREFIX = "es.entitlements.policy.";
-    private static final String SERVER_POLICY_OVERRIDE = "es.entitlements.server_policy";
+    private static final String POLICY_PATCH_PREFIX = "es.entitlements.policy.";
+    private static final String SERVER_POLICY_PATCH_NAME = POLICY_PATCH_PREFIX + "server";
 
     /**
      * Main entry point for starting elasticsearch.
@@ -253,10 +253,10 @@ private static void initPhase2(Bootstrap bootstrap) throws IOException {
                     .map(bundle -> new PolicyUtils.PluginData(bundle.getDir(), bundle.pluginDescriptor().isModular(), true))
             ).toList();
 
-            var pluginPolicyOverrides = collectPluginPolicyOverrides(modulesBundles, pluginsBundles, logger);
-            var pluginPolicies = PolicyUtils.createPluginPolicies(pluginData, pluginPolicyOverrides, Build.current().version());
+            var pluginPolicyPatches = collectPluginPolicyPatches(modulesBundles, pluginsBundles, logger);
+            var pluginPolicies = PolicyUtils.createPluginPolicies(pluginData, pluginPolicyPatches, Build.current().version());
             var serverPolicyPatch = PolicyUtils.parseEncodedPolicyIfExists(
-                System.getProperty(SERVER_POLICY_OVERRIDE),
+                System.getProperty(SERVER_POLICY_PATCH_NAME),
                 Build.current().version(),
                 false,
                 "server",
@@ -331,33 +331,36 @@ private static void logSystemInfo() {
         }
     }
 
-    private static Map<String, String> collectPluginPolicyOverrides(
+    private static Map<String, String> collectPluginPolicyPatches(
         Set<PluginBundle> modulesBundles,
         Set<PluginBundle> pluginsBundles,
         Logger logger
     ) {
-        var policyOverrides = new HashMap<String, String>();
+        var policyPatches = new HashMap<String, String>();
         var systemProperties = BootstrapInfo.getSystemProperties();
         systemProperties.keys().asIterator().forEachRemaining(key -> {
             var value = systemProperties.get(key);
-            if (key instanceof String k && k.startsWith(PLUGIN_POLICY_OVERRIDE_PREFIX) && value instanceof String v) {
-                policyOverrides.put(k.substring(PLUGIN_POLICY_OVERRIDE_PREFIX.length()), v);
+            if (key instanceof String k
+                && value instanceof String v
+                && k.startsWith(POLICY_PATCH_PREFIX)
+                && k.equals(SERVER_POLICY_PATCH_NAME) == false) {
+                policyPatches.put(k.substring(POLICY_PATCH_PREFIX.length()), v);
             }
         });
         var pluginNames = Stream.concat(modulesBundles.stream(), pluginsBundles.stream())
             .map(bundle -> bundle.pluginDescriptor().getName())
             .collect(Collectors.toUnmodifiableSet());
 
-        for (var overriddenPluginName : policyOverrides.keySet()) {
-            if (pluginNames.contains(overriddenPluginName) == false) {
+        for (var patchedPluginName : policyPatches.keySet()) {
+            if (pluginNames.contains(patchedPluginName) == false) {
                 logger.warn(
-                    "Found command-line override for unknown plugin [{}] (available plugins: [{}])",
-                    overriddenPluginName,
+                    "Found command-line policy patch for unknown plugin [{}] (available plugins: [{}])",
+                    patchedPluginName,
                     String.join(", ", pluginNames)
                 );
             }
         }
-        return policyOverrides;
+        return policyPatches;
     }
 
     private static class EntitlementSelfTester {

server/src/main/java/org/elasticsearch/common/ReferenceDocs.java

@@ -11,7 +11,6 @@
 
 import org.elasticsearch.Build;
 import org.elasticsearch.core.SuppressForbidden;
-import org.elasticsearch.core.UpdateForV9;
 
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
@@ -28,7 +27,6 @@
  * {@link #toString()} yields (a string representation of) a URL for the relevant docs. Links are defined in the resource file
  * {@code reference-docs-links.txt} which must include definitions for exactly the set of values of this enum.
  */
-@UpdateForV9(owner = UpdateForV9.Owner.DOCS) // the docs are completely different in v9 so these links all need fixing
 public enum ReferenceDocs {
     /*
      * Note that the docs subsystem parses {@code reference-docs-links.txt} differently. See {@code sub check_elasticsearch_links} in
@@ -89,7 +87,22 @@ public enum ReferenceDocs {
     // this comment keeps the ';' on the next line so every entry above has a trailing ',' which makes the diff for adding new links cleaner
     ;
 
-    private static final Map<String, String> linksBySymbol;
+    private static final Map<String, LinkComponents> linksBySymbol;
+
+    record LinkComponents(String path, String fragment) {
+        static LinkComponents ofLink(String link) {
+            if (link.indexOf('?') != -1) {
+                throw new IllegalStateException("ReferenceDocs does not support links containing pre-existing query parameters: " + link);
+            }
+
+            final var fragmentIndex = link.indexOf('#');
+            if (fragmentIndex == -1) {
+                return new LinkComponents(link, "");
+            } else {
+                return new LinkComponents(link.substring(0, fragmentIndex), link.substring(fragmentIndex));
+            }
+        }
+    }
 
     static {
         try (var resourceStream = readFromJarResourceUrl(ReferenceDocs.class.getResource("reference-docs-links.txt"))) {
@@ -106,30 +119,17 @@ public enum ReferenceDocs {
 
     static final int SYMBOL_COLUMN_WIDTH = 64; // increase as needed to accommodate yet longer symbols
 
-    static Map<String, String> readLinksBySymbol(InputStream inputStream) throws IOException {
-        final var padding = " ".repeat(SYMBOL_COLUMN_WIDTH);
-
-        record LinksBySymbolEntry(String symbol, String link) implements Map.Entry<String, String> {
-            @Override
-            public String getKey() {
-                return symbol;
-            }
-
-            @Override
-            public String getValue() {
-                return link;
-            }
-
-            @Override
-            public String setValue(String value) {
-                assert false;
-                throw new UnsupportedOperationException();
-            }
-        }
-
-        final var symbolCount = values().length;
-        final var linksBySymbolEntries = new LinksBySymbolEntry[symbolCount];
+    // exposed for tests
+    @SuppressWarnings({ "unchecked", "rawtypes" })
+    static Map<String, LinkComponents> readLinksBySymbol(InputStream inputStream) throws IOException {
+        final var linksBySymbolEntries = new Map.Entry[values().length];
+        createLinkComponentEntries(inputStream, linksBySymbolEntries);
+        return Map.ofEntries(linksBySymbolEntries);
+    }
 
+    private static void createLinkComponentEntries(InputStream inputStream, Map.Entry<?, ?>[] linksBySymbolEntries) throws IOException {
+        final var padding = " ".repeat(SYMBOL_COLUMN_WIDTH);
+        final var symbolCount = linksBySymbolEntries.length;
         try (var reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) {
             for (int i = 0; i < symbolCount; i++) {
                 final var currentLine = reader.readLine();
@@ -158,15 +158,13 @@ public String setValue(String value) {
                         "found auto-generated fragment ID in link [" + link + "] for [" + symbol + "] at line " + (i + 1)
                     );
                 }
-                linksBySymbolEntries[i] = new LinksBySymbolEntry(symbol, link);
+                linksBySymbolEntries[i] = Map.entry(symbol, LinkComponents.ofLink(link));
             }
 
             if (reader.readLine() != null) {
                 throw new IllegalStateException("unexpected trailing content at line " + (symbolCount + 1));
             }
         }
-
-        return Map.ofEntries(linksBySymbolEntries);
     }
 
     /**
@@ -196,13 +194,14 @@ static String getVersionComponent(String version, boolean isSnapshot) {
             return UNRELEASED_VERSION_COMPONENT;
         }
         // Non-semantic, released version -> point to latest information (current release documentation, e.g.
-        // https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-plugins.html)
+        // https://www.elastic.co/docs/modules-plugins?version=current)
         return CURRENT_VERSION_COMPONENT;
     }
 
     @Override
     public String toString() {
-        return "https://www.elastic.co/guide/en/elasticsearch/reference/" + VERSION_COMPONENT + "/" + linksBySymbol.get(name());
+        final var linkComponents = linksBySymbol.get(name());
+        return "https://www.elastic.co/docs/" + linkComponents.path() + "?version=" + VERSION_COMPONENT + linkComponents.fragment();
     }
 
     @SuppressForbidden(reason = "reads resource from jar")