[8.18][ML] Disable machine learning on macOS x86_64 (#104125) (#117737)

As previously advised in #104087, machine learning functionality will no longer be available on macOS x86_64.

Machine learning functionality is still available on macOS by using an arm64 machine (Apple silicon). It is also possible to run Elasticsearch with machine learning functionality within a Docker container on macOS x86_64.

This PR should be merged to main after the branch is split for the last minor release scheduled for before December 2024. For example, suppose 8.17.0 is scheduled for release in November 2024 and 8.18.0 is scheduled for release in January 2025. Then this PR should be merged to main after the 8.17 branch is split.

One this PR is merged a followup PR should be opened against the ml-cpp repo to remove the build system for darwin-x86_64. It has been confirmed that with this change in place the Elasticsearch build system works with an ml-cpp bundle that does not contain a platform/darwin-x86_64 directory. It still produces an Elasticsearch build that will run providing xpack.ml.enabled is not explicitly set to true.

After the build system for darwin-x86_64 has been removed from the ml-cpp repo, we will be able to do another PyTorch upgrade without having to worry about tweaking the build system to work on Intel macOS.

Backports #104125
---------

Co-authored-by: David Roberts <[email protected]>
Co-authored-by: Valeriy Khakhutskyy <[email protected]>

Author: 3 people

docs/changelog/104125.yaml

@@ -0,0 +1,18 @@
+pr: 104125
+summary: Disable machine learning on macOS x86_64
+area: Machine Learning
+type: breaking
+issues: []
+breaking:
+  title: Disable machine learning on macOS x86_64
+  area: Packaging
+  details: The machine learning plugin is permanently disabled on macOS x86_64.
+    For the last three years Apple has been selling hardware based on the arm64
+    architecture, and support will increasingly focus on this architecture in
+    the future. Changes to upstream dependencies of Elastic's machine learning
+    functionality have made it unviable for Elastic to continue to build machine
+    learning on macOS x86_64.
+  impact: To continue to use machine learning functionality on macOS please switch to
+    an arm64 machine (Apple silicon). Alternatively, it will still be possible to run
+    Elasticsearch with machine learning enabled in a Docker container on macOS x86_64.
+  notable: false

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -7,12 +7,16 @@
 
 package org.elasticsearch.xpack.core;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
 import org.elasticsearch.common.ssl.SslClientAuthenticationMode;
 import org.elasticsearch.common.ssl.SslVerificationMode;
 import org.elasticsearch.core.Strings;
+import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.transport.RemoteClusterPortSettings;
 import org.elasticsearch.xpack.core.security.SecurityField;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
@@ -26,6 +30,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Function;
 
 import javax.crypto.SecretKeyFactory;
@@ -40,6 +45,8 @@
  */
 public class XPackSettings {
 
+    private static final Logger logger = LogManager.getLogger(XPackSettings.class);
+
     private XPackSettings() {
         throw new IllegalStateException("Utility class should not be instantiated");
     }
@@ -76,10 +83,21 @@ public Iterator<Setting<?>> settings() {
     /** Setting for enabling or disabling graph. Defaults to true. */
     public static final Setting<Boolean> GRAPH_ENABLED = Setting.boolSetting("xpack.graph.enabled", true, Setting.Property.NodeScope);
 
-    /** Setting for enabling or disabling machine learning. Defaults to true. */
+    public static final Set<String> ML_NATIVE_CODE_PLATFORMS = Set.of("darwin-aarch64", "linux-aarch64", "linux-x86_64", "windows-x86_64");
+
+    /** Setting for enabling or disabling machine learning. Defaults to true on platforms that have the ML native code available. */
     public static final Setting<Boolean> MACHINE_LEARNING_ENABLED = Setting.boolSetting(
         "xpack.ml.enabled",
-        true,
+        ML_NATIVE_CODE_PLATFORMS.contains(Platforms.PLATFORM_NAME),
+        enabled -> {
+            if (enabled && ML_NATIVE_CODE_PLATFORMS.contains(Platforms.PLATFORM_NAME) == false) {
+                SettingsException e = new SettingsException("xpack.ml.enabled cannot be set to [true] on [{}]", Platforms.PLATFORM_NAME);
+                // The exception doesn't get logged nicely on the console because it's thrown during initial plugin loading,
+                // so log separately here to make absolutely clear what happened
+                logger.fatal(e.getMessage());
+                throw e;
+            }
+        },
         Setting.Property.NodeScope
     );
 

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -32,7 +32,6 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.breaker.CircuitBreaker;
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
-import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
@@ -69,7 +68,6 @@
 import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.PersistentTaskPlugin;
-import org.elasticsearch.plugins.Platforms;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.SearchPlugin;
 import org.elasticsearch.plugins.ShutdownAwarePlugin;
@@ -931,15 +929,6 @@ public Collection<?> createComponents(PluginServices services) {
             return List.of(new JobManagerHolder(), new MachineLearningExtensionHolder());
         }
 
-        if ("darwin-x86_64".equals(Platforms.PLATFORM_NAME)) {
-            String msg = "The machine learning plugin will be permanently disabled on macOS x86_64 in new minor versions released "
-                + "from December 2024 onwards. To continue to use machine learning functionality on macOS please switch to an arm64 "
-                + "machine (Apple silicon). Alternatively, it will still be possible to run Elasticsearch with machine learning "
-                + "enabled in a Docker container on macOS x86_64.";
-            logger.warn(msg);
-            deprecationLogger.warn(DeprecationCategory.PLUGINS, "ml-darwin-x86_64", msg);
-        }
-
         machineLearningExtension.get().configure(environment.settings());
 
         this.mlUpgradeModeActionFilter.set(new MlUpgradeModeActionFilter(clusterService));