[ML] Remove the cluster state listener when the adaptive allocations service stops (#113524) (#113530)

Author: davidkyle

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/adaptiveallocations/AdaptiveAllocationsScalerService.java

@@ -259,12 +259,17 @@ public synchronized void start() {
     }
 
     public synchronized void stop() {
+        clusterService.removeListener(this);
         stopScheduling();
         metrics.close();
     }
 
     @Override
     public void clusterChanged(ClusterChangedEvent event) {
+        if (event.metadataChanged() == false) {
+            return;
+        }
+
         updateAutoscalers(event.state());
         if (scalers.isEmpty() == false) {
             startScheduling();