[7.11] require master node for watcher template (#69859)

Backporting #69147 to 7.11 branch.

This commit requires Watcher to install composable index templates
only from the master node since during a rolling upgrade it could
try to talk to a master node that predates composable index templates.

Some tests have also been adjusted to that allows for this change.
Those test changes have been discussed offline and can generally be
summarized as the things they are testing is an edge case where if that
edge case (breaking changes to the mappings of watcher history index) between
versions then there would be other concerns beyond the need for which
node installs the watcher templates.

closes #66837

Co-authored-by: Martijn van Groningen <[email protected]>
Co-authored-by: Jake Landis <[email protected]>

Author: martijnvg

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistry.java

@@ -22,6 +22,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Stream;
 
 import static org.elasticsearch.xpack.core.ClientHelper.WATCHER_ORIGIN;
 
@@ -135,22 +136,23 @@ protected String getOrigin() {
     }
 
     public static boolean validate(ClusterState state) {
-        if(state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){
-            return (state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME) ||
-                state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM)) &&
-                state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME) &&
-                state.getMetadata().templatesV2().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME);
-        } else if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_7_0)) {
-            return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_11) ||
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_11)) &&
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) &&
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11);
+        final Stream<String> watcherHistoryTemplateIds;
+        if (state.nodes().getMinNodeVersion().onOrAfter(Version.V_7_9_0)){
+            watcherHistoryTemplateIds = state.getMetadata().templatesV2().keySet().stream();
         } else {
-            return (state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10) ||
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_NO_ILM_10)) &&
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME_11) &&
-                state.getMetadata().getTemplates().containsKey(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME_11);
+            watcherHistoryTemplateIds = Arrays.stream(state.getMetadata().getTemplates().keys().toArray(String.class));
         }
+        return watcherHistoryTemplateIds.filter(s -> s.startsWith(".watch-history-"))
+            .map(s -> Integer.valueOf(s.substring(s.lastIndexOf('-') + 1)))
+            .anyMatch(version -> version >= 9);
     }
 
+
+    @Override
+    protected boolean requiresMasterNode() {
+        // These installs a composable index template which is only supported in early versions of 7.x
+        // In mixed cluster without this set to true can result in errors in the logs during rolling upgrades.
+        // If these template(s) are only installed via elected master node then composable templates are available.
+        return true;
+    }
 }

x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/support/WatcherIndexTemplateRegistryTests.java

@@ -57,7 +57,6 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.elasticsearch.mock.orig.Mockito.verify;
@@ -70,7 +69,6 @@
 import static org.mockito.Matchers.anyObject;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.spy;
@@ -256,64 +254,6 @@ public void testThatTemplatesExist() {
         }
     }
 
-    // if a node is newer than the master node, the template needs to be applied as well
-    // otherwise a rolling upgrade would not work as expected, when the node has a .watches shard on it
-    public void testThatTemplatesAreAppliedOnNewerNodes() {
-        DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
-        DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
-        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("master").add(localNode).add(masterNode).build();
-
-        Map<String, Integer> existingTemplates = new HashMap<>();
-        existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
-        existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
-        existingTemplates.put(".watch-history-6", 6);
-        ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
-        registry.clusterChanged(event);
-
-        ArgumentCaptor<PutComposableIndexTemplateAction.Request> argumentCaptor =
-            ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class);
-        verify(client, times(3)).execute(same(PutComposableIndexTemplateAction.INSTANCE), argumentCaptor.capture(), anyObject());
-        assertTrue(argumentCaptor.getAllValues().stream()
-            .anyMatch(r -> r.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME)));
-    }
-
-    public void testThatTemplatesWithHiddenAreAppliedOnNewerNodes() {
-        DiscoveryNode node = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
-        DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.V_6_0_0);
-        DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(node).add(masterNode).build();
-
-        Map<String, Integer> existingTemplates = new HashMap<>();
-        existingTemplates.put(WatcherIndexTemplateRegistryField.TRIGGERED_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
-        existingTemplates.put(WatcherIndexTemplateRegistryField.WATCHES_TEMPLATE_NAME, INDEX_TEMPLATE_VERSION);
-        existingTemplates.put(".watch-history-6", 6);
-        ClusterChangedEvent event = createClusterChangedEvent(existingTemplates, nodes);
-        registry.clusterChanged(event);
-
-        ArgumentCaptor<PutIndexTemplateRequest> argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
-        verify(client.admin().indices(), atLeastOnce()).putTemplate(argumentCaptor.capture(), anyObject());
-        assertTrue(argumentCaptor.getAllValues().stream()
-            .anyMatch(i -> i.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)));
-
-        existingTemplates.remove(".watch-history-6");
-        existingTemplates.put(".watch-history-10", 10);
-        masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
-        nodes = DiscoveryNodes.builder().localNodeId("master").masterNodeId("master").add(masterNode).add(node).build();
-        event = createClusterChangedEvent(existingTemplates, nodes);
-        registry.clusterChanged(event);
-
-        argumentCaptor = ArgumentCaptor.forClass(PutIndexTemplateRequest.class);
-        verify(client.admin().indices(), atLeastOnce()).putTemplate(argumentCaptor.capture(), anyObject());
-        assertTrue(argumentCaptor.getAllValues().stream()
-            .anyMatch(i -> i.name().equals(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10)));
-        ArgumentCaptor<PutComposableIndexTemplateAction.Request> captor =
-            ArgumentCaptor.forClass(PutComposableIndexTemplateAction.Request.class);
-        verify(client, atLeastOnce()).execute(same(PutComposableIndexTemplateAction.INSTANCE), captor.capture(), anyObject());
-        Set<String> templateNames =
-            captor.getAllValues().stream().map(PutComposableIndexTemplateAction.Request::name).collect(Collectors.toSet());
-        assertTrue(templateNames.contains(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME));
-        assertFalse(templateNames.contains(WatcherIndexTemplateRegistryField.HISTORY_TEMPLATE_NAME_10));
-    }
-
     public void testThatTemplatesAreNotAppliedOnSameVersionNodes() {
         DiscoveryNode localNode = new DiscoveryNode("node", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);
         DiscoveryNode masterNode = new DiscoveryNode("master", ESTestCase.buildNewFakeTransportAddress(), Version.CURRENT);

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/WatcherRestartIT.java

@@ -39,7 +39,9 @@ public void testWatcherRestart() throws Exception {
         client().performRequest(new Request("POST", "/_watcher/_start"));
         ensureWatcherStarted();
 
-        validateHistoryTemplate();
+        if (CLUSTER_TYPE.equals(ClusterType.UPGRADED)) {
+            validateHistoryTemplate();
+        }
     }
 
     private void validateHistoryTemplate() throws Exception {