SOLR-17620: SolrCloud "liveNode" now has version & roles (#3305)

And added ZkStateReader.fetchLowestSolrVersion for use in gating cluster options.

---------

Co-authored-by: yqu63 <[email protected]>
Co-authored-by: David Smiley <[email protected]>

(cherry picked from commit 52b5d10)

Author: QFREEE

solr/CHANGES.txt

@@ -38,6 +38,8 @@ Other Changes
 ---------------------
 (No changes)
 
+* SOLR-17620: SolrCloud "live_node" now has metadata: version of Solr, roles (Yuntong Qu, David Smiley)
+
 ==================  9.9.1 ==================
 Bug Fixes
 ---------------------

solr/core/src/java/org/apache/solr/cloud/ZkController.java

@@ -21,6 +21,9 @@
 import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.CORE_NODE_NAME_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.ELECTION_NODE_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_NODE_NAME;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_ROLES;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_SOLR_VERSION;
 import static org.apache.solr.common.cloud.ZkStateReader.NODE_NAME_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.REJOIN_AT_HEAD_PROP;
 import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
@@ -39,6 +42,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
@@ -55,6 +59,7 @@
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
+import org.apache.solr.client.api.util.SolrVersion;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.client.solrj.cloud.SolrCloudManager;
 import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
@@ -1166,13 +1171,15 @@ private void createEphemeralLiveNode() throws KeeperException, InterruptedExcept
 
     String nodeName = getNodeName();
     String nodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
-    log.info("Register node as live in ZooKeeper:{}", nodePath);
+    log.info("Register node as live in ZooKeeper: {}", nodePath);
     Map<NodeRoles.Role, String> roles = cc.nodeRoles.getRoles();
+
     List<Op> ops = new ArrayList<>(roles.size() + 1);
+
     ops.add(
         Op.create(
             nodePath,
-            null,
+            buildLiveNodeData(),
             zkClient.getZkACLProvider().getACLsToAdd(nodePath),
             CreateMode.EPHEMERAL));
 
@@ -1189,6 +1196,17 @@ private void createEphemeralLiveNode() throws KeeperException, InterruptedExcept
     zkClient.multi(ops, true);
   }
 
+  private byte[] buildLiveNodeData() {
+    Map<String, Object> props = new LinkedHashMap<>();
+    props.put(LIVE_NODE_SOLR_VERSION, SolrVersion.LATEST.toString());
+    props.put(LIVE_NODE_NODE_NAME, getNodeName());
+
+    Map<NodeRoles.Role, String> roles = cc.nodeRoles.getRoles();
+    props.put(LIVE_NODE_ROLES, roles);
+
+    return Utils.toJSON(props);
+  }
+
   public void removeEphemeralLiveNode() throws KeeperException, InterruptedException {
     if (zkRunOnly) {
       return;

solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java

@@ -17,6 +17,8 @@
 package org.apache.solr.cloud;
 
 import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_NODE_NAME;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_SOLR_VERSION;
 import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
 import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
 import static org.mockito.Mockito.mock;
@@ -36,6 +38,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.api.util.SolrVersion;
 import org.apache.solr.client.solrj.impl.Http2SolrClient;
 import org.apache.solr.common.MapWriter;
 import org.apache.solr.common.cloud.ClusterProperties;
@@ -215,6 +218,40 @@ public void testGetHostName() throws Exception {
     }
   }
 
+  @LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG")
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testLiveNodeDataStored() throws Exception {
+    Path zkDir = createTempDir("testLiveNodeDataStored");
+    ZkTestServer server = new ZkTestServer(zkDir);
+    try {
+      server.run();
+      CoreContainer cc = getCoreContainer();
+      CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983).build();
+
+      ZkController zkController = new ZkController(cc, server.getZkAddress(), 10000, cloudConfig);
+
+      String nodeName = zkController.getNodeName();
+      String liveNodePath = ZkStateReader.LIVE_NODES_ZKNODE + "/" + nodeName;
+      SolrZkClient zkClient = zkController.getZkClient();
+      byte[] actualData = zkClient.getData(liveNodePath, null, null, true);
+
+      Map<String, Object> liveProps = (Map<String, Object>) Utils.fromJSON(actualData);
+      String expectedSolrVersion = SolrVersion.LATEST.toString();
+
+      assertEquals(
+          "Live node solrVersion incorrect",
+          expectedSolrVersion,
+          liveProps.get(LIVE_NODE_SOLR_VERSION));
+      assertEquals("Live node nodeName incorrect", nodeName, liveProps.get(LIVE_NODE_NODE_NAME));
+
+      zkController.close();
+      cc.shutdown();
+    } finally {
+      server.shutdown();
+    }
+  }
+
   @LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG")
   @Test
   public void testPublishAndWaitForDownStates() throws Exception {

solr/core/src/test/org/apache/solr/cloud/overseer/ZkStateReaderTest.java

@@ -16,6 +16,9 @@
  */
 package org.apache.solr.cloud.overseer;
 
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_NODE_NAME;
+import static org.apache.solr.common.cloud.ZkStateReader.LIVE_NODE_SOLR_VERSION;
+
 import java.io.Closeable;
 import java.io.IOException;
 import java.lang.invoke.MethodHandles;
@@ -24,6 +27,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
@@ -38,6 +42,7 @@
 import java.util.concurrent.atomic.LongAdder;
 import org.apache.lucene.util.IOUtils;
 import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.client.api.util.SolrVersion;
 import org.apache.solr.cloud.OverseerTest;
 import org.apache.solr.cloud.Stats;
 import org.apache.solr.cloud.ZkController;
@@ -61,6 +66,7 @@
 import org.apache.solr.handler.admin.ConfigSetsHandler;
 import org.apache.solr.util.LogLevel;
 import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.CreateMode;
 import org.apache.zookeeper.KeeperException;
 import org.apache.zookeeper.data.Stat;
 import org.junit.After;
@@ -888,4 +894,82 @@ public void testDeletePrsCollection() throws Exception {
       assertTrue(prsZkNodeNotFoundExceptionThrown.get());
     }
   }
+
+  /** Test when two live nodes have valid SemVer strings */
+  public void testFetchLowestSolrVersion_validNodes() throws Exception {
+    SolrZkClient zkClient = fixture.zkClient;
+    ZkStateReader reader = fixture.reader;
+    String livePath = ZkStateReader.LIVE_NODES_ZKNODE;
+
+    // Clear any existing live node children.
+    List<String> nodes = zkClient.getChildren(livePath, null, true);
+    for (String node : nodes) {
+      zkClient.delete(livePath + "/" + node, -1, true);
+    }
+
+    // Create two live nodes with valid SemVer strings.
+    String node1 = "node1_solr";
+    Map<String, Object> props1 = new HashMap<>();
+    props1.put(LIVE_NODE_SOLR_VERSION, "9.1.2");
+    props1.put(LIVE_NODE_NODE_NAME, node1);
+    byte[] data1 = Utils.toJSON(props1);
+    zkClient.create(livePath + "/" + node1, data1, CreateMode.EPHEMERAL, true);
+
+    String node2 = "node2_solr";
+    Map<String, Object> props2 = new HashMap<>();
+    props2.put(LIVE_NODE_SOLR_VERSION, "9.0.1");
+    props2.put(LIVE_NODE_NODE_NAME, node2);
+    byte[] data2 = Utils.toJSON(props2);
+    zkClient.create(livePath + "/" + node2, data2, CreateMode.EPHEMERAL, true);
+
+    var lowestVersion = reader.fetchLowestSolrVersion();
+    assertEquals(
+        "Expected lowest version to be 9.0.1", SolrVersion.valueOf("9.0.1"), lowestVersion);
+  }
+
+  /** Test when the only live node has empty data. */
+  public void testFetchLowestSolrVersion_noData() throws Exception {
+    SolrZkClient zkClient = fixture.zkClient;
+    ZkStateReader reader = fixture.reader;
+    String livePath = ZkStateReader.LIVE_NODES_ZKNODE;
+
+    // Clear any existing live node children.
+    List<String> nodes = zkClient.getChildren(livePath, null, true);
+    for (String node : nodes) {
+      zkClient.delete(livePath + "/" + node, -1, true);
+    }
+
+    // Create a live node with empty data.
+    String emptyNode = "empty_node";
+    zkClient.create(livePath + "/" + emptyNode, new byte[0], CreateMode.EPHEMERAL, true);
+
+    assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), reader.fetchLowestSolrVersion());
+  }
+
+  /** Test when two live nodes exist; one is blank and the other has a high version */
+  public void testFetchLowestSolrVersion_blankAndHighVersion() throws Exception {
+    SolrZkClient zkClient = fixture.zkClient;
+    ZkStateReader reader = fixture.reader;
+    String livePath = ZkStateReader.LIVE_NODES_ZKNODE;
+
+    // Clear any existing live node children.
+    List<String> nodes = zkClient.getChildren(livePath, null, true);
+    for (String node : nodes) {
+      zkClient.delete(livePath + "/" + node, -1, true);
+    }
+
+    String node1 = "node1_solr";
+    zkClient.create(
+        livePath + "/" + node1,
+        Utils.toJSON(Map.<String, Object>of(LIVE_NODE_SOLR_VERSION, "888.0.0")),
+        CreateMode.EPHEMERAL,
+        true);
+
+    assertEquals("after high node", SolrVersion.LATEST, reader.fetchLowestSolrVersion());
+
+    String node2 = "node2_solr";
+    zkClient.create(livePath + "/" + node2, new byte[0], CreateMode.EPHEMERAL, true);
+
+    assertEquals("after empty node", SolrVersion.valueOf("9.9.0"), reader.fetchLowestSolrVersion());
+  }
 }

solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java

@@ -46,6 +46,7 @@
 import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
 import java.util.stream.Collectors;
+import org.apache.solr.client.api.util.SolrVersion;
 import org.apache.solr.client.solrj.impl.CloudSolrClient;
 import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
 import org.apache.solr.common.AlreadyClosedException;
@@ -165,6 +166,12 @@ public class ZkStateReader implements SolrCloseable {
   public static final String SHARD_LEADERS_ZKNODE = "leaders";
   public static final String ELECTION_NODE = "election";
 
+  /** Live node JSON property keys. */
+  public static final String LIVE_NODE_SOLR_VERSION = "solrVersion";
+
+  public static final String LIVE_NODE_NODE_NAME = "nodeName";
+  public static final String LIVE_NODE_ROLES = "roles";
+
   /** "Interesting" but not actively watched Collections. */
   private final ConcurrentHashMap<String, LazyCollectionRef> lazyCollectionStates =
       new ConcurrentHashMap<>();
@@ -857,6 +864,41 @@ public void removeLiveNodesListener(LiveNodesListener listener) {
     liveNodesListeners.remove(listener);
   }
 
+  /**
+   * Returns the lowest Solr version among all live nodes in the cluster. It's not greater than
+   * {@link SolrVersion#LATEST_STRING}. Will not return null. If older Solr nodes have joined that
+   * don't declare their version, the result won't be accurate, but it's at least an upper bound on
+   * the possible version it might be.
+   *
+   * @return the lowest Solr version of the cluster; not null
+   */
+  public SolrVersion fetchLowestSolrVersion() throws KeeperException, InterruptedException {
+    List<String> liveNodeNames = zkClient.getChildren(LIVE_NODES_ZKNODE, null, true);
+    SolrVersion lowest = SolrVersion.LATEST; // current software
+    // the last version to not specify its version in live nodes
+    final SolrVersion UNSPECIFIED_VERSION = SolrVersion.valueOf("9.9.0");
+    for (String nodeName : liveNodeNames) {
+      String path = LIVE_NODES_ZKNODE + "/" + nodeName;
+      byte[] data = zkClient.getData(path, null, null, true);
+      if (data == null || data.length == 0) {
+        return UNSPECIFIED_VERSION;
+      }
+
+      @SuppressWarnings("unchecked")
+      Map<String, Object> props = (Map<String, Object>) Utils.fromJSON(data);
+      String nodeVersionStr = (String) props.get(LIVE_NODE_SOLR_VERSION);
+      if (nodeVersionStr == null) { // weird
+        log.warn("No Solr version found: {}", props);
+        return UNSPECIFIED_VERSION;
+      }
+      SolrVersion nodeVersion = SolrVersion.valueOf(nodeVersionStr);
+      if (nodeVersion.compareTo(lowest) < 0) {
+        lowest = nodeVersion;
+      }
+    }
+    return lowest;
+  }
+
   /**
    * @return information about the cluster from ZooKeeper
    */