SOLR-15472: New shards.preference option for preferring replicas based on their leader status (apache#2515)

Author: thelabdude

solr/CHANGES.txt

@@ -10,7 +10,8 @@ Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this r
 
 New Features
 ---------------------
-(No changes)
+* SOLR-15472: New shards.preference option for preferring replicas based on their leader status, i.e.
+  `shards.preference=replica.leader:false` would prefer non-leader replicas. (wei wang via Timothy Potter)
 
 Improvements
 ---------------------

solr/solr-ref-guide/src/distributed-requests.adoc

@@ -192,6 +192,11 @@ Applied after sorting by inherent replica attributes, this property defines a fa
 `node.sysprop`::
 Query will be routed to nodes with same defined system properties as the current one. For example, if you start Solr nodes on different racks, you'll want to identify those nodes by a <<configuring-solrconfig-xml.adoc#jvm-system-properties,system property>> (e.g., `-Drack=rack1`). Then, queries can contain `shards.preference=node.sysprop:sysprop.rack`, to make sure you always hit shards with the same value of `rack`.
 
+`replica.leader`::
+Prefer replicas based on their leader status, set to either `true` or `false`. Consider a shard with two `TLOG` replicas and four `PULL` replicas (six replicas in total, one of which is the leader).
+With `shards.preference=replica.leader:false`, 5 out of 6 replicas will be preferred. Contrast this with `shards.preference=replica.type:PULL` where only 4 of 6 replicas will be preferred.
+Note that the non-leader `TLOG` replica behaves like a `PULL` replica from a search perspective; it pulls index updates from the leader just like a `PULL` replica and does not perform soft-commits.
+The difference is that the non-leader `TLOG` replica also captures updates in its TLOG, so that it is a candidate to replace the current leader if it is lost.
 
 Examples:
 
@@ -216,4 +221,7 @@ Examples:
 * Prefer local replicas, and among them PULL replicas when available TLOG otherwise:
    `shards.preference=replica.location:local,replica.type:PULL,replica.type:TLOG`
 
+* Prefer any replica that is not a leader:
+    `shards.preference=replica.leader:false`
+
 Note that if you provide the settings in a query string, they need to be properly URL-encoded.

solr/solrj/src/java/org/apache/solr/client/solrj/routing/NodePreferenceRulesComparator.java

@@ -112,6 +112,10 @@ public int compare(Object left, Object right) {
             lhs = hasCoreUrlPrefix(left, preferenceRule.value);
             rhs = hasCoreUrlPrefix(right, preferenceRule.value);
             break;
+          case ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER:
+            lhs = hasLeaderStatus(left, preferenceRule.value);
+            rhs = hasLeaderStatus(right, preferenceRule.value);
+            break;
           case ShardParams.SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP:
             if (sysPropsCache == null) {
               throw new IllegalArgumentException("Unable to get the NodesSysPropsCacher on sorting replicas by preference:"+ preferenceRule.value);
@@ -161,6 +165,7 @@ else if (o instanceof Replica) {
       return s.startsWith(prefix);
     }
   }
+
   private static boolean hasReplicaType(Object o, String preferred) {
     if (!(o instanceof Replica)) {
       return false;
@@ -169,6 +174,14 @@ private static boolean hasReplicaType(Object o, String preferred) {
     return s.equalsIgnoreCase(preferred);
   }
 
+  private static boolean hasLeaderStatus(Object o, String status) {
+    if (!(o instanceof Replica)) {
+      return false;
+    }
+    final boolean leaderStatus = ((Replica) o).isLeader();
+    return leaderStatus == Boolean.parseBoolean(status);
+  }
+
   public List<PreferenceRule> getSortRules() {
     return sortRules;
   }

solr/solrj/src/java/org/apache/solr/common/params/ShardParams.java

@@ -66,6 +66,9 @@ public interface ShardParams {
   /** Replica location sort rule */
   String SHARDS_PREFERENCE_REPLICA_LOCATION = "replica.location";
 
+  /** Replica leader status sort rule, value= true/false */
+  String SHARDS_PREFERENCE_REPLICA_LEADER = "replica.leader";
+
   /** Node with same system property sort rule */
   String SHARDS_PREFERENCE_NODE_WITH_SAME_SYSPROP = "node.sysprop";
 

solr/solrj/src/test/org/apache/solr/client/solrj/routing/NodePreferenceRulesComparatorTest.java

@@ -21,8 +21,8 @@
 import java.util.List;
 
 import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.common.cloud.UrlScheme;
 import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.UrlScheme;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.ShardParams;
 import org.junit.Test;
@@ -93,6 +93,67 @@ public void replicaTypeAndReplicaLocationTest() {
     assertEquals("node3", getHost(replicas.get(3).getNodeName()));
   }
 
+  @Test
+  public void replicaLeaderTest() {
+    List<Replica> replicas = getBasicReplicaList();
+    // Add a non-leader NRT replica so that sorting by replica.type:NRT will cause a tie, order is decided by leader status
+    replicas.add(
+        new Replica(
+            "node4",
+            map(ZkStateReader.BASE_URL_PROP, "http://host2:8983/solr",
+                ZkStateReader.NODE_NAME_PROP, "node4:8983_solr",
+                ZkStateReader.CORE_NAME_PROP, "collection1",
+                ZkStateReader.REPLICA_TYPE, "NRT"
+            ), "collection1", "shard1"
+        )
+    );
+    // Prefer non-leader only, therefore node1 has lowest priority
+    List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER + ":false");
+    NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);
+    replicas.sort(comparator);
+    assertEquals("node1:8983_solr", replicas.get(3).getNodeName());
+
+    // Prefer NRT replica, followed by non-leader
+    rules = PreferenceRule.from(
+        ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
+            ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER + ":false");
+    comparator = new NodePreferenceRulesComparator(rules, null);
+    replicas.sort(comparator);
+    assertEquals("node4", getHost(replicas.get(0).getNodeName()));
+    assertEquals("node1", getHost(replicas.get(1).getNodeName()));
+
+    // Prefer non-leader first, followed by PULL replica and location host2
+    rules = PreferenceRule.from(
+        ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER + ":false," +
+            ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":PULL," +
+            ShardParams.SHARDS_PREFERENCE_REPLICA_LOCATION + ":http://host2");
+    comparator = new NodePreferenceRulesComparator(rules, null);
+
+    replicas.sort(comparator);
+    assertEquals("node3", getHost(replicas.get(0).getNodeName()));
+    assertEquals("node4", getHost(replicas.get(1).getNodeName()));
+    assertEquals("node2", getHost(replicas.get(2).getNodeName()));
+    assertEquals("node1", getHost(replicas.get(3).getNodeName()));
+
+    // make sure a single leader only still gets selected
+    List<Replica> onlyLeader = new ArrayList<>();
+    onlyLeader.add(
+        new Replica(
+            "node1",
+            map(
+                ZkStateReader.NODE_NAME_PROP, "node1:8983_solr",
+                ZkStateReader.CORE_NAME_PROP, "collection1",
+                ZkStateReader.REPLICA_TYPE, "NRT",
+                ZkStateReader.LEADER_PROP, "true"
+            ),"collection1","shard1"
+        )
+    );
+    rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER + ":false");
+    comparator = new NodePreferenceRulesComparator(rules, null);
+    replicas.sort(comparator);
+    assertEquals("node1:8983_solr", onlyLeader.get(0).getNodeName());
+  }
+
   @Test(expected = IllegalArgumentException.class)
   public void badRuleTest() {
     try {
@@ -126,7 +187,8 @@ private static List<Replica> getBasicReplicaList() {
             map(
                 ZkStateReader.NODE_NAME_PROP, "node1:8983_solr",
                 ZkStateReader.CORE_NAME_PROP, "collection1",
-                ZkStateReader.REPLICA_TYPE, "NRT"
+                ZkStateReader.REPLICA_TYPE, "NRT",
+                ZkStateReader.LEADER_PROP, "true"
             ),"collection1","shard1"
         )
     );

solr/solrj/src/test/org/apache/solr/client/solrj/routing/RequestReplicaListTransformerGeneratorTest.java

@@ -104,8 +104,9 @@ public void replicaTypeAndReplicaBase() {
         )
     );
 
-    // replicaType and replicaBase combined rule param
-    String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
+    // replica leader status, replicaType and replicaBase combined rule param
+    String rulesParam = ShardParams.SHARDS_PREFERENCE_REPLICA_LEADER + ":true," +
+        ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
         ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
         ShardParams.SHARDS_PREFERENCE_REPLICA_BASE + ":stable:dividend:routingPreference";