Stop repair scheduler if two major versions are detected

patch by Himanshu Jindal; reviewed by Jaydeepkumar Chovatia, Andy Tolbert for CASSANDRA-20048

Author: himanshujindal

CHANGES.txt

@@ -1,4 +1,5 @@
 5.1
+ * Stop repair scheduler if two major versions are detected (CASSANDRA-20048)
  * Optimize audit logic for batch operations especially when audit is not enabled for DML (CASSANDRA-20885)
  * Implement nodetool history (CASSANDRA-20851)
  * Expose StorageService.dropPreparedStatements via JMX (CASSANDRA-20870)

conf/cassandra.yaml

@@ -2782,6 +2782,9 @@ storage_compatibility_mode: NONE
 #   # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
 #   # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
 #   history_clear_delete_hosts_buffer_interval: 2h
+#   # By default repair is disabled if there are mixed major versions detected - which would happen
+#   # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+#   mixed_major_version_repair_enabled: false
 #   # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
 #   global_settings:
 #     # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired

conf/cassandra_latest.yaml

@@ -2467,6 +2467,9 @@ storage_compatibility_mode: NONE
 #   # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
 #   # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
 #   history_clear_delete_hosts_buffer_interval: 2h
+#   # By default repair is disabled if there are mixed major versions detected - which would happen
+#   # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+#   mixed_major_version_repair_enabled: false
 #   # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
 #   global_settings:
 #     # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired

doc/modules/cassandra/pages/managing/operating/auto_repair.adoc

@@ -167,6 +167,10 @@ is time to schedule repairs.
 | history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring.
 Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments
 are made to the schedule.
+| mixed_major_version_repair_enabled | false | Enable/Disable running repairs on the cluster when there are mixed
+major versions detected, which usually occurs when the cluster is being upgraded. Repairs between nodes of
+different major versions is not something that is tested, so this may lead to data compatibility issues.
+It is strongly discouraged to set this to true without doing extensive testing beforehand.
 |===
 
 

src/java/org/apache/cassandra/repair/autorepair/AutoRepair.java

@@ -165,6 +165,11 @@ public void repair(AutoRepairConfig.RepairType repairType)
             logger.debug("Auto-repair is disabled for repair type {}", repairType);
             return;
         }
+        if (!config.isMixedMajorVersionRepairEnabled() && AutoRepairUtils.hasMultipleLiveMajorVersions())
+        {
+            logger.info("Auto-repair is disabled when nodes in the cluster have different major versions");
+            return;
+        }
         AutoRepairService.instance.checkCanRun(repairType);
         AutoRepairState repairState = repairStates.get(repairType);
         try

src/java/org/apache/cassandra/repair/autorepair/AutoRepairConfig.java

@@ -59,6 +59,8 @@ public class AutoRepairConfig implements Serializable
     // Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
     // the node by scheduling too many repair tasks in a short period of time.
     public volatile DurationSpec.LongSecondsBound repair_task_min_duration = new DurationSpec.LongSecondsBound("5s");
+    // by default repair is disabled if there are mixed major versions detected, but you can enable it using this flag
+    public volatile boolean mixed_major_version_repair_enabled = false;
 
     // global_settings overides Options.defaultOptions for all repair types
     public volatile Options global_settings;
@@ -149,6 +151,11 @@ public void setAutoRepairSchedulingEnabled(boolean enabled)
         this.enabled = enabled;
     }
 
+    public boolean isMixedMajorVersionRepairEnabled()
+    {
+        return mixed_major_version_repair_enabled;
+    }
+
     public DurationSpec.IntSecondsBound getAutoRepairHistoryClearDeleteHostsBufferInterval()
     {
         return history_clear_delete_hosts_buffer_interval;
@@ -366,6 +373,16 @@ public void setRepairRetryBackoff(RepairType repairType, String interval)
         getOptions(repairType).repair_retry_backoff = new DurationSpec.LongSecondsBound(interval);
     }
 
+    public boolean getMixedMajorVersionRepairEnabled()
+    {
+        return this.mixed_major_version_repair_enabled;
+    }
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        this.mixed_major_version_repair_enabled = enabled;
+    }
+
     @VisibleForTesting
     static IAutoRepairTokenRangeSplitter newAutoRepairTokenRangeSplitter(RepairType repairType, ParameterizedClass parameterizedClass) throws ConfigurationException
     {

src/java/org/apache/cassandra/repair/autorepair/AutoRepairUtils.java

@@ -425,6 +425,21 @@ public static CurrentRepairStatus getCurrentRepairStatus(RepairType repairType,
         return null;
     }
 
+    /**
+     * Checks whether the cluster has multiple major versions
+     * @return
+     *  true if more than one major versions are detected
+     *  false if only one major version is detected
+     *
+     */
+    public static boolean hasMultipleLiveMajorVersions()
+    {
+        ClusterMetadata metadata = ClusterMetadata.current();
+        int maxMajorVersion = ClusterMetadata.current().directory.clusterMaxVersion.cassandraVersion.major;
+        int minMajorVersion = ClusterMetadata.current().directory.clusterMinVersion.cassandraVersion.major;
+        return maxMajorVersion != minMajorVersion;
+    }
+
     @VisibleForTesting
     protected static TreeSet<UUID> getHostIdsInCurrentRing(RepairType repairType, Collection<NodeAddresses> allNodesInRing)
     {

src/java/org/apache/cassandra/service/AutoRepairService.java

@@ -102,6 +102,7 @@ public String getAutoRepairConfiguration()
         appendConfig(sb, "repair_check_interval", config.getRepairCheckInterval());
         appendConfig(sb, "repair_task_min_duration", config.getRepairTaskMinDuration());
         appendConfig(sb, "history_clear_delete_hosts_buffer_interval", config.getAutoRepairHistoryClearDeleteHostsBufferInterval());
+        appendConfig(sb, "mixed_major_version_repair_enabled", config.getMixedMajorVersionRepairEnabled());
         for (RepairType repairType : RepairType.values())
         {
             sb.append(formatRepairTypeConfig(repairType, config));
@@ -271,6 +272,12 @@ public void setAutoRepairRetryBackoff(String repairType, String interval)
         config.setRepairRetryBackoff(RepairType.parse(repairType), interval);
     }
 
+    @Override
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        config.setMixedMajorVersionRepairEnabled(enabled);
+    }
+
     private String formatRepairTypeConfig(RepairType repairType, AutoRepairConfig config)
     {
         StringBuilder sb = new StringBuilder();

src/java/org/apache/cassandra/service/AutoRepairServiceMBean.java

@@ -74,4 +74,6 @@ public interface AutoRepairServiceMBean
     public void setAutoRepairMaxRetriesCount(String repairType, int retries);
 
     public void setAutoRepairRetryBackoff(String repairType, String interval);
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled);
 }

src/java/org/apache/cassandra/tools/NodeProbe.java

@@ -2677,6 +2677,11 @@ public GuardrailsMBean getGuardrailsMBean()
     {
         return grProxy;
     }
+
+    public void setMixedMajorVersionRepairEnabled(boolean enabled)
+    {
+        autoRepairProxy.setMixedMajorVersionRepairEnabled(enabled);
+    }
 }
 
 class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String, ColumnFamilyStoreMBean>>