diff --git a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java index d3596bc536d..13dc962159d 100644 --- a/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java +++ b/src/java/org/apache/cassandra/config/CassandraRelevantProperties.java @@ -951,6 +951,7 @@ public enum CassandraRelevantProperties UCS_L0_SHARDS_ENABLED("unified_compaction.l0_shards_enabled", "true"), UCS_MAX_ADAPTIVE_COMPACTIONS("unified_compaction.max_adaptive_compactions", "5"), UCS_MAX_SPACE_OVERHEAD("unified_compaction.max_space_overhead", "0.2"), + UCS_MAX_SSTABLES_PER_SHARD_FACTOR("unified_compaction.max_sstables_per_shard_factor", "10"), UCS_MIN_SSTABLE_SIZE("unified_compaction.min_sstable_size", "100MiB"), UCS_NUM_SHARDS("unified_compaction.num_shards"), UCS_OVERLAP_INCLUSION_METHOD("unified_compaction.overlap_inclusion_method"), diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java index f55b0e029e1..e2bf441fed8 100644 --- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.java @@ -998,7 +998,7 @@ private List getPendingCompactionAggregate for (Level level : entry.getValue()) { - Collection aggregates = level.getCompactionAggregates(arena, controller, spaceAvailable); + Collection aggregates = level.getCompactionAggregates(arena, controller, getShardManager(), spaceAvailable); // Note: We allow empty aggregates into the list of pending compactions. The pending compactions list // is for progress tracking only, and it is helpful to see empty levels there. pending.addAll(aggregates); @@ -1506,14 +1506,96 @@ void complete() logger.trace("Level: {}", this); } + private List getOversizeShardsAggregates(Arena arena, + Controller controller, + ShardManager shardManager) + { + List aggregates = new ArrayList<>(); + double shardThreshold = fanout * controller.getMaxSstablesPerShardFactor(); + if (sstables.size() > shardThreshold) + { + List> groups = shardManager.splitSSTablesInShards(sstables, + controller.getNumShards(max), + (sstableShard, shardRange) -> Sets.newHashSet(sstableShard)); + + Set sstablesInOversizeGroup = new HashSet<>(); + for (Set ssTables : groups) + { + if (ssTables.size() > shardThreshold) + { + sstablesInOversizeGroup.addAll(ssTables); + } + } + + if (!sstablesInOversizeGroup.isEmpty()) + { + // Now combine the groups that share an sstable so that we have valid independent transactions. + // Only keep the groups that were combined with an oversize group. + groups = Overlaps.combineSetsWithCommonElement(groups); + List unbucketed = new ArrayList<>(); + + for (Set group : groups) + { + boolean inOverSizeGroup = false; + for (CompactionSSTable sstable : group) + { + if (sstablesInOversizeGroup.contains(sstable)) + { + inOverSizeGroup = true; + break; + } + } + if (inOverSizeGroup) + { + aggregates.add( + CompactionAggregate.createUnified(group, + Overlaps.maxOverlap(group, + CompactionSSTable.startsAfter, + CompactionSSTable.firstKeyComparator, + CompactionSSTable.lastKeyComparator), + createPick(controller, nextTimeUUID(), index, group), + Collections.emptyList(), + arena, + this) + ); + } + else + { + unbucketed.addAll(group); + } + } + // Add all unbucketed sstables separately. Note that this will list the level (with its set of sstables) + // even if it does not need compaction. + if (!unbucketed.isEmpty()) + aggregates.add(CompactionAggregate.createUnified(unbucketed, + maxOverlap, + CompactionPick.EMPTY, + Collections.emptySet(), + arena, + this)); + return aggregates; + } + } + return aggregates; + } + /// Return the compaction aggregate Collection getCompactionAggregates(Arena arena, Controller controller, + ShardManager shardManager, long spaceAvailable) { if (logger.isTraceEnabled()) logger.trace("Creating compaction aggregate with sstable set {}", sstables); + List aggregates = new ArrayList<>(); + + if (sstables.isEmpty()) + { + if (logger.isTraceEnabled()) + logger.trace("No sstables in level {} of arena {}, skipping compaction", this, arena); + return aggregates; + } // Note that adjacent overlap sets may include deduplicated sstable List> overlaps = Overlaps.constructOverlapSets(sstables, @@ -1530,9 +1612,19 @@ Collection getCompactionAggregates(Arena a this::makeBucket, unbucketed::addAll); - List aggregates = new ArrayList<>(); - for (Bucket bucket : buckets) - aggregates.add(bucket.constructAggregate(controller, spaceAvailable, arena)); + if (!buckets.isEmpty()) + { + for (Bucket bucket : buckets) + aggregates.add(bucket.constructAggregate(controller, spaceAvailable, arena)); + } + else + { + // CNDB-14577: If there are no overlaps, we look if some shards have too many SSTables. + // If that's the case, we perform a major compaction on those shards. + List oversizeShardsAggregates = getOversizeShardsAggregates(arena, controller, shardManager); + if (!oversizeShardsAggregates.isEmpty()) + return oversizeShardsAggregates; + } // Add all unbucketed sstables separately. Note that this will list the level (with its set of sstables) // even if it does not need compaction. diff --git a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md index 71c45664e83..2e6561c1712 100644 --- a/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md +++ b/src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md @@ -319,6 +319,24 @@ on the number of overlapping sources we compact; in that case we use the collect select at most limit-many in any included overlap set, making sure that if an sstable is included in this compaction, all older ones are also included to maintain time order. +## Non-overlapping sstables trigger + +In some scenarios it is possible for (small) non-overlapping sstables to accumulate in numbers that can cause problems +due to the sheer number of sstables present. For example, in tables with regularly scheduled snapshots, which also use +time-based partitioning, that regular snapshot will often flush single-partition sstables. When the partition time +window passes, the normal overlap processing will no longer find newly flushed data that overlaps with older sstables +and will thus leave those sstables alone. Eventually we can end up with thousands of sstables on a lower level that are +never compacted. + +This can be a problem, especially in combination with SAI indexing, which prefers a lower number of sstables overall. +To address it, the strategy offers a threshold for the number of sstables that can be present on any of the shards that +the sharding strategy assigns for a given level. The threshold is specified as a multiple of a level's fan factor: if +there are no normal compactions to perform on a level, and we can find a shard that has more sstables than the +threshold, we perform the equivalent of a major compaction for the smallest set of shards that contains it. The result +is split on each output shard boundary and results in a single sstable for each of the output shards. This should +create sstables that span many partitions and that will thus progress nicely through the normal processing in the +next levels of the hierarchy. + ## Prioritization of compactions Compaction strategies aim to minimize the read amplification of queries, which is defined by the number of sstables @@ -533,6 +551,14 @@ UCS accepts these compaction strategy parameters: Sets $b$ to the specified value, $\lambda$ to 1, and the default minimum sstable size to 'auto'. Disabled by default and cannot be used in combination with `base_shard_count`, `target_sstable_size` or `sstable_growth`. +* `max_sstables_per_shard_factor` Limits the number of SSTables per shard. If the number of sstables in a shard + exceeds this factor times the shard compaction threshold, a major compaction of the shard will be triggered. + Some conditions like slow writes can lead to SSTables being very small, and never overlap with enough other SSTables + to be compacted. + So this setting is useful to prevent the number of SSTables in a shard from growing too large, which can cause + problems due to the per-sstable overhead. Also these small SSTables may still have overlaps even if under the + compaction threshold (eg. due to write replicas) and never compacting them wastes storage space. + The default value is 10. All UCS options can also be supplied as system properties, using the prefix `unified_compaction.`, e.g. `-Dunified_compaction.sstable_growth=0.5` sets the default `sstable_growth` to 0.5. diff --git a/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java b/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java index 9c3e9bf409c..ba5a16b32a2 100644 --- a/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java +++ b/src/java/org/apache/cassandra/db/compaction/unified/AdaptiveController.java @@ -126,6 +126,7 @@ public AdaptiveController(MonotonicClock clock, Overlaps.InclusionMethod overlapInclusionMethod, boolean parallelizeOutputShards, boolean hasVectorType, + double maxSstablesPerShardFactor, int intervalSec, int minScalingParameter, int maxScalingParameter, @@ -154,6 +155,7 @@ public AdaptiveController(MonotonicClock clock, overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, metadata); this.scalingParameters = scalingParameters; @@ -184,6 +186,7 @@ static Controller fromOptions(Environment env, Overlaps.InclusionMethod overlapInclusionMethod, boolean parallelizeOutputShards, boolean hasVectorType, + double maxSstablesPerShardFactor, TableMetadata metadata, Map options) { @@ -292,6 +295,7 @@ else if (staticScalingFactors != null) overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, intervalSec, minScalingParameter, maxScalingParameter, diff --git a/src/java/org/apache/cassandra/db/compaction/unified/Controller.java b/src/java/org/apache/cassandra/db/compaction/unified/Controller.java index 4f15c705930..b83b4b88a9f 100644 --- a/src/java/org/apache/cassandra/db/compaction/unified/Controller.java +++ b/src/java/org/apache/cassandra/db/compaction/unified/Controller.java @@ -318,6 +318,9 @@ public abstract class Controller @Deprecated(since = "CC 4.0") static final String STATIC_SCALING_FACTORS_OPTION = "static_scaling_factors"; + static final String MAX_SSTABLES_PER_SHARD_FACTOR_OPTION = "max_sstables_per_shard_factor"; + static final double DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR = UCS_MAX_SSTABLES_PER_SHARD_FACTOR.getDoubleWithLegacyFallback(); + protected final MonotonicClock clock; protected final Environment env; protected final double[] survivalFactors; @@ -349,6 +352,8 @@ public abstract class Controller final boolean l0ShardsEnabled; final boolean hasVectorType; + final double maxSstablesPerShardFactor; + Controller(MonotonicClock clock, Environment env, double[] survivalFactors, @@ -369,6 +374,7 @@ public abstract class Controller Overlaps.InclusionMethod overlapInclusionMethod, boolean parallelizeOutputShards, boolean hasVectorType, + double maxSstablesPerShardFactor, TableMetadata metadata) { this.clock = clock; @@ -390,6 +396,7 @@ public abstract class Controller this.l0ShardsEnabled = UCS_L0_SHARDS_ENABLED.getBooleanWithLegacyFallback(false); // FIXME VECTOR-23 this.parallelizeOutputShards = parallelizeOutputShards; this.hasVectorType = hasVectorType; + this.maxSstablesPerShardFactor = maxSstablesPerShardFactor; this.metadata = metadata; if (maxSSTablesToCompact <= 0) // use half the maximum permitted compaction size as upper bound by default @@ -800,6 +807,11 @@ public boolean hasVectorType() return hasVectorType; } + public double getMaxSstablesPerShardFactor() + { + return maxSstablesPerShardFactor; + } + /** * @return true if the controller is running */ @@ -1049,6 +1061,10 @@ public static Controller fromOptions(CompactionRealm realm, Map ? Boolean.parseBoolean(options.get(PARALLELIZE_OUTPUT_SHARDS_OPTION)) : DEFAULT_PARALLELIZE_OUTPUT_SHARDS; + double maxSstablesPerShardFactor = options.containsKey(MAX_SSTABLES_PER_SHARD_FACTOR_OPTION) + ? Double.parseDouble(options.get(MAX_SSTABLES_PER_SHARD_FACTOR_OPTION)) + : DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR; + return adaptive ? AdaptiveController.fromOptions(env, survivalFactors, @@ -1068,6 +1084,7 @@ public static Controller fromOptions(CompactionRealm realm, Map overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, realm.metadata(), options) : StaticController.fromOptions(env, @@ -1088,6 +1105,7 @@ public static Controller fromOptions(CompactionRealm realm, Map overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, realm.metadata(), options, useVectorOptions); @@ -1317,6 +1335,26 @@ public static Map validateOptions(Map options) t } } + s = options.remove(MAX_SSTABLES_PER_SHARD_FACTOR_OPTION); + if (s != null) + { + try + { + double maxSstablesPerShardFactor = Double.parseDouble(s); + if (maxSstablesPerShardFactor < 1) + throw new ConfigurationException(String.format("%s %s must be a float >= 1", + MAX_SSTABLES_PER_SHARD_FACTOR_OPTION, + s)); + } + catch (NumberFormatException e) + { + throw new ConfigurationException(String.format(floatParseErr, + s, + MAX_SSTABLES_PER_SHARD_FACTOR_OPTION), + e); + } + } + return adaptive ? AdaptiveController.validateOptions(options) : StaticController.validateOptions(options); } diff --git a/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java b/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java index 0c4bd51c95f..8fc978ff59e 100644 --- a/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java +++ b/src/java/org/apache/cassandra/db/compaction/unified/StaticController.java @@ -76,6 +76,7 @@ public StaticController(Environment env, Overlaps.InclusionMethod overlapInclusionMethod, boolean parallelizeOutputShards, boolean hasVectorType, + double maxSstablesPerShardFactor, TableMetadata metadata) { super(MonotonicClock.Global.preciseTime, @@ -98,6 +99,7 @@ public StaticController(Environment env, overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, metadata); this.scalingParameters = scalingParameters; } @@ -120,6 +122,7 @@ static Controller fromOptions(Environment env, Overlaps.InclusionMethod overlapInclusionMethod, boolean parallelizeOutputShards, boolean hasVectorType, + double maxSstablesPerShardFactor, TableMetadata metadata, Map options, boolean useVectorOptions) @@ -182,6 +185,7 @@ static Controller fromOptions(Environment env, overlapInclusionMethod, parallelizeOutputShards, hasVectorType, + maxSstablesPerShardFactor, metadata); } diff --git a/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java b/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java index 2ab8851f3a9..ba6e738877e 100644 --- a/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/CompactionSimulationTest.java @@ -224,6 +224,9 @@ public class CompactionSimulationTest extends BaseCompactionStrategyTest @Option(name= {"-overlap-inclusion-method"}, description = "Overlap inclusion method, NONE, SINGLE or TRANSITIVE") Overlaps.InclusionMethod overlapInclusionMethod = Overlaps.InclusionMethod.TRANSITIVE; + @Option(name= {"-max-sstables-per-shard-factor"}, description = "Factor used to determine the maximum number of sstables per level shard") + double maxSstablesPerShardFactor = 10; + @BeforeClass public static void setUpClass() { @@ -414,6 +417,7 @@ private UnifiedCompactionStrategy createUnifiedCompactionStrategy(Counters count overlapInclusionMethod, true, false, + maxSstablesPerShardFactor, updateTimeSec, minW, maxW, @@ -441,6 +445,7 @@ private UnifiedCompactionStrategy createUnifiedCompactionStrategy(Counters count overlapInclusionMethod, true, false, + maxSstablesPerShardFactor, metadata); return new UnifiedCompactionStrategy(strategyFactory, controller); diff --git a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java index b37a8f12d45..1246af9c05c 100644 --- a/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/UnifiedCompactionStrategyTest.java @@ -227,6 +227,8 @@ private void testGetBucketsOneArena(Map sstableMap, int[] Ws, when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod(); when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0); + // Set high value to prevent shard compaction from interfering with bucket threshold testing + when(controller.getMaxSstablesPerShardFactor()).thenReturn(1000.0); when(controller.random()).thenCallRealMethod(); UnifiedCompactionStrategy strategy = new UnifiedCompactionStrategy(strategyFactory, controller); @@ -266,7 +268,7 @@ private void testGetBucketsOneArena(Map sstableMap, int[] Ws, assertEquals(i, level.getIndex()); Collection compactionAggregates = - level.getCompactionAggregates(entry.getKey(), controller, dataSetSizeBytes); + level.getCompactionAggregates(entry.getKey(), controller, strategy.getShardManager(), dataSetSizeBytes); long selectedCount = compactionAggregates.stream() .filter(a -> !a.isEmpty()) @@ -325,7 +327,7 @@ private void testGetMultipleBucketsOneArenaNonOverlappingAggregates(Map compactionAggregates = - level.getCompactionAggregates(entry.getKey(), controller, dataSetSizeBytes); + level.getCompactionAggregates(entry.getKey(), controller, strategy.getShardManager(), dataSetSizeBytes); Set selectedSSTables = new HashSet<>(); for (CompactionAggregate.UnifiedAggregate aggregate : compactionAggregates) @@ -795,6 +799,7 @@ private UnifiedCompactionStrategy prepareStrategyWithLimits(int maxCount, when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod(); when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0); when(controller.getNumShards(anyDouble())).thenReturn(numShards); + when(controller.getMaxSstablesPerShardFactor()).thenReturn(10.0); when(controller.getMaxSpaceOverhead()).thenReturn(maxSpaceOverhead); when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minSstableSizeBytes); when(controller.maxConcurrentCompactions()).thenReturn(maxCount); @@ -1391,6 +1396,7 @@ private void testDropExpiredFromBucket(int numShards, boolean parallelizeOutputS when(controller.getThreshold(anyInt())).thenCallRealMethod(); when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0); when(controller.getNumShards(anyDouble())).thenReturn(numShards); + when(controller.getMaxSstablesPerShardFactor()).thenReturn(10.0); when(controller.getBaseSstableSize(anyInt())).thenReturn((double) minimalSizeBytes); when(controller.maxConcurrentCompactions()).thenReturn(1000); // let it generate as many candidates as it can when(controller.maxCompactionSpaceBytes()).thenReturn(Long.MAX_VALUE); @@ -2018,6 +2024,49 @@ public void testBucketSelectionHalvesMissing() testBucketSelection(repeats(4, arr(6, 3)), arr(6, 6), Overlaps.InclusionMethod.TRANSITIVE, 4, 2, 3); } + // We use a level with only non-overlapping SSTables, so there won't be any standard bucketing compaction. + // The per-shard threshold is fanout(3) * maxSstablesPerShardFactor(10) = 30. + // We use a number of SSTables per shard just superior to the per-shard threshold. + // We use a total number of SSTables just superior to numShards * sstablesPerShard so the SSTables are not aligned + // with the shard boundaries. This means that all shards will have an SSTable in common with the next shard. + // We drop the SSTables that cross the boundaries between the first and second shard, and between the second and + // third shard. + // In the end, we get 3 groups of shards with SSTables in common: + // - a group with the first shard which has perShardThreshold + 1 SSTables, so will get compacted. + // - a group with the second shard which has perShardThreshold SSTables, so will not get compacted. + // - a group with the rest of the shards, one of those shards having perShardThreshold + 1 SSTables, so will get compacted. + @Test + public void testBucketSelectionNonOverlapping() + { + int numShards = 16; + double maxSstablesPerShardFactor = 10.0; + int perShardThreshold = (int) (3 * maxSstablesPerShardFactor); + int sstablesPerShard = perShardThreshold + 1; + testBucketSelection(arr(numShards * sstablesPerShard + 1), + arr(sstablesPerShard, sstablesPerShard * (numShards - 2)), + Overlaps.InclusionMethod.TRANSITIVE, + numShards, + maxSstablesPerShardFactor, + perShardThreshold, + sstablesPerShard, sstablesPerShard * 2); + } + + // Same as testBucketSelectionNonOverlapping, but with an infinite maxSstablesPerShardFactor + // to deactivate compaction of oversize shards. + @Test + public void testBucketSelectionNonOverlappingInfiniteFactor() + { + int numShards = 16; + int sstablesPerShard = (3 * 10) + 1; + testBucketSelection(arr(numShards * sstablesPerShard + 1), + arr(), + Overlaps.InclusionMethod.TRANSITIVE, + numShards, + Double.POSITIVE_INFINITY, + numShards * sstablesPerShard - 1, + sstablesPerShard, sstablesPerShard * 2); + } + private int[] arr(int... values) { @@ -2053,6 +2102,11 @@ public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.Inclusio } public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.InclusionMethod overlapInclusionMethod, int expectedRemaining, int... dropFromFirst) + { + testBucketSelection(counts, expecteds, overlapInclusionMethod, 16, 10.0, expectedRemaining, dropFromFirst); + } + + public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.InclusionMethod overlapInclusionMethod, int numShards, double maxSstablesPerShardFactor, int expectedRemaining, int... dropFromFirst) { Set allSSTables = new HashSet<>(); int fanout = counts.length; @@ -2072,6 +2126,8 @@ public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.Inclusio when(controller.getFanout(anyInt())).thenCallRealMethod(); when(controller.getThreshold(anyInt())).thenCallRealMethod(); when(controller.getMaxLevelDensity(anyInt(), anyDouble())).thenCallRealMethod(); + when(controller.getNumShards(anyDouble())).thenReturn(numShards); + when(controller.getMaxSstablesPerShardFactor()).thenReturn(maxSstablesPerShardFactor); when(controller.getSurvivalFactor(anyInt())).thenReturn(1.0); when(controller.getBaseSstableSize(anyInt())).thenReturn((double) (90 << 20)); when(controller.overlapInclusionMethod()).thenReturn(overlapInclusionMethod); @@ -2129,7 +2185,6 @@ public void testBucketSelection(int[] counts, int[] expecteds, Overlaps.Inclusio assertEquals(1.3333, pick.overheadToDataRatio(), 0.0001); } - Mockito.when(controller.getNumShards(anyDouble())).thenReturn(16); // co-prime with counts to ensure multiple sstables fall in each shard // Make sure getMaxOverlapsMap does not fail. System.out.println(strategy.getMaxOverlapsMap()); diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java index 70c0ec85b7d..0400ec7aae4 100644 --- a/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/unified/AdaptiveControllerTest.java @@ -87,6 +87,7 @@ private AdaptiveController makeController(long dataSizeGB, int numShards, long s Controller.DEFAULT_OVERLAP_INCLUSION_METHOD, true, false, + Controller.DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR, interval, minW, maxW, diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java index 190ff565e95..f6bcc32cb92 100644 --- a/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/unified/ControllerTest.java @@ -626,6 +626,30 @@ public void testParallelizeOutputShards() testBooleanOption(Controller.PARALLELIZE_OUTPUT_SHARDS_OPTION, Controller.DEFAULT_PARALLELIZE_OUTPUT_SHARDS, Controller::parallelizeOutputShards); } + @Test + public void testMaxSstablesPerShardFactor() + { + HashMap options = new HashMap<>(); + Controller controller = Controller.fromOptions(cfs, options); + assertEquals(Controller.DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR, controller.getMaxSstablesPerShardFactor(), epsilon); + + options.put(Controller.MAX_SSTABLES_PER_SHARD_FACTOR_OPTION, "123.456"); + Controller.validateOptions(options); + controller = Controller.fromOptions(cfs, options); + assertEquals(123.456, controller.getMaxSstablesPerShardFactor(), epsilon); + + options.put(Controller.MAX_SSTABLES_PER_SHARD_FACTOR_OPTION, "1e1000"); + Controller.validateOptions(options); + controller = Controller.fromOptions(cfs, options); + assertEquals(Double.POSITIVE_INFINITY, controller.getMaxSstablesPerShardFactor(), epsilon); + + options.put(Controller.MAX_SSTABLES_PER_SHARD_FACTOR_OPTION, "0.9"); + assertThrows(ConfigurationException.class, () -> Controller.validateOptions(options)); + + options.put(Controller.MAX_SSTABLES_PER_SHARD_FACTOR_OPTION, "invalid"); + assertThrows(ConfigurationException.class, () -> Controller.validateOptions(options)); + } + public void testBooleanOption(String name, boolean defaultValue, Predicate getter, String... extraSettings) { Controller controller = Controller.fromOptions(cfs, newOptions(extraSettings)); diff --git a/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java b/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java index 9dc1bffea01..2d8388b75b8 100644 --- a/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/unified/StaticControllerTest.java @@ -210,6 +210,7 @@ public void testStartShutdown() Controller.DEFAULT_OVERLAP_INCLUSION_METHOD, true, false, + Controller.DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR, metadata); super.testStartShutdown(controller); } @@ -237,6 +238,7 @@ public void testShutdownNotStarted() Controller.DEFAULT_OVERLAP_INCLUSION_METHOD, true, false, + Controller.DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR, metadata); super.testShutdownNotStarted(controller); } @@ -264,6 +266,7 @@ public void testStartAlreadyStarted() Controller.DEFAULT_OVERLAP_INCLUSION_METHOD, true, false, + Controller.DEFAULT_MAX_SSTABLES_PER_SHARD_FACTOR, metadata); super.testStartAlreadyStarted(controller); }