Merge pull request #502 from IABTechLab/gdm-UID2-5601-salt-rotate-limit-per-age

Added max rotatable salts per age limit

Author: gmsdelmundo

src/main/java/com/uid2/admin/salt/SaltRotation.java

@@ -17,6 +17,7 @@
 
 public class SaltRotation {
     private static final long THIRTY_DAYS_IN_MS = Duration.ofDays(30).toMillis();
+    private static final double MAX_SALT_PERCENTAGE = 0.8;
 
     private final IKeyGenerator keyGenerator;
     private final boolean isRefreshFromEnabled;
@@ -133,6 +134,8 @@ private List<SaltEntry> pickSaltsToRotate(
             TargetDate targetDate,
             Duration[] minAges,
             int numSaltsToRotate) {
+        var maxSaltsPerAge = this.isRefreshFromEnabled ? (int) (numSaltsToRotate * MAX_SALT_PERCENTAGE) : numSaltsToRotate;
+
         var thresholds = Arrays.stream(minAges)
                 .map(minAge -> targetDate.asInstant().minusSeconds(minAge.getSeconds()))
                 .sorted()
@@ -146,7 +149,7 @@ private List<SaltEntry> pickSaltsToRotate(
             var maxIndexes = numSaltsToRotate - indexesToRotate.size();
             var saltsToRotate = pickSaltsToRotateInTimeWindow(
                     refreshableSalts,
-                    maxIndexes,
+                    Math.min(maxIndexes, maxSaltsPerAge),
                     minLastUpdated.toEpochMilli(),
                     maxLastUpdated.toEpochMilli()
             );

src/test/java/com/uid2/admin/salt/SaltRotationTest.java

@@ -56,7 +56,7 @@ void teardown() throws Exception {
     }
 
     @Test
-    void rotateSaltsLastSnapshotIsUpToDate() throws Exception {
+    void testRotateSaltsLastSnapshotIsUpToDate() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(1),
                 Duration.ofDays(2),
@@ -74,7 +74,7 @@ void rotateSaltsLastSnapshotIsUpToDate() throws Exception {
     }
 
     @Test
-    void rotateSaltsAllSaltsUpToDate() throws Exception {
+    void testRotateSaltsAllSaltsUpToDate() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(1),
                 Duration.ofDays(2),
@@ -90,7 +90,7 @@ void rotateSaltsAllSaltsUpToDate() throws Exception {
     }
 
     @Test
-    void rotateSaltsAllSaltsOld() throws Exception {
+    void testRotateSaltsAllSaltsOld() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(1),
                 Duration.ofDays(2),
@@ -110,7 +110,7 @@ void rotateSaltsAllSaltsOld() throws Exception {
     }
 
     @Test
-    void rotateSaltsRotateSaltsFromOldestBucketOnly() throws Exception {
+    void testRotateSaltsRotateSaltsFromOldestBucketOnly() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(5),
                 Duration.ofDays(4),
@@ -135,7 +135,7 @@ void rotateSaltsRotateSaltsFromOldestBucketOnly() throws Exception {
     }
 
     @Test
-    void rotateSaltsRotateSaltsFromNewerBucketOnly() throws Exception {
+    void testRotateSaltsRotateSaltsFromNewerBucketOnly() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(5),
                 Duration.ofDays(3),
@@ -158,7 +158,7 @@ void rotateSaltsRotateSaltsFromNewerBucketOnly() throws Exception {
     }
 
     @Test
-    void rotateSaltsRotateSaltsFromMultipleBuckets() throws Exception {
+    void testRotateSaltsRotateSaltsFromMultipleBuckets() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(5),
                 Duration.ofDays(4),
@@ -183,7 +183,7 @@ void rotateSaltsRotateSaltsFromMultipleBuckets() throws Exception {
     }
 
     @Test
-    void rotateSaltsRotateSaltsInsufficientOutdatedSalts() throws Exception {
+    void testRotateSaltsRotateSaltsInsufficientOutdatedSalts() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(5),
                 Duration.ofDays(3),
@@ -232,7 +232,7 @@ void testRefreshFromCalculation(int lastRotationDaysAgo, int lastRotationMsOffse
     }
 
     @Test
-    void rotateSaltsPopulatePreviousSaltsOnRotation() throws Exception {
+    void testRotateSaltsPopulatePreviousSaltsOnRotation() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(90),
                 Duration.ofDays(60),
@@ -260,7 +260,7 @@ void rotateSaltsPopulatePreviousSaltsOnRotation() throws Exception {
     }
 
     @Test
-    void rotateSaltsPreservePreviousSaltsLessThan90DaysOld() throws Exception {
+    void testRotateSaltsPreservePreviousSaltsLessThan90DaysOld() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(60),
         };
@@ -285,7 +285,7 @@ void rotateSaltsPreservePreviousSaltsLessThan90DaysOld() throws Exception {
     }
 
     @Test
-    void rotateSaltsRemovePreviousSaltsOver90DaysOld() throws Exception {
+    void testRotateSaltsRemovePreviousSaltsOver90DaysOld() throws Exception {
         final Duration[] minAges = {
                 Duration.ofDays(100),
         };
@@ -309,9 +309,8 @@ void rotateSaltsRemovePreviousSaltsOver90DaysOld() throws Exception {
         assertNull(salts[1].previousSalt());
     }
 
-
     @Test
-    void rotateSaltsRotateWhenRefreshFromIsTargetDate() throws Exception {
+    void testRotateSaltsRotateWhenRefreshFromIsTargetDate() throws Exception {
         JsonObject config = new JsonObject();
         config.put(AdminConst.ENABLE_SALT_ROTATION_REFRESH_FROM, Boolean.TRUE);
         saltRotation = new SaltRotation(config, keyGenerator);
@@ -351,35 +350,40 @@ void rotateSaltsRotateWhenRefreshFromIsTargetDate() throws Exception {
     }
 
     @Test
-    void logsSaltAgesOnRotation() throws Exception {
+    void testLogFewSaltAgesOnRotation() throws Exception {
         JsonObject config = new JsonObject();
         config.put(AdminConst.ENABLE_SALT_ROTATION_REFRESH_FROM, Boolean.TRUE);
         saltRotation = new SaltRotation(config, keyGenerator);
 
+        // 7 salts total, 5 refreshable, 3 will rotate (6 * 0.4 rounded up), up to 2 will rotate per age (3 * 0.8)
         var lastSnapshot = SaltSnapshotBuilder.start()
                 .entries(
-                        // 5 salts total, 3 refreshable, 2 rotated given 40% fraction
-                        SaltBuilder.start().lastUpdated(daysEarlier(65)).refreshFrom(targetDate()), // Refreshable, old enough, rotated
+                        SaltBuilder.start().lastUpdated(daysEarlier(65)).refreshFrom(targetDate()), // Refreshable, old enough
                         SaltBuilder.start().lastUpdated(daysEarlier(5)).refreshFrom(targetDate()), // Refreshable, too new
+                        SaltBuilder.start().lastUpdated(daysEarlier(33)).refreshFrom(targetDate()), // Refreshable, old enough
                         SaltBuilder.start().lastUpdated(daysEarlier(50)).refreshFrom(daysLater(1)), // Not refreshable, old enough
-                        SaltBuilder.start().lastUpdated(daysEarlier(65)).refreshFrom(targetDate()), // Refreshable, old enough, rotated
+                        SaltBuilder.start().lastUpdated(daysEarlier(65)).refreshFrom(targetDate()), // Refreshable, old enough
+                        SaltBuilder.start().lastUpdated(daysEarlier(65)).refreshFrom(targetDate()), // Refreshable, old enough
                         SaltBuilder.start().lastUpdated(daysEarlier(10)).refreshFrom(daysLater(10)) // Not refreshable, too new
                 )
                 .build();
 
         var expected = Set.of(
                 "[INFO] Salt rotation complete target_date=2025-01-01",
                 // Post-rotation ages, we want to look at current state
-                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=0 salt_count=2", // The two rotated salts, used to be 65 and 50 days old
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=0 salt_count=3",
                 "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=5 salt_count=1",
                 "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=10 salt_count=1",
                 "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=50 salt_count=1",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=65 salt_count=1",
 
                 // Pre-rotation ages, we want to see at which ages salts become refreshable, post rotation some will be 0
                 "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=5 salt_count=1",
-                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=65 salt_count=2",
+                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=33 salt_count=1",
+                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=65 salt_count=3",
 
                 // Pre-rotation ages, post rotation they will all have age 0
+                "[INFO] salt_count_type=rotated-salts target_date=2025-01-01 age=33 salt_count=1",
                 "[INFO] salt_count_type=rotated-salts target_date=2025-01-01 age=65 salt_count=2"
         );
 
@@ -390,12 +394,53 @@ void logsSaltAgesOnRotation() throws Exception {
         assertThat(actual).isEqualTo(expected);
     }
 
+    @Test
+    void testLogManySaltAgesOnRotation() throws Exception {
+        JsonObject config = new JsonObject();
+        config.put(AdminConst.ENABLE_SALT_ROTATION_REFRESH_FROM, Boolean.TRUE);
+        saltRotation = new SaltRotation(config, keyGenerator);
+
+        // 50 salts total, 16 refreshable, 10 will rotate (18 * 0.2 rounded up), up to 8 will rotate per age (10 * 0.8)
+        var lastSnapshot = SaltSnapshotBuilder.start()
+                .entries(10, daysEarlier(5), targetDate()) // Refreshable, too new
+                .entries(10, daysEarlier(10), daysLater(10)) // Not refreshable, too new
+                .entries(10, daysEarlier(33), targetDate()) // Refreshable, old enough
+                .entries(10, daysEarlier(50), daysLater(1)) // Not refreshable, old enough
+                .entries(10, daysEarlier(65), targetDate()) // Refreshable, old enough
+                .build();
+
+        var expected = Set.of(
+                "[INFO] Salt rotation complete target_date=2025-01-01",
+                // Post-rotation ages, we want to look at current state
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=0 salt_count=10",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=5 salt_count=10",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=10 salt_count=10",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=33 salt_count=8",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=50 salt_count=10",
+                "[INFO] salt_count_type=total-salts target_date=2025-01-01 age=65 salt_count=2",
+
+                // Pre-rotation ages, we want to see at which ages salts become refreshable, post rotation some will be 0
+                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=5 salt_count=10",
+                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=33 salt_count=10",
+                "[INFO] salt_count_type=refreshable-salts target_date=2025-01-01 age=65 salt_count=10",
+
+                // Pre-rotation ages, post rotation they will all have age 0
+                "[INFO] salt_count_type=rotated-salts target_date=2025-01-01 age=33 salt_count=2",
+                "[INFO] salt_count_type=rotated-salts target_date=2025-01-01 age=65 salt_count=8"
+        );
+
+        var minAges = new Duration[]{Duration.ofDays(30), Duration.ofDays(60)};
+        saltRotation.rotateSalts(lastSnapshot, minAges, 0.2, targetDate());
+
+        var actual = appender.list.stream().map(Object::toString).collect(Collectors.toSet());
+        assertThat(actual).isEqualTo(expected);
+    }
+
     private int countEntriesWithLastUpdated(SaltEntry[] entries, TargetDate lastUpdated) {
         return countEntriesWithLastUpdated(entries, lastUpdated.asInstant());
     }
 
     private int countEntriesWithLastUpdated(SaltEntry[] entries, Instant lastUpdated) {
         return (int) Arrays.stream(entries).filter(e -> e.lastUpdated() == lastUpdated.toEpochMilli()).count();
     }
-
 }

src/test/java/com/uid2/admin/salt/helper/SaltSnapshotBuilder.java

@@ -1,6 +1,5 @@
 package com.uid2.admin.salt.helper;
 
-import com.uid2.admin.salt.SaltRotation;
 import com.uid2.admin.salt.TargetDate;
 import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.store.salt.RotatingSaltProvider;
@@ -33,6 +32,13 @@ public SaltSnapshotBuilder entries(int count, TargetDate lastUpdated) {
         return this;
     }
 
+    public SaltSnapshotBuilder entries(int count, TargetDate lastUpdated, TargetDate refreshFrom) {
+        for (int i = 0; i < count; ++i) {
+            entries.add(SaltBuilder.start().lastUpdated(lastUpdated).refreshFrom(refreshFrom).build());
+        }
+        return this;
+    }
+
     public SaltSnapshotBuilder entries(SaltBuilder... salts) {
         SaltEntry[] builtSalts = Arrays.stream(salts).map(SaltBuilder::build).toArray(SaltEntry[]::new);
         Collections.addAll(this.entries, builtSalts);