Merge pull request #568 from IABTechLab/sch-UID2-5853-metrics-and-dashboard-for-key-rotation

sch-UID2-5853 Added logs for key bucket count in salt rotation

Author: sophia-chen-ttd

src/main/java/com/uid2/admin/salt/SaltRotation.java

@@ -62,6 +62,7 @@ public Result rotateSalts(
         logSaltAges("refreshable-salts", targetDate, refreshableSalts);
         logSaltAges("rotated-salts", targetDate, saltsToRotate);
         logSaltAges("total-salts", targetDate, Arrays.asList(postRotationSalts));
+        logBucketFormatCount(targetDate, postRotationSalts);
 
         var nextSnapshot = new SaltSnapshot(
                 nextEffective,
@@ -252,6 +253,24 @@ private void logSaltAges(String saltCountType, TargetDate targetDate, Collection
         }
     }
 
+
+    /** Logging to monitor migration of buckets from salts (old format - v2/v3) to encryption keys (new format - v4) **/
+    private void logBucketFormatCount(TargetDate targetDate, SaltEntry[] postRotationBuckets) {
+        int totalKeys = 0, totalSalts = 0, totalPreviousKeys = 0, totalPreviousSalts = 0;
+
+        for (SaltEntry bucket : postRotationBuckets) {
+            if (bucket.currentKeySalt() != null) totalKeys++;
+            if (bucket.currentSalt() != null) totalSalts++;
+            if (bucket.previousKeySalt() != null) totalPreviousKeys++;
+            if (bucket.previousSalt() != null) totalPreviousSalts++;
+        }
+
+        LOGGER.info("UID bucket format: target_date={} bucket_format={} bucket_count={}", targetDate, "total-current-key-buckets", totalKeys);
+        LOGGER.info("UID bucket format: target_date={} bucket_format={} bucket_count={}", targetDate, "total-current-salt-buckets", totalSalts);
+        LOGGER.info("UID bucket format: target_date={} bucket_format={} bucket_count={}", targetDate, "total-previous-key-buckets", totalPreviousKeys);
+        LOGGER.info("UID bucket format: target_date={} bucket_format={} bucket_count={}", targetDate, "total-previous-salt-buckets", totalPreviousSalts);
+    }
+
     @Getter
     public static final class Result {
         private final SaltSnapshot snapshot; // can be null if new snapshot is not needed

src/main/java/com/uid2/admin/store/writer/SaltStoreWriter.java

@@ -4,7 +4,6 @@
 import com.uid2.admin.store.version.VersionGenerator;
 import com.uid2.shared.cloud.CloudStorageException;
 import com.uid2.shared.cloud.TaggableCloudStorage;
-import com.uid2.shared.model.SaltEntry;
 import com.uid2.shared.store.CloudPath;
 import com.uid2.shared.store.salt.RotatingSaltProvider;
 import io.vertx.core.json.JsonArray;

src/test/java/com/uid2/admin/salt/SaltRotationTest.java

@@ -388,7 +388,7 @@ void testLogFewSaltAgesOnRotation() throws Exception {
         var minAges = new Duration[]{Duration.ofDays(30), Duration.ofDays(60)};
         saltRotation.rotateSalts(lastSnapshot, minAges, 0.4, targetDate());
 
-        var actual = appender.list.stream().map(Object::toString).collect(Collectors.toSet());
+        var actual = appender.list.stream().map(Object::toString).filter(s -> s.contains("salt_count_type") || s.contains("Salt rotation complete")).collect(Collectors.toSet());
         assertThat(actual).isEqualTo(expected);
     }
 
@@ -428,7 +428,7 @@ void testLogManySaltAgesOnRotation() throws Exception {
         var minAges = new Duration[]{Duration.ofDays(30), Duration.ofDays(60)};
         saltRotation.rotateSalts(lastSnapshot, minAges, 0.2, targetDate());
 
-        var actual = appender.list.stream().map(Object::toString).collect(Collectors.toSet());
+        var actual = appender.list.stream().map(Object::toString).filter(s -> s.contains("salt_count_type") || s.contains("Salt rotation complete")).collect(Collectors.toSet());
         assertThat(actual).isEqualTo(expected);
     }
 
@@ -656,4 +656,38 @@ void testKeyRotationKeyToSaltRotation() throws Exception {
         assertThat(buckets[0].currentKeySalt()).isNull();
         assertThat(buckets[0].previousKeySalt()).isEqualTo(new SaltEntry.KeyMaterial(0, "keyKey1", "keySalt1"));
     }
+
+    @ParameterizedTest
+    @CsvSource({
+            "true, 3, 1",
+            "false, 1, 3"
+    })
+    void testKeyRotationLogBucketFormat(boolean v4Enabled, int expectedTotalKeyBuckets, int expectedTotalSaltBuckets) throws Exception {
+        saltRotation = new SaltRotation(keyGenerator, JsonObject.of(AdminConst.ENABLE_V4_RAW_UID, v4Enabled));
+        when(keyGenerator.generateRandomKeyString(anyInt())).thenReturn("random-key-string");
+
+        final Duration[] minAges = {
+                Duration.ofDays(30)
+        };
+
+        var willRefresh = targetDate();
+        var willNotRefresh = targetDate().plusDays(30);
+        var lastSnapshot = SaltSnapshotBuilder.start()
+                .entries(SaltBuilder.start().lastUpdated(targetDate().minusDays(60)).refreshFrom(willRefresh).currentSalt(),
+                        SaltBuilder.start().lastUpdated(targetDate().minusDays(60)).refreshFrom(willNotRefresh).currentSalt(),
+                        SaltBuilder.start().lastUpdated(targetDate().minusDays(60)).refreshFrom(willRefresh).currentKeySalt(1),
+                        SaltBuilder.start().lastUpdated(targetDate().minusDays(60)).refreshFrom(willNotRefresh).currentKeySalt(2))
+                .build();
+
+        saltRotation.rotateSalts(lastSnapshot, minAges, 1, targetDate());
+
+        var expected = Set.of(
+                String.format("[INFO] UID bucket format: target_date=2025-01-01 bucket_format=total-current-key-buckets bucket_count=%d", expectedTotalKeyBuckets),
+                String.format("[INFO] UID bucket format: target_date=2025-01-01 bucket_format=total-current-salt-buckets bucket_count=%d", expectedTotalSaltBuckets),
+                String.format("[INFO] UID bucket format: target_date=2025-01-01 bucket_format=total-previous-key-buckets bucket_count=%d", 1),
+                String.format("[INFO] UID bucket format: target_date=2025-01-01 bucket_format=total-previous-salt-buckets bucket_count=%d", 1)
+        );
+        var actual = appender.list.stream().map(Object::toString).filter(s -> s.contains("UID bucket format")).collect(Collectors.toSet());
+        assertThat(actual).isEqualTo(expected);
+    }
 }