Fixed OOM during error message construction (#790)

Co-authored-by: Jegan <jponrama@netflix.com>

Author: spjegan

hollow/src/main/java/com/netflix/hollow/api/producer/validation/DuplicateDataDetectionValidator.java

@@ -46,6 +46,7 @@
  * record's type will never fail.
  */
 public class DuplicateDataDetectionValidator implements ValidatorListener {
+    private static final int MAX_DISPLAYED_DUPLICATE_KEYS = 100;
     private static final String DUPLICATE_KEYS_FOUND_ERRRO_MSG_FORMAT =
             "Duplicate keys found for type %s. Primarykey in schema is %s. "
                     + "Duplicate IDs are: %s";
@@ -189,7 +190,17 @@ public int hashCode() {
     }
 
     private String duplicateKeysToString(Collection<Object[]> duplicateKeys) {
-        return duplicateKeys.stream().map(Arrays::toString).collect(Collectors.joining(","));
+        long totalCount = duplicateKeys.size();
+        String duplicateKeysString = duplicateKeys.stream()
+                .limit(MAX_DISPLAYED_DUPLICATE_KEYS)
+                .map(Arrays::toString)
+                .collect(Collectors.joining(", "));
+
+        if (totalCount > MAX_DISPLAYED_DUPLICATE_KEYS) {
+            return String.format("%s ... (showing %d of %d duplicates)",
+                    duplicateKeysString, MAX_DISPLAYED_DUPLICATE_KEYS, totalCount);
+        }
+        return duplicateKeysString;
     }
 
     /**

hollow/src/main/java/com/netflix/hollow/api/producer/validation/NullPrimaryKeyFieldValidator.java

@@ -27,6 +27,7 @@
  * instantiating this validator.
  */
 public class NullPrimaryKeyFieldValidator implements ValidatorListener {
+    private static final int MAX_DISPLAYED_NULL_KEYS = 100;
     private static final String NAME = NullPrimaryKeyFieldValidator.class.getName();
     private static final String NULL_PRIMARY_KEYS_FOUND_ERROR_MSG_FORMAT =
             "Null primary key fields found for type %s. Primary Key in schema is %s. "
@@ -171,10 +172,18 @@ private Object getPrimaryKeyFieldValue(HollowObjectTypeReadState typeState, int[
     }
 
     private String nullKeysToString(Map<Integer, Object[]> nullPrimaryKeyValues) {
-        return nullPrimaryKeyValues.entrySet().stream()
+        long totalCount = nullPrimaryKeyValues.size();
+        String nullPrimaryKeysString = nullPrimaryKeyValues.entrySet().stream()
+                .limit(MAX_DISPLAYED_NULL_KEYS)
                 .map(entry -> {
                     return "(ordinal=" + entry.getKey() + ", key=" + Arrays.toString(entry.getValue()) + ")";
                 })
                 .collect(Collectors.joining(", "));
+
+        if (totalCount > MAX_DISPLAYED_NULL_KEYS) {
+            return String.format("%s ... (showing %d of %d null keys)",
+                    nullPrimaryKeysString, MAX_DISPLAYED_NULL_KEYS, totalCount);
+        }
+        return nullPrimaryKeysString;
     }
 }

hollow/src/test/java/com/netflix/hollow/api/producer/validation/DuplicateDataDetectionValidatorTests.java

@@ -18,6 +18,7 @@
 
 import com.netflix.hollow.api.producer.HollowProducer;
 import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager;
+import com.netflix.hollow.core.write.objectmapper.HollowPrimaryKey;
 import com.netflix.hollow.test.InMemoryBlobStore;
 import org.junit.Assert;
 import org.junit.Before;
@@ -44,4 +45,36 @@ public void failTestMissingSchema() {
                     .endsWith("(see initializeDataModel)"));
         }
     }
+
+    @Test
+    public void test_errorMessageIsTruncatedWhenThereAreManyDuplicates() {
+        HollowProducer producer = HollowProducer.withPublisher(blobStore)
+                .withBlobStager(new HollowInMemoryBlobStager())
+                .withListener(new DuplicateDataDetectionValidator(TypeWithPrimaryKey.class))
+                .build();
+
+        try {
+            producer.runCycle(writeState -> {
+                // Add 1000 records with the same primary key
+                for (int i = 0; i < 1000; i++) {
+                    writeState.add(new TypeWithPrimaryKey(1, "Duplicate" + i));
+                }
+            });
+            Assert.fail("Expected ValidationStatusException");
+        } catch (ValidationStatusException expected) {
+            String message = expected.getValidationStatus().getResults().get(0).getMessage();
+            Assert.assertTrue(message.contains("showing 100 of 499500 duplicates")); // 499500 is the number of duplicate pairs for 1000 identical keys (n * (n-1)/2)
+        }
+    }
+
+    @HollowPrimaryKey(fields = {"id"})
+    static class TypeWithPrimaryKey {
+        int id;
+        String name;
+
+        TypeWithPrimaryKey(int id, String name) {
+            this.id = id;
+            this.name = name;
+        }
+    }
 }

hollow/src/test/java/com/netflix/hollow/api/producer/validation/NullPrimaryKeyFieldValidatorTest.java

@@ -154,6 +154,29 @@ public void testInvalidNullNestedPrimaryKey() {
         }
     }
 
+    @Test
+    public void test_errorMessageIsTruncatedWhenThereAreManyNullKeys() {
+        HollowProducer producer = HollowProducer.withPublisher(blobStore)
+                .withBlobStager(new HollowInMemoryBlobStager())
+                .withListener(new NullPrimaryKeyFieldValidator(TypeWithNullablePrimaryKeyAndData.class))
+                .build();
+
+        try {
+            producer.runCycle(state -> {
+                // Add 1000 records with null primary keys but different data fields
+                for (int i = 0; i < 1000; i++) {
+                    state.add(new TypeWithNullablePrimaryKeyAndData(null, "data" + i));
+                }
+            });
+            fail("Expected ValidationStatusException");
+        } catch (Exception e) {
+            assertTrue(e instanceof ValidationStatusException);
+            ValidationStatusException expected = (ValidationStatusException) e;
+            String message = expected.getValidationStatus().getResults().get(0).getMessage();
+            assertTrue(message.contains("showing 100 of 1000 null keys"));
+        }
+    }
+
 
     @HollowPrimaryKey(fields = "id")
     static class TypeWithSinglePrimaryKey {
@@ -191,4 +214,15 @@ public TypeWithoutPrimaryKey(Integer id) {
             this.id = id;
         }
     }
+
+    @HollowPrimaryKey(fields = "id")
+    static class TypeWithNullablePrimaryKeyAndData {
+        private final Integer id;
+        private final String data;
+
+        public TypeWithNullablePrimaryKeyAndData(Integer id, String data) {
+            this.id = id;
+            this.data = data;
+        }
+    }
 }