Adding options to only generate valid formatted zips and vins.

Author: jzonthemtn

src/main/java/ai/philterd/phileas/data/generators/VINGenerator.java

@@ -24,23 +24,92 @@
  */
 public class VINGenerator implements DataGenerator.Generator<String> {
     private final Random random;
+    private final boolean onlyValid;
     private final String chars = "0123456789ABCDEFGHJKLMNPRSTUVWXYZ";
 
     /**
      * Creates a new VIN generator.
      * @param random The {@link Random} to use.
      */
     public VINGenerator(final Random random) {
+        this(random, false);
+    }
+
+    /**
+     * Creates a new VIN generator.
+     * @param random The {@link Random} to use.
+     * @param onlyValid If <code>true</code>, only valid VINs will be generated.
+     */
+    public VINGenerator(final Random random, final boolean onlyValid) {
         this.random = random;
+        this.onlyValid = onlyValid;
     }
 
     @Override
     public String random() {
-        final StringBuilder sb = new StringBuilder();
-        for (int i = 0; i < 17; i++) {
-            sb.append(chars.charAt(random.nextInt(chars.length())));
+        if (onlyValid) {
+            return generateValidVin();
+        } else {
+            final StringBuilder sb = new StringBuilder();
+            for (int i = 0; i < 17; i++) {
+                sb.append(chars.charAt(random.nextInt(chars.length())));
+            }
+            return sb.toString();
+        }
+    }
+
+    /**
+     * This generates a VIN that matches the general VIN pattern. These may or may not be actually valid VINs.
+     * @return A VIN that matches the general pattern.
+     */
+    private String generateValidVin() {
+
+        final int[] values = { 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 0, 7, 0, 9, 2, 3, 4, 5, 6, 7, 8, 9 };
+        final int[] weights = { 8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2 };
+
+        String vin = "";
+        boolean valid = false;
+
+        while (!valid) {
+            final StringBuilder sb = new StringBuilder();
+            int sum = 0;
+
+            for (int i = 0; i < 17; i++) {
+                char c;
+                if (i == 8) {
+                    // Check digit placeholder
+                    c = '0';
+                } else {
+                    c = chars.charAt(random.nextInt(chars.length()));
+                }
+
+                sb.append(c);
+
+                int value;
+                if (c >= 'A' && c <= 'Z') {
+                    value = values[c - 'A'];
+                } else {
+                    value = c - '0';
+                }
+
+                sum += value * weights[i];
+            }
+
+            final int checkDigitValue = sum % 11;
+            char checkDigit;
+            if (checkDigitValue == 10) {
+                checkDigit = 'X';
+            } else {
+                checkDigit = (char) (checkDigitValue + '0');
+            }
+
+            sb.setCharAt(8, checkDigit);
+            vin = sb.toString();
+            valid = true;
         }
-        return sb.toString();
+
+        return vin;
+
     }
 
     @Override

src/main/java/ai/philterd/phileas/data/generators/ZipCodeGenerator.java

@@ -17,30 +17,68 @@
 
 import ai.philterd.phileas.data.DataGenerator;
 
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Random;
 
 /**
  * Generates random zip codes.
  */
-public class ZipCodeGenerator implements DataGenerator.Generator<String> {
+public class ZipCodeGenerator extends AbstractGenerator<String> implements DataGenerator.Generator<String> {
     private final Random random;
+    private final boolean onlyValid;
+    private List<String> validZipCodes;
 
     /**
      * Creates a new zip code generator.
      * @param random The {@link Random} to use.
      */
     public ZipCodeGenerator(final Random random) {
+        this(random, false);
+    }
+
+    /**
+     * Creates a new zip code generator.
+     * @param random The {@link Random} to use.
+     * @param onlyValid If <code>true</code>, only valid zip codes from the census will be used.
+     */
+    public ZipCodeGenerator(final Random random, final boolean onlyValid) {
         this.random = random;
+        this.onlyValid = onlyValid;
+
+        if (onlyValid) {
+            try {
+                this.validZipCodes = new ArrayList<>();
+                final List<String> lines = loadNames("/zip-code-population.csv");
+                for (final String line : lines) {
+                    if (!line.startsWith("#")) {
+                        final String[] parts = line.split(",");
+                        validZipCodes.add(parts[0]);
+                    }
+                }
+            } catch (IOException e) {
+                throw new RuntimeException("Unable to load zip code data file.", e);
+            }
+        }
     }
 
     @Override
     public String random() {
-        return String.format("%05d", random.nextInt(100000));
+        if (onlyValid) {
+            return validZipCodes.get(random.nextInt(validZipCodes.size()));
+        } else {
+            return String.format("%05d", random.nextInt(100000));
+        }
     }
 
     @Override
     public long poolSize() {
-        return 100000L;
+        if (onlyValid) {
+            return validZipCodes.size();
+        } else {
+            return 100000L;
+        }
     }
 
 }

src/test/java/ai/philterd/phileas/data/generators/VINGeneratorTest.java

@@ -18,18 +18,118 @@
 import org.junit.jupiter.api.Test;
 
 import java.security.SecureRandom;
+import java.util.regex.Pattern;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class VINGeneratorTest {
 
+    private static final String VIN_PATTERN = "^[A-HJ-NPR-Z0-9]{17}$";
+
     @Test
     public void testGenerateVIN() {
         final VINGenerator generator = new VINGenerator(new SecureRandom());
         final String vin = generator.random();
         assertNotNull(vin);
         assertEquals(17, vin.length());
+        assertTrue(Pattern.matches(VIN_PATTERN, vin));
+    }
+
+    @Test
+    public void testGenerateValidVIN() {
+        final VINGenerator generator = new VINGenerator(new SecureRandom(), true);
+        final String vin = generator.random();
+        assertNotNull(vin);
+        assertEquals(17, vin.length());
+        assertTrue(Pattern.matches(VIN_PATTERN, vin));
+    }
+
+    @Test
+    public void testGenerateMultipleValidVINs() throws Exception {
+        final VINGenerator generator = new VINGenerator(new SecureRandom(), true);
+
+        // We need a VinFilter to verify the VINs.
+        // It requires a FilterConfiguration which we can't easily create here without more setup,
+        // but we can look at how VinFilter handles validation.
+        // Actually, let's just use the same logic as in VinFilter.
+        for (int i = 0; i < 100; i++) {
+            final String vin = generator.random();
+            assertNotNull(vin);
+            assertEquals(17, vin.length());
+            assertTrue(Pattern.matches(VIN_PATTERN, vin));
+            assertTrue(isVinValid(vin), "VIN should be valid: " + vin);
+        }
+    }
+
+    private boolean isVinValid(String vin) {
+        int[] values = { 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 0, 7, 0, 9, 2, 3, 4, 5, 6, 7, 8, 9 };
+        int[] weights = { 8, 7, 6, 5, 4, 3, 2, 10, 0, 9, 8, 7, 6, 5, 4, 3, 2 };
+
+        String s = vin.toUpperCase();
+        if (s.length() != 17) {
+            return false;
+        }
+
+        int sum = 0;
+        for (int i = 0; i < 17; i++) {
+            char c = s.charAt(i);
+            int value;
+            int weight = weights[i];
+
+            if (c >= 'A' && c <= 'Z') {
+                value = values[c - 'A'];
+                if (value == 0 && c != 'I' && c != 'O' && c != 'Q' && i != 8) {
+                    // This is a bit complex because some letters have value 0 in the values array but are not illegal.
+                    // However, I, O, Q are illegal and should have value 0.
+                    // The 9th character (index 8) is the check digit and shouldn't be used in the sum calculation for validation?
+                    // Wait, VinFilter.java line 82 uses weights[i]. weights[8] is 0.
+                    // So it doesn't matter what value is at index 8.
+                }
+            } else if (c >= '0' && c <= '9') {
+                value = c - '0';
+            } else {
+                return false;
+            }
+
+            sum = sum + weight * value;
+        }
+
+        sum = sum % 11;
+        char check = s.charAt(8);
+        if (sum == 10 && check == 'X') {
+            return true;
+        } else if (sum == (check >= '0' && check <= '9' ? check - '0' : transliterate(check))) {
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private int transliterate(char check) {
+
+        if(check == 'A' || check == 'J'){
+            return 1;
+        } else if(check == 'B' || check == 'K' || check == 'S'){
+            return 2;
+        } else if(check == 'C' || check == 'L' || check == 'T'){
+            return 3;
+        } else if(check == 'D' || check == 'M' || check == 'U'){
+            return 4;
+        } else if(check == 'E' || check == 'N' || check == 'V'){
+            return 5;
+        } else if(check == 'F' || check == 'W'){
+            return 6;
+        } else if(check == 'G' || check == 'P' || check == 'X'){
+            return 7;
+        } else if(check == 'H' || check == 'Y'){
+            return 8;
+        } else if(check == 'R' || check == 'Z'){
+            return 9;
+        }
+        return -1;
+
     }
 
     @Test

src/test/java/ai/philterd/phileas/data/generators/ZipCodeGeneratorTest.java

@@ -17,7 +17,13 @@
 
 import org.junit.jupiter.api.Test;
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.security.SecureRandom;
+import java.util.ArrayList;
+import java.util.List;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -33,10 +39,68 @@ public void testGenerateZipCode() {
         assertTrue(zip.matches("\\d{5}"));
     }
 
+    @Test
+    public void testGenerateMultipleZipCodes() {
+        final ZipCodeGenerator generator = new ZipCodeGenerator(new SecureRandom());
+        for (int i = 0; i < 100; i++) {
+            final String zip = generator.random();
+            assertNotNull(zip);
+            assertTrue(zip.matches("\\d{5}"));
+        }
+    }
+
+    @Test
+    public void testGenerateMultipleValidZipCodes() {
+        final ZipCodeGenerator generator = new ZipCodeGenerator(new SecureRandom(), true);
+        for (int i = 0; i < 100; i++) {
+            final String zip = generator.random();
+            assertNotNull(zip);
+            assertTrue(zip.matches("\\d{5}"));
+        }
+    }
+
+    @Test
+    public void testGenerateValidZipCode() throws IOException {
+        final ZipCodeGenerator generator = new ZipCodeGenerator(new SecureRandom(), true);
+        final String zip = generator.random();
+        assertNotNull(zip);
+        assertTrue(zip.matches("\\d{5}"));
+
+        // Verify it is in the list of valid zip codes.
+        final List<String> validZipCodes = new ArrayList<>();
+        try (final InputStream is = getClass().getResourceAsStream("/zip-code-population.csv")) {
+            try (final BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+                String line;
+                while ((line = reader.readLine()) != null) {
+                    if (!line.startsWith("#")) {
+                        final String[] parts = line.split(",");
+                        validZipCodes.add(parts[0]);
+                    }
+                }
+            }
+        }
+
+        assertTrue(validZipCodes.contains(zip));
+        assertEquals(validZipCodes.size(), generator.poolSize());
+    }
+
     @Test
     public void testPoolSize() {
         final ZipCodeGenerator generator = new ZipCodeGenerator(new SecureRandom());
         assertEquals(100000L, generator.poolSize());
     }
 
+    @Test
+    public void testVariety() {
+        final ZipCodeGenerator generator = new ZipCodeGenerator(new SecureRandom());
+        final List<String> zips = new ArrayList<>();
+
+        for(int i = 0; i < 100; i++) {
+            zips.add(generator.random());
+        }
+
+        // We should have some variety in the generated zip codes.
+        assertTrue(zips.stream().distinct().count() > 1);
+    }
+
 }