BoxStoreBuilder: add key value validation options #186

Author: greenrobot

objectbox-java/src/main/java/io/objectbox/BoxStoreBuilder.java

@@ -41,6 +41,7 @@
 import io.objectbox.ideasonly.ModelUpdate;
 import io.objectbox.model.FlatStoreOptions;
 import io.objectbox.model.ValidateOnOpenMode;
+import io.objectbox.model.ValidateOnOpenModeKv;
 import org.greenrobot.essentials.io.IoUtils;
 
 /**
@@ -81,8 +82,10 @@ public class BoxStoreBuilder {
     long maxDataSizeInKByte;
 
     /** On Android used for native library loading. */
-    @Nullable Object context;
-    @Nullable Object relinker;
+    @Nullable
+    Object context;
+    @Nullable
+    Object relinker;
 
     ModelUpdate modelUpdate;
 
@@ -105,9 +108,11 @@ public class BoxStoreBuilder {
     boolean readOnly;
     boolean usePreviousCommit;
 
-    short validateOnOpenMode;
+    short validateOnOpenModePages;
     long validateOnOpenPageLimit;
 
+    short validateOnOpenModeKv;
+
     TxCallback<?> failedReadTxAttemptCallback;
 
     final List<EntityInfo<?>> entityInfoList = new ArrayList<>();
@@ -404,14 +409,17 @@ public BoxStoreBuilder usePreviousCommit() {
      * OSes, file systems, or hardware.
      * <p>
      * Note: ObjectBox builds upon ACID storage, which already has strong consistency mechanisms in place.
+     * <p>
+     * See also {@link #validateOnOpenPageLimit(long)} to fine-tune this check and {@link #validateOnOpenKv(short)} for
+     * additional checks.
      *
      * @param validateOnOpenMode One of {@link ValidateOnOpenMode}.
      */
     public BoxStoreBuilder validateOnOpen(short validateOnOpenMode) {
         if (validateOnOpenMode < ValidateOnOpenMode.None || validateOnOpenMode > ValidateOnOpenMode.Full) {
             throw new IllegalArgumentException("Must be one of ValidateOnOpenMode");
         }
-        this.validateOnOpenMode = validateOnOpenMode;
+        this.validateOnOpenModePages = validateOnOpenMode;
         return this;
     }
 
@@ -423,7 +431,7 @@ public BoxStoreBuilder validateOnOpen(short validateOnOpenMode) {
      * This can only be used with {@link ValidateOnOpenMode#Regular} and {@link ValidateOnOpenMode#WithLeaves}.
      */
     public BoxStoreBuilder validateOnOpenPageLimit(long limit) {
-        if (validateOnOpenMode != ValidateOnOpenMode.Regular && validateOnOpenMode != ValidateOnOpenMode.WithLeaves) {
+        if (validateOnOpenModePages != ValidateOnOpenMode.Regular && validateOnOpenModePages != ValidateOnOpenMode.WithLeaves) {
             throw new IllegalStateException("Must call validateOnOpen(mode) with mode Regular or WithLeaves first");
         }
         if (limit < 1) {
@@ -433,6 +441,33 @@ public BoxStoreBuilder validateOnOpenPageLimit(long limit) {
         return this;
     }
 
+    /**
+     * When a database is opened, ObjectBox can perform additional consistency checks on its database structure.
+     * This enables validation checks on a key/value level.
+     * <p>
+     * This is a shortcut for {@link #validateOnOpenKv(short) validateOnOpenKv(ValidateOnOpenModeKv.Regular)}.
+     */
+    public BoxStoreBuilder validateOnOpenKv() {
+        this.validateOnOpenModeKv = ValidateOnOpenModeKv.Regular;
+        return this;
+    }
+
+    /**
+     * When a database is opened, ObjectBox can perform additional consistency checks on its database structure.
+     * This enables validation checks on a key/value level.
+     * <p>
+     * See also {@link #validateOnOpen(short)} for additional consistency checks.
+     *
+     * @param mode One of {@link ValidateOnOpenMode}.
+     */
+    public BoxStoreBuilder validateOnOpenKv(short mode) {
+        if (mode < ValidateOnOpenModeKv.Regular || mode > ValidateOnOpenMode.Regular) {
+            throw new IllegalArgumentException("Must be one of ValidateOnOpenModeKv");
+        }
+        this.validateOnOpenModeKv = mode;
+        return this;
+    }
+
     /**
      * @deprecated Use {@link #debugFlags} instead.
      */
@@ -465,7 +500,7 @@ public BoxStoreBuilder debugRelations() {
      * {@link DbException} are thrown during query execution).
      *
      * @param queryAttempts number of attempts a query find operation will be executed before failing.
-     * Recommended values are in the range of 2 to 5, e.g. a value of 3 as a starting point.
+     *                      Recommended values are in the range of 2 to 5, e.g. a value of 3 as a starting point.
      */
     @Experimental
     public BoxStoreBuilder queryAttempts(int queryAttempts) {
@@ -519,12 +554,15 @@ byte[] buildFlatStoreOptions(String canonicalPath) {
         FlatStoreOptions.addMaxDbSizeInKbyte(fbb, maxSizeInKByte);
         FlatStoreOptions.addFileMode(fbb, fileMode);
         FlatStoreOptions.addMaxReaders(fbb, maxReaders);
-        if (validateOnOpenMode != 0) {
-            FlatStoreOptions.addValidateOnOpenPages(fbb, validateOnOpenMode);
+        if (validateOnOpenModePages != 0) {
+            FlatStoreOptions.addValidateOnOpenPages(fbb, validateOnOpenModePages);
             if (validateOnOpenPageLimit != 0) {
                 FlatStoreOptions.addValidateOnOpenPageLimit(fbb, validateOnOpenPageLimit);
             }
         }
+        if (validateOnOpenModeKv != 0) {
+            FlatStoreOptions.addValidateOnOpenKv(fbb, validateOnOpenModeKv);
+        }
         if (skipReadSchema) FlatStoreOptions.addSkipReadSchema(fbb, true);
         if (usePreviousCommit) FlatStoreOptions.addUsePreviousCommit(fbb, true);
         if (readOnly) FlatStoreOptions.addReadOnly(fbb, true);

tests/objectbox-java-test/src/test/java/io/objectbox/BoxStoreValidationTest.java

@@ -5,13 +5,16 @@
 import java.io.IOException;
 import java.io.InputStream;
 
+import io.objectbox.exception.FileCorruptException;
 import io.objectbox.exception.PagesCorruptException;
 import io.objectbox.model.ValidateOnOpenMode;
 import org.greenrobot.essentials.io.IoUtils;
 import org.junit.Before;
 import org.junit.Test;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertThrows;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
@@ -38,14 +41,7 @@ public void setUpBuilder() {
     public void validateOnOpen() {
         // Create a database first; we must create the model only once (ID/UID sequences would be different 2nd time)
         byte[] model = createTestModel(null);
-        builder = new BoxStoreBuilder(model).directory(boxStoreDir);
-        builder.entity(new TestEntity_());
-        store = builder.build();
-
-        TestEntity object = new TestEntity(0);
-        object.setSimpleString("hello hello");
-        long id = getTestEntityBox().put(object);
-        store.close();
+        long id = buildNotCorruptedDatabase(model);
 
         // Then re-open database with validation and ensure db is operational
         builder = new BoxStoreBuilder(model).directory(boxStoreDir);
@@ -57,27 +53,26 @@ public void validateOnOpen() {
     }
 
 
-    @Test(expected = PagesCorruptException.class)
+    @Test
     public void validateOnOpenCorruptFile() throws IOException {
         File dir = prepareTempDir("object-store-test-corrupted");
-        File badDataFile = prepareBadDataFile(dir);
+        prepareBadDataFile(dir, "corrupt-pageno-in-branch-data.mdb");
 
         builder = BoxStoreBuilder.createDebugWithoutModel().directory(dir);
         builder.validateOnOpen(ValidateOnOpenMode.Full);
-        try {
-            store = builder.build();
-        } finally {
-            boolean delOk = badDataFile.delete();
-            delOk &= new File(dir, "lock.mdb").delete();
-            delOk &= dir.delete();
-            assertTrue(delOk);  // Try to delete all before asserting
-        }
+
+        @SuppressWarnings("resource")
+        FileCorruptException ex = assertThrows(PagesCorruptException.class, () -> builder.build());
+        assertEquals("Validating pages failed (page not found)", ex.getMessage());
+
+        // Clean up
+        deleteAllFiles(dir);
     }
 
     @Test
     public void usePreviousCommitWithCorruptFile() throws IOException {
         File dir = prepareTempDir("object-store-test-corrupted");
-        prepareBadDataFile(dir);
+        prepareBadDataFile(dir, "corrupt-pageno-in-branch-data.mdb");
         builder = BoxStoreBuilder.createDebugWithoutModel().directory(dir);
         builder.validateOnOpen(ValidateOnOpenMode.Full).usePreviousCommit();
         store = builder.build();
@@ -91,7 +86,7 @@ public void usePreviousCommitWithCorruptFile() throws IOException {
     @Test
     public void usePreviousCommitAfterFileCorruptException() throws IOException {
         File dir = prepareTempDir("object-store-test-corrupted");
-        prepareBadDataFile(dir);
+        prepareBadDataFile(dir, "corrupt-pageno-in-branch-data.mdb");
         builder = BoxStoreBuilder.createDebugWithoutModel().directory(dir);
         builder.validateOnOpen(ValidateOnOpenMode.Full);
         try {
@@ -109,15 +104,65 @@ public void usePreviousCommitAfterFileCorruptException() throws IOException {
         assertTrue(store.deleteAllFiles());
     }
 
-    private File prepareBadDataFile(File dir) throws IOException {
+    @Test
+    public void validateOnOpenKv() {
+        // Create a database first; we must create the model only once (ID/UID sequences would be different 2nd time)
+        byte[] model = createTestModel(null);
+        long id = buildNotCorruptedDatabase(model);
+
+        // Then re-open database with validation and ensure db is operational
+        builder = new BoxStoreBuilder(model).directory(boxStoreDir);
+        builder.entity(new TestEntity_());
+        builder.validateOnOpenKv();
+        store = builder.build();
+        assertNotNull(getTestEntityBox().get(id));
+        getTestEntityBox().put(new TestEntity(0));
+    }
+
+    @Test
+    public void validateOnOpenKvCorruptFile() throws IOException {
+        File dir = prepareTempDir("obx-store-validate-kv-corrupted");
+        prepareBadDataFile(dir, "corrupt-keysize0-data.mdb");
+
+        builder = BoxStoreBuilder.createDebugWithoutModel().directory(dir);
+        builder.validateOnOpenKv();
+
+        @SuppressWarnings("resource")
+        FileCorruptException ex = assertThrows(FileCorruptException.class, () -> builder.build());
+        assertEquals("KV validation failed; key is empty (KV pair number: 1, key size: 0, data size: 112)",
+                ex.getMessage());
+
+        // Clean up
+        deleteAllFiles(dir);
+    }
+
+    /**
+     * Returns the id of the inserted test entity.
+     */
+    private long buildNotCorruptedDatabase(byte[] model) {
+        builder = new BoxStoreBuilder(model).directory(boxStoreDir);
+        builder.entity(new TestEntity_());
+        store = builder.build();
+
+        TestEntity object = new TestEntity(0);
+        object.setSimpleString("hello hello");
+        long id = getTestEntityBox().put(object);
+        store.close();
+        return id;
+    }
+
+    /**
+     * Copies the given file from resources to the given directory as "data.mdb".
+     */
+    private void prepareBadDataFile(File dir, String resourceName) throws IOException {
         assertTrue(dir.mkdir());
         File badDataFile = new File(dir, "data.mdb");
-        try (InputStream badIn = getClass().getResourceAsStream("corrupt-pageno-in-branch-data.mdb")) {
+        try (InputStream badIn = getClass().getResourceAsStream(resourceName)) {
+            assertNotNull(badIn);
             try (FileOutputStream badOut = new FileOutputStream(badDataFile)) {
                 IoUtils.copyAllBytes(badIn, badOut);
             }
         }
-        return badDataFile;
     }
 
 }