[feature] Add checksums for binary files to the Journal

Author: adamretter

src/org/exist/storage/AbstractBinaryLoggable.java

@@ -23,8 +23,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.exist.storage.journal.AbstractLoggable;
+import org.exist.util.crypto.digest.DigestType;
+import org.exist.util.crypto.digest.MessageDigest;
 
 import javax.annotation.Nullable;
+import java.nio.ByteBuffer;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
@@ -36,6 +39,8 @@
 public abstract class AbstractBinaryLoggable extends AbstractLoggable {
     private static final Logger LOG = LogManager.getLogger(AbstractBinaryLoggable.class);
 
+    protected final static byte NO_DIGEST_TYPE = 0x0;
+
     public AbstractBinaryLoggable(final byte type, final long transactionId) {
         super(type, transactionId);
     }
@@ -111,4 +116,38 @@ protected static void checkPathLen(final String loggableName, final String pathN
             LOG.error(loggableName + ": " + pathName + " path needs more than 65,535 bytes. Path will be truncated: " + new String(path, UTF_8));
         }
     }
+
+    /**
+     * Writes a message digest to a buffer.
+     *
+     * @param out the buffer to write the message digest to.
+     * @param messageDigest the message digest to write to the buffer.
+     */
+    protected static void writeMessageDigest(final ByteBuffer out, @Nullable final MessageDigest messageDigest) {
+        if (messageDigest == null) {
+            out.put(NO_DIGEST_TYPE);
+        } else {
+            out.put(messageDigest.getDigestType().getId());
+            out.put(messageDigest.getValue(), 0, messageDigest.getDigestType().getDigestLengthBytes());
+        }
+    }
+
+    /**
+     * Reads a message digest from a buffer.
+     *
+     * @param in the buffer to read the message digest from.
+     *
+     * @return the message digest read from the buffer.
+     */
+    protected static @Nullable MessageDigest readMessageDigest(final ByteBuffer in) {
+        final byte digestTypeId = in.get();
+        if (digestTypeId == NO_DIGEST_TYPE) {
+            return null;
+        } else {
+            final DigestType digestType = DigestType.forId(digestTypeId);
+            final byte[] digestValue = new byte[digestType.getDigestLengthBytes()];
+            in.get(digestValue);
+            return new MessageDigest(digestType, digestValue);
+        }
+    }
 }

src/org/exist/storage/CreateBinaryLoggable.java

@@ -25,38 +25,49 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.util.Arrays;
 
 import org.exist.storage.journal.LogException;
 import org.exist.storage.txn.Txn;
+import org.exist.util.FileUtils;
+import org.exist.util.crypto.digest.MessageDigest;
+import org.exist.util.crypto.digest.StreamableDigest;
 
 /**
  * @author Adam Retter <[email protected]>
  *
  * Serialized binary format is as follows:
  *
- * [walDataPathLen, walDataPath, createPathLen, createPath]
+ * [walDataPathLen, walDataPath, walDataDigestType, walDataDigest, createPathLen, createPath]
  *
- * walDataPathLen:  2 bytes, unsigned short
- * walDataPath:     var length bytes, UTF-8 encoded java.lang.String
- * createPathLen:   2 bytes, unsigned short
- * createPath:      var length bytes, UTF-8 encoded java.lang.String
+ * walDataPathLen:          2 bytes, unsigned short
+ * walDataPath:             var length bytes, UTF-8 encoded java.lang.String
+ * walDataDigestType:       1 byte
+ * walDataDigest:           n-bytes, where n is deteremined by {@code walDataDigestType}
+ * createPathLen:           2 bytes, unsigned short
+ * createPath:              var length bytes, UTF-8 encoded java.lang.String
  */
 public class CreateBinaryLoggable extends AbstractBinaryLoggable {
-    private byte[] walDataPath;  // the data to use for the file to be created
-    private byte[] createPath;   // the file to be created
+    private byte[] walDataPath;             // the data to use for the file to be created
+    private MessageDigest walDataDigest;
+    private byte[] createPath;              // the file to be created
 
     /**
      * Creates a new instance of CreateBinaryLoggable.
      *
      * @param broker The database broker.
      * @param txn The database transaction.
      * @param walData A copy of the data that was stored for {@code create} file before it was actually created (i.e. the new value).
+     * @param walDataDigest digest of the {@code walData} content
      * @param create The file that is to be created in the database.
      */
-    public CreateBinaryLoggable(final DBBroker broker, final Txn txn, final Path walData, final Path create) {
+    public CreateBinaryLoggable(final DBBroker broker, final Txn txn, final Path walData,
+            final MessageDigest walDataDigest, final Path create) {
         super(NativeBroker.LOG_CREATE_BINARY, txn.getId());
         this.walDataPath = getPathData(walData);
         checkPathLen(getClass().getSimpleName(), "walDataPath", walDataPath);
+        this.walDataDigest = walDataDigest;
+
         this.createPath = getPathData(create);
         checkPathLen(getClass().getSimpleName(), "createPath", createPath);
     }
@@ -76,6 +87,8 @@ public int getLogSize() {
         return
                 2 +
                 walDataPath.length +
+                1 +
+                walDataDigest.getDigestType().getDigestLengthBytes() +
                 2 +
                 createPath.length;
     }
@@ -84,6 +97,8 @@ public int getLogSize() {
     public void write(final ByteBuffer out) {
         out.putShort(asUnsignedShort(walDataPath.length));
         out.put(walDataPath);
+        writeMessageDigest(out, walDataDigest);
+
         out.putShort(asUnsignedShort(createPath.length));
         out.put(createPath);
     }
@@ -93,6 +108,7 @@ public void read(final ByteBuffer in) {
         final int walDataPathLen = asSignedInt(in.getShort());
         this.walDataPath = new byte[walDataPathLen];
         in.get(walDataPath);
+        this.walDataDigest = readMessageDigest(in);
 
         final int createPathLen = asSignedInt(in.getShort());
         this.createPath = new byte[createPathLen];
@@ -113,7 +129,28 @@ public void redo() throws LogException {
         }
 
         try {
+            // ensure the integrity of the walData file
+            final StreamableDigest walDataStreamableDigest = walDataDigest.getDigestType().newStreamableDigest();
+            FileUtils.digest(walData, walDataStreamableDigest);
+            if (!Arrays.equals(walDataStreamableDigest.getMessageDigest(), walDataDigest.getValue())) {
+                throw new LogException("Cannot redo creation of binary resource: "
+                        + create.toAbsolutePath().toString() + " from "
+                        + walData.toAbsolutePath().toString() + ", digest of walData file is invalid");
+            }
+
+            // perform the redo - copy
             Files.copy(walData, create, StandardCopyOption.REPLACE_EXISTING);
+
+            // ensure the integrity of the copy
+            walDataStreamableDigest.reset();
+            FileUtils.digest(create, walDataStreamableDigest);
+            if (!Arrays.equals(walDataStreamableDigest.getMessageDigest(), walDataDigest.getValue())) {
+                FileUtils.deleteQuietly(create);
+                throw new LogException("Cannot redo creation of binary resource: "
+                        + create.toAbsolutePath().toString() + " from "
+                        + walData.toAbsolutePath().toString() + ", checksum of new create file is invalid");
+            }
+
         } catch(final IOException ioe) {
             throw new LogException("Cannot redo creation of binary resource: "
                     + create.toAbsolutePath().toString(), ioe);
@@ -127,20 +164,36 @@ public void undo() throws LogException {
         final Path walData = getPath(walDataPath);
         final Path create = getPath(createPath);
 
-        // ensure integrity of write-ahead-data file first!
-        if (!Files.exists(walData)) {
-            throw new LogException("Cannot redo creation of binary resource: "
-                    + create.toAbsolutePath().toString() + ", missing write ahead data: "
-                    + walData.toAbsolutePath().toString());
-        }
-
-
-        // cover the use-case where the previous operation was a delete
-        if (!Files.exists(create)) {
-            return;
-        }
-
         try {
+            // ensure integrity of the walData file first!
+            if (!Files.exists(walData)) {
+                throw new LogException("Cannot redo creation of binary resource: "
+                        + create.toAbsolutePath().toString() + ", missing write ahead data: "
+                        + walData.toAbsolutePath().toString());
+            }
+
+            // ensure integrity of walData file by checksum
+            final StreamableDigest walDataStreamableDigest = walDataDigest.getDigestType().newStreamableDigest();
+            FileUtils.digest(walData, walDataStreamableDigest);
+            if (!Arrays.equals(walDataStreamableDigest.getMessageDigest(), walDataDigest.getValue())) {
+                throw new LogException("Cannot undo creation of binary resource: "
+                        + create.toAbsolutePath().toString() + ", checksum of walData file is invalid");
+            }
+
+            // cover the use-case where the previous operation was a delete
+            if (!Files.exists(create)) {
+                return;
+            }
+
+            // ensure that no one has interfered with the createdFile
+            walDataStreamableDigest.reset();
+            FileUtils.digest(create, walDataStreamableDigest);
+            if (!Arrays.equals(walDataStreamableDigest.getMessageDigest(), walDataDigest.getValue())) {
+                throw new LogException("Cannot undo creation of binary resource: "
+                        + create.toAbsolutePath().toString() + ", checksum is invalid");
+            }
+
+            // preform the undo - delete
             Files.delete(create);
         } catch(final IOException ioe) {
             throw new LogException("Cannot undo creation of binary resource: "

src/org/exist/storage/DeleteBinaryLoggable.java

@@ -23,42 +23,57 @@
 import org.exist.storage.journal.LogException;
 import org.exist.storage.txn.Txn;
 import org.exist.util.FileUtils;
+import org.exist.util.crypto.digest.MessageDigest;
+import org.exist.util.crypto.digest.StreamableDigest;
 
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 
 /**
  * @author Adam Retter <[email protected]>
  *
  * Serialized binary format is as follows:
  *
- * [deletePathLen, deletePath, dataPathLen, dataPath]
+ * [deletePathLen, deletePath, deleteDigestType, deleteDigest, dataPathLen, dataPath, dataDigestType, dataDigest]
  *
- * deletePathLen:  2 bytes, unsigned short
- * deletePath:     var length bytes, UTF-8 encoded java.lang.String
- * dataPathLen:    2 bytes, unsigned short
- * dataPath:       var length bytes, UTF-8 encoded java.lang.String
+ * deletePathLen:       2 bytes, unsigned short
+ * deletePath:          var length bytes, UTF-8 encoded java.lang.String
+ * deleteDigestType:    1 byte
+ * deleteDigest:        n-bytes, where n is deteremined by {@code deleteDigestType}
+ * dataPathLen:         2 bytes, unsigned short
+ * dataPath:            var length bytes, UTF-8 encoded java.lang.String
+ * dataDigestType:      1 byte
+ * dataDigest:          n-bytes, where n is deteremined by {@code dataDigestType}
  */
 public class DeleteBinaryLoggable extends AbstractBinaryLoggable {
-    private byte[] deletePath;  // the file to be deleted
-    private byte[] dataPath;    // the data before the file was deleted (i.e. the current value)
+    private byte[] deletePath;              // the file to be deleted
+    private MessageDigest deleteDigest;
+    private byte[] dataPath;                // the data before the file was deleted (i.e. the current value)
+    private MessageDigest dataDigest;
 
     /**
      * Creates a new instance of DeleteBinaryLoggable.
      *
      * @param broker The database broker.
      * @param txn The database transaction.
      * @param delete The file that is to be deleted from the database.
+     * @param deleteDigest digest of the {@code delete} content
      * @param data A copy of the data before the file was deleted (i.e. the current value).
+     * @param dataDigest digest of the {@code data} content
      */
-    public DeleteBinaryLoggable(final DBBroker broker, final Txn txn, final Path delete, final Path data) {
+    public DeleteBinaryLoggable(final DBBroker broker, final Txn txn, final Path delete,
+            final MessageDigest deleteDigest, final Path data, final MessageDigest dataDigest) {
         super(NativeBroker.LOG_DELETE_BINARY, txn.getId());
         this.deletePath = getPathData(delete);
         checkPathLen(getClass().getSimpleName(), "deletePath", deletePath);
+        this.deleteDigest = deleteDigest;
+
         this.dataPath = getPathData(data);
         checkPathLen(getClass().getSimpleName(), "dataPath", dataPath);
+        this.dataDigest = dataDigest;
     }
 
     /**
@@ -76,27 +91,36 @@ public int getLogSize() {
         return
                 2 +
                 deletePath.length +
+                1 +
+                deleteDigest.getDigestType().getDigestLengthBytes() +
                 2 +
-                dataPath.length;
+                dataPath.length +
+                1 +
+                dataDigest.getDigestType().getDigestLengthBytes();
     }
 
     @Override
     public void write(final ByteBuffer out) {
         out.putShort(asUnsignedShort(deletePath.length));
         out.put(deletePath);
+        writeMessageDigest(out, deleteDigest);
+
         out.putShort(asUnsignedShort(dataPath.length));
         out.put(dataPath);
+        writeMessageDigest(out, dataDigest);
     }
 
     @Override
     public void read(final ByteBuffer in) {
         final int replacePathLen = asSignedInt(in.getShort());
         this.deletePath = new byte[replacePathLen];
         in.get(deletePath);
+        this.deleteDigest = readMessageDigest(in);
 
         final int dataPathLen = asSignedInt(in.getShort());
         this.dataPath = new byte[dataPathLen];
         in.get(dataPath);
+        this.dataDigest = readMessageDigest(in);
     }
 
     @Override
@@ -111,6 +135,17 @@ public void redo() throws LogException {
         }
 
         try {
+            final Path data = getPath(dataPath);
+
+            // ensure integrity of data file first!
+            final StreamableDigest dataStreamableDigest = dataDigest.getDigestType().newStreamableDigest();
+            FileUtils.digest(data, dataStreamableDigest);
+            if (!Arrays.equals(dataStreamableDigest.getMessageDigest(), dataDigest.getValue())) {
+                throw new LogException("Cannot redo delete of binary resource: "
+                        + delete.toAbsolutePath().toString() + ", checksum of data file is invalid");
+            }
+
+            // preform the redo - delete
             FileUtils.delete(delete);
         } catch(final IOException ioe) {
             throw new LogException("Cannot redo delete of binary resource: "
@@ -130,14 +165,32 @@ public void undo() throws LogException {
                     + delete.toAbsolutePath().toString() + ", already exists");
         }
 
-        // ensure integrity of data file first!
-        if (!Files.exists(data)) {
-            throw new LogException("Cannot undo delete of binary resource: "
-                    + data.toAbsolutePath().toString() + ", missing data file");
-        }
-
         try {
+            // ensure integrity of the data file first!
+            if (!Files.exists(data)) {
+                throw new LogException("Cannot undo delete of binary resource: "
+                        + data.toAbsolutePath().toString() + ", missing data file");
+            }
+
+            // ensure integrity of data file by checksum
+            final StreamableDigest dataStreamableDigest = dataDigest.getDigestType().newStreamableDigest();
+            FileUtils.digest(data, dataStreamableDigest);
+            if (!Arrays.equals(dataStreamableDigest.getMessageDigest(), dataDigest.getValue())) {
+                throw new LogException("Cannot undo delete of binary resource: "
+                        + data.toAbsolutePath().toString() + ", data file checksum is invalid");
+            }
+
+            // perform the undo - copy
             FileUtils.copy(data, delete);
+
+            // ensure the integrity of the copy
+            dataStreamableDigest.reset();
+            FileUtils.digest(delete, dataStreamableDigest);
+            if (!Arrays.equals(dataStreamableDigest.getMessageDigest(), dataDigest.getValue())) {
+                throw new LogException("Cannot undo delete of binary resource: "
+                        + delete.toAbsolutePath().toString() + " from "
+                        + data.toAbsolutePath().toString() + ", checksum of new delete file is invalid");
+            }
         } catch(final IOException ioe) {
             throw new LogException("Cannot undo delete of binary resource: "
                     + delete.toAbsolutePath().toString(), ioe);