[feature] Facilities for working with Hashes

Author: adamretter

src/org/exist/util/FileUtils.java

@@ -24,9 +24,13 @@
 
 import com.evolvedbinary.j8fu.Either;
 import com.evolvedbinary.j8fu.function.FunctionE;
+import org.exist.util.crypto.digest.DigestOutputStream;
+import org.exist.util.crypto.digest.StreamableDigest;
 
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
@@ -35,6 +39,10 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING;
+import static java.nio.file.StandardOpenOption.WRITE;
+
 /**
  * @author Adam Retter <[email protected]>
  * @author alex
@@ -398,4 +406,77 @@ public static String addPaths(final String path1, final String path2) {
         }
     }
 
+    /**
+     * Copy a file whilst generating a digest of the bytes that are being written.
+     *
+     * Destination file will be overwritten.
+     *
+     * @param src the source file.
+     * @param dst the destination file
+     * @param streamableDigest the digest
+     */
+    public static void copyWithDigest(final Path src, final Path dst, final StreamableDigest streamableDigest) throws IOException {
+        copyWithDigest(src, dst, streamableDigest, WRITE, TRUNCATE_EXISTING);
+    }
+
+    /**
+     * Copy a file whilst generating a digest of the bytes that are being written.
+     *
+     * @param src the source file.
+     * @param dst the destination file
+     * @param streamableDigest the digest
+     */
+    public static void copyWithDigest(final Path src, final Path dst, final StreamableDigest streamableDigest, final OpenOption... dstOptions) throws IOException {
+        try (final InputStream is = Files.newInputStream(src, READ)) {
+            copyWithDigest(is, dst, streamableDigest);
+        }
+    }
+
+    /**
+     * Copy data to a file whilst generating a digest of the bytes that are being written.
+     *
+     * Destination file will be overwritten.
+     *
+     * @param is the source data.
+     * @param dst the destination file
+     * @param streamableDigest the digest
+     */
+    public static void copyWithDigest(final InputStream is, final Path dst, final StreamableDigest streamableDigest) throws IOException {
+        copyWithDigest(is, dst, streamableDigest, WRITE, TRUNCATE_EXISTING);
+    }
+
+    /**
+     * Copy data to a file whilst generating a digest of the bytes that are being written.
+     *
+     * @param is the source data.
+     * @param dst the destination file
+     * @param streamableDigest the digest
+     */
+    public static void copyWithDigest(final InputStream is, final Path dst, final StreamableDigest streamableDigest, final OpenOption... dstOptions) throws IOException {
+        try (final OutputStream os = new DigestOutputStream(Files.newOutputStream(dst, dstOptions), streamableDigest)) {
+
+            final byte[] buf = new byte[8192];
+            int read;
+            while ((read = is.read(buf)) != -1) {
+                os.write(buf, 0, read);
+            }
+        }
+    }
+
+    /**
+     * Calculates the digest for a file.
+     *
+     * @param path the file to calculate the digest for.
+     * @param streamableDigest the digest.
+     */
+    public static void digest(final Path path, final StreamableDigest streamableDigest) throws IOException {
+        try (final InputStream is = Files.newInputStream(path, READ)) {
+            final byte[] buf = new byte[8192];
+
+            int read;
+            while ((read = is.read(buf)) != -1) {
+                streamableDigest.update(buf, 0, read);
+            }
+        }
+    }
 }

src/org/exist/util/crypto/digest/Blake256StreamableDigest.java

@@ -0,0 +1,71 @@
+/*
+ * eXist Open Source Native XML Database
+ * Copyright (C) 2001-2018 The eXist Project
+ * http://exist-db.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+
+package org.exist.util.crypto.digest;
+
+import org.bouncycastle.crypto.ExtendedDigest;
+import org.bouncycastle.crypto.digests.Blake2bDigest;
+
+import java.util.Arrays;
+
+import static org.exist.util.crypto.digest.DigestType.BLAKE_256;
+
+/**
+ * Implementation of Blake2b 256 bit streamable digest.
+ *
+ * @author Adam Retter <[email protected]>
+ */
+public class Blake256StreamableDigest implements StreamableDigest {
+    private final ExtendedDigest ed = new Blake2bDigest(BLAKE_256.getDigestLength());
+
+    @Override
+    public void update(final byte b) {
+        ed.update(b);
+    }
+
+    @Override
+    public void update(final byte[] buf, final int offset, final int len) {
+        ed.update(buf, offset, len);
+    }
+
+    @Override
+    public DigestType getDigestType() {
+        return BLAKE_256;
+    }
+
+    @Override
+    public byte[] getMessageDigest() {
+        final byte[] digestBytes = new byte[BLAKE_256.getDigestLengthBytes()];
+        ed.doFinal(digestBytes, 0);
+        return digestBytes;
+    }
+
+    @Override
+    public MessageDigest copyMessageDigest() {
+        return new MessageDigest(BLAKE_256,
+                Arrays.copyOf(getMessageDigest(), BLAKE_256.getDigestLengthBytes())
+        );
+    }
+
+    @Override
+    public void reset() {
+        ed.reset();
+    }
+}

src/org/exist/util/crypto/digest/DigestInputStream.java

@@ -0,0 +1,84 @@
+/*
+ * eXist Open Source Native XML Database
+ * Copyright (C) 2001-2018 The eXist Project
+ * http://exist-db.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.util.crypto.digest;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An input stream which calculates a digest of the
+ * data that is read.
+ *
+ * @author Adam Retter <[email protected]>
+ */
+public class DigestInputStream extends FilterInputStream {
+
+    private final StreamableDigest streamableDigest;
+
+    /**
+     * Creates an input stream filter which calculates a digest
+     * as the underlying input stream is read.
+     *
+     * @param is the input stream
+     * @param streamableDigest the streamable digest
+     */
+    public DigestInputStream(final InputStream is, final StreamableDigest streamableDigest) {
+        super(is);
+        this.streamableDigest = streamableDigest;
+    }
+
+    @Override
+    public int read() throws IOException {
+        int b = in.read();
+        if (b != -1) {
+            streamableDigest.update((byte)(b & 0xFF));
+        }
+        return b;
+    }
+
+    @Override
+    public int read(final byte[] buf, final int off, int len) throws IOException {
+        len = in.read(buf, off, len);
+        if (len != -1) {
+            streamableDigest.update(buf, off, len);
+        }
+        return len;
+    }
+
+    @Override
+    public long skip(final long n) throws IOException {
+        final byte[] buf = new byte[512];
+        long total = 0;
+        while (total < n) {
+            long len = n - total;
+            len = read(buf, 0, len < buf.length ? (int)len : buf.length);
+            if (len == -1) {
+                return total;
+            }
+            total += len;
+        }
+        return total;
+    }
+
+    public StreamableDigest getStreamableDigest() {
+        return streamableDigest;
+    }
+}

src/org/exist/util/crypto/digest/DigestOutputStream.java

@@ -0,0 +1,63 @@
+/*
+ * eXist Open Source Native XML Database
+ * Copyright (C) 2001-2018 The eXist Project
+ * http://exist-db.org
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+package org.exist.util.crypto.digest;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * An input stream which calculates a digest of the
+ * data that is written.
+ *
+ * @author Adam Retter <[email protected]>
+ */
+public class DigestOutputStream extends FilterOutputStream {
+
+    private final StreamableDigest streamableDigest;
+
+    /**
+     * Creates an output stream filter which calculates a digest
+     * as the underlying output stream is written.
+     *
+     * @param os the input stream
+     * @param streamableDigest the streamable digest
+     */
+    public DigestOutputStream(final OutputStream os, final StreamableDigest streamableDigest) {
+        super(os);
+        this.streamableDigest = streamableDigest;
+    }
+
+    @Override
+    public void write(final int b) throws IOException {
+        out.write(b);
+        streamableDigest.update((byte) (b & 0xFF));
+    }
+
+    @Override
+    public void write(final byte[] b, final int off, final int len) throws IOException {
+        out.write(b, off, len);
+        streamableDigest.update(b, off, len);
+    }
+
+    public StreamableDigest getStreamableDigest() {
+        return streamableDigest;
+    }
+}