Set QuickXorHash as checksum to compare file changes

box/src/main/java/ch/cyberduck/core/box/BoxAttributesFinderFeature.java

@@ -24,6 +24,7 @@
 import ch.cyberduck.core.box.io.swagger.client.model.File;
 import ch.cyberduck.core.box.io.swagger.client.model.Folder;
 import ch.cyberduck.core.exception.BackgroundException;
+import ch.cyberduck.core.exception.UnsupportedException;
 import ch.cyberduck.core.features.AttributesAdapter;
 import ch.cyberduck.core.features.AttributesFinder;
 import ch.cyberduck.core.io.Checksum;
@@ -75,7 +76,12 @@ public PathAttributes toAttributes(final File f) {
             attrs.setSize(f.getSize());
         }
         attrs.setFileId(f.getId());
-        attrs.setChecksum(new Checksum(HashAlgorithm.sha1, f.getSha1()));
+        try {
+            attrs.setChecksum(new Checksum(HashAlgorithm.sha1, f.getSha1()));
+        }
+        catch(UnsupportedException e) {
+            attrs.setChecksum(Checksum.NONE);
+        }
         attrs.setETag(f.getEtag());
         return attrs;
     }

core/src/main/java/ch/cyberduck/core/io/CRC32ChecksumCompute.java

@@ -17,6 +17,7 @@
 
 import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.exception.ChecksumException;
+import ch.cyberduck.core.exception.UnsupportedException;
 import ch.cyberduck.core.transfer.TransferStatus;
 
 import org.apache.commons.io.IOUtils;
@@ -35,6 +36,7 @@ public Checksum compute(final InputStream in, final TransferStatus status) throw
             byte[] buffer = new byte[16384];
             int bytesRead;
             while((bytesRead = normalized.read(buffer, 0, buffer.length)) != -1) {
+                status.validate();
                 crc32.update(buffer, 0, bytesRead);
             }
         }
@@ -44,6 +46,11 @@ public Checksum compute(final InputStream in, final TransferStatus status) throw
         finally {
             IOUtils.closeQuietly(normalized);
         }
-        return new Checksum(HashAlgorithm.crc32, String.format("%08x", crc32.getValue()));
+        try {
+            return new Checksum(HashAlgorithm.crc32, String.format("%08x", crc32.getValue()));
+        }
+        catch(UnsupportedException e) {
+            return Checksum.NONE;
+        }
     }
 }

core/src/main/java/ch/cyberduck/core/io/Checksum.java

@@ -18,6 +18,9 @@
  * [email protected]
  */
 
+import ch.cyberduck.core.exception.UnsupportedException;
+
+import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.codec.binary.Hex;
 import org.apache.commons.lang3.StringUtils;
@@ -42,11 +45,16 @@ public Checksum(final HashAlgorithm algorithm, final byte[] digest) {
         this.base64 = Base64.encodeBase64String(digest);
     }
 
-    public Checksum(final HashAlgorithm algorithm, final String hexString) {
+    public Checksum(final HashAlgorithm algorithm, final String hexString) throws UnsupportedException {
         this.algorithm = algorithm;
         this.hash = hexString;
         this.hex = hexString;
-        this.base64 = null;
+        try {
+            this.base64 = Base64.encodeBase64String(Hex.decodeHex(hexString));
+        }
+        catch(DecoderException e) {
+            throw new UnsupportedException(e);
+        }
     }
 
     public Checksum(final HashAlgorithm algorithm, final String hexString, final String base64String) {
@@ -65,41 +73,58 @@ public Checksum(final Checksum other) {
 
     @Override
     public String toString() {
-        return hash;
+        return hex;
     }
 
     public static Checksum parse(final String hash) {
         if(StringUtils.isBlank(hash)) {
             return Checksum.NONE;
         }
-        switch(hash.length()) {
-            case 8:
-                if(hash.matches("[a-fA-F0-9]{8}")) {
-                    return new Checksum(HashAlgorithm.crc32, hash);
-                }
-                break;
-            case 32:
-                if(hash.matches("[a-fA-F0-9]{32}")) {
-                    return new Checksum(HashAlgorithm.md5, hash);
-                }
-                break;
-            case 40:
-                if(hash.matches("[a-fA-F0-9]{40}")) {
-                    return new Checksum(HashAlgorithm.sha1, hash);
-                }
-                break;
-            case 64:
-                if(hash.matches("[A-Fa-f0-9]{64}")) {
-                    return new Checksum(HashAlgorithm.sha256, hash);
-                }
-                break;
-            case 128:
-                if(hash.matches("[A-Fa-f0-9]{128}")) {
-                    return new Checksum(HashAlgorithm.sha512, hash);
-                }
-                break;
-            default:
-                log.warn("Failure to detect algorithm for checksum {}", hash);
+        // Check for Base64 with proper padding
+        if(hash.matches("^[A-Za-z0-9+/]+={0,2}$") && hash.length() % 4 == 0) {
+            final Checksum result = parseHex(Hex.encodeHexString(Base64.decodeBase64(hash)));
+            if(result != Checksum.NONE) {
+                return new Checksum(result.algorithm, result.hex, hash);
+            }
+        }
+        // Parse as hex string
+        return parseHex(hash);
+    }
+
+    private static Checksum parseHex(final String hexString) {
+        try {
+            switch(hexString.length()) {
+                case 8:
+                    if(hexString.matches("[a-fA-F0-9]{8}")) {
+                        return new Checksum(HashAlgorithm.crc32, hexString);
+                    }
+                    break;
+                case 32:
+                    if(hexString.matches("[a-fA-F0-9]{32}")) {
+                        return new Checksum(HashAlgorithm.md5, hexString);
+                    }
+                    break;
+                case 40:
+                    if(hexString.matches("[a-fA-F0-9]{40}")) {
+                        return new Checksum(HashAlgorithm.sha1, hexString);
+                    }
+                    break;
+                case 64:
+                    if(hexString.matches("[A-Fa-f0-9]{64}")) {
+                        return new Checksum(HashAlgorithm.sha256, hexString);
+                    }
+                    break;
+                case 128:
+                    if(hexString.matches("[A-Fa-f0-9]{128}")) {
+                        return new Checksum(HashAlgorithm.sha512, hexString);
+                    }
+                    break;
+                default:
+                    log.warn("Failure to detect algorithm for checksum {}", hexString);
+            }
+        }
+        catch(UnsupportedException e) {
+            return Checksum.NONE;
         }
         return Checksum.NONE;
     }
@@ -116,7 +141,7 @@ public boolean equals(final Object o) {
         if(algorithm != checksum.algorithm) {
             return false;
         }
-        if(!StringUtils.equalsIgnoreCase(hash, checksum.hash)) {
+        if(!StringUtils.equalsIgnoreCase(hex, checksum.hex)) {
             return false;
         }
         return true;
@@ -125,7 +150,7 @@ public boolean equals(final Object o) {
     @Override
     public int hashCode() {
         int result = algorithm != null ? algorithm.hashCode() : 0;
-        result = 31 * result + (hash != null ? hash.hashCode() : 0);
+        result = 31 * result + (hex != null ? hex.hashCode() : 0);
         return result;
     }
 

core/src/main/java/ch/cyberduck/core/io/MD5FastChecksumCompute.java

@@ -18,6 +18,7 @@
 import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.exception.ChecksumCanceledException;
 import ch.cyberduck.core.exception.ChecksumException;
+import ch.cyberduck.core.exception.ConnectionCanceledException;
 import ch.cyberduck.core.transfer.TransferStatus;
 
 import org.apache.commons.io.IOUtils;
@@ -41,12 +42,13 @@ public Checksum compute(final InputStream in, final TransferStatus status) throw
                 this.normalize(in, status), status));
     }
 
-    protected byte[] digest(final String algorithm, final InputStream in, final StreamCancelation cancelation) throws ChecksumException, ChecksumCanceledException {
+    protected byte[] digest(final String algorithm, final InputStream in, final StreamCancelation cancelation) throws ChecksumException, ConnectionCanceledException {
         final MD5 md = new MD5();
         try {
             byte[] buffer = new byte[16384];
             int bytesRead;
             while((bytesRead = in.read(buffer, 0, buffer.length)) != -1) {
+                cancelation.validate();
                 md.Update(buffer, 0, bytesRead);
             }
         }

core/src/test/java/ch/cyberduck/core/PathAttributesTest.java

@@ -77,7 +77,7 @@ public void testSerialize() {
         acl.addAll(new Acl.CanonicalUser("user1"), new Acl.Role(Acl.Role.READ), new Acl.Role(Acl.Role.WRITE));
         acl.addAll(new Acl.CanonicalUser("user2"), new Acl.Role(Acl.Role.FULL));
         attributes.setAcl(acl);
-        attributes.setChecksum(new Checksum(HashAlgorithm.crc32, "abcdefab"));
+        attributes.setChecksum(new Checksum(HashAlgorithm.crc32, "abcdefab", null));
         attributes.setVersionId("v-1");
         attributes.setFileId("f-1");
         attributes.setDisplayname("d-1");

core/src/test/java/ch/cyberduck/core/io/ChecksumTest.java

@@ -25,7 +25,7 @@
 public class ChecksumTest {
 
     @Test
-    public void testParse() {
+    public void testParse() throws Exception {
         assertEquals(new Checksum(HashAlgorithm.md5, "d41d8cd98f00b204e9800998ecf8427e"),
                 Checksum.parse("d41d8cd98f00b204e9800998ecf8427e"));
         assertEquals(new Checksum(HashAlgorithm.sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709"),

core/src/test/java/ch/cyberduck/core/synchronization/ChecksumComparisonServiceTest.java

@@ -12,22 +12,22 @@
 public class ChecksumComparisonServiceTest {
 
     @Test
-    public void testCompare() {
+    public void testCompare() throws Exception {
         ComparisonService s = new ChecksumComparisonService();
         assertEquals(Comparison.equal, s.compare(Path.Type.file, new PathAttributes() {
                     @Override
                     public Checksum getChecksum() {
-                        return new Checksum(HashAlgorithm.md5, "a");
+                        return new Checksum(HashAlgorithm.md5, "a", null);
                     }
-                }, new PathAttributes().setChecksum(new Checksum(HashAlgorithm.md5, "a"))
+                }, new PathAttributes().setChecksum(new Checksum(HashAlgorithm.md5, "a", null))
         ));
 
         assertEquals(Comparison.notequal, s.compare(Path.Type.file, new PathAttributes() {
             @Override
             public Checksum getChecksum() {
-                return new Checksum(HashAlgorithm.md5, "b");
+                return new Checksum(HashAlgorithm.md5, "b", null);
             }
-        }, new PathAttributes().setChecksum(new Checksum(HashAlgorithm.md5, "a"))));
+        }, new PathAttributes().setChecksum(new Checksum(HashAlgorithm.md5, "a", null))));
     }
 
     @Test

core/src/test/java/ch/cyberduck/core/synchronization/DefaultComparePathFilterTest.java

@@ -39,7 +39,7 @@ public PathAttributes find(final Path file, final ListProgressListener listener)
                 return new PathAttributes() {
                     @Override
                     public Checksum getChecksum() {
-                        return new Checksum(HashAlgorithm.md5, "a");
+                        return new Checksum(HashAlgorithm.md5, "a", null);
                     }
                 };
             }
@@ -54,7 +54,7 @@ public boolean find(final Path file, final ListProgressListener listener) {
         ComparePathFilter s = new DefaultComparePathFilter(new NullSession(new Host(new TestProtocol())), find, attributes) {
             @Override
             protected Checksum checksum(final HashAlgorithm algorithm, final Local local) {
-                return new Checksum(HashAlgorithm.md5, "a");
+                return new Checksum(HashAlgorithm.md5, "a", null);
             }
         };
         final String path = new AlphanumericRandomStringService().random();
@@ -166,7 +166,7 @@ public PathAttributes find(final Path file, final ListProgressListener listener)
                 return new PathAttributes() {
                     @Override
                     public Checksum getChecksum() {
-                        return new Checksum(HashAlgorithm.md5, "b");
+                        return new Checksum(HashAlgorithm.md5, "b", null);
                     }
 
                     @Override
@@ -189,7 +189,7 @@ public long getModificationDate() {
         ComparePathFilter s = new DefaultComparePathFilter(new NullSession(new Host(new TestProtocol())), find, attributes) {
             @Override
             protected Checksum checksum(final HashAlgorithm algorithm, final Local local) {
-                return new Checksum(HashAlgorithm.md5, "a");
+                return new Checksum(HashAlgorithm.md5, "a", null);
             }
         };
         assertEquals(Comparison.local, s.compare(new Path("t", EnumSet.of(Path.Type.file)), new NullLocal("t") {

dropbox/src/main/java/ch/cyberduck/core/dropbox/DropboxAttributesFinderFeature.java

@@ -21,6 +21,7 @@
 import ch.cyberduck.core.PathContainerService;
 import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.exception.NotfoundException;
+import ch.cyberduck.core.exception.UnsupportedException;
 import ch.cyberduck.core.features.AttributesAdapter;
 import ch.cyberduck.core.features.AttributesFinder;
 import ch.cyberduck.core.io.Checksum;
@@ -86,7 +87,12 @@ public PathAttributes toAttributes(final Metadata metadata) {
             if(file.getFileLockInfo() != null) {
                 attributes.setLockId(String.valueOf(file.getFileLockInfo().getIsLockholder()));
             }
-            attributes.setChecksum(new Checksum(HashAlgorithm.dropbox_content_hash, file.getContentHash()));
+            try {
+                attributes.setChecksum(new Checksum(HashAlgorithm.dropbox_content_hash, file.getContentHash()));
+            }
+            catch(UnsupportedException e) {
+                attributes.setChecksum(Checksum.NONE);
+            }
             attributes.setVersionId(file.getRev());
         }
         if(metadata instanceof FolderMetadata) {

eue/src/main/java/ch/cyberduck/core/eue/ChunkListSHA256ChecksumCompute.java

@@ -20,6 +20,7 @@
 
 import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.exception.ChecksumException;
+import ch.cyberduck.core.exception.UnsupportedException;
 import ch.cyberduck.core.io.AbstractChecksumCompute;
 import ch.cyberduck.core.io.Checksum;
 import ch.cyberduck.core.io.HashAlgorithm;
@@ -59,6 +60,11 @@ public Checksum compute(final Number length, final byte[] contentDigest) throws
         }
         digest.update(contentDigest);
         digest.update(intToBytes(length.intValue()));
-        return new Checksum(HashAlgorithm.cdash64, Base64.encodeBase64URLSafeString(digest.digest()));
+        try {
+            return new Checksum(HashAlgorithm.cdash64, Base64.encodeBase64URLSafeString(digest.digest()));
+        }
+        catch(UnsupportedException e) {
+            return Checksum.NONE;
+        }
     }
 }

importer/src/main/java/ch/cyberduck/core/importer/ThirdpartyBookmarkCollection.java

@@ -30,6 +30,7 @@
 import ch.cyberduck.core.exception.AccessDeniedException;
 import ch.cyberduck.core.exception.BackgroundException;
 import ch.cyberduck.core.exception.LocalAccessDeniedException;
+import ch.cyberduck.core.exception.UnsupportedException;
 import ch.cyberduck.core.io.Checksum;
 import ch.cyberduck.core.io.ChecksumComputeFactory;
 import ch.cyberduck.core.io.HashAlgorithm;
@@ -75,10 +76,10 @@ public void load() throws AccessDeniedException {
             }
             if(preferences.getBoolean(this.getConfiguration())) {
                 // Previously imported
-                final Checksum previous = new Checksum(HashAlgorithm.md5,
-                    preferences.getProperty(String.format("%s.checksum", this.getConfiguration())));
-                log.debug("Saved previous checksum {} for bookmark {}", previous, file);
-                if(StringUtils.isNotBlank(previous.hash)) {
+                try {
+                    final Checksum previous = new Checksum(HashAlgorithm.md5,
+                            preferences.getProperty(String.format("%s.checksum", this.getConfiguration())));
+                    log.debug("Saved previous checksum {} for bookmark {}", previous, file);
                     if(previous.equals(current)) {
                         log.info("Skip importing bookmarks from {} with previously saved checksum {}", file, previous);
                     }
@@ -87,8 +88,7 @@ public void load() throws AccessDeniedException {
                         // Should filter existing bookmarks. Skip import
                     }
                 }
-                else {
-                    // Skip flagged
+                catch(UnsupportedException e) {
                     log.debug("Skip importing bookmarks from {}", file);
                 }
             }
@@ -155,7 +155,7 @@ public boolean add(final Host bookmark) {
             if(credentials.isPublicKeyAuthentication()) {
                 try {
                     keychain.addPassword(bookmark.getHostname(), credentials.getIdentity().getAbbreviatedPath(),
-                        credentials.getPassword());
+                            credentials.getPassword());
                 }
                 catch(LocalAccessDeniedException e) {
                     log.error("Failure {} saving credentials for {} in password store", e, bookmark);
@@ -164,7 +164,7 @@ public boolean add(final Host bookmark) {
             else if(!credentials.isAnonymousLogin()) {
                 try {
                     keychain.addPassword(bookmark.getProtocol().getScheme(), bookmark.getPort(),
-                        bookmark.getHostname(), credentials.getUsername(), credentials.getPassword());
+                            bookmark.getHostname(), credentials.getUsername(), credentials.getPassword());
                 }
                 catch(LocalAccessDeniedException e) {
                     log.error("Failure {} saving credentials for {} in password store", e, bookmark);