iRODS: Add iRODS profile, TLS support, fix PAM and various bugs

Author: korydraughn

irods/pom.xml

@@ -27,7 +27,7 @@
         <dependency>
             <groupId>org.irods</groupId>
             <artifactId>irods4j</artifactId>
-            <version>0.3.0-java8-SNAPSHOT</version>
+            <version>0.4.0-java8</version>
         </dependency>
         <dependency>
             <groupId>ch.cyberduck</groupId>

irods/src/main/java/ch/cyberduck/core/irods/IRODSAttributesFinderFeature.java

@@ -22,22 +22,23 @@
 import ch.cyberduck.core.exception.NotfoundException;
 import ch.cyberduck.core.features.AttributesAdapter;
 import ch.cyberduck.core.features.AttributesFinder;
-import ch.cyberduck.core.io.Checksum;
 
-import org.apache.commons.io.FilenameUtils;
-import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.irods.irods4j.high_level.catalog.IRODSQuery;
-import org.irods.irods4j.high_level.catalog.IRODSQuery.GenQuery1QueryArgs;
 import org.irods.irods4j.high_level.connection.IRODSConnection;
 import org.irods.irods4j.high_level.vfs.IRODSFilesystem;
-import org.irods.irods4j.low_level.api.GenQuery1Columns;
+import org.irods.irods4j.high_level.vfs.LogicalPath;
+import org.irods.irods4j.high_level.vfs.ObjectStatus;
 import org.irods.irods4j.low_level.api.IRODSException;
 
 import java.io.IOException;
 import java.util.List;
 
 public class IRODSAttributesFinderFeature implements AttributesFinder, AttributesAdapter<List<String>> {
 
+    private static final Logger log = LogManager.getLogger(IRODSAttributesFinderFeature.class);
+
     private final IRODSSession session;
 
     public IRODSAttributesFinderFeature(final IRODSSession session) {
@@ -47,42 +48,56 @@ public IRODSAttributesFinderFeature(final IRODSSession session) {
     @Override
     public PathAttributes find(final Path file, final ListProgressListener listener) throws BackgroundException {
         try {
-            final IRODSConnection conn = session.getClient();
+            log.debug("looking up path attributes.");
+
             final String logicalPath = file.getAbsolute();
-            if(!IRODSFilesystem.exists(session.getClient().getRcComm(), logicalPath)) {
-                throw new NotfoundException(file.getAbsolute());
-            }
+            final IRODSConnection conn = session.getClient();
 
-            GenQuery1QueryArgs input = new GenQuery1QueryArgs();
+            ObjectStatus status = IRODSFilesystem.status(session.getClient().getRcComm(), logicalPath);
 
-            // select DATA_MODIFY_TIME, DATA_CREATE_TIME, DATA_SIZE, DATA_CHECKSUM ...
-            input.addColumnToSelectClause(GenQuery1Columns.COL_D_MODIFY_TIME);
-            input.addColumnToSelectClause(GenQuery1Columns.COL_D_CREATE_TIME);
-            input.addColumnToSelectClause(GenQuery1Columns.COL_DATA_SIZE);
-            input.addColumnToSelectClause(GenQuery1Columns.COL_D_DATA_CHECKSUM);
+            if(IRODSFilesystem.isDataObject(status)) {
+                log.debug("data object exists in iRODS. fetching data using GenQuery2.");
+                String query = String.format(
+                        "select DATA_CREATE_TIME, DATA_MODIFY_TIME, DATA_SIZE, DATA_CHECKSUM, DATA_REPL_STATUS where COLL_NAME = '%s' and DATA_NAME = '%s' order by DATA_REPL_STATUS desc, DATA_MODIFY_TIME desc",
+                        LogicalPath.parentPath(logicalPath),
+                        LogicalPath.objectName(logicalPath));
+                log.debug("query = [{}]", query);
+                List<List<String>> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query);
 
-            // where COLL_NAME = '<parent_path>' and DATA_NAME = '<filename>'
-            String collNameCondStr = String.format("= '%s'", FilenameUtils.getFullPathNoEndSeparator(logicalPath));
-            String dataNameCondStr = String.format("= '%s'", FilenameUtils.getName(logicalPath));
-            input.addConditionToWhereClause(GenQuery1Columns.COL_COLL_NAME, collNameCondStr);
-            input.addConditionToWhereClause(GenQuery1Columns.COL_DATA_NAME, dataNameCondStr);
+                PathAttributes attrs = new PathAttributes();
 
-            final PathAttributes attrs = new PathAttributes();
+                if(!rows.isEmpty()) {
+                    List<String> row = rows.get(0);
+                    if("0".equals(row.get(4)) || "1".equals(row.get(4))) {
+                        setAttributes(attrs, row);
+                    }
+                }
 
-            IRODSQuery.executeGenQuery1(conn.getRcComm(), input, row -> {
-                attrs.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms
-                attrs.setCreationDate(Long.parseLong(row.get(1)) * 1000);
-                attrs.setSize(Long.parseLong(row.get(2)));
+                return attrs;
+            }
 
-                String checksum = row.get(3);
-                if(!StringUtils.isEmpty(checksum)) {
-                    attrs.setChecksum(Checksum.parse(checksum));
+            if(IRODSFilesystem.isCollection(status)) {
+                log.debug("collection exists in iRODS. fetching data using GenQuery2.");
+                String query = String.format("select COLL_CREATE_TIME, COLL_MODIFY_TIME where COLL_NAME = '%s'", logicalPath);
+                log.debug("query = [{}]", query);
+                List<List<String>> rows = IRODSQuery.executeGenQuery2(conn.getRcComm(), query);
+
+                PathAttributes attrs = new PathAttributes();
+
+                if(!rows.isEmpty()) {
+                    // Collections do not have the same properties as data objects
+                    // so fill in the gaps to satisfy requirements of setAttributes.
+                    List<String> row = rows.get(0);
+                    row.add("0"); // Data size
+                    row.add("");  // Checksum
+                    row.add("");  // Replica status
+                    setAttributes(attrs, row);
                 }
 
-                return false;
-            });
+                return attrs;
+            }
 
-            return attrs;
+            throw new NotfoundException(logicalPath);
         }
         catch(IOException | IRODSException e) {
             throw new IRODSExceptionMappingService().map("Failure to read attributes of {0}", e, file);
@@ -91,18 +106,18 @@ public PathAttributes find(final Path file, final ListProgressListener listener)
 
     @Override
     public PathAttributes toAttributes(final List<String> row) {
-        final IRODSConnection conn = session.getClient();
-        final PathAttributes attributes = new PathAttributes();
-
-        attributes.setModificationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms
-        attributes.setCreationDate(Long.parseLong(row.get(1)) * 1000);
-        attributes.setSize(Long.parseLong(row.get(2)));
-
-        String checksum = row.get(3);
-        if(!StringUtils.isEmpty(checksum)) {
-            attributes.setChecksum(Checksum.parse(checksum));
-        }
+        PathAttributes attrs = new PathAttributes();
+        setAttributes(attrs, row);
+        return attrs;
+    }
 
-        return attributes;
+    private static void setAttributes(final PathAttributes attrs, final List<String> row) {
+        log.debug("path attribute info: created at [{}], modified at [{}], data size = [{}], checksum = [{}]",
+                row.get(0), row.get(1), row.get(2), row.get(3));
+        attrs.setCreationDate(Long.parseLong(row.get(0)) * 1000); // seconds to ms
+        attrs.setModificationDate(Long.parseLong(row.get(1)) * 1000);
+        attrs.setSize(Long.parseLong(row.get(2)));
+        attrs.setChecksum(IRODSChecksumUtils.toChecksum(row.get(3)));
     }
+
 }

irods/src/main/java/ch/cyberduck/core/irods/IRODSChecksumUtils.java

@@ -0,0 +1,52 @@
+package ch.cyberduck.core.irods;
+
+/*
+ * Copyright (c) 2002-2025 iterate GmbH. All rights reserved.
+ * https://cyberduck.io/
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+import ch.cyberduck.core.io.Checksum;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public final class IRODSChecksumUtils {
+
+    private static final Logger log = LogManager.getLogger(IRODSChecksumUtils.class);
+
+    public static Checksum toChecksum(String irodsChecksum) {
+        if(StringUtils.isBlank(irodsChecksum)) {
+            return Checksum.NONE;
+        }
+
+        int colon = irodsChecksum.indexOf(':');
+        if(-1 == colon) {
+            log.debug("no hash algorithm prefix found in iRODS checksum. ignoring checksum.");
+            return Checksum.NONE;
+        }
+
+        if(colon + 1 >= irodsChecksum.length()) {
+            log.debug("iRODS checksum may be corrupted. ignoring checksum.");
+            return Checksum.NONE;
+        }
+
+        log.debug("checksum from iRODS server is [{}].", irodsChecksum);
+        String checksum = irodsChecksum.substring(colon + 1);
+        checksum = Hex.encodeHexString(Base64.decodeBase64(checksum));
+        log.debug("base64-decoded, hex-encoded checksum is [{}].", checksum);
+        return Checksum.parse(checksum);
+    }
+}

irods/src/main/java/ch/cyberduck/core/irods/IRODSChunkWorker.java

@@ -15,105 +15,82 @@
  * GNU General Public License for more details.
  */
 
-import org.irods.irods4j.high_level.connection.IRODSConnectionPool;
-import org.irods.irods4j.high_level.io.IRODSDataObjectInputStream;
-import org.irods.irods4j.high_level.io.IRODSDataObjectOutputStream;
-import org.irods.irods4j.high_level.io.IRODSDataObjectStream;
+import ch.cyberduck.core.exception.ConnectionCanceledException;
+import ch.cyberduck.core.io.StreamListener;
+import ch.cyberduck.core.transfer.TransferStatus;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.irods.irods4j.low_level.api.IRODSException;
 
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
 
 public class IRODSChunkWorker implements Runnable {
 
-    private final IRODSConnectionPool.PoolConnection conn;
+    private static final Logger log = LogManager.getLogger(IRODSChunkWorker.class);
+
+    private final TransferStatus status;
+    private final StreamListener streamListener;
     private final InputStream in;
     private final OutputStream out;
     private final long offset;
     private final long chunkSize;
     private final byte[] buffer;
 
-    public IRODSChunkWorker(IRODSConnectionPool.PoolConnection conn, InputStream in, OutputStream out, long offset, long chunkSize, int bufferSize) throws IOException, IRODSException {
-        this.conn = conn;
+    public IRODSChunkWorker(TransferStatus status, StreamListener streamListener, InputStream in, OutputStream out, long offset, long chunkSize, int bufferSize) {
+        log.info("constructing iRODS chunk worker.");
+        log.info("offset      = [{}]", offset);
+        log.info("chunk size  = [{}]", chunkSize);
+        log.info("buffer size = [{}]", bufferSize);
+        this.status = status;
+        this.streamListener = streamListener;
         this.in = in;
         this.out = out;
         this.offset = offset;
         this.chunkSize = chunkSize;
         this.buffer = new byte[bufferSize];
+        log.info("iRODS chunk worker constructed.");
     }
 
     @Override
     public void run() {
         try {
-            seek(in);
-            seek(out);
+            IRODSStreamUtils.seek(in, offset);
+            IRODSStreamUtils.seek(out, offset);
 
             long remaining = chunkSize;
             while(remaining > 0) {
+                try {
+                    status.validate();
+                }
+                catch(ConnectionCanceledException e) {
+                    log.info("transfer cancelled.");
+                    return;
+                }
+
                 int count = (int) Math.min(buffer.length, remaining);
 
                 int bytesRead = in.read(buffer, 0, count);
+                log.info("read [{}] of [{}] requested bytes from input stream.", bytesRead, count);
                 if(-1 == bytesRead) {
                     break;
                 }
 
+                streamListener.recv(bytesRead);
                 out.write(buffer, 0, bytesRead);
+                log.info("wrote [{}] bytes to output stream.", bytesRead);
+                streamListener.sent(bytesRead);
                 remaining -= bytesRead;
             }
-        }
-        catch(IOException | IRODSException e) {
-            // TODO Log error
-        }
-        finally {
-            try {
-                in.close();
-            }
-            catch(Exception e) { /* Ignored */ }
-            try {
-                out.close();
-            }
-            catch(Exception e) { /* Ignored */ }
-        }
-    }
 
-    private void seek(InputStream in) throws IRODSException, IOException {
-        if(in instanceof IRODSDataObjectInputStream) {
-            IRODSDataObjectInputStream stream = (IRODSDataObjectInputStream) in;
-            long totalOffset = offset;
-            while(totalOffset > 0) {
-                if(totalOffset >= Integer.MAX_VALUE) {
-                    totalOffset -= Integer.MAX_VALUE;
-                    stream.seek(Integer.MAX_VALUE, IRODSDataObjectStream.SeekDirection.CURRENT);
-                }
-                else {
-                    stream.seek((int) totalOffset, IRODSDataObjectStream.SeekDirection.CURRENT);
-                }
-            }
+            log.info("total bytes remaining = [{}]", remaining);
+            log.info("done. wrote [{}] of [{}] bytes to the replica.", chunkSize - remaining, chunkSize);
         }
-        else if(in instanceof FileInputStream) {
-            ((FileInputStream) in).getChannel().position(offset);
+        catch(IOException | IRODSException e) {
+            log.error(e.getMessage());
         }
     }
 
-    private void seek(OutputStream out) throws IRODSException, IOException {
-        if(out instanceof IRODSDataObjectOutputStream) {
-            IRODSDataObjectOutputStream stream = (IRODSDataObjectOutputStream) out;
-            long totalOffset = offset;
-            while(totalOffset > 0) {
-                if(totalOffset >= Integer.MAX_VALUE) {
-                    totalOffset -= Integer.MAX_VALUE;
-                    stream.seek(Integer.MAX_VALUE, IRODSDataObjectStream.SeekDirection.CURRENT);
-                }
-                else {
-                    stream.seek((int) totalOffset, IRODSDataObjectStream.SeekDirection.CURRENT);
-                }
-            }
-        }
-        else if(out instanceof FileOutputStream) {
-            ((FileOutputStream) out).getChannel().position(offset);
-        }
-    }
 }