quota: implement protocols specific return codes

Motivation:
----------

When quota was introduced, proper exception handling was not
added to the doors other than NFS door - an oversight. As
the result over quota condition tiggers generic exception handling
even though some protocols provide for specific return
codes in this case.

Modification:
------------

Handle quota exceeded exception in all doors with the exception
of DCap door.

Result:
-------

Protocol compliant return codes on quota exceeded exception

Ticket: #7747
Patch:  https://rb.dcache.org/r/14476/
Acked-by: Paul, Tigran

Target: trunk
Request: 11.0
Request: 10.2
Request: 10.1
Request: 10.0
Request: 9.2

Require-bool: no
Require-notes: yes

Author: DmitryLitvintsev

modules/dcache-chimera/src/main/java/org/dcache/chimera/namespace/ChimeraNameSpaceProvider.java

@@ -51,6 +51,7 @@
 import diskCacheV111.util.NotDirCacheException;
 import diskCacheV111.util.NotFileCacheException;
 import diskCacheV111.util.PermissionDeniedCacheException;
+import diskCacheV111.util.QuotaExceededCacheException;
 import diskCacheV111.util.PnfsId;
 import diskCacheV111.util.RetentionPolicy;
 import diskCacheV111.vehicles.StorageInfo;
@@ -92,6 +93,7 @@
 import org.dcache.chimera.DirectoryStreamB;
 import org.dcache.chimera.FileExistsChimeraFsException;
 import org.dcache.chimera.FileNotFoundChimeraFsException;
+import org.dcache.chimera.QuotaChimeraFsException;
 import org.dcache.chimera.FileState;
 import org.dcache.chimera.FileSystemProvider;
 import org.dcache.chimera.FileSystemProvider.SetXattrMode;
@@ -383,6 +385,8 @@ public FileAttributes createFile(Subject subject, String path,
             throw new FileNotFoundCacheException("No such directory: " + parentPath);
         } catch (FileExistsChimeraFsException e) {
             throw new FileExistsCacheException("File exists: " + path);
+        } catch (QuotaChimeraFsException e) {
+            throw new QuotaExceededCacheException(e.getMessage());
         } catch (IOException e) {
             throw new CacheException(CacheException.UNEXPECTED_SYSTEM_EXCEPTION,
                   e.getMessage());

modules/dcache-ftp/src/main/java/org/dcache/ftp/door/AbstractFtpDoorV1.java

@@ -127,6 +127,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import diskCacheV111.util.PermissionDeniedCacheException;
 import diskCacheV111.util.PnfsHandler;
 import diskCacheV111.util.PnfsId;
+import diskCacheV111.util.QuotaExceededCacheException;
 import diskCacheV111.util.TimeoutCacheException;
 import diskCacheV111.vehicles.DoorCancelledUploadNotificationMessage;
 import diskCacheV111.vehicles.DoorRequestInfoMessage;
@@ -3575,6 +3576,8 @@ private void store(String file, Mode mode, TransferMode xferMode,
             transfer.abort(451, "Operation failed: " + e.getMessage());
         } catch (PermissionDeniedCacheException e) {
             transfer.abort(550, "Permission denied");
+        } catch (QuotaExceededCacheException e) {
+            transfer.abort(552, "Quota exceeded");
         } catch (CacheException e) {
             switch (e.getRc()) {
                 case CacheException.FILE_NOT_FOUND:

modules/dcache-srm/src/main/java/diskCacheV111/srm/dcache/Storage.java

@@ -116,6 +116,7 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 import diskCacheV111.util.PermissionDeniedCacheException;
 import diskCacheV111.util.PnfsHandler;
 import diskCacheV111.util.PnfsId;
+import diskCacheV111.util.QuotaExceededCacheException;
 import diskCacheV111.util.RetentionPolicy;
 import diskCacheV111.util.TimeoutCacheException;
 import diskCacheV111.vehicles.CopyManagerMessage;
@@ -1166,6 +1167,8 @@ public void putDone(SRMUser user, String localTransferPath, URI surl, boolean ov
             throw new SRMException(e.getMessage(), e);
         } catch (PermissionDeniedCacheException e) {
             throw new SRMAuthorizationException("Permission denied.", e);
+        } catch (QuotaExceededCacheException e) {
+            throw new SRMAuthorizationException("Quota exceeded.", e);
         } catch (FileExistsCacheException e) {
             throw new SRMDuplicationException(surl + " exists.", e);
         } catch (CacheException e) {

modules/dcache-vehicles/src/main/java/diskCacheV111/util/CacheException.java

@@ -188,6 +188,11 @@ public class CacheException extends Exception {
      */
     public static final int INVALID_UPDATE = 10031;
 
+    /**
+     * Quota exceeded
+     */
+    public static final int QUOTA_EXCEEDED = 10032;
+
     /**
      * default error code. <b>It's recommended to use more specific error codes</b>
      */

modules/dcache-vehicles/src/main/java/diskCacheV111/util/QuotaExceededCacheException.java

@@ -0,0 +1,74 @@
+/*
+COPYRIGHT STATUS:
+Dec 1st 2001, Fermi National Accelerator Laboratory (FNAL) documents and
+software are sponsored by the U.S. Department of Energy under Contract No.
+DE-AC02-76CH03000. Therefore, the U.S. Government retains a  world-wide
+non-exclusive, royalty-free license to publish or reproduce these documents
+and software for U.S. Government purposes.  All documents and software
+available from this server are protected under the U.S. and Foreign
+Copyright Laws, and FNAL reserves all rights.
+
+Distribution of the software available from this server is free of
+charge subject to the user following the terms of the Fermitools
+Software Legal Information.
+
+Redistribution and/or modification of the software shall be accompanied
+by the Fermitools Software Legal Information  (including the copyright
+notice).
+
+The user is asked to feed back problems, benefits, and/or suggestions
+about the software to the Fermilab Software Providers.
+
+Neither the name of Fermilab, the  URA, nor the names of the contributors
+may be used to endorse or promote products derived from this software
+without specific prior written permission.
+
+DISCLAIMER OF LIABILITY (BSD):
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED  WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED  WARRANTIES OF MERCHANTABILITY AND FITNESS
+FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL FERMILAB,
+OR THE URA, OR THE U.S. DEPARTMENT of ENERGY, OR CONTRIBUTORS BE LIABLE
+FOR  ANY  DIRECT, INDIRECT,  INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES  (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
+OF SUBSTITUTE  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY  OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT  OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE  POSSIBILITY OF SUCH DAMAGE.
+
+Liabilities of the Government:
+
+This software is provided by URA, independent from its Prime Contract
+with the U.S. Department of Energy. URA is acting independently from
+the Government and in its own private capacity and is not acting on
+behalf of the U.S. Government, nor as its contractor nor its agent.
+Correspondingly, it is understood and agreed that the U.S. Government
+has no connection to this software and in no manner whatsoever shall
+be liable for nor assume any responsibility or obligation for any claim,
+cost, or damages arising out of or resulting from the use of the software
+available from this server.
+
+Export Control:
+
+All documents and software available from this server are subject to U.S.
+export control laws.  Anyone downloading information from this server is
+obligated to secure any necessary Government licenses before exporting
+documents or software obtained from this server.
+*/
+
+package diskCacheV111.util;
+
+public class QuotaExceededCacheException extends CacheException {
+
+    private static final long serialVersionUID = -1L;
+
+    public QuotaExceededCacheException(String msg) {
+        super(CacheException.QUOTA_EXCEEDED, msg);
+    }
+
+    public QuotaExceededCacheException(String msg, Throwable cause) {
+        super(CacheException.QUOTA_EXCEEDED, msg, cause);
+    }
+}

modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheResourceFactory.java

@@ -4,6 +4,8 @@
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.collect.Iterables.cycle;
 import static com.google.common.collect.Iterables.limit;
+import static io.milton.http.quota.StorageChecker.StorageErrorReason.SER_DISK_FULL;
+import static io.milton.http.quota.StorageChecker.StorageErrorReason.SER_QUOTA_EXCEEDED;
 import static java.util.Arrays.asList;
 import static java.util.Objects.requireNonNull;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
@@ -50,6 +52,7 @@
 import diskCacheV111.util.PermissionDeniedCacheException;
 import diskCacheV111.util.PnfsHandler;
 import diskCacheV111.util.PnfsId;
+import diskCacheV111.util.QuotaExceededCacheException;
 import diskCacheV111.util.TimeoutCacheException;
 import diskCacheV111.vehicles.DoorRequestInfoMessage;
 import diskCacheV111.vehicles.DoorTransferFinishedMessage;
@@ -735,10 +738,9 @@ && isImpatientClientProxied()
      */
     public DcacheResource createFile(FsPath path, InputStream inputStream, Long length)
           throws CacheException, InterruptedException, IOException,
-          URISyntaxException, BadRequestException {
+                 URISyntaxException, BadRequestException {
         Subject subject = getSubject();
         Restriction restriction = getRestriction();
-
         checkUploadSize(length);
 
         WriteTransfer transfer = new WriteTransfer(_pnfs, subject, restriction, path);
@@ -802,6 +804,11 @@ public DcacheResource createFile(FsPath path, InputStream inputStream, Long leng
                     transfer.deleteNameSpaceEntry();
                 }
             }
+        } catch (QuotaExceededCacheException e) {
+            throw new InsufficientStorageException(e.getMessage(),
+                                                   null,
+                                                   SER_QUOTA_EXCEEDED);
+
         } finally {
             _transfers.remove((int) transfer.getId());
         }
@@ -811,7 +818,7 @@ public DcacheResource createFile(FsPath path, InputStream inputStream, Long leng
 
     public String getWriteUrl(FsPath path, Long length)
           throws CacheException, InterruptedException,
-          URISyntaxException {
+                 URISyntaxException {
         Subject subject = getSubject();
         Restriction restriction = getRestriction();
 
@@ -853,6 +860,10 @@ public String getWriteUrl(FsPath path, Long length)
                     transfer.deleteNameSpaceEntry();
                 }
             }
+        } catch (QuotaExceededCacheException e) {
+            throw new InsufficientStorageException(e.getMessage(),
+                                                   null,
+                                                   SER_QUOTA_EXCEEDED);
         } finally {
             if (uri == null) {
                 _transfers.remove((int) transfer.getId());
@@ -1437,7 +1448,9 @@ private OptionalLong getMaxUploadSize() {
     private void checkUploadSize(Long length) {
         OptionalLong maxUploadSize = getMaxUploadSize();
         checkStorageSufficient(!maxUploadSize.isPresent() || length == null
-              || length <= maxUploadSize.getAsLong(), "Upload too large");
+                               || length <= maxUploadSize.getAsLong(),
+                               SER_DISK_FULL,
+                               "Upload too large");
     }
 
     private boolean isAdmin() {
@@ -1944,7 +1957,8 @@ public void relayData(InputStream inputStream)
                             throw new BadRequestException(connection.getResponseMessage());
                         case 507: // Insufficient Storage
                             throw new InsufficientStorageException(connection.getResponseMessage(),
-                                  null);
+                                                                   null,
+                                                                   SER_DISK_FULL);
                         case ResponseStatus.SC_INTERNAL_SERVER_ERROR:
                             throw new CacheException(
                                   "Pool error: " + connection.getResponseMessage());

modules/dcache-webdav/src/main/java/org/dcache/webdav/DcacheStandardFilter.java

@@ -75,8 +75,9 @@ public void process(FilterChain chain, Request request, Response response) {
             LOGGER.debug("Client supplied bad request parameters: {}", e.getMessage());
             responseHandler.respondBadRequest(e.getResource(), response, request);
         } catch (InsufficientStorageException e) {
-            responseHandler.respondInsufficientStorage(request, response,
-                  StorageChecker.StorageErrorReason.SER_DISK_FULL);
+            responseHandler.respondInsufficientStorage(request,
+                                                       response,
+                                                       e.getReason());
         } catch (ConflictException e) {
             responseHandler.respondConflict(e.getResource(), response, request, e.getMessage());
         } catch (NotAuthorizedException e) {

modules/dcache-webdav/src/main/java/org/dcache/webdav/InsufficientStorageException.java

@@ -1,7 +1,9 @@
 package org.dcache.webdav;
 
+import static java.util.Objects.requireNonNull;
 import static org.dcache.util.Exceptions.genericCheck;
 
+import io.milton.http.quota.StorageChecker;
 import io.milton.resource.Resource;
 
 /**
@@ -10,16 +12,32 @@
  */
 public class InsufficientStorageException extends WebDavException {
 
-    public static void checkStorageSufficient(boolean isOK, String template, Object... arguments)
+    private final StorageChecker.StorageErrorReason reason;
+
+    public static void checkStorageSufficient(boolean isOK,
+                                              StorageChecker.StorageErrorReason reason,
+                                              String template,
+                                              Object... arguments)
           throws InsufficientStorageException {
-        genericCheck(isOK, s -> new InsufficientStorageException(s, null), template, arguments);
+        genericCheck(isOK, s -> new InsufficientStorageException(s, null, reason), template, arguments);
+    }
+
+    public StorageChecker.StorageErrorReason getReason() {
+        return reason;
     }
 
-    public InsufficientStorageException(String message, Resource resource) {
+    public InsufficientStorageException(String message,
+                                        Resource resource,
+                                        StorageChecker.StorageErrorReason reason) {
         super(message, resource);
+        this.reason = requireNonNull(reason);
     }
 
-    public InsufficientStorageException(String message, Throwable cause, Resource resource) {
+    public InsufficientStorageException(String message,
+                                        Throwable cause,
+                                        Resource resource,
+                                        StorageChecker.StorageErrorReason reason) {
         super(message, cause, resource);
+        this.reason = requireNonNull(reason);
     }
 }

modules/dcache-webdav/src/main/java/org/dcache/webdav/WebDavException.java

@@ -1,8 +1,11 @@
 package org.dcache.webdav;
 
+import static io.milton.http.quota.StorageChecker.StorageErrorReason.SER_DISK_FULL;
+import static io.milton.http.quota.StorageChecker.StorageErrorReason.SER_QUOTA_EXCEEDED;
 import com.google.common.collect.ImmutableSet;
 import diskCacheV111.util.CacheException;
 import diskCacheV111.util.PermissionDeniedCacheException;
+import diskCacheV111.util.QuotaExceededCacheException;
 import io.milton.resource.Resource;
 import javax.annotation.Nonnull;
 
@@ -59,17 +62,22 @@ public static WebDavException of(@Nonnull CacheException e, Resource resource)
     {
         if (e instanceof PermissionDeniedCacheException) {
             return WebDavExceptions.permissionDenied(resource);
+        } else if (e instanceof QuotaExceededCacheException) {
+            return new InsufficientStorageException(e.getMessage(),
+                                                    e,
+                                                    resource,
+                                                    SER_QUOTA_EXCEEDED);
         }
 
         switch (e.getRc()) {
         case 192: // Pool-to-pool required, but destination cost exceeded.
         case 194: // Pool-to-pool required, but source cost exceeded.
             return new InsufficientStorageException("Unable to ready file for access",
-                    e, resource);
+                                                    e, resource, SER_DISK_FULL);
         }
 
         if (FULL_POOL_MESSAGE.contains(e.getMessage())) {
-            return new InsufficientStorageException(e.getMessage(), e, resource);
+            return new InsufficientStorageException(e.getMessage(), e, resource, SER_DISK_FULL);
         }
 
         return new WebDavException(e.getMessage(), e, resource);

modules/dcache-webdav/src/main/java/org/dcache/webdav/transfer/CopyFilter.java

@@ -38,12 +38,14 @@
 import io.milton.servlet.ServletResponse;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
 import java.security.AccessController;
 import java.security.cert.X509Certificate;
 import java.util.Collections;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -268,16 +270,36 @@ public void validateOidcClientParameters() {
     }
 
     @Override
-    public void process(FilterChain chain, Request request, Response response) {
+    public void process(FilterChain chain, Request request, Response response ) {
         try {
             if (isRequestThirdPartyCopy(request)) {
                 processThirdPartyCopy(request, response);
             } else {
                 chain.process(request, response);
             }
         } catch (ErrorResponseException e) {
-            ServletResponse.getResponse().setStatus(e.getStatus().code,
-                  e.getMessage());
+            var r = ServletResponse.getResponse();
+            int code = e.getStatus().code;
+            r.setStatus(code, e.getMessage());
+            if (code == 507) {
+                /**
+                 * https://www.rfc-editor.org/rfc/rfc4331.html#section-6
+                 * stipulates that insufficient storage response error
+                 * must be accompanied by the following error response:
+                 */
+                r.setContentType("application/xml");
+                r.setCharacterEncoding("UTF-8");
+                var body = "<?xml version=\"1.0\">\n<error xmlns=\"DAV:\">\n<quota-not-exceeded/>\n</error>\n";
+                int len = StandardCharsets.UTF_8.encode(body).limit();
+                r.setContentLength(len);
+                try {
+                    var out = r.getWriter();
+                    out.write(body);
+                } catch (IOException ioe) {
+                    LOGGER.warn("Failed to write error response body: {}",
+                                ioe.toString());
+                }
+            }
         } catch (BadRequestException e) {
             ServletResponse.getResponse().setStatus(HttpServletResponse.SC_BAD_REQUEST,
                   e.getMessage());