nfs41: add support for read delegations

Motivation:
Read/write delegation allow server to delegate file access check to
clients. As a result, client don't need to send a new open requests to
the server, thus, less network overhead. This is especially useful when
a client opens the same time from multiple processes (HPC workload).

Modification:

1. add record to tack issued delegations
2. Update FileTracker to return OpenRecord instead of stateid on open
3. Populate OpenRecord with delegation params, if issued
4. Issue read-delegation if requested by client if:
  - there are no accesses for write exist
  - a call-back channel to the client is established
  - client doesn't already have a delegation for the given file

A delegation is recalled if conflicting open-for-write arrives.

Result:
Server can issue read-delegation, if requested.

Acked-by: Lea Morschel
Closes: #137
Target: master

Author: kofemann

core/src/main/java/org/dcache/nfs/v4/FileTracker.java

@@ -20,6 +20,8 @@
 package org.dcache.nfs.v4;
 
 import com.google.common.util.concurrent.Striped;
+
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Iterator;
@@ -30,9 +32,13 @@
 import java.util.stream.Collectors;
 import org.dcache.nfs.ChimeraNFSException;
 import org.dcache.nfs.status.BadStateidException;
+import org.dcache.nfs.status.DelayException;
 import org.dcache.nfs.status.InvalException;
 import org.dcache.nfs.status.ShareDeniedException;
+import org.dcache.nfs.status.StaleException;
 import org.dcache.nfs.v4.xdr.nfs4_prot;
+import org.dcache.nfs.v4.xdr.nfs_fh4;
+import org.dcache.nfs.v4.xdr.open_delegation_type4;
 import org.dcache.nfs.v4.xdr.stateid4;
 import org.dcache.nfs.vfs.Inode;
 import org.dcache.nfs.util.Opaque;
@@ -55,6 +61,11 @@ public class FileTracker {
     private final Striped<Lock> filesLock = Striped.lock(Runtime.getRuntime().availableProcessors()*4);
     private final Map<Opaque, List<OpenState>> files = new ConcurrentHashMap<>();
 
+    /**
+     * Delegation records associated with open files.
+     */
+    private final Map<Opaque, List<DelegationState>> delegations = new ConcurrentHashMap<>();
+
     private static class OpenState {
 
         private final NFS4Client client;
@@ -118,6 +129,27 @@ public NFS4Client getClient() {
         }
     }
 
+    /**
+     * Record associated with open-delegation.
+     * @param client
+     * @param stateid
+     * @param delegationType
+     */
+    record DelegationState(NFS4Client client, stateid4 openStateId, stateid4 stateid, int delegationType) {
+
+    }
+
+    /**
+     * Record associated with an open file.
+     *
+     * @param openStateId
+     * @param delegationStateId
+     * @param hasDelegation
+     */
+    public record OpenRecord(stateid4 openStateId, stateid4 delegationStateId, boolean hasDelegation) {
+
+    }
+
     /**
      * Add a new open to the list of open files. If provided {@code shareAccess}
      * and {@code shareDeny} conflicts with existing opens, @{link ShareDeniedException}
@@ -127,11 +159,14 @@ public NFS4Client getClient() {
      * @param inode of opened file.
      * @param shareAccess type of access required.
      * @param shareDeny type of access to deny others.
-     * @return a snapshot of the stateid associated with open.
+     * @return a snapshot of an OpenRecord associated with open.
      * @throws ShareDeniedException if share reservation conflicts with an existing open.
      * @throws ChimeraNFSException
      */
-    public stateid4 addOpen(NFS4Client client, StateOwner owner, Inode inode, int shareAccess, int shareDeny) throws  ChimeraNFSException {
+    public OpenRecord addOpen(NFS4Client client, StateOwner owner, Inode inode, int shareAccess, int shareDeny) throws  ChimeraNFSException {
+
+        boolean wantReadDelegation = (shareAccess & nfs4_prot.OPEN4_SHARE_ACCESS_WANT_READ_DELEG) != 0;
+        boolean wantWriteDelegation = (shareAccess & nfs4_prot.OPEN4_SHARE_ACCESS_WANT_WRITE_DELEG) != 0;
 
         Opaque fileId = new Opaque(inode.getFileId());
         Lock lock = filesLock.get(fileId);
@@ -170,7 +205,43 @@ public stateid4 addOpen(NFS4Client client, StateOwner owner, Inode inode, int sh
 
                         os.stateid.seqid++;
                         //we need to return copy to avoid modification by concurrent opens
-                        return new stateid4(os.stateid.other, os.stateid.seqid);
+                        var openStateid = new stateid4(os.stateid.other, os.stateid.seqid);
+                        return new OpenRecord(openStateid, null, false);
+                }
+            }
+
+            /*
+             * REVISIT: currently only read-delegations are supported
+             */
+            var existingDelegations = delegations.get(fileId);
+
+            /*
+             * delegation is possible if:
+             * - client has a callback channel
+             * - client does not have a delegation for this file
+             * - no other open has write access
+             */
+            boolean canDelegate = client.getCB() != null &&
+                  (existingDelegations == null || existingDelegations.stream().noneMatch(d -> d.client().getId() == client.getId())) &&
+                  opens.stream().noneMatch(os -> (os.shareAccess & nfs4_prot.OPEN4_SHARE_ACCESS_WRITE) != 0);
+
+            // recall any read delegations if write
+            if ((existingDelegations != null) && (shareAccess & nfs4_prot.OPEN4_SHARE_ACCESS_WRITE) != 0) {
+
+                // REVISIT: usage of Stream#peek is an anti-pattern
+                boolean haveRecalled = existingDelegations.stream()
+                      .filter(d -> client.isLeaseValid())
+                      .peek(d -> {
+                          try {
+                              d.client().getCB()
+                                    .cbDelegationRecall(new nfs_fh4(inode.toNfsHandle()), d.stateid(), false);
+                          } catch (IOException e) {
+                              // ignore
+                          }
+                      }).count() > 0;
+
+                if (haveRecalled) {
+                    throw new DelayException("Recalling read delegations");
                 }
             }
 
@@ -180,8 +251,21 @@ public stateid4 addOpen(NFS4Client client, StateOwner owner, Inode inode, int sh
             opens.add(openState);
             state.addDisposeListener(s -> removeOpen(inode, stateid));
             stateid.seqid++;
+
             //we need to return copy to avoid modification by concurrent opens
-            return new stateid4(stateid.other, stateid.seqid);
+            var openStateid = new stateid4(stateid.other, stateid.seqid);
+
+            // REVISIT: currently only read-delegations are supported
+            if (wantReadDelegation && canDelegate) {
+                // REVISIT: currently only read-delegations are supported
+                stateid4 delegationStateid = client.createState(state.getStateOwner(), state).stateid();
+                delegations.computeIfAbsent(fileId, x -> new ArrayList<>(1))
+                        .add(new DelegationState(client, openStateid, delegationStateid, open_delegation_type4.OPEN_DELEGATE_READ));
+                return new OpenRecord(openStateid, delegationStateid, true);
+            } else {
+                //we need to return copy to avoid modification by concurrent opens
+                return new OpenRecord(openStateid, null, false);
+            }
         } finally {
             lock.unlock();
         }
@@ -241,6 +325,42 @@ public stateid4 downgradeOpen(NFS4Client client, stateid4 stateid, Inode inode,
         }
     }
 
+    /**
+     * Return delegation for the given file
+     * @param client nfs client who returns the delegation.
+     * @param stateid delegation stateid
+     * @param inode the inode of the delegated file.
+     */
+    public void delegationReturn(NFS4Client client, stateid4 stateid, Inode inode)
+          throws StaleException {
+
+        Opaque fileId = new Opaque(inode.getFileId());
+        Lock lock = filesLock.get(fileId);
+        lock.lock();
+        try {
+
+            var fileDelegations = delegations.get(fileId);
+            if (fileDelegations == null) {
+                throw new StaleException("no delegation found");
+            }
+
+            DelegationState delegation = fileDelegations.stream()
+                    .filter(d -> d.client().getId() == client.getId())
+                    .filter(d -> d.stateid().equals(stateid))
+                    .findFirst()
+                    .orElseThrow(StaleException::new);
+
+            fileDelegations.remove(delegation);
+            if (fileDelegations.isEmpty()) {
+                delegations.remove(fileId);
+            }
+
+        } finally {
+            lock.unlock();
+        }
+    }
+
+
     /**
      * Remove an open from the list.
      * @param inode of the opened file
@@ -271,6 +391,23 @@ void removeOpen(Inode inode, stateid4 stateid) {
                     files.remove(fileId);
                 }
             }
+
+            var existingDelegations = delegations.get(fileId);
+            if (existingDelegations != null) {
+                Iterator<DelegationState> dsi = existingDelegations.iterator();
+                while (dsi.hasNext()) {
+                    stateid4 os = dsi.next().openStateId();
+                    if (os.equals(stateid)) {
+                        dsi.remove();
+                        break;
+                    }
+                }
+
+                if (existingDelegations.isEmpty()) {
+                    delegations.remove(fileId);
+                }
+            }
+
         } finally {
             lock.unlock();
         }

core/src/main/java/org/dcache/nfs/v4/OperationDELEGRETURN.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009 - 2012 Deutsches Elektronen-Synchroton,
+ * Copyright (c) 2009 - 2025 Deutsches Elektronen-Synchroton,
  * Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY
  *
  * This library is free software; you can redistribute it and/or modify
@@ -19,10 +19,13 @@
  */
 package org.dcache.nfs.v4;
 
+import org.dcache.nfs.ChimeraNFSException;
 import org.dcache.nfs.nfsstat;
 import org.dcache.nfs.v4.xdr.nfs_argop4;
 import org.dcache.nfs.v4.xdr.nfs_opnum4;
 import org.dcache.nfs.v4.xdr.nfs_resop4;
+import org.dcache.nfs.v4.xdr.stateid4;
+import org.dcache.nfs.vfs.Inode;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -35,7 +38,18 @@ public OperationDELEGRETURN(nfs_argop4 args) {
     }
 
     @Override
-    public void process(CompoundContext context, nfs_resop4 result) {
-        result.opdelegreturn.status = nfsstat.NFSERR_NOTSUPP;
+    public void process(CompoundContext context, nfs_resop4 result) throws ChimeraNFSException {
+
+        final Inode inode = context.currentInode();
+        stateid4 stateid = Stateids.getCurrentStateidIfNeeded(context, _args.opdelegreturn.deleg_stateid);
+        NFS4Client client;
+        if (context.getMinorversion() > 0) {
+            client = context.getSession().getClient();
+        } else {
+            client = context.getStateHandler().getClientIdByStateId(stateid);
+        }
+
+        context.getStateHandler().getFileTracker().delegationReturn(client, stateid, inode);
+        result.opdelegreturn.status = nfsstat.NFS_OK;
     }
 }

core/src/main/java/org/dcache/nfs/v4/OperationOPEN.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2009 - 2023 Deutsches Elektronen-Synchroton,
+ * Copyright (c) 2009 - 2025 Deutsches Elektronen-Synchroton,
  * Member of the Helmholtz Association, (DESY), HAMBURG, GERMANY
  *
  * This library is free software; you can redistribute it and/or modify
@@ -22,13 +22,19 @@
 import java.io.IOException;
 import java.util.Optional;
 
+import org.dcache.nfs.v4.xdr.aceflag4;
+import org.dcache.nfs.v4.xdr.acemask4;
+import org.dcache.nfs.v4.xdr.acetype4;
+import org.dcache.nfs.v4.xdr.nfsace4;
 import org.dcache.nfs.v4.xdr.open_delegation_type4;
 import org.dcache.nfs.v4.xdr.change_info4;
 import org.dcache.nfs.v4.xdr.bitmap4;
 import org.dcache.nfs.v4.xdr.nfs4_prot;
 import org.dcache.nfs.v4.xdr.nfs_argop4;
 import org.dcache.nfs.v4.xdr.changeid4;
 import org.dcache.nfs.nfsstat;
+import org.dcache.nfs.v4.xdr.open_none_delegation4;
+import org.dcache.nfs.v4.xdr.open_read_delegation4;
 import org.dcache.nfs.v4.xdr.uint32_t;
 import org.dcache.nfs.v4.xdr.opentype4;
 import org.dcache.nfs.v4.xdr.open_claim_type4;
@@ -51,6 +57,8 @@
 import org.dcache.nfs.v4.xdr.mode4;
 import org.dcache.nfs.v4.xdr.nfs_resop4;
 import org.dcache.nfs.v4.xdr.stateid4;
+import org.dcache.nfs.v4.xdr.utf8str_mixed;
+import org.dcache.nfs.v4.xdr.why_no_delegation4;
 import org.dcache.nfs.vfs.Inode;
 import org.dcache.nfs.vfs.Stat;
 import org.dcache.oncrpc4j.rpc.OncRpcException;
@@ -85,7 +93,9 @@ public void process(CompoundContext context, nfs_resop4 result) throws ChimeraNF
         res.resok4 = new OPEN4resok();
         res.resok4.attrset = new bitmap4();
         res.resok4.delegation = new open_delegation4();
-        res.resok4.delegation.delegation_type = open_delegation_type4.OPEN_DELEGATE_NONE;
+        res.resok4.delegation.delegation_type = context.getMinorversion() == 0 ? open_delegation_type4.OPEN_DELEGATE_NONE : open_delegation_type4.OPEN_DELEGATE_NONE_EXT;
+        res.resok4.delegation.od_whynone =  new open_none_delegation4();
+        res.resok4.delegation.od_whynone.ond_why =  why_no_delegation4.WND4_NOT_WANTED;
         res.resok4.cinfo = new change_info4();
         res.resok4.cinfo.atomic = true;
 
@@ -264,15 +274,28 @@ public void process(CompoundContext context, nfs_resop4 result) throws ChimeraNF
          * THis is a perfectly a valid situation as at the end file is created and only
          * one writer is allowed.
          */
-        stateid4 stateid = context
+        var openRecord = context
                 .getStateHandler()
                 .getFileTracker()
                 .addOpen(client, owner, context.currentInode(),
                 _args.opopen.share_access.value,
                 _args.opopen.share_deny.value);
 
-        context.currentStateid(stateid);
-        res.resok4.stateid = stateid;
+        context.currentStateid(openRecord.openStateId());
+        res.resok4.stateid = openRecord.openStateId();
+        if (openRecord.hasDelegation()) {
+            res.resok4.delegation.delegation_type = open_delegation_type4.OPEN_DELEGATE_READ;
+            res.resok4.delegation.read = new open_read_delegation4();
+            res.resok4.delegation.read.stateid = openRecord.delegationStateId();
+            res.resok4.delegation.read.permissions = new nfsace4();
+            res.resok4.delegation.read.permissions.type = new acetype4(nfs4_prot.ACE4_ACCESS_ALLOWED_ACE_TYPE);
+            res.resok4.delegation.read.permissions.flag = new aceflag4(0);
+            res.resok4.delegation.read.permissions.access_mask = new acemask4(nfs4_prot.ACCESS4_READ);
+            res.resok4.delegation.read.permissions.who = new utf8str_mixed(context.getPrincipal().getName());
+        } else if ((_args.opopen.share_access.value & nfs4_prot.OPEN4_SHARE_ACCESS_WANT_ANY_DELEG) != 0) {
+            // REVISIT: shall we return something less general?
+            res.resok4.delegation.od_whynone.ond_why = why_no_delegation4.WND4_RESOURCE;
+        }
         res.status = nfsstat.NFS_OK;
 
     }