Use a single worker thread for LWCA key retrieval

Lightweight CA key retrieval is failing in an IPA test scenario
where many sub-CAs are created in rapid succession.  The test output
suggests resource exhaustion (e.g. LDAP server connections).  The
current implementation spawns a separate, concurrent key retriever
thread for each new LWCA.  This behaviour is suspected to be the
cause of the failure.

Modify LWCA key retrieval to use a single retriever thread per
replica.  Retrievals are processed sequentially.  The exponential
backoff on failure is retained.

The queue behaviour lives in the new `KeyRetrieverWorker` class,
which uses `java.util.Timer` under the hood.  `CAEngine`
instantiates a single instance of `KeyRetrieverWorker`, which in
turn starts the single timer thread and manages the queue.

Fixes: https://issues.redhat.com/browse/RHEL-27181
Fixes: dogtagpki#4677

Author: frasertweedale

base/ca/src/main/java/com/netscape/ca/CertificateAuthority.java

@@ -219,8 +219,6 @@ public class CertificateAuthority extends Subsystem implements IAuthority, IOCSP
     private ResponderID mResponderIDByName = null;
     private ResponderID mResponderIDByHash = null;
 
-    private Thread keyRetrieverThread;
-
     /**
      * Internal constants
      */
@@ -649,12 +647,7 @@ public synchronized void startKeyRetriever() throws EBaseException {
             return;
         }
 
-        if (keyRetrieverThread != null) {
-            logger.info("CertificateAuthority: KeyRetriever already running for authority " + authorityID);
-            return;
-        }
-
-        logger.info("CertificateAuthority: Starting KeyRetriever for authority " + authorityID);
+        logger.info("CertificateAuthority: Initialising KeyRetriever for authority " + authorityID);
 
         CAEngine engine = CAEngine.getInstance();
         CAEngineConfig engineConfig = engine.getConfig();
@@ -688,13 +681,7 @@ public synchronized void startKeyRetriever() throws EBaseException {
         }
 
         KeyRetrieverRunner runner = new KeyRetrieverRunner(keyRetriever, this);
-
-        keyRetrieverThread = new Thread(runner, "KeyRetriever-" + authorityID);
-        keyRetrieverThread.start();
-    }
-
-    public synchronized void removeKeyRetriever() {
-        keyRetrieverThread = null;
+        engine.getKeyRetrieverWorker().requestKeyRetrieval(runner);
     }
 
     public void initSigningUnits() throws Exception {

base/ca/src/main/java/com/netscape/ca/KeyRetrieverRunner.java

@@ -33,7 +33,7 @@
 import com.netscape.certsrv.ca.CAMissingKeyException;
 import com.netscape.cmsutil.crypto.CryptoUtil;
 
-public class KeyRetrieverRunner implements Runnable {
+public class KeyRetrieverRunner {
 
     public final static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(KeyRetrieverRunner.class);
 
@@ -61,24 +61,10 @@ public KeyRetrieverRunner(KeyRetriever keyRetriever, CertificateAuthority certif
 
     }
 
-    @Override
-    public void run() {
-        try {
-            long d = 10000;  // initial delay of 10 seconds
-
-            while (!_run()) {
-                logger.debug("Retrying in " + d / 1000 + " seconds");
-                try {
-                    Thread.sleep(d);
-                } catch (InterruptedException e) {
-                    break;
-                }
-                d += d / 2;  // back off
-            }
-
-        } finally {
-            ca.removeKeyRetriever();
-        }
+    /** Return the AuthorityID of the authority whose keys this runner
+     * attempts to retrieve. */
+    public AuthorityID getAuthorityID() {
+        return aid;
     }
 
     /**
@@ -89,8 +75,12 @@ public void run() {
      *         does not necessarily imply that the process fully
      *         completed.  See comments at sites of 'return true;'
      *         below.
+     *
+     * This method has package visibility to allow KeyRetrieverWorker
+     * to access it.  You do not need to call it directly when running
+     * a KeyRetrieverRunner in its own thread.
      */
-    private boolean _run() {
+    boolean attemptRetrieval() {
 
         CAEngine engine = CAEngine.getInstance();
         CAEngineConfig cs = engine.getConfig();

base/ca/src/main/java/com/netscape/ca/KeyRetrieverWorker.java

@@ -0,0 +1,83 @@
+//
+// Copyright Red Hat, Inc.
+//
+// SPDX-License-Identifier: GPL-2.0-or-later
+//
+package com.netscape.ca;
+
+import java.util.Collections;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import com.netscape.certsrv.ca.AuthorityID;
+
+/**
+ * Wrapper around a Timer to execute key retrieval attempts,
+ * rescheduling failed attempts with backoff.
+ */
+public class KeyRetrieverWorker {
+
+    public final static org.slf4j.Logger logger =
+        org.slf4j.LoggerFactory.getLogger(KeyRetrieverWorker.class);
+
+    private SortedSet<AuthorityID> aidsInQueue;
+
+    protected Timer timer;
+
+    /** Constructor initialises the queue and **starts the thread**.
+     *
+     * Only one KeyRetrieverWorker should ever be constructed.
+     */
+    public KeyRetrieverWorker() {
+        aidsInQueue = Collections.synchronizedSortedSet(new TreeSet());
+        timer = new Timer("KeyRetrieverWorker");
+    }
+
+    /** Register a key retriever with the worker thread.
+     *
+     * Following registration, no further action is required
+     * by the caller.  The worker thread takes care of retrieval
+     * attempts and backoff.
+     */
+    public void requestKeyRetrieval(KeyRetrieverRunner krr) {
+        AuthorityID aid = krr.getAuthorityID();
+        if (aidsInQueue.contains(aid)) {
+            logger.info("KeyRetriever already enqueued for authority " + aid);
+            return;
+        }
+
+        logger.info("Queuing KeyRetriever for authority " + aid);
+        aidsInQueue.add(aid);
+        Request req = new Request(krr, 10000 /* initial backoff = 10s */);
+        timer.schedule(req, 0);     // attempt immediately
+    }
+
+    private class Request extends TimerTask {
+        KeyRetrieverRunner krr;
+        long backoff_ms;
+
+        public Request(KeyRetrieverRunner krr, long backoff_ms) {
+            this.krr = krr;
+            this.backoff_ms = backoff_ms;
+        }
+
+        public void run() {
+            AuthorityID aid = krr.getAuthorityID();
+            logger.debug(aid + ": attempt retrieval");
+            if (krr.attemptRetrieval()) {
+                logger.info(aid + ": successfully retrieved key");
+                aidsInQueue.remove(aid);
+            } else {
+                logger.info(aid + ": failed to retrieve key; try again after "
+                        + backoff_ms / 1000 + "s");
+                long new_backoff = backoff_ms + backoff_ms / 2;
+                // cannot reschedule "this" -> IllegalStateException;
+                // therefore create a new Request and schedule that.
+                timer.schedule(new Request(krr, new_backoff), backoff_ms);
+            }
+        }
+    }
+
+}

base/ca/src/main/java/org/dogtagpki/server/ca/CAEngine.java

@@ -78,6 +78,7 @@
 import com.netscape.ca.CRLIssuingPoint;
 import com.netscape.ca.CRLIssuingPointConfig;
 import com.netscape.ca.CertificateAuthority;
+import com.netscape.ca.KeyRetrieverWorker;
 import com.netscape.certsrv.authentication.ISharedToken;
 import com.netscape.certsrv.base.BadRequestDataException;
 import com.netscape.certsrv.base.EBaseException;
@@ -208,6 +209,8 @@ public class CAEngine extends CMSEngine {
     protected AuthorityMonitor authorityMonitor;
     protected boolean enableAuthorityMonitor = true;
 
+    private KeyRetrieverWorker keyRetrieverWorker;
+
     // is the current KRA-related info authoritative?
     private static boolean kraInfoAuthoritative = false;
 
@@ -328,6 +331,10 @@ public boolean getEnableOCSP() {
         return enableOCSP;
     }
 
+    public KeyRetrieverWorker getKeyRetrieverWorker() {
+        return keyRetrieverWorker;
+    }
+
     /**
      * Allows certificates to have validities that are longer
      * than this certificate authority's.
@@ -1087,6 +1094,10 @@ public void initSubsystems() throws Exception {
             initCRLPublisher();
             initPublisherProcessor();
 
+            // initialise key retriever worker (before
+            // initialising signing units)
+            keyRetrieverWorker = new KeyRetrieverWorker();
+
             // initialize host CA signing units
             hostCA.initSigningUnits();