fix: add retries to certificate provisioning (#2841)

The p2p-forge service at libp2p.direct sometimes rejects requests
with 401 errors, the only thing to do is to retry.

This can probably be reverted in future if it becomes clear why
some requests are rejected.

Author: achingbrain

packages/auto-tls/README.md

@@ -25,7 +25,7 @@ repo and examine the changes made.
 -->
 
 When a publicly dialable address is detected, use the p2p-forge service at
-<https://registration.libp2p.direct> to acquire a valid Let's Encrypted-backed
+<https://registration.libp2p.direct> to acquire a valid Let's Encrypt-backed
 TLS certificate, which the node can then use with the relevant transports.
 
 The node must be configured with a listener for at least one of the following
@@ -82,27 +82,19 @@ console.info(node.getMultiaddrs())
 # Install
 
 ```console
-$ npm i @libp2p/plaintext
-```
-
-## Browser `<script>` tag
-
-Loading this module through a script tag will make it's exports available as `Libp2pPlaintext` in the global namespace.
-
-```html
-<script src="https://unpkg.com/@libp2p/plaintext/dist/index.min.js"></script>
+$ npm i @libp2p/auto-tls
 ```
 
 # API Docs
 
-- <https://libp2p.github.io/js-libp2p/modules/_libp2p_plaintext.html>
+- <https://libp2p.github.io/js-libp2p/modules/_libp2p_auto-tls.html>
 
 # License
 
 Licensed under either of
 
-- Apache 2.0, ([LICENSE-APACHE](https://github.com/libp2p/js-libp2p/blob/main/packages/connection-encrypter-plaintext/LICENSE-APACHE) / <http://www.apache.org/licenses/LICENSE-2.0>)
-- MIT ([LICENSE-MIT](https://github.com/libp2p/js-libp2p/blob/main/packages/connection-encrypter-plaintext/LICENSE-MIT) / <http://opensource.org/licenses/MIT>)
+- Apache 2.0, ([LICENSE-APACHE](https://github.com/libp2p/js-libp2p/blob/main/packages/auto-tls/LICENSE-APACHE) / <http://www.apache.org/licenses/LICENSE-2.0>)
+- MIT ([LICENSE-MIT](https://github.com/libp2p/js-libp2p/blob/main/packages/auto-tls/LICENSE-MIT) / <http://opensource.org/licenses/MIT>)
 
 # Contribution
 

packages/auto-tls/package.json

@@ -57,6 +57,8 @@
     "@multiformats/multiaddr-matcher": "^1.4.0",
     "@peculiar/x509": "^1.12.3",
     "acme-client": "^5.4.0",
+    "any-signal": "^4.1.1",
+    "delay": "^6.0.0",
     "interface-datastore": "^8.3.1",
     "multiformats": "^13.3.1",
     "uint8arrays": "^5.1.0"
@@ -66,7 +68,6 @@
     "@libp2p/peer-id": "^5.0.7",
     "aegir": "^44.0.1",
     "datastore-core": "^10.0.2",
-    "delay": "^6.0.0",
     "p-event": "^6.0.1",
     "sinon": "^19.0.2",
     "sinon-ts": "^2.0.0"

packages/auto-tls/src/auto-tls.ts

@@ -1,14 +1,16 @@
 import { ClientAuth } from '@libp2p/http-fetch/auth'
-import { serviceCapabilities, serviceDependencies, start, stop } from '@libp2p/interface'
+import { serviceCapabilities, serviceDependencies, setMaxListeners, start, stop } from '@libp2p/interface'
 import { debounce } from '@libp2p/utils/debounce'
 import { X509Certificate } from '@peculiar/x509'
 import * as acme from 'acme-client'
+import { anySignal } from 'any-signal'
+import delay from 'delay'
 import { Key } from 'interface-datastore'
 import { base36 } from 'multiformats/bases/base36'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
-import { DEFAULT_ACCOUNT_PRIVATE_KEY_BITS, DEFAULT_ACCOUNT_PRIVATE_KEY_NAME, DEFAULT_ACME_DIRECTORY, DEFAULT_CERTIFICATE_DATASTORE_KEY, DEFAULT_CERTIFICATE_PRIVATE_KEY_BITS, DEFAULT_CERTIFICATE_PRIVATE_KEY_NAME, DEFAULT_FORGE_DOMAIN, DEFAULT_FORGE_ENDPOINT, DEFAULT_PROVISION_DELAY, DEFAULT_PROVISION_TIMEOUT, DEFAULT_RENEWAL_THRESHOLD } from './constants.js'
+import { DEFAULT_ACCOUNT_PRIVATE_KEY_BITS, DEFAULT_ACCOUNT_PRIVATE_KEY_NAME, DEFAULT_ACME_DIRECTORY, DEFAULT_CERTIFICATE_DATASTORE_KEY, DEFAULT_CERTIFICATE_PRIVATE_KEY_BITS, DEFAULT_CERTIFICATE_PRIVATE_KEY_NAME, DEFAULT_FORGE_DOMAIN, DEFAULT_FORGE_ENDPOINT, DEFAULT_PROVISION_DELAY, DEFAULT_PROVISION_REQUEST_TIMEOUT, DEFAULT_PROVISION_TIMEOUT, DEFAULT_RENEWAL_THRESHOLD } from './constants.js'
 import { DomainMapper } from './domain-mapper.js'
 import { createCsr, importFromPem, loadOrCreateKey, supportedAddressesFilter } from './utils.js'
 import type { AutoTLSComponents, AutoTLSInit, AutoTLS as AutoTLSInterface } from './index.js'
@@ -19,6 +21,8 @@ import type { DebouncedFunction } from '@libp2p/utils/debounce'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { Datastore } from 'interface-datastore'
 
+const RETRY_DELAY = 5_000
+
 type CertificateEvent = 'certificate:provision' | 'certificate:renew'
 
 interface Certificate {
@@ -40,6 +44,7 @@ export class AutoTLS implements AutoTLSInterface {
   private readonly acmeDirectory: URL
   private readonly clientAuth: ClientAuth
   private readonly provisionTimeout: number
+  private readonly provisionRequestTimeout: number
   private readonly renewThreshold: number
   private started: boolean
   private shutdownController?: AbortController
@@ -68,6 +73,7 @@ export class AutoTLS implements AutoTLSInterface {
     this.forgeDomain = init.forgeDomain ?? DEFAULT_FORGE_DOMAIN
     this.acmeDirectory = new URL(init.acmeDirectory ?? DEFAULT_ACME_DIRECTORY)
     this.provisionTimeout = init.provisionTimeout ?? DEFAULT_PROVISION_TIMEOUT
+    this.provisionRequestTimeout = init.provisionRequestTimeout ?? DEFAULT_PROVISION_REQUEST_TIMEOUT
     this.renewThreshold = init.renewThreshold ?? DEFAULT_RENEWAL_THRESHOLD
     this.accountPrivateKeyName = init.accountPrivateKeyName ?? DEFAULT_ACCOUNT_PRIVATE_KEY_NAME
     this.accountPrivateKeyBits = init.accountPrivateKeyBits ?? DEFAULT_ACCOUNT_PRIVATE_KEY_BITS
@@ -108,6 +114,7 @@ export class AutoTLS implements AutoTLSInterface {
     await start(this.domainMapper)
     this.events.addEventListener('self:peer:update', this.onSelfPeerUpdate)
     this.shutdownController = new AbortController()
+    setMaxListeners(Infinity, this.shutdownController.signal)
     this.started = true
   }
 
@@ -120,7 +127,8 @@ export class AutoTLS implements AutoTLSInterface {
   }
 
   private _onSelfPeerUpdate (): void {
-    const addresses = this.addressManager.getAddresses().filter(supportedAddressesFilter)
+    const addresses = this.addressManager.getAddresses()
+      .filter(supportedAddressesFilter)
 
     if (addresses.length === 0) {
       this.log('not fetching certificate as we have no public addresses')
@@ -139,11 +147,29 @@ export class AutoTLS implements AutoTLSInterface {
 
     this.fetching = true
 
-    this.fetchCertificate(addresses, {
-      signal: AbortSignal.timeout(this.provisionTimeout)
+    Promise.resolve().then(async () => {
+      let attempt = 0
+
+      while (true) {
+        if (this.shutdownController?.signal.aborted === true) {
+          throw this.shutdownController.signal.reason
+        }
+
+        try {
+          await this.fetchCertificate(addresses, {
+            signal: AbortSignal.timeout(this.provisionTimeout)
+          })
+
+          return
+        } catch (err) {
+          this.log.error('provisioning certificate failed on attempt %d - %e', attempt++, err)
+        }
+
+        await delay(RETRY_DELAY)
+      }
     })
       .catch(err => {
-        this.log.error('error fetching certificates - %e', err)
+        this.log.error('giving up provisioning certificate - %e', err)
       })
       .finally(() => {
         this.fetching = false
@@ -190,7 +216,9 @@ export class AutoTLS implements AutoTLSInterface {
     // emit a certificate event
     this.log('dispatching %s', event)
     this.events.safeDispatchEvent(event, {
-      detail: this.certificate
+      detail: {
+        ...this.certificate
+      }
     })
   }
 
@@ -271,7 +299,33 @@ export class AutoTLS implements AutoTLSInterface {
       email: this.email,
       termsOfServiceAgreed: true,
       challengeCreateFn: async (authz, challenge, keyAuthorization) => {
-        await this.configureAcmeChallengeResponse(multiaddrs, keyAuthorization, options)
+        const signal = anySignal([this.shutdownController?.signal, options?.signal])
+        setMaxListeners(Infinity, signal)
+
+        let attempt = 0
+
+        while (true) {
+          if (signal.aborted) {
+            throw signal.reason
+          }
+
+          try {
+            const timeout = AbortSignal.timeout(this.provisionRequestTimeout)
+            const signal = anySignal([timeout, options?.signal])
+            setMaxListeners(Infinity, timeout, signal)
+
+            await this.configureAcmeChallengeResponse(multiaddrs, keyAuthorization, {
+              ...options,
+              signal
+            })
+
+            return
+          } catch (err: any) {
+            this.log.error('contacting %s failed on attempt %d - %e', this.forgeEndpoint, attempt++, err.cause ?? err)
+          }
+
+          await delay(RETRY_DELAY)
+        }
       },
       challengeRemoveFn: async (authz, challenge, keyAuthorization) => {
         // no-op
@@ -285,8 +339,9 @@ export class AutoTLS implements AutoTLSInterface {
     const addresses = multiaddrs.map(ma => ma.toString())
 
     const endpoint = `${this.forgeEndpoint}v1/_acme-challenge`
-    this.log('asking %sv1/_acme-challenge to respond to the acme DNS challenge on our behalf', endpoint)
+    this.log('asking %s to respond to the acme DNS challenge on our behalf', endpoint)
     this.log('dialback public addresses: %s', addresses.join(', '))
+
     const response = await this.clientAuth.authenticatedFetch(endpoint, {
       method: 'POST',
       headers: {

packages/auto-tls/src/constants.ts

@@ -1,7 +1,8 @@
 export const DEFAULT_FORGE_ENDPOINT = 'https://registration.libp2p.direct'
 export const DEFAULT_FORGE_DOMAIN = 'libp2p.direct'
 export const DEFAULT_ACME_DIRECTORY = 'https://acme-v02.api.letsencrypt.org/directory'
-export const DEFAULT_PROVISION_TIMEOUT = 10_000
+export const DEFAULT_PROVISION_TIMEOUT = 120_000
+export const DEFAULT_PROVISION_REQUEST_TIMEOUT = 10_000
 export const DEFAULT_PROVISION_DELAY = 5_000
 export const DEFAULT_RENEWAL_THRESHOLD = 86_400_000
 export const DEFAULT_ACCOUNT_PRIVATE_KEY_NAME = 'auto-tls-acme-account-private-key'

packages/auto-tls/src/index.ts

@@ -104,10 +104,18 @@ export interface AutoTLSInit {
   /**
    * How long to attempt to acquire a certificate before timing out in ms
    *
-   * @default 10000
+   * @default 120_000
    */
   provisionTimeout?: number
 
+  /**
+   * How long asking the forge endpoint to answer a DNS challenge can take
+   * before we retry
+   *
+   * @default 10_000
+   */
+  provisionRequestTimeout?: number
+
   /**
    * Certificates are acquired when the `self:peer:update` event fires, which
    * happens when the node's addresses change. To avoid starting to map ports