ACME: set a limit on server-indicated retry intervals.

Previously, it was possible to suspend the module execution by sending a
sufficiently high Retry-After value.  Now we will refuse to wait longer
than one minute.

Author: bavshin-f5

src/acme.rs

@@ -34,7 +34,13 @@ pub mod solvers;
 pub mod types;
 
 const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(1);
-const MAX_RETRY_INTERVAL: Duration = Duration::from_secs(8);
+
+/// Upper limit for locally generated increasing backoff interval.
+const MAX_BACKOFF_INTERVAL: Duration = Duration::from_secs(8);
+
+/// Upper limit for server-generated retry intervals (Retry-After).
+const MAX_SERVER_RETRY_INTERVAL: Duration = Duration::from_secs(60);
+
 static REPLAY_NONCE: http::HeaderName = http::HeaderName::from_static("replay-nonce");
 
 pub enum NewAccountOutput<'a> {
@@ -232,10 +238,22 @@ where
 
             let err: types::Problem = deserialize_body(res.body())?;
 
-            let retriable = matches!(
-                err.kind,
-                types::ErrorKind::BadNonce | types::ErrorKind::RateLimited
-            );
+            let retriable = match err.kind {
+                types::ErrorKind::RateLimited => {
+                    // The server may ask us to retry in several hours or days.
+                    if let Some(val) = res
+                        .headers()
+                        .get(http::header::RETRY_AFTER)
+                        .and_then(parse_retry_after)
+                        .filter(|x| x > &MAX_SERVER_RETRY_INTERVAL)
+                    {
+                        return Err(RequestError::RateLimited(val));
+                    }
+                    true
+                }
+                types::ErrorKind::BadNonce => true,
+                _ => false,
+            };
 
             if retriable && wait_for_retry(&res, &mut tries).await {
                 ngx_log_debug!(self.log.as_ptr(), "retrying failed request ({err})");
@@ -416,7 +434,7 @@ where
             _ => order.status = OrderStatus::Processing,
         };
 
-        let mut tries = backoff(MAX_RETRY_INTERVAL, self.finalize_timeout);
+        let mut tries = backoff(MAX_BACKOFF_INTERVAL, self.finalize_timeout);
 
         while order.status == OrderStatus::Processing && wait_for_retry(&res, &mut tries).await {
             drop(order);
@@ -466,7 +484,7 @@ where
             return Err(NewCertificateError::ChallengeStatus(result.status));
         }
 
-        let mut tries = backoff(MAX_RETRY_INTERVAL, self.authorization_timeout);
+        let mut tries = backoff(MAX_BACKOFF_INTERVAL, self.authorization_timeout);
         wait_for_retry(&res, &mut tries).await;
 
         let result = loop {
@@ -552,7 +570,8 @@ async fn wait_for_retry<B>(
         .headers()
         .get(http::header::RETRY_AFTER)
         .and_then(parse_retry_after)
-        .unwrap_or(interval);
+        .unwrap_or(interval)
+        .min(MAX_SERVER_RETRY_INTERVAL);
 
     sleep(retry_after).await;
     true

src/acme/error.rs

@@ -4,6 +4,7 @@
 // LICENSE file in the root directory of this source tree.
 
 use core::error::Error as StdError;
+use core::time::Duration;
 
 use ngx::allocator::{unsize_box, Box};
 use thiserror::Error;
@@ -133,6 +134,9 @@ pub enum RequestError {
     #[error(transparent)]
     Protocol(#[from] Problem),
 
+    #[error("rate limit exceeded, next attempt in {0:?}")]
+    RateLimited(Duration),
+
     #[error("cannot serialize request ({0})")]
     RequestFormat(serde_json::Error),
 

src/lib.rs

@@ -23,6 +23,7 @@ use openssl::x509::X509;
 use time::TimeRange;
 use zeroize::Zeroizing;
 
+use crate::acme::error::RequestError;
 use crate::acme::AcmeClient;
 use crate::conf::{AcmeMainConfig, AcmeServerConfig, NGX_HTTP_ACME_COMMANDS};
 use crate::net::http::NgxHttpClient;
@@ -312,6 +313,15 @@ async fn ngx_http_acme_update_certificates_for_issuer(
                     issuer.set_invalid(&err);
                     return Ok(Time::MAX);
                 }
+                Err(acme::error::NewAccountError::Request(err @ RequestError::RateLimited(x))) => {
+                    ngx_log_error!(
+                        NGX_LOG_WARN,
+                        log.as_ptr(),
+                        "{err} while creating account for acme issuer \"{issuer}\"",
+                        issuer = issuer.name
+                    );
+                    return Ok(Time::now() + x);
+                }
                 Err(err) => {
                     ngx_log_error!(
                         NGX_LOG_WARN,
@@ -373,6 +383,15 @@ async fn ngx_http_acme_update_certificates_for_issuer(
 
                 next
             }
+            Err(acme::error::NewCertificateError::Request(err @ RequestError::RateLimited(x))) => {
+                ngx_log_error!(
+                    NGX_LOG_WARN,
+                    log.as_ptr(),
+                    "{err} while updating certificate \"{issuer}/{order_id}\"",
+                    issuer = issuer.name,
+                );
+                return Ok(Time::now() + x);
+            }
             Err(ref err) if err.is_invalid() => {
                 ngx_log_error!(
                     NGX_LOG_ERR,