ACME: restore previously issued certs from "state_path".

Improves time to readiness after a full restart of the server.

Author: bavshin-f5

Cargo.lock

@@ -11,6 +11,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 http = "1.3.1"
+libc = "0.2.174"
 openssl = { version = "0.10.73", features = ["bindgen"] }
 openssl-foreign-types = { package = "foreign-types", version = "0.3" }
 openssl-sys = { version = "0.9.109", features = ["bindgen"] }

Cargo.toml

@@ -8,6 +8,7 @@ use std::env;
 /// [1]: https://github.com/rust-lang/cargo/issues/3544
 fn main() {
     detect_nginx_features();
+    detect_libssl_features();
 
     // Generate required compiler flags
     if cfg!(target_os = "macos") {
@@ -57,3 +58,33 @@ fn detect_nginx_features() {
         }
     }
 }
+
+/// Detects libssl implementation and version.
+fn detect_libssl_features() {
+    // OpenSSL
+    let openssl_features = ["awslc", "boringssl", "libressl", "openssl", "openssl111"];
+    let openssl_version = env::var("DEP_OPENSSL_VERSION_NUMBER").unwrap_or_default();
+    let openssl_version = u64::from_str_radix(&openssl_version, 16).unwrap_or(0);
+
+    println!(
+        "cargo::rustc-check-cfg=cfg(openssl, values(\"{}\"))",
+        openssl_features.join("\",\"")
+    );
+
+    #[allow(clippy::unusual_byte_groupings)]
+    let openssl = if env::var("DEP_OPENSSL_AWSLC").is_ok() {
+        "awslc"
+    } else if env::var("DEP_OPENSSL_BORINGSSL").is_ok() {
+        "boringssl"
+    } else if env::var("DEP_OPENSSL_LIBRESSL").is_ok() {
+        "libressl"
+    } else {
+        if openssl_version >= 0x01_01_01_00_0 {
+            println!("cargo::rustc-cfg=openssl=\"openssl111\"");
+        }
+
+        "openssl"
+    };
+
+    println!("cargo::rustc-cfg=openssl=\"{openssl}\"");
+}

build.rs

@@ -17,12 +17,14 @@ use ngx::sync::RwLock;
 use openssl::pkey::{PKey, Private};
 use thiserror::Error;
 
+use super::ext::NgxConfExt;
 use super::order::CertificateOrder;
 use super::pkey::PrivateKey;
 use super::ssl::NgxSsl;
 use super::AcmeMainConfig;
-use crate::state::certificate::CertificateContext;
+use crate::state::certificate::{CertificateContext, CertificateContextInner};
 use crate::state::issuer::IssuerContext;
+use crate::time::{Time, TimeRange};
 
 const ACCOUNT_KEY_FILE: &str = "account.key";
 const NGX_ACME_DEFAULT_RESOLVER_TIMEOUT: ngx_msec_t = 30000;
@@ -163,7 +165,32 @@ impl Issuer {
                 self.name
             );
 
-            let cert = CertificateContext::Empty;
+            let mut cert = CertificateContext::Empty;
+
+            if let Some(state_dir) = unsafe { StateDir::from_ptr(self.state_path) } {
+                match state_dir.load_certificate(cf, order) {
+                    Ok(x) => {
+                        ngx_log_debug!(
+                            cf.log,
+                            "acme: found cached certificate {}/{}, next update in {:?}",
+                            self.name,
+                            order.cache_key(),
+                            (x.next - Time::now()),
+                        );
+                        cert = CertificateContext::Local(x);
+                    }
+                    Err(CachedCertificateError::NotFound) => (),
+                    Err(err) => {
+                        ngx_log_debug!(
+                            cf.log,
+                            "acme: cannot load certificate {}/{} from state path: {}",
+                            self.name,
+                            order.cache_key(),
+                            err
+                        );
+                    }
+                }
+            }
 
             if self.orders.try_insert(order.clone(), cert).is_err() {
                 return Err(Status::NGX_ERROR);
@@ -239,6 +266,20 @@ impl Issuer {
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+enum CachedCertificateError {
+    #[error(transparent)]
+    Alloc(#[from] AllocError),
+    #[error("X509_check_private_key() failed: {0}")]
+    Mismatch(openssl::error::ErrorStack),
+    #[error("file not found")]
+    NotFound,
+    #[error(transparent)]
+    Ssl(#[from] openssl::error::ErrorStack),
+    #[error("failed to load file: {0}")]
+    CertificateFetch(#[from] super::ssl::CertificateFetchError),
+}
+
 /// The StateDir helper encapsulates operations with a persistent state in the state directory.
 #[repr(transparent)]
 struct StateDir(ngx_path_t);
@@ -264,4 +305,52 @@ impl StateDir {
     pub fn write(&self, path: &std::path::Path, data: &[u8]) -> Result<(), std::io::Error> {
         std::fs::write(path, data)
     }
+
+    pub fn load_certificate(
+        &self,
+        cf: &mut ngx_conf_t,
+        order: &CertificateOrder<ngx_str_t, Pool>,
+    ) -> Result<CertificateContextInner<Pool>, CachedCertificateError> {
+        use openssl_foreign_types::ForeignType;
+        #[cfg(ngx_ssl_cache)]
+        use openssl_foreign_types::ForeignTypeRef;
+
+        let name = order.cache_key();
+
+        let cert = std::format!("{}/{}.crt", self.0.name, name);
+        if matches!(std::fs::exists(&cert), Ok(false)) {
+            return Err(CachedCertificateError::NotFound);
+        }
+
+        let key = std::format!("{}/{}.key", self.0.name, name);
+        if matches!(std::fs::exists(&key), Ok(false)) {
+            return Err(CachedCertificateError::NotFound);
+        }
+
+        let stack = super::ssl::conf_read_certificate(cf, &cert)?;
+        #[allow(clippy::get_first)] // ^ can return Stack or Vec, depending on the NGINX version
+        let cert = stack
+            .get(0)
+            .ok_or(super::ssl::CertificateFetchError::Fetch(c"no certificates"))?;
+        let pkey = super::ssl::conf_read_private_key(cf, &key)?;
+
+        if unsafe { openssl_sys::X509_check_private_key(cert.as_ptr(), pkey.as_ptr()) } != 1 {
+            return Err(CachedCertificateError::Mismatch(
+                openssl::error::ErrorStack::get(),
+            ));
+        }
+
+        let valid = TimeRange::from_x509(cert).unwrap_or_default();
+        let temp_alloc = unsafe { Pool::from_ngx_pool(cf.temp_pool) };
+
+        let mut chain: Vec<u8, Pool> = Vec::new_in(temp_alloc.clone());
+        for x509 in stack.into_iter() {
+            chain.extend(x509.to_pem()?.into_iter());
+        }
+
+        let mut cert = CertificateContextInner::new_in(cf.pool());
+        cert.set(&chain, &pkey.private_key_to_pem_pkcs8()?, valid)?;
+
+        Ok(cert)
+    }
 }

src/conf/issuer.rs

@@ -9,6 +9,7 @@ use nginx_sys::{
 use ngx::allocator::AllocError;
 use ngx::core::Status;
 use openssl::pkey::{PKey, Private};
+use openssl::x509::X509;
 use openssl_sys::SSL_CTX_set_default_verify_paths;
 use thiserror::Error;
 
@@ -21,6 +22,30 @@ pub enum CertificateFetchError {
     #[error("{0:?} {1}")]
     Ssl(&'static CStr, openssl::error::ErrorStack),
 }
+
+#[cfg(ngx_ssl_cache)]
+pub fn conf_read_certificate(
+    cf: &mut ngx_conf_t,
+    name: &str,
+) -> Result<openssl::stack::Stack<X509>, CertificateFetchError> {
+    conf_ssl_cache_fetch(cf, nginx_sys::NGX_SSL_CACHE_CERT as _, name)
+}
+
+#[cfg(not(ngx_ssl_cache))]
+pub fn conf_read_certificate(
+    _cf: &mut ngx_conf_t,
+    name: &str,
+) -> Result<std::vec::Vec<X509>, CertificateFetchError> {
+    let Ok(buf) = std::fs::read_to_string(name) else {
+        return Err(CertificateFetchError::Fetch(c"cannot load certificate"));
+    };
+
+    match X509::stack_from_pem(buf.as_bytes()) {
+        Ok(x) => Ok(x),
+        Err(err) => Err(CertificateFetchError::Ssl(c"cannot load key", err)),
+    }
+}
+
 #[cfg(ngx_ssl_cache)]
 pub fn conf_read_private_key(
     cf: &mut ngx_conf_t,

src/conf/ssl.rs

@@ -1,6 +1,6 @@
 use ngx::allocator::{AllocError, Allocator, TryCloneIn};
 use ngx::collections::Vec;
-use ngx::core::SlabPool;
+use ngx::core::{Pool, SlabPool};
 use ngx::sync::RwLock;
 
 use crate::time::{jitter, Time, TimeRange};
@@ -11,6 +11,8 @@ pub type SharedCertificateContext = RwLock<CertificateContextInner<SlabPool>>;
 pub enum CertificateContext {
     #[default]
     Empty,
+    // Previously issued certificate, restored from the state directory.
+    Local(CertificateContextInner<Pool>),
     // Ready to use certificate in shared memory.
     Shared(&'static SharedCertificateContext),
 }

src/state/certificate.rs

@@ -1,6 +1,6 @@
 use core::ptr;
 
-use ngx::allocator::AllocError;
+use ngx::allocator::{AllocError, TryCloneIn};
 use ngx::collections::Queue;
 use ngx::core::SlabPool;
 use ngx::sync::RwLock;
@@ -21,7 +21,12 @@ impl IssuerContext {
         let mut certificates = Queue::try_new_in(alloc.clone())?;
 
         for (_, value) in issuer.orders.iter_mut() {
-            let ctx = CertificateContextInner::new_in(alloc.clone());
+            let ctx = if let CertificateContext::Local(value) = value {
+                value.try_clone_in(alloc.clone())?
+            } else {
+                CertificateContextInner::new_in(alloc.clone())
+            };
+
             let ctx = certificates.push_back(RwLock::new(ctx))?;
             *value = CertificateContext::Shared(unsafe { &*ptr::from_ref(ctx) });
         }

src/state/issuer.rs

@@ -2,6 +2,9 @@ use core::ops;
 use core::time::Duration;
 
 use nginx_sys::{ngx_random, ngx_time, time_t};
+use openssl::asn1::Asn1TimeRef;
+use openssl::x509::X509Ref;
+use openssl_foreign_types::ForeignTypeRef;
 use thiserror::Error;
 
 #[derive(Debug, Error)]
@@ -16,6 +19,66 @@ pub struct InvalidTime;
 #[repr(transparent)]
 pub struct Time(time_t);
 
+impl TryFrom<&Asn1TimeRef> for Time {
+    type Error = InvalidTime;
+
+    #[cfg(openssl = "openssl111")]
+    fn try_from(asn1time: &Asn1TimeRef) -> Result<Self, Self::Error> {
+        let val = unsafe {
+            let mut tm: libc::tm = core::mem::zeroed();
+            if openssl_sys::ASN1_TIME_to_tm(asn1time.as_ptr(), &mut tm) != 1 {
+                return Err(InvalidTime);
+            }
+            libc::timegm(&mut tm) as _
+        };
+
+        Ok(Time(val))
+    }
+
+    #[cfg(any(openssl = "awslc", openssl = "boringssl"))]
+    fn try_from(asn1time: &Asn1TimeRef) -> Result<Self, Self::Error> {
+        let mut val: time_t = 0;
+        if unsafe { openssl_sys::ASN1_TIME_to_time_t(asn1time.as_ptr(), &mut val) } != 1 {
+            return Err(InvalidTime);
+        }
+        Ok(Time(val))
+    }
+
+    #[cfg(not(any(openssl = "openssl111", openssl = "awslc", openssl = "boringssl")))]
+    fn try_from(asn1time: &Asn1TimeRef) -> Result<Self, Self::Error> {
+        pub const NGX_INVALID_TIME: time_t = nginx_sys::NGX_ERROR as _;
+
+        use openssl_sys::{
+            ASN1_TIME_print, BIO_free, BIO_get_mem_data, BIO_new, BIO_s_mem, BIO_write,
+        };
+
+        let val = unsafe {
+            let bio = BIO_new(BIO_s_mem());
+            if bio.is_null() {
+                openssl::error::ErrorStack::get(); // clear errors
+                return Err(InvalidTime);
+            }
+
+            let mut value: *mut core::ffi::c_char = core::ptr::null_mut();
+            /* fake weekday prepended to match C asctime() format */
+            let prefix = c"Tue ";
+            BIO_write(bio, prefix.as_ptr().cast(), prefix.count_bytes() as _);
+            ASN1_TIME_print(bio, asn1time.as_ptr());
+            let len = BIO_get_mem_data(bio, &mut value);
+            let val = ngx_parse_http_time(value.cast(), len as _);
+
+            BIO_free(bio);
+            val
+        };
+
+        if val == NGX_INVALID_TIME {
+            return Err(InvalidTime);
+        }
+
+        Ok(Time(val))
+    }
+}
+
 impl Time {
     // time_t can be signed, but is not supposed to be negative
     pub const MIN: Self = Self(0);
@@ -33,6 +96,18 @@ pub struct TimeRange {
 }
 
 impl TimeRange {
+    pub fn new(start: Time, end: Time) -> Self {
+        // ensure that end >= start
+        let end = end.max(start);
+        Self { start, end }
+    }
+
+    pub fn from_x509(x509: &X509Ref) -> Option<Self> {
+        let start = Time::try_from(x509.not_before()).ok()?;
+        let end = Time::try_from(x509.not_after()).ok()?;
+        Some(Self::new(start, end))
+    }
+
     /// Returns duration between the start and the end of the interval.
     #[inline]
     pub fn duration(&self) -> Duration {