ACME: define and allocate shared data structures.

Author: bavshin-f5

src/conf.rs

@@ -16,6 +16,7 @@ use self::issuer::Issuer;
 use self::order::CertificateOrder;
 use self::pkey::PrivateKey;
 use self::shared_zone::{SharedZone, ACME_ZONE_NAME, ACME_ZONE_SIZE};
+use crate::state::AcmeSharedData;
 
 pub mod ext;
 pub mod identifier;
@@ -31,6 +32,7 @@ const NGX_CONF_DUPLICATE: *mut c_char = c"is duplicate".as_ptr().cast_mut();
 #[derive(Debug, Default)]
 pub struct AcmeMainConfig {
     pub issuers: Vec<Issuer>,
+    pub data: Option<&'static AcmeSharedData>,
     pub shm_zone: shared_zone::SharedZone,
 }
 
@@ -500,7 +502,10 @@ impl AcmeMainConfig {
             self.shm_zone = SharedZone::Configured(ACME_ZONE_NAME, ACME_ZONE_SIZE);
         }
 
+        let amcfp = ptr::from_mut(self).cast();
         let shm_zone = self.shm_zone.request(cf)?;
+        shm_zone.init = Some(crate::state::ngx_acme_shared_zone_init);
+        shm_zone.data = amcfp;
         shm_zone.noreuse = 1;
 
         Ok(())

src/conf/issuer.rs

@@ -13,13 +13,16 @@ use ngx::collections::{RbTreeMap, Vec};
 use ngx::core::{Pool, Status};
 use ngx::http::{HttpModuleLocationConf, NgxHttpCoreModule};
 use ngx::ngx_log_debug;
+use ngx::sync::RwLock;
 use openssl::pkey::{PKey, Private};
 use thiserror::Error;
 
 use super::order::CertificateOrder;
 use super::pkey::PrivateKey;
 use super::ssl::NgxSsl;
 use super::AcmeMainConfig;
+use crate::state::certificate::CertificateContext;
+use crate::state::issuer::IssuerContext;
 
 const ACCOUNT_KEY_FILE: &str = "account.key";
 const NGX_ACME_DEFAULT_RESOLVER_TIMEOUT: ngx_msec_t = 30000;
@@ -41,8 +44,9 @@ pub struct Issuer {
     // Generated fields
     // ngx_ssl_t stores a pointer to itself in SSL_CTX ex_data.
     pub ssl: Box<NgxSsl, Pool>,
-    pub orders: RbTreeMap<CertificateOrder<ngx_str_t, Pool>, (), Pool>,
+    pub orders: RbTreeMap<CertificateOrder<ngx_str_t, Pool>, CertificateContext, Pool>,
     pub pkey: Option<PKey<Private>>,
+    pub data: Option<&'static RwLock<IssuerContext>>,
 }
 
 #[derive(Debug, Error)]
@@ -83,6 +87,7 @@ impl Issuer {
             ssl,
             pkey: None,
             orders: RbTreeMap::try_new_in(alloc)?,
+            data: None,
         })
     }
 
@@ -158,7 +163,9 @@ impl Issuer {
                 self.name
             );
 
-            if self.orders.try_insert(order.clone(), ()).is_err() {
+            let cert = CertificateContext::Empty;
+
+            if self.orders.try_insert(order.clone(), cert).is_err() {
                 return Err(Status::NGX_ERROR);
             }
         } else {

src/conf/shared_zone.rs

@@ -2,7 +2,7 @@ use core::ffi::c_void;
 use core::ptr::{self, NonNull};
 
 use nginx_sys::{ngx_conf_t, ngx_int_t, ngx_shm_zone_t, ngx_str_t, NGX_ERROR};
-use ngx::core::Status;
+use ngx::core::{SlabPool, Status};
 use ngx::http::HttpModule;
 use ngx::log::ngx_cycle_log;
 use ngx::{ngx_log_debug, ngx_string};
@@ -32,6 +32,13 @@ pub enum SharedZoneError {
 }
 
 impl SharedZone {
+    pub fn allocator(&self) -> Option<SlabPool> {
+        match self {
+            Self::Ready(zone) => unsafe { SlabPool::from_shm_zone(zone.as_ref()) },
+            _ => None,
+        }
+    }
+
     pub fn is_configured(&self) -> bool {
         !matches!(self, Self::Unset)
     }

src/lib.rs

@@ -14,6 +14,8 @@ use crate::conf::{AcmeMainConfig, AcmeServerConfig, NGX_HTTP_ACME_COMMANDS};
 use crate::variables::NGX_HTTP_ACME_VARS;
 
 mod conf;
+mod state;
+mod time;
 mod variables;
 
 #[derive(Debug)]

src/state.rs

@@ -0,0 +1,140 @@
+//! Shared runtime state of the module.
+use core::ffi::c_void;
+use core::ptr;
+
+use nginx_sys::{ngx_int_t, ngx_shm_zone_t, NGX_LOG_EMERG};
+use ngx::allocator::{AllocError, Allocator, Box, TryCloneIn};
+use ngx::collections::Queue;
+use ngx::core::{SlabPool, Status};
+use ngx::log::ngx_cycle_log;
+use ngx::sync::RwLock;
+use ngx::{ngx_log_debug, ngx_log_error};
+
+use crate::conf::shared_zone::SharedZone;
+use crate::conf::AcmeMainConfig;
+
+pub use self::certificate::CertificateContext;
+pub use self::issuer::IssuerContext;
+
+pub mod certificate;
+pub mod issuer;
+
+#[derive(Debug)]
+pub struct AcmeSharedData<A = SlabPool>
+where
+    A: Allocator + Clone,
+{
+    pub issuers: Queue<RwLock<IssuerContext>, A>,
+}
+
+impl<A> AcmeSharedData<A>
+where
+    A: Allocator + Clone,
+{
+    pub fn try_new_in(alloc: A) -> Result<Self, AllocError> {
+        Ok(Self {
+            issuers: Queue::try_new_in(alloc)?,
+        })
+    }
+}
+
+pub extern "C" fn ngx_acme_shared_zone_init(
+    shm_zone: *mut ngx_shm_zone_t,
+    data: *mut c_void,
+) -> ngx_int_t {
+    // SAFETY: shm_zone is always valid in this callback
+    let shm_zone = unsafe { &mut *shm_zone };
+    let log = ngx_cycle_log().as_ptr();
+
+    ngx_log_debug!(
+        log,
+        "acme: init shared zone \"{}:{}\"",
+        shm_zone.shm.name,
+        shm_zone.shm.size,
+    );
+
+    let oamcf = unsafe { data.cast::<AcmeMainConfig>().as_ref() };
+    let amcf = unsafe { shm_zone.data.cast::<AcmeMainConfig>().as_mut().unwrap() };
+    let zone = SharedZone::Ready(shm_zone.into());
+
+    let mut alloc = zone.allocator().expect("shared zone allocator");
+
+    // Our shared zone is `noreuse`, meaning that we get an empty zone every time unless we are
+    // running on Windows.
+
+    let Ok(mut data) =
+        AcmeSharedData::try_new_in(alloc.clone()).and_then(|x| Box::try_new_in(x, alloc.clone()))
+    else {
+        ngx_log_error!(NGX_LOG_EMERG, log, "cannot allocate acme shared data");
+        return Status::NGX_ERROR.into();
+    };
+
+    for issuer in &mut amcf.issuers[..] {
+        // Create new shared data.
+        let Ok(ctx) = IssuerContext::try_new_in(issuer, alloc.clone()) else {
+            ngx_log_error!(
+                NGX_LOG_EMERG,
+                log,
+                "cannot allocate acme issuer \"{}\"",
+                issuer.name,
+            );
+            return Status::NGX_ERROR.into();
+        };
+
+        // Copy data from the previous cycle.
+        if let Some(oissuer) = oamcf.and_then(|x| x.issuer(&issuer.name)) {
+            ngx_log_debug!(log, "acme: copy old data for issuer \"{}\"", issuer.name);
+
+            for (order, ctx) in issuer.orders.iter_mut() {
+                // Should not fail as we just allocated all the certificate contexts.
+                let CertificateContext::Shared(ctx) = ctx else {
+                    continue;
+                };
+
+                let Some(CertificateContext::Shared(octx)) = oissuer.orders.get(order) else {
+                    continue;
+                };
+
+                // The old shared zone is going away as soon as we're done, so we have to copy the
+                // data to the new slab pool.
+                let Ok(cloned) = octx.read().try_clone_in(alloc.clone()) else {
+                    return Status::NGX_ERROR.into();
+                };
+
+                *ctx.write() = cloned;
+            }
+        }
+
+        if let Ok(ctx) = data.issuers.push_back(RwLock::new(ctx)) {
+            // SAFETY: we ensured that the chosen data structure will not move the IssuerContext,
+            // thus the pointer will remain valid beyond this scope.
+            //
+            // The assigned lifetime is a bit misleading though; shared zone will be unmapped
+            // while the main config is still present, right before calling the cycle pool cleanup.
+            // A proper ownership-tracking pointer could attempt to unref the data from the config
+            // destructor _after_ the zone is unmapped and thus trip on an invalid address.
+            //
+            // Of all the ways to handle that, we are picking the most obviously unsafe to make
+            // sure this detail is not missed while reading.
+            issuer.data = Some(unsafe { &*ptr::from_ref(ctx) });
+        } else {
+            ngx_log_error!(
+                NGX_LOG_EMERG,
+                log,
+                "cannot allocate acme issuer \"{}\"",
+                issuer.name,
+            );
+            return Status::NGX_ERROR.into();
+        }
+    }
+
+    // Will be freed when the zone is unmapped.
+    let data = Box::leak(data);
+
+    alloc.as_mut().data = ptr::from_mut(data).cast();
+
+    amcf.data = Some(data);
+    amcf.shm_zone = zone;
+
+    Status::NGX_OK.into()
+}

src/state/certificate.rs

@@ -0,0 +1,150 @@
+use ngx::allocator::{AllocError, Allocator, TryCloneIn};
+use ngx::collections::Vec;
+use ngx::core::SlabPool;
+use ngx::sync::RwLock;
+
+use crate::time::{jitter, Time, TimeRange};
+
+pub type SharedCertificateContext = RwLock<CertificateContextInner<SlabPool>>;
+
+#[derive(Debug, Default)]
+pub enum CertificateContext {
+    #[default]
+    Empty,
+    // Ready to use certificate in shared memory.
+    Shared(&'static SharedCertificateContext),
+}
+
+impl CertificateContext {
+    pub fn as_ref(&self) -> Option<&'static SharedCertificateContext> {
+        if let CertificateContext::Shared(data) = self {
+            Some(data)
+        } else {
+            None
+        }
+    }
+}
+
+#[derive(Debug, Default, PartialEq, Eq)]
+pub enum CertificateState {
+    #[default]
+    Pending,
+    Ready,
+}
+
+#[derive(Debug)]
+pub struct CertificateContextInner<A>
+where
+    A: Allocator + Clone,
+{
+    pub state: CertificateState,
+    pub chain: Vec<u8, A>,
+    pub pkey: Vec<u8, A>,
+    pub valid: TimeRange,
+    pub next: Time,
+}
+
+impl<OA> TryCloneIn for CertificateContextInner<OA>
+where
+    OA: Allocator + Clone,
+{
+    type Target<A: Allocator + Clone> = CertificateContextInner<A>;
+
+    fn try_clone_in<A: Allocator + Clone>(&self, alloc: A) -> Result<Self::Target<A>, AllocError> {
+        let mut chain = Vec::new_in(alloc.clone());
+        chain
+            .try_reserve_exact(self.chain.len())
+            .map_err(|_| AllocError)?;
+        chain.extend(self.chain.iter());
+
+        let mut pkey = Vec::new_in(alloc);
+        pkey.try_reserve_exact(self.pkey.len())
+            .map_err(|_| AllocError)?;
+        pkey.extend(self.pkey.iter());
+
+        Ok(Self::Target {
+            state: CertificateState::Ready,
+            chain,
+            pkey,
+            valid: self.valid.clone(),
+            next: self.next,
+        })
+    }
+}
+
+impl<A> CertificateContextInner<A>
+where
+    A: Allocator + Clone,
+{
+    pub fn new_in(alloc: A) -> Self {
+        Self {
+            state: CertificateState::Pending,
+            chain: Vec::new_in(alloc.clone()),
+            pkey: Vec::new_in(alloc.clone()),
+            valid: Default::default(),
+            next: Default::default(),
+        }
+    }
+
+    pub fn set(&mut self, chain: &[u8], pkey: &[u8], valid: TimeRange) -> Result<Time, AllocError> {
+        const PREFIX: &[u8] = b"data:";
+
+        // reallocate the storage only if the current capacity is insufficient
+
+        fn needs_realloc<A: Allocator>(buf: &Vec<u8, A>, new_size: usize) -> bool {
+            buf.capacity() < PREFIX.len() + new_size
+        }
+
+        if needs_realloc(&self.chain, chain.len()) || needs_realloc(&self.pkey, pkey.len()) {
+            let alloc = self.chain.allocator();
+
+            let mut new_chain: Vec<u8, A> = Vec::new_in(alloc.clone());
+            new_chain
+                .try_reserve_exact(PREFIX.len() + chain.len())
+                .map_err(|_| AllocError)?;
+
+            let mut new_pkey: Vec<u8, A> = Vec::new_in(alloc.clone());
+            new_pkey
+                .try_reserve_exact(PREFIX.len() + pkey.len())
+                .map_err(|_| AllocError)?;
+
+            self.chain = new_chain;
+            self.pkey = new_pkey;
+        }
+
+        // update the stored data in-place
+
+        self.chain.clear();
+        self.chain.extend(PREFIX);
+        self.chain.extend(chain);
+
+        self.pkey.clear();
+        self.pkey.extend(PREFIX);
+        self.pkey.extend(pkey);
+
+        // Schedule the next update at around 2/3 of the cert lifetime,
+        // as recommended in Let's Encrypt integration guide
+        self.next = valid.start + jitter(valid.duration() * 2 / 3, 2);
+        self.valid = valid;
+
+        self.state = CertificateState::Ready;
+
+        Ok(self.next)
+    }
+
+    pub fn chain(&self) -> Option<&[u8]> {
+        if matches!(self.state, CertificateState::Ready) {
+            return Some(&self.chain);
+        }
+
+        None
+    }
+
+    pub fn pkey(&self) -> Option<&[u8]> {
+        if matches!(self.state, CertificateState::Ready) {
+            return Some(&self.pkey);
+        }
+
+        None
+    }
+}