rofl-containers: Add support for persistent storage

Author: kostko

Cargo.lock

@@ -1,7 +1,7 @@
 //! REST API daemon accessible by ROFL apps.
 
 mod routes;
-pub(crate) mod services;
+pub mod services;
 pub(crate) mod state;
 
 use std::sync::Arc;
@@ -10,26 +10,29 @@ use rocket::{figment::Figment, routes};
 
 use oasis_runtime_sdk::modules::rofl::app::{App, Environment};
 
+/// API server configuration.
+#[derive(Clone)]
+pub struct Config<'a> {
+    /// Address where the service should listen on.
+    pub address: &'a str,
+    /// Key management service to use.
+    pub kms: Arc<dyn services::kms::KmsService>,
+}
+
 /// Start the REST API server.
-pub async fn start<A>(address: &str, env: Environment<A>) -> Result<(), rocket::Error>
+pub async fn start<A>(cfg: Config<'_>, env: Environment<A>) -> Result<(), rocket::Error>
 where
     A: App,
 {
-    // KMS service.
-    let kms_service: Arc<dyn services::kms::KmsService> =
-        Arc::new(services::kms::OasisKmsService::new(env.clone()));
-    let kms_service_task = kms_service.clone();
-    tokio::spawn(async move { kms_service_task.start().await });
-
     // Oasis runtime environment.
     let env: Arc<dyn state::Env> = Arc::new(state::EnvImpl::new(env));
 
     // Server configuration.
-    let cfg = Figment::new().join(("address", address));
+    let rocket_cfg = Figment::new().join(("address", cfg.address));
 
-    rocket::custom(cfg)
+    rocket::custom(rocket_cfg)
         .manage(env)
-        .manage(kms_service)
+        .manage(cfg.kms)
         .mount("/rofl/v1/app", routes![routes::app::id,])
         .mount("/rofl/v1/keys", routes![routes::keys::generate,])
         .launch()

rofl-appd/src/lib.rs

@@ -4,6 +4,7 @@ use std::sync::{
 };
 
 use sp800_185::KMac;
+use tokio::sync::Notify;
 
 use oasis_runtime_sdk::{
     core::common::logger::get_logger,
@@ -17,6 +18,9 @@ pub trait KmsService: Send + Sync {
     /// Start the KMS service.
     async fn start(&self) -> Result<(), Error>;
 
+    /// Waits for the service to become ready to accept requests.
+    async fn wait_ready(&self) -> Result<(), Error>;
+
     /// Generate a key based on the passed parameters.
     async fn generate(&self, request: &GenerateRequest<'_>) -> Result<GenerateResponse, Error>;
 }
@@ -94,6 +98,7 @@ pub struct OasisKmsService<A: App> {
     root_key: Arc<Mutex<Option<Vec<u8>>>>,
     env: Environment<A>,
     logger: slog::Logger,
+    ready_notify: Notify,
 }
 
 impl<A: App> OasisKmsService<A> {
@@ -103,6 +108,7 @@ impl<A: App> OasisKmsService<A> {
             root_key: Arc::new(Mutex::new(None)),
             env,
             logger: get_logger("appd/services/kms"),
+            ready_notify: Notify::new(),
         }
     }
 }
@@ -146,11 +152,25 @@ impl<A: App> KmsService for OasisKmsService<A> {
         // Store the key in memory.
         *self.root_key.lock().unwrap() = Some(root_key.key);
 
+        self.ready_notify.notify_waiters();
+
         slog::info!(self.logger, "KMS service initialized");
 
         Ok(())
     }
 
+    async fn wait_ready(&self) -> Result<(), Error> {
+        let handle = self.ready_notify.notified();
+
+        if self.root_key.lock().unwrap().is_some() {
+            return Ok(());
+        }
+
+        handle.await;
+
+        Ok(())
+    }
+
     async fn generate(&self, request: &GenerateRequest<'_>) -> Result<GenerateResponse, Error> {
         let root_key_guard = self.root_key.lock().unwrap();
         let root_key = root_key_guard.as_ref().ok_or(Error::NotInitialized)?;
@@ -173,6 +193,10 @@ impl KmsService for MockKmsService {
         Ok(())
     }
 
+    async fn wait_ready(&self) -> Result<(), Error> {
+        Ok(())
+    }
+
     async fn generate(&self, request: &GenerateRequest<'_>) -> Result<GenerateResponse, Error> {
         let key = Kdf::derive_key(
             INSECURE_MOCK_ROOT_KEY,

rofl-appd/src/services/kms.rs

@@ -1,6 +1,6 @@
 [package]
 name = "rofl-containers"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2021"
 
 [dependencies]
@@ -9,4 +9,9 @@ oasis-runtime-sdk = { path = "../runtime-sdk", features = ["tdx"] }
 rofl-appd = { path = "../rofl-appd" }
 
 # Third party.
+anyhow = "1.0.86"
 base64 = "0.22.1"
+cmd_lib = "1.9.5"
+hex = "0.4.3"
+nix = { version = "0.29.0", features = ["signal"] }
+tokio = { version = "1.38", features = ["rt", "rt-multi-thread", "sync", "time", "macros"] }

rofl-containers/Cargo.toml

@@ -11,7 +11,15 @@
 use std::env;
 
 use base64::prelude::*;
-use oasis_runtime_sdk::{cbor, modules::rofl::app::prelude::*};
+use oasis_runtime_sdk::{
+    cbor,
+    core::common::{logger::get_logger, process},
+    modules::rofl::app::prelude::*,
+};
+use rofl_appd::services;
+
+mod reaper;
+mod storage;
 
 /// UNIX socket address where the REST API server will listen on.
 const ROFL_APPD_ADDRESS: &str = "unix:/run/rofl-appd.sock";
@@ -42,12 +50,39 @@ impl App for ContainersApp {
         .expect("Corrupted ROFL_CONSENSUS_TRUST_ROOT (must be Base64-encoded CBOR).")
     }
 
-    async fn run(self: Arc<Self>, env: Environment<Self>) {
+    async fn post_registration_init(self: Arc<Self>, env: Environment<Self>) {
+        // Temporarily disable the default process reaper as it interferes with scripts.
+        let _guard = reaper::disable_default_reaper();
+        let logger = get_logger("post_registration_init");
+
+        // Start the key management service and wait for it to initialize.
+        let kms: Arc<dyn services::kms::KmsService> =
+            Arc::new(services::kms::OasisKmsService::new(env.clone()));
+        let kms_task = kms.clone();
+        tokio::spawn(async move { kms_task.start().await });
+        let _ = kms.wait_ready().await;
+
+        // Initialize storage when configured in the kernel cmdline.
+        if let Err(err) = storage::init(kms.clone()).await {
+            slog::error!(logger, "failed to initialize stage 2 storage"; "err" => ?err);
+            process::abort();
+        }
+
         // Start the REST API server.
-        let _ = rofl_appd::start(ROFL_APPD_ADDRESS, env).await;
+        let cfg = rofl_appd::Config {
+            address: ROFL_APPD_ADDRESS,
+            kms,
+        };
+        let _ = rofl_appd::start(cfg, env).await;
     }
 }
 
 fn main() {
+    // Configure the binary search path.
+    // SAFETY: This is safe as no other threads are running yet.
+    unsafe {
+        env::set_var("PATH", "/usr/sbin:/usr/bin:/sbin:/bin");
+    }
+
     ContainersApp.start();
 }

rofl-containers/src/main.rs

@@ -0,0 +1,33 @@
+use nix::sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal};
+
+/// Guard that re-enables the default process reaper when dropped.
+pub struct DisableReaperGuard {
+    _internal: (),
+}
+
+impl Drop for DisableReaperGuard {
+    fn drop(&mut self) {
+        // Re-enable default kernel process reaper.
+        unsafe {
+            let _ = sigaction(
+                Signal::SIGCHLD,
+                &SigAction::new(SigHandler::SigIgn, SaFlags::empty(), SigSet::empty()),
+            );
+        }
+    }
+}
+
+/// Temporarily disables the default process reaper. When the returned guard gets out of scope, the
+/// default reaper is re-enabled.
+///
+/// This assumes that the default reaper has been previously configured by core init.
+pub fn disable_default_reaper() -> DisableReaperGuard {
+    unsafe {
+        let _ = sigaction(
+            Signal::SIGCHLD,
+            &SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::empty()),
+        );
+    }
+
+    DisableReaperGuard { _internal: () }
+}