refactor: separate out metadata loading stage

Author: Devdutt Shenoi

server/src/handlers/http/modal/ingest_server.rs

@@ -23,7 +23,6 @@ use super::IngestorMetadata;
 use super::OpenIdClient;
 use super::ParseableServer;
 use crate::analytics;
-use crate::banner;
 use crate::handlers::airplane;
 use crate::handlers::http::ingest;
 use crate::handlers::http::logstream;
@@ -34,9 +33,7 @@ use crate::localcache::LocalCacheManager;
 use crate::metrics;
 use crate::migration;
 use crate::migration::metadata_migration::migrate_ingester_metadata;
-use crate::rbac;
 use crate::rbac::role::Action;
-use crate::storage;
 use crate::storage::object_storage::ingestor_metadata_path;
 use crate::storage::object_storage::parseable_json_path;
 use crate::storage::staging;
@@ -84,8 +81,8 @@ impl ParseableServer for IngestServer {
             .service(Server::get_ingest_otel_factory());
     }
 
-    /// configure the server and start an instance to ingest data
-    async fn init(&self) -> anyhow::Result<()> {
+    async fn load_metadata(&self) -> anyhow::Result<Option<Bytes>> {
+        // parseable can't use local storage for persistence when running a distributed setup
         if CONFIG.get_storage_mode_string() == "Local drive" {
             return Err(anyhow::Error::msg(
                 // Error Message can be better
@@ -97,13 +94,12 @@ impl ParseableServer for IngestServer {
         let parseable_json = self.check_querier_state().await?;
         // to get the .parseable.json file in staging
         self.validate_credentials().await?;
-        let metadata = storage::resolve_parseable_metadata(&parseable_json).await?;
 
-        banner::print(&CONFIG, &metadata).await;
-        rbac::map::init(&metadata);
-        // set the info in the global metadata
-        metadata.set_global();
+        Ok(parseable_json)
+    }
 
+    /// configure the server and start an instance to ingest data
+    async fn init(&self) -> anyhow::Result<()> {
         // ! Undefined and Untested behaviour
         if let Some(cache_manager) = LocalCacheManager::global() {
             cache_manager

server/src/handlers/http/modal/mod.rs

@@ -33,6 +33,7 @@ use actix_web::HttpServer;
 use actix_web_prometheus::PrometheusMetrics;
 use async_trait::async_trait;
 use base64::Engine;
+use bytes::Bytes;
 use openid::Discovered;
 use serde::Deserialize;
 use serde::Serialize;
@@ -60,6 +61,12 @@ pub trait ParseableServer {
     where
         Self: Sized;
 
+    /// load metadata/configuration from persistence for previous sessions of parseable
+    async fn load_metadata(&self) -> anyhow::Result<Option<Bytes>>;
+
+    /// code that describes starting and setup procedures for each type of server
+    async fn init(&self) -> anyhow::Result<()>;
+
     /// configure the server
     async fn start(
         &self,
@@ -165,8 +172,6 @@ pub trait ParseableServer {
 
         Ok(())
     }
-
-    async fn init(&self) -> anyhow::Result<()>;
 }
 
 #[derive(Serialize, Debug, Deserialize, Default, Clone, Eq, PartialEq)]

server/src/handlers/http/modal/query_server.rs

@@ -28,10 +28,11 @@ use crate::rbac::role::Action;
 use crate::sync;
 use crate::users::dashboards::DASHBOARDS;
 use crate::users::filters::FILTERS;
-use crate::{analytics, banner, metrics, migration, rbac, storage};
+use crate::{analytics, metrics, migration, storage};
 use actix_web::web::{resource, ServiceConfig};
 use actix_web::{web, Scope};
 use async_trait::async_trait;
+use bytes::Bytes;
 
 use crate::option::CONFIG;
 
@@ -67,8 +68,8 @@ impl ParseableServer for QueryServer {
             .service(Server::get_generated());
     }
 
-    /// initialize the server, run migrations as needed and start an instance
-    async fn init(&self) -> anyhow::Result<()> {
+    async fn load_metadata(&self) -> anyhow::Result<Option<Bytes>> {
+        // parseable can't use local storage for persistence when running a distributed setup
         if CONFIG.get_storage_mode_string() == "Local drive" {
             return Err(anyhow::anyhow!(
                  "Query Server cannot be started in local storage mode. Please start the server in a supported storage mode.",
@@ -78,13 +79,12 @@ impl ParseableServer for QueryServer {
         migration::run_file_migration(&CONFIG).await?;
         let parseable_json = CONFIG.validate_storage().await?;
         migration::run_metadata_migration(&CONFIG, &parseable_json).await?;
-        let metadata = storage::resolve_parseable_metadata(&parseable_json).await?;
-        banner::print(&CONFIG, &metadata).await;
-        // initialize the rbac map
-        rbac::map::init(&metadata);
-        // keep metadata info in mem
-        metadata.set_global();
 
+        Ok(parseable_json)
+    }
+
+    /// initialize the server, run migrations as needed and start an instance
+    async fn init(&self) -> anyhow::Result<()> {
         let prometheus = metrics::build_metrics_handler();
         CONFIG.storage().register_store_metrics(&prometheus);
 

server/src/handlers/http/modal/server.rs

@@ -17,7 +17,6 @@
  */
 
 use crate::analytics;
-use crate::banner;
 use crate::handlers;
 use crate::handlers::http::about;
 use crate::handlers::http::base_path;
@@ -31,7 +30,6 @@ use crate::hottier::HotTierManager;
 use crate::localcache::LocalCacheManager;
 use crate::metrics;
 use crate::migration;
-use crate::rbac;
 use crate::storage;
 use crate::sync;
 use crate::users::dashboards::DASHBOARDS;
@@ -43,6 +41,7 @@ use actix_web::Resource;
 use actix_web::Scope;
 use actix_web_static_files::ResourceFiles;
 use async_trait::async_trait;
+use bytes::Bytes;
 
 use crate::{
     handlers::http::{
@@ -88,16 +87,16 @@ impl ParseableServer for Server {
             .service(Self::get_generated());
     }
 
-    /// configure the server and start an instance of the single server setup
-    async fn init(&self) -> anyhow::Result<()> {
+    async fn load_metadata(&self) -> anyhow::Result<Option<Bytes>> {
         migration::run_file_migration(&CONFIG).await?;
         let parseable_json = CONFIG.validate_storage().await?;
         migration::run_metadata_migration(&CONFIG, &parseable_json).await?;
-        let metadata = storage::resolve_parseable_metadata(&parseable_json).await?;
-        banner::print(&CONFIG, &metadata).await;
-        rbac::map::init(&metadata);
-        metadata.set_global();
 
+        Ok(parseable_json)
+    }
+
+    // configure the server and start an instance of the single server setup
+    async fn init(&self) -> anyhow::Result<()> {
         if let Some(cache_manager) = LocalCacheManager::global() {
             cache_manager
                 .validate(CONFIG.parseable.local_cache_size)

server/src/main.rs

@@ -63,5 +63,14 @@ async fn main() -> anyhow::Result<()> {
         Mode::All => Box::new(Server),
     };
 
+    // load metadata from persistence
+    let parseable_json = server.load_metadata().await?;
+    let metadata = storage::resolve_parseable_metadata(&parseable_json).await?;
+    banner::print(&CONFIG, &metadata).await;
+    // initialize the rbac map
+    rbac::map::init(&metadata);
+    // keep metadata info in mem
+    metadata.set_global();
+
     server.init().await
 }