feat: extensible log management

Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "plugins/c8y_firmware_plugin",
     "plugins/c8y_remote_access_plugin",
     "plugins/tedge_apt_plugin",
+    "plugins/tedge_file_log_plugin",
 ]
 resolver = "2"
 
@@ -43,6 +44,7 @@ mqtt_tests = { path = "crates/tests/mqtt_tests" }
 plugin_sm = { path = "crates/core/plugin_sm" }
 tedge-agent = { path = "crates/core/tedge_agent" }
 tedge-apt-plugin = { path = "plugins/tedge_apt_plugin" }
+tedge-file-log-plugin = { path = "plugins/tedge_file_log_plugin" }
 tedge-mapper = { path = "crates/core/tedge_mapper", default-features = false }
 tedge-p11-server = { path = "crates/extensions/tedge-p11-server" }
 tedge-watchdog = { path = "crates/core/tedge_watchdog" }

configuration/package_scripts/_generated/tedge/apk/preinst

@@ -56,6 +56,7 @@ fi
 if [ -d /etc/sudoers.d ]; then
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge, /etc/tedge/sm-plugins/[a-zA-Z0-9]*, /bin/sync, /sbin/init" > /etc/sudoers.d/tedge
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge-write /etc/*" >> /etc/sudoers.d/tedge
+    echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/share/tedge/log-plugins/[a-zA-Z0-9]*" >> /etc/sudoers.d/tedge
 fi
 
 

configuration/package_scripts/_generated/tedge/deb/preinst

@@ -56,6 +56,7 @@ fi
 if [ -d /etc/sudoers.d ]; then
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge, /etc/tedge/sm-plugins/[a-zA-Z0-9]*, /bin/sync, /sbin/init" > /etc/sudoers.d/tedge
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge-write /etc/*" >> /etc/sudoers.d/tedge
+    echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/share/tedge/log-plugins/[a-zA-Z0-9]*" >> /etc/sudoers.d/tedge
 fi
 
 

configuration/package_scripts/_generated/tedge/rpm/preinst

@@ -56,6 +56,7 @@ fi
 if [ -d /etc/sudoers.d ]; then
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge, /etc/tedge/sm-plugins/[a-zA-Z0-9]*, /bin/sync, /sbin/init" > /etc/sudoers.d/tedge
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge-write /etc/*" >> /etc/sudoers.d/tedge
+    echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/share/tedge/log-plugins/[a-zA-Z0-9]*" >> /etc/sudoers.d/tedge
 fi
 
 

configuration/package_scripts/tedge/preinst

@@ -56,6 +56,7 @@ fi
 if [ -d /etc/sudoers.d ]; then
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge, /etc/tedge/sm-plugins/[a-zA-Z0-9]*, /bin/sync, /sbin/init" > /etc/sudoers.d/tedge
     echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/bin/tedge-write /etc/*" >> /etc/sudoers.d/tedge
+    echo "tedge    ALL = (ALL) NOPASSWD:SETENV: /usr/share/tedge/log-plugins/[a-zA-Z0-9]*" >> /etc/sudoers.d/tedge
 fi
 
 

crates/common/tedge_config/src/tedge_toml/tedge_config.rs

@@ -936,7 +936,13 @@ define_tedge_config! {
         /// The directories where diagnostic plugins are stored
         #[tedge_config(example = "/usr/share/diag-plugins,/etc/tedge/diag-plugins", default(value = "/usr/share/tedge/diag-plugins"))]
         plugin_paths: TemplatesSet,
-    }
+    },
+
+    log: {
+        /// The directories where log plugins are stored
+        #[tedge_config(example = "/usr/share/log-plugins,/etc/tedge/log-plugins", default(value = "/usr/share/tedge/log-plugins"))]
+        plugin_paths: TemplatesSet,
+    },
 }
 
 static CLOUD_ROOT_CERTIFICATES: OnceLock<Arc<[Certificate]>> = OnceLock::new();

crates/core/tedge/Cargo.toml

@@ -49,6 +49,7 @@ strum_macros = { workspace = true }
 tar = { workspace = true }
 tedge-agent = { workspace = true }
 tedge-apt-plugin = { workspace = true }
+tedge-file-log-plugin = { workspace = true }
 tedge-mapper = { workspace = true, default-features = false }
 tedge-watchdog = { workspace = true }
 tedge-write = { workspace = true }

crates/core/tedge/src/cli/init.rs

@@ -95,6 +95,14 @@ impl TEdgeInitCmd {
         create_directory(&config.logs.path, &permissions).await?;
         create_directory(&config.data.path, &permissions).await?;
 
+        for log_plugins_dir in &config.log.plugin_paths.0 {
+            create_directory(&log_plugins_dir, &permissions).await?;
+        }
+        // The last directory of the log plugin path list is the location for thin-edge provided plugins
+        if let Some(main_log_plugins_dir) = config.log.plugin_paths.0.last() {
+            create_symlinks_for("file", target, Path::new(main_log_plugins_dir), &RealEnv).await?;
+        }
+
         let entity_store_file = config_dir.join(".agent").join("entity_store.jsonl");
 
         if entity_store_file.exists() {

crates/core/tedge/src/cli/mod.rs

@@ -10,6 +10,7 @@ use tedge_agent::AgentOpt;
 use tedge_apt_plugin::AptCli;
 use tedge_config::cli::CommonArgs;
 use tedge_config::TEdgeConfig;
+use tedge_file_log_plugin::bin::FileLogCli;
 use tedge_mapper::MapperOpt;
 use tedge_watchdog::WatchdogOpt;
 use tedge_write::bin::Args as TedgeWriteOpt;
@@ -70,6 +71,8 @@ pub enum Component {
     #[clap(alias = "apt")]
     TedgeAptPlugin(AptCli),
 
+    TedgeFileLogPlugin(FileLogCli),
+
     TedgeMapper(MapperOpt),
 
     TedgeWatchdog(WatchdogOpt),

crates/core/tedge/src/main.rs

@@ -24,6 +24,8 @@ use tedge_apt_plugin::AptCli;
 use tedge_config::cli::CommonArgs;
 use tedge_config::log_init;
 use tedge_config::unconfigured_logger;
+use tedge_file_log_plugin::bin::FileLogCli;
+use tedge_file_log_plugin::bin::TEdgeConfigView;
 use tracing::log;
 
 #[global_allocator]
@@ -68,6 +70,13 @@ async fn main() -> anyhow::Result<()> {
                 .await
                 .context("failed to run tedge apt plugin")?
         }
+        TEdgeOptMulticall::Component(Component::TedgeFileLogPlugin(opt)) => {
+            let tedge_config = tedge_config::TEdgeConfig::load(&opt.common.config_dir).await?;
+            let plugin_config = TEdgeConfigView::new(tedge_config.tmp.path.as_path());
+            tokio::task::spawn_blocking(move || tedge_file_log_plugin::bin::run(opt, plugin_config))
+                .await
+                .context("failed to run tedge file log plugin")?
+        }
         TEdgeOptMulticall::Tedge(TEdgeCli { cmd, common }) => {
             log_init(
                 "tedge",
@@ -133,7 +142,22 @@ where
         match AptCli::try_parse() {
             Ok(apt) => return TEdgeOptMulticall::Component(Component::TedgeAptPlugin(apt)),
             Err(e) => {
-                eprintln!("{}", RichFormatter::format_error(&e));
+                eprintln!("{e}");
+                std::process::exit(1);
+            }
+        }
+    }
+
+    if matches!(
+        executable_name.as_deref(),
+        Some("file" | "tedge-file-log-plugin")
+    ) {
+        // the file log plugin must be treated apart
+        // as we want to exit 1 and not 2 when the command line cannot be parsed
+        match FileLogCli::try_parse() {
+            Ok(cli) => return TEdgeOptMulticall::Component(Component::TedgeFileLogPlugin(cli)),
+            Err(e) => {
+                eprintln!("{e}");
                 std::process::exit(1);
             }
         }

crates/core/tedge_agent/src/agent.rs

@@ -30,6 +30,7 @@ use reqwest::Identity;
 use std::fmt::Debug;
 use std::net::SocketAddr;
 use std::sync::Arc;
+use std::vec;
 use tedge_actors::Concurrent;
 use tedge_actors::ConvertingActor;
 use tedge_actors::ConvertingActorBuilder;
@@ -97,6 +98,7 @@ pub(crate) struct AgentConfig {
     pub fts_url: Arc<str>,
     pub is_sudo_enabled: bool,
     pub capabilities: Capabilities,
+    pub log_plugin_dirs: Vec<Utf8PathBuf>,
     entity_auto_register: bool,
     entity_store_clean_start: bool,
 }
@@ -186,6 +188,13 @@ impl AgentConfig {
 
         let entity_auto_register = tedge_config.agent.entity_store.auto_register;
         let entity_store_clean_start = tedge_config.agent.entity_store.clean_start;
+        let log_plugin_dirs = tedge_config
+            .log
+            .plugin_paths
+            .0
+            .iter()
+            .map(Utf8PathBuf::from)
+            .collect();
 
         Ok(Self {
             mqtt_config,
@@ -211,6 +220,7 @@ impl AgentConfig {
             is_sudo_enabled,
             service: tedge_config.service.clone(),
             capabilities,
+            log_plugin_dirs,
             entity_auto_register,
             entity_store_clean_start,
         })
@@ -361,10 +371,11 @@ impl Agent {
         let log_actor_builder = if self.config.capabilities.log_upload {
             let log_manager_config = LogManagerConfig::from_options(LogManagerOptions {
                 config_dir: self.config.config_dir.clone().into(),
-                tmp_dir: self.config.tmp_dir.to_path_buf().into(),
+                tmp_dir: self.config.tmp_dir.clone(),
                 log_dir: self.config.log_dir,
                 mqtt_schema: mqtt_schema.clone(),
                 mqtt_device_topic_id: device_topic_id.clone(),
+                plugin_dirs: self.config.log_plugin_dirs,
             })?;
             let mut log_actor = LogManagerBuilder::try_new(
                 log_manager_config,

crates/extensions/tedge_file_system_ext/src/lib.rs

@@ -91,6 +91,14 @@ impl MessageSource<FsWatchEvent, PathBuf> for FsWatchActorBuilder {
     }
 }
 
+impl MessageSource<FsWatchEvent, Vec<PathBuf>> for FsWatchActorBuilder {
+    fn connect_sink(&mut self, paths: Vec<PathBuf>, peer: &impl MessageSink<FsWatchEvent>) {
+        for path in paths {
+            self.connect_sink(path, peer);
+        }
+    }
+}
+
 impl RuntimeRequestSink for FsWatchActorBuilder {
     fn get_signal_sender(&self) -> DynSender<RuntimeRequest> {
         Box::new(self.signal_sender.clone())

crates/extensions/tedge_log_manager/Cargo.toml

@@ -10,14 +10,9 @@ homepage = { workspace = true }
 repository = { workspace = true }
 
 [dependencies]
+anyhow = { workspace = true }
 async-trait = { workspace = true }
 camino = { workspace = true }
-easy_reader = { workspace = true }
-glob = { workspace = true }
-log = { workspace = true }
-rand = { workspace = true }
-regex = { workspace = true }
-serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
 tedge_actors = { workspace = true }
 tedge_api = { workspace = true }
@@ -30,10 +25,9 @@ thiserror = { workspace = true }
 time = { workspace = true, features = ["formatting"] }
 tokio = { workspace = true, features = ["macros"] }
 toml = { workspace = true }
+tracing = { workspace = true }
 
 [dev-dependencies]
-anyhow = { workspace = true }
-filetime = { workspace = true }
 tedge_actors = { workspace = true, features = ["test-helpers"] }
 tedge_test_utils = { workspace = true }
 time = { workspace = true, features = ["macros"] }