Merge pull request #3902 from jarhodes314/feat/mapper-config-migration-tool

feat: Support automated config migration for separate mapper config

Author: jarhodes314

Cargo.lock

@@ -21,6 +21,7 @@ clap_complete = { workspace = true }
 doku = { workspace = true }
 figment = { workspace = true, features = ["env", "toml"] }
 futures = { workspace = true }
+glob = { workspace = true }
 humantime = { workspace = true }
 mqtt_channel = { workspace = true }
 once_cell = { workspace = true }

crates/common/tedge_config/Cargo.toml

@@ -17,7 +17,9 @@ pub use tedge_toml::tedge_config_location::*;
 pub use camino::Utf8Path as Path;
 pub use camino::Utf8PathBuf as PathBuf;
 pub use certificate::CertificateError;
+use models::CloudType;
 use std::path::Path as StdPath;
+use strum::IntoEnumIterator;
 pub use tedge_config_macros::all_or_nothing;
 pub use tedge_config_macros::OptionalConfig;
 
@@ -27,13 +29,36 @@ impl TEdgeConfig {
         config_location.load().await
     }
 
+    /// Load [TEdgeConfig], using a separate mapper config file as the default
+    /// behaviour if no clouds are already configured
+    ///
+    /// As of 2026-01-05, this is only used for testing how the new default will
+    /// work before we fully adopt the new format. When we do this, we should
+    /// probably also include `tedge config upgrade` commands in the
+    /// relevant package postinstall scripts.
+    #[cfg(feature = "test")]
+    pub async fn load_prefer_separate_mapper_config(
+        config_dir: impl AsRef<StdPath>,
+    ) -> Result<Self, TEdgeConfigError> {
+        let mut config_location = TEdgeConfigLocation::from_custom_root(config_dir.as_ref());
+        config_location.default_to_mapper_config_dir();
+        config_location.load().await
+    }
+
     pub async fn update_toml(
         self,
         update: &impl Fn(&mut TEdgeConfigDto, &TEdgeConfigReader) -> ConfigSettingResult<()>,
     ) -> Result<(), TEdgeConfigError> {
         self.location().update_toml(update).await
     }
 
+    pub async fn migrate_mapper_configs(self) -> Result<(), TEdgeConfigError> {
+        for cloud_type in CloudType::iter() {
+            self.location().migrate_mapper_config(cloud_type).await?;
+        }
+        Ok(())
+    }
+
     #[cfg(feature = "test")]
     /// A test only method designed for injecting configuration into tests
     ///

crates/common/tedge_config/src/lib.rs

@@ -55,6 +55,7 @@ pub use topic_prefix::TopicPrefix;
     serde::Serialize,
     serde::Deserialize,
     strum::EnumIter,
+    strum::AsRefStr,
 )]
 #[serde(rename_all = "lowercase")]
 #[strum(serialize_all = "lowercase")]

crates/common/tedge_config/src/tedge_toml/models/mod.rs

@@ -1,7 +1,6 @@
 mod version;
 use futures::Stream;
 use reqwest::NoProxy;
-use serde::Deserialize;
 use version::TEdgeTomlVersion;
 
 mod append_remove;
@@ -62,7 +61,6 @@ use std::net::Ipv4Addr;
 use std::num::NonZeroU16;
 use std::path::PathBuf;
 use std::sync::Arc;
-use strum::IntoEnumIterator;
 use tedge_api::mqtt_topics::EntityTopicId;
 pub use tedge_config_macros::ConfigNotSet;
 pub use tedge_config_macros::MultiError;
@@ -104,16 +102,18 @@ impl std::ops::Deref for TEdgeConfig {
     }
 }
 
-async fn read_file_if_exists(path: &Utf8Path) -> anyhow::Result<Option<String>> {
+async fn read_file_if_exists(
+    path: &Utf8Path,
+    config_dir: &Utf8Path,
+) -> anyhow::Result<Option<String>> {
     match tokio::fs::read_to_string(path).await {
         Ok(contents) => Ok(Some(contents)),
         Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
         Err(e) => {
-            let dir = path.parent().unwrap();
             // If the error is actually with the mappers directory as a whole,
             // feed that back to the user
-            if let Err(dir_error) = tokio::fs::read_dir(dir).await {
-                Err(dir_error).context(format!("failed to read {dir}"))
+            if let Err(dir_error) = tokio::fs::read_dir(config_dir).await {
+                Err(dir_error).context(format!("failed to read {config_dir}"))
             } else {
                 Err(e).context(format!("failed to read mapper configuration from {path}"))
             }
@@ -132,34 +132,41 @@ impl TEdgeConfigDto {
         use futures::StreamExt;
         use futures::TryStreamExt;
 
-        let mappers_dir = location.tedge_config_root_path().join("mappers");
+        let mappers_dir = location.mappers_config_dir();
         let all_profiles = location.mapper_config_profiles::<T>().await;
         let ty = T::expected_cloud_type();
         if let Some(profiles) = all_profiles {
+            let config_paths = location.config_path::<T>();
+            let default_profile_path = config_paths.path_for(None::<&ProfileName>);
+
             if !dto.is_default() {
-                tracing::warn!("{ty} configuration found in `tedge.toml`, but this will be ignored in favour of configuration in {mappers_dir}/{ty}.toml and {mappers_dir}/{ty}.d")
+                let wildcard_profile_path = config_paths.path_for(Some("*"));
+                tracing::warn!("{ty} configuration found in `tedge.toml`, but this will be ignored in favour of configuration in {default_profile_path} and {wildcard_profile_path}")
             }
-            let toml_path = mappers_dir.join(format!("{ty}.toml"));
-            let default_profile_toml = read_file_if_exists(&toml_path).await?;
+
+            let default_profile_toml =
+                read_file_if_exists(&default_profile_path, &config_paths.base_dir).await?;
             let mut default_profile_config: T::CloudDto = default_profile_toml.map_or_else(
                 || Ok(<_>::default()),
                 |toml| {
                     toml::from_str(&toml).with_context(|| {
-                        format!("failed to deserialise mapper config in {toml_path}")
+                        format!("failed to deserialise mapper config in {default_profile_path}")
                     })
                 },
             )?;
-            default_profile_config.set_mapper_config_dir(mappers_dir.clone());
+            default_profile_config.set_mappers_root_dir(mappers_dir.clone());
+            default_profile_config.set_mapper_config_file(default_profile_path);
             dto.non_profile = default_profile_config;
 
-            dto.profiles = profiles
+            dto.profiles = futures::stream::iter(profiles)
                 .filter_map(futures::future::ready)
                 .then(|profile| async {
-                    let toml_path = mappers_dir.join(format!("{ty}.d/{profile}.toml"));
+                    let toml_path = config_paths.path_for(Some(&profile));
                     let profile_toml = tokio::fs::read_to_string(&toml_path).await?;
                     let mut profiled_config: T::CloudDto = toml::from_str(&profile_toml)
                         .context("failed to deserialise mapper config")?;
-                    profiled_config.set_mapper_config_dir(mappers_dir.clone());
+                    profiled_config.set_mappers_root_dir(mappers_dir.clone());
+                    profiled_config.set_mapper_config_file(toml_path);
                     Ok::<_, anyhow::Error>((profile, profiled_config))
                 })
                 .try_collect()
@@ -227,10 +234,6 @@ impl TEdgeConfig {
         )?)
     }
 
-    pub fn profiled_config_directories(&self) -> impl Iterator<Item = Utf8PathBuf> + use<'_> {
-        CloudType::iter().map(|ty| self.location.mappers_config_dir().join(format!("{ty}.d")))
-    }
-
     fn all_profiles<'a, T>(&'a self) -> Box<dyn Iterator<Item = Option<ProfileName>> + 'a>
     where
         T: ExpectedCloudType,
@@ -421,6 +424,12 @@ impl CloudConfig for DynCloudConfig<'_> {
             Self::Borrow(config) => config.key_pin(),
         }
     }
+    fn mapper_config_location(&self) -> &Utf8Path {
+        match self {
+            Self::Arc(config) => config.mapper_config_location(),
+            Self::Borrow(config) => config.mapper_config_location(),
+        }
+    }
 }
 
 /// The keys that can be read from the configuration
@@ -439,12 +448,6 @@ pub static READABLE_KEYS: Lazy<Vec<(Cow<'static, str>, doku::Type)>> = Lazy::new
     struct_field_paths(None, &fields)
 });
 
-#[derive(Debug, Clone, PartialEq, Eq, Deserialize, doku::Document, serde::Serialize)]
-pub enum MapperConfigLocation {
-    TedgeToml,
-    SeparateFile(#[doku(as = "String")] camino::Utf8PathBuf),
-}
-
 define_tedge_config! {
     #[tedge_config(reader(skip))]
     config: {
@@ -576,6 +579,10 @@ define_tedge_config! {
         #[serde(skip)]
         mapper_config_dir: Utf8PathBuf,
 
+        #[tedge_config(reader(skip))]
+        #[serde(skip)]
+        mapper_config_file: Utf8PathBuf,
+
         /// Endpoint URL of Cumulocity tenant
         #[tedge_config(example = "your-tenant.cumulocity.com")]
         // Config consumers should use `c8y.http`/`c8y.mqtt` as appropriate, hence this field is private
@@ -820,6 +827,10 @@ define_tedge_config! {
         #[serde(skip)]
         mapper_config_dir: Utf8PathBuf,
 
+        #[tedge_config(reader(skip))]
+        #[serde(skip)]
+        mapper_config_file: Utf8PathBuf,
+
         /// Endpoint URL of Azure IoT tenant
         #[tedge_config(example = "myazure.azure-devices.net")]
         url: ConnectUrl,
@@ -911,6 +922,10 @@ define_tedge_config! {
         #[serde(skip)]
         mapper_config_dir: Utf8PathBuf,
 
+        #[tedge_config(reader(skip))]
+        #[serde(skip)]
+        mapper_config_file: Utf8PathBuf,
+
         /// Endpoint URL of AWS IoT tenant
         #[tedge_config(example = "your-endpoint.amazonaws.com")]
         url: ConnectUrl,
@@ -1411,6 +1426,7 @@ pub trait CloudConfig {
     fn root_cert_path(&self) -> &Utf8Path;
     fn key_uri(&self) -> Option<Arc<str>>;
     fn key_pin(&self) -> Option<Arc<str>>;
+    fn mapper_config_location(&self) -> &Utf8Path;
 }
 
 impl<T: SpecialisedCloudConfig> CloudConfig for MapperConfig<T> {
@@ -1433,6 +1449,10 @@ impl<T: SpecialisedCloudConfig> CloudConfig for MapperConfig<T> {
     fn key_pin(&self) -> Option<Arc<str>> {
         self.device.key_pin.clone()
     }
+
+    fn mapper_config_location(&self) -> &Utf8Path {
+        &self.location
+    }
 }
 
 fn c8y_topic_prefix() -> TopicPrefix {
@@ -1816,7 +1836,8 @@ mod tests {
     async fn mapper_config_reads_non_profiled_mapper() {
         let ttd = TempTedgeDir::new();
         ttd.dir("mappers")
-            .file("c8y.toml")
+            .dir("c8y")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "example.com"
 
@@ -1838,8 +1859,8 @@ mod tests {
     async fn mapper_config_reads_profiled_mapper() {
         let ttd = TempTedgeDir::new();
         ttd.dir("mappers")
-            .dir("c8y.d")
-            .file("myprofile.toml")
+            .dir("c8y.myprofile")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "example.com"
 
@@ -1861,8 +1882,8 @@ mod tests {
     async fn mapper_config_fails_if_profile_is_not_migrated_but_directory_exists() {
         let ttd = TempTedgeDir::new();
         ttd.dir("mappers")
-            .dir("c8y.d")
-            .file("myprofile.toml")
+            .dir("c8y.myprofile")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "example.com"
 
@@ -1882,7 +1903,8 @@ mod tests {
     async fn mapper_config_fails_if_profile_is_not_migrated_but_default_profile_is() {
         let ttd = TempTedgeDir::new();
         ttd.dir("mappers")
-            .file("c8y.toml")
+            .dir("c8y")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "example.com"
 
@@ -1936,7 +1958,7 @@ mod tests {
     async fn mapper_config_falls_back_to_tedge_toml_config_if_only_another_cloud_is_using_new_format(
     ) {
         let ttd = TempTedgeDir::new();
-        ttd.dir("mappers").file("c8y.toml");
+        ttd.dir("mappers").dir("c8y").file("tedge.toml");
         ttd.file("tedge.toml").with_toml_content(toml::toml! {
             az.url = "az.url"
         });
@@ -1987,7 +2009,8 @@ mod tests {
     async fn all_mapper_configs_succeeds_when_profile_directory_does_not_exist() {
         let ttd = TempTedgeDir::new();
         ttd.dir("mappers")
-            .file("c8y.toml")
+            .dir("c8y")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "example.com"
             });
@@ -2007,18 +2030,21 @@ mod tests {
     async fn all_mapper_configs_includes_both_default_and_profiles_when_directory_exists() {
         let ttd = TempTedgeDir::new();
         let mappers_dir = ttd.dir("mappers");
-        mappers_dir.file("c8y.toml").with_toml_content(toml::toml! {
-            url = "default.example.com"
-        });
         mappers_dir
-            .dir("c8y.d")
-            .file("profile1.toml")
+            .dir("c8y")
+            .file("tedge.toml")
+            .with_toml_content(toml::toml! {
+                url = "default.example.com"
+            });
+        mappers_dir
+            .dir("c8y.profile1")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "profile1.example.com"
             });
         mappers_dir
-            .dir("c8y.d")
-            .file("profile2.toml")
+            .dir("c8y.profile2")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "profile2.example.com"
             });
@@ -2049,13 +2075,15 @@ mod tests {
         let ttd = TempTedgeDir::new();
         // C8y mapper with no profile directory
         ttd.dir("mappers")
-            .file("c8y.toml")
+            .dir("c8y")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "c8y.example.com"
             });
         // Az mapper with no profile directory
         ttd.dir("mappers")
-            .file("az.toml")
+            .dir("az")
+            .file("tedge.toml")
             .with_toml_content(toml::toml! {
                 url = "az.example.com"
             });