Merge pull request #83 from sky1122/k8s-device-stradegy-to-list

add: change K8s device list strategy from a single value to a list

Author: ytsssun

Cargo.toml

@@ -57,7 +57,7 @@ bottlerocket-template-helper = { path = "./bottlerocket-template-helper", versio
 
 # Settings Models
 bottlerocket-model-derive = { path = "./bottlerocket-settings-models/model-derive", version = "0.1" }
-bottlerocket-modeled-types = { path = "./bottlerocket-settings-models/modeled-types", version = "0.8" }
+bottlerocket-modeled-types = { path = "./bottlerocket-settings-models/modeled-types", version = "0.9" }
 bottlerocket-scalar = { path = "./bottlerocket-settings-models/scalar", version = "0.1" }
 bottlerocket-scalar-derive = { path = "./bottlerocket-settings-models/scalar-derive", version = "0.1" }
 bottlerocket-string-impls-for = { path = "./bottlerocket-settings-models/string-impls-for", version = "0.1" }
@@ -75,7 +75,7 @@ settings-extension-ecs = { path = "./bottlerocket-settings-models/settings-exten
 settings-extension-host-containers = { path = "./bottlerocket-settings-models/settings-extensions/host-containers", version = "0.1" }
 settings-extension-kernel = { path = "./bottlerocket-settings-models/settings-extensions/kernel", version = "0.1" }
 settings-extension-kubernetes = { path = "./bottlerocket-settings-models/settings-extensions/kubernetes", version = "0.3" }
-settings-extension-kubelet-device-plugins = { path = "./bottlerocket-settings-models/settings-extensions/kubelet-device-plugins", version = "0.2" }
+settings-extension-kubelet-device-plugins = { path = "./bottlerocket-settings-models/settings-extensions/kubelet-device-plugins", version = "0.3" }
 settings-extension-metrics = { path = "./bottlerocket-settings-models/settings-extensions/metrics", version = "0.1" }
 settings-extension-motd = { path = "./bottlerocket-settings-models/settings-extensions/motd", version = "0.1" }
 settings-extension-network = { path = "./bottlerocket-settings-models/settings-extensions/network", version = "0.1" }

bottlerocket-settings-models/modeled-types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bottlerocket-modeled-types"
-version = "0.8.0"
+version = "0.9.0"
 authors = []
 license = "Apache-2.0 OR MIT"
 edition = "2021"

bottlerocket-settings-models/modeled-types/src/kubernetes.rs

@@ -1481,10 +1481,40 @@ pub enum NvidiaDeviceIdStrategy {
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
-#[serde(rename_all = "kebab-case")]
+#[serde(untagged)]
 pub enum NvidiaDeviceListStrategy {
+    Scalar(NvidiaDeviceListStrategyValues),
+    Vector(Vec<NvidiaDeviceListStrategyValues>),
+}
+
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum NvidiaDeviceListStrategyValues {
     Envvar,
     VolumeMounts,
+    CdiCri,
+}
+
+impl NvidiaDeviceListStrategy {
+    pub fn iter(&self) -> Box<dyn Iterator<Item = &NvidiaDeviceListStrategyValues> + '_> {
+        match self {
+            Self::Scalar(inner) => Box::new(std::iter::once(inner)),
+            Self::Vector(inner) => Box::new(inner.iter()),
+        }
+    }
+}
+
+impl IntoIterator for NvidiaDeviceListStrategy {
+    type Item = NvidiaDeviceListStrategyValues;
+    type IntoIter = std::vec::IntoIter<Self::Item>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            Self::Scalar(inner) => vec![inner],
+            Self::Vector(inner) => inner,
+        }
+        .into_iter()
+    }
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -1580,6 +1610,15 @@ string_impls_for!(MigProfile, "MigProfile");
 mod test_nvidia_device_plugins {
     use super::*;
 
+    fn helper_with_device_list_strategy(
+        device_list_strategy: Option<NvidiaDeviceListStrategy>,
+    ) -> NvidiaDevicePluginSettings {
+        NvidiaDevicePluginSettings {
+            device_list_strategy: device_list_strategy,
+            ..NvidiaDevicePluginSettings::default()
+        }
+    }
+
     #[test]
     fn valid_gpu_model() {
         for ok in &["a100.40gb", "a100.80gb", "h100.80gb", "h100.141gb"] {
@@ -1628,7 +1667,9 @@ mod test_nvidia_device_plugins {
             NvidiaDevicePluginSettings {
                 pass_device_specs: Some(false),
                 device_id_strategy: Some(NvidiaDeviceIdStrategy::Uuid),
-                device_list_strategy: Some(NvidiaDeviceListStrategy::Envvar),
+                device_list_strategy: Some(NvidiaDeviceListStrategy::Scalar(
+                    NvidiaDeviceListStrategyValues::Envvar
+                ),),
                 device_sharing_strategy: None,
                 time_slicing: None,
                 device_partitioning_strategy: None,
@@ -1649,7 +1690,9 @@ mod test_nvidia_device_plugins {
             NvidiaDevicePluginSettings {
                 pass_device_specs: Some(false),
                 device_id_strategy: Some(NvidiaDeviceIdStrategy::Uuid),
-                device_list_strategy: Some(NvidiaDeviceListStrategy::Envvar),
+                device_list_strategy: Some(NvidiaDeviceListStrategy::Scalar(
+                    NvidiaDeviceListStrategyValues::Envvar
+                ),),
                 device_sharing_strategy: Some(NvidiaDeviceSharingStrategy::TimeSlicing),
                 time_slicing: None,
                 device_partitioning_strategy: None,
@@ -1678,7 +1721,9 @@ mod test_nvidia_device_plugins {
             NvidiaDevicePluginSettings {
                 pass_device_specs: Some(false),
                 device_id_strategy: Some(NvidiaDeviceIdStrategy::Uuid),
-                device_list_strategy: Some(NvidiaDeviceListStrategy::Envvar),
+                device_list_strategy: Some(NvidiaDeviceListStrategy::Scalar(
+                    NvidiaDeviceListStrategyValues::Envvar
+                ),),
                 device_sharing_strategy: None,
                 time_slicing: None,
                 device_partitioning_strategy: Some(NvidiaDevicePartitioningStrategy::MIG),
@@ -1700,7 +1745,9 @@ mod test_nvidia_device_plugins {
             NvidiaDevicePluginSettings {
                 pass_device_specs: Some(false),
                 device_id_strategy: Some(NvidiaDeviceIdStrategy::Uuid),
-                device_list_strategy: Some(NvidiaDeviceListStrategy::Envvar),
+                device_list_strategy: Some(NvidiaDeviceListStrategy::Scalar(
+                    NvidiaDeviceListStrategyValues::Envvar
+                ),),
                 device_sharing_strategy: None,
                 time_slicing: None,
                 device_partitioning_strategy: Some(NvidiaDevicePartitioningStrategy::MIG),
@@ -1716,4 +1763,94 @@ mod test_nvidia_device_plugins {
         let results = serde_json::to_string(&nvidia_device_plugins).unwrap();
         assert_eq!(results, test_json);
     }
+
+    #[test]
+    fn test_serde_nvidia_device_plugins_with_list_shape_nvidia_list_stradegy() {
+        let test_json_1 = r#"{"device-list-strategy":["volume-mounts","envvar","cdi-cri"]}"#;
+        let test_json_2 = r#"{"device-list-strategy":["volume-mounts","cdi-cri","envvar"]}"#;
+        let test_json_3 = r#"{"device-list-strategy":["envvar","volume-mounts","cdi-cri"]}"#;
+        let test_json_4 = r#"{"device-list-strategy":["envvar","cdi-cri","volume-mounts"]}"#;
+        let test_json_5 = r#"{"device-list-strategy":["cdi-cri","volume-mounts","envvar"]}"#;
+        let test_json_6 = r#"{"device-list-strategy":["cdi-cri","envvar","volume-mounts"]}"#;
+
+        let device_plugins_1: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_1).unwrap();
+        assert_eq!(
+            device_plugins_1,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+                NvidiaDeviceListStrategyValues::Envvar,
+                NvidiaDeviceListStrategyValues::CdiCri,
+            ])))
+        );
+
+        let device_plugins_2: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_2).unwrap();
+        assert_eq!(
+            device_plugins_2,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+                NvidiaDeviceListStrategyValues::CdiCri,
+                NvidiaDeviceListStrategyValues::Envvar,
+            ]))),
+        );
+
+        let device_plugins_3: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_3).unwrap();
+        assert_eq!(
+            device_plugins_3,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::Envvar,
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+                NvidiaDeviceListStrategyValues::CdiCri,
+            ]))),
+        );
+
+        let device_plugins_4: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_4).unwrap();
+        assert_eq!(
+            device_plugins_4,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::Envvar,
+                NvidiaDeviceListStrategyValues::CdiCri,
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+            ]))),
+        );
+
+        let device_plugins_5: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_5).unwrap();
+        assert_eq!(
+            device_plugins_5,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::CdiCri,
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+                NvidiaDeviceListStrategyValues::Envvar,
+            ]))),
+        );
+
+        let device_plugins_6: NvidiaDevicePluginSettings =
+            serde_json::from_str(test_json_6).unwrap();
+        assert_eq!(
+            device_plugins_6,
+            helper_with_device_list_strategy(Some(NvidiaDeviceListStrategy::Vector(vec![
+                NvidiaDeviceListStrategyValues::CdiCri,
+                NvidiaDeviceListStrategyValues::Envvar,
+                NvidiaDeviceListStrategyValues::VolumeMounts,
+            ]))),
+        );
+
+        let results_1 = serde_json::to_string(&device_plugins_1).unwrap();
+        let results_2 = serde_json::to_string(&device_plugins_2).unwrap();
+        let results_3 = serde_json::to_string(&device_plugins_3).unwrap();
+        let results_4 = serde_json::to_string(&device_plugins_4).unwrap();
+        let results_5 = serde_json::to_string(&device_plugins_5).unwrap();
+        let results_6 = serde_json::to_string(&device_plugins_6).unwrap();
+
+        assert_eq!(results_1, test_json_1);
+        assert_eq!(results_2, test_json_2);
+        assert_eq!(results_3, test_json_3);
+        assert_eq!(results_4, test_json_4);
+        assert_eq!(results_5, test_json_5);
+        assert_eq!(results_6, test_json_6);
+    }
 }

bottlerocket-settings-models/settings-extensions/kubelet-device-plugins/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "settings-extension-kubelet-device-plugins"
-version = "0.2.0"
+version = "0.3.0"
 authors = ["Arnaldo Garcia Rincon <[email protected]>"]
 license = "Apache-2.0 OR MIT"
 edition = "2021"

bottlerocket-settings-models/settings-extensions/kubelet-device-plugins/src/lib.rs

@@ -44,8 +44,8 @@ mod test {
     use super::*;
     use bottlerocket_modeled_types::{
         MigProfile, NvidiaDeviceIdStrategy, NvidiaDeviceListStrategy,
-        NvidiaDevicePartitioningStrategy, NvidiaDeviceSharingStrategy, NvidiaGpuModel,
-        NvidiaMigSettings, NvidiaTimeSlicingSettings,
+        NvidiaDeviceListStrategyValues, NvidiaDevicePartitioningStrategy,
+        NvidiaDeviceSharingStrategy, NvidiaGpuModel, NvidiaMigSettings, NvidiaTimeSlicingSettings,
     };
     use bounded_integer::BoundedI32;
     use std::collections::HashMap;
@@ -60,7 +60,43 @@ mod test {
     }
 
     #[test]
-    fn test_serde_kubelet_device_plugins() {
+    fn test_serde_kubelet_device_plugins_vec() {
+        let test_json = r#"{"nvidia":{"pass-device-specs":true,"device-id-strategy":"index","device-list-strategy":["volume-mounts","envvar"],"device-sharing-strategy":"time-slicing","time-slicing":{"replicas":2,"rename-by-default":true,"fail-requests-greater-than-one":true},"device-partitioning-strategy":"mig","mig":{"profile":{"a100.40gb":"1g.5gb"}}}}"#;
+
+        let device_plugins: KubeletDevicePluginsV1 = serde_json::from_str(test_json).unwrap();
+        assert_eq!(
+            device_plugins,
+            KubeletDevicePluginsV1 {
+                nvidia: Some(NvidiaDevicePluginSettings {
+                    pass_device_specs: Some(true),
+                    device_id_strategy: Some(NvidiaDeviceIdStrategy::Index),
+                    device_list_strategy: Some(NvidiaDeviceListStrategy::Vector(vec![
+                        NvidiaDeviceListStrategyValues::VolumeMounts,
+                        NvidiaDeviceListStrategyValues::Envvar,
+                    ])),
+                    device_sharing_strategy: Some(NvidiaDeviceSharingStrategy::TimeSlicing),
+                    time_slicing: Some(NvidiaTimeSlicingSettings {
+                        replicas: Some(BoundedI32::new(2).unwrap()),
+                        rename_by_default: Some(true),
+                        fail_requests_greater_than_one: Some(true),
+                    }),
+                    device_partitioning_strategy: Some(NvidiaDevicePartitioningStrategy::MIG),
+                    mig: Some(NvidiaMigSettings {
+                        profile: Some(HashMap::from([(
+                            NvidiaGpuModel::try_from("a100.40gb").unwrap(),
+                            MigProfile::try_from("1g.5gb").unwrap()
+                        )]))
+                    }),
+                })
+            }
+        );
+
+        let results = serde_json::to_string(&device_plugins).unwrap();
+        assert_eq!(results, test_json);
+    }
+
+    #[test]
+    fn test_serde_kubelet_device_plugins_scalar() {
         let test_json = r#"{"nvidia":{"pass-device-specs":true,"device-id-strategy":"index","device-list-strategy":"volume-mounts","device-sharing-strategy":"time-slicing","time-slicing":{"replicas":2,"rename-by-default":true,"fail-requests-greater-than-one":true},"device-partitioning-strategy":"mig","mig":{"profile":{"a100.40gb":"1g.5gb"}}}}"#;
 
         let device_plugins: KubeletDevicePluginsV1 = serde_json::from_str(test_json).unwrap();
@@ -70,7 +106,9 @@ mod test {
                 nvidia: Some(NvidiaDevicePluginSettings {
                     pass_device_specs: Some(true),
                     device_id_strategy: Some(NvidiaDeviceIdStrategy::Index),
-                    device_list_strategy: Some(NvidiaDeviceListStrategy::VolumeMounts),
+                    device_list_strategy: Some(NvidiaDeviceListStrategy::Scalar(
+                        NvidiaDeviceListStrategyValues::VolumeMounts
+                    )),
                     device_sharing_strategy: Some(NvidiaDeviceSharingStrategy::TimeSlicing),
                     time_slicing: Some(NvidiaTimeSlicingSettings {
                         replicas: Some(BoundedI32::new(2).unwrap()),
@@ -84,7 +122,7 @@ mod test {
                             MigProfile::try_from("1g.5gb").unwrap()
                         )]))
                     }),
-                }),
+                })
             }
         );
 

bottlerocket-settings-models/settings-models/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bottlerocket-settings-models"
-version = "0.9.0"
+version = "0.10.0"
 authors = ["Tom Kirchner <[email protected]>"]
 license = "Apache-2.0 OR MIT"
 edition = "2021"

bottlerocket-settings-plugin/src/settings.rs

@@ -10,7 +10,7 @@ functions.
 #![expect(non_local_definitions)]
 // Avoid `elide the lifetimes` warnings by clippy. The suggested changes do not make the code more
 // readable
-#![expect(clippy::needless_lifetimes)]
+#![allow(clippy::needless_lifetimes)]
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_json::Value as JsonValue;
 use std::path::PathBuf;