Skip to content

Commit 6e9ca12

Browse files
fix(dpf): wire service VPC slots
Signed-off-by: Frank Spitulski <fspitulski@nvidia.com>
1 parent c6ebc75 commit 6e9ca12

13 files changed

Lines changed: 1073 additions & 342 deletions

File tree

Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

crates/api-core/src/cfg/file.rs

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3172,6 +3172,14 @@ impl CarbideConfig {
31723172
Ok(())
31733173
}
31743174

3175+
pub(crate) fn validate_service_vpc_slots(&self) -> eyre::Result<()> {
3176+
eyre::ensure!(
3177+
self.dpu_config.service_vpc_slot_count == 0 || self.site_global_vpc_vni.is_none(),
3178+
"dpu_config.service_vpc_slot_count requires site_global_vpc_vni to be unset because service VPCs require distinct HBN VRFs"
3179+
);
3180+
Ok(())
3181+
}
3182+
31753183
/// validate_supernic_firmware_profiles checks that each profile's inner
31763184
/// part_number and psid match the HashMap keys they are nested under.
31773185
/// Logs a warning for any mismatches (the inner values are authoritative
@@ -5564,6 +5572,26 @@ path = "credentials.yaml"
55645572
assert!(config.validate_web_ui_sidebar_tools().is_err());
55655573
}
55665574

5575+
#[test]
5576+
fn validate_service_vpc_slots_rejects_site_global_vpc_vni() {
5577+
let mut config: CarbideConfig = Figment::new()
5578+
.merge(Toml::file(format!("{TEST_DATA_DIR}/min_config.toml")))
5579+
.extract()
5580+
.unwrap();
5581+
5582+
config.dpu_config.service_vpc_slot_count = 1;
5583+
assert!(config.validate_service_vpc_slots().is_ok());
5584+
5585+
config.site_global_vpc_vni = Some(6_000);
5586+
assert!(
5587+
config
5588+
.validate_service_vpc_slots()
5589+
.unwrap_err()
5590+
.to_string()
5591+
.contains("requires site_global_vpc_vni to be unset")
5592+
);
5593+
}
5594+
55675595
#[test]
55685596
fn periodic_state_republish_defaults_enabled() {
55695597
let config = PeriodicStateRepublishConfig::default();

crates/api-core/src/cfg/load.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -307,6 +307,7 @@ pub fn parse_carbide_config(
307307

308308
// Validate that admin-UI tool entries have unique names.
309309
config.validate_web_ui_sidebar_tools()?;
310+
config.validate_service_vpc_slots()?;
310311
config.api_admission_control.validate()?;
311312

312313
if let Some(config) = &config.dsx_exchange_event_bus {

crates/api-core/src/dpf_services.rs

Lines changed: 32 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,14 @@ use std::collections::BTreeMap;
2121
use std::fmt::Write;
2222

2323
use carbide_dpf::types::{
24-
DHCP_SERVER_SERVICE_NAME, DOCA_HBN_SERVICE_NAME, DOCA_WEAVE_DHCP_AGENT_SERVICE_NAME,
25-
DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_NAME, DOCA_XPLANE_SERVICE_NAME, DPU_AGENT_SERVICE_NAME,
26-
DTS_SERVICE_NAME, DpuServiceInterfaceTemplateDefinition, FMDS_SERVICE_NAME,
27-
OTEL_COLLECTOR_SERVICE_NAME,
24+
DHCP_SERVER_SERVICE_NAME, DOCA_HBN_SERVICE_NAME, DOCA_HBN_SERVICE_NETWORK,
25+
DOCA_WEAVE_DHCP_AGENT_SERVICE_NAME, DOCA_WEAVE_FLOW_CONTROLLER_SERVICE_NAME,
26+
DOCA_XPLANE_SERVICE_NAME, DPU_AGENT_SERVICE_NAME, DTS_SERVICE_NAME,
27+
DpuServiceInterfaceTemplateDefinition, FMDS_SERVICE_NAME, OTEL_COLLECTOR_SERVICE_NAME,
2828
};
2929
use carbide_dpf::{
3030
IntOrString, ServiceDefinition, ServiceInterface, ServiceNAD, ServiceNADResourceType,
31+
ServiceVpcSlots,
3132
};
3233

3334
use crate::cfg::file::{
@@ -59,7 +60,6 @@ pub(crate) const DOCA_HBN_SERVICE_HELM_NAME: &str = "doca-hbn";
5960
pub(crate) const DOCA_HBN_SERVICE_HELM_VERSION: &str = "3.4.0";
6061
pub(crate) const DOCA_HBN_SERVICE_IMAGE_NAME: &str = "doca_hbn";
6162
pub(crate) const DOCA_HBN_SERVICE_IMAGE_TAG: &str = "3.4.0-doca3.4.0";
62-
pub(crate) const DOCA_HBN_SERVICE_NETWORK: &str = "mybrhbn";
6363

6464
/// DHCP Service Definitions
6565
pub(crate) const DHCP_SERVER_SERVICE_HELM_NAME: &str = "nico-dhcp-server";
@@ -132,48 +132,14 @@ pub(crate) const COMPILE_TIME_IMAGE_TAG: &str = match option_env!("CARBIDE_BUILD
132132

133133
fn doca_hbn_service_interfaces(
134134
interfaces: &[DpuServiceInterfaceTemplateDefinition],
135-
extra_interfaces: &[String],
135+
service_vpc_slots: ServiceVpcSlots,
136136
) -> Vec<ServiceInterface> {
137137
let mut service_interfaces =
138138
dpu_service_interfaces(interfaces, DOCA_HBN_SERVICE_NAME, DOCA_HBN_SERVICE_NETWORK);
139-
service_interfaces.extend(extra_interfaces.iter().map(|name| ServiceInterface {
140-
name: name.clone(),
141-
network: DOCA_HBN_SERVICE_NETWORK.to_string(),
142-
}));
139+
service_vpc_slots.append_hbn_interfaces(&mut service_interfaces);
143140
service_interfaces
144141
}
145142

146-
/// Generates deterministic service-VPC interface names after checking HBN capacity.
147-
pub(crate) fn service_vpc_interfaces(
148-
interfaces: &[DpuServiceInterfaceTemplateDefinition],
149-
slot_count: u32,
150-
) -> Result<Vec<String>, String> {
151-
let mut hbn_interface_names = doca_hbn_service_interfaces(interfaces, &[])
152-
.into_iter()
153-
.map(|interface| interface.name)
154-
.collect::<std::collections::BTreeSet<_>>();
155-
156-
let total = usize::try_from(slot_count)
157-
.ok()
158-
.and_then(|slot_count| hbn_interface_names.len().checked_add(slot_count))
159-
.ok_or_else(|| "HBN interface count exceeds usize".to_string())?;
160-
if total > 32 {
161-
return Err(format!(
162-
"HBN interface count {total} exceeds the supported maximum of 32"
163-
));
164-
}
165-
166-
let generated = (0..slot_count)
167-
.map(|slot| format!("iface_svc_{slot}"))
168-
.collect::<Vec<_>>();
169-
for name in &generated {
170-
if !hbn_interface_names.insert(name.clone()) {
171-
return Err(format!("HBN interface name {name:?} is not unique"));
172-
}
173-
}
174-
175-
Ok(generated)
176-
}
177143
fn dhcp_server_service_interfaces(
178144
interfaces: &[DpuServiceInterfaceTemplateDefinition],
179145
) -> Vec<ServiceInterface> {
@@ -482,9 +448,9 @@ fn reassert_api_owned_value(
482448
pub(crate) fn doca_hbn_service(
483449
cfg: &DpfServiceConfig,
484450
dpu_interfaces: &[DpuServiceInterfaceTemplateDefinition],
485-
extra_interfaces: &[String],
451+
service_vpc_slots: ServiceVpcSlots,
486452
) -> ServiceDefinition {
487-
let interfaces = doca_hbn_service_interfaces(dpu_interfaces, extra_interfaces);
453+
let interfaces = doca_hbn_service_interfaces(dpu_interfaces, service_vpc_slots);
488454
let mut helm_values = serde_json::json!({
489455
"image": {
490456
"repository": cfg.docker_repo_url,
@@ -892,12 +858,12 @@ pub(crate) fn mandatory_services(
892858
resolved: &DpfResolvedMandatoryServicesConfig,
893859
bootstrap_ca: &DpfDpuAgentBootstrapCa,
894860
interfaces: &[DpuServiceInterfaceTemplateDefinition],
895-
service_vpc_interfaces: &[String],
861+
service_vpc_slots: ServiceVpcSlots,
896862
node_auth: &NodeAuthConfig,
897863
) -> Vec<ServiceDefinition> {
898864
let mut service_vec = vec![
899865
dts_service(&resolved.base.dts),
900-
doca_hbn_service(&resolved.base.doca_hbn, interfaces, service_vpc_interfaces),
866+
doca_hbn_service(&resolved.base.doca_hbn, interfaces, service_vpc_slots),
901867
dhcp_server_service(&resolved.base.dhcp_server, interfaces),
902868
dpu_agent_service(&resolved.base.dpu_agent, bootstrap_ca),
903869
// Not `node_auth.enabled` directly: an operator staging a disable
@@ -974,12 +940,8 @@ mod tests {
974940

975941
// HBN receives p0, p1, the PF, the VF, and the configured external attachment; its SF
976942
// count and startup YAML agree.
977-
let service_vpc_interfaces = service_vpc_interfaces(&interfaces, 1).unwrap();
978-
let hbn = doca_hbn_service(
979-
&default_doca_hbn_service(),
980-
&interfaces,
981-
&service_vpc_interfaces,
982-
);
943+
let service_vpc_slots = ServiceVpcSlots::new(1).unwrap();
944+
let hbn = doca_hbn_service(&default_doca_hbn_service(), &interfaces, service_vpc_slots);
983945
assert_eq!(hbn.interfaces.len(), 5);
984946
assert_eq!(
985947
hbn.helm_values.as_ref().unwrap()["resources"]["nvidia.com/bf_sf"],
@@ -1013,16 +975,6 @@ mod tests {
1013975
);
1014976
}
1015977

1016-
#[test]
1017-
fn service_vpc_interfaces_are_deterministic_and_capacity_checked() {
1018-
let interfaces = build_effective_dpu_interfaces(16, None);
1019-
assert_eq!(
1020-
service_vpc_interfaces(&interfaces, 2).unwrap(),
1021-
["iface_svc_0".to_string(), "iface_svc_1".to_string()]
1022-
);
1023-
assert!(service_vpc_interfaces(&interfaces, 15).is_err());
1024-
}
1025-
1026978
/// Verifies operator Helm values cannot disconnect HBN's SF request from its interfaces.
1027979
#[test]
1028980
fn hbn_sf_count_remains_topology_derived() {
@@ -1039,7 +991,7 @@ mod tests {
1039991
let interfaces = build_dpu_interfaces_vec();
1040992

1041993
// Ordinary resource overrides remain effective, while the SF count follows inventory.
1042-
let hbn = doca_hbn_service(&config, &interfaces, &[]);
994+
let hbn = doca_hbn_service(&config, &interfaces, ServiceVpcSlots::default());
1043995
let helm_values = hbn.helm_values.unwrap();
1044996
assert_eq!(helm_values["resources"]["memory"], "8Gi");
1045997
assert_eq!(
@@ -1159,7 +1111,11 @@ mod tests {
11591111
fn hbn_and_dts_omit_image_pull_secrets_by_default() {
11601112
// HBN and DTS pull from the public DOCA registry: no imagePullSecrets unless configured.
11611113
let interfaces = build_dpu_interfaces_vec();
1162-
let hbn = doca_hbn_service(&default_doca_hbn_service(), &interfaces, &[]);
1114+
let hbn = doca_hbn_service(
1115+
&default_doca_hbn_service(),
1116+
&interfaces,
1117+
ServiceVpcSlots::default(),
1118+
);
11631119
assert!(
11641120
hbn.helm_values.unwrap().get("imagePullSecrets").is_none(),
11651121
"HBN must not emit imagePullSecrets without a configured secret"
@@ -1180,7 +1136,7 @@ mod tests {
11801136
let mut hbn_cfg = default_doca_hbn_service();
11811137
hbn_cfg.docker_image_pull_secret = Some("private-pull-secret".to_string());
11821138
assert_eq!(
1183-
doca_hbn_service(&hbn_cfg, &interfaces, &[])
1139+
doca_hbn_service(&hbn_cfg, &interfaces, ServiceVpcSlots::default())
11841140
.helm_values
11851141
.unwrap()["imagePullSecrets"],
11861142
expected
@@ -1646,12 +1602,18 @@ mod tests {
16461602
let bootstrap_ca = DpfDpuAgentBootstrapCa::default();
16471603

16481604
let fmds_mode = |node_auth: &NodeAuthConfig| {
1649-
mandatory_services(&resolved, &bootstrap_ca, &[], &[], node_auth)
1650-
.into_iter()
1651-
.find(|s| s.name == FMDS_SERVICE_NAME)
1652-
.and_then(|s| s.helm_values)
1653-
.and_then(|v| v.get("useNodeTokens").and_then(serde_json::Value::as_bool))
1654-
.expect("fmds renders useNodeTokens")
1605+
mandatory_services(
1606+
&resolved,
1607+
&bootstrap_ca,
1608+
&[],
1609+
ServiceVpcSlots::default(),
1610+
node_auth,
1611+
)
1612+
.into_iter()
1613+
.find(|s| s.name == FMDS_SERVICE_NAME)
1614+
.and_then(|s| s.helm_values)
1615+
.and_then(|v| v.get("useNodeTokens").and_then(serde_json::Value::as_bool))
1616+
.expect("fmds renders useNodeTokens")
16551617
};
16561618

16571619
let derived_on = NodeAuthConfig {

0 commit comments

Comments
 (0)