From b8c7d29a61f91f6100b8bba6c1b5c8d26ac78b97 Mon Sep 17 00:00:00 2001 From: Bohdan Dobrelia Date: Mon, 11 Aug 2025 17:37:56 +0200 Subject: [PATCH] Add nova04delta DT and nvidia-vfio VA Add DT and related VA for full PCI device passthrough for testing GPU workloads on RHOSO. The VA is based on nvidia-mdev-passthrough, DT is based on nova02beta. Jira: #OSPRH-18904 Signed-off-by: Bohdan Dobrelia --- .gitignore | 1 + README.md | 7 + automation/mocks/nova04delta.yaml | 42 ++ automation/net-env/nova04delta.yaml | 405 ++++++++++++++++++ .../net-env/nvidia-vfio-passthrough.yaml | 405 ++++++++++++++++++ automation/vars/nova04delta-adoption.yaml | 29 ++ automation/vars/nova04delta.yaml | 72 ++++ .../nvidia-vfio-passthrough-adoption.yaml | 29 ++ automation/vars/nvidia-vfio-passthrough.yaml | 71 +++ dt/nova/nova04delta/README.md | 12 + .../edpm/deployment/kustomization.yaml | 21 + .../edpm/nodeset/baremetalhost.yaml | 15 + .../edpm/nodeset/baremetalhost_template.yaml | 20 + .../edpm/nodeset/kustomization.yaml | 214 +++++++++ .../edpm/nodeset/network-data-secrets.yaml | 8 + .../nova04delta/edpm/nodeset/nova_gpu.yaml | 92 ++++ .../edpm/nodeset/provisioning.yaml | 14 + dt/nova/nova04delta/kustomization.yaml | 135 ++++++ dt/nova/nova04delta/namespace.yaml | 12 + .../nova04delta/networking/kustomization.yaml | 23 + .../networking/nncp/kustomization.yaml | 21 + examples/dt/nova/nova04delta/README.md | 91 ++++ .../nova/nova04delta/control-plane/README.md | 54 +++ .../control-plane/kustomization.yaml | 10 + .../networking/kustomization.yaml | 10 + .../control-plane/networking/nncp/.gitignore | 2 + .../networking/nncp/kustomization.yaml | 9 + .../control-plane/networking/nncp/values.yaml | 199 +++++++++ .../control-plane/service-values.yaml | 64 +++ examples/dt/nova/nova04delta/data-plane.md | 87 ++++ .../edpm/deployment/kustomization.yaml | 9 + .../nova04delta/edpm/deployment/values.yaml | 10 + .../edpm/nodeset/kustomization.yaml | 9 + .../nova/nova04delta/edpm/nodeset/values.yaml | 220 ++++++++++ .../va/nvidia-vfio-passthrough/.gitignore | 1 + examples/va/nvidia-vfio-passthrough/README.md | 85 ++++ .../control-plane/README.md | 69 +++ .../control-plane/kustomization.yaml | 10 + .../networking/kustomization.yaml | 9 + .../networking/nncp/kustomization.yaml | 9 + .../control-plane/networking/nncp/values.yaml | 209 +++++++++ .../control-plane/service-values.yaml | 61 +++ .../nvidia-vfio-passthrough/data-plane-pre.md | 80 ++++ .../edpm/deployment/.gitignore | 2 + .../edpm/deployment/kustomization.yaml | 12 + .../edpm/deployment/values.yaml | 10 + .../edpm/nodeset/kustomization.yaml | 12 + .../edpm/nodeset/values.yaml | 218 ++++++++++ .../edpm/deployment/kustomization.yaml | 6 + .../edpm/nodeset/kustomization.yaml | 6 + va/nvidia-vfio-passthrough/kustomization.yaml | 6 + .../networking/kustomization.yaml | 6 + .../networking/nncp/kustomization.yaml | 6 + zuul.d/projects.yaml | 4 + zuul.d/validations.yaml | 48 +++ 55 files changed, 3291 insertions(+) create mode 100644 automation/mocks/nova04delta.yaml create mode 100644 automation/net-env/nova04delta.yaml create mode 100644 automation/net-env/nvidia-vfio-passthrough.yaml create mode 100644 automation/vars/nova04delta-adoption.yaml create mode 100644 automation/vars/nova04delta.yaml create mode 100644 automation/vars/nvidia-vfio-passthrough-adoption.yaml create mode 100644 automation/vars/nvidia-vfio-passthrough.yaml create mode 100644 dt/nova/nova04delta/README.md create mode 100644 dt/nova/nova04delta/edpm/deployment/kustomization.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/baremetalhost.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/baremetalhost_template.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/kustomization.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/network-data-secrets.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/nova_gpu.yaml create mode 100644 dt/nova/nova04delta/edpm/nodeset/provisioning.yaml create mode 100644 dt/nova/nova04delta/kustomization.yaml create mode 100644 dt/nova/nova04delta/namespace.yaml create mode 100644 dt/nova/nova04delta/networking/kustomization.yaml create mode 100644 dt/nova/nova04delta/networking/nncp/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/README.md create mode 100644 examples/dt/nova/nova04delta/control-plane/README.md create mode 100644 examples/dt/nova/nova04delta/control-plane/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/control-plane/networking/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/control-plane/networking/nncp/.gitignore create mode 100644 examples/dt/nova/nova04delta/control-plane/networking/nncp/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/control-plane/networking/nncp/values.yaml create mode 100644 examples/dt/nova/nova04delta/control-plane/service-values.yaml create mode 100644 examples/dt/nova/nova04delta/data-plane.md create mode 100644 examples/dt/nova/nova04delta/edpm/deployment/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/edpm/deployment/values.yaml create mode 100644 examples/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml create mode 100644 examples/dt/nova/nova04delta/edpm/nodeset/values.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/.gitignore create mode 100644 examples/va/nvidia-vfio-passthrough/README.md create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/README.md create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/kustomization.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/networking/kustomization.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/kustomization.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/values.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/control-plane/service-values.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/data-plane-pre.md create mode 100644 examples/va/nvidia-vfio-passthrough/edpm/deployment/.gitignore create mode 100644 examples/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/edpm/deployment/values.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml create mode 100644 examples/va/nvidia-vfio-passthrough/edpm/nodeset/values.yaml create mode 100644 va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml create mode 100644 va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml create mode 100644 va/nvidia-vfio-passthrough/kustomization.yaml create mode 100644 va/nvidia-vfio-passthrough/networking/kustomization.yaml create mode 100644 va/nvidia-vfio-passthrough/networking/nncp/kustomization.yaml diff --git a/.gitignore b/.gitignore index 9de5703b0..ce4a07247 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ examples/dt/uni*/control-plane/nncp/nncp.yaml examples/dt/uni*/data-plane.yaml examples/dt/uni*/networker/edpm-networker.yaml .idea/ +.aider* diff --git a/README.md b/README.md index e2583ddbd..d1b8dc8a5 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,10 @@ The following VAs are available. - [Network Functions Virtualization with SRIOV](examples/va/nfv/sriov/) - [Network Functions Virtualization with OvS DPDK](examples/va/nfv/ovs-dpdk/) - [Network Functions Virtualization with OvS DPDK & SRIOV](examples/va/nfv/ovs-dpdk-sriov/) [untested] +- [Nvidia GPU Passthrough (VFIO)](examples/va/nvidia-vfio-passthrough/) [untested] + +## Deployment Topologies + +The following DTs are available. + +- [Nova with GPU Passthrough (VFIO)](examples/dt/nova/nova04delta/) [untested] diff --git a/automation/mocks/nova04delta.yaml b/automation/mocks/nova04delta.yaml new file mode 100644 index 000000000..95cc934ea --- /dev/null +++ b/automation/mocks/nova04delta.yaml @@ -0,0 +1,42 @@ +--- +cifmw_install_ca_url: http://example.com/example.pem +cifmw_repo_setup_rhos_release_rpm: http://example.com/rhos-release.rpm +cifmw_repo_setup_rhos_release_args: example-args +cifmw_ci_gen_kustomize_values_remove_keys_expressions: + - ^node(_[0-9]+)?$ +cifmw_baremetal_hosts: + edpm-compute-0: + connection: idrac-virtualmedia://foo.example.com/redfish/v1/Systems/System.Embedded.1 + boot_mode: UEFI + status: running + nics: + - mac: "aa:bb:cc:dd:ee:ff" + network: provision + root_device_hint: /dev/disk/by-path/pci-0000:65:00.0-scsi-0:1:234:5 + root_device_hint_field: deviceName + credentials_file: "/home/zuul/idrac_access.yaml" + nmstate: + interfaces: + - name: nic1 + type: ethernet + state: up + ipv4: + enabled: true + dhcp: true + - name: nic2 + type: ethernet + state: up + ipv4: + enabled: true + address: + - ip: 10.0.0.10 + prefix-length: 24 + dns-resolver: + config: + server: + - 10.0.0.1 + routes: + config: + - destination: 0.0.0.0/0 + next-hop-address: 10.0.0.1 + next-hop-interface: nic1 diff --git a/automation/net-env/nova04delta.yaml b/automation/net-env/nova04delta.yaml new file mode 100644 index 000000000..b6d3b4245 --- /dev/null +++ b/automation/net-env/nova04delta.yaml @@ -0,0 +1,405 @@ +--- +instances: + compute-0: + name: compute-0 + networks: + ctlplane: + interface_name: eth1 + ip_v4: 192.168.122.100 + is_trunk_parent: true + mac_addr: 52:54:03:b1:78:6c + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: eth1.20 + ip_v4: 172.17.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:1e:58:da + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: eth2 + ip_v4: 192.168.111.100 + mac_addr: 52:54:03:16:9c:42 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: eth1.21 + ip_v4: 172.18.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:7e:be:ad + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: eth1.22 + ip_v4: 172.19.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:33:ee:37 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-0: + hostname: master-0 + name: ocp-master-0 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.10 + is_trunk_parent: true + mac_addr: 52:54:00:15:07:f4 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:68:4b:ce + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.10 + mac_addr: 52:54:00:df:8c:54 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:05:23:2c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:06:80:f0 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-1: + hostname: master-1 + name: ocp-master-1 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.11 + is_trunk_parent: true + mac_addr: 52:54:01:43:50:83 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:71:78:e6 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.11 + mac_addr: 52:54:01:51:68:1e + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:76:3d:ba + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:7a:5d:1d + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-2: + hostname: master-2 + name: ocp-master-2 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.12 + is_trunk_parent: true + mac_addr: 52:54:02:fd:b8:5a + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:66:a0:7c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.12 + mac_addr: 52:54:02:50:e9:87 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:09:45:3b + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:22:70:9f + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 +networks: + ctlplane: + dns_v4: + - 192.168.122.1 + dns_v6: [] + gw_v4: 192.168.122.1 + mtu: 1500 + network_name: ctlplane + network_v4: 192.168.122.0/24 + search_domain: ctlplane.example.com + tools: + metallb: + ipv4_ranges: + - end: 192.168.122.90 + end_host: 90 + length: 11 + start: 192.168.122.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 192.168.122.70 + end_host: 70 + length: 41 + start: 192.168.122.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 192.168.122.120 + end_host: 120 + length: 21 + start: 192.168.122.100 + start_host: 100 + - end: 192.168.122.170 + end_host: 170 + length: 21 + start: 192.168.122.150 + start_host: 150 + ipv6_ranges: [] + external: + dns_v4: [] + dns_v6: [] + mtu: 1500 + network_name: external + network_v4: 10.0.0.0/24 + search_domain: external.example.com + tools: + netconfig: + ipv4_ranges: + - end: 10.0.0.250 + end_host: 250 + length: 151 + start: 10.0.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 + internalapi: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: internalapi + network_v4: 172.17.0.0/24 + search_domain: internalapi.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.17.0.90 + end_host: 90 + length: 11 + start: 172.17.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.17.0.70 + end_host: 70 + length: 41 + start: 172.17.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.17.0.250 + end_host: 250 + length: 151 + start: 172.17.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 20 + ocpbm: + dns_v4: [] + dns_v6: [] + network_name: ocpbm + network_v4: 192.168.111.0/24 + search_domain: ocpbm.example.com + tools: {} + storage: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storage + network_v4: 172.18.0.0/24 + search_domain: storage.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.18.0.90 + end_host: 90 + length: 11 + start: 172.18.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.18.0.70 + end_host: 70 + length: 41 + start: 172.18.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.18.0.250 + end_host: 250 + length: 151 + start: 172.18.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 21 + tenant: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: tenant + network_v4: 172.19.0.0/24 + search_domain: tenant.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.19.0.90 + end_host: 90 + length: 11 + start: 172.19.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.19.0.70 + end_host: 70 + length: 41 + start: 172.19.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.19.0.250 + end_host: 250 + length: 151 + start: 172.19.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 +routers: {} diff --git a/automation/net-env/nvidia-vfio-passthrough.yaml b/automation/net-env/nvidia-vfio-passthrough.yaml new file mode 100644 index 000000000..b6d3b4245 --- /dev/null +++ b/automation/net-env/nvidia-vfio-passthrough.yaml @@ -0,0 +1,405 @@ +--- +instances: + compute-0: + name: compute-0 + networks: + ctlplane: + interface_name: eth1 + ip_v4: 192.168.122.100 + is_trunk_parent: true + mac_addr: 52:54:03:b1:78:6c + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: eth1.20 + ip_v4: 172.17.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:1e:58:da + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: eth2 + ip_v4: 192.168.111.100 + mac_addr: 52:54:03:16:9c:42 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: eth1.21 + ip_v4: 172.18.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:7e:be:ad + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: eth1.22 + ip_v4: 172.19.0.100 + is_trunk_parent: false + mac_addr: 52:54:00:33:ee:37 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-0: + hostname: master-0 + name: ocp-master-0 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.10 + is_trunk_parent: true + mac_addr: 52:54:00:15:07:f4 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:68:4b:ce + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.10 + mac_addr: 52:54:00:df:8c:54 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:05:23:2c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:06:80:f0 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-1: + hostname: master-1 + name: ocp-master-1 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.11 + is_trunk_parent: true + mac_addr: 52:54:01:43:50:83 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:71:78:e6 + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.11 + mac_addr: 52:54:01:51:68:1e + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:76:3d:ba + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:7a:5d:1d + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 + ocp-master-2: + hostname: master-2 + name: ocp-master-2 + networks: + ctlplane: + interface_name: enp6s0 + ip_v4: 192.168.122.12 + is_trunk_parent: true + mac_addr: 52:54:02:fd:b8:5a + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: ctlplane + prefix_length_v4: 24 + skip_nm: false + internalapi: + interface_name: enp6s0.20 + ip_v4: 172.17.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:66:a0:7c + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: internalapi + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 20 + ocpbm: + interface_name: enp2s0 + ip_v4: 192.168.111.12 + mac_addr: 52:54:02:50:e9:87 + netmask_v4: 255.255.255.0 + network_name: ocpbm + prefix_length_v4: 24 + skip_nm: false + storage: + interface_name: enp6s0.21 + ip_v4: 172.18.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:09:45:3b + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: storage + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 21 + tenant: + interface_name: enp6s0.22 + ip_v4: 172.19.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:22:70:9f + mtu: 1496 + netmask_v4: 255.255.255.0 + network_name: tenant + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 22 +networks: + ctlplane: + dns_v4: + - 192.168.122.1 + dns_v6: [] + gw_v4: 192.168.122.1 + mtu: 1500 + network_name: ctlplane + network_v4: 192.168.122.0/24 + search_domain: ctlplane.example.com + tools: + metallb: + ipv4_ranges: + - end: 192.168.122.90 + end_host: 90 + length: 11 + start: 192.168.122.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 192.168.122.70 + end_host: 70 + length: 41 + start: 192.168.122.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 192.168.122.120 + end_host: 120 + length: 21 + start: 192.168.122.100 + start_host: 100 + - end: 192.168.122.170 + end_host: 170 + length: 21 + start: 192.168.122.150 + start_host: 150 + ipv6_ranges: [] + external: + dns_v4: [] + dns_v6: [] + mtu: 1500 + network_name: external + network_v4: 10.0.0.0/24 + search_domain: external.example.com + tools: + netconfig: + ipv4_ranges: + - end: 10.0.0.250 + end_host: 250 + length: 151 + start: 10.0.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 + internalapi: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: internalapi + network_v4: 172.17.0.0/24 + search_domain: internalapi.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.17.0.90 + end_host: 90 + length: 11 + start: 172.17.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.17.0.70 + end_host: 70 + length: 41 + start: 172.17.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.17.0.250 + end_host: 250 + length: 151 + start: 172.17.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 20 + ocpbm: + dns_v4: [] + dns_v6: [] + network_name: ocpbm + network_v4: 192.168.111.0/24 + search_domain: ocpbm.example.com + tools: {} + storage: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: storage + network_v4: 172.18.0.0/24 + search_domain: storage.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.18.0.90 + end_host: 90 + length: 11 + start: 172.18.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.18.0.70 + end_host: 70 + length: 41 + start: 172.18.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.18.0.250 + end_host: 250 + length: 151 + start: 172.18.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 21 + tenant: + dns_v4: [] + dns_v6: [] + mtu: 1496 + network_name: tenant + network_v4: 172.19.0.0/24 + search_domain: tenant.example.com + tools: + metallb: + ipv4_ranges: + - end: 172.19.0.90 + end_host: 90 + length: 11 + start: 172.19.0.80 + start_host: 80 + ipv6_ranges: [] + multus: + ipv4_ranges: + - end: 172.19.0.70 + end_host: 70 + length: 41 + start: 172.19.0.30 + start_host: 30 + ipv4_routes: [] + ipv6_ranges: [] + ipv6_routes: [] + netconfig: + ipv4_ranges: + - end: 172.19.0.250 + end_host: 250 + length: 151 + start: 172.19.0.100 + start_host: 100 + ipv6_ranges: [] + vlan_id: 22 +routers: {} diff --git a/automation/vars/nova04delta-adoption.yaml b/automation/vars/nova04delta-adoption.yaml new file mode 100644 index 000000000..f23f6abf4 --- /dev/null +++ b/automation/vars/nova04delta-adoption.yaml @@ -0,0 +1,29 @@ +--- +vas: + nova04delta-adoption: + stages: + - name: nncp-configuration + path: examples/dt/nova/nova04delta/control-plane/networking/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=5m + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - name: network-configuration + path: examples/dt/nova/nova04delta/control-plane/networking + wait_conditions: + - >- + oc -n metallb-system wait pod + -l app=metallb -l component=speaker + --for condition=Ready + --timeout=5min + values: + - name: network-values + src_file: nncp/values.yaml + build_output: networking.yaml diff --git a/automation/vars/nova04delta.yaml b/automation/vars/nova04delta.yaml new file mode 100644 index 000000000..d92cca5a1 --- /dev/null +++ b/automation/vars/nova04delta.yaml @@ -0,0 +1,72 @@ +--- +vas: + nova04delta: + stages: + - name: nncp-configuration + path: examples/dt/nova/nova04delta/control-plane/networking/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=5m + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - name: network-configuration + path: examples/dt/nova/nova04delta/control-plane/networking + wait_conditions: + - >- + oc -n metallb-system wait pod + -l app=metallb -l component=speaker + --for condition=Ready + --timeout=5m + values: + - name: network-values + src_file: nncp/values.yaml + build_output: network.yaml + + - name: control-plane + path: examples/dt/nova/nova04delta/control-plane + wait_conditions: + - >- + oc -n openstack wait osctlplane controlplane + --for condition=Ready + --timeout=60m + values: + - name: network-values + src_file: networking/nncp/values.yaml + - name: service-values.yaml + src_file: service-values.yaml + build_output: control-plane.yaml + + - name: edpm-nodeset + path: examples/dt/nova/nova04delta/edpm/nodeset + wait_conditions: + - >- + oc -n openstack wait baremetalhosts.metal3.io edpm-compute-0 + --for=jsonpath='{.status.provisioning.state}'=available + --timeout=10m + - >- + oc -n openstack wait + osdpns openstack-edpm --for condition=SetupReady + --timeout=60m + values: + - name: edpm-nodeset-values + src_file: values.yaml + build_output: nodeset.yaml + + + - name: edpm-deployment + path: examples/dt/nova/nova04delta/edpm/deployment + wait_conditions: + - >- + oc -n openstack wait + osdpd edpm-deployment --for condition=Ready + --timeout=90m + values: + - name: edpm-deployment-values + src_file: values.yaml + build_output: deployment.yaml diff --git a/automation/vars/nvidia-vfio-passthrough-adoption.yaml b/automation/vars/nvidia-vfio-passthrough-adoption.yaml new file mode 100644 index 000000000..d6c0473ab --- /dev/null +++ b/automation/vars/nvidia-vfio-passthrough-adoption.yaml @@ -0,0 +1,29 @@ +--- +vas: + nvidia-vfio-passthrough-adoption: + stages: + - name: nncp-configuration + path: examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=5m + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - name: network-configuration + path: examples/va/nvidia-vfio-passthrough/control-plane/networking + wait_conditions: + - >- + oc -n metallb-system wait pod + -l app=metallb -l component=speaker + --for condition=Ready + --timeout=5min + values: + - name: network-values + src_file: nncp/values.yaml + build_output: networking.yaml diff --git a/automation/vars/nvidia-vfio-passthrough.yaml b/automation/vars/nvidia-vfio-passthrough.yaml new file mode 100644 index 000000000..3c4c6e9cd --- /dev/null +++ b/automation/vars/nvidia-vfio-passthrough.yaml @@ -0,0 +1,71 @@ +--- +vas: + nvidia-vfio-passthrough: + stages: + - name: nncp-configuration + path: examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp + wait_conditions: + - >- + oc -n openstack wait nncp + -l osp/nncm-config-type=standard + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured + --timeout=5m + values: + - name: network-values + src_file: values.yaml + build_output: nncp.yaml + + - name: network-configuration + path: examples/va/nvidia-vfio-passthrough/control-plane/networking + wait_conditions: + - >- + oc -n metallb-system wait pod + -l app=metallb -l component=speaker + --for condition=Ready + --timeout=5m + values: + - name: network-values + src_file: nncp/values.yaml + build_output: network.yaml + + - name: control-plane + path: examples/va/nvidia-vfio-passthrough/control-plane + wait_conditions: + - >- + oc -n openstack wait osctlplane controlplane + --for condition=Ready + --timeout=60m + values: + - name: network-values + src_file: networking/nncp/values.yaml + - name: service-values.yaml + src_file: service-values.yaml + build_output: control-plane.yaml + + - name: edpm-nodeset + path: examples/va/nvidia-vfio-passthrough/edpm/nodeset + wait_conditions: + - >- + oc -n openstack wait baremetalhosts.metal3.io edpm-compute-0 + --for=jsonpath='{.status.provisioning.state}'=available + --timeout=10m + - >- + oc -n openstack wait + osdpns openstack-edpm --for condition=SetupReady + --timeout=60m + values: + - name: edpm-nodeset-values + src_file: values.yaml + build_output: nodeset.yaml + + - name: edpm-deployment + path: examples/va/nvidia-vfio-passthrough/edpm/deployment + wait_conditions: + - >- + oc -n openstack wait + osdpd edpm-deployment --for condition=Ready + --timeout=90m + values: + - name: edpm-deployment-values + src_file: values.yaml + build_output: deployment.yaml diff --git a/dt/nova/nova04delta/README.md b/dt/nova/nova04delta/README.md new file mode 100644 index 000000000..2dc46fe03 --- /dev/null +++ b/dt/nova/nova04delta/README.md @@ -0,0 +1,12 @@ +# Deployed Topology - Nova/nova04delta + +If you are looking for information on how to deploy the nova04delta based DT, then +please see the +[README](../../../examples/dt/nova/nova04delta/README.md) in the examples +directory. + +This directory ,`dt/nova/nova04delta/`, exists so that the +[kustomization.yaml](../../../examples/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml) +in the examples directory of nova04delta topology, reference it by path as a +component. It's contents are likely uninteresting unless you want to understand +how kustomize was implemented in this repository. \ No newline at end of file diff --git a/dt/nova/nova04delta/edpm/deployment/kustomization.yaml b/dt/nova/nova04delta/edpm/deployment/kustomization.yaml new file mode 100644 index 000000000..35ccc4852 --- /dev/null +++ b/dt/nova/nova04delta/edpm/deployment/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../lib/dataplane/deployment diff --git a/dt/nova/nova04delta/edpm/nodeset/baremetalhost.yaml b/dt/nova/nova04delta/edpm/nodeset/baremetalhost.yaml new file mode 100644 index 000000000..d95ba43cb --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/baremetalhost.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: edpm-compute-0-preprovision-network-data + namespace: openstack +type: Opaque +stringData: {} +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + labels: {} + name: edpm-compute-0 + namespace: openstack diff --git a/dt/nova/nova04delta/edpm/nodeset/baremetalhost_template.yaml b/dt/nova/nova04delta/edpm/nodeset/baremetalhost_template.yaml new file mode 100644 index 000000000..717dd1579 --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/baremetalhost_template.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + labels: {} + name: _ignored_ + namespace: _replaced_ + annotations: + inspect.metal3.io: _replaced_ +spec: + architecture: x86_64 + automatedCleaningMode: metadata + bmc: + address: _replaced_ + credentialsName: _replaced_ + bootMACAddress: _replaced_ + bootMode: UEFI + rootDeviceHints: {} + online: true + preprovisioningNetworkDataName: _replaced_ diff --git a/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml b/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml new file mode 100644 index 000000000..de3d2b2e0 --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml @@ -0,0 +1,214 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +secretGenerator: + - name: baremetalset-password-secret + behavior: create + literals: + - NodeRootPassword=redhat + options: + disableNameSuffixHash: true + +components: + - ../../../../../lib/dataplane/nodeset + +resources: + - network-data-secrets.yaml + - nova_gpu.yaml + - baremetalhost.yaml + - provisioning.yaml + +patches: + - target: + kind: BareMetalHost + path: baremetalhost_template.yaml + +replacements: + # Nova compute CPU pinning customization + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nova.compute.conf + targets: + - select: + kind: ConfigMap + name: cpu-pinning-nova + fieldPaths: + - data.25-cpu-pinning-nova\.conf + options: + create: true + # Nova compute PCI passthrough customization + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.nova.pci.conf + targets: + - select: + kind: ConfigMap + name: gpu-nova + fieldPaths: + - data.03-gpu-nova\.conf + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.preProvisioned + targets: + - select: + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm + fieldPaths: + - spec.preProvisioned + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalSetTemplate + targets: + - select: + kind: OpenStackDataPlaneNodeSet + name: openstack-edpm + fieldPaths: + - spec.baremetalSetTemplate + options: + create: true + # BareMetalHost + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.labels + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - metadata.labels + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.metal3_inspection + targets: + - select: + kind: BareMetalHost + fieldPaths: + - metadata.annotations.inspect\.metal3\.io + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.bmc + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - spec.bmc + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.online + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - spec.online + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.preprovisioningNetworkDataName + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - spec.preprovisioningNetworkDataName + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.bootMACAddress + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - spec.bootMACAddress + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.rootDeviceHints + targets: + - select: + kind: BareMetalHost + name: edpm-compute-0 + fieldPaths: + - spec.rootDeviceHints + options: + create: true + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalhosts.edpm-compute-0.preprovisioningNetworkData + targets: + - select: + kind: Secret + name: edpm-compute-0-preprovision-network-data + fieldPaths: + - stringData + options: + create: true + # BMO root password for provisioned host + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.root_password + targets: + - select: + kind: Secret + name: baremetalset-password-secret + fieldPaths: + - data.NodeRootPassword + options: + create: true + # BMO networkData for provisioned host + - source: + kind: ConfigMap + name: edpm-nodeset-values + fieldPath: data.baremetalHostsNetworkData.edpm-compute-0 + targets: + - select: + kind: Secret + name: edpm-compute-0-network-data + fieldPaths: + - stringData + options: + create: true diff --git a/dt/nova/nova04delta/edpm/nodeset/network-data-secrets.yaml b/dt/nova/nova04delta/edpm/nodeset/network-data-secrets.yaml new file mode 100644 index 000000000..8ab2638f4 --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/network-data-secrets.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: edpm-compute-0-network-data + namespace: openstack +type: Opaque +stringData: {} diff --git a/dt/nova/nova04delta/edpm/nodeset/nova_gpu.yaml b/dt/nova/nova04delta/edpm/nodeset/nova_gpu.yaml new file mode 100644 index 000000000..687b1a5c9 --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/nova_gpu.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cpu-pinning-nova +data: + 25-cpu-pinning-nova.conf: _replaced_ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gpu-nova +data: + 03-gpu-nova.conf: _replaced_ +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneService +metadata: + name: nova-custom-gpu +spec: + edpmServiceType: nova + dataSources: + - configMapRef: + name: cpu-pinning-nova + - configMapRef: + name: gpu-nova + - secretRef: + name: nova-cell1-compute-config + - secretRef: + name: nova-migration-ssh-key + playbook: osp.edpm.nova + tlsCerts: + default: + contents: + - dnsnames + - ips + networks: + - ctlplane + issuer: osp-rootca-issuer-internal + caCerts: combined-ca-bundle +--- +apiVersion: dataplane.openstack.org/v1beta1 +kind: OpenStackDataPlaneService +metadata: + name: vfio-pci-bind + namespace: openstack +spec: + playbookContents: | + - name: Bind vfio-pci to devices + hosts: all + tasks: + - name: Blacklist nouveau and nvidia + become: true + ansible.builtin.copy: + dest: "/etc/modprobe.d/blacklist-nvidia.conf" + mode: "0644" + content: |- + blacklist nouveau + blacklist nvidia + options nouveau modeset=0 + force: false + register: _blacklist_nvidia + + - name: Ensure vfio and vfio-pci modules are loaded at boot + become: true + ansible.builtin.copy: + dest: /etc/modules-load.d/vfio.conf + mode: "0644" + content: | + vfio + vfio-pci + force: false + register: _load_vfio + + - name: Check if grub2-mkconfig has --update-bls-cmdline option + ansible.builtin.shell: + cmd: grub2-mkconfig --help | grep '\-\-update-bls-cmdline' + ignore_errors: true + register: check_update_bls_cmdline + changed_when: false + + - name: Regenerate initramfs + become: true + ansible.builtin.command: "{{ item }}" + loop: + - 'dracut --force' + - >- + grub2-mkconfig -o /boot/grub2/grub.cfg + {{ '--update-bls-cmdline' + if check_update_bls_cmdline.rc == 0 + else '' }} + when: (_blacklist_nvidia.changed or _load_vfio.changed) diff --git a/dt/nova/nova04delta/edpm/nodeset/provisioning.yaml b/dt/nova/nova04delta/edpm/nodeset/provisioning.yaml new file mode 100644 index 000000000..d4465abac --- /dev/null +++ b/dt/nova/nova04delta/edpm/nodeset/provisioning.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: metal3.io/v1alpha1 +kind: Provisioning +metadata: + name: provisioning-configuration +spec: + provisioningDHCPRange: "" + provisioningIP: "" + provisioningInterface: "" + provisioningMacAddresses: [] + provisioningNetworkCIDR: "" + provisioningNetwork: Disabled + watchAllNamespaces: true + virtualMediaViaExternalNetwork: true diff --git a/dt/nova/nova04delta/kustomization.yaml b/dt/nova/nova04delta/kustomization.yaml new file mode 100644 index 000000000..bb14cee2c --- /dev/null +++ b/dt/nova/nova04delta/kustomization.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../lib/control-plane + +replacements: + # Control plane customization + - source: + kind: ConfigMap + name: service-values + fieldPath: data.neutron.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.neutron.template.customServiceConfig + options: + create: true + # OVN control plane SRIOV customization + - source: + kind: ConfigMap + name: service-values + fieldPath: data.ovn.ovnController.nicMappings + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.ovn.template.ovnController.nicMappings + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.glance.default.replicas + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.glance.template.glanceAPIs.default.replicas + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.swift.enabled + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.swift.enabled + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.telemetry + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.telemetry + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.apiServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.apiServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell0.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell0.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.cell1.conductorServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.cellTemplates.cell1.conductorServiceTemplate.customServiceConfig + options: + create: true + - source: + kind: ConfigMap + name: service-values + fieldPath: data.nova.schedulerServiceTemplate.customServiceConfig + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.nova.template.schedulerServiceTemplate.customServiceConfig + options: + create: true diff --git a/dt/nova/nova04delta/namespace.yaml b/dt/nova/nova04delta/namespace.yaml new file mode 100644 index 000000000..60a6e8c42 --- /dev/null +++ b/dt/nova/nova04delta/namespace.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: builtin +kind: NamespaceTransformer +metadata: + name: _ignored_ + namespace: openstack +setRoleBindingSubjects: none +unsetOnly: true +fieldSpecs: + - path: metadata/name + kind: Namespace + create: true diff --git a/dt/nova/nova04delta/networking/kustomization.yaml b/dt/nova/nova04delta/networking/kustomization.yaml new file mode 100644 index 000000000..9265b54e7 --- /dev/null +++ b/dt/nova/nova04delta/networking/kustomization.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../lib/networking/metallb + - ../../../../lib/networking/netconfig + - ../../../../lib/networking/nad diff --git a/dt/nova/nova04delta/networking/nncp/kustomization.yaml b/dt/nova/nova04delta/networking/nncp/kustomization.yaml new file mode 100644 index 000000000..383f59913 --- /dev/null +++ b/dt/nova/nova04delta/networking/nncp/kustomization.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../lib/nncp diff --git a/examples/dt/nova/nova04delta/README.md b/examples/dt/nova/nova04delta/README.md new file mode 100644 index 000000000..29a9c5dc1 --- /dev/null +++ b/examples/dt/nova/nova04delta/README.md @@ -0,0 +1,91 @@ +# Nova GPU Passthrough (VFIO) + +This directory contains the necessary configurations to deploy OpenStack with Nova configured for full GPU device passthrough (VFIO). This setup allows entire physical GPUs on compute nodes to be passed directly to virtual machines, providing near-native performance. Nova control plane is configured for requesting PCI devices from Placement. + +## Overview + +This configuration performs the following actions: + +1. **Host Kernel Configuration**: It configures the compute node's kernel to enable IOMMU and bind specific GPUs to the `vfio-pci` driver, preventing the host from using them. +2. **Nova Scheduler Configuration**: It configures the Nova scheduler to be aware of the PCI devices available for passthrough. +3. **Nova Compute Configuration**: It whitelists the passthrough-capable GPUs in Nova on the compute nodes. + +Unlike SR-IOV or mdev (mediated device) setups, this configuration does not require installing the NVIDIA driver on the host. The driver is only installed inside the guest VM that consumes the GPU. + +## Host Configuration (`examples/dt/nova/nova04delta/edpm/nodeset/values.yaml`) + +The following parameters are crucial for host-level configuration: + +* **BareMetalHost configuration**: `baremetalhosts` section contains information required by metal3 to provision baremetal nodes. + * `bmc.address`: The IP address of the Baseboard Management Controller (BMC). + * `bootMACAddress`: The MAC address of the network interface that the node will use to PXE boot. + * `rootDeviceHints`: Hints for Metal3 to identify the root device for the OS installation. + * `preprovisioningNetworkData`: Static nmstate network config to be applied to a `BaremetalHost` via ironic-python-agent ramdisk during provisioning. The config is embedded in the ISO attached as virtual media via the BMC, so no DHCP is required. + * `baremetalHostsNetworkData`: Final nmstate network configuration for EDPM nodes. + +* `edpm_kernel_args`: Appends necessary kernel arguments for VFIO passthrough. + * `intel_iommu=on iommu=pt`: Enables the IOMMU for device passthrough. + * `vfio-pci.ids=10de:20f1`: Instructs the `vfio-pci` driver to claim the specified GPU(s) by their vendor and product IDs at boot time. The example IDs `10de:20f1` are for an NVIDIA A100 GPU. + * `rd.driver.pre=vfio-pci`: Avoids race conditions during boot by loading vfio-pci kernel module early. + +* `edpm_tuned_profile` and `edpm_tuned_isolated_cores`: These parameters configure the `tuned` service. + * `edpm_tuned_profile` is set to `cpu-partitioning-powersave` to enable CPU isolation features. + * `edpm_tuned_isolated_cores` specifies the cores to be isolated. For CPU isolation we strongly recommend using the Tuned approach rather than `isolcpus` kernel argument. + +* **VFIO-PCI Binding Service**: The `vfio-pci-bind` service in `dt/nova/nova04delta/edpm/nodeset/nova_gpu.yaml` blacklists the `nouveau` and `nvidia` kernel modules to ensure they do not interfere with the `vfio-pci` driver. The service also regenerates the initramfs and grub configuration to apply these changes. A reboot is required for these changes to take effect. + +## Nova Configuration + +A count of `X` PCI devices may be requested through `"pci_passthrough:alias"="nvidia_a2:X"` flavor extra specs: +``` +$ openstack --os-compute-api=2.86 flavor set --property "pci_passthrough:alias"="nvidia_a2:1" device_passthrough +``` + +### Control Plane (`examples/dt/nova/nova04delta/control-plane/service-values.yaml`) + +See [README.md](control-plane/README.md) for deployment instructions. +There are most essential configuration values to define: + +* `[pci]alias`: Creates an alias for a specific GPU type. This allows users to request a GPU by a friendly name (e.g., `nvidia_a2`) when creating a VM. This configuration should match the configuration found on the compute nodes. + ```yaml + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + ``` +* `[filter_scheduler]pci_in_placement`: Enables PCI in Placement. It should only be enabled after all the computes in the system become configured to report PCI inventory in Placement via enabling `[pci]report_in_placement` in EDPM nodesets configuration. However, this order must be ensured during major upgrades only, where the dataplane deployment to upate EDPM computes configurataion must come before reconfiguring control plane resources. +* `device_type` in the alias is dependent on the actual hardware: + * `type-PF`: The device supports SR-IOV and is the parent or root device. + * `type-VF`: The device is a child device of a device that supports SR-IOV. + * `type-PCI`: The device does not support SR-IOV. + +### Compute Node (`examples/dt/nova/nova04delta/edpm/nodeset/values.yaml`) + +See [dataplane section](data-plane.md) for deployment instructions. +There are most essential configuration values to define: + +* `[pci]report_in_placement`: Required for PCI in placement to work. +* `[pci]device_spec`: Whitelists the physical GPUs that are available for passthrough. You must create a `device_spec` entry for each physical GPU you want to make available. For example: + ```yaml + nova: + pci: + conf: | + [pci] + device_spec = { "vendor_id":"10de", "product_id":"20f1", "address": "0000:04:00.0" } + device_spec = { "vendor_id":"10de", "product_id":"20f1", "address": "0000:82:00.0" } + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + ``` + +In addition to PCI device configuration, the `nova.compute.conf` section includes parameters for resource management on the compute node: + +* `[DEFAULT]reserved_host_memory_mb`: Specifies the amount of memory (in megabytes) to reserve for the host operating system and other non-OpenStack services. This memory will not be available for allocation to virtual machines. +* `[compute]cpu_shared_set`: A list of physical CPUs that are available for host processes and for virtual machines that do not have dedicated CPUs (i.e., unpinned VMs). These should be the CPUs that are **not** isolated by `edpm_tuned_isolated_cores`. +* `[compute]cpu_dedicated_set`: A list of physical CPUs that are exclusively reserved for virtual machines with dedicated CPU pinning policies. To ensure performance isolation, this list should correspond directly to the CPUs isolated using `edpm_tuned_isolated_cores` parameter. +* `[DEFAULT]reserved_huge_pages`: Defines the number and size of huge pages to reserve for the host, making them unavailable for guest VMs. This configuration works in conjunction with the `hugepages` and `hugepagesz` kernel arguments, which define the total pool of huge pages on the host. + +**Note**: In a full device passthrough scenario, the `[devices]enabled_vgpu_types` option in Nova's configuration is not used. This option is specific to mediated device (mdev) configurations. + +## Guest VM + +To use the passthrough GPU, the guest operating system inside the VM must have the appropriate native NVIDIA driver installed. You will need a standard NVIDIA driver. Do not use vGPU-enabled guest drivers. The GPU will appear as a physical PCI device within the guest. diff --git a/examples/dt/nova/nova04delta/control-plane/README.md b/examples/dt/nova/nova04delta/control-plane/README.md new file mode 100644 index 000000000..1e8016372 --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/README.md @@ -0,0 +1,54 @@ +# Configuring networking and deploy the OpenStack control plane + +## Assumptions + +- A storage class called `local-storage` should already exist. + +## Initialize + +Switch to the "openstack" namespace +``` +oc project openstack +``` +Change to the nova/nova04delta directory +``` +cd architecture/examples/dt/nova/nova04delta +``` +Edit the [nncp/values.yaml](networking/nncp/values.yaml) and +[service-values.yaml](service-values.yaml) files to suit +your environment. +``` +vi networking/nncp/values.yaml +vi service-values.yaml +``` + +## Apply node network configuration + +Generate the node network configuration +``` +kustomize build nncp > nncp.yaml +``` +Apply the NNCP CRs +``` +oc apply -f nncp.yaml +``` +Wait for NNCPs to be available +``` +oc wait nncp -l osp/nncm-config-type=standard --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured --timeout=300s +``` + +## Apply networking and control-plane configuration + +Generate the control-plane and networking CRs. +``` +kustomize build > control-plane.yaml +``` +Apply the CRs +``` +oc apply -f control-plane.yaml +``` + +Wait for control plane to be available +``` +oc wait osctlplane controlplane --for condition=Ready --timeout=600s +``` diff --git a/examples/dt/nova/nova04delta/control-plane/kustomization.yaml b/examples/dt/nova/nova04delta/control-plane/kustomization.yaml new file mode 100644 index 000000000..dc40ae056 --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../dt/nova/nova04delta/ + +resources: + - networking/nncp/values.yaml + - service-values.yaml diff --git a/examples/dt/nova/nova04delta/control-plane/networking/kustomization.yaml b/examples/dt/nova/nova04delta/control-plane/networking/kustomization.yaml new file mode 100644 index 000000000..052c6f42a --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/networking/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../dt/nova/nova04delta/networking/nncp + - ../../../../../../dt/nova/nova04delta/networking + +resources: + - nncp/values.yaml diff --git a/examples/dt/nova/nova04delta/control-plane/networking/nncp/.gitignore b/examples/dt/nova/nova04delta/control-plane/networking/nncp/.gitignore new file mode 100644 index 000000000..4c57ec1a7 --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/networking/nncp/.gitignore @@ -0,0 +1,2 @@ +# Generated files +nncp.yaml diff --git a/examples/dt/nova/nova04delta/control-plane/networking/nncp/kustomization.yaml b/examples/dt/nova/nova04delta/control-plane/networking/nncp/kustomization.yaml new file mode 100644 index 000000000..fb8a75db5 --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/networking/nncp/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../../dt/nova/nova04delta/networking/nncp + +resources: + - values.yaml diff --git a/examples/dt/nova/nova04delta/control-plane/networking/nncp/values.yaml b/examples/dt/nova/nova04delta/control-plane/networking/nncp/values.yaml new file mode 100644 index 000000000..94ff59cf1 --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/networking/nncp/values.yaml @@ -0,0 +1,199 @@ +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-values + annotations: + config.kubernetes.io/local-config: "true" +data: + # nodes + node_0: + name: ostest-master-0 + internalapi_ip: 172.17.0.5 + tenant_ip: 172.19.0.5 + ctlplane_ip: 192.168.122.10 + storage_ip: 172.18.0.5 + node_1: + name: ostest-master-1 + internalapi_ip: 172.17.0.6 + tenant_ip: 172.19.0.6 + ctlplane_ip: 192.168.122.11 + storage_ip: 172.18.0.6 + node_2: + name: ostest-master-2 + internalapi_ip: 172.17.0.7 + tenant_ip: 172.19.0.7 + ctlplane_ip: 192.168.122.12 + storage_ip: 172.18.0.7 + + # networks + ctlplane: + dnsDomain: ctlplane.example.com + subnets: + - allocationRanges: + - end: 192.168.122.120 + start: 192.168.122.100 + - end: 192.168.122.200 + start: 192.168.122.150 + cidr: 192.168.122.0/24 + gateway: 192.168.122.1 + name: subnet1 + prefix-length: 24 + iface: enp7s0 + mtu: 1500 + lb_addresses: + - 192.168.122.80-192.168.122.90 + endpoint_annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "ctlplane", + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "192.168.122.0/24", + "range_start": "192.168.122.30", + "range_end": "192.168.122.70" + } + } + internalapi: + dnsDomain: internalapi.example.com + subnets: + - allocationRanges: + - end: 172.17.0.250 + start: 172.17.0.100 + cidr: 172.17.0.0/24 + name: subnet1 + vlan: 20 + mtu: 1500 + prefix-length: 24 + iface: internalapi + vlan: 20 + base_iface: enp7s0 + lb_addresses: + - 172.17.0.80-172.17.0.90 + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "internalapi", + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "range_start": "172.17.0.30", + "range_end": "172.17.0.70" + } + } + storage: + dnsDomain: storage.example.com + subnets: + - allocationRanges: + - end: 172.18.0.250 + start: 172.18.0.100 + cidr: 172.18.0.0/24 + name: subnet1 + vlan: 21 + mtu: 1500 + prefix-length: 24 + iface: storage + vlan: 21 + base_iface: enp7s0 + lb_addresses: + - 172.18.0.80-172.18.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.30", + "range_end": "172.18.0.70" + } + } + tenant: + dnsDomain: tenant.example.com + subnets: + - allocationRanges: + - end: 172.19.0.250 + start: 172.19.0.100 + cidr: 172.19.0.0/24 + name: subnet1 + vlan: 22 + mtu: 1500 + prefix-length: 24 + iface: tenant + vlan: 22 + base_iface: enp7s0 + lb_addresses: + - 172.19.0.80-172.19.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "tenant", + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "172.19.0.0/24", + "range_start": "172.19.0.30", + "range_end": "172.19.0.70" + } + } + external: + dnsDomain: external.example.com + subnets: + - allocationRanges: + - end: 10.0.0.250 + start: 10.0.0.100 + cidr: 10.0.0.0/24 + gateway: 10.0.0.1 + name: subnet1 + mtu: 1500 + datacentre: + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "datacentre", + "type": "bridge", + "bridge": "ospbr", + "ipam": {} + } + + dns-resolver: + config: + server: + - 192.168.122.1 + search: [] + options: + - key: server + values: + - 192.168.122.1 + + routes: + config: [] + + rabbitmq: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.85 + rabbitmq-cell1: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.86 + + lbServiceType: LoadBalancer + storageClass: local-storage + bridgeName: ospbr diff --git a/examples/dt/nova/nova04delta/control-plane/service-values.yaml b/examples/dt/nova/nova04delta/control-plane/service-values.yaml new file mode 100644 index 000000000..c19288fcb --- /dev/null +++ b/examples/dt/nova/nova04delta/control-plane/service-values.yaml @@ -0,0 +1,64 @@ +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: false + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + ovn: + ovnController: + nicMappings: + datacentre: ospbr + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:swift + [glance_store] + default_backend = default_backend + [default_backend] + swift_store_create_container_on_put = True + swift_store_auth_version = 3 + swift_store_auth_address = {{ .KeystoneInternalURL }} + swift_store_endpoint_type = internalURL + swift_store_user = service:glance + swift_store_key = {{ .ServicePassword }} + default: + replicas: 1 + swift: + enabled: true + telemetry: + enabled: true + template: + ceilometer: + enabled: true + logging: + enabled: false + # cloNamespace: "" + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True diff --git a/examples/dt/nova/nova04delta/data-plane.md b/examples/dt/nova/nova04delta/data-plane.md new file mode 100644 index 000000000..28352d506 --- /dev/null +++ b/examples/dt/nova/nova04delta/data-plane.md @@ -0,0 +1,87 @@ +# Configuring and deploying the dataplane + +## Assumptions + +- The [control plane](control-plane/README.md) has been created and successfully deployed + +## Initialize + +Switch to the "openstack" namespace +``` +oc project openstack +``` +Change to the nova/nova04delta directory +``` +cd architecture/examples/dt/nova/nova04delta +``` +Edit the [edpm/nodeset/values.yaml](edpm/nodeset/values.yaml) +file to suit your environment. +``` +vi edpm/nodeset/values.yaml +``` +In `nodeset/values.yaml`, pay special attention to the `baremetalhosts` section. You will need to provide details for each of your baremetal compute nodes, including: +- `bmc.address`: The IP address of the Baseboard Management Controller (BMC). +- `bootMACAddress`: The MAC address of the network interface that the node will use to PXE boot. +- Other parameters as described in the main [README.md](README.md). + +Additionally, you need to provide SSH keys for Nova live migration. The following keys in `edpm/nodeset/values.yaml` must be populated with base64 encoded values: +- `nova.migration.ssh_keys.private` +- `nova.migration.ssh_keys.public` + +You can encode your keys using the `base64` command, for example: `cat ~/.ssh/id_rsa | base64 -w0`. +``` + +### Create the BareMetalHost CRs + +Also, ensure the `bmhLabelSelector` in `baremetalSetTemplate` matches the labels you have defined for your `baremetalhosts`. For example, if you use `app: openstack`, your `baremetalhosts` should have a corresponding label. + +Before applying the nodeset configuration, you must also create the `bmc-secret` secret that contains the BMC credentials. You can create it with the following command: +``` +oc create secret generic bmc-secret --from-literal=username=CHANGEME --from-literal=password=CHANGEME +``` + +Generate the dataplane nodeset CR, which includes the BareMetalHost definitions. +``` +kustomize build nodeset > dataplane-nodeset.yaml +``` + +Apply the CR to create the BareMetalHost and the nodeset. +``` +oc apply -f dataplane-nodeset.yaml +``` + +Wait for the BareMetalHosts to become available. You can monitor the status with: +``` +oc get bmh -w +``` +The state should change to `available`. + +### Configure and deploy the dataplane + +Edit `edpm/deployment/values.yaml` if needed. +``` +vi edpm/deployment/values.yaml +``` + +Build it with kustomize. +``` +kustomize build deployment > dataplane-deployment.yaml +``` + +Wait for dataplane nodeset setup to finish. +``` +oc wait osdpns openstack-edpm --for condition=SetupReady --timeout=600s +``` + +Start the deployment +``` +oc apply -f dataplane-deployment.yaml +``` + +Wait for dataplane deployment to finish +``` +oc wait osdpd edpm-deployment --for condition=Ready --timeout=40m +``` + +After the Nvidia drivers have been blacklisted on the EDPM nodes, and kernel args updated, edpm hosts will be +automatically rebooted. diff --git a/examples/dt/nova/nova04delta/edpm/deployment/kustomization.yaml b/examples/dt/nova/nova04delta/edpm/deployment/kustomization.yaml new file mode 100644 index 000000000..0e3d66aad --- /dev/null +++ b/examples/dt/nova/nova04delta/edpm/deployment/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../dt/nova/nova04delta/edpm/deployment + +resources: + - values.yaml diff --git a/examples/dt/nova/nova04delta/edpm/deployment/values.yaml b/examples/dt/nova/nova04delta/edpm/deployment/values.yaml new file mode 100644 index 000000000..7f371b2cc --- /dev/null +++ b/examples/dt/nova/nova04delta/edpm/deployment/values.yaml @@ -0,0 +1,10 @@ +# yamllint disable rule:line-length +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-deployment-values + annotations: + config.kubernetes.io/local-config: "true" +data: {} diff --git a/examples/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml b/examples/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml new file mode 100644 index 000000000..d087ad455 --- /dev/null +++ b/examples/dt/nova/nova04delta/edpm/nodeset/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../dt/nova/nova04delta/edpm/nodeset/ + +resources: + - values.yaml diff --git a/examples/dt/nova/nova04delta/edpm/nodeset/values.yaml b/examples/dt/nova/nova04delta/edpm/nodeset/values.yaml new file mode 100644 index 000000000..8ffa3800a --- /dev/null +++ b/examples/dt/nova/nova04delta/edpm/nodeset/values.yaml @@ -0,0 +1,220 @@ +# yamllint disable rule:line-length +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" +data: + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to baremetalset-password-secret.data.NodeRootPassword + root_password: cmVkaGF0Cg== + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to OpenStackDataPlaneNodeSet.spec.preProvisioned + preProvisioned: false + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to BareMetalHost.metadata.annotations.inspect.metal3.io + # Uncomment to allow HW inspection of the node, for example after a blank RAID device was manually created in BIOS. + metal3_inspection: "disabled" + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to BareMetalHost resource properties + baremetalhosts: + edpm-compute-0: + online: true + labels: + nodeName: edpm-compute-0 + bmc: + address: CHANGEME + # NOTE: in CI systems, we use credentials_file from cifmw_baremetal_hosts. + # credentialsName: bmc-secret + disableCertificateVerification: true + bootMACAddress: CHANGEME + rootDeviceHints: + deviceName: /dev/vda + preprovisioningNetworkDataName: edpm-compute-0-preprovision-network-data + preprovisioningNetworkData: + nmstate: | + CHANGEME + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to edpm-compute-0-network-data Secret + # Only provide it if it should not be auto-generated by EDPM operator + baremetalHostsNetworkData: + edpm-compute-0: + networkData: | + CHANGEME + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to OpenStackDataPlaneNodeSet.spec.baremetalSetTemplate + baremetalSetTemplate: + ctlplaneInterface: eno2 # CHANGEME + cloudUserName: cloud-admin + bmhNamespace: openstack + # NOTE: provisioningInterface should not be provided when using virtual-media + bmhLabelSelector: + app: openstack # CHANGEME + passwordSecret: + name: baremetalset-password-secret + namespace: openstack + # Mapped in lib/dataplane/nodeset/kustomization.yaml to the dataplane-ansible-ssh-private-key-secret + ssh_keys: + # Authorized keys that will have access to the dataplane computes via SSH + authorized: CHANGEME + # The private key that will have access to the dataplane computes via SSH + private: CHANGEME2 + # The public key that will have access to the dataplane computes via SSH + public: CHANGEME3 + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + # see https://access.redhat.com/solutions/253273 + edpm_bootstrap_command: | + echo CHANGEME + rhc_release: 9.4 + rhc_repositories: + - {name: "*", state: disabled} + - {name: "CHANGEME"} + edpm_bootstrap_release_version_package: [] + timesync_ntp_servers: + - hostname: pool.ntp.org + # GPU passthrough settings for a PCI Vendor and Device ID pair. + # Here, 10de represents NVIDIA Corporation, and 20f1 corresponds to the GA100 [A100 PCIe 40GB] GPU, according to Device Hunt + # https://devicehunt.com/view/type/pci/vendor/10DE/device/20F1 + # TODO(bogdando): add noveau.blacklist=1 nvidia.blacklist=1 nvidia-drm.blacklist=1 nvidia-modeset.blacklist=1 nvidia-uvm.blacklist=1 rd.driver.blacklist=nouveau,nvidia,nvidia-drm,nvidia-modeset,nvidia-uvm ? + edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt vfio-pci.ids=10de:20f1 rd.driver.pre=vfio-pci" + edpm_reboot_strategy: force + edpm_tuned_profile: "cpu-partitioning-powersave" + edpm_tuned_isolated_cores: "4-23,28-47" + # edpm_network_config + # These vars are edpm_network_config role vars + edpm_network_config_hide_sensitive_logs: false + edpm_ovn_bridge_mappings: + - datacentre:br-ex + # Nics to MAC addresses mappings for the dataplane nodes + edpm_network_config_os_net_config_mappings: + edpm-compute-0: + nic1: CHANGEME + edpm_network_config_template: | + --- + # CHANGEME + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: ovs_bridge + name: {{ neutron_physical_bridge_name }} + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + # force the MAC address of the bridge to this interface + primary: true + {% for network in nodeset_networks %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: + {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endfor %} + + # These vars are for the network config templates themselves and are + # considered EDPM network defaults. + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: eth0 + # edpm_nodes_validation + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + edpm_network_config_nmstate: false + edpm_network_config_update: false + dns_search_domains: [] + gather_facts: false + # edpm firewall, change the allowed CIDR if needed + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - 192.168.122.0/24 + networks: # mapped to spec.nodeTemplate.networks + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + nodes: # mapped to spec.nodes + edpm-compute-0: + hostName: edpm-compute-0 + bmhLabelSelector: + nodeName: edpm-compute-0 + networkData: + name: edpm-compute-0-network-data + namespace: openstack + # must be defined if custom networkData is used for BMO, and there is a static IP defined for ctplane interface + networks: + - name: ctlplane + subnetName: subnet1 + defaultRoute: true + fixedIP: 192.168.122.100 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + services: # mapped to spec.services + - vfio-pci-bind + - bootstrap + - download-cache + - configure-network + - validate-network + - install-os + - configure-os + - ssh-known-hosts + - run-os + - reboot-os + - install-certs + - libvirt + - ovn + - neutron-ovn + - nova-custom-gpu + - neutron-metadata + - telemetry + nova: + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to cpu-pinning-nova ConfigMap + compute: + conf: | + # CHANGEME + [DEFAULT] + reserved_host_memory_mb = 4096 + reserved_huge_pages = node:0,size:4,count:524160 + reserved_huge_pages = node:1,size:4,count:524160 + [compute] + cpu_shared_set = 0-3,24-27 + cpu_dedicated_set = 8-23,32-47 + # Mapped in lib/dataplane/nodeset/kustomization.yaml to the nova-migration-ssh-key secret + migration: + ssh_keys: + private: CHANGEME4 + public: CHANGEME5 + # Mapped in dt/nova/nova04delta/edpm/nodeset/kustomization.yaml to gpu-nova ConfigMap + pci: + # You will have one device_spec line for each physical GPU you want to make available for passthrough: + # "vendor_id": The vendor ID of the GPU manufacturer (e.g., "10de" for NVIDIA). + # "product_id": The product ID of the specific GPU model (e.g., "20f1" for an NVIDIA A100). + # "address": The PCI address of the GPU on the host machine. You can find this using `lspci | grep -i nvidia`. + # "physical_network": This is used for SR-IOV networking passthrough. For a GPU, you can typically leave this as null. + # "alias": must match the alias configuration in the API service-values.yaml + conf: | + # CHANGEME + [pci] + device_spec = {"vendor_id":"10de", "product_id":"20f1", "address": "CHANGEME" } + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + report_in_placement = True diff --git a/examples/va/nvidia-vfio-passthrough/.gitignore b/examples/va/nvidia-vfio-passthrough/.gitignore new file mode 100644 index 000000000..1e82fc7de --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/.gitignore @@ -0,0 +1 @@ +*.yaml diff --git a/examples/va/nvidia-vfio-passthrough/README.md b/examples/va/nvidia-vfio-passthrough/README.md new file mode 100644 index 000000000..0f2f40a15 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/README.md @@ -0,0 +1,85 @@ +# Validated Architecture - Nvidia GPU Passthrough (VFIO) + +This document describes the CR's and deployment workflow to create an +environment with EDPM Compute Nodes capable of supplying Nvidia GPUs via +passthrough (VFIO). This setup allows entire physical GPUs on compute nodes to +be passed directly to virtual machines, providing near-native performance. +EDPM Compute node is provisioned by BMO via virtual-media. + +## Purpose + +This topology is used for e2e verification of environments that provide Nvidia GPU +passthrough and to confirm guests are able to take advantage of the resource +correctly. It should be noted that this type of deployment cannot be simulated +with nested virtualization and requires real baremetal hosts. + +## Environment + +### Nova + +Nova control plane is configured for requesting PCI devices from Placement. + +### Guest VM + +To use the passthrough GPU, the guest operating system inside the VM must have +the appropriate NVIDIA driver installed. You will need a standard guest NVIDIA +driver or advanced drivers, like GRID. The GPU will appear as a physical +PCI device within the guest. + +### Host Configuration (`examples/va/nvidia-vfio-passthrough/edpm/nodeset/values.yaml`) + +The following parameters are crucial for host-level configuration: + +* **BareMetalHost configuration**: `baremetalhosts` section contains information required by metal3 to provision baremetal nodes. + * `bmc.address`: The IP address of the Baseboard Management Controller (BMC). + * `bootMACAddress`: The MAC address of the network interface that the node will use to PXE boot. + * `rootDeviceHints`: Hints for metal3 to identify the root device for the OS installation. + * `preprovisioningNetworkData`: Static nmstate network config to be applied to a `BaremetalHost` via ironic-python-agent ramdisk during provisioning. The config is embedded in the ISO attached as virtual media via the BMC, so no DHCP is required. + * `baremetalHostsNetworkData`: Final nmstate network configuration for EDPM nodes. + +* `edpm_kernel_args`: Appends necessary kernel arguments for VFIO passthrough. + * `intel_iommu=on iommu=pt`: Enables the IOMMU for device passthrough. + * `vfio-pci.ids=10de:20f1`: Instructs the `vfio-pci` driver to claim the specified GPU(s) by their vendor and product IDs at boot time. The example IDs `10de:20f1` are for an NVIDIA A100 GPU. + * `rd.driver.pre=vfio-pci`: Avoids race conditions during boot by loading vfio-pci kernel module early. + +* `edpm_tuned_profile` and `edpm_tuned_isolated_cores`: These parameters configure the `tuned` service. + * `edpm_tuned_profile` is set to `cpu-partitioning-powersave` to enable CPU isolation features. + * `edpm_tuned_isolated_cores` specifies the cores to be isolated. For CPU isolation we strongly recommend using the Tuned approach rather than `isolcpus` kernel argument. + +* **VFIO-PCI Binding Service**: The `vfio-pci-bind` service in `va/nvidia-vfio-passthrough/edpm/nodeset/nova_gpu.yaml` blacklists the `nouveau` and `nvidia` kernel modules to ensure they do not interfere with the `vfio-pci` driver. The service also regenerates the initramfs and grub configuration to apply these changes. A reboot is required for these changes to take effect. + +### Nodes + +| Role | Machine Type | Count | +| --------------------------- | ------------ | ----- | +| Compact OpenShift | vm* | 3 | +| OpenStack Baremetal Compute | Baremetal | 1 | + +\* OCP setup is not covered in this VA, but in the parent nova04delta DT. + +#### VLAN networks in RH OSP + +| Name | Type | CIDR | +| ----------- | ----------- | ----------------- | +| ctlplane | untagged | 192.168.122.0/24 | +| internalapi | VLAN tagged | 172.17.0.0/24 | +| storage | VLAN tagged | 172.18.0.0/24 | +| storagemgmt | VLAN tagged | 172.20.0.0/24 | +| tenant | VLAN tagged | 172.19.0.0/24 | +| external | untagged | 10.0.0.0/24 | + +A network attach definition is not provided for storagemgmt because only +RHEL EDPM nodes are going to use it, while control plane pods on OpenShift do +not need it. + +Also the base DT configures the default gateway for external network instead of ctlplane. +It is attached to EDPM and CP to transfer virtual media traffic during BMO setup. + +## Stages +All stages must be executed in the order listed below. Everything is required unless otherwise indicated. + +1. [Install the OpenStack K8S operators and their dependencies](../../common/) +2. [Configuring networking and deploy the OpenStack control plane](control-plane/README.md) +3. [Configure and deploy the initial dataplane](data-plane-pre.md) + +Note that EDPM host will be rebooted to apply required changes to kernel arguments. \ No newline at end of file diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/README.md b/examples/va/nvidia-vfio-passthrough/control-plane/README.md new file mode 100644 index 000000000..13e446196 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/README.md @@ -0,0 +1,69 @@ +# Configuring networking and deploy the OpenStack control plane + +## Assumptions + +- A storage class called `local-storage` should already exist. + +## Initialize + +Switch to the "openstack" namespace +```bash +oc project openstack +``` + +Change to nvidia-vfio-passthrough directory +```bash +cd architecture/examples/va/nvidia-vfio-passthrough +``` + +# Apply the required node network configurations. +```bash +vi control-plane/networking/nncp/values.yaml +``` + +```bash +# Change the Node Network Configuration folder. +pushd control-plane/networking/nncp + +# Generate the configuration +kustomize build > nncp.yaml + +# Apply the generated configuration +oc apply -f nncp.yaml + +# Wait till the network configuration is applied. +oc wait nncp -l osp/nncm-config-type=standard \ + --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured \ + --timeout=300s + +# change the working directory +popd +``` +# Apply remaining networking configuration + +Generate the remaining networking configuration +```bash +kustomize build control-plane/networking > networking.yaml +``` +Apply the networking CRs +```bash +oc apply -f networking.yaml +``` + +# Generate and apply the control-plane configurations. + +```bash +# Navigate to control-panel +pushd control-plane + +# Generate the CR +kustomize build > control-plane.yaml + +# Apply the CR +oc apply -f control-plane.yaml + +# Wait till the control plane is ready. +oc wait openstackcontrolplane --for condition=Ready --timeout=600s + +# change the work_dir +popd diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/kustomization.yaml b/examples/va/nvidia-vfio-passthrough/control-plane/kustomization.yaml new file mode 100644 index 000000000..b32478036 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/kustomization.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../va/nvidia-vfio-passthrough + +resources: + - networking/nncp/values.yaml + - service-values.yaml diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/networking/kustomization.yaml b/examples/va/nvidia-vfio-passthrough/control-plane/networking/kustomization.yaml new file mode 100644 index 000000000..a0acfd6b0 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/networking/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../va/nvidia-vfio-passthrough/networking + +resources: + - nncp/values.yaml diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/kustomization.yaml b/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/kustomization.yaml new file mode 100644 index 000000000..ea0ea05c0 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../../va/nvidia-vfio-passthrough/networking/nncp + +resources: + - values.yaml diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/values.yaml b/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/values.yaml new file mode 100644 index 000000000..ddd922f50 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp/values.yaml @@ -0,0 +1,209 @@ +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: network-values + annotations: + config.kubernetes.io/local-config: "true" +data: + # nodes + node_0: + name: ostest-master-0 + internalapi_ip: 172.17.0.5 + tenant_ip: 172.19.0.5 + ctlplane_ip: 192.168.122.10 + storage_ip: 172.18.0.5 + node_1: + name: ostest-master-1 + internalapi_ip: 172.17.0.6 + tenant_ip: 172.19.0.6 + ctlplane_ip: 192.168.122.11 + storage_ip: 172.18.0.6 + node_2: + name: ostest-master-2 + internalapi_ip: 172.17.0.7 + tenant_ip: 172.19.0.7 + ctlplane_ip: 192.168.122.12 + storage_ip: 172.18.0.7 + + # networks + ctlplane: + dnsDomain: ctlplane.example.com + subnets: + - allocationRanges: + - end: 192.168.122.120 + start: 192.168.122.100 + - end: 192.168.122.200 + start: 192.168.122.150 + cidr: 192.168.122.0/24 + gateway: 192.168.122.1 + name: subnet1 + prefix-length: 24 + iface: enp7s0 + mtu: 1500 + lb_addresses: + - 192.168.122.80-192.168.122.90 + endpoint_annotations: + metallb.universe.tf/address-pool: ctlplane + metallb.universe.tf/allow-shared-ip: ctlplane + metallb.universe.tf/loadBalancerIPs: 192.168.122.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "ctlplane", + "type": "macvlan", + "master": "ospbr", + "ipam": { + "type": "whereabouts", + "range": "192.168.122.0/24", + "range_start": "192.168.122.30", + "range_end": "192.168.122.70" + } + } + internalapi: + dnsDomain: internalapi.example.com + subnets: + - allocationRanges: + - end: 172.17.0.250 + start: 172.17.0.100 + cidr: 172.17.0.0/24 + name: subnet1 + vlan: 20 + mtu: 1500 + prefix-length: 24 + iface: internalapi + vlan: 20 + base_iface: enp7s0 + lb_addresses: + - 172.17.0.80-172.17.0.90 + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/allow-shared-ip: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.80 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "internalapi", + "type": "macvlan", + "master": "internalapi", + "ipam": { + "type": "whereabouts", + "range": "172.17.0.0/24", + "range_start": "172.17.0.30", + "range_end": "172.17.0.70" + } + } + storage: + dnsDomain: storage.example.com + subnets: + - allocationRanges: + - end: 172.18.0.250 + start: 172.18.0.100 + cidr: 172.18.0.0/24 + name: subnet1 + vlan: 21 + mtu: 1500 + prefix-length: 24 + iface: storage + vlan: 21 + base_iface: enp7s0 + lb_addresses: + - 172.18.0.80-172.18.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "storage", + "type": "macvlan", + "master": "storage", + "ipam": { + "type": "whereabouts", + "range": "172.18.0.0/24", + "range_start": "172.18.0.30", + "range_end": "172.18.0.70" + } + } + storagemgmt: # used on RHEL, not used on OpenShift + dnsDomain: storagemgmt.example.com + subnets: + - allocationRanges: + - end: 172.20.0.250 + start: 172.20.0.100 + cidr: 172.20.0.0/24 + name: subnet1 + vlan: 23 + mtu: 1500 + tenant: + dnsDomain: tenant.example.com + subnets: + - allocationRanges: + - end: 172.19.0.250 + start: 172.19.0.100 + cidr: 172.19.0.0/24 + name: subnet1 + vlan: 22 + mtu: 1500 + prefix-length: 24 + iface: tenant + vlan: 22 + base_iface: enp7s0 + lb_addresses: + - 172.19.0.80-172.19.0.90 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "tenant", + "type": "macvlan", + "master": "tenant", + "ipam": { + "type": "whereabouts", + "range": "172.19.0.0/24", + "range_start": "172.19.0.30", + "range_end": "172.19.0.70" + } + } + external: + dnsDomain: external.example.com + subnets: + - allocationRanges: + - end: 10.0.0.250 + start: 10.0.0.100 + cidr: 10.0.0.0/24 + gateway: 10.0.0.1 + name: subnet1 + mtu: 1500 + datacentre: + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "datacentre", + "type": "bridge", + "bridge": "ospbr", + "ipam": {} + } + + dns-resolver: + config: + server: + - 192.168.122.1 + search: [] + options: + - key: server + values: + - 192.168.122.1 + + routes: + config: [] + + rabbitmq: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.85 + rabbitmq-cell1: + endpoint_annotations: + metallb.universe.tf/address-pool: internalapi + metallb.universe.tf/loadBalancerIPs: 172.17.0.86 + + lbServiceType: LoadBalancer + storageClass: local-storage + bridgeName: ospbr diff --git a/examples/va/nvidia-vfio-passthrough/control-plane/service-values.yaml b/examples/va/nvidia-vfio-passthrough/control-plane/service-values.yaml new file mode 100644 index 000000000..f0a94ae0b --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/control-plane/service-values.yaml @@ -0,0 +1,61 @@ +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: service-values + annotations: + config.kubernetes.io/local-config: "true" +data: + preserveJobs: false + neutron: + customServiceConfig: | + [ml2] + mechanism_drivers = ovn + ovn: + ovnController: + nicMappings: + datacentre: ospbr + glance: + customServiceConfig: | + [DEFAULT] + enabled_backends = default_backend:swift + [glance_store] + default_backend = default_backend + [default_backend] + swift_store_create_container_on_put = True + swift_store_auth_version = 3 + swift_store_auth_address = {{ .KeystoneInternalURL }} + swift_store_endpoint_type = internalURL + swift_store_user = service:glance + swift_store_key = {{ .ServicePassword }} + default: + replicas: 1 + swift: + enabled: true + telemetry: + enabled: true + template: + ceilometer: + enabled: true + nova: + apiServiceTemplate: + customServiceConfig: | + [pci] + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + [filter_scheduler] + pci_in_placement = True + cell0: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + cell1: + conductorServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True + schedulerServiceTemplate: + customServiceConfig: | + [filter_scheduler] + pci_in_placement = True diff --git a/examples/va/nvidia-vfio-passthrough/data-plane-pre.md b/examples/va/nvidia-vfio-passthrough/data-plane-pre.md new file mode 100644 index 000000000..86756e7d1 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/data-plane-pre.md @@ -0,0 +1,80 @@ +# Configuring and deploying the dataplane + +## Assumptions + +- The [control plane](control-plane/README.md) has been created and successfully deployed + +## Initialize + +Switch to the "openstack" namespace +``` +oc project openstack +``` +Change to the nvidia-vfio-passthrough directory +``` +cd architecture/examples/va/nvidia-vfio-passthrough +``` + +### Create the BareMetalHost CRs + +Edit the `edpm/nodeset/values.yaml` file to suit your environment. Pay special attention to the `baremetalhosts` section, where you will need to provide details for each of your baremetal compute nodes, including: +- `bmc.address`: The IP address of the Baseboard Management Controller (BMC). +- `bootMACAddress`: The MAC address of the network interface that the node will use to PXE boot. +- Other parameters as described in the main [README.md](README.md). + +Additionally, you need to provide SSH keys for Nova live migration. The following keys in `edpm/nodeset/values.yaml` must be populated with base64 encoded values: +- `nova.migration.ssh_keys.private` +- `nova.migration.ssh_keys.public` + +You can encode your keys using the `base64` command, for example: `cat ~/.ssh/id_rsa | base64 -w0`. + +Also, ensure the `bmhLabelSelector` in `baremetalSetTemplate` matches the labels you have defined for your `baremetalhosts`. For example, if you use `app: openstack`, your `baremetalhosts` should have a corresponding label. + +Before applying the nodeset configuration, you must also create the `bmc-secret` secret that contains the BMC credentials. You can create it with the following command: +``` +oc create secret generic bmc-secret --from-literal=username=CHANGEME --from-literal=password=CHANGEME +``` + +Generate the dataplane nodeset CR, which includes the BareMetalHost definitions. +``` +kustomize build edpm/nodeset > dataplane-nodeset.yaml +``` +Apply the CRs to create the BareMetalHosts and the nodeset. +``` +oc apply -f dataplane-nodeset.yaml +``` + +Wait for the BareMetalHosts to become available. You can monitor the status with: +``` +oc get bmh -w +``` +The state should change to `available`. + +### Configure and deploy the dataplane + +Edit `edpm/deployment/values.yaml` if needed. +``` +vi edpm/deployment/values.yaml +``` +Generate the dataplane deployment CR. +``` +kustomize build edpm/deployment > dataplane-deployment.yaml +``` + +Wait for dataplane nodeset setup to finish. +``` +oc wait osdpns openstack-edpm --for condition=SetupReady --timeout=600s +``` + +Start the deployment. +``` +oc apply -f dataplane-deployment.yaml +``` + +Wait for dataplane deployment to finish. +``` +oc wait osdpns openstack-edpm --for condition=Ready --timeout=60m +``` + +After the Nvidia drivers have been blacklisted on the EDPM nodes, and kernel args updated, edpm hosts will be +automatically rebooted. diff --git a/examples/va/nvidia-vfio-passthrough/edpm/deployment/.gitignore b/examples/va/nvidia-vfio-passthrough/edpm/deployment/.gitignore new file mode 100644 index 000000000..285e9100a --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/edpm/deployment/.gitignore @@ -0,0 +1,2 @@ +dataplane-deployment.yaml +deployment.yaml diff --git a/examples/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml b/examples/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml new file mode 100644 index 000000000..6ebfbd293 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../va/nvidia-vfio-passthrough/edpm/deployment + # - https://github.com/openstack-k8s-operators/architecture/va/nvidia-vfio-passthrough/edpm/deployment?ref=main + ## It's possible to replace ../../../../../va/nvidia-vfio-passthrough/edpm/deployment/ with a git checkout URL as per: + ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml diff --git a/examples/va/nvidia-vfio-passthrough/edpm/deployment/values.yaml b/examples/va/nvidia-vfio-passthrough/edpm/deployment/values.yaml new file mode 100644 index 000000000..7f371b2cc --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/edpm/deployment/values.yaml @@ -0,0 +1,10 @@ +# yamllint disable rule:line-length +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-deployment-values + annotations: + config.kubernetes.io/local-config: "true" +data: {} diff --git a/examples/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml b/examples/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml new file mode 100644 index 000000000..4dda1cfc2 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../va/nvidia-vfio-passthrough/edpm/nodeset + # - https://github.com/openstack-k8s-operators/architecture/va/nvidia-vfio-passthrough/edpm/nodeset?ref=main + ## It's possible to replace ../../../../../va/nvidia-vfio-passthrough/edpm/nodeset/ with a git checkout URL as per: + ## https://github.com/kubernetes-sigs/kustomize/blob/master/examples/remoteBuild.md + +resources: + - values.yaml diff --git a/examples/va/nvidia-vfio-passthrough/edpm/nodeset/values.yaml b/examples/va/nvidia-vfio-passthrough/edpm/nodeset/values.yaml new file mode 100644 index 000000000..96cd55093 --- /dev/null +++ b/examples/va/nvidia-vfio-passthrough/edpm/nodeset/values.yaml @@ -0,0 +1,218 @@ +# yamllint disable rule:line-length +# local-config: referenced, but not emitted by kustomize +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: edpm-nodeset-values + annotations: + config.kubernetes.io/local-config: "true" +data: + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to baremetalset-password-secret.data.NodeRootPassword + root_password: cmVkaGF0Cg== + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to OpenStackDataPlaneNodeSet.spec.preProvisioned + preProvisioned: false + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to BareMetalHost.metadata.annotations.inspect.metal3.io + # Uncomment to allow HW inspection of the node, for example after a blank RAID device was manually created in BIOS. + metal3_inspection: "disabled" + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to BareMetalHost resource properties + baremetalhosts: + edpm-compute-0: + online: true + labels: + nodeName: edpm-compute-0 + bmc: + address: CHANGEME + credentialsName: bmc-secret + disableCertificateVerification: false + bootMACAddress: CHANGEME + rootDeviceHints: + deviceName: /dev/vda + preprovisioningNetworkDataName: edpm-compute-0-preprovision-network-data + preprovisioningNetworkData: + nmstate: | + CHANGEME + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to edpm-compute-0-network-data and edpm-compute-1-network-data Secrets + # Only provide it if it should not be auto-generated by EDPM operator + baremetalHostsNetworkData: + edpm-compute-0: + networkData: | + CHANGEME + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to OpenStackDataPlaneNodeSet.spec.baremetalSetTemplate + baremetalSetTemplate: + ctlplaneInterface: eno2 # CHANGEME + cloudUserName: cloud-admin + bmhNamespace: openstack + # NOTE: provisioningInterface should not be provided when using virtual-media + bmhLabelSelector: + app: openstack # CHANGEME + passwordSecret: + name: baremetalset-password-secret + namespace: openstack + # Mapped in lib/dataplane/nodeset/kustomization.yaml to the dataplane-ansible-ssh-private-key-secret + ssh_keys: + # Authorized keys that will have access to the dataplane computes via SSH + authorized: CHANGEME + # The private key that will have access to the dataplane computes via SSH + private: CHANGEME2 + # The public key that will have access to the dataplane computes via SSH + public: CHANGEME3 + nodeset: + ansible: + ansibleUser: cloud-admin + ansiblePort: 22 + ansibleVars: + # see https://access.redhat.com/solutions/253273 + edpm_bootstrap_command: | + echo CHANGEME + rhc_release: 9.4 + rhc_repositories: + - {name: "*", state: disabled} + - {name: "CHANGEME"} + edpm_bootstrap_release_version_package: [] + timesync_ntp_servers: + - hostname: pool.ntp.org + # GPU passthrough settings for a PCI Vendor and Device ID pair. + # Here, 10de represents NVIDIA Corporation, and 20f1 corresponds to the GA100 [A100 PCIe 40GB] GPU, according to Device Hunt + # https://devicehunt.com/view/type/pci/vendor/10DE/device/20F1 + # TODO(bogdando): add noveau.blacklist=1 nvidia.blacklist=1 nvidia-drm.blacklist=1 nvidia-modeset.blacklist=1 nvidia-uvm.blacklist=1 rd.driver.blacklist=nouveau,nvidia,nvidia-drm,nvidia-modeset,nvidia-uvm ? + edpm_kernel_args: "default_hugepagesz=1GB hugepagesz=1G hugepages=16 intel_iommu=on iommu=pt vfio-pci.ids=10de:20f1 rd.driver.pre=vfio-pci" + edpm_reboot_strategy: auto + edpm_tuned_profile: "cpu-partitioning-powersave" + edpm_tuned_isolated_cores: "4-23,28-47" + # edpm_network_config + # These vars are edpm_network_config role vars + edpm_network_config_hide_sensitive_logs: true + # Nics to MAC addresses mappings for the dataplane nodes + edpm_network_config_os_net_config_mappings: + edpm-compute-0: + nic1: CHANGEME + nic2: CHANGEME + edpm_network_config_template: | + --- + # CHANGEME + {% set mtu_list = [ctlplane_mtu] %} + {% for network in nodeset_networks %} + {{ mtu_list.append(lookup('vars', networks_lower[network] ~ '_mtu')) }} + {%- endfor %} + {% set min_viable_mtu = mtu_list | max %} + network_config: + - type: ovs_bridge + name: {{ neutron_physical_bridge_name }} + mtu: {{ min_viable_mtu }} + use_dhcp: false + dns_servers: {{ ctlplane_dns_nameservers }} + domain: {{ dns_search_domains }} + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: {{ ctlplane_host_routes }} + members: + - type: interface + name: nic2 + mtu: {{ min_viable_mtu }} + # force the MAC address of the bridge to this interface + primary: true + {% for network in nodeset_networks %} + - type: vlan + mtu: {{ lookup('vars', networks_lower[network] ~ '_mtu') }} + vlan_id: {{ lookup('vars', networks_lower[network] ~ '_vlan_id') }} + addresses: + - ip_netmask: + {{ lookup('vars', networks_lower[network] ~ '_ip') }}/{{ lookup('vars', networks_lower[network] ~ '_cidr') }} + routes: {{ lookup('vars', networks_lower[network] ~ '_host_routes') }} + {% endfor %} + + # These vars are for the network config templates themselves and are + # considered EDPM network defaults. + neutron_physical_bridge_name: br-ex + neutron_public_interface_name: eth0 + # edpm_nodes_validation + edpm_nodes_validation_validate_controllers_icmp: false + edpm_nodes_validation_validate_gateway_icmp: false + edpm_network_config_nmstate: false + edpm_network_config_update: false + dns_search_domains: [] + gather_facts: false + # edpm firewall, change the allowed CIDR if needed + edpm_sshd_configure_firewall: true + edpm_sshd_allowed_ranges: + - 192.168.122.0/24 + networks: # mapped to spec.nodeTemplate.networks + - defaultRoute: true + name: ctlplane + subnetName: subnet1 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + nodes: # mapped to spec.nodes + edpm-compute-0: + hostName: edpm-compute-0 + bmhLabelSelector: + nodeName: edpm-compute-0 + networkData: + name: edpm-compute-0-network-data + namespace: openstack + # must be defined if custom networkData is used for BMO, and there is a static IP defined for ctplane interface + networks: + - name: ctlplane + subnetName: subnet1 + defaultRoute: true + fixedIP: 192.168.122.100 + - name: internalapi + subnetName: subnet1 + - name: storage + subnetName: subnet1 + - name: tenant + subnetName: subnet1 + services: + - vfio-pci-bind + - bootstrap + - download-cache + - configure-network + - validate-network + - install-os + - configure-os + - ssh-known-hosts + - run-os + - reboot-os + - install-certs + - libvirt + - ovn + - neutron-ovn + - nova-custom-gpu + - neutron-metadata + - telemetry + nova: + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to cpu-pinning-nova ConfigMap + compute: + conf: | + # CHANGEME + [DEFAULT] + reserved_host_memory_mb = 4096 + reserved_huge_pages = node:0,size:4,count:524160 + reserved_huge_pages = node:1,size:4,count:524160 + [compute] + cpu_shared_set = 0-3,24-27 + cpu_dedicated_set = 8-23,32-47 + # Mapped in lib/dataplane/nodeset/kustomization.yaml to the nova-migration-ssh-key secret + migration: + ssh_keys: + private: CHANGEME4 + public: CHANGEME5 + # Mapped in va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml to gpu-nova ConfigMap + pci: + # You will have one device_spec line for each physical GPU you want to make available for passthrough: + # "vendor_id": The vendor ID of the GPU manufacturer (e.g., "10de" for NVIDIA). + # "product_id": The product ID of the specific GPU model (e.g., "20f1" for an NVIDIA A100). + # "address": The PCI address of the GPU on the host machine. You can find this using `lspci | grep -i nvidia`. + # "physical_network": This is used for SR-IOV networking passthrough. For a GPU, you can typically leave this as null. + # "alias": must match the alias configuration in the API service-values.yaml + conf: | + # CHANGEME + [pci] + device_spec = {"vendor_id":"10de", "product_id":"20f1", "address": "CHANGEME" } + alias = { "vendor_id":"10de", "product_id":"20f1", "device_type":"type-PF", "name":"nvidia_a2" } + report_in_placement = True diff --git a/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml b/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml new file mode 100644 index 000000000..bc49358eb --- /dev/null +++ b/va/nvidia-vfio-passthrough/edpm/deployment/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +components: + - ../../../../dt/nova/nova04delta/edpm/deployment diff --git a/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml b/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml new file mode 100644 index 000000000..aaccd5911 --- /dev/null +++ b/va/nvidia-vfio-passthrough/edpm/nodeset/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +components: + - ../../../../dt/nova/nova04delta/edpm/nodeset diff --git a/va/nvidia-vfio-passthrough/kustomization.yaml b/va/nvidia-vfio-passthrough/kustomization.yaml new file mode 100644 index 000000000..acc2110e0 --- /dev/null +++ b/va/nvidia-vfio-passthrough/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +components: + - ../../dt/nova/nova04delta diff --git a/va/nvidia-vfio-passthrough/networking/kustomization.yaml b/va/nvidia-vfio-passthrough/networking/kustomization.yaml new file mode 100644 index 000000000..a69636365 --- /dev/null +++ b/va/nvidia-vfio-passthrough/networking/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +components: + - ../../../dt/nova/nova04delta/networking diff --git a/va/nvidia-vfio-passthrough/networking/nncp/kustomization.yaml b/va/nvidia-vfio-passthrough/networking/nncp/kustomization.yaml new file mode 100644 index 000000000..06733d392 --- /dev/null +++ b/va/nvidia-vfio-passthrough/networking/nncp/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +components: + - ../../../../dt/nova/nova04delta/networking/nncp diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml index e545029fe..488856f9f 100644 --- a/zuul.d/projects.yaml +++ b/zuul.d/projects.yaml @@ -16,7 +16,11 @@ - rhoso-architecture-validate-nova-three-cells - rhoso-architecture-validate-nova01alpha - rhoso-architecture-validate-nova02beta + - rhoso-architecture-validate-nova04delta + - rhoso-architecture-validate-nova04delta-adoption - rhoso-architecture-validate-nvidia-mdev + - rhoso-architecture-validate-nvidia-vfio-passthrough + - rhoso-architecture-validate-nvidia-vfio-passthrough-adoption - rhoso-architecture-validate-osasinfra - rhoso-architecture-validate-osasinfra-ipv6 - rhoso-architecture-validate-ovs-dpdk diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index ce48a082d..2a79695bb 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -207,6 +207,30 @@ parent: rhoso-architecture-base-job vars: cifmw_architecture_scenario: nova02beta +- job: + files: + - automation/mocks/nova04delta.yaml + - automation/net-env/nova04delta.yaml + - examples/dt/nova/nova04delta/control-plane + - examples/dt/nova/nova04delta/control-plane/networking + - examples/dt/nova/nova04delta/control-plane/networking/nncp + - examples/dt/nova/nova04delta/edpm/deployment + - examples/dt/nova/nova04delta/edpm/nodeset + - lib + name: rhoso-architecture-validate-nova04delta + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: nova04delta + cifmw_networking_env_def_file: automation/net-env/nova04delta.yaml +- job: + files: + - examples/dt/nova/nova04delta/control-plane/networking + - examples/dt/nova/nova04delta/control-plane/networking/nncp + - lib + name: rhoso-architecture-validate-nova04delta-adoption + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: nova04delta-adoption - job: files: - examples/va/nvidia-mdev/control-plane @@ -221,6 +245,30 @@ parent: rhoso-architecture-base-job vars: cifmw_architecture_scenario: nvidia-mdev +- job: + files: + - automation/net-env/nvidia-vfio-passthrough.yaml + - examples/va/nvidia-vfio-passthrough/control-plane + - examples/va/nvidia-vfio-passthrough/control-plane/networking + - examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp + - examples/va/nvidia-vfio-passthrough/edpm/deployment + - examples/va/nvidia-vfio-passthrough/edpm/nodeset + - lib + - va/nvidia-vfio-passthrough + name: rhoso-architecture-validate-nvidia-vfio-passthrough + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: nvidia-vfio-passthrough + cifmw_networking_env_def_file: automation/net-env/nvidia-vfio-passthrough.yaml +- job: + files: + - examples/va/nvidia-vfio-passthrough/control-plane/networking + - examples/va/nvidia-vfio-passthrough/control-plane/networking/nncp + - lib + name: rhoso-architecture-validate-nvidia-vfio-passthrough-adoption + parent: rhoso-architecture-base-job + vars: + cifmw_architecture_scenario: nvidia-vfio-passthrough-adoption - job: files: - dt/osasinfra