diff --git a/automation/net-env/dcn.yaml b/automation/net-env/dcn.yaml index ca78435ad..ec94ff84e 100644 --- a/automation/net-env/dcn.yaml +++ b/automation/net-env/dcn.yaml @@ -660,6 +660,19 @@ instances: skip_nm: false trunk_parent: ctlplane vlan_id: 21 + storagemgmt: + interface_name: enp7s0.23 + ip_v4: 172.20.0.10 + is_trunk_parent: false + mac_addr: 52:54:00:05:23:40 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + parent_interface: enp7s0 + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 tenant: interface_name: enp7s0.22 ip_v4: 172.19.0.10 @@ -726,6 +739,19 @@ instances: skip_nm: false trunk_parent: ctlplane vlan_id: 21 + storagemgmt: + interface_name: enp7s0.23 + ip_v4: 172.20.0.11 + is_trunk_parent: false + mac_addr: 52:54:00:76:3d:41 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + parent_interface: enp7s0 + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 tenant: interface_name: enp7s0.22 ip_v4: 172.19.0.11 @@ -792,6 +818,19 @@ instances: skip_nm: false trunk_parent: ctlplane vlan_id: 21 + storagemgmt: + interface_name: enp7s0.23 + ip_v4: 172.20.0.12 + is_trunk_parent: false + mac_addr: 52:54:00:09:45:42 + mtu: 1500 + netmask_v4: 255.255.255.0 + network_name: storagemgmt + parent_interface: enp7s0 + prefix_length_v4: 24 + skip_nm: false + trunk_parent: ctlplane + vlan_id: 23 tenant: interface_name: enp7s0.22 ip_v4: 172.19.0.12 diff --git a/automation/vars/dcn.yaml b/automation/vars/dcn.yaml index 4a7dbbe5a..5bebc5ec1 100644 --- a/automation/vars/dcn.yaml +++ b/automation/vars/dcn.yaml @@ -3,7 +3,7 @@ vas: dcn: stages: - name: nncp-configuration - path: examples/dt/dcn/control-plane/nncp + path: examples/dt/dcn/control-plane/networking/nncp wait_conditions: - >- oc -n openstack wait nncp @@ -15,6 +15,19 @@ vas: src_file: values.yaml build_output: nncp.yaml + - name: networking + path: examples/dt/dcn/control-plane/networking + wait_conditions: + - >- + oc -n metallb-system wait pod + -l app=metallb -l component=speaker + --for condition=Ready + --timeout=5m + values: + - name: network-values + src_file: nncp/values.yaml + build_output: network.yaml + - name: control-plane path: examples/dt/dcn/control-plane wait_conditions: @@ -23,7 +36,7 @@ vas: --timeout=30m values: - name: network-values - src_file: nncp/values.yaml + src_file: networking/nncp/values.yaml build_output: ../control-plane.yaml post_stage_run: - name: Deploy DCN diff --git a/dt/dcn/kustomization.yaml b/dt/dcn/kustomization.yaml index afb7e3773..ed986cb9e 100644 --- a/dt/dcn/kustomization.yaml +++ b/dt/dcn/kustomization.yaml @@ -25,71 +25,4 @@ transformers: create: true components: - - ../../lib/networking/metallb - - ../../lib/networking/netconfig - - ../../lib/networking/nad - ../../lib/control-plane - -resources: - - nad.yaml - -# Add storagemgmt network template, as it is needed for CephHCI -patches: - - target: - version: v1beta1 - kind: NetConfig - name: netconfig - patch: |- - - op: add - path: /spec/networks/- - value: - dnsDomain: _replaced_ - name: storagemgmt - subnets: - - _replaced_ - mtu: 1500 - -# Add storagemgmt network replacements -replacements: - # NetConfig dnsDomain specific to this VA - - source: - kind: ConfigMap - name: network-values - fieldPath: data.storagemgmt.dnsDomain - targets: - - select: - kind: NetConfig - fieldPaths: - - spec.networks.[name=storagemgmt].dnsDomain - # NetConfig MTU specific to this VA - - source: - kind: ConfigMap - name: network-values - fieldPath: data.storagemgmt.mtu - targets: - - select: - kind: NetConfig - fieldPaths: - - spec.networks.[name=storagemgmt].mtu - # NetConfig subnets specific to this VA - - source: - kind: ConfigMap - name: network-values - fieldPath: data.storagemgmt.subnets - targets: - - select: - kind: NetConfig - fieldPaths: - - spec.networks.[name=storagemgmt].subnets - - # Octavia support - - source: - kind: ConfigMap - name: network-values - fieldPath: data.octavia.net-attach-def - targets: - - select: - kind: NetworkAttachmentDefinition - name: octavia - fieldPaths: - - spec.config diff --git a/dt/dcn/networking/kustomization.yaml b/dt/dcn/networking/kustomization.yaml new file mode 100644 index 000000000..3427b0e81 --- /dev/null +++ b/dt/dcn/networking/kustomization.yaml @@ -0,0 +1,137 @@ +--- +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../lib/networking/metallb + - ../../../lib/networking/netconfig + - ../../../lib/networking/nad + +resources: + - nad.yaml + - storagemgmt-nad.yaml + - storagemgmt-metallb.yaml + +# Add storagemgmt network template, as it is needed for CephHCI +patches: + - target: + version: v1beta1 + kind: NetConfig + name: netconfig + patch: |- + - op: add + path: /spec/networks/- + value: + dnsDomain: _replaced_ + name: storagemgmt + subnets: + - _replaced_ + mtu: 1500 + +# Add storagemgmt network replacements +replacements: + # NetConfig dnsDomain specific to this VA + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.dnsDomain + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].dnsDomain + # NetConfig MTU specific to this VA + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.mtu + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].mtu + # NetConfig subnets specific to this VA + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.subnets + targets: + - select: + kind: NetConfig + fieldPaths: + - spec.networks.[name=storagemgmt].subnets + + # Octavia support + - source: + kind: ConfigMap + name: network-values + fieldPath: data.octavia.net-attach-def + targets: + - select: + kind: NetworkAttachmentDefinition + name: octavia + fieldPaths: + - spec.config + + # Storagemgmt NAD support + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.net-attach-def + targets: + - select: + kind: NetworkAttachmentDefinition + name: storagemgmt + fieldPaths: + - spec.config + + # Storagemgmt IPAddressPool + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.lb_addresses + targets: + - select: + kind: IPAddressPool + name: storagemgmt + fieldPaths: + - spec.addresses + + # Storagemgmt L2Advertisement interface + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.iface + targets: + - select: + kind: L2Advertisement + name: storagemgmt + fieldPaths: + - spec.interfaces.0 + + # Override ctlplane L2Advertisement to use bridgeName (ospbr) instead of ctlplane.iface (enp7s0) + - source: + kind: ConfigMap + name: network-values + fieldPath: data.bridgeName + targets: + - select: + kind: L2Advertisement + name: ctlplane + fieldPaths: + - spec.interfaces.0 diff --git a/dt/dcn/networking/nad.yaml b/dt/dcn/networking/nad.yaml new file mode 100644 index 000000000..d59e60095 --- /dev/null +++ b/dt/dcn/networking/nad.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: octavia + labels: + osp/net: octavia + osp/net-attach-def-type: standard diff --git a/dt/dcn/networking/storagemgmt-metallb.yaml b/dt/dcn/networking/storagemgmt-metallb.yaml new file mode 100644 index 000000000..ebb4db7e8 --- /dev/null +++ b/dt/dcn/networking/storagemgmt-metallb.yaml @@ -0,0 +1,22 @@ +--- +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: storagemgmt + namespace: metallb-system + labels: + osp/lb-addresses-type: standard +spec: + addresses: + - _replaced_ +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: storagemgmt + namespace: metallb-system +spec: + ipAddressPools: + - storagemgmt + interfaces: + - _replaced_ diff --git a/dt/dcn/networking/storagemgmt-nad.yaml b/dt/dcn/networking/storagemgmt-nad.yaml new file mode 100644 index 000000000..c1d4bf2be --- /dev/null +++ b/dt/dcn/networking/storagemgmt-nad.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: storagemgmt + labels: + osp/net: storagemgmt + osp/net-attach-def-type: standard diff --git a/examples/dt/dcn/README.md b/examples/dt/dcn/README.md index 43d06ab3f..27c4e9c81 100644 --- a/examples/dt/dcn/README.md +++ b/examples/dt/dcn/README.md @@ -28,7 +28,7 @@ This is a collection of CR templates that represent a Red Hat OpenStack Services 2. The CRs are applied against an OpenShift cluster in _stages_. That is, there is an ordering in which each grouping of CRs is fed to the cluster. It is _not_ a case of simply taking all CRs from all stages and applying them all at once. -3. In stages 1 and 2 [kustomize](https://kustomize.io/) is used to genereate the control plane CRs dynamically. The `control-plane/nncp/values.yaml` file(s) must be updated to fit your environment. kustomize version 5 or newer required. +3. In stages 1 and 2 [kustomize](https://kustomize.io/) is used to generate the control plane CRs dynamically. The `control-plane/networking/nncp/values.yaml` file(s) must be updated to fit your environment. kustomize version 5 or newer required. 4. In stages 3 and 4 [kustomize](https://kustomize.io/) is used to generate the dataplane CRs dynamically. The `edpm-pre-ceph/values.yaml`, `values.yaml` and `service-values.yaml` files must be updated to fit your environment. kustomize version 5 or newer required. diff --git a/examples/dt/dcn/control-plane.md b/examples/dt/dcn/control-plane.md index baefd720e..10b753542 100644 --- a/examples/dt/dcn/control-plane.md +++ b/examples/dt/dcn/control-plane.md @@ -18,10 +18,10 @@ Change to the dcn directory cd architecture/examples/dt/dcn ``` -Edit the [control-plane/nncp/values.yaml](control-plane/nncp/values.yaml) file to suit your environment. +Edit the [control-plane/networking/nncp/values.yaml](control-plane/networking/nncp/values.yaml) file to suit your environment. ```shell -vi control-plane/nncp/values.yaml +vi control-plane/networking/nncp/values.yaml ``` ## Apply node network configuration @@ -29,7 +29,7 @@ vi control-plane/nncp/values.yaml Generate the node network configuration ```shell -kustomize build control-plane/nncp > nncp.yaml +kustomize build control-plane/networking/nncp > nncp.yaml ``` Apply the NNCP CRs @@ -44,9 +44,29 @@ Wait for NNCPs to be available oc wait nncp -l osp/nncm-config-type=standard --for jsonpath='{.status.conditions[0].reason}'=SuccessfullyConfigured --timeout=300s ``` -## Apply networking and control-plane configuration +## Apply networking configuration -Generate the control-plane and networking CRs. +Generate the networking CRs. + +```shell +kustomize build control-plane/networking > networking.yaml +``` + +Apply the networking CRs + +```shell +oc apply -f networking.yaml +``` + +Wait for MetalLB to be available + +```shell +oc -n metallb-system wait pod -l app=metallb -l component=speaker --for condition=Ready --timeout=5m +``` + +## Apply control-plane configuration + +Generate the control-plane CRs. ```shell kustomize build control-plane > control-plane.yaml diff --git a/examples/dt/dcn/control-plane/kustomization.yaml b/examples/dt/dcn/control-plane/kustomization.yaml index bab9b637a..363da6c37 100644 --- a/examples/dt/dcn/control-plane/kustomization.yaml +++ b/examples/dt/dcn/control-plane/kustomization.yaml @@ -6,5 +6,5 @@ components: - ../../../../dt/dcn/ resources: - - nncp/values.yaml + - networking/nncp/values.yaml - service-values.yaml diff --git a/examples/dt/dcn/control-plane/networking/kustomization.yaml b/examples/dt/dcn/control-plane/networking/kustomization.yaml new file mode 100644 index 000000000..ab6f29226 --- /dev/null +++ b/examples/dt/dcn/control-plane/networking/kustomization.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +components: + - ../../../../../dt/dcn/networking + +resources: + - nncp/values.yaml diff --git a/examples/dt/dcn/control-plane/nncp/.gitignore b/examples/dt/dcn/control-plane/networking/nncp/.gitignore similarity index 100% rename from examples/dt/dcn/control-plane/nncp/.gitignore rename to examples/dt/dcn/control-plane/networking/nncp/.gitignore diff --git a/examples/dt/dcn/control-plane/networking/nncp/kustomization.yaml b/examples/dt/dcn/control-plane/networking/nncp/kustomization.yaml new file mode 100644 index 000000000..dd649d65f --- /dev/null +++ b/examples/dt/dcn/control-plane/networking/nncp/kustomization.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +transformers: + # Set namespace to OpenStack on all namespaced objects without a namespace + - |- + apiVersion: builtin + kind: NamespaceTransformer + metadata: + name: _ignored_ + namespace: openstack + setRoleBindingSubjects: none + unsetOnly: true + fieldSpecs: + - path: metadata/name + kind: Namespace + create: true + +components: + - ../../../../../../dt/dcn/nncp + + +resources: + - values.yaml + +# storagemgmt_ip below should only be used for Swift here (not necessary for Ceph) + +patches: + - target: + kind: NodeNetworkConfigurationPolicy + name: "master-.*" + patch: |- + - op: add + path: /spec/desiredState/interfaces/- + value: + description: storagemgmt vlan host interface + name: storagemgmt + state: up + type: vlan + mtu: _mtu_replaced_ + ipv4: + address: + - ip: _ip_replaced_ + prefix-length: _prefix_replaced_ + dhcp: false + enabled: true + ipv6: + enabled: false + vlan: + base-iface: _base_iface_replaced_ + id: _vlan_id_replaced_ + +replacements: + - source: + kind: ConfigMap + name: network-values + fieldPath: data.node_0.storagemgmt_ip + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-0 + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.ip + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.node_1.storagemgmt_ip + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-1 + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.ip + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.node_2.storagemgmt_ip + targets: + - select: + kind: NodeNetworkConfigurationPolicy + name: master-2 + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.ip + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.base_iface + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].vlan.base-iface + options: + create: true + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.vlan + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].vlan.id + options: + create: true + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.mtu + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].mtu + options: + create: true + + - source: + kind: ConfigMap + name: network-values + fieldPath: data.storagemgmt.prefix-length + targets: + - select: + kind: NodeNetworkConfigurationPolicy + fieldPaths: + - spec.desiredState.interfaces.[name=storagemgmt].ipv4.address.0.prefix-length + options: + create: true + +# storagemgmt_ip above should only be used for Swift here (not necessary for Ceph) diff --git a/examples/dt/dcn/control-plane/nncp/values.yaml b/examples/dt/dcn/control-plane/networking/nncp/values.yaml similarity index 96% rename from examples/dt/dcn/control-plane/nncp/values.yaml rename to examples/dt/dcn/control-plane/networking/nncp/values.yaml index 74f4f8487..d1035fb99 100644 --- a/examples/dt/dcn/control-plane/nncp/values.yaml +++ b/examples/dt/dcn/control-plane/networking/nncp/values.yaml @@ -62,7 +62,7 @@ data: - destination: 192.168.122.0/24 nexthop: 192.168.133.1 - destination: 192.168.144.0/24 - nexthop: 192.168.133.1ØØ + nexthop: 192.168.133.1 - allocationRanges: - end: 192.168.144.120 start: 192.168.144.100 @@ -147,8 +147,8 @@ data: { "cniVersion": "0.3.1", "name": "datacentre", - "type": "bridge", - "bridge": "ospbr", + "type": "host-device", + "device": "enp8s0", "ipam": {} } dns-resolver: @@ -170,8 +170,8 @@ data: start: 10.0.0.100 cidr: 10.0.0.0/24 name: subnet1 - vlan: 22 - vlan: 22 + vlan: 44 + vlan: 44 internalapi: base_iface: enp7s0 dnsDomain: internalapi.example.com @@ -309,23 +309,27 @@ data: vlan: 40 vlan: 40 lbServiceType: LoadBalancer + # storagemgmt_ip should only be used for Swift here (not necessary for Ceph) node_0: ctlplane_ip: 192.168.122.10 internalapi_ip: 172.17.0.10 name: master-0 storage_ip: 172.18.0.10 + storagemgmt_ip: 172.20.0.10 tenant_ip: 172.19.0.10 node_1: ctlplane_ip: 192.168.122.11 internalapi_ip: 172.17.0.11 name: master-1 storage_ip: 172.18.0.11 + storagemgmt_ip: 172.20.0.11 tenant_ip: 172.19.0.11 node_2: ctlplane_ip: 192.168.122.12 internalapi_ip: 172.17.0.12 name: master-2 storage_ip: 172.18.0.12 + storagemgmt_ip: 172.20.0.12 tenant_ip: 172.19.0.12 rabbitmq: endpoint_annotations: @@ -507,10 +511,25 @@ data: vlan: 41 vlan: 41 storagemgmt: - base_iface: eth1 + base_iface: enp7s0 dnsDomain: storagemgmt.example.com iface: storagemgmt + lb_addresses: + - 172.20.0.80-172.20.0.90 mtu: 1500 + net-attach-def: | + { + "cniVersion": "0.3.1", + "name": "storagemgmt", + "type": "macvlan", + "master": "storagemgmt", + "ipam": { + "type": "whereabouts", + "range": "172.20.0.0/24", + "range_start": "172.20.0.30", + "range_end": "172.20.0.70" + } + } prefix-length: 24 subnets: - allocationRanges: diff --git a/examples/dt/dcn/control-plane/nncp/kustomization.yaml b/examples/dt/dcn/control-plane/nncp/kustomization.yaml deleted file mode 100644 index 9b001be99..000000000 --- a/examples/dt/dcn/control-plane/nncp/kustomization.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -transformers: - # Set namespace to OpenStack on all namespaced objects without a namespace - - |- - apiVersion: builtin - kind: NamespaceTransformer - metadata: - name: _ignored_ - namespace: openstack - setRoleBindingSubjects: none - unsetOnly: true - fieldSpecs: - - path: metadata/name - kind: Namespace - create: true - -components: - - ../../../../../dt/dcn/nncp - - -resources: - - values.yaml diff --git a/examples/dt/dcn/control-plane/scaledown/kustomization.yaml b/examples/dt/dcn/control-plane/scaledown/kustomization.yaml index 010b62661..80d036fd5 100644 --- a/examples/dt/dcn/control-plane/scaledown/kustomization.yaml +++ b/examples/dt/dcn/control-plane/scaledown/kustomization.yaml @@ -6,5 +6,5 @@ components: - ../../../../../dt/dcn/control-plane resources: - - ../nncp + - ../networking/nncp - service-values.yaml diff --git a/examples/dt/dcn/kustomization.yaml b/examples/dt/dcn/kustomization.yaml index df1d8982e..e2f2a48c4 100644 --- a/examples/dt/dcn/kustomization.yaml +++ b/examples/dt/dcn/kustomization.yaml @@ -7,7 +7,7 @@ components: - ../../../dt/dcn/edpm-post-ceph/nodeset resources: - - control-plane/nncp/values.yaml + - control-plane/networking/nncp/values.yaml - edpm-pre-ceph/nodeset/values.yaml - service-values.yaml - values.yaml diff --git a/zuul.d/validations.yaml b/zuul.d/validations.yaml index 493da6f06..1b92d8a7a 100644 --- a/zuul.d/validations.yaml +++ b/zuul.d/validations.yaml @@ -84,7 +84,8 @@ - automation/net-env/dcn.yaml - dt/dcn - examples/dt/dcn/control-plane - - examples/dt/dcn/control-plane/nncp + - examples/dt/dcn/control-plane/networking + - examples/dt/dcn/control-plane/networking/nncp - lib name: rhoso-architecture-validate-dcn parent: rhoso-architecture-base-job