Skip to content

Commit d125fda

Browse files
DanielXiaoDanielXiao
authored
✨Support multi-networking for NSX-VPC and vsphere-network providers (#3530)
* Support multi-networking for NSX-VPC and vsphere-network providers This PR enhances Node network management to support multi-networking for the NSX-VPC and vsphere-network providers, while preserving existing behavior for backward compatibility. Current Behavior CAPV manages Node network lifecycle based on the configured network provider: * vsphere-network (VDS): Uses the single default Network in the Supervisor Namespace. * NSX (NSX-T Tier-1): Automatically creates a VirtualNetwork for the Node network. * NSX-VPC (NSX-T VPC): Automatically creates a SubnetSet for the Node network. What's New NSX-VPC: * Primary network: Users can now specify an existing SubnetSet as the Kubernetes primary network (used for load balancing, service discovery, Pod traffic, management traffic, etc.). * Secondary networks: Users can configure additional SubnetSets or Subnets to handle specialized traffic (e.g., dedicated Pod networking, NFS access). vsphere-network: * Primary network: Continues to use the default Network in the Supervisor Namespace (no change). * Secondary networks: Users can now specify additional Networks from the Supervisor Namespace as secondary networks. Compatibility If users do not specify custom networking configurations, the default Node network behavior remains unchanged. * Fixup api-lint * Address review comments * Fix github lint * Fix second review
1 parent 24b0b35 commit d125fda

31 files changed

+3188
-75
lines changed

‎apis/vmware/v1beta1/vspherecluster_types.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ limitations under the License.
1717
package v1beta1
1818

1919
import (
20+
"reflect"
21+
2022
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2123
clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1"
2224
)
@@ -123,10 +125,42 @@ const (
123125
VSphereClusterServiceDiscoveryNotReadyV1Beta2Reason = clusterv1beta1.NotReadyV1Beta2Reason
124126
)
125127

128+
// NSXVPC defines the configuration when the network provider is NSX-VPC.
129+
// +kubebuilder:validation:XValidation:rule="has(self.createSubnetSet) == has(oldSelf.createSubnetSet) && self.createSubnetSet == oldSelf.createSubnetSet",message="createSubnetSet value cannot be changed after creation"
130+
// +kubebuilder:validation:MinProperties=1
131+
type NSXVPC struct {
132+
// createSubnetSet is a flag to indicate whether to create a SubnetSet or not as the primary network. If not set, the default is true.
133+
// +optional
134+
CreateSubnetSet *bool `json:"createSubnetSet,omitempty"`
135+
}
136+
137+
// IsDefined returns true if the NSXVPC is defined.
138+
func (r *NSXVPC) IsDefined() bool {
139+
return !reflect.DeepEqual(r, &NSXVPC{})
140+
}
141+
142+
// Network defines the network configuration for the cluster with different network providers.
143+
// +kubebuilder:validation:XValidation:rule="has(self.nsxVPC) == has(oldSelf.nsxVPC)",message="field 'nsxVPC' cannot be added or removed after creation"
144+
// +kubebuilder:validation:MinProperties=1
145+
type Network struct {
146+
// nsxVPC defines the configuration when the network provider is NSX-VPC.
147+
// +optional
148+
NSXVPC NSXVPC `json:"nsxVPC,omitempty,omitzero"`
149+
}
150+
151+
// IsDefined returns true if the Network is defined.
152+
func (r *Network) IsDefined() bool {
153+
return !reflect.DeepEqual(r, &Network{})
154+
}
155+
126156
// VSphereClusterSpec defines the desired state of VSphereCluster.
157+
// +kubebuilder:validation:XValidation:rule="has(self.network) == has(oldSelf.network)",message="field 'network' cannot be added or removed after creation"
127158
type VSphereClusterSpec struct {
128159
// +optional
129160
ControlPlaneEndpoint clusterv1beta1.APIEndpoint `json:"controlPlaneEndpoint"`
161+
// network defines the network configuration for the cluster with different network providers.
162+
// +optional
163+
Network Network `json:"network,omitempty,omitzero"`
130164
}
131165

132166
// VSphereClusterStatus defines the observed state of VSphereClusterSpec.

‎apis/vmware/v1beta1/vspheremachine_types.go

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@ limitations under the License.
1717
package v1beta1
1818

1919
import (
20+
"reflect"
21+
2022
corev1 "k8s.io/api/core/v1"
2123
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/runtime/schema"
2225
clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1"
2326
"sigs.k8s.io/cluster-api/errors"
2427
)
@@ -35,6 +38,7 @@ type VSphereMachineVolume struct {
3538
}
3639

3740
// VSphereMachineSpec defines the desired state of VSphereMachine.
41+
// +kubebuilder:validation:XValidation:rule="has(self.network) == has(oldSelf.network)",message="field 'network' cannot be added or removed after creation"
3842
type VSphereMachineSpec struct {
3943
// ProviderID is the virtual machine's BIOS UUID formatted as
4044
// vsphere://12345678-1234-1234-1234-123456789abc.
@@ -64,6 +68,10 @@ type VSphereMachineSpec struct {
6468
// +optional
6569
Volumes []VSphereMachineVolume `json:"volumes,omitempty"`
6670

71+
// network is the network configuration for the VSphereMachine
72+
// +optional
73+
Network VSphereMachineNetworkSpec `json:"network,omitempty,omitzero"`
74+
6775
// PowerOffMode describes the desired behavior when powering off a VM.
6876
//
6977
// There are three, supported power off modes: hard, soft, and
@@ -93,6 +101,153 @@ type VSphereMachineSpec struct {
93101
NamingStrategy *VirtualMachineNamingStrategy `json:"namingStrategy,omitempty"`
94102
}
95103

104+
// VSphereMachineNetworkSpec defines the network configuration of a VSphereMachine.
105+
// +kubebuilder:validation:XValidation:rule="has(self.interfaces) == has(oldSelf.interfaces)",message="field 'interfaces' cannot be added or removed after creation"
106+
// +kubebuilder:validation:MinProperties=1
107+
type VSphereMachineNetworkSpec struct {
108+
// interfaces is the list of network interfaces attached to this VSphereMachine.
109+
//
110+
// +optional
111+
Interfaces InterfacesSpec `json:"interfaces,omitempty,omitzero"`
112+
}
113+
114+
// IsDefined returns true if the VSphereMachineNetworkSpec is defined.
115+
func (r *VSphereMachineNetworkSpec) IsDefined() bool {
116+
return !reflect.DeepEqual(r, &VSphereMachineNetworkSpec{})
117+
}
118+
119+
// InterfacesSpec defines all the network interfaces of a VSphereMachine from Kubernetes perspective.
120+
// +kubebuilder:validation:XValidation:rule="has(self.primary) == has(oldSelf.primary)",message="field 'primary' cannot be added or removed after creation"
121+
// +kubebuilder:validation:MinProperties=1
122+
type InterfacesSpec struct {
123+
// primary is the primary network interface.
124+
//
125+
// It is used to connect the Kubernetes primary network for Load balancer,
126+
// Service discovery, Pod traffic and management traffic etc.
127+
// Leave it unset if you don't want to customize the primary network and interface.
128+
// Customization is only supported with network provider NSX-VPC.
129+
// It should be set only when VSphereCluster spec.network.nsxVPC.createSubnetSet is set to false.
130+
//
131+
// +optional
132+
Primary InterfaceSpec `json:"primary,omitempty,omitzero"`
133+
134+
// secondary are the secondary network interfaces.
135+
//
136+
// It is used for any purpose like deploying Antrea secondary network,
137+
// Multus, mounting NFS etc.
138+
// Secondary network is supported with network provider NSX-VPC and vsphere-network.
139+
//
140+
// +kubebuilder:validation:MinItems=1
141+
// +kubebuilder:validation:MaxItems=9
142+
// +listType=atomic
143+
// +optional
144+
Secondary []SecondaryInterfaceSpec `json:"secondary,omitempty"`
145+
}
146+
147+
// IsDefined returns true if the InterfacesSpec is defined.
148+
func (r *InterfacesSpec) IsDefined() bool {
149+
return !reflect.DeepEqual(r, &InterfacesSpec{})
150+
}
151+
152+
// SecondaryInterfaceSpec defines a secondary network interface for a VSphereMachine.
153+
type SecondaryInterfaceSpec struct {
154+
// name describes the unique name of this network interface, used to
155+
// distinguish it from other network interfaces attached to this VSphereMachine.
156+
//
157+
// +kubebuilder:validation:Pattern="^[a-z0-9]{2,}$"
158+
// +kubebuilder:validation:MinLength=2
159+
// +kubebuilder:validation:MaxLength=15
160+
// +required
161+
Name string `json:"name,omitempty"`
162+
163+
InterfaceSpec `json:",inline"`
164+
}
165+
166+
// InterfaceSpec defines properties of a network interface.
167+
type InterfaceSpec struct {
168+
// network is the name of the network resource to which this interface is
169+
// connected.
170+
// +required
171+
Network InterfaceNetworkReference `json:"network,omitempty,omitzero"`
172+
173+
// mtu is the Maximum Transmission Unit size in bytes.
174+
//
175+
// +kubebuilder:validation:Minimum=68
176+
// +kubebuilder:validation:Maximum=9000
177+
// +optional
178+
MTU int32 `json:"mtu,omitempty"`
179+
180+
// routes is a list of optional, static routes.
181+
//
182+
// Please note this feature is available only with the following bootstrap
183+
// providers: CloudInit.
184+
//
185+
// +kubebuilder:validation:MinItems=1
186+
// +kubebuilder:validation:MaxItems=100
187+
// +listType=atomic
188+
// +optional
189+
Routes []RouteSpec `json:"routes,omitempty"`
190+
}
191+
192+
// IsDefined returns true if the InterfaceSpec is defined.
193+
func (r *InterfaceSpec) IsDefined() bool {
194+
return !reflect.DeepEqual(r, &InterfaceSpec{})
195+
}
196+
197+
// InterfaceNetworkReference describes a reference to another object in the same
198+
// namespace as the referrer.
199+
type InterfaceNetworkReference struct {
200+
// kind of the remediation template.
201+
// kind must consist of alphanumeric characters or '-', start with an alphabetic character, and end with an alphanumeric character.
202+
// +required
203+
// +kubebuilder:validation:MinLength=1
204+
// +kubebuilder:validation:MaxLength=63
205+
// +kubebuilder:validation:Pattern=`^[a-zA-Z]([-a-zA-Z0-9]*[a-zA-Z0-9])?$`
206+
Kind string `json:"kind,omitempty"`
207+
208+
// name of the remediation template.
209+
// name must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character.
210+
// +required
211+
// +kubebuilder:validation:MinLength=1
212+
// +kubebuilder:validation:MaxLength=253
213+
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`
214+
Name string `json:"name,omitempty"`
215+
216+
// apiVersion of the remediation template.
217+
// apiVersion must be fully qualified domain name followed by / and a version.
218+
// NOTE: This field must be kept in sync with the APIVersion of the remediation template.
219+
// +required
220+
// +kubebuilder:validation:MinLength=1
221+
// +kubebuilder:validation:MaxLength=317
222+
// +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*\/[a-z]([-a-z0-9]*[a-z0-9])?$`
223+
APIVersion string `json:"apiVersion,omitempty"`
224+
}
225+
226+
// GroupVersionKind gets the GroupVersionKind for an InterfaceNetworkReference.
227+
func (r *InterfaceNetworkReference) GroupVersionKind() schema.GroupVersionKind {
228+
return schema.FromAPIVersionAndKind(r.APIVersion, r.Kind)
229+
}
230+
231+
// RouteSpec defines a static route for a guest.
232+
type RouteSpec struct {
233+
// to is an IP4 CIDR. IP6 is not supported yet.
234+
// Examples: 192.168.1.0/24, 192.168.100.100/32, 0.0.0.0/0
235+
//
236+
// +kubebuilder:validation:Pattern=`^([0-9]{1,3}\.){3}[0-9]{1,3}\/[0-9]{1,2}$`
237+
// +kubebuilder:validation:MinLength=9
238+
// +kubebuilder:validation:MaxLength=18
239+
// +required
240+
To string `json:"to,omitempty"`
241+
242+
// via is an IP4 address. IP6 is not supported yet.
243+
//
244+
// +kubebuilder:validation:Pattern=`^([0-9]{1,3}\.){3}[0-9]{1,3}$`
245+
// +kubebuilder:validation:MinLength=7
246+
// +kubebuilder:validation:MaxLength=15
247+
// +required
248+
Via string `json:"via,omitempty"`
249+
}
250+
96251
// VirtualMachineNamingStrategy defines the naming strategy for the VirtualMachines.
97252
type VirtualMachineNamingStrategy struct {
98253
// Template defines the template to use for generating the name of the VirtualMachine object.
@@ -121,6 +276,10 @@ type VSphereMachineStatus struct {
121276
Ready bool `json:"ready"`
122277

123278
// Addresses contains the instance associated addresses.
279+
// +kubebuilder:validation:MinItems=1
280+
// +kubebuilder:validation:MaxItems=10
281+
// +listType=atomic
282+
// +optional
124283
Addresses []corev1.NodeAddress `json:"addresses,omitempty"`
125284

126285
// ID is used to identify the virtual machine.
@@ -180,6 +339,12 @@ type VSphereMachineStatus struct {
180339
// v1beta2 groups all the fields that will be added or modified in VSphereMachine's status with the V1Beta2 version.
181340
// +optional
182341
V1Beta2 *VSphereMachineV1Beta2Status `json:"v1beta2,omitempty"`
342+
343+
// network describes the observed state of the VM's network configuration.
344+
// Please note much of the network status information is only available if
345+
// the guest has VM Tools installed.
346+
// +optional
347+
Network VSphereMachineNetworkStatus `json:"network,omitempty,omitzero"`
183348
}
184349

185350
// VSphereMachineV1Beta2Status groups all the fields that will be added or modified in VSphereMachineStatus with the V1Beta2 version.

0 commit comments

Comments
 (0)