Skip to content

Commit bb0fdd1

Browse files
mazdaknmazdakn
authored
cluster routing mode (#4511)
1 parent f56b597 commit bb0fdd1

File tree

11 files changed

+369
-12
lines changed

11 files changed

+369
-12
lines changed

‎api/v1/installation_types.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,15 @@ func BGPOptionPtr(b BGPOption) *BGPOption {
537537
return &b
538538
}
539539

540+
// ClusterRoutingMode describes the mode of cluster routing.
541+
// +kubebuilder:validation:Enum=BIRD;Felix
542+
type ClusterRoutingMode string
543+
544+
const (
545+
ClusterRoutingModeBIRD ClusterRoutingMode = "BIRD"
546+
ClusterRoutingModeFelix ClusterRoutingMode = "Felix"
547+
)
548+
540549
const (
541550
BGPEnabled BGPOption = "Enabled"
542551
BGPDisabled BGPOption = "Disabled"
@@ -614,6 +623,13 @@ type CalicoNetworkSpec struct {
614623
// +kubebuilder:validation:Enum=Enabled;Disabled
615624
BGP *BGPOption `json:"bgp,omitempty"`
616625

626+
// ClusterRoutingMode controls how nodes get a route to a workload on another node,
627+
// when that workload's IP comes from an IP Pool with vxlanMode: Never. When ClusterRoutingMode is BIRD,
628+
// confd and BIRD program that route. When ClusterRoutingMode is Felix, it is expected that Felix will program that route.
629+
// Felix always programs such routes for IP Pools with vxlanMode: Always or vxlanMode: CrossSubnet. [Default: BIRD]
630+
// +optional
631+
ClusterRoutingMode *ClusterRoutingMode `json:"clusterRoutingMode,omitempty"`
632+
617633
// IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool
618634
// will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead
619635
// wait for IP pools to be created out-of-band.

‎api/v1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ require (
2929
github.com/sirupsen/logrus v1.9.4
3030
github.com/snowzach/rotatefilehook v0.0.0-20220211133110-53752135082d
3131
github.com/stretchr/testify v1.11.1
32-
github.com/tigera/api v0.0.0-20260227222130-df0b9e289a34
32+
github.com/tigera/api v0.0.0-20260310182635-546021df243c
3333
github.com/tigera/operator/api v0.0.0-20260120220012-4a3f8a7d8399
3434
github.com/urfave/cli/v3 v3.6.2
3535
go.uber.org/zap v1.27.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -406,8 +406,8 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
406406
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
407407
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
408408
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
409-
github.com/tigera/api v0.0.0-20260227222130-df0b9e289a34 h1:bFz27L2XauCys8mBwGQUXzYL8fS4oC5/ModvfujYKaQ=
410-
github.com/tigera/api v0.0.0-20260227222130-df0b9e289a34/go.mod h1:CX/80DoAQD88kMvkvujV4h3XPmIBo0yeSf9OZmvZY0c=
409+
github.com/tigera/api v0.0.0-20260310182635-546021df243c h1:XPwyK52lSkCFeTcUIOFUXp8+3HQr5ZHg83gk2gBCCKw=
410+
github.com/tigera/api v0.0.0-20260310182635-546021df243c/go.mod h1:CX/80DoAQD88kMvkvujV4h3XPmIBo0yeSf9OZmvZY0c=
411411
github.com/urfave/cli/v3 v3.6.2 h1:lQuqiPrZ1cIz8hz+HcrG0TNZFxU70dPZ3Yl+pSrH9A8=
412412
github.com/urfave/cli/v3 v3.6.2/go.mod h1:ysVLtOEmg2tOy6PknnYVhDoouyC/6N42TMeoMzskhso=
413413
github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo=

‎pkg/controller/installation/core_controller.go‎

Lines changed: 79 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1144,9 +1144,34 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile
11441144
if err != nil {
11451145
return false, err
11461146
}
1147-
return u || u2, nil
1147+
1148+
// Configure cluster routing mode.
1149+
u3, err := setClusterRoutingOnFelixConfiguration(instance, fc, reqLogger)
1150+
if err != nil {
1151+
return false, err
1152+
}
1153+
1154+
updated := u || u2 || u3
1155+
return updated, nil
1156+
})
1157+
if err != nil {
1158+
return reconcile.Result{}, err
1159+
}
1160+
1161+
// Set any non-default BGPConfiguration values that we need.
1162+
_, err = utils.PatchBGPConfiguration(ctx, r.client, func(bgpConfig *v3.BGPConfiguration) (bool, error) {
1163+
// Configure cluster routing mode.
1164+
u, err := setClusterRoutingOnBGPConfiguration(instance, bgpConfig, reqLogger)
1165+
if err != nil {
1166+
return false, err
1167+
}
1168+
1169+
return u, nil
11481170
})
11491171
if err != nil {
1172+
// Since, programClusterRoutes in FelixConfiguration is already updated earlier,
1173+
// failure in updating programClusterRouting in BGPConfiguration, essentially results in inconsistency
1174+
// between the configuration of BIRD and Felix in programming cluster routes, until the next reconcile convergence
11501175
return reconcile.Result{}, err
11511176
}
11521177

@@ -2011,6 +2036,59 @@ func (r *ReconcileInstallation) setDefaultsOnFelixConfiguration(ctx context.Cont
20112036
return updated, nil
20122037
}
20132038

2039+
// setClusterRoutingOnFelixConfiguration sets programClusterRoutes in the FelixConfiguration resource
2040+
// based on the value of clusterRoutingMode in the install config.
2041+
func setClusterRoutingOnFelixConfiguration(
2042+
install *operatorv1.Installation,
2043+
fc *v3.FelixConfiguration,
2044+
reqLogger logr.Logger,
2045+
) (bool, error) {
2046+
updated := false
2047+
desiredValue := "Disabled"
2048+
if felixProgramsClusterRoutes(install) {
2049+
desiredValue = "Enabled"
2050+
}
2051+
2052+
if fc.Spec.ProgramClusterRoutes == nil || *fc.Spec.ProgramClusterRoutes != desiredValue {
2053+
fc.Spec.ProgramClusterRoutes = &desiredValue
2054+
updated = true
2055+
reqLogger.Info("Patching FelixConfiguration", "programClusterRoutes", desiredValue)
2056+
}
2057+
2058+
return updated, nil
2059+
}
2060+
2061+
// setClusterRoutingOnBGPConfiguration sets programClusterRoutes in the BGPConfiguration resource
2062+
// based on the value of clusterRoutingMode in the install config.
2063+
func setClusterRoutingOnBGPConfiguration(
2064+
install *operatorv1.Installation,
2065+
bgpConfig *v3.BGPConfiguration,
2066+
reqLogger logr.Logger,
2067+
) (bool, error) {
2068+
updated := false
2069+
desiredValue := "Enabled"
2070+
2071+
if felixProgramsClusterRoutes(install) {
2072+
desiredValue = "Disabled"
2073+
}
2074+
2075+
if bgpConfig.Spec.ProgramClusterRoutes == nil || *bgpConfig.Spec.ProgramClusterRoutes != desiredValue {
2076+
bgpConfig.Spec.ProgramClusterRoutes = &desiredValue
2077+
updated = true
2078+
reqLogger.Info("Patching BGPConfiguration", "programClusterRoutes", desiredValue)
2079+
}
2080+
2081+
return updated, nil
2082+
}
2083+
2084+
func felixProgramsClusterRoutes(install *operatorv1.Installation) bool {
2085+
if install.Spec.CalicoNetwork != nil && install.Spec.CalicoNetwork.ClusterRoutingMode != nil &&
2086+
*install.Spec.CalicoNetwork.ClusterRoutingMode == operatorv1.ClusterRoutingModeFelix {
2087+
return true
2088+
}
2089+
return false
2090+
}
2091+
20142092
// setBPFUpdatesOnFelixConfiguration will take the passed in fc and update any BPF properties needed
20152093
// based on the install config and the daemonset.
20162094
func (r *ReconcileInstallation) setBPFUpdatesOnFelixConfiguration(ctx context.Context, install *operatorv1.Installation, fc *v3.FelixConfiguration, reqLogger logr.Logger) (bool, error) {

‎pkg/controller/installation/core_controller_test.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1427,6 +1427,92 @@ var _ = Describe("Testing core-controller installation", func() {
14271427
Expect(pullSecret.Kind).To(Equal("Installation"))
14281428
})
14291429

1430+
It("should correctly patch FelixConfig and BGPConfig with ClusterRouteMode not set", func() {
1431+
cr.Spec.CalicoNetwork = &operator.CalicoNetworkSpec{}
1432+
Expect(c.Create(ctx, cr)).NotTo(HaveOccurred())
1433+
_, err := r.Reconcile(ctx, reconcile.Request{})
1434+
Expect(err).ShouldNot(HaveOccurred())
1435+
1436+
fc := &v3.FelixConfiguration{}
1437+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, fc)
1438+
Expect(err).ShouldNot(HaveOccurred())
1439+
Expect(fc.Spec.ProgramClusterRoutes).NotTo(BeNil())
1440+
Expect(*fc.Spec.ProgramClusterRoutes).To(Equal("Disabled"))
1441+
1442+
bgpConfig := &v3.BGPConfiguration{}
1443+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, bgpConfig)
1444+
Expect(err).ShouldNot(HaveOccurred())
1445+
Expect(bgpConfig.Spec.ProgramClusterRoutes).NotTo(BeNil())
1446+
Expect(*bgpConfig.Spec.ProgramClusterRoutes).To(Equal("Enabled"))
1447+
})
1448+
1449+
It("should correctly patch FelixConfig and BGPConfig with ClusterRouteMode set to BIRD", func() {
1450+
bird := operator.ClusterRoutingModeBIRD
1451+
cr.Spec.CalicoNetwork = &operator.CalicoNetworkSpec{ClusterRoutingMode: &bird}
1452+
Expect(c.Create(ctx, cr)).NotTo(HaveOccurred())
1453+
_, err := r.Reconcile(ctx, reconcile.Request{})
1454+
Expect(err).ShouldNot(HaveOccurred())
1455+
1456+
fc := &v3.FelixConfiguration{}
1457+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, fc)
1458+
Expect(err).ShouldNot(HaveOccurred())
1459+
Expect(fc.Spec.ProgramClusterRoutes).NotTo(BeNil())
1460+
Expect(*fc.Spec.ProgramClusterRoutes).To(Equal("Disabled"))
1461+
1462+
bgpConfig := &v3.BGPConfiguration{}
1463+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, bgpConfig)
1464+
Expect(err).ShouldNot(HaveOccurred())
1465+
Expect(bgpConfig.Spec.ProgramClusterRoutes).NotTo(BeNil())
1466+
Expect(*bgpConfig.Spec.ProgramClusterRoutes).To(Equal("Enabled"))
1467+
})
1468+
1469+
It("should correctly patch FelixConfig and BGPConfig with ClusterRouteMode set to Felix", func() {
1470+
felix := operator.ClusterRoutingModeFelix
1471+
cr.Spec.CalicoNetwork = &operator.CalicoNetworkSpec{ClusterRoutingMode: &felix}
1472+
Expect(c.Create(ctx, cr)).NotTo(HaveOccurred())
1473+
_, err := r.Reconcile(ctx, reconcile.Request{})
1474+
Expect(err).ShouldNot(HaveOccurred())
1475+
1476+
fc := &v3.FelixConfiguration{}
1477+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, fc)
1478+
Expect(err).ShouldNot(HaveOccurred())
1479+
Expect(fc.Spec.ProgramClusterRoutes).NotTo(BeNil())
1480+
Expect(*fc.Spec.ProgramClusterRoutes).To(Equal("Enabled"))
1481+
1482+
bgpConfig := &v3.BGPConfiguration{}
1483+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, bgpConfig)
1484+
Expect(err).ShouldNot(HaveOccurred())
1485+
Expect(bgpConfig.Spec.ProgramClusterRoutes).NotTo(BeNil())
1486+
Expect(*bgpConfig.Spec.ProgramClusterRoutes).To(Equal("Disabled"))
1487+
})
1488+
1489+
It("should create the default BGPConfig and FelixConfig with ClusterRoutingMode set", func() {
1490+
bgpConfig := &v3.BGPConfiguration{}
1491+
err := c.Get(ctx, types.NamespacedName{Name: "default"}, bgpConfig)
1492+
Expect(err).Should(HaveOccurred())
1493+
1494+
fc := &v3.FelixConfiguration{}
1495+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, fc)
1496+
Expect(err).Should(HaveOccurred())
1497+
1498+
felix := operator.ClusterRoutingModeFelix
1499+
cr.Spec.CalicoNetwork = &operator.CalicoNetworkSpec{ClusterRoutingMode: &felix}
1500+
Expect(c.Create(ctx, cr)).NotTo(HaveOccurred())
1501+
_, err = r.Reconcile(ctx, reconcile.Request{})
1502+
Expect(err).ShouldNot(HaveOccurred())
1503+
1504+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, fc)
1505+
Expect(err).ShouldNot(HaveOccurred())
1506+
Expect(fc.Spec.ProgramClusterRoutes).NotTo(BeNil())
1507+
Expect(*fc.Spec.ProgramClusterRoutes).To(Equal("Enabled"))
1508+
1509+
bgpConfig = &v3.BGPConfiguration{}
1510+
err = c.Get(ctx, types.NamespacedName{Name: "default"}, bgpConfig)
1511+
Expect(err).ShouldNot(HaveOccurred())
1512+
Expect(bgpConfig.Spec.ProgramClusterRoutes).NotTo(BeNil())
1513+
Expect(*bgpConfig.Spec.ProgramClusterRoutes).To(Equal("Disabled"))
1514+
})
1515+
14301516
It("should set vxlanVNI to 10000 when provider is DockerEE", func() {
14311517
cr.Spec.KubernetesProvider = operator.ProviderDockerEE
14321518
Expect(c.Create(ctx, cr)).NotTo(HaveOccurred())

‎pkg/controller/installation/validation.go‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ func validateCustomResource(instance *operatorv1.Installation) error {
139139
// Verify Calico settings, if specified.
140140
if instance.Spec.CalicoNetwork != nil {
141141
bpfDataplane := instance.Spec.CalicoNetwork.LinuxDataplane != nil && *instance.Spec.CalicoNetwork.LinuxDataplane == operatorv1.LinuxDataplaneBPF
142+
felixClusterRoutingMode := felixProgramsClusterRoutes(instance)
142143

143144
// Perform validation on non-IPPool fields that rely on IP pool configuration. Validation of the IP pools themselves
144145
// happens in the IP pool controller.
@@ -167,15 +168,17 @@ func validateCustomResource(instance *operatorv1.Installation) error {
167168
// Verify the specified encapsulation type is valid.
168169
switch pool.Encapsulation {
169170
case operatorv1.EncapsulationIPIP, operatorv1.EncapsulationIPIPCrossSubnet:
170-
// IPIP currently requires BGP to be running in order to program routes.
171-
if instance.Spec.CalicoNetwork.BGP == nil || *instance.Spec.CalicoNetwork.BGP == operatorv1.BGPDisabled {
172-
return fmt.Errorf("IPIP encapsulation requires that BGP is enabled")
171+
// In BIRD cluster routing mode, IPIP currently requires BGP to be running in order to program routes.
172+
if !felixClusterRoutingMode &&
173+
(instance.Spec.CalicoNetwork.BGP == nil || *instance.Spec.CalicoNetwork.BGP == operatorv1.BGPDisabled) {
174+
return fmt.Errorf("with BIRD cluster routing mode, IPIP encapsulation requires that BGP is enabled")
173175
}
174176
case operatorv1.EncapsulationVXLAN, operatorv1.EncapsulationVXLANCrossSubnet:
175177
case operatorv1.EncapsulationNone:
176-
// Unencapsulated currently requires BGP to be running in order to program routes.
177-
if instance.Spec.CalicoNetwork.BGP == nil || *instance.Spec.CalicoNetwork.BGP == operatorv1.BGPDisabled {
178-
return fmt.Errorf("unencapsulated IP pools require that BGP is enabled")
178+
// In BIRD cluster routing mode, Unencapsulated currently requires BGP to be running in order to program routes.
179+
if !felixClusterRoutingMode &&
180+
(instance.Spec.CalicoNetwork.BGP == nil || *instance.Spec.CalicoNetwork.BGP == operatorv1.BGPDisabled) {
181+
return fmt.Errorf("with BIRD cluster routing mode, unencapsulated IP pools require that BGP is enabled")
179182
}
180183
}
181184
case operatorv1.IPAMPluginHostLocal:

‎pkg/controller/installation/validation_test.go‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@ var _ = Describe("Installation validation tests", func() {
245245
Expect(err).NotTo(HaveOccurred())
246246
})
247247

248-
It("should prevent IPIP if BGP is disabled", func() {
248+
It("should prevent IPIP with BGP disabled in BIRD cluster routing mode", func() {
249249
disabled := operator.BGPDisabled
250250
instance.Spec.CalicoNetwork.BGP = &disabled
251251
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
@@ -260,7 +260,7 @@ var _ = Describe("Installation validation tests", func() {
260260
Expect(err).To(HaveOccurred())
261261
})
262262

263-
It("should prevent IPIP cross-subnet if BGP is disabled", func() {
263+
It("should prevent IPIP cross-subnet with BGP disabled in BIRD cluster routing mode", func() {
264264
disabled := operator.BGPDisabled
265265
instance.Spec.CalicoNetwork.BGP = &disabled
266266
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
@@ -275,6 +275,72 @@ var _ = Describe("Installation validation tests", func() {
275275
Expect(err).To(HaveOccurred())
276276
})
277277

278+
It("should prevent no-encap with BGP disabled in BIRD cluster routing mode", func() {
279+
disabled := operator.BGPDisabled
280+
instance.Spec.CalicoNetwork.BGP = &disabled
281+
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
282+
{
283+
CIDR: "192.168.0.0/24",
284+
Encapsulation: operator.EncapsulationNone,
285+
NATOutgoing: operator.NATOutgoingEnabled,
286+
NodeSelector: "all()",
287+
},
288+
}
289+
err := validateCustomResource(instance)
290+
Expect(err).To(HaveOccurred())
291+
})
292+
293+
It("should allow IPIP with BGP disabled in Felix cluster routing mode", func() {
294+
disabled := operator.BGPDisabled
295+
instance.Spec.CalicoNetwork.BGP = &disabled
296+
clusterRoutingMode := operator.ClusterRoutingModeFelix
297+
instance.Spec.CalicoNetwork.ClusterRoutingMode = &clusterRoutingMode
298+
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
299+
{
300+
CIDR: "192.168.0.0/24",
301+
Encapsulation: operator.EncapsulationIPIP,
302+
NATOutgoing: operator.NATOutgoingEnabled,
303+
NodeSelector: "all()",
304+
},
305+
}
306+
err := validateCustomResource(instance)
307+
Expect(err).NotTo(HaveOccurred())
308+
})
309+
310+
It("should allow IPIP cross-subnet with BGP disabled in Felix cluster routing mode", func() {
311+
disabled := operator.BGPDisabled
312+
instance.Spec.CalicoNetwork.BGP = &disabled
313+
clusterRoutingMode := operator.ClusterRoutingModeFelix
314+
instance.Spec.CalicoNetwork.ClusterRoutingMode = &clusterRoutingMode
315+
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
316+
{
317+
CIDR: "192.168.0.0/24",
318+
Encapsulation: operator.EncapsulationIPIPCrossSubnet,
319+
NATOutgoing: operator.NATOutgoingEnabled,
320+
NodeSelector: "all()",
321+
},
322+
}
323+
err := validateCustomResource(instance)
324+
Expect(err).NotTo(HaveOccurred())
325+
})
326+
327+
It("should allow no-encap with BGP disabled in Felix cluster routing mode", func() {
328+
disabled := operator.BGPDisabled
329+
instance.Spec.CalicoNetwork.BGP = &disabled
330+
clusterRoutingMode := operator.ClusterRoutingModeFelix
331+
instance.Spec.CalicoNetwork.ClusterRoutingMode = &clusterRoutingMode
332+
instance.Spec.CalicoNetwork.IPPools = []operator.IPPool{
333+
{
334+
CIDR: "192.168.0.0/24",
335+
Encapsulation: operator.EncapsulationNone,
336+
NATOutgoing: operator.NATOutgoingEnabled,
337+
NodeSelector: "all()",
338+
},
339+
}
340+
err := validateCustomResource(instance)
341+
Expect(err).NotTo(HaveOccurred())
342+
})
343+
278344
It("should not error if CalicoNetwork is provided on EKS", func() {
279345
instance := &operator.Installation{}
280346
instance.Spec.CNI = &operator.CNISpec{Type: operator.PluginCalico}

0 commit comments

Comments
 (0)