Skip to content

Commit a78ff3e

Browse files
jukiejukie
authored
feat: Implement ZoneAware loadbalancing - ZoneAware Lb Config (splitup envoyproxy#6482) (envoyproxy#6485)
* First phase - Implement ZoneAware Lb Config Signed-off-by: Jukie <[email protected]> * Adjust conditional and description Signed-off-by: Jukie <[email protected]> * consolidate preferlocal Signed-off-by: jukie <[email protected]>
1 parent 6be1747 commit a78ff3e

15 files changed

+453
-82
lines changed

‎internal/gatewayapi/route.go‎

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1426,7 +1426,7 @@ func (t *Translator) processDestination(name string, backendRefContext BackendRe
14261426
ds = t.processServiceDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, resources, envoyProxy)
14271427
svc := resources.GetService(backendNamespace, string(backendRef.Name))
14281428
ds.IPFamily = getServiceIPFamily(svc)
1429-
ds.ZoneAwareRouting = processZoneAwareRouting(svc)
1429+
ds.PreferLocal = processPreferLocalZone(svc)
14301430

14311431
case egv1a1.KindBackend:
14321432
ds = t.processBackendDestinationSetting(name, backendRef.BackendObjectReference, backendNamespace, protocol, resources)
@@ -1599,12 +1599,12 @@ func (t *Translator) processServiceDestinationSetting(
15991599
}
16001600

16011601
return &ir.DestinationSetting{
1602-
Name: name,
1603-
Protocol: protocol,
1604-
Endpoints: endpoints,
1605-
AddressType: addrType,
1606-
ZoneAwareRouting: processZoneAwareRouting(service),
1607-
Metadata: buildResourceMetadata(service, ptr.To(gwapiv1.SectionName(strconv.Itoa(int(*backendRef.Port))))),
1602+
Name: name,
1603+
Protocol: protocol,
1604+
Endpoints: endpoints,
1605+
AddressType: addrType,
1606+
PreferLocal: processPreferLocalZone(service),
1607+
Metadata: buildResourceMetadata(service, ptr.To(gwapiv1.SectionName(strconv.Itoa(int(*backendRef.Port))))),
16081608
}
16091609
}
16101610

@@ -1624,14 +1624,17 @@ func getBackendFilters(routeType gwapiv1.Kind, backendRefContext BackendRefConte
16241624
return nil
16251625
}
16261626

1627-
func processZoneAwareRouting(svc *corev1.Service) *ir.ZoneAwareRouting {
1627+
func processPreferLocalZone(svc *corev1.Service) *ir.PreferLocalZone {
16281628
if svc == nil {
16291629
return nil
16301630
}
16311631

16321632
if trafficDist := svc.Spec.TrafficDistribution; trafficDist != nil {
1633-
return &ir.ZoneAwareRouting{
1634-
MinSize: 1,
1633+
return &ir.PreferLocalZone{
1634+
MinEndpointsThreshold: ptr.To[uint64](1),
1635+
Force: &ir.ForceLocalZone{
1636+
MinEndpointsInZoneThreshold: ptr.To[uint32](1),
1637+
},
16351638
}
16361639
}
16371640

@@ -1640,8 +1643,11 @@ func processZoneAwareRouting(svc *corev1.Service) *ir.ZoneAwareRouting {
16401643
// https://kubernetes.io/docs/concepts/services-networking/topology-aware-routing/#enabling-topology-aware-routing
16411644
// https://github.com/kubernetes/kubernetes/blob/9d9e1afdf78bce0a517cc22557457f942040ca19/staging/src/k8s.io/endpointslice/utils.go#L355-L368
16421645
if val, ok := svc.Annotations[corev1.AnnotationTopologyMode]; ok && val == "Auto" || val == "auto" {
1643-
return &ir.ZoneAwareRouting{
1644-
MinSize: 3,
1646+
return &ir.PreferLocalZone{
1647+
MinEndpointsThreshold: ptr.To[uint64](3),
1648+
Force: &ir.ForceLocalZone{
1649+
MinEndpointsInZoneThreshold: ptr.To[uint32](3),
1650+
},
16451651
}
16461652
}
16471653

‎internal/gatewayapi/testdata/httproute-with-enable-zone-discovery.out.yaml‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -157,10 +157,12 @@ xdsIR:
157157
namespace: default
158158
sectionName: "8080"
159159
name: httproute/default/httproute-1/rule/1/backend/0
160+
preferLocal:
161+
force:
162+
minEndpointsInZoneThreshold: 1
163+
minEndpointsThreshold: 1
160164
protocol: HTTP
161165
weight: 1
162-
zoneAwareRouting:
163-
minSize: 1
164166
headerMatches:
165167
- distinct: false
166168
exact: bar
@@ -195,10 +197,12 @@ xdsIR:
195197
namespace: default
196198
sectionName: "8080"
197199
name: httproute/default/httproute-1/rule/0/backend/0
200+
preferLocal:
201+
force:
202+
minEndpointsInZoneThreshold: 1
203+
minEndpointsThreshold: 1
198204
protocol: HTTP
199205
weight: 1
200-
zoneAwareRouting:
201-
minSize: 1
202206
hostname: '*'
203207
isHTTP2: false
204208
metadata:

‎internal/ir/xds.go‎

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -832,6 +832,15 @@ func (h *HTTPRoute) GetRetry() *Retry {
832832
return nil
833833
}
834834

835+
func (h *HTTPRoute) NeedsClusterPerSetting() bool {
836+
if h.Traffic != nil &&
837+
h.Traffic.LoadBalancer != nil &&
838+
h.Traffic.LoadBalancer.PreferLocal != nil {
839+
return true
840+
}
841+
return h.Destination.NeedsClusterPerSetting()
842+
}
843+
835844
// DNS contains configuration options for DNS resolution.
836845
// +k8s:deepcopy-gen=true
837846
type DNS struct {
@@ -1582,7 +1591,7 @@ func (r *RouteDestination) Validate() error {
15821591
func (r *RouteDestination) NeedsClusterPerSetting() bool {
15831592
return r.HasMixedEndpoints() ||
15841593
r.HasFiltersInSettings() ||
1585-
(len(r.Settings) > 1 && r.HasZoneAwareRouting())
1594+
(len(r.Settings) > 1 && r.HasPreferLocalZone())
15861595
}
15871596

15881597
// HasMixedEndpoints returns true if the RouteDestination has endpoints of multiple types
@@ -1607,10 +1616,10 @@ func (r *RouteDestination) HasFiltersInSettings() bool {
16071616
return false
16081617
}
16091618

1610-
// HasZoneAwareRouting returns true if any setting in the destination has ZoneAwareRoutingEnabled set
1611-
func (r *RouteDestination) HasZoneAwareRouting() bool {
1619+
// HasPreferLocalZone returns true if any setting in the destination has PreferLocalZone set
1620+
func (r *RouteDestination) HasPreferLocalZone() bool {
16121621
for _, setting := range r.Settings {
1613-
if setting.ZoneAwareRouting != nil {
1622+
if setting.PreferLocal != nil {
16141623
return true
16151624
}
16161625
}
@@ -1673,11 +1682,9 @@ type DestinationSetting struct {
16731682
IPFamily *egv1a1.IPFamily `json:"ipFamily,omitempty" yaml:"ipFamily,omitempty"`
16741683
TLS *TLSUpstreamConfig `json:"tls,omitempty" yaml:"tls,omitempty"`
16751684
Filters *DestinationFilters `json:"filters,omitempty" yaml:"filters,omitempty"`
1676-
// ZoneAwareRouting specifies whether to enable Zone Aware Routing for this destination's endpoints.
1685+
// PreferLocal specifies whether to enable Zone Aware Routing for this destination's endpoints.
16771686
// This is derived from the backend service and depends on having Kubernetes Topology Aware Routing or Traffic Distribution enabled.
1678-
//
1679-
// +optional
1680-
ZoneAwareRouting *ZoneAwareRouting `json:"zoneAwareRouting,omitempty" yaml:"zoneAwareRouting,omitempty"`
1687+
PreferLocal *PreferLocalZone `json:"preferLocal,omitempty" yaml:"preferLocal,omitempty"`
16811688
// Metadata is used to enrich envoy route metadata with user and provider-specific information
16821689
// The primary metadata for DestinationSettings comes from the Backend resource reference in BackendRef
16831690
Metadata *ResourceMetadata `json:"metadata,omitempty" yaml:"metadata,omitempty"`
@@ -2523,6 +2530,8 @@ type LoadBalancer struct {
25232530
Random *Random `json:"random,omitempty" yaml:"random,omitempty"`
25242531
// ConsistentHash load balancer policy
25252532
ConsistentHash *ConsistentHash `json:"consistentHash,omitempty" yaml:"consistentHash,omitempty"`
2533+
// PreferLocal defines the configuration related to the distribution of requests between locality zones.
2534+
PreferLocal *PreferLocalZone `json:"preferLocal,omitempty" yaml:"preferLocal,omitempty"`
25262535
}
25272536

25282537
// Validate the fields within the LoadBalancer structure
@@ -3203,8 +3212,21 @@ type RequestBuffer struct {
32033212
Limit resource.Quantity `json:"limit" yaml:"limit"`
32043213
}
32053214

3206-
// ZoneAwareRouting holds the zone aware routing configuration
3215+
// PreferLocalZone configures zone-aware routing to prefer sending traffic to the local locality zone.
3216+
// +k8s:deepcopy-gen=true
3217+
type PreferLocalZone struct {
3218+
// ForceLocalZone defines override configuration for forcing all traffic to stay within the local zone instead of the default behavior
3219+
// which maintains equal distribution among upstream endpoints while sending as much traffic as possible locally.
3220+
Force *ForceLocalZone `json:"force,omitempty" yaml:"force,omitempty"`
3221+
// MinEndpointsThreshold is the minimum number of total upstream endpoints across all zones required to enable zone-aware routing.
3222+
MinEndpointsThreshold *uint64 `json:"minEndpointsThreshold,omitempty" yaml:"minEndpointsThreshold,omitempty"`
3223+
}
3224+
3225+
// ForceLocalZone defines override configuration for forcing all traffic to stay within the local zone instead of the default behavior
3226+
// which maintains equal distribution among upstream endpoints while sending as much traffic as possible locally.
32073227
// +k8s:deepcopy-gen=true
3208-
type ZoneAwareRouting struct {
3209-
MinSize int `json:"minSize" yaml:"minSize"`
3228+
type ForceLocalZone struct {
3229+
// MinEndpointsInZoneThreshold is the minimum number of upstream endpoints in the local zone required to honor the forceLocalZone
3230+
// override. This is useful for protecting zones with fewer endpoints.
3231+
MinEndpointsInZoneThreshold *uint32 `json:"minEndpointsInZoneThreshold,omitempty" yaml:"minEndpointsInZoneThreshold,omitempty"`
32103232
}

‎internal/ir/xds_test.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1125,8 +1125,10 @@ func TestRouteDestination_NeedsClusterPerSetting(t *testing.T) {
11251125
Port: 8080,
11261126
},
11271127
},
1128-
AddressType: ptr.To(FQDN),
1129-
ZoneAwareRouting: &ZoneAwareRouting{MinSize: 1},
1128+
AddressType: ptr.To(FQDN),
1129+
PreferLocal: &PreferLocalZone{
1130+
Force: &ForceLocalZone{MinEndpointsInZoneThreshold: ptr.To[uint32](1)},
1131+
},
11301132
},
11311133
{
11321134
Endpoints: []*DestinationEndpoint{
@@ -1153,8 +1155,10 @@ func TestRouteDestination_NeedsClusterPerSetting(t *testing.T) {
11531155
Port: 8080,
11541156
},
11551157
},
1156-
AddressType: ptr.To(FQDN),
1157-
ZoneAwareRouting: &ZoneAwareRouting{MinSize: 1},
1158+
AddressType: ptr.To(FQDN),
1159+
PreferLocal: &PreferLocalZone{
1160+
Force: &ForceLocalZone{MinEndpointsInZoneThreshold: ptr.To[uint32](1)},
1161+
},
11581162
},
11591163
},
11601164
},

‎internal/ir/zz_generated.deepcopy.go‎

Lines changed: 54 additions & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/xds/translator/cluster.go‎

Lines changed: 44 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -228,25 +228,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
228228
cluster.TypedExtensionProtocolOptions = epo
229229
}
230230

231-
// Set default localityLbConfig
232-
localityLbConfig := &commonv3.LocalityLbConfig{
233-
LocalityConfigSpecifier: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig_{
234-
LocalityWeightedLbConfig: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig{},
235-
},
236-
}
237-
238-
// Override LocalityWeightedLbConfig if zone aware routing is enabled.
239-
// Zone aware enabled backendRefs always have a single DestinationSetting per-cluster.
240-
if len(args.settings) == 1 && args.settings[0].ZoneAwareRouting != nil {
241-
localityLbConfig.LocalityConfigSpecifier = &commonv3.LocalityLbConfig_ZoneAwareLbConfig_{
242-
ZoneAwareLbConfig: &commonv3.LocalityLbConfig_ZoneAwareLbConfig{
243-
MinClusterSize: wrapperspb.UInt64(1),
244-
ForceLocalZone: &commonv3.LocalityLbConfig_ZoneAwareLbConfig_ForceLocalZone{
245-
MinSize: wrapperspb.UInt32(uint32(args.settings[0].ZoneAwareRouting.MinSize)),
246-
},
247-
},
248-
}
249-
}
231+
localityLbConfig := buildLocalityLbConfig(args)
250232

251233
// Set Load Balancer policy
252234
//nolint:gocritic
@@ -439,6 +421,48 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
439421
}, nil
440422
}
441423

424+
func buildLocalityLbConfig(args *xdsClusterArgs) *commonv3.LocalityLbConfig {
425+
// Default to LocalityWeightedLbConfig
426+
localityLbConfig := &commonv3.LocalityLbConfig{
427+
LocalityConfigSpecifier: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig_{
428+
LocalityWeightedLbConfig: &commonv3.LocalityLbConfig_LocalityWeightedLbConfig{},
429+
},
430+
}
431+
432+
// Check for LoadBalancer.PreferLocal configuration or if backendRef enables
433+
// PreferLocal (such as Topology Aware Routing or Traffic Distribution)
434+
if preferLocal := ptr.Deref(args.loadBalancer, ir.LoadBalancer{}).PreferLocal; preferLocal != nil {
435+
if cfg := buildZoneAwareLbConfig(preferLocal); cfg != nil {
436+
localityLbConfig.LocalityConfigSpecifier = cfg
437+
}
438+
// Zone aware enabled backendRefs use weighted clusters and
439+
// always have a single DestinationSetting per-cluster.
440+
} else if len(args.settings) == 1 && args.settings[0].PreferLocal != nil {
441+
if cfg := buildZoneAwareLbConfig(args.settings[0].PreferLocal); cfg != nil {
442+
localityLbConfig.LocalityConfigSpecifier = cfg
443+
}
444+
}
445+
446+
return localityLbConfig
447+
}
448+
449+
func buildZoneAwareLbConfig(preferLocal *ir.PreferLocalZone) *commonv3.LocalityLbConfig_ZoneAwareLbConfig_ {
450+
if preferLocal == nil {
451+
return nil
452+
}
453+
lbConfig := &commonv3.LocalityLbConfig_ZoneAwareLbConfig_{
454+
ZoneAwareLbConfig: &commonv3.LocalityLbConfig_ZoneAwareLbConfig{
455+
MinClusterSize: wrapperspb.UInt64(ptr.Deref(preferLocal.MinEndpointsThreshold, 6)),
456+
},
457+
}
458+
if preferLocal.Force != nil {
459+
lbConfig.ZoneAwareLbConfig.ForceLocalZone = &commonv3.LocalityLbConfig_ZoneAwareLbConfig_ForceLocalZone{
460+
MinSize: wrapperspb.UInt32(ptr.Deref(preferLocal.Force.MinEndpointsInZoneThreshold, 1)),
461+
}
462+
}
463+
return lbConfig
464+
}
465+
442466
func buildXdsHealthCheck(healthcheck *ir.ActiveHealthCheck) []*corev3.HealthCheck {
443467
hc := &corev3.HealthCheck{
444468
Timeout: durationpb.New(healthcheck.Timeout.Duration),
@@ -650,7 +674,7 @@ func buildXdsClusterLoadAssignment(clusterName string, destSettings []*ir.Destin
650674
// if multiple backendRefs exist. This pushes part of the routing logic higher up the stack which can
651675
// limit host selection controls during retries and session affinity.
652676
// For more details see https://github.com/envoyproxy/gateway/issues/5307#issuecomment-2688767482
653-
if ds.ZoneAwareRouting != nil {
677+
if ds.PreferLocal != nil {
654678
localities = append(localities, buildZonalLocalities(metadata, ds)...)
655679
} else {
656680
localities = append(localities, buildWeightedLocalities(metadata, ds))

0 commit comments

Comments
 (0)