Skip to content

Commit bcd8033

Browse files
LucasgranetLucasgranet
authored
feat(occm/lb): octavia proxy protocol v2 support (#2635)
1 parent 799ebfe commit bcd8033

File tree

3 files changed

+147
-27
lines changed

3 files changed

+147
-27
lines changed

‎docs/openstack-cloud-controller-manager/expose-applications-using-loadbalancer-type-service.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,11 @@ Request Body:
136136

137137
- `loadbalancer.openstack.org/proxy-protocol`
138138

139-
If 'true', the loadbalancer pool protocol will be set as `PROXY`. Default is 'false'.
139+
Enable the ProxyProtocol on all listeners. Default is 'false'.
140+
141+
Values:
142+
- `v1`, `true`: enable the ProxyProtocol version 1
143+
- `v2`: enable the ProxyProtocol version 2
140144

141145
Not supported when `lb-provider=ovn` is configured in openstack-cloud-controller-manager.
142146

@@ -402,6 +406,8 @@ To enable PROXY protocol support, the either the openstack-cloud-controller-mana
402406
app: nginx-ingress
403407
```
404408

409+
> To use the ProxyProtocol's version 2, set the annotation's value to `v2`. By default, ProxyProtocol's version 1 is used.
410+
405411
Wait until the service gets an external IP.
406412

407413
```bash
@@ -500,6 +506,8 @@ To enable PROXY protocol support, the either the openstack-cloud-controller-mana
500506
-no body in request-
501507
```
502508

509+
> Note: the Proxy Protocol is only available with TCP services.
510+
503511
### Sharing load balancer with multiple Services
504512

505513
By default, different Services of LoadBalancer type should have different corresponding cloud load balancers, however, openstack-cloud-controller-manager allows multiple Services to share a single load balancer if the Octavia service supports the tag feature (since API version 2.5).

‎pkg/openstack/loadbalancer.go‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ type serviceConfig struct {
124124
lbPublicSubnetSpec *floatingSubnetSpec
125125
nodeSelectors map[string]string
126126
keepClientIP bool
127-
enableProxyProtocol bool
127+
proxyProtocolVersion *v2pools.Protocol
128128
timeoutClientData int
129129
timeoutMemberConnect int
130130
timeoutMemberData int
@@ -474,6 +474,21 @@ func getBoolFromServiceAnnotation(service *corev1.Service, annotationKey string,
474474
return defaultSetting
475475
}
476476

477+
// getProxyProtocolFromServiceAnnotation searches a given v1.Service the ServiceAnnotationLoadBalancerProxyEnabled to guess if the proxyProtocol needs to be
478+
// enabled and return the ProxyProtocol's version which is need to be applied
479+
func getProxyProtocolFromServiceAnnotation(service *corev1.Service) *v2pools.Protocol {
480+
switch getStringFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, "false") {
481+
case "true":
482+
return ptr.To(v2pools.ProtocolPROXY)
483+
case "v1":
484+
return ptr.To(v2pools.ProtocolPROXY)
485+
case "v2":
486+
return ptr.To(v2pools.ProtocolPROXYV2)
487+
default:
488+
return nil
489+
}
490+
}
491+
477492
// getSubnetIDForLB returns subnet-id for a specific node
478493
func getSubnetIDForLB(network *gophercloud.ServiceClient, node corev1.Node, preferredIPFamily corev1.IPFamily) (string, error) {
479494
ipAddress, err := nodeAddressForLB(&node, preferredIPFamily)
@@ -868,8 +883,8 @@ func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, name string, listener *list
868883

869884
// By default, use the protocol of the listener
870885
poolProto := v2pools.Protocol(listener.Protocol)
871-
if svcConf.enableProxyProtocol {
872-
poolProto = v2pools.ProtocolPROXY
886+
if svcConf.proxyProtocolVersion != nil {
887+
poolProto = *svcConf.proxyProtocolVersion
873888
} else if (svcConf.keepClientIP || svcConf.tlsContainerRef != "") && poolProto != v2pools.ProtocolHTTP {
874889
poolProto = v2pools.ProtocolHTTP
875890
}
@@ -935,8 +950,8 @@ func (lbaas *LbaasV2) ensureOctaviaPool(lbID string, name string, listener *list
935950
func (lbaas *LbaasV2) buildPoolCreateOpt(listenerProtocol string, service *corev1.Service, svcConf *serviceConfig, name string) v2pools.CreateOpts {
936951
// By default, use the protocol of the listener
937952
poolProto := v2pools.Protocol(listenerProtocol)
938-
if svcConf.enableProxyProtocol {
939-
poolProto = v2pools.ProtocolPROXY
953+
if svcConf.proxyProtocolVersion != nil {
954+
poolProto = *svcConf.proxyProtocolVersion
940955
} else if (svcConf.keepClientIP || svcConf.tlsContainerRef != "") && poolProto != v2pools.ProtocolHTTP {
941956
if svcConf.keepClientIP && svcConf.tlsContainerRef != "" {
942957
klog.V(4).Infof("Forcing to use %q protocol for pool because annotations %q %q are set", v2pools.ProtocolHTTP, ServiceAnnotationLoadBalancerXForwardedFor, ServiceAnnotationTlsContainerRef)
@@ -1310,12 +1325,11 @@ func (lbaas *LbaasV2) checkServiceUpdate(service *corev1.Service, nodes []*corev
13101325

13111326
// This affects the protocol of listener and pool
13121327
keepClientIP := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerXForwardedFor, false)
1313-
useProxyProtocol := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, false)
1314-
if useProxyProtocol && keepClientIP {
1328+
svcConf.proxyProtocolVersion = getProxyProtocolFromServiceAnnotation(service)
1329+
if svcConf.proxyProtocolVersion != nil && keepClientIP {
13151330
return fmt.Errorf("annotation %s and %s cannot be used together", ServiceAnnotationLoadBalancerProxyEnabled, ServiceAnnotationLoadBalancerXForwardedFor)
13161331
}
13171332
svcConf.keepClientIP = keepClientIP
1318-
svcConf.enableProxyProtocol = useProxyProtocol
13191333

13201334
svcConf.tlsContainerRef = getStringFromServiceAnnotation(service, ServiceAnnotationTlsContainerRef, lbaas.opts.TlsContainerRef)
13211335
svcConf.enableMonitor = getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerEnableHealthMonitor, lbaas.opts.CreateMonitor)
@@ -1335,7 +1349,7 @@ func (lbaas *LbaasV2) checkServiceDelete(service *corev1.Service, svcConf *servi
13351349

13361350
// This affects the protocol of listener and pool
13371351
svcConf.keepClientIP = getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerXForwardedFor, false)
1338-
svcConf.enableProxyProtocol = getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, false)
1352+
svcConf.proxyProtocolVersion = getProxyProtocolFromServiceAnnotation(service)
13391353
svcConf.tlsContainerRef = getStringFromServiceAnnotation(service, ServiceAnnotationTlsContainerRef, lbaas.opts.TlsContainerRef)
13401354

13411355
return nil
@@ -1537,12 +1551,11 @@ func (lbaas *LbaasV2) checkService(service *corev1.Service, nodes []*corev1.Node
15371551
}
15381552

15391553
keepClientIP := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerXForwardedFor, false)
1540-
useProxyProtocol := getBoolFromServiceAnnotation(service, ServiceAnnotationLoadBalancerProxyEnabled, false)
1541-
if useProxyProtocol && keepClientIP {
1554+
svcConf.proxyProtocolVersion = getProxyProtocolFromServiceAnnotation(service)
1555+
if svcConf.proxyProtocolVersion != nil && keepClientIP {
15421556
return fmt.Errorf("annotation %s and %s cannot be used together", ServiceAnnotationLoadBalancerProxyEnabled, ServiceAnnotationLoadBalancerXForwardedFor)
15431557
}
15441558
svcConf.keepClientIP = keepClientIP
1545-
svcConf.enableProxyProtocol = useProxyProtocol
15461559

15471560
if openstackutil.IsOctaviaFeatureSupported(lbaas.lb, openstackutil.OctaviaFeatureTimeout, lbaas.opts.LBProvider) {
15481561
svcConf.timeoutClientData = getIntFromServiceAnnotation(service, ServiceAnnotationLoadBalancerTimeoutClientData, 50000)
@@ -1627,7 +1640,7 @@ func (lbaas *LbaasV2) createLoadBalancerStatus(service *corev1.Service, svcConf
16271640
}
16281641

16291642
ipMode := corev1.LoadBalancerIPModeVIP
1630-
if svcConf.enableProxyProtocol {
1643+
if svcConf.proxyProtocolVersion != nil {
16311644
// If the load balancer is using the PROXY protocol, expose its IP address via
16321645
// the Hostname field to prevent kube-proxy from injecting an iptables bypass.
16331646
// Setting must be removed by the user to allow the use of the LoadBalancerIPModeProxy.

‎pkg/openstack/loadbalancer_test.go‎

Lines changed: 112 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package openstack
33
import (
44
"context"
55
"fmt"
6+
"k8s.io/utils/ptr"
67
"reflect"
78
"sort"
89
"testing"
@@ -747,7 +748,7 @@ func TestLbaasV2_createLoadBalancerStatus(t *testing.T) {
747748
},
748749
},
749750
svcConf: &serviceConfig{
750-
enableProxyProtocol: false,
751+
proxyProtocolVersion: nil,
751752
},
752753
addr: "10.10.0.6",
753754
},
@@ -772,7 +773,7 @@ func TestLbaasV2_createLoadBalancerStatus(t *testing.T) {
772773
},
773774
},
774775
svcConf: &serviceConfig{
775-
enableProxyProtocol: true,
776+
proxyProtocolVersion: ptr.To(pools.ProtocolPROXY),
776777
},
777778
addr: "10.10.0.6",
778779
},
@@ -797,7 +798,7 @@ func TestLbaasV2_createLoadBalancerStatus(t *testing.T) {
797798
},
798799
},
799800
svcConf: &serviceConfig{
800-
enableProxyProtocol: false,
801+
proxyProtocolVersion: nil,
801802
},
802803
addr: "10.10.0.6",
803804
},
@@ -823,7 +824,7 @@ func TestLbaasV2_createLoadBalancerStatus(t *testing.T) {
823824
},
824825
},
825826
svcConf: &serviceConfig{
826-
enableProxyProtocol: true,
827+
proxyProtocolVersion: ptr.To(pools.ProtocolPROXY),
827828
},
828829
addr: "10.10.0.6",
829830
},
@@ -981,9 +982,9 @@ func Test_buildPoolCreateOpt(t *testing.T) {
981982
args: args{
982983
protocol: "TCP",
983984
svcConf: &serviceConfig{
984-
keepClientIP: true,
985-
tlsContainerRef: "tls-container-ref",
986-
enableProxyProtocol: true,
985+
keepClientIP: true,
986+
tlsContainerRef: "tls-container-ref",
987+
proxyProtocolVersion: ptr.To(pools.ProtocolPROXY),
987988
},
988989
lbaasV2: &LbaasV2{
989990
LoadBalancer{
@@ -1011,9 +1012,9 @@ func Test_buildPoolCreateOpt(t *testing.T) {
10111012
args: args{
10121013
protocol: "HTTP",
10131014
svcConf: &serviceConfig{
1014-
keepClientIP: true,
1015-
tlsContainerRef: "tls-container-ref",
1016-
enableProxyProtocol: false,
1015+
keepClientIP: true,
1016+
tlsContainerRef: "tls-container-ref",
1017+
proxyProtocolVersion: nil,
10171018
},
10181019
lbaasV2: &LbaasV2{
10191020
LoadBalancer{
@@ -1041,9 +1042,9 @@ func Test_buildPoolCreateOpt(t *testing.T) {
10411042
args: args{
10421043
protocol: "UDP",
10431044
svcConf: &serviceConfig{
1044-
keepClientIP: true,
1045-
tlsContainerRef: "tls-container-ref",
1046-
enableProxyProtocol: false,
1045+
keepClientIP: true,
1046+
tlsContainerRef: "tls-container-ref",
1047+
proxyProtocolVersion: nil,
10471048
},
10481049
lbaasV2: &LbaasV2{
10491050
LoadBalancer{
@@ -1124,6 +1125,36 @@ func Test_buildPoolCreateOpt(t *testing.T) {
11241125
Persistence: &pools.SessionPersistence{Type: "SOURCE_IP"},
11251126
},
11261127
},
1128+
{
1129+
name: "test for proxy protocol v2 enabled",
1130+
args: args{
1131+
protocol: "TCP",
1132+
svcConf: &serviceConfig{
1133+
keepClientIP: true,
1134+
tlsContainerRef: "tls-container-ref",
1135+
proxyProtocolVersion: ptr.To(pools.ProtocolPROXYV2),
1136+
},
1137+
lbaasV2: &LbaasV2{
1138+
LoadBalancer{
1139+
opts: LoadBalancerOpts{
1140+
LBProvider: "ovn",
1141+
LBMethod: "SOURCE_IP_PORT",
1142+
},
1143+
},
1144+
},
1145+
service: &corev1.Service{
1146+
Spec: corev1.ServiceSpec{
1147+
SessionAffinity: corev1.ServiceAffinityClientIP,
1148+
},
1149+
},
1150+
},
1151+
want: pools.CreateOpts{
1152+
Name: "test for proxy protocol v2 enabled",
1153+
Protocol: pools.ProtocolPROXYV2,
1154+
LBMethod: "SOURCE_IP_PORT",
1155+
Persistence: &pools.SessionPersistence{Type: "SOURCE_IP"},
1156+
},
1157+
},
11271158
}
11281159

11291160
for _, tt := range tests {
@@ -2553,3 +2584,71 @@ func TestFilterNodes(t *testing.T) {
25532584
})
25542585
}
25552586
}
2587+
2588+
func Test_getProxyProtocolFromServiceAnnotation(t *testing.T) {
2589+
type args struct {
2590+
service *corev1.Service
2591+
}
2592+
tests := []struct {
2593+
name string
2594+
args args
2595+
want *pools.Protocol
2596+
}{
2597+
{
2598+
name: "no proxy protocol",
2599+
args: args{service: &corev1.Service{
2600+
ObjectMeta: v1.ObjectMeta{
2601+
Annotations: map[string]string{ServiceAnnotationLoadBalancerProxyEnabled: "false"},
2602+
},
2603+
}},
2604+
want: nil,
2605+
},
2606+
{
2607+
name: "proxy protocol true",
2608+
args: args{service: &corev1.Service{
2609+
ObjectMeta: v1.ObjectMeta{
2610+
Annotations: map[string]string{ServiceAnnotationLoadBalancerProxyEnabled: "true"},
2611+
},
2612+
}},
2613+
want: ptr.To(pools.ProtocolPROXY),
2614+
},
2615+
{
2616+
name: "proxy protocol v1",
2617+
args: args{service: &corev1.Service{
2618+
ObjectMeta: v1.ObjectMeta{
2619+
Annotations: map[string]string{ServiceAnnotationLoadBalancerProxyEnabled: "v1"},
2620+
},
2621+
}},
2622+
want: ptr.To(pools.ProtocolPROXY),
2623+
},
2624+
{
2625+
name: "proxy protocol v2",
2626+
args: args{service: &corev1.Service{
2627+
ObjectMeta: v1.ObjectMeta{
2628+
Annotations: map[string]string{ServiceAnnotationLoadBalancerProxyEnabled: "v2"},
2629+
},
2630+
}},
2631+
want: ptr.To(pools.ProtocolPROXYV2),
2632+
},
2633+
{
2634+
name: "no proxy protocol annotation",
2635+
args: args{service: &corev1.Service{}},
2636+
want: nil,
2637+
},
2638+
{
2639+
name: "unknown proxy protocol annotation's value",
2640+
args: args{service: &corev1.Service{
2641+
ObjectMeta: v1.ObjectMeta{
2642+
Annotations: map[string]string{ServiceAnnotationLoadBalancerProxyEnabled: "not valid value"},
2643+
},
2644+
}},
2645+
want: nil,
2646+
},
2647+
}
2648+
for _, tt := range tests {
2649+
t.Run(tt.name, func(t *testing.T) {
2650+
got := getProxyProtocolFromServiceAnnotation(tt.args.service)
2651+
assert.Equalf(t, tt.want, got, "getProxyProtocolFromServiceAnnotation(%v)", tt.args.service)
2652+
})
2653+
}
2654+
}

0 commit comments

Comments
 (0)