Skip to content

Commit 8c42248

Browse files
dspodspo
committed
fix: add support for EndpointSubset in GatewayProxy and update RBAC permissions
1 parent 2647195 commit 8c42248

File tree

11 files changed

+182
-87
lines changed

11 files changed

+182
-87
lines changed

‎api/v1alpha1/gatewayproxy_types.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,9 +116,10 @@ type ControlPlaneAuth struct {
116116
}
117117

118118
// ControlPlaneProvider defines the configuration for control plane provider.
119+
// +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)"
119120
type ControlPlaneProvider struct {
120121
// Endpoints specifies the list of control plane endpoints.
121-
// +kubebuilder:validation:Required
122+
// +kubebuilder:validation:Optional
122123
// +kubebuilder:validation:MinItems=1
123124
Endpoints []string `json:"endpoints"`
124125

‎config/crd/bases/apisix.apache.org_gatewayproxies.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,9 @@ spec:
137137
type: boolean
138138
required:
139139
- auth
140-
- endpoints
141140
type: object
141+
x-kubernetes-validations:
142+
- rule: has(self.endpoints) != has(self.service)
142143
type:
143144
description: Type specifies the type of provider. Can only be
144145
`ControlPlane`.

‎config/rbac/role.yaml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,7 @@ rules:
77
- apiGroups:
88
- ""
99
resources:
10-
- events
11-
verbs:
12-
- create
13-
- patch
14-
- apiGroups:
15-
- ""
16-
resources:
10+
- endpoints
1711
- namespaces
1812
- pods
1913
- secrets
@@ -22,6 +16,13 @@ rules:
2216
- get
2317
- list
2418
- watch
19+
- apiGroups:
20+
- ""
21+
resources:
22+
- events
23+
verbs:
24+
- create
25+
- patch
2526
- apiGroups:
2627
- apisix.apache.org
2728
resources:

‎internal/controller/gatewayclass_congroller.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,6 @@ const (
4343
FinalizerGatewayClassProtection = "apisix.apache.org/gc-protection"
4444
)
4545

46-
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update
47-
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;update
48-
4946
// GatewayClassReconciler reconciles a GatewayClass object.
5047
type GatewayClassReconciler struct { //nolint:revive
5148
client.Client

‎internal/controller/ingressclass_controller.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
networkingv1 "k8s.io/api/networking/v1"
2828
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2929
"k8s.io/apimachinery/pkg/runtime"
30+
"k8s.io/apimachinery/pkg/types"
3031
ctrl "sigs.k8s.io/controller-runtime"
3132
"sigs.k8s.io/controller-runtime/pkg/builder"
3233
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -227,6 +228,16 @@ func (r *IngressClassReconciler) processInfrastructure(tctx *provider.TranslateC
227228
}] = secret
228229
}
229230
}
231+
232+
if service := gatewayProxy.Spec.Provider.ControlPlane.Service; service != nil {
233+
if err := addProviderEndpointsToTranslateContext(tctx, r.Client, types.NamespacedName{
234+
Namespace: gatewayProxy.GetNamespace(),
235+
Name: service.Name,
236+
}); err != nil {
237+
return err
238+
}
239+
}
240+
230241
}
231242

232243
_, ok := tctx.GatewayProxies[rk]

‎internal/controller/utils.go‎

Lines changed: 97 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -921,34 +921,44 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew
921921
tctx.ResourceParentRefs[rk] = append(tctx.ResourceParentRefs[rk], gatewayKind)
922922

923923
// Process provider secrets if provider exists
924-
if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane {
925-
if gatewayProxy.Spec.Provider.ControlPlane != nil &&
926-
gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey &&
927-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil &&
928-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil &&
929-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
930-
931-
secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef
932-
secret := &corev1.Secret{}
933-
if err := r.Get(context.Background(), client.ObjectKey{
934-
Namespace: ns,
935-
Name: secretRef.Name,
936-
}, secret); err != nil {
937-
log.Error(err, "failed to get secret for GatewayProxy provider",
938-
"namespace", ns,
939-
"name", secretRef.Name)
940-
return err
941-
}
924+
if prov := gatewayProxy.Spec.Provider; prov != nil && prov.Type == v1alpha1.ProviderTypeControlPlane {
925+
if cp := prov.ControlPlane; cp != nil {
926+
if cp.Auth.Type == v1alpha1.AuthTypeAdminKey &&
927+
cp.Auth.AdminKey != nil &&
928+
cp.Auth.AdminKey.ValueFrom != nil &&
929+
cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
930+
931+
secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
932+
secret := &corev1.Secret{}
933+
if err := r.Get(context.Background(), client.ObjectKey{
934+
Namespace: ns,
935+
Name: secretRef.Name,
936+
}, secret); err != nil {
937+
log.Error(err, "failed to get secret for GatewayProxy provider",
938+
"namespace", ns,
939+
"name", secretRef.Name)
940+
return err
941+
}
942942

943-
log.Info("found secret for GatewayProxy provider",
944-
"gateway", gateway.Name,
945-
"gatewayproxy", gatewayProxy.Name,
946-
"secret", secretRef.Name)
943+
log.Info("found secret for GatewayProxy provider",
944+
"gateway", gateway.Name,
945+
"gatewayproxy", gatewayProxy.Name,
946+
"secret", secretRef.Name)
947947

948-
tctx.Secrets[k8stypes.NamespacedName{
949-
Namespace: ns,
950-
Name: secretRef.Name,
951-
}] = secret
948+
tctx.Secrets[k8stypes.NamespacedName{
949+
Namespace: ns,
950+
Name: secretRef.Name,
951+
}] = secret
952+
}
953+
954+
if cp.Service != nil {
955+
if err := addProviderEndpointsToTranslateContext(tctx, r, k8stypes.NamespacedName{
956+
Namespace: gatewayProxy.GetNamespace(),
957+
Name: cp.Service.Name,
958+
}); err != nil {
959+
return err
960+
}
961+
}
952962
}
953963
}
954964
}
@@ -1340,40 +1350,76 @@ func ProcessIngressClassParameters(tctx *provider.TranslateContext, c client.Cli
13401350

13411351
// check if the provider field references a secret
13421352
if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane {
1343-
if gatewayProxy.Spec.Provider.ControlPlane != nil &&
1344-
gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey &&
1345-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil &&
1346-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil &&
1347-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
1348-
1349-
secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef
1350-
secret := &corev1.Secret{}
1351-
if err := c.Get(tctx, client.ObjectKey{
1352-
Namespace: ns,
1353-
Name: secretRef.Name,
1354-
}, secret); err != nil {
1355-
log.Error(err, "failed to get secret for GatewayProxy provider",
1356-
"namespace", ns,
1357-
"name", secretRef.Name)
1358-
return err
1359-
}
1353+
if cp := gatewayProxy.Spec.Provider.ControlPlane; cp != nil {
1354+
// process control plane provider auth
1355+
if cp.Auth.Type == v1alpha1.AuthTypeAdminKey &&
1356+
cp.Auth.AdminKey != nil &&
1357+
cp.Auth.AdminKey.ValueFrom != nil &&
1358+
cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
1359+
1360+
secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
1361+
secret := &corev1.Secret{}
1362+
if err := c.Get(tctx, client.ObjectKey{
1363+
Namespace: ns,
1364+
Name: secretRef.Name,
1365+
}, secret); err != nil {
1366+
log.Error(err, "failed to get secret for GatewayProxy provider",
1367+
"namespace", ns,
1368+
"name", secretRef.Name)
1369+
return err
1370+
}
13601371

1361-
log.Info("found secret for GatewayProxy provider",
1362-
"ingressClass", ingressClass.Name,
1363-
"gatewayproxy", gatewayProxy.Name,
1364-
"secret", secretRef.Name)
1372+
log.Info("found secret for GatewayProxy provider",
1373+
"ingressClass", ingressClass.Name,
1374+
"gatewayproxy", gatewayProxy.Name,
1375+
"secret", secretRef.Name)
1376+
1377+
tctx.Secrets[k8stypes.NamespacedName{
1378+
Namespace: ns,
1379+
Name: secretRef.Name,
1380+
}] = secret
1381+
}
13651382

1366-
tctx.Secrets[k8stypes.NamespacedName{
1367-
Namespace: ns,
1368-
Name: secretRef.Name,
1369-
}] = secret
1383+
// process control plane provider service
1384+
if cp.Service != nil {
1385+
if err := addProviderEndpointsToTranslateContext(tctx, c, client.ObjectKey{
1386+
Namespace: gatewayProxy.GetNamespace(),
1387+
Name: cp.Service.Name,
1388+
}); err != nil {
1389+
return err
1390+
}
1391+
}
13701392
}
13711393
}
13721394
}
13731395

13741396
return nil
13751397
}
13761398

1399+
func addProviderEndpointsToTranslateContext(tctx *provider.TranslateContext, c client.Client, serviceNN k8stypes.NamespacedName) error {
1400+
log.Debugf("to process provider endpints by provider.service: %s", serviceNN)
1401+
var (
1402+
service corev1.Service
1403+
)
1404+
if err := c.Get(tctx, serviceNN, &service); err != nil {
1405+
log.Error(err, "failed to get service from GatewayProxy provider", "key", serviceNN)
1406+
return err
1407+
}
1408+
tctx.Services[serviceNN] = &service
1409+
1410+
// get endpoints
1411+
var (
1412+
endpoints corev1.Endpoints
1413+
)
1414+
if err := c.Get(tctx, serviceNN, &endpoints); err != nil {
1415+
log.Error(err, "failed to get endpoints for GatewayProxy provider", "endpoints", serviceNN)
1416+
return err
1417+
}
1418+
tctx.EndpointSubset[serviceNN] = endpoints.Subsets
1419+
1420+
return nil
1421+
}
1422+
13771423
func GetIngressClass(ctx context.Context, c client.Client, log logr.Logger, ingressClassName string) (*networkingv1.IngressClass, error) {
13781424
if ingressClassName == "" {
13791425
// Check for default ingress class

‎internal/manager/controllers.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
// +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
3535
// +kubebuilder:rbac:groups="discovery.k8s.io",resources=endpointslices,verbs=get;list;watch
3636
// +kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch
37+
// +kubebuilder:rbac:groups="",resources=endpoints,verbs=get;list;watch
3738
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch
3839
// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
3940
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch

‎internal/provider/adc/config.go‎

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,19 @@
1818
package adc
1919

2020
import (
21-
"errors"
22-
"fmt"
21+
"net"
2322
"slices"
23+
"strconv"
2424

2525
"github.com/api7/gopkg/pkg/log"
26+
"github.com/pkg/errors"
2627
"go.uber.org/zap"
28+
corev1 "k8s.io/api/core/v1"
2729
k8stypes "k8s.io/apimachinery/pkg/types"
28-
"k8s.io/utils/ptr"
29-
v1 "sigs.k8s.io/gateway-api/apis/v1"
3030

3131
"github.com/apache/apisix-ingress-controller/api/v1alpha1"
3232
"github.com/apache/apisix-ingress-controller/internal/provider"
33-
types "github.com/apache/apisix-ingress-controller/internal/types"
33+
"github.com/apache/apisix-ingress-controller/internal/types"
3434
)
3535

3636
func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) {
@@ -86,24 +86,34 @@ func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, g
8686
}
8787
_, ok := tctx.Services[namespacedName]
8888
if !ok {
89-
return nil, errors.New("no service found for service reference")
89+
return nil, errors.Errorf("no service found for service reference: %s", namespacedName)
9090
}
91-
endpoint := tctx.EndpointSlices[namespacedName]
92-
if endpoint == nil {
93-
return nil, nil
94-
}
95-
upstreamNodes, err := d.translator.TranslateBackendRef(tctx, v1.BackendRef{
96-
BackendObjectReference: v1.BackendObjectReference{
97-
Name: v1.ObjectName(provider.ControlPlane.Service.Name),
98-
Port: ptr.To(v1.PortNumber(provider.ControlPlane.Service.Port)),
99-
},
100-
})
101-
if err != nil {
102-
return nil, err
103-
}
104-
for _, node := range upstreamNodes {
105-
config.ServerAddrs = append(config.ServerAddrs, fmt.Sprintf("http://%s:%d", node.Host, node.Port))
91+
92+
var (
93+
subsets = tctx.EndpointSubset[namespacedName]
94+
port = strconv.FormatInt(int64(provider.ControlPlane.Service.Port), 10)
95+
)
96+
for _, endpoint := range subsets {
97+
if !slices.ContainsFunc(endpoint.Ports, func(port corev1.EndpointPort) bool {
98+
return port.Port == provider.ControlPlane.Service.Port
99+
}) {
100+
continue
101+
}
102+
103+
for _, addresses := range [][]corev1.EndpointAddress{
104+
endpoint.NotReadyAddresses,
105+
endpoint.Addresses,
106+
} {
107+
for _, addr := range addresses {
108+
serverAddr := "http://" + net.JoinHostPort(addr.IP, port)
109+
if tlsVerify := provider.ControlPlane.TlsVerify; tlsVerify != nil && *tlsVerify {
110+
serverAddr = "https://" + net.JoinHostPort(addr.IP, port)
111+
}
112+
config.ServerAddrs = append(config.ServerAddrs, serverAddr)
113+
}
114+
}
106115
}
116+
log.Debugf("add server address to config.ServiceAddrs: %v", config.ServerAddrs)
107117
}
108118

109119
return &config, nil

‎internal/provider/provider.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ type TranslateContext struct {
4747
Credentials []v1alpha1.Credential
4848

4949
EndpointSlices map[k8stypes.NamespacedName][]discoveryv1.EndpointSlice
50+
EndpointSubset map[k8stypes.NamespacedName][]corev1.EndpointSubset
5051
Secrets map[k8stypes.NamespacedName]*corev1.Secret
5152
PluginConfigs map[k8stypes.NamespacedName]*v1alpha1.PluginConfig
5253
ApisixPluginConfigs map[k8stypes.NamespacedName]*apiv2.ApisixPluginConfig
@@ -64,6 +65,7 @@ func NewDefaultTranslateContext(ctx context.Context) *TranslateContext {
6465
return &TranslateContext{
6566
Context: ctx,
6667
EndpointSlices: make(map[k8stypes.NamespacedName][]discoveryv1.EndpointSlice),
68+
EndpointSubset: make(map[k8stypes.NamespacedName][]corev1.EndpointSubset),
6769
Secrets: make(map[k8stypes.NamespacedName]*corev1.Secret),
6870
PluginConfigs: make(map[k8stypes.NamespacedName]*v1alpha1.PluginConfig),
6971
ApisixPluginConfigs: make(map[k8stypes.NamespacedName]*apiv2.ApisixPluginConfig),

‎test/e2e/framework/manifests/ingress.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ rules:
8181
- ""
8282
resources:
8383
- namespaces
84+
- endpoints
8485
verbs:
8586
- get
8687
- list

0 commit comments

Comments
 (0)