Skip to content

Commit 73aaf39

Browse files
dspoAlinsRan
dspo
authored and
AlinsRan
committed
fix: Add provider endpoints to translate context (#2442)
1 parent f9cdfb6 commit 73aaf39

File tree

7 files changed

+154
-69
lines changed

7 files changed

+154
-69
lines changed

‎api/v1alpha1/gatewayproxy_types.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@ import (
2222
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2323
)
2424

25-
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
26-
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
27-
2825
// GatewayProxySpec defines the desired state of GatewayProxy.
2926
type GatewayProxySpec struct {
3027
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
@@ -116,9 +113,10 @@ type ControlPlaneAuth struct {
116113
}
117114

118115
// ControlPlaneProvider defines the configuration for control plane provider.
116+
// +kubebuilder:validation:XValidation:rule="has(self.endpoints) != has(self.service)"
119117
type ControlPlaneProvider struct {
120118
// Endpoints specifies the list of control plane endpoints.
121-
// +kubebuilder:validation:Required
119+
// +kubebuilder:validation:Optional
122120
// +kubebuilder:validation:MinItems=1
123121
Endpoints []string `json:"endpoints"`
124122

‎config/crd/bases/apisix.apache.org_gatewayproxies.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,8 +137,9 @@ spec:
137137
type: boolean
138138
required:
139139
- auth
140-
- endpoints
141140
type: object
141+
x-kubernetes-validations:
142+
- rule: has(self.endpoints) != has(self.service)
142143
type:
143144
description: Type specifies the type of provider. Can only be
144145
`ControlPlane`.

‎internal/controller/gatewayclass_congroller.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,9 +43,6 @@ const (
4343
FinalizerGatewayClassProtection = "apisix.apache.org/gc-protection"
4444
)
4545

46-
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=get;list;watch;update
47-
// +kubebuilder:rbac:groups=gateway.networking.k8s.io,resources=gatewayclasses/status,verbs=get;update
48-
4946
// GatewayClassReconciler reconciles a GatewayClass object.
5047
type GatewayClassReconciler struct { //nolint:revive
5148
client.Client

‎internal/controller/ingressclass_controller.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
networkingv1 "k8s.io/api/networking/v1"
2828
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2929
"k8s.io/apimachinery/pkg/runtime"
30+
"k8s.io/apimachinery/pkg/types"
3031
ctrl "sigs.k8s.io/controller-runtime"
3132
"sigs.k8s.io/controller-runtime/pkg/builder"
3233
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -229,6 +230,15 @@ func (r *IngressClassReconciler) processInfrastructure(tctx *provider.TranslateC
229230
}
230231
}
231232

233+
if service := gatewayProxy.Spec.Provider.ControlPlane.Service; service != nil {
234+
if err := addProviderEndpointsToTranslateContext(tctx, r.Client, types.NamespacedName{
235+
Namespace: gatewayProxy.GetNamespace(),
236+
Name: service.Name,
237+
}); err != nil {
238+
return err
239+
}
240+
}
241+
232242
_, ok := tctx.GatewayProxies[rk]
233243
if !ok {
234244
return fmt.Errorf("no gateway proxy found for ingress class")

‎internal/controller/utils.go‎

Lines changed: 102 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
"github.com/samber/lo"
3434
"go.uber.org/zap"
3535
corev1 "k8s.io/api/core/v1"
36+
discoveryv1 "k8s.io/api/discovery/v1"
3637
networkingv1 "k8s.io/api/networking/v1"
3738
"k8s.io/apimachinery/pkg/api/meta"
3839
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -921,34 +922,44 @@ func ProcessGatewayProxy(r client.Client, tctx *provider.TranslateContext, gatew
921922
tctx.ResourceParentRefs[rk] = append(tctx.ResourceParentRefs[rk], gatewayKind)
922923

923924
// Process provider secrets if provider exists
924-
if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane {
925-
if gatewayProxy.Spec.Provider.ControlPlane != nil &&
926-
gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey &&
927-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil &&
928-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil &&
929-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
930-
931-
secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef
932-
secret := &corev1.Secret{}
933-
if err := r.Get(context.Background(), client.ObjectKey{
934-
Namespace: ns,
935-
Name: secretRef.Name,
936-
}, secret); err != nil {
937-
log.Error(err, "failed to get secret for GatewayProxy provider",
938-
"namespace", ns,
939-
"name", secretRef.Name)
940-
return err
941-
}
925+
if prov := gatewayProxy.Spec.Provider; prov != nil && prov.Type == v1alpha1.ProviderTypeControlPlane {
926+
if cp := prov.ControlPlane; cp != nil {
927+
if cp.Auth.Type == v1alpha1.AuthTypeAdminKey &&
928+
cp.Auth.AdminKey != nil &&
929+
cp.Auth.AdminKey.ValueFrom != nil &&
930+
cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
931+
932+
secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
933+
secret := &corev1.Secret{}
934+
if err := r.Get(context.Background(), client.ObjectKey{
935+
Namespace: ns,
936+
Name: secretRef.Name,
937+
}, secret); err != nil {
938+
log.Error(err, "failed to get secret for GatewayProxy provider",
939+
"namespace", ns,
940+
"name", secretRef.Name)
941+
return err
942+
}
942943

943-
log.Info("found secret for GatewayProxy provider",
944-
"gateway", gateway.Name,
945-
"gatewayproxy", gatewayProxy.Name,
946-
"secret", secretRef.Name)
944+
log.Info("found secret for GatewayProxy provider",
945+
"gateway", gateway.Name,
946+
"gatewayproxy", gatewayProxy.Name,
947+
"secret", secretRef.Name)
947948

948-
tctx.Secrets[k8stypes.NamespacedName{
949-
Namespace: ns,
950-
Name: secretRef.Name,
951-
}] = secret
949+
tctx.Secrets[k8stypes.NamespacedName{
950+
Namespace: ns,
951+
Name: secretRef.Name,
952+
}] = secret
953+
}
954+
955+
if cp.Service != nil {
956+
if err := addProviderEndpointsToTranslateContext(tctx, r, k8stypes.NamespacedName{
957+
Namespace: gatewayProxy.GetNamespace(),
958+
Name: cp.Service.Name,
959+
}); err != nil {
960+
return err
961+
}
962+
}
952963
}
953964
}
954965
}
@@ -1340,33 +1351,45 @@ func ProcessIngressClassParameters(tctx *provider.TranslateContext, c client.Cli
13401351

13411352
// check if the provider field references a secret
13421353
if gatewayProxy.Spec.Provider != nil && gatewayProxy.Spec.Provider.Type == v1alpha1.ProviderTypeControlPlane {
1343-
if gatewayProxy.Spec.Provider.ControlPlane != nil &&
1344-
gatewayProxy.Spec.Provider.ControlPlane.Auth.Type == v1alpha1.AuthTypeAdminKey &&
1345-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey != nil &&
1346-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom != nil &&
1347-
gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
1348-
1349-
secretRef := gatewayProxy.Spec.Provider.ControlPlane.Auth.AdminKey.ValueFrom.SecretKeyRef
1350-
secret := &corev1.Secret{}
1351-
if err := c.Get(tctx, client.ObjectKey{
1352-
Namespace: ns,
1353-
Name: secretRef.Name,
1354-
}, secret); err != nil {
1355-
log.Error(err, "failed to get secret for GatewayProxy provider",
1356-
"namespace", ns,
1357-
"name", secretRef.Name)
1358-
return err
1359-
}
1354+
if cp := gatewayProxy.Spec.Provider.ControlPlane; cp != nil {
1355+
// process control plane provider auth
1356+
if cp.Auth.Type == v1alpha1.AuthTypeAdminKey &&
1357+
cp.Auth.AdminKey != nil &&
1358+
cp.Auth.AdminKey.ValueFrom != nil &&
1359+
cp.Auth.AdminKey.ValueFrom.SecretKeyRef != nil {
1360+
1361+
secretRef := cp.Auth.AdminKey.ValueFrom.SecretKeyRef
1362+
secret := &corev1.Secret{}
1363+
if err := c.Get(tctx, client.ObjectKey{
1364+
Namespace: ns,
1365+
Name: secretRef.Name,
1366+
}, secret); err != nil {
1367+
log.Error(err, "failed to get secret for GatewayProxy provider",
1368+
"namespace", ns,
1369+
"name", secretRef.Name)
1370+
return err
1371+
}
13601372

1361-
log.Info("found secret for GatewayProxy provider",
1362-
"ingressClass", ingressClass.Name,
1363-
"gatewayproxy", gatewayProxy.Name,
1364-
"secret", secretRef.Name)
1373+
log.Info("found secret for GatewayProxy provider",
1374+
"ingressClass", ingressClass.Name,
1375+
"gatewayproxy", gatewayProxy.Name,
1376+
"secret", secretRef.Name)
1377+
1378+
tctx.Secrets[k8stypes.NamespacedName{
1379+
Namespace: ns,
1380+
Name: secretRef.Name,
1381+
}] = secret
1382+
}
13651383

1366-
tctx.Secrets[k8stypes.NamespacedName{
1367-
Namespace: ns,
1368-
Name: secretRef.Name,
1369-
}] = secret
1384+
// process control plane provider service
1385+
if cp.Service != nil {
1386+
if err := addProviderEndpointsToTranslateContext(tctx, c, client.ObjectKey{
1387+
Namespace: gatewayProxy.GetNamespace(),
1388+
Name: cp.Service.Name,
1389+
}); err != nil {
1390+
return err
1391+
}
1392+
}
13701393
}
13711394
}
13721395
}
@@ -1420,3 +1443,31 @@ func distinctRequests(requests []reconcile.Request) []reconcile.Request {
14201443
}
14211444
return distinctRequests
14221445
}
1446+
1447+
func addProviderEndpointsToTranslateContext(tctx *provider.TranslateContext, c client.Client, serviceNN k8stypes.NamespacedName) error {
1448+
log.Debugf("to process provider endpints by provider.service: %s", serviceNN)
1449+
var (
1450+
service corev1.Service
1451+
)
1452+
if err := c.Get(tctx, serviceNN, &service); err != nil {
1453+
log.Error(err, "failed to get service from GatewayProxy provider", "key", serviceNN)
1454+
return err
1455+
}
1456+
tctx.Services[serviceNN] = &service
1457+
1458+
// get es
1459+
var (
1460+
esList discoveryv1.EndpointSliceList
1461+
)
1462+
if err := c.List(tctx, &esList,
1463+
client.InNamespace(serviceNN.Namespace),
1464+
client.MatchingLabels{
1465+
discoveryv1.LabelServiceName: serviceNN.Name,
1466+
}); err != nil {
1467+
log.Error(err, "failed to get endpoints for GatewayProxy provider", "endpoints", serviceNN)
1468+
return err
1469+
}
1470+
tctx.EndpointSlices[serviceNN] = esList.Items
1471+
1472+
return nil
1473+
}

‎internal/provider/adc/config.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,20 @@
1818
package adc
1919

2020
import (
21-
"errors"
22-
"fmt"
21+
"net"
2322
"slices"
23+
"strconv"
2424

2525
"github.com/api7/gopkg/pkg/log"
26+
"github.com/pkg/errors"
2627
"go.uber.org/zap"
2728
k8stypes "k8s.io/apimachinery/pkg/types"
2829
"k8s.io/utils/ptr"
2930
v1 "sigs.k8s.io/gateway-api/apis/v1"
3031

3132
"github.com/apache/apisix-ingress-controller/api/v1alpha1"
3233
"github.com/apache/apisix-ingress-controller/internal/provider"
33-
types "github.com/apache/apisix-ingress-controller/internal/types"
34+
"github.com/apache/apisix-ingress-controller/internal/types"
3435
)
3536

3637
func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, gatewayProxy *v1alpha1.GatewayProxy) (*adcConfig, error) {
@@ -86,24 +87,27 @@ func (d *adcClient) getConfigsForGatewayProxy(tctx *provider.TranslateContext, g
8687
}
8788
_, ok := tctx.Services[namespacedName]
8889
if !ok {
89-
return nil, errors.New("no service found for service reference")
90+
return nil, errors.Errorf("no service found for service reference: %s", namespacedName)
9091
}
9192
endpoint := tctx.EndpointSlices[namespacedName]
9293
if endpoint == nil {
9394
return nil, nil
9495
}
9596
upstreamNodes, err := d.translator.TranslateBackendRef(tctx, v1.BackendRef{
9697
BackendObjectReference: v1.BackendObjectReference{
97-
Name: v1.ObjectName(provider.ControlPlane.Service.Name),
98-
Port: ptr.To(v1.PortNumber(provider.ControlPlane.Service.Port)),
98+
Name: v1.ObjectName(provider.ControlPlane.Service.Name),
99+
Namespace: (*v1.Namespace)(&gatewayProxy.Namespace),
100+
Port: ptr.To(v1.PortNumber(provider.ControlPlane.Service.Port)),
99101
},
100102
})
101103
if err != nil {
102104
return nil, err
103105
}
104106
for _, node := range upstreamNodes {
105-
config.ServerAddrs = append(config.ServerAddrs, fmt.Sprintf("http://%s:%d", node.Host, node.Port))
107+
config.ServerAddrs = append(config.ServerAddrs, "http://"+net.JoinHostPort(node.Host, strconv.Itoa(node.Port)))
106108
}
109+
110+
log.Debugf("add server address to config.ServiceAddrs: %v", config.ServerAddrs)
107111
}
108112

109113
return &config, nil

‎test/e2e/gatewayapi/httproute.go‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,36 @@ import (
3434
"sigs.k8s.io/gateway-api/apis/v1alpha2"
3535

3636
"github.com/apache/apisix-ingress-controller/api/v1alpha1"
37+
"github.com/apache/apisix-ingress-controller/internal/provider/adc"
3738
"github.com/apache/apisix-ingress-controller/test/e2e/framework"
3839
"github.com/apache/apisix-ingress-controller/test/e2e/scaffold"
3940
)
4041

4142
var _ = Describe("Test HTTPRoute", Label("networking.k8s.io", "httproute"), func() {
4243
s := scaffold.NewDefaultScaffold()
4344

44-
var gatewayProxyYaml = `
45+
getGatewayProxySpec := func(endpoint, adminKey string) string {
46+
if framework.ProviderType == adc.BackendModeAPISIXStandalone {
47+
var gatewayProxyYaml = `
48+
apiVersion: apisix.apache.org/v1alpha1
49+
kind: GatewayProxy
50+
metadata:
51+
name: apisix-proxy-config
52+
spec:
53+
provider:
54+
type: ControlPlane
55+
controlPlane:
56+
service:
57+
name: apisix-standalone
58+
port: 9180
59+
auth:
60+
type: AdminKey
61+
adminKey:
62+
value: "%s"
63+
`
64+
return fmt.Sprintf(gatewayProxyYaml, adminKey)
65+
}
66+
var gatewayProxyYaml = `
4567
apiVersion: apisix.apache.org/v1alpha1
4668
kind: GatewayProxy
4769
metadata:
@@ -57,6 +79,8 @@ spec:
5779
adminKey:
5880
value: "%s"
5981
`
82+
return fmt.Sprintf(gatewayProxyYaml, endpoint, adminKey)
83+
}
6084

6185
var gatewayClassYaml = `
6286
apiVersion: gateway.networking.k8s.io/v1
@@ -129,7 +153,7 @@ spec:
129153

130154
var beforeEachHTTP = func() {
131155
By("create GatewayProxy")
132-
gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Deployer.GetAdminEndpoint(), s.AdminKey())
156+
gatewayProxy := getGatewayProxySpec(s.Deployer.GetAdminEndpoint(), s.AdminKey())
133157
err := s.CreateResourceFromString(gatewayProxy)
134158
Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
135159
time.Sleep(5 * time.Second)
@@ -160,7 +184,7 @@ spec:
160184

161185
var beforeEachHTTPS = func() {
162186
By("create GatewayProxy")
163-
gatewayProxy := fmt.Sprintf(gatewayProxyYaml, s.Deployer.GetAdminEndpoint(), s.AdminKey())
187+
gatewayProxy := getGatewayProxySpec(s.Deployer.GetAdminEndpoint(), s.AdminKey())
164188
err := s.CreateResourceFromString(gatewayProxy)
165189
Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy")
166190
time.Sleep(5 * time.Second)

0 commit comments

Comments
 (0)