Skip to content

Commit d5c885a

Browse files
committed
report warning events for ingress and ingressclass reconciliation failures
1 parent 3fed355 commit d5c885a

File tree

44 files changed

+2731
-107
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2731
-107
lines changed

go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
github.com/go-openapi/swag v0.22.3 // indirect
3636
github.com/gofrs/flock v0.8.1 // indirect
3737
github.com/gogo/protobuf v1.3.2 // indirect
38+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
3839
github.com/golang/protobuf v1.5.3 // indirect
3940
github.com/google/gnostic v0.5.7-v3refs // indirect
4041
github.com/google/go-cmp v0.6.0 // indirect

go.sum

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
3838
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
3939
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
4040
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
41+
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
4142
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
4243
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
4344
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

helm/oci-native-ingress-controller/templates/deployment.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ spec:
6969
- --metrics-port={{.Values.metrics.port}}
7070
- --v=4
7171
- --use-lb-compartment-for-certificates={{ .Values.useLbCompartmentForCertificates }}
72+
- --emit-events={{ .Values.emitEvents }}
7273
env:
7374
- name: OCI_RESOURCE_PRINCIPAL_VERSION
7475
value: "2.2"

helm/oci-native-ingress-controller/templates/rbac.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@ metadata:
1212
labels:
1313
{{- include "oci-native-ingress-controller.labels" . | nindent 4 }}
1414
rules:
15-
- apiGroups: [""]
15+
- apiGroups: ["", "events.k8s.io"]
1616
resources: [events]
17-
verbs: [create, patch]
17+
verbs: [create, patch, update]
1818
- apiGroups: [""]
1919
resources: [pods]
2020
verbs: [get, list, watch]

helm/oci-native-ingress-controller/values.yaml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,5 +124,9 @@ metrics:
124124
backend: prometheus
125125
port: 2223
126126

127-
# Use the compartment supplied in IngressClassParam.spec.compartmentId for certificate management
127+
# Use the compartment supplied in IngressClassParameters.spec.compartmentId for certificate management.
128+
# If set to false, the default compartment_id specified in this file is used for this purpose instead.
128129
useLbCompartmentForCertificates: false
130+
131+
# Emit kubernetes events for Ingress/IngressClass errors observed during reconciliation
132+
emitEvents: false

main.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import (
3434
clientset "k8s.io/client-go/kubernetes"
3535
"k8s.io/client-go/kubernetes/scheme"
3636
"k8s.io/client-go/tools/cache"
37+
events "k8s.io/client-go/tools/events"
3738
"k8s.io/client-go/tools/leaderelection"
3839
"k8s.io/client-go/tools/leaderelection/resourcelock"
3940

@@ -61,6 +62,7 @@ func main() {
6162
flag.IntVar(&opts.MetricsPort, "metrics-port", 2223, "Metrics port for metrics backend")
6263
flag.BoolVar(&opts.UseLbCompartmentForCertificates, "use-lb-compartment-for-certificates", false,
6364
"use the compartment supplied in IngressClassParam.spec.compartmentId for certificate management")
65+
flag.BoolVar(&opts.EmitEvents, "emit-events", false, "emit kubernetes events for Ingress/IngressClass errors observed during reconciliation")
6466

6567
var logFile string
6668
flag.StringVar(&logFile, "log-file", "", "absolute path to the file where application logs will be stored")
@@ -171,11 +173,19 @@ func main() {
171173

172174
ingressClassInformer, ingressInformer, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer := setupInformers(informerFactory, ctx, classParamInformer)
173175

176+
var eventRecorder events.EventRecorder
177+
if opts.EmitEvents {
178+
eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
179+
eventBroadcaster.StartStructuredLogging(5)
180+
eventBroadcaster.StartRecordingToSink(make(chan struct{}))
181+
eventRecorder = eventBroadcaster.NewRecorder(scheme.Scheme, opts.ControllerClass)
182+
}
183+
174184
server.SetupWebhookServer(ingressInformer, serviceInformer, client, ctx)
175185
mux := http.NewServeMux()
176186
reg, err := server.SetupMetricsServer(opts.MetricsBackend, opts.MetricsPort, mux, ctx)
177187

178-
run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer, c, reg)
188+
run := server.SetUpControllers(opts, ingressClassInformer, ingressInformer, client, serviceInformer, secretInformer, endpointInformer, podInformer, nodeInformer, serviceAccountInformer, c, reg, eventRecorder)
179189

180190
metric.ServeMetrics(opts.MetricsPort, mux)
181191
// we use the Lease lock type since edits to Leases are less common

pkg/certificate/certificate.go

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -122,12 +122,14 @@ func (certificatesClient *CertificatesClient) ListCertificates(ctx context.Conte
122122
func (certificatesClient *CertificatesClient) UpdateCertificate(ctx context.Context,
123123
req certificatesmanagement.UpdateCertificateRequest) (*certificatesmanagement.Certificate, string, error) {
124124
_, err := certificatesClient.ManagementClient.UpdateCertificate(ctx, req)
125+
125126
if err != nil {
126-
if !util.IsServiceError(err, 409) {
127-
klog.Errorf("Error updating certificate %s: %s", *req.CertificateId, err)
128-
} else {
127+
if util.IsServiceError(err, 409) {
129128
klog.Errorf("Error updating certificate %s due to 409-Conflict", *req.CertificateId)
129+
} else {
130+
klog.Errorf("Error updating certificate %s: %s", *req.CertificateId, err)
130131
}
132+
131133
return nil, "", err
132134
}
133135

@@ -223,12 +225,14 @@ func (certificatesClient *CertificatesClient) ListCaBundles(ctx context.Context,
223225
func (certificatesClient *CertificatesClient) UpdateCaBundle(ctx context.Context,
224226
req certificatesmanagement.UpdateCaBundleRequest) (*certificatesmanagement.CaBundle, string, error) {
225227
_, err := certificatesClient.ManagementClient.UpdateCaBundle(ctx, req)
228+
226229
if err != nil {
227-
if !util.IsServiceError(err, 409) {
228-
klog.Errorf("Error updating ca bundle %s: %s", *req.CaBundleId, err)
229-
} else {
230+
if util.IsServiceError(err, 409) {
230231
klog.Errorf("Error updating ca bundle %s due to 409-Conflict", *req.CaBundleId)
232+
} else {
233+
klog.Errorf("Error updating ca bundle %s: %s", *req.CaBundleId, err)
231234
}
235+
232236
return nil, "", err
233237
}
234238

pkg/controllers/backend/backend.go

Lines changed: 61 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"encoding/json"
1515
"fmt"
1616
coreinformers "k8s.io/client-go/informers/core/v1"
17+
"k8s.io/client-go/tools/events"
1718
"time"
1819

1920
"github.com/oracle/oci-native-ingress-controller/pkg/client"
@@ -47,6 +48,7 @@ type Controller struct {
4748
podLister corelisters.PodLister
4849
endpointLister corelisters.EndpointsLister
4950
saLister corelisters.ServiceAccountLister
51+
eventRecorder events.EventRecorder
5052

5153
queue workqueue.RateLimitingInterface
5254
client *client.ClientProvider
@@ -61,6 +63,7 @@ func NewController(
6163
endpointLister corelisters.EndpointsLister,
6264
podLister corelisters.PodLister,
6365
client *client.ClientProvider,
66+
eventRecorder events.EventRecorder,
6467
) *Controller {
6568

6669
c := &Controller{
@@ -72,6 +75,7 @@ func NewController(
7275
podLister: podLister,
7376
saLister: saInformer.Lister(),
7477
client: client,
78+
eventRecorder: eventRecorder,
7579
queue: workqueue.NewRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(10*time.Second, 5*time.Minute)),
7680
}
7781

@@ -93,7 +97,7 @@ func (c *Controller) processNextItem() bool {
9397
err := c.sync(key.(string))
9498

9599
// Handle the error if something went wrong during the execution of the business logic
96-
util.HandleErr(c.queue, err, "Error syncing backends for ingress class", key)
100+
util.HandleErrForBackendController(c.eventRecorder, c.ingressClassLister, c.queue, err, "Error syncing backends for ingress class", key)
97101
return true
98102
}
99103

@@ -157,50 +161,69 @@ func (c *Controller) ensureBackends(ctx context.Context, ingressClass *networkin
157161
}
158162

159163
for _, ingress := range ingresses {
160-
for _, rule := range ingress.Spec.Rules {
161-
for _, path := range rule.HTTP.Paths {
162-
pSvc, svc, err := util.ExtractServices(path, c.serviceLister, ingress)
163-
if err != nil {
164-
return err
165-
}
166-
svcName, svcPort, targetPort, err := util.PathToServiceAndTargetPort(c.endpointLister, svc, pSvc, ingress.Namespace, false)
167-
if err != nil {
168-
return err
169-
}
170-
epAddrs, err := util.GetEndpoints(c.endpointLister, ingress.Namespace, svcName)
171-
if err != nil {
172-
return fmt.Errorf("unable to fetch endpoints for %s/%s/%d: %w", ingress.Namespace, svcName, targetPort, err)
173-
}
174-
backends := []ociloadbalancer.BackendDetails{}
175-
for _, epAddr := range epAddrs {
176-
backends = append(backends, util.NewBackend(epAddr.IP, targetPort))
177-
}
178-
backendSetName := util.GenerateBackendSetName(ingress.Namespace, svcName, svcPort)
179-
err = wrapperClient.GetLbClient().UpdateBackends(context.TODO(), lbID, backendSetName, backends)
164+
err := c.ensureBackendsForIngress(ctx, ingress, ingressClass, lbID, wrapperClient)
165+
if err != nil {
166+
return fmt.Errorf("for ingress %s, encountered error: %w", klog.KObj(ingress), err)
167+
}
168+
}
169+
170+
// Sync default backends
171+
c.syncDefaultBackend(ctx, lbID, ingresses)
172+
return nil
173+
}
174+
175+
func (c *Controller) ensureBackendsForIngress(ctx context.Context, ingress *networkingv1.Ingress, ingressClass *networkingv1.IngressClass,
176+
lbID string, wrapperClient *client.WrapperClient) error {
177+
178+
for _, rule := range ingress.Spec.Rules {
179+
for _, path := range rule.HTTP.Paths {
180+
pSvc, svc, err := util.ExtractServices(path, c.serviceLister, ingress)
181+
if err != nil {
182+
return err
183+
}
184+
svcName, svcPort, targetPort, err := util.PathToServiceAndTargetPort(c.endpointLister, svc, pSvc, ingress.Namespace, false)
185+
if err != nil {
186+
return err
187+
}
188+
189+
epAddrs, err := util.GetEndpoints(c.endpointLister, ingress.Namespace, svcName)
190+
if err != nil {
191+
return fmt.Errorf("unable to fetch endpoints for %s/%s/%d: %w", ingress.Namespace, svcName, targetPort, err)
192+
}
193+
194+
backends := []ociloadbalancer.BackendDetails{}
195+
for _, epAddr := range epAddrs {
196+
backends = append(backends, util.NewBackend(epAddr.IP, targetPort))
197+
}
198+
199+
backendSetName := util.GenerateBackendSetName(ingress.Namespace, svcName, svcPort)
200+
err = wrapperClient.GetLbClient().UpdateBackends(context.TODO(), lbID, backendSetName, backends)
201+
if err != nil {
202+
return fmt.Errorf("unable to update backends for %s/%s: %w", ingressClass.Name, backendSetName, err)
203+
}
204+
205+
backendSetHealth, err := wrapperClient.GetLbClient().GetBackendSetHealth(context.TODO(), lbID, backendSetName)
206+
if err != nil {
207+
return fmt.Errorf("unable to fetch backendset health: %w", err)
208+
}
209+
210+
for _, epAddr := range epAddrs {
211+
pod, err := c.podLister.Pods(ingress.Namespace).Get(epAddr.TargetRef.Name)
180212
if err != nil {
181-
return fmt.Errorf("unable to update backends for %s/%s: %w", ingressClass.Name, backendSetName, err)
213+
return fmt.Errorf("failed to fetch pod %s/%s: %w", ingress.Namespace, epAddr.TargetRef.Name, err)
182214
}
183-
backendSetHealth, err := wrapperClient.GetLbClient().GetBackendSetHealth(context.TODO(), lbID, backendSetName)
215+
216+
backendName := fmt.Sprintf("%s:%d", epAddr.IP, targetPort)
217+
readinessCondition := util.GetPodReadinessCondition(ingress.Name, rule.Host, path)
218+
219+
err = c.ensurePodReadinessCondition(ctx, pod, readinessCondition, backendSetHealth, backendName)
184220
if err != nil {
185-
return fmt.Errorf("unable to fetch backendset health: %w", err)
186-
}
187-
for _, epAddr := range epAddrs {
188-
pod, err := c.podLister.Pods(ingress.Namespace).Get(epAddr.TargetRef.Name)
189-
if err != nil {
190-
return fmt.Errorf("failed to fetch pod %s/%s: %w", ingress.Namespace, epAddr.TargetRef.Name, err)
191-
}
192-
backendName := fmt.Sprintf("%s:%d", epAddr.IP, targetPort)
193-
readinessCondition := util.GetPodReadinessCondition(ingress.Name, rule.Host, path)
194-
err = c.ensurePodReadinessCondition(ctx, pod, readinessCondition, backendSetHealth, backendName)
195-
if err != nil {
196-
return fmt.Errorf("%w", err)
197-
}
221+
return fmt.Errorf("%w", err)
198222
}
199223
}
200224
}
201225
}
202-
// Sync default backends
203-
c.syncDefaultBackend(ctx, lbID, ingresses)
226+
204227
return nil
205228
}
206229

pkg/controllers/backend/backend_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"context"
66
coreinformers "k8s.io/client-go/informers/core/v1"
7+
"k8s.io/client-go/tools/events"
78
"sync"
89
"testing"
910
"time"
@@ -223,7 +224,8 @@ func inits(ctx context.Context, ingressClassList *networkingv1.IngressClassList,
223224
DefaultConfigGetter: &MockConfigGetter{},
224225
Cache: NewMockCacheStore(wrapperClient),
225226
}
226-
c := NewController("oci.oraclecloud.com/native-ingress-controller", ingressClassInformer, ingressInformer, saInformer, serviceLister, endpointLister, podLister, client)
227+
fakeRecorder := events.NewFakeRecorder(10)
228+
c := NewController("oci.oraclecloud.com/native-ingress-controller", ingressClassInformer, ingressInformer, saInformer, serviceLister, endpointLister, podLister, client, fakeRecorder)
227229
return c
228230
}
229231

pkg/controllers/ingress/certificate_util.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"github.com/oracle/oci-go-sdk/v65/certificatesmanagement"
1616
"github.com/oracle/oci-go-sdk/v65/common"
1717
"github.com/oracle/oci-native-ingress-controller/pkg/certificate"
18+
"github.com/oracle/oci-native-ingress-controller/pkg/exception"
1819
"github.com/oracle/oci-native-ingress-controller/pkg/util"
1920
"k8s.io/klog/v2"
2021
"time"
@@ -152,7 +153,7 @@ func UpdateImportedTypeCertificate(certificateId *string, tlsSecretData *TLSSecr
152153
if util.IsServiceError(err, 409) {
153154
klog.Infof("Update certificate returned code %d for certificate %s. Refreshing cache.", 409, *certificateId)
154155
getCertificateBustCache(*certificateId, certificatesClient)
155-
return nil, fmt.Errorf("unable to update certificate %s due to conflict, controller will retry later", *certificateId)
156+
return nil, exception.NewTransientError(fmt.Errorf("unable to update certificate %s due to conflict, controller will retry later", *certificateId))
156157
}
157158

158159
if err != nil {
@@ -353,7 +354,7 @@ func UpdateCaBundle(caBundleId string, certificatesClient *certificate.Certifica
353354
if util.IsServiceError(err, 409) {
354355
klog.Infof("Update ca bundle returned code %d for ca bundle %s. Refreshing cache.", 409, caBundleId)
355356
getCaBundleBustCache(caBundleId, certificatesClient)
356-
return nil, fmt.Errorf("unable to update ca bundle %s due to conflict, controller will retry later", caBundleId)
357+
return nil, exception.NewTransientError(fmt.Errorf("unable to update ca bundle %s due to conflict, controller will retry later", caBundleId))
357358
}
358359

359360
if err != nil {

0 commit comments

Comments
 (0)