diff --git a/PROJECT b/PROJECT index c4e93ad4d..a9b4a1555 100644 --- a/PROJECT +++ b/PROJECT @@ -2,7 +2,6 @@ # This file is used to track the info used to scaffold your project # and allow the plugins properly work. # More info: https://book.kubebuilder.io/reference/project-config.html -domain: github.com layout: - go.kubebuilder.io/v4 projectName: apisix-ingress-controller @@ -12,16 +11,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com - group: apisix.apache.org - kind: Guestbook - path: github.com/apache/apisix-ingress-controller/api/v1alpha1 - version: v1alpha1 -- api: - crdVersion: v1 - namespaced: true - controller: true - domain: github.com group: apisix.apache.org kind: GatewayProxy path: github.com/apache/apisix-ingress-controller/api/v1alpha1 @@ -29,7 +18,6 @@ resources: - api: crdVersion: v1 namespaced: true - domain: github.com group: apisix.apache.org kind: HTTPRoutePolicy path: github.com/apache/apisix-ingress-controller/api/v1alpha1 @@ -38,7 +26,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com group: apisix.apache.org kind: ApisixRoute path: github.com/apache/apisix-ingress-controller/api/v2 @@ -47,7 +34,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com group: apisix.apache.org kind: ApisixConsumer path: github.com/apache/apisix-ingress-controller/api/v2 @@ -56,7 +42,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com group: apisix.apache.org kind: ApisixGlobalRule path: github.com/apache/apisix-ingress-controller/api/v2 @@ -65,7 +50,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com group: apisix.apache.org kind: ApisixTls path: github.com/apache/apisix-ingress-controller/api/v2 @@ -74,7 +58,6 @@ resources: crdVersion: v1 namespaced: true controller: true - domain: github.com group: apisix.apache.org kind: ApisixUpstream path: github.com/apache/apisix-ingress-controller/api/v2 @@ -82,9 +65,17 @@ resources: - api: crdVersion: v1 namespaced: true - domain: github.com group: apisix.apache.org kind: ApisixPluginConfig path: github.com/apache/apisix-ingress-controller/api/v2 version: v2 +- core: true + domain: k8s.io + group: networking + kind: Ingress + path: k8s.io/api/networking/v1 + version: v1 + webhooks: + validation: true + webhookVersion: v1 version: "3" diff --git a/config/certmanager/certificate-metrics.yaml b/config/certmanager/certificate-metrics.yaml new file mode 100644 index 000000000..b47c8989a --- /dev/null +++ b/config/certmanager/certificate-metrics.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a metrics certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: metrics-certs # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + dnsNames: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: metrics-server-cert diff --git a/config/certmanager/certificate-webhook.yaml b/config/certmanager/certificate-webhook.yaml new file mode 100644 index 000000000..c390d091b --- /dev/null +++ b/config/certmanager/certificate-webhook.yaml @@ -0,0 +1,20 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # SERVICE_NAME and SERVICE_NAMESPACE will be substituted by kustomize + # replacements in the config/default/kustomization.yaml file. + dnsNames: + - SERVICE_NAME.SERVICE_NAMESPACE.svc + - SERVICE_NAME.SERVICE_NAMESPACE.svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert diff --git a/config/certmanager/issuer.yaml b/config/certmanager/issuer.yaml new file mode 100644 index 000000000..8af548bfb --- /dev/null +++ b/config/certmanager/issuer.yaml @@ -0,0 +1,13 @@ +# The following manifest contains a self-signed issuer CR. +# More information can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 000000000..fcb7498e4 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- issuer.yaml +- certificate-webhook.yaml +- certificate-metrics.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 000000000..cf6f89e88 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,8 @@ +# This configuration is for teaching kustomize how to update name ref substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 2f12a658b..4148a24d3 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -42,5 +42,5 @@ patches: # [WEBHOOK] To enable webhook, uncomment the following section # the following config is for teaching kustomize how to do kustomization for CRDs. -#configurations: -#- kustomizeconfig.yaml +configurations: +- kustomizeconfig.yaml diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 87b8a237e..b822a1505 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -20,17 +20,18 @@ resources: - ../manager # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus # [METRICS] Expose the controller manager metrics service. - metrics_service.yaml - ../samples +- ../network-policy # Uncomment the patches line if you enable Metrics, and/or are using webhooks and cert-manager -#patches: +patches: # [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. # More info: https://book.kubebuilder.io/reference/metrics #- path: manager_patch.yaml @@ -39,109 +40,115 @@ resources: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- path: manager_webhook_patch.yaml +- path: manager_webhook_patch.yaml + target: + kind: Deployment + name: controller-manager # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- path: webhookcainjection_patch.yaml +- path: webhookcainjection_patch.yaml + target: + kind: ValidatingWebhookConfiguration + name: validating-webhook-configuration # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. # Uncomment the following replacements to add the cert-manager CA injection annotations -#replacements: -# - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.namespace # namespace of the certificate CR -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 0 -# create: true -# - source: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldPath: .metadata.name -# targets: -# - select: -# kind: ValidatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: MutatingWebhookConfiguration -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - select: -# kind: CustomResourceDefinition -# fieldPaths: -# - .metadata.annotations.[cert-manager.io/inject-ca-from] -# options: -# delimiter: '/' -# index: 1 -# create: true -# - source: # Add cert-manager annotation to the webhook Service -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.name # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 0 -# create: true -# - source: -# kind: Service -# version: v1 -# name: webhook-service -# fieldPath: .metadata.namespace # namespace of the service -# targets: -# - select: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# fieldPaths: -# - .spec.dnsNames.0 -# - .spec.dnsNames.1 -# options: -# delimiter: '.' -# index: 1 -# create: true +replacements: + - source: # Add cert-manager annotation to ValidatingWebhookConfiguration, MutatingWebhookConfiguration and CRDs + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.namespace # namespace of the certificate CR + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 0 + create: true + - source: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldPath: .metadata.name + targets: + - select: + kind: ValidatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: MutatingWebhookConfiguration + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - select: + kind: CustomResourceDefinition + fieldPaths: + - .metadata.annotations.[cert-manager.io/inject-ca-from] + options: + delimiter: '/' + index: 1 + create: true + - source: # Add cert-manager annotation to the webhook Service + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.name # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 0 + create: true + - source: + kind: Service + version: v1 + name: webhook-service + fieldPath: .metadata.namespace # namespace of the service + targets: + - select: + kind: Certificate + group: cert-manager.io + version: v1 + fieldPaths: + - .spec.dnsNames.0 + - .spec.dnsNames.1 + options: + delimiter: '.' + index: 1 + create: true diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 000000000..7c12cdb43 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,26 @@ +# This patch ensures the webhook certificates are properly mounted in the manager container. +# It configures the necessary arguments, volumes, volume mounts, and container ports. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + args: + - --webhook-cert-path=/tmp/k8s-webhook-server/serving-certs + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: webhook-certs + readOnly: true + volumes: + - name: webhook-certs + secret: + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 000000000..d23c3c933 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,8 @@ +# This patch adds the cert-manager CA injection annotation to the webhook configuration. +# This file is auto-generated; do not edit directly. +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/network-policy/allow-webhook-traffic.yaml b/config/network-policy/allow-webhook-traffic.yaml new file mode 100644 index 000000000..e3f2f4083 --- /dev/null +++ b/config/network-policy/allow-webhook-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic to your webhook server running +# as part of the controller-manager from specific namespaces and pods. CR(s) which uses webhooks +# will only work when applied in namespaces labeled with 'webhook: enabled' +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: allow-webhook-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: apisix-ingress-controller + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label webhook: enabled + - from: + - namespaceSelector: + matchLabels: + webhook: enabled # Only from namespaces with this label + ports: + - port: 443 + protocol: TCP diff --git a/config/network-policy/kustomization.yaml b/config/network-policy/kustomization.yaml new file mode 100644 index 000000000..a67bd684b --- /dev/null +++ b/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-webhook-traffic.yaml diff --git a/config/samples/config.yaml b/config/samples/config.yaml index 55a74e3a5..6656cb90f 100644 --- a/config/samples/config.yaml +++ b/config/samples/config.yaml @@ -40,3 +40,15 @@ provider: # If you want to enable the sync, set it to a positive value. init_sync_delay: 20m # The initial delay before the first sync, only used when the controller is started. # The default value is 20 minutes. + +webhook: + enable: false # Whether to enable the webhook server. + # The default value is false. + tls_cert_file: "tls.crt" # The filename within tls_cert_dir containing the webhook server TLS certificate. + # The default value is "tls.crt". + tls_key_file: "tls.key" # The filename within tls_cert_dir containing the webhook server TLS private key. + # The default value is "tls.key". + tls_cert_dir: "/certs" # The directory containing the webhook server TLS certificate files. + # The default value is "/certs". + port: 9443 # The port for the webhook server to listen on. + # The default value is 9443. diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 000000000..9cf26134e --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 000000000..206316e54 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,22 @@ +# the following config is for teaching kustomize where to look at when substituting nameReference. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + - kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +- kind: ValidatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 000000000..8632054a2 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-networking-k8s-io-v1-ingress + failurePolicy: Fail + name: vingress-v1.kb.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 000000000..1f0b7c349 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: apisix-ingress-controller diff --git a/internal/controller/config/config.go b/internal/controller/config/config.go index 774f2728f..ea8dfbac3 100644 --- a/internal/controller/config/config.go +++ b/internal/controller/config/config.go @@ -55,6 +55,7 @@ func NewDefaultConfig() *Config { SyncPeriod: types.TimeDuration{Duration: 0}, InitSyncDelay: types.TimeDuration{Duration: 20 * time.Minute}, }, + Webhook: NewWebhookConfig(), } } @@ -67,6 +68,16 @@ func NewLeaderElection() *LeaderElection { } } +func NewWebhookConfig() *WebhookConfig { + return &WebhookConfig{ + Enable: false, + TLSCertFile: DefaultWebhookTLSCert, + TLSKeyFile: DefaultWebhookTLSKey, + TLSCertDir: DefaultWebhookTLSCertDir, + Port: DefaultWebhookPort, + } +} + // NewConfigFromFile creates a Config object and fills all config items according // to the configuration file. The file can be in JSON/YAML format, which will be // distinguished according to the file suffix. @@ -113,6 +124,9 @@ func (c *Config) Validate() error { if err := validateProvider(c.ProviderConfig); err != nil { return err } + if err := validateWebhook(c.Webhook); err != nil { + return err + } return nil } @@ -133,3 +147,25 @@ func validateProvider(config ProviderConfig) error { func GetControllerName() string { return ControllerConfig.ControllerName } + +func validateWebhook(config *WebhookConfig) error { + if config == nil { + return nil + } + if config.Enable { + if config.Port <= 0 { + return fmt.Errorf("port must be greater than 0 for webhook") + } + if config.TLSCertFile == "" { + return fmt.Errorf("tls_cert_file is required for webhook") + } + if config.TLSKeyFile == "" { + return fmt.Errorf("tls_key_file is required for webhook") + } + if config.TLSCertDir == "" { + return fmt.Errorf("tls_cert_dir is required for webhook") + } + } + + return nil +} diff --git a/internal/controller/config/types.go b/internal/controller/config/types.go index 8669268f4..8d167a2be 100644 --- a/internal/controller/config/types.go +++ b/internal/controller/config/types.go @@ -44,6 +44,13 @@ const ( DefaultMetricsAddr = ":8080" DefaultProbeAddr = ":8081" + DefaultServerAddr = ":9092" + + // Webhook configuration defaults + DefaultWebhookTLSCert = "tls.crt" + DefaultWebhookTLSKey = "tls.key" + DefaultWebhookTLSCertDir = "/certs" + DefaultWebhookPort = 9443 ) // Config contains all config items which are necessary for @@ -59,6 +66,7 @@ type Config struct { LeaderElection *LeaderElection `json:"leader_election" yaml:"leader_election"` ExecADCTimeout types.TimeDuration `json:"exec_adc_timeout" yaml:"exec_adc_timeout"` ProviderConfig ProviderConfig `json:"provider" yaml:"provider"` + Webhook *WebhookConfig `json:"webhook" yaml:"webhook"` } type GatewayConfig struct { @@ -85,3 +93,11 @@ type ProviderConfig struct { SyncPeriod types.TimeDuration `json:"sync_period" yaml:"sync_period"` InitSyncDelay types.TimeDuration `json:"init_sync_delay" yaml:"init_sync_delay"` } + +type WebhookConfig struct { + Enable bool `json:"enable" yaml:"enable"` + TLSCertFile string `json:"tls_cert_file" yaml:"tls_cert_file"` + TLSKeyFile string `json:"tls_key_file" yaml:"tls_key_file"` + TLSCertDir string `json:"tls_cert_dir" yaml:"tls_cert_dir"` + Port int `json:"port" yaml:"port"` +} diff --git a/internal/manager/run.go b/internal/manager/run.go index 2042b859d..12a074ea4 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -95,10 +95,6 @@ func Run(ctx context.Context, logger logr.Logger) error { tlsOpts = append(tlsOpts, disableHTTP2) } - webhookServer := webhook.NewServer(webhook.Options{ - TLSOpts: tlsOpts, - }) - // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.4/pkg/metrics/server @@ -128,10 +124,9 @@ func Run(ctx context.Context, logger logr.Logger) error { namespace = "default" } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + mgrOptions := ctrl.Options{ Scheme: scheme, Metrics: metricsServerOptions, - WebhookServer: webhookServer, HealthProbeBindAddress: cfg.ProbeAddr, LeaderElection: !config.ControllerConfig.LeaderElection.Disable, LeaderElectionID: cfg.LeaderElectionID, @@ -150,7 +145,19 @@ func Run(ctx context.Context, logger logr.Logger) error { // if you are doing or is intended to do any operation such as perform cleanups // after the manager stops then its usage might be unsafe. // LeaderElectionReleaseOnCancel: true, - }) + } + + if cfg.Webhook != nil && cfg.Webhook.Enable { + webhookServer := webhook.NewServer(webhook.Options{ + Port: cfg.Webhook.Port, + CertDir: cfg.Webhook.TLSCertDir, + CertName: cfg.Webhook.TLSCertFile, + KeyName: cfg.Webhook.TLSKeyFile, + }) + mgrOptions.WebhookServer = webhookServer + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOptions) if err != nil { setupLog.Error(err, "unable to start manager") return err @@ -213,6 +220,16 @@ func Run(ctx context.Context, logger logr.Logger) error { // +kubebuilder:scaffold:builder + if cfg.Webhook != nil && cfg.Webhook.Enable { + setupLog.Info("setting up webhooks") + if err := setupWebhooks(ctx, mgr); err != nil { + setupLog.Error(err, "unable to create webhook", "webhook", "Ingress") + return err + } + } else { + setupLog.Info("webhooks disabled, skipping webhook setup") + } + setupLog.Info("setting up health checks") if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") diff --git a/internal/manager/webhooks.go b/internal/manager/webhooks.go new file mode 100644 index 000000000..0a67f07a0 --- /dev/null +++ b/internal/manager/webhooks.go @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package manager + +import ( + "context" + + "sigs.k8s.io/controller-runtime/pkg/manager" + + webhookv1 "github.com/apache/apisix-ingress-controller/internal/webhook/v1" +) + +func setupWebhooks(_ context.Context, mgr manager.Manager) error { + if err := webhookv1.SetupIngressWebhookWithManager(mgr); err != nil { + return err + } + return nil +} diff --git a/internal/webhook/v1/ingress_webhook.go b/internal/webhook/v1/ingress_webhook.go new file mode 100644 index 000000000..777d1e1be --- /dev/null +++ b/internal/webhook/v1/ingress_webhook.go @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "context" + "fmt" + "slices" + + networkingk8siov1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +var ingresslog = logf.Log.WithName("ingress-resource") + +// unsupportedAnnotations contains all the APISIX Ingress annotations that are not supported in 2.0.0 +// ref: https://apisix.apache.org/docs/ingress-controller/upgrade-guide/#limited-support-for-ingress-annotations +var unsupportedAnnotations = []string{ + "k8s.apisix.apache.org/use-regex", + "k8s.apisix.apache.org/enable-websocket", + "k8s.apisix.apache.org/plugin-config-name", + "k8s.apisix.apache.org/upstream-scheme", + "k8s.apisix.apache.org/upstream-retries", + "k8s.apisix.apache.org/upstream-connect-timeout", + "k8s.apisix.apache.org/upstream-read-timeout", + "k8s.apisix.apache.org/upstream-send-timeout", + "k8s.apisix.apache.org/enable-cors", + "k8s.apisix.apache.org/cors-allow-origin", + "k8s.apisix.apache.org/cors-allow-headers", + "k8s.apisix.apache.org/cors-allow-methods", + "k8s.apisix.apache.org/enable-csrf", + "k8s.apisix.apache.org/csrf-key", + "k8s.apisix.apache.org/http-to-https", + "k8s.apisix.apache.org/http-redirect", + "k8s.apisix.apache.org/http-redirect-code", + "k8s.apisix.apache.org/rewrite-target", + "k8s.apisix.apache.org/rewrite-target-regex", + "k8s.apisix.apache.org/rewrite-target-regex-template", + "k8s.apisix.apache.org/enable-response-rewrite", + "k8s.apisix.apache.org/response-rewrite-status-code", + "k8s.apisix.apache.org/response-rewrite-body", + "k8s.apisix.apache.org/response-rewrite-body-base64", + "k8s.apisix.apache.org/response-rewrite-add-header", + "k8s.apisix.apache.org/response-rewrite-set-header", + "k8s.apisix.apache.org/response-rewrite-remove-header", + "k8s.apisix.apache.org/auth-uri", + "k8s.apisix.apache.org/auth-ssl-verify", + "k8s.apisix.apache.org/auth-request-headers", + "k8s.apisix.apache.org/auth-upstream-headers", + "k8s.apisix.apache.org/auth-client-headers", + "k8s.apisix.apache.org/allowlist-source-range", + "k8s.apisix.apache.org/blocklist-source-range", + "k8s.apisix.apache.org/http-allow-methods", + "k8s.apisix.apache.org/http-block-methods", + "k8s.apisix.apache.org/auth-type", + "k8s.apisix.apache.org/svc-namespace", +} + +// checkUnsupportedAnnotations checks if the Ingress contains any unsupported annotations +// and returns appropriate warnings +func checkUnsupportedAnnotations(ingress *networkingk8siov1.Ingress) admission.Warnings { + var warnings admission.Warnings + + if len(ingress.Annotations) == 0 { + return warnings + } + + for annotation := range ingress.Annotations { + if slices.Contains(unsupportedAnnotations, annotation) { + warningMsg := fmt.Sprintf("Annotation '%s' is not supported in APISIX Ingress Controller 2.0.0.", annotation) + warnings = append(warnings, warningMsg) + ingresslog.Info("Detected unsupported annotation", + "ingress", ingress.GetName(), + "namespace", ingress.GetNamespace(), + "annotation", annotation) + } + } + + return warnings +} + +// SetupIngressWebhookWithManager registers the webhook for Ingress in the manager. +func SetupIngressWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&networkingk8siov1.Ingress{}). + WithValidator(&IngressCustomValidator{}). + Complete() +} + +// NOTE: The 'path' attribute must follow a specific pattern and should not be modified directly here. +// Modifying the path for an invalid path can cause API server errors; failing to locate the webhook. +// +kubebuilder:webhook:path=/validate-networking-k8s-io-v1-ingress,mutating=false,failurePolicy=fail,sideEffects=None,groups=networking.k8s.io,resources=ingresses,verbs=create;update,versions=v1,name=vingress-v1.kb.io,admissionReviewVersions=v1 + +// IngressCustomValidator struct is responsible for validating the Ingress resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type IngressCustomValidator struct{} + +var _ webhook.CustomValidator = &IngressCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type Ingress. +func (v *IngressCustomValidator) ValidateCreate(_ context.Context, obj runtime.Object) (admission.Warnings, error) { + ingress, ok := obj.(*networkingk8siov1.Ingress) + if !ok { + return nil, fmt.Errorf("expected a Ingress object but got %T", obj) + } + ingresslog.Info("Validation for Ingress upon creation", "name", ingress.GetName(), "namespace", ingress.GetNamespace()) + + // Check for unsupported annotations and generate warnings + warnings := checkUnsupportedAnnotations(ingress) + + return warnings, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type Ingress. +func (v *IngressCustomValidator) ValidateUpdate(_ context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + ingress, ok := newObj.(*networkingk8siov1.Ingress) + if !ok { + return nil, fmt.Errorf("expected a Ingress object for the newObj but got %T", newObj) + } + ingresslog.Info("Validation for Ingress upon update", "name", ingress.GetName(), "namespace", ingress.GetNamespace()) + + // Check for unsupported annotations and generate warnings + warnings := checkUnsupportedAnnotations(ingress) + + return warnings, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type Ingress. +func (v *IngressCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil +} diff --git a/internal/webhook/v1/ingress_webhook_test.go b/internal/webhook/v1/ingress_webhook_test.go new file mode 100644 index 000000000..5ae9bd2a9 --- /dev/null +++ b/internal/webhook/v1/ingress_webhook_test.go @@ -0,0 +1,121 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to You under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + "context" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + networkingk8siov1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestIngressCustomValidator_ValidateCreate_UnsupportedAnnotations(t *testing.T) { + validator := IngressCustomValidator{} + obj := &networkingk8siov1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "k8s.apisix.apache.org/use-regex": "true", + "k8s.apisix.apache.org/enable-websocket": "true", + }, + }, + } + + warnings, err := validator.ValidateCreate(context.TODO(), obj) + assert.NoError(t, err) + assert.Len(t, warnings, 2) + + // Check that warnings contain the expected unsupported annotations + warningsStr := strings.Join(warnings, " ") + assert.Contains(t, warningsStr, "k8s.apisix.apache.org/use-regex") + assert.Contains(t, warningsStr, "k8s.apisix.apache.org/enable-websocket") +} + +func TestIngressCustomValidator_ValidateCreate_SupportedAnnotations(t *testing.T) { + validator := IngressCustomValidator{} + obj := &networkingk8siov1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "ingressclass.kubernetes.io/is-default-class": "true", + }, + }, + } + + warnings, err := validator.ValidateCreate(context.TODO(), obj) + assert.NoError(t, err) + assert.Empty(t, warnings) +} + +func TestIngressCustomValidator_ValidateUpdate_UnsupportedAnnotations(t *testing.T) { + validator := IngressCustomValidator{} + oldObj := &networkingk8siov1.Ingress{} + obj := &networkingk8siov1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "k8s.apisix.apache.org/enable-cors": "true", + "k8s.apisix.apache.org/cors-allow-origin": "*", + }, + }, + } + + warnings, err := validator.ValidateUpdate(context.TODO(), oldObj, obj) + assert.NoError(t, err) + assert.Len(t, warnings, 2) + + // Check that warnings contain the expected unsupported annotations + warningsStr := strings.Join(warnings, " ") + assert.Contains(t, warningsStr, "k8s.apisix.apache.org/enable-cors") + assert.Contains(t, warningsStr, "k8s.apisix.apache.org/cors-allow-origin") +} + +func TestIngressCustomValidator_ValidateDelete_NoWarnings(t *testing.T) { + validator := IngressCustomValidator{} + obj := &networkingk8siov1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + Annotations: map[string]string{ + "k8s.apisix.apache.org/use-regex": "true", + }, + }, + } + + warnings, err := validator.ValidateDelete(context.TODO(), obj) + assert.NoError(t, err) + assert.Empty(t, warnings) +} + +func TestIngressCustomValidator_ValidateCreate_NoAnnotations(t *testing.T) { + validator := IngressCustomValidator{} + obj := &networkingk8siov1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-ingress", + Namespace: "default", + }, + } + + warnings, err := validator.ValidateCreate(context.TODO(), obj) + assert.NoError(t, err) + assert.Empty(t, warnings) +} diff --git a/test/e2e/framework/apisix_consts.go b/test/e2e/framework/apisix_consts.go index cb9e7531a..65207e935 100644 --- a/test/e2e/framework/apisix_consts.go +++ b/test/e2e/framework/apisix_consts.go @@ -42,6 +42,10 @@ var ( //go:embed manifests/etcd.yaml EtcdSpec string + + //go:embed manifests/webhook.yaml + validatingWebhookTemplate string + ValidatingWebhookTpl *template.Template ) var ( @@ -106,4 +110,10 @@ func init() { panic(err) } APISIXStandaloneTpl = tpl + + webhookTpl, err := template.New("validating-webhook").Funcs(sprig.TxtFuncMap()).Parse(validatingWebhookTemplate) + if err != nil { + panic(err) + } + ValidatingWebhookTpl = webhookTpl } diff --git a/test/e2e/framework/ingress.go b/test/e2e/framework/ingress.go index e77db052a..9c2e71998 100644 --- a/test/e2e/framework/ingress.go +++ b/test/e2e/framework/ingress.go @@ -51,6 +51,8 @@ type IngressDeployOpts struct { StatusAddress string Replicas *int InitSyncDelay time.Duration + WebhookEnable bool + WebhookPort int } func (f *Framework) DeployIngress(opts IngressDeployOpts) { diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index f8097c03f..94972cc8b 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -351,6 +351,10 @@ data: # The default value is 0 seconds, which means the controller will not sync. # If you want to enable the sync, set it to a positive value. init_sync_delay: {{ .InitSyncDelay | default "20m" }} + webhook: + enable: {{ .WebhookEnable | default false }} + port: {{ .WebhookPort | default 9443 }} + tls_cert_dir: "/tmp/certs" --- apiVersion: v1 kind: Service @@ -409,6 +413,11 @@ spec: - name: ingress-config mountPath: /app/conf/config.yaml subPath: config.yaml + {{ if .WebhookEnable -}} + - name: webhook-certs + mountPath: /tmp/certs + readOnly: true + {{ end -}} livenessProbe: httpGet: path: /healthz @@ -420,6 +429,9 @@ spec: - name: metrics containerPort: 8080 protocol: TCP + - name: webhook-server + containerPort: 9443 + protocol: TCP readinessProbe: httpGet: path: /readyz @@ -471,5 +483,30 @@ spec: - name: ingress-config configMap: name: ingress-config + {{ if .WebhookEnable -}} + - name: webhook-certs + secret: + secretName: webhook-server-certs + optional: true + {{ end -}} serviceAccountName: apisix-ingress-controller-manager terminationGracePeriodSeconds: 10 +{{ if .WebhookEnable -}} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: apisix-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: {{ .Namespace }} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager + app: apisix-ingress-controller +{{ end -}} diff --git a/test/e2e/framework/manifests/webhook.yaml b/test/e2e/framework/manifests/webhook.yaml new file mode 100644 index 000000000..4d96e4540 --- /dev/null +++ b/test/e2e/framework/manifests/webhook.yaml @@ -0,0 +1,43 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: test-webhook-{{ .Namespace }} +webhooks: +- name: vingress-v1.kb.io + clientConfig: + service: + name: webhook-service + namespace: {{ .Namespace }} + path: /validate-networking-k8s-io-v1-ingress + caBundle: {{ .CABundle }} + admissionReviewVersions: + - v1 + rules: + - operations: + - CREATE + - UPDATE + apiGroups: + - networking.k8s.io + apiVersions: + - v1 + resources: + - ingresses + failurePolicy: Fail + sideEffects: None diff --git a/test/e2e/ingress/webhook.go b/test/e2e/ingress/webhook.go new file mode 100644 index 000000000..2f8c9eff2 --- /dev/null +++ b/test/e2e/ingress/webhook.go @@ -0,0 +1,160 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package ingress + +import ( + "fmt" + "net/http" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/apache/apisix-ingress-controller/test/e2e/scaffold" +) + +var _ = Describe("Test Ingress Webhook", Label("networking.k8s.io", "ingress"), func() { + s := scaffold.NewScaffold(scaffold.Options{ + Name: "webhook-test", + EnableWebhook: true, + }) + + BeforeEach(func() { + By("create GatewayProxy") + err := s.CreateResourceFromString(s.GetGatewayProxySpec()) + Expect(err).NotTo(HaveOccurred(), "creating GatewayProxy") + time.Sleep(5 * time.Second) + + By("create IngressClass") + err = s.CreateResourceFromStringWithNamespace(s.GetIngressClassYaml(), "") + Expect(err).NotTo(HaveOccurred(), "creating IngressClass") + time.Sleep(5 * time.Second) + }) + + Context("Ingress Validation", func() { + It("should warn about unsupported annotations on create", func() { + + By("creating Ingress with unsupported annotations") + ingressYAML := fmt.Sprintf(` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-webhook-unsupported + namespace: %s + annotations: + k8s.apisix.apache.org/use-regex: "true" + k8s.apisix.apache.org/enable-websocket: "true" +spec: + ingressClassName: %s + rules: + - host: webhook-test.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +`, s.Namespace(), s.Namespace()) + + output, err := s.CreateResourceFromStringAndGetOutput(ingressYAML) + Expect(err).ShouldNot(HaveOccurred()) + Expect(output).To(ContainSubstring(`Warning: Annotation 'k8s.apisix.apache.org/enable-websocket' is not supported`)) + Expect(output).To(ContainSubstring(`Warning: Annotation 'k8s.apisix.apache.org/use-regex' is not supported`)) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "webhook-test.example.com", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + }) + + It("should warn about unsupported annotations on update", func() { + By("creating Ingress without unsupported annotations") + initialIngressYAML := fmt.Sprintf(` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-webhook-update + namespace: %s +spec: + ingressClassName: %s + rules: + - host: webhook-test-update.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +`, s.Namespace(), s.Namespace()) + + output, err := s.CreateResourceFromStringAndGetOutput(initialIngressYAML) + Expect(err).ShouldNot(HaveOccurred()) + Expect(output).ShouldNot(ContainSubstring(`Warning`)) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "webhook-test-update.example.com", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + + By("updating Ingress with unsupported annotations") + updatedIngressYAML := fmt.Sprintf(` +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: test-webhook-update + namespace: %s + annotations: + k8s.apisix.apache.org/enable-cors: "true" +spec: + ingressClassName: %s + rules: + - host: webhook-test-update.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: httpbin-service-e2e-test + port: + number: 80 +`, s.Namespace(), s.Namespace()) + + output, err = s.CreateResourceFromStringAndGetOutput(updatedIngressYAML) + Expect(err).ShouldNot(HaveOccurred()) + Expect(output).To(ContainSubstring(`Warning: Annotation 'k8s.apisix.apache.org/enable-cors' is not supported`)) + + s.RequestAssert(&scaffold.RequestAssert{ + Method: "GET", + Path: "/get", + Host: "webhook-test-update.example.com", + Check: scaffold.WithExpectedStatus(http.StatusOK), + }) + }) + }) +}) diff --git a/test/e2e/scaffold/api7_deployer.go b/test/e2e/scaffold/api7_deployer.go index 0ac02fff2..45faf073a 100644 --- a/test/e2e/scaffold/api7_deployer.go +++ b/test/e2e/scaffold/api7_deployer.go @@ -187,11 +187,17 @@ func (s *API7Deployer) newAPISIXTunnels(serviceName string) error { } func (s *API7Deployer) DeployIngress() { + if s.runtimeOpts.EnableWebhook { + err := s.SetupWebhookResources() + Expect(err).NotTo(HaveOccurred(), "setting up webhook resources") + } + s.Framework.DeployIngress(framework.IngressDeployOpts{ ProviderType: "api7ee", ControllerName: s.runtimeOpts.ControllerName, Namespace: s.namespace, Replicas: ptr.To(1), + WebhookEnable: s.runtimeOpts.EnableWebhook, }) } diff --git a/test/e2e/scaffold/apisix_deployer.go b/test/e2e/scaffold/apisix_deployer.go index 91b6234e8..62221879f 100644 --- a/test/e2e/scaffold/apisix_deployer.go +++ b/test/e2e/scaffold/apisix_deployer.go @@ -258,12 +258,18 @@ func (s *APISIXDeployer) DeployIngress() { if s.runtimeOpts.SyncPeriod != 0 { syncPeriod = s.runtimeOpts.SyncPeriod } + if s.runtimeOpts.EnableWebhook { + err := s.SetupWebhookResources() + Expect(err).NotTo(HaveOccurred(), "setting up webhook resources") + } + s.Framework.DeployIngress(framework.IngressDeployOpts{ ControllerName: s.runtimeOpts.ControllerName, ProviderType: framework.ProviderType, ProviderSyncPeriod: syncPeriod, Namespace: s.namespace, Replicas: ptr.To(1), + WebhookEnable: s.runtimeOpts.EnableWebhook, }) } diff --git a/test/e2e/scaffold/assertion.go b/test/e2e/scaffold/assertion.go index b17cab6bd..ec0d2f210 100644 --- a/test/e2e/scaffold/assertion.go +++ b/test/e2e/scaffold/assertion.go @@ -40,11 +40,11 @@ type ErrorReporter struct { err error } -func (r *ErrorReporter) Errorf(message string, args ...interface{}) { +func (r *ErrorReporter) Errorf(message string, args ...any) { r.err = fmt.Errorf(message, args...) } -func (r *ErrorReporter) Fatalf(message string, args ...interface{}) { +func (r *ErrorReporter) Fatalf(message string, args ...any) { r.err = fmt.Errorf(message, args...) } diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index cd13f3b4d..a3547b61a 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -18,9 +18,12 @@ package scaffold import ( + "bytes" "cmp" "context" + "encoding/base64" "fmt" + "os" "os/exec" "strings" "time" @@ -47,6 +50,18 @@ func (s *Scaffold) CreateResourceFromString(yaml string) error { return k8s.KubectlApplyFromStringE(s.t, s.kubectlOptions, yaml) } +// CreateResourceFromStringAndGetOutput creates resource from a loaded yaml string and returns the output of the command. +func (s *Scaffold) CreateResourceFromStringAndGetOutput(yaml string) (string, error) { + tmpfile, err := k8s.StoreConfigToTempFileE(s.t, yaml) + if err != nil { + return "", err + } + defer func() { + _ = os.Remove(tmpfile) + }() + return k8s.RunKubectlAndGetOutputE(s.t, s.kubectlOptions, "apply", "-f", tmpfile) +} + func (s *Scaffold) DeleteResourceFromString(yaml string) error { return k8s.KubectlDeleteFromStringE(s.t, s.kubectlOptions, yaml) } @@ -387,3 +402,31 @@ func (s *Scaffold) RunDigDNSClientFromK8s(args ...string) (string, error) { kubectlArgs = append(kubectlArgs, args...) return s.RunKubectlAndGetOutput(kubectlArgs...) } + +type WebhookData struct { + Namespace string + CABundle string +} + +func (s *Scaffold) SetupWebhookResources() error { + // Generate TLS certificates + caCert, serverCert, serverKey, _, _ := s.GenerateMACert(s.GinkgoT, []string{fmt.Sprintf("webhook-service.%s.svc", s.Namespace())}) + + err := s.NewKubeTlsSecret("webhook-server-certs", serverCert.String(), serverKey.String()) + if err != nil { + return err + } + + data := WebhookData{ + Namespace: s.Namespace(), + CABundle: base64.StdEncoding.EncodeToString(caCert.Bytes()), + } + + var buf bytes.Buffer + err = framework.ValidatingWebhookTpl.Execute(&buf, data) + if err != nil { + return err + } + + return s.CreateResourceFromStringWithNamespace(buf.String(), "") +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 6959decb6..aede2130d 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -53,6 +53,8 @@ type Options struct { NamespaceSelectorLabel map[string][]string DisableNamespaceLabel bool SkipHooks bool + + EnableWebhook bool } type Scaffold struct { diff --git a/test/e2e/scaffold/ssl.go b/test/e2e/scaffold/ssl.go index 0eb3246a5..e5e72ec79 100644 --- a/test/e2e/scaffold/ssl.go +++ b/test/e2e/scaffold/ssl.go @@ -150,7 +150,6 @@ func (s *Scaffold) GenerateMACert( caTemplate := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ - CommonName: dnsNames[0] + "-ca", Organization: []string{"Acme Co"}, }, NotBefore: time.Now(), @@ -179,7 +178,6 @@ func (s *Scaffold) GenerateMACert( serverTemplate := x509.Certificate{ SerialNumber: serialNumber, Subject: pkix.Name{ - CommonName: dnsNames[0], Organization: []string{"Acme Co"}, }, NotBefore: time.Now(), @@ -188,6 +186,7 @@ func (s *Scaffold) GenerateMACert( KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, + DNSNames: dnsNames, } serverBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, &caTemplate, &serverKey.PublicKey, caKey)