diff --git a/.golangci.yml b/.golangci.yml index cfadc12b6..0e2b6caa0 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -24,6 +24,8 @@ linters: revive: rules: - name: comment-spacings + lll: + line-length: 160 exclusions: generated: lax rules: diff --git a/PROJECT b/PROJECT index d591634af..f0db07576 100644 --- a/PROJECT +++ b/PROJECT @@ -34,4 +34,12 @@ resources: kind: HTTPRoutePolicy path: github.com/api7/api7-ingress-controller/api/v1alpha1 version: v1alpha1 +- external: true + group: gateway.networking.k8s.io + kind: GatewayClass + path: sigs.k8s.io/gateway-api/apis/v1 + version: v1 + webhooks: + validation: true + webhookVersion: v1 version: "3" diff --git a/internal/manager/run.go b/internal/manager/run.go index a5352e746..deba0580a 100644 --- a/internal/manager/run.go +++ b/internal/manager/run.go @@ -20,6 +20,7 @@ import ( "github.com/api7/api7-ingress-controller/api/v1alpha1" "github.com/api7/api7-ingress-controller/internal/controller/config" "github.com/api7/api7-ingress-controller/internal/provider/adc" + webhookv1 "github.com/api7/api7-ingress-controller/internal/webhook/v1" ) var ( @@ -138,6 +139,9 @@ func Run(ctx context.Context, logger logr.Logger) error { return err } } + if err = webhookv1.SetupGatewayClassWebhookWithManager(mgr); err != nil { + return err + } // +kubebuilder:scaffold:builder diff --git a/internal/webhook/v1/gatewayclass_webhook.go b/internal/webhook/v1/gatewayclass_webhook.go new file mode 100644 index 000000000..dd14fcb8b --- /dev/null +++ b/internal/webhook/v1/gatewayclass_webhook.go @@ -0,0 +1,83 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + gatewayv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +var _gcLog = logf.Log.WithName("gatewayclass-resource") + +// SetupGatewayClassWebhookWithManager registers the webhook for GatewayClass in the manager. +func SetupGatewayClassWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr).For(&gatewayv1.GatewayClass{}). + WithValidator(&GatewayClassCustomValidator{ + Client: mgr.GetClient(), + Logger: _gcLog, + }). + Complete() +} + +// +kubebuilder:webhook:path=/validate-gateway-networking-k8s-io-v1-gatewayclass,mutating=false,failurePolicy=fail,sideEffects=None,groups=gateway.networking.k8s.io,resources=gatewayclasses,verbs=create;update;delete,versions=v1,name=vgatewayclass-v1.kb.io,admissionReviewVersions=v1 + +// GatewayClassCustomValidator struct is responsible for validating the GatewayClass resource +// when it is created, updated, or deleted. +// +// NOTE: The +kubebuilder:object:generate=false marker prevents controller-gen from generating DeepCopy methods, +// as this struct is used only for temporary operations and does not need to be deeply copied. +type GatewayClassCustomValidator struct { + client.Client + logr.Logger +} + +var _ webhook.CustomValidator = &GatewayClassCustomValidator{} + +// ValidateCreate implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass. +func (v *GatewayClassCustomValidator) ValidateCreate(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// ValidateUpdate implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass. +func (v *GatewayClassCustomValidator) ValidateUpdate(ctx context.Context, oldObj, newObj runtime.Object) (admission.Warnings, error) { + return nil, nil +} + +// ValidateDelete implements webhook.CustomValidator so a webhook will be registered for the type GatewayClass. +func (v *GatewayClassCustomValidator) ValidateDelete(ctx context.Context, obj runtime.Object) (admission.Warnings, error) { + gc, ok := obj.(*gatewayv1.GatewayClass) + if !ok { + return nil, fmt.Errorf("expected a GatewayClass object but got %T", obj) + } + v.Info("Validation for GatewayClass upon deletion", "name", gc.GetName()) + + var gatewayList gatewayv1.GatewayList + if err := v.List(ctx, &gatewayList); err != nil { + v.Error(err, "failed to list gateway for the GatewayClass") + return nil, err + } + var gateways []types.NamespacedName + for _, gateway := range gatewayList.Items { + if string(gateway.Spec.GatewayClassName) == gc.Name { + gateways = append(gateways, types.NamespacedName{ + Namespace: gateway.GetNamespace(), + Name: gateway.GetName(), + }) + } + } + if len(gateways) > 0 { + err := fmt.Errorf("the GatewayClass is still in using by Gateways: %v", gateways) + v.Error(err, "can not delete the GatewayClass") + return nil, err + } + + return nil, nil +} diff --git a/test/e2e/framework/ingress.go b/test/e2e/framework/ingress.go index 0f041c76f..c493f3a1f 100644 --- a/test/e2e/framework/ingress.go +++ b/test/e2e/framework/ingress.go @@ -2,14 +2,26 @@ package framework import ( "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" _ "embed" + "encoding/base64" + "encoding/pem" "text/template" "time" "github.com/Masterminds/sprig/v3" "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/retry" . "github.com/onsi/gomega" //nolint:staticcheck + "github.com/pkg/errors" + certificatesv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) var ( @@ -33,12 +45,18 @@ type IngressDeployOpts struct { Namespace string AdminEnpoint string StatusAddress string + TLSKey string + TLSCRT string + CaBundle string } func (f *Framework) DeployIngress(opts IngressDeployOpts) { + err := f.setWebhookCertificate(&opts) + f.GomegaT.Expect(err).NotTo(HaveOccurred(), "set certificates info for webhook service") + buf := bytes.NewBuffer(nil) - err := IngressSpecTpl.Execute(buf, opts) + err = IngressSpecTpl.Execute(buf, opts) f.GomegaT.Expect(err).ToNot(HaveOccurred(), "rendering ingress spec") kubectlOpts := k8s.NewKubectlOptions("", "", opts.Namespace) @@ -51,3 +69,114 @@ func (f *Framework) DeployIngress(opts IngressDeployOpts) { f.GomegaT.Expect(err).ToNot(HaveOccurred(), "waiting for controller-manager pod ready") f.WaitControllerManagerLog("All cache synced successfully", 0, time.Minute) } + +func (f *Framework) setWebhookCertificate(opts *IngressDeployOpts) error { + // generate private key + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return errors.Wrap(err, "failed to GenerateKey") + } + var privateKeyBuf = bytes.NewBuffer(nil) + if err = pem.Encode(privateKeyBuf, &pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + }); err != nil { + return errors.Wrap(err, "failed to pem.Encode private key") + } + + // prepare CertificateSigningRequest + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{ + Subject: pkix.Name{ + Organization: []string{"system:nodes"}, + CommonName: "system:node:webhook-service." + opts.Namespace, + }, + DNSNames: []string{ + "webhook-service", + "webhook-service." + opts.Namespace, + "webhook-service." + opts.Namespace + ".svc", + }, + }, privateKey) + if err != nil { + return errors.Wrap(err, "failed to CreateCertificateRequest") + } + + certificateSigningRequest := &certificatesv1.CertificateSigningRequest{ + ObjectMeta: metav1.ObjectMeta{ + Name: "webhook-csr", + }, + Spec: certificatesv1.CertificateSigningRequestSpec{ + Request: pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE REQUEST", + Bytes: csrBytes, + }), + SignerName: "kubernetes.io/kubelet-serving", + ExpirationSeconds: nil, + Usages: []certificatesv1.KeyUsage{ + certificatesv1.UsageServerAuth, + certificatesv1.UsageDigitalSignature, + certificatesv1.UsageKeyEncipherment, + }, + }, + } + + // try to delete the CertificateSigningRequest before creating + err = f.clientset.CertificatesV1().CertificateSigningRequests(). + Delete(context.Background(), certificateSigningRequest.GetName(), metav1.DeleteOptions{}) + if err != nil { + f.Logf("failed to CertificateSigningRequests().Delete: %v", err) + } + + // create CertificateSigningRequest + certificateSigningRequest, err = f.clientset.CertificatesV1().CertificateSigningRequests(). + Create(context.Background(), certificateSigningRequest, metav1.CreateOptions{}) + if err != nil { + return errors.Wrap(err, "failed to CertificateSigningRequests().Create") + } + + // to approve the CertificateSigningRequest + condition := certificatesv1.CertificateSigningRequestCondition{ + Type: certificatesv1.CertificateApproved, + Status: "True", + Reason: "AdminApproval", + Message: "CSR approved by admin", + LastUpdateTime: metav1.Now(), + LastTransitionTime: metav1.Now(), + } + certificateSigningRequest.Status.Conditions = append(certificateSigningRequest.Status.Conditions, condition) + certificateSigningRequest, err = f.clientset.CertificatesV1().CertificateSigningRequests(). + UpdateApproval(context.Background(), certificateSigningRequest.GetName(), certificateSigningRequest, metav1.UpdateOptions{}) + if err != nil { + return errors.Wrap(err, "failed to CertificateSigningRequests().UpdateApproval") + } + + // try to get the Certificate issued by K8s + certPEM := retry.DoWithRetry(f.GinkgoT, "get approved certificate", 10, time.Second, func() (string, error) { + csr, err := f.clientset.CertificatesV1().CertificateSigningRequests().Get(context.Background(), certificateSigningRequest.GetName(), metav1.GetOptions{}) + if err != nil { + return "", err + } + if csr.Status.Certificate == nil { + return "", errors.New("certificate is not signed yet") + } + return string(csr.Status.Certificate), nil + }) + + // get client-ca-file as the caBundle for webhook ValidatingWebhookConfiguration + var cm corev1.ConfigMap + var cmKey = client.ObjectKey{ + Namespace: "kube-system", + Name: "extension-apiserver-authentication", + } + err = f.K8sClient.Get(context.Background(), cmKey, &cm) + if err != nil { + return errors.Wrapf(err, "failed to get ConfigMap: %v", cmKey) + } + + // set certificate info + opts.TLSKey = "\n" + privateKeyBuf.String() + opts.TLSCRT = "\n" + certPEM + opts.CaBundle = base64.StdEncoding.EncodeToString([]byte(cm.Data["client-ca-file"])) + + return nil +} diff --git a/test/e2e/framework/manifests/ingress.yaml b/test/e2e/framework/manifests/ingress.yaml index 61958d58a..6cc4c0dfb 100644 --- a/test/e2e/framework/manifests/ingress.yaml +++ b/test/e2e/framework/manifests/ingress.yaml @@ -340,6 +340,10 @@ data: controller_name: {{ .ControllerName | default "apisix.apache.org/api7-ingress-controller" }} leader_election_id: "api7-ingress-controller-leader" + tls.key: | + {{- .TLSKey | indent 4 }} + tls.crt: | + {{- .TLSCRT | indent 4 }} --- apiVersion: v1 kind: Service @@ -398,6 +402,13 @@ spec: - name: ingress-config mountPath: /app/conf/config.yaml subPath: config.yaml + - name: ingress-config + mountPath: /tmp/k8s-webhook-server/serving-certs + readOnly: true + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP livenessProbe: httpGet: path: /healthz @@ -429,3 +440,47 @@ spec: name: ingress-config serviceAccountName: api7-ingress-controller-manager terminationGracePeriodSeconds: 10 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: api7-ingress-controller + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: {{ .Namespace }} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: validating-webhook-configuration +webhooks: + - admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: {{ .Namespace }} + path: /validate-gateway-networking-k8s-io-v1-gatewayclass + caBundle: {{ .CaBundle }} + failurePolicy: Fail + name: vgatewayclass-v1.kb.io + rules: + - apiGroups: + - gateway.networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - gatewayclasses + sideEffects: None diff --git a/test/e2e/gatewayapi/gatewayclass.go b/test/e2e/gatewayapi/gatewayclass.go index 4bde9a9f7..e76dc2ede 100644 --- a/test/e2e/gatewayapi/gatewayclass.go +++ b/test/e2e/gatewayapi/gatewayclass.go @@ -31,6 +31,18 @@ metadata: name: api7-not-accepeted spec: controllerName: "apisix.apache.org/not-exist" +` + const defaultGateway = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: api7ee +spec: + gatewayClassName: api7 + listeners: + - name: http1 + protocol: HTTP + port: 80 ` It("Create GatewayClass", func() { By("create default GatewayClass") @@ -53,5 +65,35 @@ spec: Expect(gcyaml).To(ContainSubstring(`status: Unknown`), "checking GatewayClass condition status") Expect(gcyaml).To(ContainSubstring("message: Waiting for controller"), "checking GatewayClass condition message") }) + + It("Delete GatewayClass", func() { + By("create default GatewayClass") + err := s.CreateResourceFromStringWithNamespace(defautlGatewayClass, "") + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + Eventually(func() string { + spec, err := s.GetResourceYaml("GatewayClass", "api7") + Expect(err).NotTo(HaveOccurred(), "get resource yaml") + return spec + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second).Should(ContainSubstring(`status: "True"`)) + + By("create a Gateway") + err = s.CreateResourceFromString(defaultGateway) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + time.Sleep(time.Second) + + By("try to delete the GatewayClass") + err = s.DeleteResource("GatewayClass", "api7") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("the GatewayClass is still in using by Gateways")) + + By("delete the Gateway") + err = s.DeleteResource("Gateway", "api7ee") + Expect(err).NotTo(HaveOccurred(), "deleting Gateway") + time.Sleep(time.Second) + + By("try to delete the GatewayClass again") + err = s.DeleteResource("GatewayClass", "api7") + Expect(err).NotTo(HaveOccurred()) + }) }) })