diff --git a/test/e2e/framework/cluster.go b/test/e2e/framework/cluster.go index b9f3d7da0..9c91fc4c2 100644 --- a/test/e2e/framework/cluster.go +++ b/test/e2e/framework/cluster.go @@ -46,6 +46,8 @@ type Cluster struct { RestMapper meta.RESTMapper PricingProvider trackers.PricingProvider SystemMastersClient client.Client + KubeSystemClient client.Client + FleetSystemClient client.Client } func NewCluster(name, svcAccountName string, scheme *runtime.Scheme, pp trackers.PricingProvider) *Cluster { @@ -62,6 +64,8 @@ func GetClusterClient(cluster *Cluster) { clusterConfig := GetClientConfig(cluster) impersonateClusterConfig := GetImpersonateClientConfig(cluster) systemMastersConfig := GetSystemMastersClientConfig(cluster) + kubeSystemConfig := GetKubeSystemClientConfig(cluster) + fleetSystemConfig := GetFleetSystemClientConfig(cluster) restConfig, err := clusterConfig.ClientConfig() if err != nil { @@ -75,7 +79,17 @@ func GetClusterClient(cluster *Cluster) { systemMastersRestConfig, err := systemMastersConfig.ClientConfig() if err != nil { - gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up impersonate rest config") + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up system masters rest config") + } + + kubeSystemRestConfig, err := kubeSystemConfig.ClientConfig() + if err != nil { + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up kube-system service account rest config") + } + + fleetSystemRestConfig, err := fleetSystemConfig.ClientConfig() + if err != nil { + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up fleet-system service account rest config") } cluster.KubeClient, err = client.New(restConfig, client.Options{Scheme: cluster.Scheme}) @@ -94,7 +108,13 @@ func GetClusterClient(cluster *Cluster) { gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Impersonate Kube Client") cluster.SystemMastersClient, err = client.New(systemMastersRestConfig, client.Options{Scheme: cluster.Scheme}) - gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Impersonate Kube Client") + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up System Masters Kube Client") + + cluster.KubeSystemClient, err = client.New(kubeSystemRestConfig, client.Options{Scheme: cluster.Scheme}) + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Kube System Service Account Client") + + cluster.FleetSystemClient, err = client.New(fleetSystemRestConfig, client.Options{Scheme: cluster.Scheme}) + gomega.Expect(err).Should(gomega.Succeed(), "Failed to set up Fleet System Service Account Client") } func GetClientConfig(cluster *Cluster) clientcmd.ClientConfig { @@ -128,3 +148,33 @@ func GetImpersonateClientConfig(cluster *Cluster) clientcmd.ClientConfig { }, }) } + +func GetKubeSystemClientConfig(cluster *Cluster) clientcmd.ClientConfig { + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ + CurrentContext: cluster.ClusterName, + AuthInfo: api.AuthInfo{ + Impersonate: "system:serviceaccount:kube-system:service-account-controller", + ImpersonateGroups: []string{ + "system:serviceaccounts:kube-system", + }, + }, + }, + ) +} + +func GetFleetSystemClientConfig(cluster *Cluster) clientcmd.ClientConfig { + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ + CurrentContext: cluster.ClusterName, + AuthInfo: api.AuthInfo{ + Impersonate: "system:serviceaccount:fleet-system:service-account-controller", + ImpersonateGroups: []string{ + "system:serviceaccounts:fleet-system", + }, + }, + }, + ) +} diff --git a/test/e2e/managed_resource_vap_test.go b/test/e2e/managed_resource_vap_test.go index 5bbd641c3..8525244b3 100644 --- a/test/e2e/managed_resource_vap_test.go +++ b/test/e2e/managed_resource_vap_test.go @@ -26,6 +26,7 @@ import ( admissionregistrationv1 "k8s.io/api/admissionregistration/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -127,6 +128,63 @@ func expectDeniedByVAP(err error) { } var _ = Describe("ValidatingAdmissionPolicy for Managed Resources", Label("managedresource"), Ordered, func() { + var clusterRole *rbacv1.ClusterRole + var clusterRoleBinding *rbacv1.ClusterRoleBinding + + BeforeAll(func() { + By("Give permissions to service accounts") + // --- Create ClusterRole --- + clusterRole = &rbacv1.ClusterRole{ + ObjectMeta: metav1.ObjectMeta{ + Name: "allow-certain-managed-resources", + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, // Core API group + Resources: []string{"resourcequotas", "namespaces"}, + Verbs: []string{"create", "update", "delete"}, + }, + { + APIGroups: []string{"networking.k8s.io"}, + Resources: []string{"networkpolicies"}, + Verbs: []string{"create", "update", "delete"}, + }, + }, + } + Expect(hubClient.Create(ctx, clusterRole)).To(Succeed()) + + // --- Create ClusterRoleBinding --- + clusterRoleBinding = &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service-accounts-binding-for-managed-resources", + }, + Subjects: []rbacv1.Subject{ + { + Kind: rbacv1.ServiceAccountKind, + Name: "service-account-controller", // The service account's name + Namespace: "kube-system", // The service account's namespace + }, + { + Kind: rbacv1.ServiceAccountKind, + Name: "service-account-controller", // The service account's name + Namespace: "fleet-system", // The service account's namespace + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: rbacv1.GroupName, + Kind: "ClusterRole", + Name: clusterRole.Name, + }, + } + Expect(hubClient.Create(ctx, clusterRoleBinding)).To(Succeed()) + }) + + AfterAll(func() { + By("Cleaning up service account permissions") + Expect(hubClient.Delete(ctx, clusterRoleBinding)).To(Succeed()) + Expect(hubClient.Delete(ctx, clusterRole)).To(Succeed()) + }) + It("The VAP and its binding should exist", func() { var vap admissionregistrationv1.ValidatingAdmissionPolicy Expect(sysMastersClient.Get(ctx, types.NamespacedName{Name: vapName}, &vap)).Should(Succeed(), "ValidatingAdmissionPolicy should be installed") @@ -169,6 +227,24 @@ var _ = Describe("ValidatingAdmissionPolicy for Managed Resources", Label("manag Expect(sysMastersClient.Delete(ctx, managedNS)).To(Succeed()) }) + + It("should allow CREATE operation on managed namespace for system:serviceaccount:kube-system user", func() { + managedNS := createManagedNamespace("test-managed-ns-kubesystem-sa") + By("expecting successful CREATE operation with system:serviceaccount:kube-system user") + Expect(kubeSystemClient.Create(ctx, managedNS)).To(Succeed()) + + By("expecting successful DELETE operation on managed namespace") + Expect(sysMastersClient.Delete(ctx, managedNS)).To(Succeed()) + }) + + It("should allow CREATE operation on managed namespace for system:serviceaccounts:fleet-system user", func() { + managedNS := createManagedNamespace("test-managed-ns-fleet-system") + By("expecting successful CREATE operation with system:serviceaccounts:fleet-system user") + Expect(fleetSystemClient.Create(ctx, managedNS)).To(Succeed()) + + By("expecting successful DELETE operation on managed namespace") + Expect(fleetSystemClient.Delete(ctx, managedNS)).To(Succeed()) + }) }) Context("When the namespace exists", Ordered, func() { @@ -227,6 +303,40 @@ var _ = Describe("ValidatingAdmissionPolicy for Managed Resources", Label("manag }, eventuallyDuration, eventuallyInterval).Should(Succeed()) }) + It("should allow UPDATE operation on managed namespace for system:serviceaccounts:kube-system user", func() { + var updateErr error + Eventually(func() error { + var ns corev1.Namespace + if err := sysMastersClient.Get(ctx, types.NamespacedName{Name: managedNS.Name}, &ns); err != nil { + return err + } + ns.Annotations = map[string]string{"test": "annotation"} + By("expecting denial of UPDATE operation on managed namespace") + updateErr = kubeSystemClient.Update(ctx, &ns) + if k8sErrors.IsConflict(updateErr) { + return updateErr + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + + It("should allow UPDATE operation on managed namespace for system:serviceaccounts:fleet-system user", func() { + var updateErr error + Eventually(func() error { + var ns corev1.Namespace + if err := sysMastersClient.Get(ctx, types.NamespacedName{Name: managedNS.Name}, &ns); err != nil { + return err + } + ns.Annotations = map[string]string{"test": "annotation"} + By("expecting denial of UPDATE operation on managed namespace") + updateErr = fleetSystemClient.Update(ctx, &ns) + if k8sErrors.IsConflict(updateErr) { + return updateErr + } + return nil + }, eventuallyDuration, eventuallyInterval).Should(Succeed()) + }) + Context("For other resources in scope", func() { It("should deny creating managed resource quotas", func() { rq := createManagedResourceQuota("default", "default") @@ -289,6 +399,42 @@ var _ = Describe("ValidatingAdmissionPolicy for Managed Resources", Label("manag err = sysMastersClient.Delete(ctx, crp) Expect(err).To(BeNil(), "system:masters user should delete managed CRP") }) + + It("should allow CREATE operation on managed ResourceQuota for kube-system service account", func() { + rq := createManagedResourceQuota(managedNS.Name, "default") + By("expecting successful CREATE operation with kube-system service account") + Expect(kubeSystemClient.Create(ctx, rq)).To(Succeed()) + + By("expecting successful DELETE operation with kube-system service account") + Expect(kubeSystemClient.Delete(ctx, rq)).To(Succeed()) + }) + + It("should allow CREATE operation on managed ResourceQuota for fleet-system service account", func() { + rq := createManagedResourceQuota(managedNS.Name, "default") + By("expecting successful CREATE operation with fleet-system service account") + Expect(fleetSystemClient.Create(ctx, rq)).To(Succeed()) + + By("expecting successful DELETE operation with fleet-system service account") + Expect(fleetSystemClient.Delete(ctx, rq)).To(Succeed()) + }) + + It("should allow CREATE operation on managed NetworkPolicy for kube-system service account", func() { + netpol := createManagedNetworkPolicy(managedNS.Name, "default") + By("expecting successful CREATE operation with kube-system service account") + Expect(kubeSystemClient.Create(ctx, netpol)).To(Succeed()) + + By("expecting successful DELETE operation with kube-system service account") + Expect(kubeSystemClient.Delete(ctx, netpol)).To(Succeed()) + }) + + It("should allow CREATE operation on managed NetworkPolicy for fleet-system service account", func() { + netpol := createManagedNetworkPolicy(managedNS.Name, "default") + By("expecting successful CREATE operation with fleet-system service account") + Expect(fleetSystemClient.Create(ctx, netpol)).To(Succeed()) + + By("expecting successful DELETE operation with fleet-system service account") + Expect(fleetSystemClient.Delete(ctx, netpol)).To(Succeed()) + }) }) AfterAll(func() { diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go index 4b2c8a302..037525fb5 100644 --- a/test/e2e/setup_test.go +++ b/test/e2e/setup_test.go @@ -109,6 +109,8 @@ var ( hubClient client.Client notMasterUser client.Client sysMastersClient client.Client + kubeSystemClient client.Client + fleetSystemClient client.Client memberCluster1EastProdClient client.Client memberCluster2EastCanaryClient client.Client memberCluster3WestProdClient client.Client @@ -336,7 +338,11 @@ func beforeSuiteForAllProcesses() { notMasterUser = hubCluster.ImpersonateKubeClient Expect(notMasterUser).NotTo(BeNil(), "Failed to initialize impersonate client for accessing Kubernetes cluster") sysMastersClient = hubCluster.SystemMastersClient - Expect(sysMastersClient).NotTo(BeNil(), "Failed to initialize impersonate client for accessing Kubernetes cluster") + Expect(sysMastersClient).NotTo(BeNil(), "Failed to initialize impersonate system masters client for accessing Kubernetes cluster") + kubeSystemClient = hubCluster.KubeSystemClient + Expect(kubeSystemClient).NotTo(BeNil(), "Failed to initialize kube-system service account client for accessing Kubernetes cluster") + fleetSystemClient = hubCluster.FleetSystemClient + Expect(fleetSystemClient).NotTo(BeNil(), "Failed to initialize fleet-system service account client for accessing Kubernetes cluster") var pricingProvider1 trackers.PricingProvider if isAzurePropertyProviderEnabled {