Skip to content

Commit 38fd067

Browse files
deads2kdeads2k
committed
demonstrate VAP restricting action per-node
1 parent fad52ae commit 38fd067

File tree

5 files changed

+301
-2
lines changed

5 files changed

+301
-2
lines changed

‎test/e2e/auth/e2edata/per_node_validatingadmissionpolicy.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
apiVersion: admissionregistration.k8s.io/v1
2+
kind: ValidatingAdmissionPolicy
3+
metadata:
4+
name: "only-allow-name-matching-node-configmaps"
5+
spec:
6+
failurePolicy: Fail
7+
matchConstraints:
8+
resourceRules:
9+
- apiGroups: [""]
10+
apiVersions: ["v1"]
11+
operations: ["CREATE", "UPDATE", "DELETE"]
12+
resources: ["configmaps"]
13+
matchConditions:
14+
- name: isRestrictedUser
15+
# e2e-ns gets replaced with the tests's namespace when running the E2E test.
16+
expression: >-
17+
request.userInfo.username == "system:serviceaccount:e2e-ns:default"
18+
variables:
19+
- name: userNodeName
20+
expression: >-
21+
request.userInfo.extra[?'authentication.kubernetes.io/node-name'][0].orValue('')
22+
- name: objectNodeName
23+
expression: >-
24+
(request.operation == 'DELETE' ? oldObject : object).?metadata.name.orValue('')
25+
validations:
26+
- expression: variables.userNodeName != ""
27+
message: >-
28+
no node association found for user, this user must run in a pod on a node and ServiceAccountTokenPodNodeInfo must be enabled
29+
- expression: variables.userNodeName == variables.objectNodeName
30+
messageExpression: >-
31+
"this user running on node '"+variables.userNodeName+"' may not modify ConfigMap '" + variables.objectNodeName +
32+
"' because the name does not match the node name"

‎test/e2e/auth/e2edata/per_node_validatingadmissionpolicybinding.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: admissionregistration.k8s.io/v1
2+
kind: ValidatingAdmissionPolicyBinding
3+
metadata:
4+
name: "only-allow-name-matching-node-configmaps"
5+
spec:
6+
policyName: "only-allow-name-matching-node-configmaps"
7+
validationActions: [Deny]
8+
matchResources:
9+
namespaceSelector:
10+
matchLabels:
11+
kubernetes.io/metadata.name: "e2e-ns"

‎test/e2e/auth/per_node_update.go‎

Lines changed: 251 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,251 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package auth
18+
19+
import (
20+
"context"
21+
_ "embed"
22+
"fmt"
23+
"strings"
24+
25+
g "github.com/onsi/ginkgo/v2"
26+
o "github.com/onsi/gomega"
27+
admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
28+
authenticationv1 "k8s.io/api/authentication/v1"
29+
v1 "k8s.io/api/core/v1"
30+
rbacv1 "k8s.io/api/rbac/v1"
31+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
32+
"k8s.io/apimachinery/pkg/runtime"
33+
"k8s.io/apiserver/pkg/authentication/serviceaccount"
34+
"k8s.io/client-go/kubernetes"
35+
cgoscheme "k8s.io/client-go/kubernetes/scheme"
36+
"k8s.io/client-go/rest"
37+
"k8s.io/kubernetes/pkg/features"
38+
"k8s.io/kubernetes/test/e2e/framework"
39+
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
40+
imageutils "k8s.io/kubernetes/test/utils/image"
41+
admissionapi "k8s.io/pod-security-admission/api"
42+
"k8s.io/utils/ptr"
43+
)
44+
45+
// Embed manifests that we leave as yaml to make it clear to users how to these permissions are created.
46+
// These will match future docs.
47+
var (
48+
//go:embed e2edata/per_node_validatingadmissionpolicy.yaml
49+
perNodeCheckValidatingAdmissionPolicy string
50+
51+
//go:embed e2edata/per_node_validatingadmissionpolicybinding.yaml
52+
perNodeCheckValidatingAdmissionPolicyBinding string
53+
)
54+
55+
var _ = SIGDescribe("ValidatingAdmissionPolicy", framework.WithFeatureGate(features.ServiceAccountTokenNodeBindingValidation), func() {
56+
defer g.GinkgoRecover()
57+
f := framework.NewDefaultFramework("node-authn")
58+
f.NamespacePodSecurityLevel = admissionapi.LevelRestricted
59+
60+
g.It("can restrict access by-node", func(ctx context.Context) {
61+
admission := strings.ReplaceAll(perNodeCheckValidatingAdmissionPolicy, "e2e-ns", f.Namespace.Name)
62+
admissionToCreate := readValidatingAdmissionPolicyV1OrDie([]byte(admission))
63+
admissionBinding := strings.ReplaceAll(perNodeCheckValidatingAdmissionPolicyBinding, "e2e-ns", f.Namespace.Name)
64+
admissionBindingToCreate := readValidatingAdmissionPolicyBindingV1OrDie([]byte(admissionBinding))
65+
66+
saTokenRoleBinding := &rbacv1.RoleBinding{
67+
ObjectMeta: metav1.ObjectMeta{
68+
Namespace: f.Namespace.Name,
69+
Name: "sa-token",
70+
},
71+
Subjects: []rbacv1.Subject{
72+
{
73+
Kind: "ServiceAccount",
74+
Name: "default",
75+
Namespace: f.Namespace.Name,
76+
},
77+
},
78+
RoleRef: rbacv1.RoleRef{
79+
APIGroup: "rbac.authorization.k8s.io",
80+
Kind: "ClusterRole",
81+
Name: "edit",
82+
},
83+
}
84+
85+
agnhost := imageutils.GetConfig(imageutils.Agnhost)
86+
sleeperPod := e2epod.MustMixinRestrictedPodSecurity(&v1.Pod{
87+
ObjectMeta: metav1.ObjectMeta{
88+
Namespace: f.Namespace.Name,
89+
Name: "sa-token",
90+
},
91+
Spec: v1.PodSpec{
92+
Containers: []v1.Container{
93+
{
94+
Name: "sleeper",
95+
Image: agnhost.GetE2EImage(),
96+
Command: []string{"sleep"},
97+
Args: []string{"1200"},
98+
SecurityContext: &v1.SecurityContext{
99+
AllowPrivilegeEscalation: ptr.To(false),
100+
Capabilities: &v1.Capabilities{
101+
Drop: []v1.Capability{"ALL"},
102+
},
103+
},
104+
},
105+
},
106+
},
107+
Status: v1.PodStatus{},
108+
})
109+
110+
// cleanup the ValidatingAdmissionPolicy.
111+
112+
var err error
113+
_, err = f.ClientSet.AdmissionregistrationV1().ValidatingAdmissionPolicies().Create(ctx, admissionToCreate, metav1.CreateOptions{})
114+
framework.ExpectNoError(err)
115+
g.DeferCleanup(f.ClientSet.AdmissionregistrationV1().ValidatingAdmissionPolicies().Delete, admissionToCreate.Name, metav1.DeleteOptions{})
116+
117+
_, err = f.ClientSet.AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Create(ctx, admissionBindingToCreate, metav1.CreateOptions{})
118+
framework.ExpectNoError(err)
119+
g.DeferCleanup(f.ClientSet.AdmissionregistrationV1().ValidatingAdmissionPolicyBindings().Delete, admissionBindingToCreate.Name, metav1.DeleteOptions{})
120+
121+
// create permissions that will allow unrestricted access to mutate configmaps in this namespace.
122+
// We limited these permissions in the step above.
123+
// This means the admission policy must fail closed or permissions will be too broad.
124+
_, err = f.ClientSet.RbacV1().RoleBindings(f.Namespace.Name).Create(ctx, saTokenRoleBinding, metav1.CreateOptions{})
125+
framework.ExpectNoError(err)
126+
127+
// run an actual pod to prove that the token is injected, not just creatable via the API
128+
actualPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(ctx, sleeperPod, metav1.CreateOptions{})
129+
framework.ExpectNoError(err)
130+
err = e2epod.WaitForPodNameRunningInNamespace(ctx, f.ClientSet, actualPod.Name, actualPod.Namespace)
131+
framework.ExpectNoError(err)
132+
// need the pod that contains the node name
133+
actualPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, actualPod.Name, metav1.GetOptions{})
134+
framework.ExpectNoError(err)
135+
136+
// get the actual projected token from the pod.
137+
nodeScopedSAToken, stderr, err := e2epod.ExecWithOptionsContext(ctx, f, e2epod.ExecOptions{
138+
Command: []string{"cat", "/var/run/secrets/kubernetes.io/serviceaccount/token"},
139+
Namespace: actualPod.Namespace,
140+
PodName: actualPod.Name,
141+
ContainerName: actualPod.Spec.Containers[0].Name,
142+
CaptureStdout: true,
143+
CaptureStderr: true,
144+
PreserveWhitespace: true,
145+
})
146+
framework.ExpectNoError(err)
147+
o.Expect(stderr).To(o.BeEmpty(), "stderr from cat")
148+
149+
// make a kubeconfig with the token and confirm the kube-apiserver has the expected claims
150+
nodeScopedClientConfig := rest.AnonymousClientConfig(f.ClientConfig())
151+
nodeScopedClientConfig.BearerToken = nodeScopedSAToken
152+
nodeScopedClient, err := kubernetes.NewForConfig(nodeScopedClientConfig)
153+
framework.ExpectNoError(err)
154+
saUser, err := nodeScopedClient.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{})
155+
framework.ExpectNoError(err)
156+
expectedUser := serviceaccount.MakeUsername(f.Namespace.Name, "default")
157+
o.Expect(saUser.Status.UserInfo.Username).To(o.Equal(expectedUser))
158+
expectedNode := authenticationv1.ExtraValue([]string{actualPod.Spec.NodeName})
159+
o.Expect(saUser.Status.UserInfo.Extra["authentication.kubernetes.io/node-name"]).To(o.Equal(expectedNode))
160+
161+
allowedConfigMap := &v1.ConfigMap{
162+
ObjectMeta: metav1.ObjectMeta{
163+
Namespace: f.Namespace.Name,
164+
Name: actualPod.Spec.NodeName,
165+
},
166+
}
167+
disallowedConfigMap := &v1.ConfigMap{
168+
ObjectMeta: metav1.ObjectMeta{
169+
Namespace: f.Namespace.Name,
170+
Name: "unlikely-node",
171+
},
172+
}
173+
disallowedMessage := fmt.Sprintf("this user running on node '%s' may not modify ConfigMap '%s' because the name does not match the node name", actualPod.Spec.NodeName, disallowedConfigMap.Name)
174+
175+
actualAllowedConfigMap, err := nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, allowedConfigMap, metav1.CreateOptions{})
176+
framework.ExpectNoError(err)
177+
_, err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, disallowedConfigMap, metav1.CreateOptions{})
178+
o.Expect(err).To(o.HaveOccurred())
179+
o.Expect(err.Error()).To(o.ContainSubstring(disallowedMessage))
180+
181+
// now create so we can see the update cases
182+
actualDisallowedConfigMap, err := f.ClientSet.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, disallowedConfigMap, metav1.CreateOptions{})
183+
framework.ExpectNoError(err)
184+
185+
actualAllowedConfigMap, err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Update(ctx, actualAllowedConfigMap, metav1.UpdateOptions{})
186+
framework.ExpectNoError(err)
187+
_, err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Update(ctx, actualDisallowedConfigMap, metav1.UpdateOptions{})
188+
o.Expect(err).To(o.HaveOccurred())
189+
o.Expect(err.Error()).To(o.ContainSubstring(disallowedMessage))
190+
191+
err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, actualAllowedConfigMap.Name, metav1.DeleteOptions{})
192+
framework.ExpectNoError(err)
193+
err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, actualDisallowedConfigMap.Name, metav1.DeleteOptions{})
194+
o.Expect(err).To(o.HaveOccurred())
195+
o.Expect(err.Error()).To(o.ContainSubstring(disallowedMessage))
196+
197+
// recreate the allowedConfigMap and then do a delete collection
198+
_, err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, allowedConfigMap, metav1.CreateOptions{})
199+
framework.ExpectNoError(err)
200+
err = nodeScopedClient.CoreV1().ConfigMaps(f.Namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
201+
o.Expect(err).To(o.HaveOccurred())
202+
// Delete collection can happen in random/racy orders. We'll match on everything except the name
203+
disallowedAnyNameMessage := fmt.Sprintf("this user running on node '%s' may not modify ConfigMap .* because the name does not match the node name", actualPod.Spec.NodeName)
204+
o.Expect(err.Error()).To(o.MatchRegexp(disallowedAnyNameMessage))
205+
206+
// ensure that if the node claim is missing from the restricted service-account user, we reject the request
207+
tokenRequest := &authenticationv1.TokenRequest{
208+
Spec: authenticationv1.TokenRequestSpec{
209+
ExpirationSeconds: ptr.To[int64](600),
210+
},
211+
}
212+
tokenRequestResponse, err := f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name).CreateToken(ctx, "default", tokenRequest, metav1.CreateOptions{})
213+
framework.ExpectNoError(err)
214+
serviceAccountConfigWithoutNodeClaim := rest.AnonymousClientConfig(f.ClientConfig())
215+
serviceAccountConfigWithoutNodeClaim.BearerToken = tokenRequestResponse.Status.Token
216+
serviceAccountClientWithoutNodeClaim, err := kubernetes.NewForConfig(serviceAccountConfigWithoutNodeClaim)
217+
framework.ExpectNoError(err)
218+
// now confirm this token lacks a node name claim.
219+
selfSubjectResults, err := serviceAccountClientWithoutNodeClaim.AuthenticationV1().SelfSubjectReviews().Create(ctx, &authenticationv1.SelfSubjectReview{}, metav1.CreateOptions{})
220+
framework.Logf("Token: %q expires at %v", serviceAccountConfigWithoutNodeClaim.BearerToken, tokenRequestResponse.Status.ExpirationTimestamp)
221+
framework.ExpectNoError(err)
222+
o.Expect(selfSubjectResults.Status.UserInfo.Extra["authentication.kubernetes.io/node-name"]).To(o.BeEmpty())
223+
224+
noNodeAssociationMessage := "no node association found for user, this user must run in a pod on a node and ServiceAccountTokenPodNodeInfo must be enabled"
225+
_, err = serviceAccountClientWithoutNodeClaim.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, actualDisallowedConfigMap, metav1.CreateOptions{})
226+
o.Expect(err).To(o.HaveOccurred())
227+
o.Expect(err.Error()).To(o.ContainSubstring(noNodeAssociationMessage))
228+
_, err = serviceAccountClientWithoutNodeClaim.CoreV1().ConfigMaps(f.Namespace.Name).Create(ctx, actualAllowedConfigMap, metav1.CreateOptions{})
229+
o.Expect(err).To(o.HaveOccurred())
230+
o.Expect(err.Error()).To(o.ContainSubstring(noNodeAssociationMessage))
231+
err = serviceAccountClientWithoutNodeClaim.CoreV1().ConfigMaps(f.Namespace.Name).Delete(ctx, actualDisallowedConfigMap.Name, metav1.DeleteOptions{})
232+
o.Expect(err).To(o.HaveOccurred())
233+
o.Expect(err.Error()).To(o.ContainSubstring(noNodeAssociationMessage))
234+
})
235+
})
236+
237+
func readValidatingAdmissionPolicyV1OrDie(objBytes []byte) *admissionregistrationv1.ValidatingAdmissionPolicy {
238+
requiredObj, err := runtime.Decode(cgoscheme.Codecs.UniversalDecoder(admissionregistrationv1.SchemeGroupVersion), objBytes)
239+
if err != nil {
240+
panic(err)
241+
}
242+
return requiredObj.(*admissionregistrationv1.ValidatingAdmissionPolicy)
243+
}
244+
245+
func readValidatingAdmissionPolicyBindingV1OrDie(objBytes []byte) *admissionregistrationv1.ValidatingAdmissionPolicyBinding {
246+
requiredObj, err := runtime.Decode(cgoscheme.Codecs.UniversalDecoder(admissionregistrationv1.SchemeGroupVersion), objBytes)
247+
if err != nil {
248+
panic(err)
249+
}
250+
return requiredObj.(*admissionregistrationv1.ValidatingAdmissionPolicyBinding)
251+
}

‎test/e2e/feature/feature.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,12 @@ var (
224224
// TODO: document the feature (owning SIG, when to use this feature for a test)
225225
NodeAuthenticator = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthenticator"))
226226

227+
// Owner: sig-auth
228+
// Marks tests that require a conforming implementation of
229+
// Node claims for serviceaccounts. Typically this means that the
230+
// ServiceAccountTokenNodeBindingValidation feature must be enabled.
231+
ServiceAccountTokenNodeBindingValidation = framework.WithFeature(framework.ValidFeatures.Add("ServiceAccountTokenNodeBindingValidation"))
232+
227233
// TODO: document the feature (owning SIG, when to use this feature for a test)
228234
NodeAuthorizer = framework.WithFeature(framework.ValidFeatures.Add("NodeAuthorizer"))
229235

‎test/e2e/framework/pod/exec_util.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,7 @@ func ExecWithOptionsContext(ctx context.Context, f *framework.Framework, options
6868
Resource("pods").
6969
Name(options.PodName).
7070
Namespace(options.Namespace).
71-
SubResource("exec").
72-
Param("container", options.ContainerName)
71+
SubResource("exec")
7372
req.VersionedParams(&v1.PodExecOptions{
7473
Container: options.ContainerName,
7574
Command: options.Command,

0 commit comments

Comments
 (0)