Skip to content

Commit 2090a01

Browse files
committed
add e2e test with the gcp-credential-provider test plugin
Signed-off-by: Anish Ramasekar <[email protected]>
1 parent ad8666c commit 2090a01

File tree

4 files changed

+163
-10
lines changed

4 files changed

+163
-10
lines changed

test/e2e/common/node/image_credential_provider.go

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
v1 "k8s.io/api/core/v1"
2525
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2626
"k8s.io/apimachinery/pkg/util/uuid"
27+
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
2728
"k8s.io/kubernetes/test/e2e/feature"
2829
"k8s.io/kubernetes/test/e2e/framework"
2930
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -34,10 +35,12 @@ import (
3435
var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProviders, func() {
3536
f := framework.NewDefaultFramework("image-credential-provider")
3637
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
38+
var serviceAccountClient typedcorev1.ServiceAccountInterface
3739
var podClient *e2epod.PodClient
3840

3941
ginkgo.BeforeEach(func() {
4042
podClient = e2epod.NewPodClient(f)
43+
serviceAccountClient = f.ClientSet.CoreV1().ServiceAccounts(f.Namespace.Name)
4144
})
4245

4346
/*
@@ -48,11 +51,28 @@ var _ = SIGDescribe("ImageCredentialProvider", feature.KubeletCredentialProvider
4851
ginkgo.It("should be able to create pod with image credentials fetched from external credential provider ", func(ctx context.Context) {
4952
privateimage := imageutils.GetConfig(imageutils.AgnhostPrivate)
5053
name := "pod-auth-image-" + string(uuid.NewUUID())
54+
55+
// The service account is required to exist for the credential provider plugin that's configured to use service account token.
56+
serviceAccount := &v1.ServiceAccount{
57+
ObjectMeta: metav1.ObjectMeta{
58+
Name: "test-service-account",
59+
// these annotations are validated by the test gcp-credential-provider-with-sa plugin
60+
// that runs in service account token mode.
61+
Annotations: map[string]string{
62+
"domain.io/identity-id": "123456",
63+
"domain.io/identity-type": "serviceaccount",
64+
},
65+
},
66+
}
67+
_, err := serviceAccountClient.Create(ctx, serviceAccount, metav1.CreateOptions{})
68+
framework.ExpectNoError(err)
69+
5170
pod := &v1.Pod{
5271
ObjectMeta: metav1.ObjectMeta{
5372
Name: name,
5473
},
5574
Spec: v1.PodSpec{
75+
ServiceAccountName: "test-service-account",
5676
Containers: []v1.Container{
5777
{
5878
Name: "container-auth-image",

test/e2e_node/plugins/gcp-credential-provider/main.go

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,29 @@ limitations under the License.
1717
package main
1818

1919
import (
20+
"encoding/base64"
2021
"encoding/json"
2122
"errors"
23+
"fmt"
2224
"io"
2325
"net/http"
2426
"os"
27+
"reflect"
28+
"strings"
2529
"time"
2630

31+
"gopkg.in/go-jose/go-jose.v2/jwt"
32+
2733
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2834
"k8s.io/klog/v2"
2935
credentialproviderv1 "k8s.io/kubelet/pkg/apis/credentialprovider/v1"
3036
)
3137

32-
const metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
38+
const (
39+
metadataTokenEndpoint = "http://metadata.google.internal./computeMetadata/v1/instance/service-accounts/default/token"
40+
41+
pluginModeEnvVar = "PLUGIN_MODE"
42+
)
3343

3444
func main() {
3545
if err := getCredentials(metadataTokenEndpoint, os.Stdin, os.Stdout); err != nil {
@@ -56,6 +66,40 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
5666
return err
5767
}
5868

69+
pluginUsingServiceAccount := os.Getenv(pluginModeEnvVar) == "serviceaccount"
70+
if pluginUsingServiceAccount {
71+
if len(authRequest.ServiceAccountToken) == 0 {
72+
return errors.New("service account token is empty")
73+
}
74+
expectedAnnotations := map[string]string{
75+
"domain.io/identity-id": "123456",
76+
"domain.io/identity-type": "serviceaccount",
77+
}
78+
if !reflect.DeepEqual(authRequest.ServiceAccountAnnotations, expectedAnnotations) {
79+
return fmt.Errorf("unexpected service account annotations, want: %v, got: %v", expectedAnnotations, authRequest.ServiceAccountAnnotations)
80+
}
81+
// The service account token is not actually used for authentication by this test plugin.
82+
// We extract the claims from the token to validate the audience.
83+
// This is solely for testing assertions and is not an actual security layer.
84+
// Post validation in this block, we proceed with the default flow for fetching credentials.
85+
c, err := getClaims(authRequest.ServiceAccountToken)
86+
if err != nil {
87+
return err
88+
}
89+
// The audience in the token should match the audience configured in tokenAttributes.serviceAccountTokenAudience
90+
// in CredentialProviderConfig.
91+
if len(c.Audience) != 1 || c.Audience[0] != "test-audience" {
92+
return fmt.Errorf("unexpected audience: %v", c.Audience)
93+
}
94+
} else {
95+
if len(authRequest.ServiceAccountToken) > 0 {
96+
return errors.New("service account token is not expected")
97+
}
98+
if len(authRequest.ServiceAccountAnnotations) > 0 {
99+
return errors.New("service account annotations are not expected")
100+
}
101+
}
102+
59103
auth, err := provider.Provide(authRequest.Image)
60104
if err != nil {
61105
return err
@@ -70,10 +114,66 @@ func getCredentials(tokenEndpoint string, r io.Reader, w io.Writer) error {
70114
Auth: auth,
71115
}
72116

117+
if pluginUsingServiceAccount {
118+
response.CacheKeyType = credentialproviderv1.GlobalPluginCacheKeyType
119+
}
120+
73121
if err := json.NewEncoder(w).Encode(response); err != nil {
74122
// The error from json.Marshal is intentionally not included so as to not leak credentials into the logs
75123
return errors.New("error marshaling response")
76124
}
77125

78126
return nil
79127
}
128+
129+
// getClaims is used to extract claims from the service account token when the plugin is running in service account mode
130+
// This is solely for testing assertions and is not an actual security layer.
131+
// We get claims and validate the audience of the token (audience in the token matches the audience configured
132+
// in tokenAttributes.serviceAccountTokenAudience in CredentialProviderConfig).
133+
func getClaims(tokenData string) (claims, error) {
134+
if strings.HasPrefix(strings.TrimSpace(tokenData), "{") {
135+
return claims{}, errors.New("token is not a JWS")
136+
}
137+
parts := strings.Split(tokenData, ".")
138+
if len(parts) != 3 {
139+
return claims{}, errors.New("token is not a JWS")
140+
}
141+
payload, err := base64.RawURLEncoding.DecodeString(parts[1])
142+
if err != nil {
143+
return claims{}, fmt.Errorf("error decoding token payload: %w", err)
144+
}
145+
146+
var c claims
147+
d := json.NewDecoder(strings.NewReader(string(payload)))
148+
d.DisallowUnknownFields()
149+
if err := d.Decode(&c); err != nil {
150+
return claims{}, fmt.Errorf("error decoding token payload: %w", err)
151+
}
152+
153+
return c, nil
154+
}
155+
156+
type claims struct {
157+
jwt.Claims
158+
privateClaims
159+
}
160+
161+
// copied from https://github.com/kubernetes/kubernetes/blob/60c4c2b2521fb454ce69dee737e3eb91a25e0535/pkg/serviceaccount/claims.go#L51-L67
162+
163+
type privateClaims struct {
164+
Kubernetes kubernetes `json:"kubernetes.io,omitempty"`
165+
}
166+
167+
type kubernetes struct {
168+
Namespace string `json:"namespace,omitempty"`
169+
Svcacct ref `json:"serviceaccount,omitempty"`
170+
Pod *ref `json:"pod,omitempty"`
171+
Secret *ref `json:"secret,omitempty"`
172+
Node *ref `json:"node,omitempty"`
173+
WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"`
174+
}
175+
176+
type ref struct {
177+
Name string `json:"name,omitempty"`
178+
UID string `json:"uid,omitempty"`
179+
}

test/e2e_node/remote/node_e2e.go

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,21 @@ func (n *NodeE2ERemote) SetupTestPackage(tardir, systemSpecName string) error {
7272
}
7373
}
7474

75+
// create a symlink of gcp-credential-provider binary to use for testing
76+
// service account token for credential providers.
77+
// feature-gate: KubeletServiceAccountTokenForCredentialProviders=true
78+
binary := "gcp-credential-provider" // Use relative path instead of full path
79+
symlink := filepath.Join(tardir, "gcp-credential-provider-with-sa")
80+
if _, err := os.Lstat(symlink); err == nil {
81+
if err := os.Remove(symlink); err != nil {
82+
return fmt.Errorf("failed to remove symlink %q: %w", symlink, err)
83+
}
84+
}
85+
klog.V(2).Infof("Creating symlink %s -> %s", symlink, binary)
86+
if err := os.Symlink(binary, symlink); err != nil {
87+
return fmt.Errorf("failed to create symlink %q: %w", symlink, err)
88+
}
89+
7590
if systemSpecName != "" {
7691
// Copy system spec file
7792
source := filepath.Join(rootDir, system.SystemSpecPath, systemSpecName+".yaml")
@@ -97,9 +112,10 @@ func prependMemcgNotificationFlag(args string) string {
97112
// a credential provider plugin.
98113
func prependCredentialProviderFlag(args, workspace string) string {
99114
credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml")
115+
featureGateFlag := "--kubelet-flags=--feature-gates=KubeletServiceAccountTokenForCredentialProviders=true"
100116
configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig)
101117
binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace)
102-
return fmt.Sprintf("%s %s %s", configFlag, binFlag, args)
118+
return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args)
103119
}
104120

105121
// osSpecificActions takes OS specific actions required for the node tests

test/e2e_node/remote/utils.go

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -53,14 +53,31 @@ const cniConfig = `{
5353
const credentialGCPProviderConfig = `kind: CredentialProviderConfig
5454
apiVersion: kubelet.config.k8s.io/v1
5555
providers:
56-
- name: gcp-credential-provider
57-
apiVersion: credentialprovider.kubelet.k8s.io/v1
58-
matchImages:
59-
- "gcr.io"
60-
- "*.gcr.io"
61-
- "container.cloud.google.com"
62-
- "*.pkg.dev"
63-
defaultCacheDuration: 1m`
56+
- name: gcp-credential-provider
57+
apiVersion: credentialprovider.kubelet.k8s.io/v1
58+
matchImages:
59+
- "gcr.io"
60+
- "*.gcr.io"
61+
- "container.cloud.google.com"
62+
- "*.pkg.dev"
63+
defaultCacheDuration: 1m
64+
- name: gcp-credential-provider-with-sa
65+
apiVersion: credentialprovider.kubelet.k8s.io/v1
66+
matchImages:
67+
- "gcr.io"
68+
- "*.gcr.io"
69+
- "container.cloud.google.com"
70+
- "*.pkg.dev"
71+
defaultCacheDuration: 1m
72+
tokenAttributes:
73+
serviceAccountTokenAudience: test-audience
74+
requireServiceAccount: true
75+
requiredServiceAccountAnnotationKeys:
76+
- "domain.io/identity-id"
77+
- "domain.io/identity-type"
78+
env:
79+
- name: PLUGIN_MODE
80+
value: "serviceaccount"`
6481

6582
const credentialAWSProviderConfig = `kind: CredentialProviderConfig
6683
apiVersion: kubelet.config.k8s.io/v1

0 commit comments

Comments
 (0)