Skip to content

Commit 48259e0

Browse files
nettoclaudionettoclaudio
authored
feat: support for dynamically fetching credentials from external command (#149)
* feat: support issuing cluster credential from external command Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * refactor: rename the dynamic credential type Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * refactor: covering dynamic credentials w/ unit tests Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * docs: adding an example of secret using Dynamic cluster credential Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * test: covering code branch of failed to build cluster credentials Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * refactor: return ClusterGateway w/ credential type Dynamic Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * feat: store/fetch credential from in-memory cache Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * test: fixing exec plugin tests Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * test: increasing test coverage of exec package Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * test: cover transport package with tests about dynamic credential Signed-off-by: Claudio Netto <nettinhorama@gmail.com> * test: covering w/ test an error-catching Signed-off-by: Claudio Netto <nettinhorama@gmail.com> --------- Signed-off-by: Claudio Netto <nettinhorama@gmail.com>
1 parent adc35a6 commit 48259e0

File tree

8 files changed

+789
-12
lines changed

8 files changed

+789
-12
lines changed

‎docs/local-run.md‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,21 @@ data:
167167
token: "..." # working jwt token
168168
```
169169
170+
2.3. (Alternatively) Create a secret containing an exec config to dynamically fetch the cluster credential from an external command:
171+
172+
```yaml
173+
apiVersion: v1
174+
kind: Secret
175+
metadata:
176+
name: managed1
177+
labels:
178+
cluster.core.oam.dev/cluster-credential-type: Dynamic
179+
type: Opaque # <--- Has to be opaque
180+
data:
181+
endpoint: "..." # ditto
182+
exec: "..." # an exec config in JSON format; see ExecConfig (https://github.com/kubernetes/kubernetes/blob/2016fab3085562b4132e6d3774b6ded5ba9939fd/staging/src/k8s.io/client-go/tools/clientcmd/api/types.go#L206, https://kubernetes.io/docs/reference/access-authn-authz/authentication/#configuration)
183+
```
184+
170185
3. Proxy to cluster `managed1`'s `/healthz` endpoint
171186

172187
```shell
@@ -195,4 +210,4 @@ KUBECONFIG=/tmp/hub-managed1.kubeconfig kubectl get ns
195210

196211
```shell
197212
$ kind delete cluster --name tmp
198-
```
213+
```

‎pkg/apis/cluster/v1alpha1/clustergateway_types.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,10 @@ const (
7979
// CredentialTypeX509Certificate means the cluster is accessible via
8080
// X509 certificate and key.
8181
CredentialTypeX509Certificate CredentialType = "X509Certificate"
82+
// CredentialTypeDynamic means that a credential will be issued before
83+
// accessing the cluster. The generated credential can be either a service
84+
// account token or X509 certificate and key.
85+
CredentialTypeDynamic CredentialType = "Dynamic"
8286
)
8387

8488
type ClusterEndpointType string

‎pkg/apis/cluster/v1alpha1/clustergateway_types_secret.go‎

Lines changed: 54 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,21 @@ package v1alpha1
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
67
"strconv"
78
"strings"
89

910
"k8s.io/apimachinery/pkg/util/yaml"
11+
"k8s.io/apiserver/pkg/registry/rest"
12+
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
1013
"k8s.io/utils/pointer"
1114

1215
"github.com/oam-dev/cluster-gateway/pkg/common"
1316
"github.com/oam-dev/cluster-gateway/pkg/config"
1417
"github.com/oam-dev/cluster-gateway/pkg/featuregates"
1518
"github.com/oam-dev/cluster-gateway/pkg/options"
19+
"github.com/oam-dev/cluster-gateway/pkg/util/exec"
1620
"github.com/oam-dev/cluster-gateway/pkg/util/singleton"
1721

1822
"github.com/pkg/errors"
@@ -22,7 +26,6 @@ import (
2226
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2327
"k8s.io/apimachinery/pkg/runtime"
2428
"k8s.io/apimachinery/pkg/runtime/schema"
25-
"k8s.io/apiserver/pkg/registry/rest"
2629
utilfeature "k8s.io/apiserver/pkg/util/feature"
2730
"k8s.io/klog/v2"
2831
clusterv1 "open-cluster-management.io/api/cluster/v1"
@@ -176,11 +179,11 @@ func getEndpointFromSecret(secret *v1.Secret) ([]byte, string, error) {
176179
func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.Secret) (*ClusterGateway, error) {
177180
c := &ClusterGateway{
178181
ObjectMeta: metav1.ObjectMeta{
179-
Name: secret.Name,
182+
Name: secret.Name,
183+
CreationTimestamp: secret.CreationTimestamp,
180184
},
181185
Spec: ClusterGatewaySpec{
182-
Provider: "",
183-
Access: ClusterAccess{},
186+
Access: ClusterAccess{},
184187
},
185188
}
186189

@@ -242,11 +245,21 @@ func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.
242245
PrivateKey: secret.Data[v1.TLSPrivateKeyKey],
243246
},
244247
}
248+
245249
case CredentialTypeServiceAccountToken:
246250
c.Spec.Access.Credential = &ClusterAccessCredential{
247251
Type: CredentialTypeServiceAccountToken,
248252
ServiceAccountToken: string(secret.Data[v1.ServiceAccountTokenKey]),
249253
}
254+
255+
case CredentialTypeDynamic:
256+
credential, err := buildCredentialFromExecConfig(secret)
257+
if err != nil {
258+
return nil, fmt.Errorf("failed to issue credential from external command: %s", err)
259+
}
260+
261+
c.Spec.Access.Credential = credential
262+
250263
default:
251264
return nil, fmt.Errorf("unrecognized secret credential type %v", credentialType)
252265
}
@@ -278,3 +291,40 @@ func convert(caData []byte, apiServerEndpoint string, insecure bool, secret *v1.
278291

279292
return c, nil
280293
}
294+
295+
func buildCredentialFromExecConfig(secret *v1.Secret) (*ClusterAccessCredential, error) {
296+
execConfigRaw := secret.Data["exec"]
297+
if len(execConfigRaw) == 0 {
298+
return nil, errors.New("missing secret data key: exec")
299+
}
300+
301+
var ec clientcmdapi.ExecConfig
302+
if err := json.Unmarshal(execConfigRaw, &ec); err != nil {
303+
return nil, fmt.Errorf("failed to decode exec config JSON from secret data: %v", err)
304+
}
305+
306+
cred, err := exec.IssueClusterCredential(secret.Name, &ec)
307+
if err != nil {
308+
return nil, err
309+
}
310+
311+
if token := cred.Status.Token; len(token) > 0 {
312+
return &ClusterAccessCredential{
313+
Type: CredentialTypeDynamic,
314+
ServiceAccountToken: token,
315+
}, nil
316+
}
317+
318+
if cert, key := cred.Status.ClientCertificateData, cred.Status.ClientKeyData; len(cert) > 0 && len(key) > 0 {
319+
return &ClusterAccessCredential{
320+
Type: CredentialTypeDynamic,
321+
X509: &X509{
322+
Certificate: []byte(cert),
323+
PrivateKey: []byte(key),
324+
},
325+
}, nil
326+
327+
}
328+
329+
return nil, fmt.Errorf("no credential type available")
330+
}

‎pkg/apis/cluster/v1alpha1/clustergateway_types_secret_test.go‎

Lines changed: 194 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,29 @@ import (
2727
)
2828

2929
var (
30-
testNamespace = "foo"
31-
testName = "bar"
32-
testCAData = "caData"
33-
testCertData = "certData"
34-
testKeyData = "keyData"
35-
testToken = "token"
36-
testEndpoint = "https://localhost:443"
30+
testNamespace = "foo"
31+
testName = "bar"
32+
testCAData = "caData"
33+
testCertData = "certData"
34+
testKeyData = "keyData"
35+
testToken = "token"
36+
testEndpoint = "https://localhost:443"
37+
testExecConfigForToken = `{
38+
"apiVersion": "client.authentication.k8s.io/v1beta1",
39+
"kind": "ExecConfig",
40+
"command": "echo",
41+
"args": [
42+
"{\"apiVersion\": \"client.authentication.k8s.io/v1beta1\", \"kind\": \"ExecCredential\", \"status\": {\"token\": \"token\"}}"
43+
]
44+
}`
45+
testExecConfigForX509 = `{
46+
"apiVersion": "client.authentication.k8s.io/v1beta1",
47+
"kind": "ExecConfig",
48+
"command": "echo",
49+
"args": [
50+
"{\"apiVersion\": \"client.authentication.k8s.io/v1beta1\", \"kind\": \"ExecCredential\", \"status\": {\"clientCertificateData\": \"certData\", \"clientKeyData\": \"keyData\"}}"
51+
]
52+
}`
3753
)
3854

3955
func TestConvertSecretToGateway(t *testing.T) {
@@ -260,6 +276,101 @@ func TestConvertSecretToGateway(t *testing.T) {
260276
},
261277
},
262278
},
279+
{
280+
name: "dynamic service account token issued from external command",
281+
inputSecret: &corev1.Secret{
282+
ObjectMeta: metav1.ObjectMeta{
283+
Name: testName,
284+
Namespace: testNamespace,
285+
Labels: map[string]string{
286+
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
287+
},
288+
},
289+
Data: map[string][]byte{
290+
"endpoint": []byte(testEndpoint),
291+
"ca.crt": []byte(testCAData),
292+
"exec": []byte(testExecConfigForToken),
293+
},
294+
},
295+
expected: &ClusterGateway{
296+
ObjectMeta: metav1.ObjectMeta{
297+
Name: testName,
298+
},
299+
Spec: ClusterGatewaySpec{
300+
Access: ClusterAccess{
301+
Credential: &ClusterAccessCredential{
302+
Type: CredentialTypeDynamic,
303+
ServiceAccountToken: testToken,
304+
},
305+
Endpoint: &ClusterEndpoint{
306+
Type: ClusterEndpointTypeConst,
307+
Const: &ClusterEndpointConst{
308+
CABundle: []byte(testCAData),
309+
Address: testEndpoint,
310+
},
311+
},
312+
},
313+
},
314+
},
315+
},
316+
{
317+
name: "dynamic x509 cert-key pair issued from external command",
318+
inputSecret: &corev1.Secret{
319+
ObjectMeta: metav1.ObjectMeta{
320+
Name: testName,
321+
Namespace: testNamespace,
322+
Labels: map[string]string{
323+
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
324+
},
325+
},
326+
Data: map[string][]byte{
327+
"endpoint": []byte(testEndpoint),
328+
"ca.crt": []byte(testCAData),
329+
"exec": []byte(testExecConfigForX509),
330+
},
331+
},
332+
expected: &ClusterGateway{
333+
ObjectMeta: metav1.ObjectMeta{
334+
Name: testName,
335+
},
336+
Spec: ClusterGatewaySpec{
337+
Access: ClusterAccess{
338+
Credential: &ClusterAccessCredential{
339+
Type: CredentialTypeDynamic,
340+
X509: &X509{
341+
Certificate: []byte(testCertData),
342+
PrivateKey: []byte(testKeyData),
343+
},
344+
},
345+
Endpoint: &ClusterEndpoint{
346+
Type: ClusterEndpointTypeConst,
347+
Const: &ClusterEndpointConst{
348+
CABundle: []byte(testCAData),
349+
Address: testEndpoint,
350+
},
351+
},
352+
},
353+
},
354+
},
355+
},
356+
{
357+
name: "failed to fetch cluster credential from dynamic auth mode",
358+
inputSecret: &corev1.Secret{
359+
ObjectMeta: metav1.ObjectMeta{
360+
Name: testName,
361+
Namespace: testNamespace,
362+
Labels: map[string]string{
363+
common.LabelKeyClusterCredentialType: string(CredentialTypeDynamic),
364+
},
365+
},
366+
Data: map[string][]byte{
367+
"endpoint": []byte(testEndpoint),
368+
"ca.crt": []byte(testCAData),
369+
"exec": []byte("invalid exec config format"),
370+
},
371+
},
372+
expectedFailure: true,
373+
},
263374
}
264375
for _, c := range cases {
265376
t.Run(c.name, func(t *testing.T) {
@@ -524,3 +635,79 @@ func TestListHybridClusterGateway(t *testing.T) {
524635
}
525636
assert.Equal(t, expectedNames, actualNames)
526637
}
638+
639+
func TestBuildCredentialFromExecConfig(t *testing.T) {
640+
cases := []struct {
641+
name string
642+
secret func(s *corev1.Secret) *corev1.Secret
643+
cluster func(ce *ClusterEndpoint) *ClusterEndpoint
644+
expectedError string
645+
expected *ClusterAccessCredential
646+
}{
647+
{
648+
name: "missing exec config",
649+
expectedError: "missing secret data key: exec",
650+
},
651+
652+
{
653+
name: "invalid exec config format",
654+
secret: func(s *corev1.Secret) *corev1.Secret {
655+
s.Data["exec"] = []byte("some invalid exec config")
656+
return s
657+
},
658+
expectedError: "failed to decode exec config JSON from secret data: invalid character 's' looking for beginning of value",
659+
},
660+
661+
{
662+
name: "returns successfully a service account token",
663+
secret: func(s *corev1.Secret) *corev1.Secret {
664+
s.Data["exec"] = []byte(`{"apiVersion": "client.authentication.k8s.io/v1", "command": "echo", "args": ["{\"apiVersion\": \"client.authentication.k8s.io/v1\", \"status\": {\"token\": \"token\"}}"]}`)
665+
return s
666+
},
667+
expected: &ClusterAccessCredential{
668+
Type: CredentialTypeDynamic,
669+
ServiceAccountToken: testToken,
670+
},
671+
},
672+
673+
{
674+
name: "returns successfully a X509 client certificate",
675+
secret: func(s *corev1.Secret) *corev1.Secret {
676+
s.Data["exec"] = []byte(`{"apiVersion": "client.authentication.k8s.io/v1", "command": "echo", "args": ["{\"apiVersion\": \"client.authentication.k8s.io/v1\", \"status\": {\"clientCertificateData\": \"certData\", \"clientKeyData\": \"keyData\"}}"]}`)
677+
return s
678+
},
679+
expected: &ClusterAccessCredential{
680+
Type: CredentialTypeDynamic,
681+
X509: &X509{
682+
Certificate: []byte(testCertData),
683+
PrivateKey: []byte(testKeyData),
684+
},
685+
},
686+
},
687+
}
688+
689+
for _, tt := range cases {
690+
t.Run(tt.name, func(t *testing.T) {
691+
secret := &corev1.Secret{
692+
ObjectMeta: metav1.ObjectMeta{
693+
Name: testName,
694+
Namespace: testNamespace,
695+
},
696+
Data: map[string][]byte{},
697+
}
698+
if tt.secret != nil {
699+
secret = tt.secret(secret)
700+
}
701+
702+
got, err := buildCredentialFromExecConfig(secret)
703+
if tt.expectedError != "" {
704+
assert.Error(t, err)
705+
assert.EqualError(t, err, tt.expectedError)
706+
return
707+
}
708+
709+
assert.NoError(t, err)
710+
assert.Equal(t, tt.expected, got)
711+
})
712+
}
713+
}

‎pkg/apis/cluster/v1alpha1/transport.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,8 +94,19 @@ func NewConfigFromCluster(ctx context.Context, c *ClusterGateway) (*restclient.C
9494
}
9595
// setting up credentials
9696
switch c.Spec.Access.Credential.Type {
97+
case CredentialTypeDynamic:
98+
if token := c.Spec.Access.Credential.ServiceAccountToken; token != "" {
99+
cfg.BearerToken = token
100+
}
101+
102+
if c.Spec.Access.Credential.X509 != nil && len(c.Spec.Access.Credential.X509.Certificate) > 0 && len(c.Spec.Access.Credential.X509.PrivateKey) > 0 {
103+
cfg.CertData = c.Spec.Access.Credential.X509.Certificate
104+
cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey
105+
}
106+
97107
case CredentialTypeServiceAccountToken:
98108
cfg.BearerToken = c.Spec.Access.Credential.ServiceAccountToken
109+
99110
case CredentialTypeX509Certificate:
100111
cfg.CertData = c.Spec.Access.Credential.X509.Certificate
101112
cfg.KeyData = c.Spec.Access.Credential.X509.PrivateKey

0 commit comments

Comments
 (0)