From adc482ec8379cec3e48149756b53dd871ec0a12a Mon Sep 17 00:00:00 2001 From: Steffen Baarsgaard Date: Sun, 3 Aug 2025 00:35:08 +0200 Subject: [PATCH 1/4] feat(Grafana): useKubeAuth enables using k8s serviceacccount --- api/v1beta1/grafana_types.go | 4 ++++ .../bases/grafana.integreatly.org_grafanas.yaml | 5 +++++ controllers/client/grafana_client.go | 14 +++++++++++++- .../crds/grafana.integreatly.org_grafanas.yaml | 5 +++++ deploy/kustomize/base/crds.yaml | 5 +++++ docs/docs/api.md | 8 ++++++++ 6 files changed, 40 insertions(+), 1 deletion(-) diff --git a/api/v1beta1/grafana_types.go b/api/v1beta1/grafana_types.go index f11e1aa68..37d2d45ff 100644 --- a/api/v1beta1/grafana_types.go +++ b/api/v1beta1/grafana_types.go @@ -129,6 +129,10 @@ type JsonnetConfig struct { // GrafanaClient contains the Grafana API client settings type GrafanaClient struct { + // Use Kubernetes Serviceaccount as authentication + // Requires configuring [auth.jwt] in the instance + // +optional + UseKubeAuth bool `json:"useKubeAuth,omitempty"` // +nullable TimeoutSeconds *int `json:"timeout,omitempty"` // +nullable diff --git a/config/crd/bases/grafana.integreatly.org_grafanas.yaml b/config/crd/bases/grafana.integreatly.org_grafanas.yaml index 77878f2b3..9ed00e28f 100644 --- a/config/crd/bases/grafana.integreatly.org_grafanas.yaml +++ b/config/crd/bases/grafana.integreatly.org_grafanas.yaml @@ -90,6 +90,11 @@ spec: x-kubernetes-validations: - message: insecureSkipVerify and certSecretRef cannot be set at the same time rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean type: object config: additionalProperties: diff --git a/controllers/client/grafana_client.go b/controllers/client/grafana_client.go index 7cd739091..26430704a 100644 --- a/controllers/client/grafana_client.go +++ b/controllers/client/grafana_client.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/url" + "os" "time" httptransport "github.com/go-openapi/runtime/client" @@ -63,6 +64,17 @@ func getExternalAdminPassword(ctx context.Context, c client.Client, cr *v1beta1. func getAdminCredentials(ctx context.Context, c client.Client, grafana *v1beta1.Grafana) (*grafanaAdminCredentials, error) { credentials := &grafanaAdminCredentials{} + if grafana.Spec.Client != nil && grafana.Spec.Client.UseKubeAuth { + b, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + if err != nil { + return nil, err + } + + credentials.apikey = string(b) + + return credentials, nil + } + if grafana.IsExternal() { // prefer api key if present if grafana.Spec.External.APIKey != nil { @@ -152,7 +164,7 @@ func InjectAuthHeaders(ctx context.Context, c client.Client, grafana *v1beta1.Gr } if creds.apikey != "" { - req.Header.Add("Authorization", "Bearer "+creds.apikey) + req.Header.Set("Authorization", "Bearer "+creds.apikey) } else { req.SetBasicAuth(creds.adminUser, creds.adminPassword) } diff --git a/deploy/helm/grafana-operator/crds/grafana.integreatly.org_grafanas.yaml b/deploy/helm/grafana-operator/crds/grafana.integreatly.org_grafanas.yaml index 77878f2b3..9ed00e28f 100644 --- a/deploy/helm/grafana-operator/crds/grafana.integreatly.org_grafanas.yaml +++ b/deploy/helm/grafana-operator/crds/grafana.integreatly.org_grafanas.yaml @@ -90,6 +90,11 @@ spec: x-kubernetes-validations: - message: insecureSkipVerify and certSecretRef cannot be set at the same time rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean type: object config: additionalProperties: diff --git a/deploy/kustomize/base/crds.yaml b/deploy/kustomize/base/crds.yaml index fe252f43e..818815830 100644 --- a/deploy/kustomize/base/crds.yaml +++ b/deploy/kustomize/base/crds.yaml @@ -3407,6 +3407,11 @@ spec: at the same time rule: (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))) + useKubeAuth: + description: |- + Use Kubernetes Serviceaccount as authentication + Requires configuring [auth.jwt] in the instance + type: boolean type: object config: additionalProperties: diff --git a/docs/docs/api.md b/docs/docs/api.md index 3e57f9adc..669a06398 100644 --- a/docs/docs/api.md +++ b/docs/docs/api.md @@ -6444,6 +6444,14 @@ Client defines how the grafana-operator talks to the grafana instance. Validations:
  • (has(self.insecureSkipVerify) && !(has(self.certSecretRef))) || (has(self.certSecretRef) && !(has(self.insecureSkipVerify))): insecureSkipVerify and certSecretRef cannot be set at the same time
    • false + + useKubeAuth + boolean + + Use Kubernetes Serviceaccount as authentication +Requires configuring [auth.jwt] in the instance
    + + false From 6976dedc3bf9c37f3811e6f37317ee5ab0e0ef4a Mon Sep 17 00:00:00 2001 From: Steffen Baarsgaard Date: Thu, 7 Aug 2025 21:28:39 +0200 Subject: [PATCH 2/4] feat: Cache token by decoding it and using 'exp' claim --- controllers/client/grafana_client.go | 64 +++++++++++++++++++++++++++- go.mod | 1 + go.sum | 2 + 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/controllers/client/grafana_client.go b/controllers/client/grafana_client.go index 26430704a..04a691e23 100644 --- a/controllers/client/grafana_client.go +++ b/controllers/client/grafana_client.go @@ -8,6 +8,8 @@ import ( "os" "time" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" httptransport "github.com/go-openapi/runtime/client" genapi "github.com/grafana/grafana-openapi-client-go/client" "github.com/grafana/grafana-operator/v5/api/v1beta1" @@ -18,12 +20,70 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +const ( + serviceAccountTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" // nolint:gosec +) + type grafanaAdminCredentials struct { adminUser string adminPassword string apikey string } +type JWTCache struct { + Token string + Expiration time.Time +} + +var jwtCache *JWTCache + +// getBearerToken will read JWT token from given file and cache it until it expires. +// accepts filepath arg for testing +func getBearerToken(bearerTokenPath string) (string, error) { + // Return cached token if not expired + if jwtCache != nil && jwtCache.Expiration.After(time.Now()) { + return jwtCache.Token, nil + } + + b, err := os.ReadFile(bearerTokenPath) + if err != nil { + return "", fmt.Errorf("reading token file at %s, %w", bearerTokenPath, err) + } + + token := string(b) + + // List of accepted JWT signing algorithms from: https://kubernetes.io/docs/reference/access-authn-authz/authentication/#:~:text=oidc-signing-algs + t, err := jwt.ParseSigned(token, []jose.SignatureAlgorithm{ + jose.RS256, jose.RS384, jose.RS512, + jose.ES256, jose.ES384, jose.ES512, + jose.PS256, jose.PS384, jose.PS512, + }) + if err != nil { + return "", err + } + + claims := jwt.Claims{} + + // TODO fetch JWKS from https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}/openid/v1/jwks + // Then verify token using the keys + err = t.UnsafeClaimsWithoutVerification(&claims) + if err != nil { + return "", fmt.Errorf("decoding ServiceAccount token %w", err) + } + + tokenExpiration := claims.Expiry.Time() + if tokenExpiration.Before(time.Now()) { + return "", fmt.Errorf("token expired at %s, expected %s to be rotated", tokenExpiration.String(), bearerTokenPath) + } + + jwtCache = &JWTCache{ + Token: token, + Expiration: tokenExpiration, + } + + return token, nil +} + func getExternalAdminUser(ctx context.Context, c client.Client, cr *v1beta1.Grafana) (string, error) { if cr.Spec.External != nil && cr.Spec.External.AdminUser != nil { adminUser, err := GetValueFromSecretKey(ctx, cr.Spec.External.AdminUser, c, cr.Namespace) @@ -65,12 +125,12 @@ func getAdminCredentials(ctx context.Context, c client.Client, grafana *v1beta1. credentials := &grafanaAdminCredentials{} if grafana.Spec.Client != nil && grafana.Spec.Client.UseKubeAuth { - b, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token") + t, err := getBearerToken(serviceAccountTokenPath) if err != nil { return nil, err } - credentials.apikey = string(b) + credentials.apikey = t return credentials, nil } diff --git a/go.mod b/go.mod index 187583fd2..a686eaa7a 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.23.0 // indirect diff --git a/go.sum b/go.sum index 3ec406afa..8a30b4253 100644 --- a/go.sum +++ b/go.sum @@ -60,6 +60,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= From 36f8a80dbf1b7ce4c6efdbb6032396c710f4780d Mon Sep 17 00:00:00 2001 From: Steffen Baarsgaard Date: Sat, 9 Aug 2025 11:37:33 +0200 Subject: [PATCH 3/4] test: getBearerToken --- controllers/client/grafana_client_test.go | 91 +++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/controllers/client/grafana_client_test.go b/controllers/client/grafana_client_test.go index f95ab4a4e..c5d95b5ad 100644 --- a/controllers/client/grafana_client_test.go +++ b/controllers/client/grafana_client_test.go @@ -2,8 +2,15 @@ package client import ( "context" + "crypto/rand" + "crypto/rsa" + "os" + "slices" "testing" + "time" + "github.com/go-jose/go-jose/v4" + "github.com/go-jose/go-jose/v4/jwt" "github.com/grafana/grafana-operator/v5/api/v1beta1" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -312,3 +319,87 @@ func TestGetAdminCredentials(t *testing.T) { assert.Nil(t, got) }) } + +func TestGetBearerToken(t *testing.T) { + t.Parallel() + + now := time.Now() + body := jwt.Claims{ + ID: "1234", + Issuer: "grafana.operator.com", + Audience: jwt.Audience{"https://grafana.operator.com"}, + IssuedAt: jwt.NewNumericDate(now), + NotBefore: jwt.NewNumericDate(now), + Expiry: jwt.NewNumericDate(now.Add(time.Duration(30 * float64(time.Second)))), + } + + // Generate key + testRSAKey, err := rsa.GenerateKey(rand.Reader, 1024) // nolint:gosec + require.NotNil(t, testRSAKey) + require.NoError(t, err) + + sk := jose.SigningKey{ + Key: &jose.JSONWebKey{ + Key: testRSAKey, + KeyID: "test-key", + }, + Algorithm: jose.RS256, + } + + // Create signer, and sign token + signer, err := jose.NewSigner(sk, &jose.SignerOptions{EmbedJWK: true}) + require.NoError(t, err) + + origToken, err := jwt.Signed(signer).Claims(body).Serialize() + require.NoError(t, err) + + // Create tmp file + tokenFile, err := os.CreateTemp(os.TempDir(), "token-*") + require.NoError(t, err) + require.NotNil(t, testRSAKey) + + // Do not read token file purposefully + jwtCache = nil + noToken, err := getBearerToken(tokenFile.Name() + "-extra") + require.Error(t, err) + require.Empty(t, noToken) + require.Nil(t, jwtCache) + + // Write token to file + writtenBytes, err := tokenFile.WriteString(origToken) + require.Equal(t, len([]byte(origToken)), writtenBytes) + require.NoError(t, err) + + // Decode token + token, err := getBearerToken(tokenFile.Name()) + require.NoError(t, err) + require.Equal(t, origToken, token) + require.Equal(t, origToken, jwtCache.Token) + require.False(t, jwtCache.Expiration.IsZero()) + + // Expire the cache and re-read token + jwtCache.Expiration = time.Now().Add(-10 * time.Second) + token, err = getBearerToken(tokenFile.Name()) + require.NoError(t, err) + require.Equal(t, origToken, token) + require.True(t, jwtCache.Expiration.After(time.Now())) + + // Mangle token + reversedOrigToken := []byte(origToken) + slices.Reverse(reversedOrigToken) + reversedWrittenBytes, err := tokenFile.Write(reversedOrigToken) + require.NoError(t, err) + require.Equal(t, len([]byte(reversedOrigToken)), reversedWrittenBytes) + + // Successfully get token from cache + token, err = getBearerToken(tokenFile.Name()) + require.NoError(t, err) + require.Equal(t, origToken, token) + + // Reset cache and error on mangled token + jwtCache = nil + mangledToken, err := getBearerToken(tokenFile.Name()) + require.Error(t, err) + require.Empty(t, mangledToken) + require.Nil(t, jwtCache) +} From 9f29f8fcf15ea54709f80ff857615a9f8be3d101 Mon Sep 17 00:00:00 2001 From: Steffen Baarsgaard Date: Sat, 30 Aug 2025 15:40:37 +0200 Subject: [PATCH 4/4] chore: Expire token 30 seconds early to mitigate mid-reconcile expirations --- controllers/client/grafana_client.go | 10 +++++++--- controllers/client/grafana_client_test.go | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/controllers/client/grafana_client.go b/controllers/client/grafana_client.go index 04a691e23..a8d4e07c2 100644 --- a/controllers/client/grafana_client.go +++ b/controllers/client/grafana_client.go @@ -37,6 +37,10 @@ type JWTCache struct { var jwtCache *JWTCache +// Revoke tokens early expecting them to be rotated hourly, see 'ExpirationSeconds' in KEP1205 +// Should mitigate mid-reconcile expiration +const tokenExpirationCompensation = -30 * time.Second + // getBearerToken will read JWT token from given file and cache it until it expires. // accepts filepath arg for testing func getBearerToken(bearerTokenPath string) (string, error) { @@ -72,13 +76,13 @@ func getBearerToken(bearerTokenPath string) (string, error) { } tokenExpiration := claims.Expiry.Time() - if tokenExpiration.Before(time.Now()) { - return "", fmt.Errorf("token expired at %s, expected %s to be rotated", tokenExpiration.String(), bearerTokenPath) + if tokenExpiration.Add(tokenExpirationCompensation).Before(time.Now()) { + return "", fmt.Errorf("token expired at %s, expected %s to be renewed. Tokens are considered expired 30 seconds early", tokenExpiration.String(), bearerTokenPath) } jwtCache = &JWTCache{ Token: token, - Expiration: tokenExpiration, + Expiration: tokenExpiration.Add(tokenExpirationCompensation), } return token, nil diff --git a/controllers/client/grafana_client_test.go b/controllers/client/grafana_client_test.go index c5d95b5ad..a627381c3 100644 --- a/controllers/client/grafana_client_test.go +++ b/controllers/client/grafana_client_test.go @@ -330,7 +330,7 @@ func TestGetBearerToken(t *testing.T) { Audience: jwt.Audience{"https://grafana.operator.com"}, IssuedAt: jwt.NewNumericDate(now), NotBefore: jwt.NewNumericDate(now), - Expiry: jwt.NewNumericDate(now.Add(time.Duration(30 * float64(time.Second)))), + Expiry: jwt.NewNumericDate(now.Add(time.Duration(60 * float64(time.Second)))), } // Generate key