Use token authentication for the compliance history API

The addon framework's hub kubeconfig is certificate based. This does not
allow authentication to the compliance history API using an OpenShift
route that handles the TLS. Because of this, token based authentication
is required.

This uses the hub kubeconfig's token first and if not set, falls back to
getting the token from a secret in the cluster namespace on the Hub
created by the policy addon controller.

Relates:
https://issues.redhat.com/browse/ACM-6889

Signed-off-by: mprahl <[email protected]>

Author: mprahl

controllers/statussync/policy_status_sync.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
@@ -44,7 +45,10 @@ import (
 	"open-cluster-management.io/governance-policy-framework-addon/controllers/utils"
 )
 
-const ControllerName string = "policy-status-sync"
+const (
+	ControllerName         string = "policy-status-sync"
+	hubComplianceAPISAName string = "open-cluster-management-compliance-history-api-recorder"
+)
 
 var (
 	clusterClaimGVR = schema.GroupVersionResource{
@@ -93,6 +97,7 @@ type PolicyReconciler struct {
 //+kubebuilder:rbac:groups=policy.open-cluster-management.io,resources=policies/finalizers,verbs=update
 //+kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=clusterclaims,resourceNames=id.k8s.io,verbs=get
 //+kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
+//+kubebuilder:rbac:groups=core,resources=secrets,verbs=get,resourceNames="open-cluster-management-compliance-history-api-recorder"
 // This is required for the status lease for the addon framework
 //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list
 
@@ -552,6 +557,12 @@ func StartComplianceEventsSyncer(
 	apiURL string,
 	events workqueue.RateLimitingInterface,
 ) error {
+	var hubToken string
+
+	if hubCfg.BearerToken != "" {
+		hubToken = hubCfg.BearerToken
+	}
+
 	managedClient, err := dynamic.NewForConfig(managedCfg)
 	if err != nil {
 		return err
@@ -583,42 +594,26 @@ func StartComplianceEventsSyncer(
 		caCertPool = x509.NewCertPool()
 	}
 
-	// Append the Kubernete API Server CA in case the Service is exposed directly as opposed to using something like the
-	// OpenShift router.
+	// Append the Kubernete API Server CAs in case the Hub cluster's ingress CA is not trusted by the system pool.
 	if hubCfg.CAData != nil {
-		caCertPool.AppendCertsFromPEM(hubCfg.CAData)
-	}
-
-	httpClient := &http.Client{Timeout: 60 * time.Second}
-
-	var usesCertAuth bool
-
-	if hubCfg.CertData != nil && hubCfg.KeyData != nil {
-		log.Info("Using certificate authentication with the compliance API server")
-
-		cert, err := tls.X509KeyPair(hubCfg.CertData, hubCfg.KeyData)
-		if err != nil {
-			log.Error(err, "Failed to load the hub kubeconfig for certificate authentication on the compliance API")
-
-			return err
+		_ = caCertPool.AppendCertsFromPEM(hubCfg.CAData)
+	} else if hubCfg.CAFile != "" {
+		caData, err := os.ReadFile(hubCfg.CAFile)
+		if err == nil {
+			log.Info("The hub kubeconfig CA file can't be read. Ignoring it.", "path", hubCfg.CAFile)
 		}
 
-		httpClient.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{
-				MinVersion:   tls.VersionTLS12,
-				Certificates: []tls.Certificate{cert},
-				RootCAs:      caCertPool,
-			},
-		}
+		_ = caCertPool.AppendCertsFromPEM(caData)
+	}
 
-		usesCertAuth = true
-	} else {
-		httpClient.Transport = &http.Transport{
+	httpClient := &http.Client{
+		Timeout: 60 * time.Second,
+		Transport: &http.Transport{
 			TLSClientConfig: &tls.Config{
 				MinVersion: tls.VersionTLS12,
 				RootCAs:    caCertPool,
 			},
-		}
+		},
 	}
 
 	for {
@@ -671,10 +666,32 @@ func StartComplianceEventsSyncer(
 
 		httpRequest.Header.Set("Content-Type", "application/json")
 
-		if !usesCertAuth && hubCfg.BearerToken != "" {
-			httpRequest.Header.Set("Authorization", "Bearer "+hubCfg.BearerToken)
+		if hubToken == "" {
+			var err error
+
+			hubToken, err = getHubComplianceAPIToken(ctx, hubCfg, clusterName)
+			if err != nil || hubToken == "" {
+				var msg string
+
+				if err != nil {
+					msg = err.Error()
+				} else {
+					msg = "the token was not set on the secret"
+				}
+
+				log.Info(
+					"Failed to get the compliance API hub token. Will requeue in 10 seconds.", "error", msg,
+				)
+
+				events.AddAfter(ceUntyped, 10*time.Second)
+				events.Done(ceUntyped)
+
+				continue
+			}
 		}
 
+		httpRequest.Header.Set("Authorization", "Bearer "+hubToken)
+
 		httpResponse, err := httpClient.Do(httpRequest)
 		if err != nil {
 			log.Info("Failed to record the compliance event with the compliance API. Will requeue in 10 seconds.")
@@ -705,6 +722,11 @@ func StartComplianceEventsSyncer(
 				"message", message,
 			)
 
+			if httpResponse.StatusCode == http.StatusUnauthorized || httpResponse.StatusCode == http.StatusForbidden {
+				// Wipe out the hubToken so that the token is fetched again on the next try.
+				hubToken = ""
+			}
+
 			events.AddRateLimited(ceUntyped)
 			events.Done(ceUntyped)
 
@@ -721,6 +743,24 @@ func StartComplianceEventsSyncer(
 	}
 }
 
+// getHubComplianceAPIToken retrieves the token associated with the service account with compliance history API
+// recording permssions in the cluster namespace.
+func getHubComplianceAPIToken(ctx context.Context, hubCfg *rest.Config, clusterNamespace string) (string, error) {
+	client, err := kubernetes.NewForConfig(hubCfg)
+	if err != nil {
+		return "", err
+	}
+
+	saTokenSecret, err := client.CoreV1().Secrets(clusterNamespace).Get(
+		ctx, hubComplianceAPISAName, metav1.GetOptions{},
+	)
+	if err != nil {
+		return "", err
+	}
+
+	return string(saTokenSecret.Data["token"]), nil
+}
+
 type historyEvent struct {
 	policiesv1.ComplianceHistory
 	eventTime metav1.MicroTime

deploy/operator.yaml

@@ -110,6 +110,14 @@ rules:
   - secrets
   verbs:
   - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - open-cluster-management-compliance-history-api-recorder
+  resources:
+  - secrets
+  verbs:
+  - get
 - apiGroups:
   - ""
   resourceNames:

deploy/rbac/role.yaml

@@ -80,6 +80,14 @@ rules:
   - secrets
   verbs:
   - create
+- apiGroups:
+  - ""
+  resourceNames:
+  - open-cluster-management-compliance-history-api-recorder
+  resources:
+  - secrets
+  verbs:
+  - get
 - apiGroups:
   - ""
   resourceNames:

test/e2e/case23_compliance_api_recording_test.go

@@ -4,7 +4,6 @@ package e2e
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"io"
 	"net/http"
@@ -15,8 +14,6 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/tools/clientcmd"
-	certutil "k8s.io/client-go/util/cert"
 	policiesv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
 	"open-cluster-management.io/governance-policy-propagator/controllers/complianceeventsapi"
 	"open-cluster-management.io/governance-policy-propagator/test/utils"
@@ -71,22 +68,15 @@ var _ = Describe("Compliance API recording", Ordered, Label("compliance-events-a
 	BeforeAll(func(ctx context.Context) {
 		mux := http.NewServeMux()
 
-		hubConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfigHub)
-		Expect(err).ToNot(HaveOccurred())
-
-		caCertPool, err := certutil.NewPoolFromBytes(hubConfig.CAData)
-		Expect(err).ToNot(HaveOccurred())
-
 		complianceAPIURL := os.Getenv("COMPLIANCE_API_URL")
 		Expect(complianceAPIURL).ToNot(BeEmpty())
 
 		parsedURL, err := url.Parse(complianceAPIURL)
 		Expect(err).ToNot(HaveOccurred())
 
 		server = &http.Server{
-			Addr:      parsedURL.Host,
-			Handler:   mux,
-			TLSConfig: &tls.Config{ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: caCertPool},
+			Addr:    parsedURL.Host,
+			Handler: mux,
 		}
 
 		mux.HandleFunc("/api/v1/compliance-events", func(w http.ResponseWriter, r *http.Request) {
@@ -98,6 +88,13 @@ var _ = Describe("Compliance API recording", Ordered, Label("compliance-events-a
 				return
 			}
 
+			if r.Header.Get("Authorization") == "" {
+				w.WriteHeader(http.StatusUnauthorized)
+				_, _ = w.Write([]byte(`{"message": "No token sent"}`))
+
+				return
+			}
+
 			body, err := io.ReadAll(r.Body)
 			if err != nil {
 				log.Error(err, "error reading request body")