Optionally read cluster resources with dryrun CLI

pkg/dryrun/cmd.go

@@ -21,6 +21,7 @@ type DryRunner struct {
 	logPath       string
 	noColors      bool
 	fullDiffs     bool
+	fromCluster   bool
 }
 
 var ErrNonCompliant = errors.New("policy is NonCompliant")
@@ -105,6 +106,18 @@ func (d *DryRunner) GetCmd() *cobra.Command {
 			"the DRYRUN_MAPPINGS_FILE environment variable.",
 	)
 
+	fromCluster := os.Getenv("DRYRUN_FROM_CLUSTER") == "true" // false if not set
+
+	cmd.Flags().BoolVar(
+		&d.fromCluster,
+		"from-cluster",
+		fromCluster,
+		"Read the current state of resources from the currently configured Kubernetes cluster instead of "+
+			"from input files. Uses the default kubeconfig or KUBECONFIG environment variable. "+
+			"Any input files representing the cluster state are ignored. "+
+			"Can also be set via the DRYRUN_FROM_CLUSTER environment variable.",
+	)
+
 	cmd.AddCommand(&cobra.Command{
 		Use:   "generate",
 		Short: "Generate an API Mappings file",

pkg/dryrun/dryrun.go

@@ -30,8 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
 	dynfake "k8s.io/client-go/dynamic/fake"
+	"k8s.io/client-go/kubernetes"
 	clientsetfake "k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	klog "k8s.io/klog/v2"
 	parentpolicyv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
@@ -57,11 +59,6 @@ func (d *DryRunner) dryRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to read input policy: %w", err)
 	}
 
-	inputObjects, err := d.readInputResources(cmd, args)
-	if err != nil {
-		return fmt.Errorf("unable to read input resources: %w", err)
-	}
-
 	if err := d.setupLogs(); err != nil {
 		return fmt.Errorf("unable to setup the logging configuration: %w", err)
 	}
@@ -74,43 +71,15 @@ func (d *DryRunner) dryRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to setup the dryrun reconciler: %w", err)
 	}
 
-	// Apply the user's resources to the fake cluster
-	for _, obj := range inputObjects {
-		gvk := obj.GroupVersionKind()
-
-		scopedGVR, err := rec.DynamicWatcher.GVKToGVR(gvk)
+	if !d.fromCluster {
+		inputObjects, err := d.readInputResources(cmd, args)
 		if err != nil {
-			if errors.Is(err, depclient.ErrNoVersionedResource) {
-				return fmt.Errorf("%w for kind %v: if this is a custom resource, it may need an "+
-					"entry in the mappings file", err, gvk.Kind)
-			}
-
-			return fmt.Errorf("unable to apply an input resource: %w", err)
+			return fmt.Errorf("unable to read input resources: %w", err)
 		}
 
-		var resInt dynamic.ResourceInterface
-
-		if scopedGVR.Namespaced {
-			if obj.GetNamespace() == "" {
-				obj.SetNamespace("default")
-			}
-
-			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource).Namespace(obj.GetNamespace())
-		} else {
-			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource)
-		}
-
-		sanitizeForCreation(obj)
-
-		if _, err := resInt.Create(ctx, obj, metav1.CreateOptions{}); err != nil &&
-			!k8serrors.IsAlreadyExists(err) {
-			return fmt.Errorf("unable to apply an input resource: %w", err)
-		}
-
-		// Manually convert resources from the dynamic client to the runtime client
-		err = rec.Client.Create(ctx, obj)
-		if err != nil && !k8serrors.IsAlreadyExists(err) {
-			return err
+		err = d.applyInputResources(ctx, rec, inputObjects)
+		if err != nil {
+			return fmt.Errorf("unable to apply input resources: %w", err)
 		}
 	}
 
@@ -367,6 +336,54 @@ func (d *DryRunner) readInputResources(cmd *cobra.Command, args []string) (
 	return rawInputs, nil
 }
 
+// applyInputResources applies the user's resources to the fake cluster
+func (d *DryRunner) applyInputResources(
+	ctx context.Context,
+	rec *ctrl.ConfigurationPolicyReconciler,
+	inputObjects []*unstructured.Unstructured,
+) error {
+	for _, obj := range inputObjects {
+		gvk := obj.GroupVersionKind()
+
+		scopedGVR, err := rec.DynamicWatcher.GVKToGVR(gvk)
+		if err != nil {
+			if errors.Is(err, depclient.ErrNoVersionedResource) {
+				return fmt.Errorf("%w for kind %v: if this is a custom resource, it may need an "+
+					"entry in the mappings file", err, gvk.Kind)
+			}
+
+			return err
+		}
+
+		var resInt dynamic.ResourceInterface
+
+		if scopedGVR.Namespaced {
+			if obj.GetNamespace() == "" {
+				obj.SetNamespace("default")
+			}
+
+			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource).Namespace(obj.GetNamespace())
+		} else {
+			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource)
+		}
+
+		sanitizeForCreation(obj)
+
+		if _, err := resInt.Create(ctx, obj, metav1.CreateOptions{}); err != nil &&
+			!k8serrors.IsAlreadyExists(err) {
+			return err
+		}
+
+		// Manually convert resources from the dynamic client to the runtime client
+		err = rec.Client.Create(ctx, obj)
+		if err != nil && !k8serrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // setupLogs configures klog and the controller-runtime logger to send logs to the
 // path defined in the configuration. If that option is empty, logs will be discarded.
 func (d *DryRunner) setupLogs() error {
@@ -417,10 +434,34 @@ func (d *DryRunner) setupReconciler(
 		return nil, err
 	}
 
-	dynamicClient := dynfake.NewSimpleDynamicClient(scheme.Scheme)
-	clientset := clientsetfake.NewSimpleClientset()
-	watcherReconciler, _ := depclient.NewControllerRuntimeSource()
+	runtimeClient := clientfake.NewClientBuilder().
+		WithScheme(scheme.Scheme).
+		WithObjects(configPolCRD, cfgPolicy).
+		WithStatusSubresource(cfgPolicy).
+		Build()
+	nsSelUpdatesChan := make(chan event.GenericEvent, 20)
+
+	var clientset kubernetes.Interface
+	var dynamicClient dynamic.Interface
+	var nsSelReconciler common.NamespaceSelectorReconciler
+
+	if d.fromCluster {
+		var nsSelClient client.Client
+		var err error
+
+		clientset, dynamicClient, nsSelClient, err = setupClusterClients()
+		if err != nil {
+			return nil, err
+		}
+
+		nsSelReconciler = common.NewNamespaceSelectorReconciler(nsSelClient, nsSelUpdatesChan)
+	} else {
+		dynamicClient = dynfake.NewSimpleDynamicClient(scheme.Scheme)
+		clientset = clientsetfake.NewSimpleClientset()
+		nsSelReconciler = common.NewNamespaceSelectorReconciler(runtimeClient, nsSelUpdatesChan)
+	}
 
+	watcherReconciler, _ := depclient.NewControllerRuntimeSource()
 	dynamicWatcher := depclient.NewWithClients(
 		dynamicClient,
 		clientset.Discovery(),
@@ -435,14 +476,28 @@ func (d *DryRunner) setupReconciler(
 		}
 	}()
 
-	runtimeClient := clientfake.NewClientBuilder().
-		WithScheme(scheme.Scheme).
-		WithObjects(configPolCRD, cfgPolicy).
-		WithStatusSubresource(cfgPolicy).
-		Build()
+	rec := ctrl.ConfigurationPolicyReconciler{
+		Client:                 runtimeClient,
+		DecryptionConcurrency:  1,
+		DynamicWatcher:         dynamicWatcher,
+		Scheme:                 scheme.Scheme,
+		Recorder:               record.NewFakeRecorder(8),
+		InstanceName:           "policy-cli",
+		TargetK8sClient:        clientset,
+		TargetK8sDynamicClient: dynamicClient,
+		SelectorReconciler:     &nsSelReconciler,
+		EnableMetrics:          false,
+		UninstallMode:          false,
+		EvalBackoffSeconds:     5,
+		FullDiffs:              d.fullDiffs,
+	}
 
-	nsSelUpdatesChan := make(chan event.GenericEvent, 20)
-	nsSelReconciler := common.NewNamespaceSelectorReconciler(runtimeClient, nsSelUpdatesChan)
+	// wait for dynamic watcher to have started
+	<-rec.DynamicWatcher.Started()
+
+	if d.fromCluster {
+		return &rec, nil
+	}
 
 	defaultNs := &unstructured.Unstructured{
 		Object: map[string]interface{}{
@@ -467,21 +522,7 @@ func (d *DryRunner) setupReconciler(
 		return nil, err
 	}
 
-	rec := ctrl.ConfigurationPolicyReconciler{
-		Client:                 runtimeClient,
-		DecryptionConcurrency:  1,
-		DynamicWatcher:         dynamicWatcher,
-		Scheme:                 scheme.Scheme,
-		Recorder:               record.NewFakeRecorder(8),
-		InstanceName:           "policy-cli",
-		TargetK8sClient:        clientset,
-		TargetK8sDynamicClient: dynamicClient,
-		SelectorReconciler:     &nsSelReconciler,
-		EnableMetrics:          false,
-		UninstallMode:          false,
-		EvalBackoffSeconds:     5,
-		FullDiffs:              d.fullDiffs,
-	}
+	fakeClientset := clientset.(*clientsetfake.Clientset)
 
 	if d.mappingsPath != "" {
 		mFile, err := os.ReadFile(d.mappingsPath)
@@ -494,19 +535,16 @@ func (d *DryRunner) setupReconciler(
 			return nil, err
 		}
 
-		clientset.Resources = mappings.ResourceLists(apiMappings)
+		fakeClientset.Resources = mappings.ResourceLists(apiMappings)
 	} else {
-		clientset.Resources, err = mappings.DefaultResourceLists()
+		fakeClientset.Resources, err = mappings.DefaultResourceLists()
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	// Add open-cluster-management policy CRD
-	addSupportedResources(clientset)
-
-	// wait for dynamic watcher to have started
-	<-rec.DynamicWatcher.Started()
+	addSupportedResources(fakeClientset)
 
 	return &rec, nil
 }
@@ -622,6 +660,40 @@ func sanitizeForCreation(obj *unstructured.Unstructured) {
 	delete(obj.Object["metadata"].(map[string]interface{}), "uid")
 }
 
+func setupClusterClients() (kubernetes.Interface, dynamic.Interface, client.Client, error) {
+	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+	clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		loadingRules, &clientcmd.ConfigOverrides{},
+	)
+
+	kubeConfig, err := clientConfig.ClientConfig()
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	clientset, err := kubernetes.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	dynamicClient, err := dynamic.NewForConfig(kubeConfig)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	readOnlyMode := true // Prevent modifications to the cluster
+
+	runtimeClient, err := client.New(kubeConfig, client.Options{
+		Scheme: scheme.Scheme,
+		DryRun: &readOnlyMode,
+	})
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	return clientset, dynamicClient, runtimeClient, nil
+}
+
 func addSupportedResources(clientset *clientsetfake.Clientset) {
 	clientset.Resources = append(clientset.Resources, &metav1.APIResourceList{
 		GroupVersion: parentpolicyv1.GroupVersion.String(),

test/dryrun/context_vars/object_cluster_scoped/output_from_cluster.txt

@@ -0,0 +1,15 @@
+# Diffs:
+v1 Namespace mega-mart:
+--- mega-mart : existing
++++ mega-mart : updated
+@@ -5,10 +5,12 @@
+     city: durham
+   labels:
+     box: big
++    name: mega-mart
++    new-label: durham
+   name: mega-mart
+ spec:
+   finalizers:
+# Compliance messages:
+NonCompliant; violation - namespaces [mega-mart] found but not as specified

test/dryrun/context_vars/object_namespaced/input.yaml

@@ -1,12 +1,5 @@
 ---
 apiVersion: v1
-kind: Namespace
-metadata:
-  name: mega-mart
-  labels:
-    box: big
----
-apiVersion: v1
 kind: ConfigMap
 metadata:
   name: inventory

test/dryrun/context_vars/object_namespaced/input_ns.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: mega-mart
+  labels:
+    box: big
+

test/dryrun/context_vars/object_namespaced/output_from_cluster.txt

@@ -0,0 +1,16 @@
+# Diffs:
+v1 ConfigMap mega-mart/inventory:
+--- mega-mart/inventory : existing
++++ mega-mart/inventory : updated
+@@ -2,10 +2,12 @@
+ data:
+   inventory.yaml: 'appliance: toaster'
+ kind: ConfigMap
+ metadata:
++  labels:
++    new-label: toaster
+   name: inventory
+   namespace: mega-mart
+
+# Compliance messages:
+NonCompliant; violation - configmaps [inventory] found but not as specified in namespace mega-mart

test/dryrun/context_vars/object_namespaced/policy.yaml

@@ -19,4 +19,4 @@ spec:
           name: '{{ .ObjectName }}'
           namespace: '{{ .ObjectNamespace }}'
           labels:
-            new-label: '{{ (fromYAML (index .Object.data "inventory.yaml")).appliance }}'
+            new-label: '{{ ne (printf "%s" .ObjectName) "inventory" | skipObject }}{{ (fromYAML (index .Object.data "inventory.yaml")).appliance }}'

test/dryrun/context_vars/object_pod/output_from_cluster.txt

@@ -0,0 +1,15 @@
+# Diffs:
+v1 Pod default/nginx-pod:
+--- default/nginx-pod : existing
++++ default/nginx-pod : updated
+@@ -1,9 +1,11 @@
+ apiVersion: v1
+ kind: Pod
+ metadata:
++  labels:
++    image: nginx:1.7.9
+   name: nginx-pod
+   namespace: default
+ spec:
+# Compliance messages:
+NonCompliant; violation - pods [nginx-pod] found but not as specified in namespace default