Add dryrun CLI flag to read cluster resources

ref: https://issues.redhat.com/browse/ACM-22932

Instead of providing input yaml files to the dryrun command args, set the --from-cluster flag to read cluster resources with the default kubeconfig. As before, the policies are set to Inform during evaluation with the fake runtime client, preventing modifications to the real cluster.

Moved the input yaml creation code to a helper function.

When --from-cluster enabled, point the reconciler's TargetK8sClient, TargetK8sDynamicClient, DynamicWatcher and SelectorReconciler to the real cluster.

Signed-off-by: Janelle Law <[email protected]>

Author: jan-law

pkg/dryrun/cmd.go

@@ -21,6 +21,7 @@ type DryRunner struct {
 	logPath       string
 	noColors      bool
 	fullDiffs     bool
+	fromCluster   bool
 }
 
 var ErrNonCompliant = errors.New("policy is NonCompliant")
@@ -105,6 +106,15 @@ func (d *DryRunner) GetCmd() *cobra.Command {
 			"the DRYRUN_MAPPINGS_FILE environment variable.",
 	)
 
+	cmd.Flags().BoolVar(
+		&d.fromCluster,
+		"from-cluster",
+		false,
+		"Read the current state of resources from a real Kubernetes cluster instead of "+
+			"from input files. Uses the default kubeconfig or KUBECONFIG environment variable. "+
+			"Any input files representing the cluster state will be ignored.",
+	)
+
 	cmd.AddCommand(&cobra.Command{
 		Use:   "generate",
 		Short: "Generate an API Mappings file",

pkg/dryrun/dryrun.go

@@ -30,8 +30,10 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/dynamic"
 	dynfake "k8s.io/client-go/dynamic/fake"
+	"k8s.io/client-go/kubernetes"
 	clientsetfake "k8s.io/client-go/kubernetes/fake"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
 	klog "k8s.io/klog/v2"
 	parentpolicyv1 "open-cluster-management.io/governance-policy-propagator/api/v1"
@@ -57,11 +59,6 @@ func (d *DryRunner) dryRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to read input policy: %w", err)
 	}
 
-	inputObjects, err := d.readInputResources(cmd, args)
-	if err != nil {
-		return fmt.Errorf("unable to read input resources: %w", err)
-	}
-
 	if err := d.setupLogs(); err != nil {
 		return fmt.Errorf("unable to setup the logging configuration: %w", err)
 	}
@@ -74,43 +71,15 @@ func (d *DryRunner) dryRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to setup the dryrun reconciler: %w", err)
 	}
 
-	// Apply the user's resources to the fake cluster
-	for _, obj := range inputObjects {
-		gvk := obj.GroupVersionKind()
-
-		scopedGVR, err := rec.DynamicWatcher.GVKToGVR(gvk)
+	if !d.fromCluster {
+		inputObjects, err := d.readInputResources(cmd, args)
 		if err != nil {
-			if errors.Is(err, depclient.ErrNoVersionedResource) {
-				return fmt.Errorf("%w for kind %v: if this is a custom resource, it may need an "+
-					"entry in the mappings file", err, gvk.Kind)
-			}
-
-			return fmt.Errorf("unable to apply an input resource: %w", err)
+			return fmt.Errorf("unable to read input resources: %w", err)
 		}
 
-		var resInt dynamic.ResourceInterface
-
-		if scopedGVR.Namespaced {
-			if obj.GetNamespace() == "" {
-				obj.SetNamespace("default")
-			}
-
-			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource).Namespace(obj.GetNamespace())
-		} else {
-			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource)
-		}
-
-		sanitizeForCreation(obj)
-
-		if _, err := resInt.Create(ctx, obj, metav1.CreateOptions{}); err != nil &&
-			!k8serrors.IsAlreadyExists(err) {
-			return fmt.Errorf("unable to apply an input resource: %w", err)
-		}
-
-		// Manually convert resources from the dynamic client to the runtime client
-		err = rec.Client.Create(ctx, obj)
-		if err != nil && !k8serrors.IsAlreadyExists(err) {
-			return err
+		err = d.applyInputResources(ctx, rec, inputObjects)
+		if err != nil {
+			return fmt.Errorf("unable to apply input resources: %w", err)
 		}
 	}
 
@@ -259,9 +228,59 @@ func (d *DryRunner) readPolicy(cmd *cobra.Command) (*policyv1.ConfigurationPolic
 		Name: parentName,
 	}}
 
+	if cfgpol.GetNamespace() == "" {
+		cfgpol.SetNamespace("default")
+	}
+
 	return &cfgpol, nil
 }
 
+func (d *DryRunner) applyInputResources(
+	ctx context.Context, rec *ctrl.ConfigurationPolicyReconciler, inputObjects []*unstructured.Unstructured,
+) error {
+	// Apply the user's resources to the fake cluster
+	for _, obj := range inputObjects {
+		gvk := obj.GroupVersionKind()
+
+		scopedGVR, err := rec.DynamicWatcher.GVKToGVR(gvk)
+		if err != nil {
+			if errors.Is(err, depclient.ErrNoVersionedResource) {
+				return fmt.Errorf("%w for kind %v: if this is a custom resource, it may need an "+
+					"entry in the mappings file", err, gvk.Kind)
+			}
+
+			return fmt.Errorf("unable to apply an input resource: %w", err)
+		}
+
+		var resInt dynamic.ResourceInterface
+
+		if scopedGVR.Namespaced {
+			if obj.GetNamespace() == "" {
+				obj.SetNamespace("default")
+			}
+
+			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource).Namespace(obj.GetNamespace())
+		} else {
+			resInt = rec.TargetK8sDynamicClient.Resource(scopedGVR.GroupVersionResource)
+		}
+
+		sanitizeForCreation(obj)
+
+		if _, err := resInt.Create(ctx, obj, metav1.CreateOptions{}); err != nil &&
+			!k8serrors.IsAlreadyExists(err) {
+			return fmt.Errorf("unable to apply an input resource: %w", err)
+		}
+
+		// Manually convert resources from the dynamic client to the runtime client
+		err = rec.Client.Create(ctx, obj)
+		if err != nil && !k8serrors.IsAlreadyExists(err) {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // readInputResources takes stdin and any paths given as "positional" arguments,
 // and decodes them from YAML into k8s resources. Directories can be passed in
 // arguments: in this case all files in that directory will be read and decoded
@@ -417,11 +436,60 @@ func (d *DryRunner) setupReconciler(
 		return nil, err
 	}
 
-	dynamicClient := dynfake.NewSimpleDynamicClient(scheme.Scheme)
-	clientset := clientsetfake.NewSimpleClientset()
+	runtimeClient := clientfake.NewClientBuilder().
+		WithScheme(scheme.Scheme).
+		WithObjects(configPolCRD, cfgPolicy).
+		WithStatusSubresource(cfgPolicy).
+		Build()
+
+	nsSelUpdatesChan := make(chan event.GenericEvent, 20)
+
 	watcherReconciler, _ := depclient.NewControllerRuntimeSource()
 
-	dynamicWatcher := depclient.NewWithClients(
+	var dynamicWatcher depclient.DynamicWatcher
+	var clientset kubernetes.Interface
+	var dynamicClient dynamic.Interface
+	var nsSelReconciler common.NamespaceSelectorReconciler
+
+	if d.fromCluster {
+		loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()
+		clientConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+			loadingRules, &clientcmd.ConfigOverrides{},
+		)
+
+		kubeConfig, err := clientConfig.ClientConfig()
+		if err != nil {
+			return nil, fmt.Errorf("unable to locate the default kubeconfig: %w", err)
+		}
+
+		clientset, err = kubernetes.NewForConfig(kubeConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create kubernetes clientset for cluster: %w", err)
+		}
+
+		dynamicClient, err = dynamic.NewForConfig(kubeConfig)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create dynamic client for cluster: %w", err)
+		}
+
+		readOnlyMode := true // Prevent modifications to the cluster
+
+		realRuntimeClient, err := client.New(kubeConfig, client.Options{
+			Scheme: scheme.Scheme,
+			DryRun: &readOnlyMode,
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create runtime client for ns selector reconciler: %w", err)
+		}
+
+		nsSelReconciler = common.NewNamespaceSelectorReconciler(realRuntimeClient, nsSelUpdatesChan)
+	} else {
+		dynamicClient = dynfake.NewSimpleDynamicClient(scheme.Scheme)
+		clientset = clientsetfake.NewSimpleClientset()
+		nsSelReconciler = common.NewNamespaceSelectorReconciler(runtimeClient, nsSelUpdatesChan)
+	}
+
+	dynamicWatcher = depclient.NewWithClients(
 		dynamicClient,
 		clientset.Discovery(),
 		watcherReconciler,
@@ -435,14 +503,28 @@ func (d *DryRunner) setupReconciler(
 		}
 	}()
 
-	runtimeClient := clientfake.NewClientBuilder().
-		WithScheme(scheme.Scheme).
-		WithObjects(configPolCRD, cfgPolicy).
-		WithStatusSubresource(cfgPolicy).
-		Build()
+	rec := ctrl.ConfigurationPolicyReconciler{
+		Client:                 runtimeClient,
+		DecryptionConcurrency:  1,
+		DynamicWatcher:         dynamicWatcher,
+		Scheme:                 scheme.Scheme,
+		Recorder:               record.NewFakeRecorder(8),
+		InstanceName:           "policy-cli",
+		TargetK8sClient:        clientset,
+		TargetK8sDynamicClient: dynamicClient,
+		SelectorReconciler:     &nsSelReconciler,
+		EnableMetrics:          false,
+		UninstallMode:          false,
+		EvalBackoffSeconds:     5,
+		FullDiffs:              d.fullDiffs,
+	}
 
-	nsSelUpdatesChan := make(chan event.GenericEvent, 20)
-	nsSelReconciler := common.NewNamespaceSelectorReconciler(runtimeClient, nsSelUpdatesChan)
+	// wait for dynamic watcher to have started
+	<-rec.DynamicWatcher.Started()
+
+	if d.fromCluster {
+		return &rec, nil
+	}
 
 	defaultNs := &unstructured.Unstructured{
 		Object: map[string]interface{}{
@@ -467,21 +549,7 @@ func (d *DryRunner) setupReconciler(
 		return nil, err
 	}
 
-	rec := ctrl.ConfigurationPolicyReconciler{
-		Client:                 runtimeClient,
-		DecryptionConcurrency:  1,
-		DynamicWatcher:         dynamicWatcher,
-		Scheme:                 scheme.Scheme,
-		Recorder:               record.NewFakeRecorder(8),
-		InstanceName:           "policy-cli",
-		TargetK8sClient:        clientset,
-		TargetK8sDynamicClient: dynamicClient,
-		SelectorReconciler:     &nsSelReconciler,
-		EnableMetrics:          false,
-		UninstallMode:          false,
-		EvalBackoffSeconds:     5,
-		FullDiffs:              d.fullDiffs,
-	}
+	fakeClientset := clientset.(*clientsetfake.Clientset)
 
 	if d.mappingsPath != "" {
 		mFile, err := os.ReadFile(d.mappingsPath)
@@ -494,19 +562,16 @@ func (d *DryRunner) setupReconciler(
 			return nil, err
 		}
 
-		clientset.Resources = mappings.ResourceLists(apiMappings)
+		fakeClientset.Resources = mappings.ResourceLists(apiMappings)
 	} else {
-		clientset.Resources, err = mappings.DefaultResourceLists()
+		fakeClientset.Resources, err = mappings.DefaultResourceLists()
 		if err != nil {
 			return nil, err
 		}
 	}
 
 	// Add open-cluster-management policy CRD
-	addSupportedResources(clientset)
-
-	// wait for dynamic watcher to have started
-	<-rec.DynamicWatcher.Started()
+	addSupportedResources(fakeClientset)
 
 	return &rec, nil
 }