Add-option-to-not-consolidate-under-single-ConfigurationPolicy-#15836 (#13)

* Add-option-to-not-consolidate-under-single-ConfigurationPolicy-#15836

Signed-off-by: Chunxi Luo [email protected]
Signed-off-by: ChunxiAlexLuo <[email protected]>

* polish

Signed-off-by: Chunxi Luo [email protected]
Signed-off-by: ChunxiAlexLuo <[email protected]>

* update comments and naming

Signed-off-by: Chunxi Luo [email protected]
Co-Authored-By: Matt Prahl <[email protected]>

Co-authored-by: Matt Prahl <[email protected]>

Author: ChunxiAlexLuo

docs/policygenerator-reference.yaml

@@ -21,6 +21,11 @@ policyDefaults:
   # annotation. This defaults to ["CM-2 Baseline Configuration"].
   controls:
     - "CM-2 Baseline Configuration"
+  # Optional. This determines if a single configuration policy should be
+  # generated for all the manifests being wrapped in the policy.
+  # If set to false, a configuration policy per manifest will be generated.
+  # This defaults to true.
+  consolidateManifests: true
   # Optional. When the policy references a Kyverno policy manifest, this determines if an additonal
   # configuration policy should be generated in order to receive policy violations in Open Cluster
   # Management when the Kyverno policy has been violated. This defaults to true.
@@ -91,6 +96,8 @@ policies:
     disabled: false
     # Optional. (See policyDefaults.informKyvernoPolicies for description.)
     informKyvernoPolicies: true
+    # Optional. (See policyDefaults.consolidateManifests for description.)
+    consolidateManifests: true
     # Optional. Determines the list of namespaces to check on the cluster for the given manifest. If
     # a namespace is specified in the manifest, the selector is not necessary. This defaults to no
     # selectors.

internal/plugin.go

@@ -211,20 +211,27 @@ func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 
 	// Policy expanders default to true unless explicitly set in the config.
 	// Gatekeeper policy expander policyDefault
-	igvValue, set := getDefaultBool(unmarshaledConfig, "informGatekeeperPolicies")
-	if set {
+	igvValue, setIgv := getDefaultBool(unmarshaledConfig, "informGatekeeperPolicies")
+	if setIgv {
 		p.PolicyDefaults.InformGatekeeperPolicies = igvValue
 	} else {
 		p.PolicyDefaults.InformGatekeeperPolicies = true
 	}
 	// Kyverno policy expander policyDefault
-	ikvValue, set := getDefaultBool(unmarshaledConfig, "informKyvernoPolicies")
-	if set {
+	ikvValue, setIkv := getDefaultBool(unmarshaledConfig, "informKyvernoPolicies")
+	if setIkv {
 		p.PolicyDefaults.InformKyvernoPolicies = ikvValue
 	} else {
 		p.PolicyDefaults.InformKyvernoPolicies = true
 	}
 
+	consolidatedValue, setConsolidated := getDefaultBool(unmarshaledConfig, "consolidateManifests")
+	if setConsolidated {
+		p.PolicyDefaults.ConsolidateManifests = consolidatedValue
+	} else {
+		p.PolicyDefaults.ConsolidateManifests = true
+	}
+
 	if p.PolicyDefaults.RemediationAction == "" {
 		p.PolicyDefaults.RemediationAction = defaults.RemediationAction
 	}
@@ -253,20 +260,27 @@ func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 
 		// Policy expanders default to the policy default unless explicitly set.
 		// Gatekeeper policy expander policy override
-		igvValue, set := getPolicyBool(unmarshaledConfig, i, "informGatekeeperPolicies")
-		if set {
+		igvValue, setIgv := getPolicyBool(unmarshaledConfig, i, "informGatekeeperPolicies")
+		if setIgv {
 			policy.InformGatekeeperPolicies = igvValue
 		} else {
 			policy.InformGatekeeperPolicies = p.PolicyDefaults.InformGatekeeperPolicies
 		}
 		// Kyverno policy expander policy override
-		ikvValue, set := getPolicyBool(unmarshaledConfig, i, "informKyvernoPolicies")
-		if set {
+		ikvValue, setIkv := getPolicyBool(unmarshaledConfig, i, "informKyvernoPolicies")
+		if setIkv {
 			policy.InformKyvernoPolicies = ikvValue
 		} else {
 			policy.InformKyvernoPolicies = p.PolicyDefaults.InformKyvernoPolicies
 		}
 
+		consolidatedValue, setConsolidated := getPolicyBool(unmarshaledConfig, i, "consolidateManifests")
+		if setConsolidated {
+			policy.ConsolidateManifests = consolidatedValue
+		} else {
+			policy.ConsolidateManifests = p.PolicyDefaults.ConsolidateManifests
+		}
+
 		// If both cluster selectors and placement rule path aren't set, then use the
 		// defaults with a priority on placement rule path.
 		if len(policy.Placement.ClusterSelectors) == 0 && policy.Placement.PlacementRulePath == "" {

internal/plugin_test.go

@@ -43,6 +43,10 @@ func TestGenerate(t *testing.T) {
 	}
 	p.Policies = append(p.Policies, policyConf, policyConf2)
 	p.applyDefaults(map[string]interface{}{})
+	// Default all policy ConsolidateManifests flags are set to true
+	// unless explicitly set
+	assertEqual(t, p.Policies[0].ConsolidateManifests, true)
+	assertEqual(t, p.Policies[1].ConsolidateManifests, true)
 	if err := p.assertValidConfig(); err != nil {
 		t.Fatal(err.Error())
 	}

internal/types/types.go

@@ -19,35 +19,37 @@ type PlacementConfig struct {
 
 // PolicyConfig represents a policy entry in the PolicyGenerator configuration.
 type PolicyConfig struct {
-	Categories               []string `json:"categories,omitempty" yaml:"categories,omitempty"`
-	ComplianceType           string   `json:"complianceType,omitempty" yaml:"complianceType,omitempty"`
-	Controls                 []string `json:"controls,omitempty" yaml:"controls,omitempty"`
-	Disabled                 bool     `json:"disabled,omitempty" yaml:"disabled,omitempty"`
-	InformGatekeeperPolicies bool     `json:"informGatekeeperPolicies,omitempty" yaml:"informGatekeeperPolicies,omitempty"`
-	InformKyvernoPolicies    bool     `json:"informKyvernoPolicies,omitempty" yaml:"informKyvernoPolicies,omitempty"`
+	Categories     []string `json:"categories,omitempty" yaml:"categories,omitempty"`
+	ComplianceType string   `json:"complianceType,omitempty" yaml:"complianceType,omitempty"`
+	Controls       []string `json:"controls,omitempty" yaml:"controls,omitempty"`
 	// This a slice of structs to allow additional configuration related to a manifest such as
 	// accepting patches.
 	Manifests         []Manifest        `json:"manifests,omitempty" yaml:"manifests,omitempty"`
 	Name              string            `json:"name,omitempty" yaml:"name,omitempty"`
 	NamespaceSelector NamespaceSelector `json:"namespaceSelector,omitempty" yaml:"namespaceSelector,omitempty"`
 	// This is named Placement so that eventually PlacementRules and Placements will be supported
-	Placement         PlacementConfig `json:"placement,omitempty" yaml:"placement,omitempty"`
-	RemediationAction string          `json:"remediationAction,omitempty" yaml:"remediationAction,omitempty"`
-	Severity          string          `json:"severity,omitempty" yaml:"severity,omitempty"`
-	Standards         []string        `json:"standards,omitempty" yaml:"standards,omitempty"`
+	Placement                PlacementConfig `json:"placement,omitempty" yaml:"placement,omitempty"`
+	RemediationAction        string          `json:"remediationAction,omitempty" yaml:"remediationAction,omitempty"`
+	Severity                 string          `json:"severity,omitempty" yaml:"severity,omitempty"`
+	Standards                []string        `json:"standards,omitempty" yaml:"standards,omitempty"`
+	ConsolidateManifests     bool            `json:"consolidateManifests,omitempty" yaml:"consolidateManifests,omitempty"`
+	Disabled                 bool            `json:"disabled,omitempty" yaml:"disabled,omitempty"`
+	InformGatekeeperPolicies bool            `json:"informGatekeeperPolicies,omitempty" yaml:"informGatekeeperPolicies,omitempty"`
+	InformKyvernoPolicies    bool            `json:"informKyvernoPolicies,omitempty" yaml:"informKyvernoPolicies,omitempty"`
 }
 
 type PolicyDefaults struct {
-	Categories               []string          `json:"categories,omitempty" yaml:"categories,omitempty"`
-	ComplianceType           string            `json:"complianceType,omitempty" yaml:"complianceType,omitempty"`
-	Controls                 []string          `json:"controls,omitempty" yaml:"controls,omitempty"`
-	InformGatekeeperPolicies bool              `json:"informGatekeeperPolicies,omitempty" yaml:"informGatekeeperPolicies,omitempty"`
-	InformKyvernoPolicies    bool              `json:"informKyvernoPolicies,omitempty" yaml:"informKyvernoPolicies,omitempty"`
-	Namespace                string            `json:"namespace,omitempty" yaml:"namespace,omitempty"`
-	NamespaceSelector        NamespaceSelector `json:"namespaceSelector,omitempty" yaml:"namespaceSelector,omitempty"`
+	Categories        []string          `json:"categories,omitempty" yaml:"categories,omitempty"`
+	ComplianceType    string            `json:"complianceType,omitempty" yaml:"complianceType,omitempty"`
+	Controls          []string          `json:"controls,omitempty" yaml:"controls,omitempty"`
+	Namespace         string            `json:"namespace,omitempty" yaml:"namespace,omitempty"`
+	NamespaceSelector NamespaceSelector `json:"namespaceSelector,omitempty" yaml:"namespaceSelector,omitempty"`
 	// This is named Placement so that eventually PlacementRules and Placements will be supported
-	Placement         PlacementConfig `json:"placement,omitempty" yaml:"placement,omitempty"`
-	RemediationAction string          `json:"remediationAction,omitempty" yaml:"remediationAction,omitempty"`
-	Severity          string          `json:"severity,omitempty" yaml:"severity,omitempty"`
-	Standards         []string        `json:"standards,omitempty" yaml:"standards,omitempty"`
+	Placement                PlacementConfig `json:"placement,omitempty" yaml:"placement,omitempty"`
+	RemediationAction        string          `json:"remediationAction,omitempty" yaml:"remediationAction,omitempty"`
+	Severity                 string          `json:"severity,omitempty" yaml:"severity,omitempty"`
+	Standards                []string        `json:"standards,omitempty" yaml:"standards,omitempty"`
+	ConsolidateManifests     bool            `json:"consolidateManifests,omitempty" yaml:"consolidateManifests,omitempty"`
+	InformGatekeeperPolicies bool            `json:"informGatekeeperPolicies,omitempty" yaml:"informGatekeeperPolicies,omitempty"`
+	InformKyvernoPolicies    bool            `json:"informKyvernoPolicies,omitempty" yaml:"informKyvernoPolicies,omitempty"`
 }

internal/utils.go

@@ -87,8 +87,11 @@ func getManifests(policyConf *types.PolicyConfig) ([]map[string]interface{}, err
 }
 
 // getPolicyTemplates generates the policy templates for the ConfigurationPolicy manifests
-// that includes the manifests specified in policyConf. An error is returned
-// if one or more manifests cannot be read or are invalid.
+// policyConf.ConsolidateManifests = true (default value) will generate a policy templates slice
+// that just has one template which includes all the manifests specified in policyConf.
+// policyConf.ConsolidateManifests = false will generate a policy templates slice
+// that each template includes a single manifest specified in policyConf.
+// An error is returned if one or more manifests cannot be read or are invalid.
 func getPolicyTemplates(policyConf *types.PolicyConfig) ([]map[string]map[string]interface{}, error) {
 	manifests, err := getManifests(policyConf)
 	if err != nil {
@@ -101,14 +104,54 @@ func getPolicyTemplates(policyConf *types.PolicyConfig) ([]map[string]map[string
 		)
 	}
 
-	objectTemplates := make([]map[string]interface{}, 0, len(manifests))
+	objectTemplatesLength := len(manifests)
+	policyTemplatesLength := 1
+	if !policyConf.ConsolidateManifests {
+		policyTemplatesLength = len(manifests)
+		objectTemplatesLength = 0
+	}
+	objectTemplates := make([]map[string]interface{}, 0, objectTemplatesLength)
+	policyTemplates := make([]map[string]map[string]interface{}, 0, policyTemplatesLength)
 	for _, manifest := range manifests {
 		objTemplate := map[string]interface{}{
 			"complianceType":   policyConf.ComplianceType,
 			"objectDefinition": manifest,
 		}
-		objectTemplates = append(objectTemplates, objTemplate)
+		if policyConf.ConsolidateManifests {
+			// put all objTemplate with manifest into single consolidated objectTemplates object
+			objectTemplates = append(objectTemplates, objTemplate)
+		} else {
+			// casting each objTemplate with manifest to objectTemplates type
+			// build policyTemplate for each objectTemplates
+			policyTemplate := buildPolicyTemplate(policyConf, &[]map[string]interface{}{objTemplate})
+			setNamespaceSelector(policyConf, policyTemplate)
+			policyTemplates = append(policyTemplates, *policyTemplate)
+		}
 	}
+
+	//  just build one policyTemplate by using the above consolidated objectTemplates
+	if policyConf.ConsolidateManifests {
+		policyTemplate := buildPolicyTemplate(policyConf, &objectTemplates)
+		setNamespaceSelector(policyConf, policyTemplate)
+		policyTemplates = append(policyTemplates, *policyTemplate)
+	}
+
+	// check the enabled expanders and add additional policy templates
+	expandedPolicyTemplates := handleExpanders(manifests, policyConf)
+	policyTemplates = append(policyTemplates, expandedPolicyTemplates...)
+
+	return policyTemplates, nil
+}
+
+// setNamespaceSelector sets the namespace selector, if set, on the input policy template.
+func setNamespaceSelector(policyConf *types.PolicyConfig, policyTemplate *map[string]map[string]interface{}) {
+	if policyConf.NamespaceSelector.Exclude != nil || policyConf.NamespaceSelector.Include != nil {
+		(*policyTemplate)["objectDefinition"]["spec"].(map[string]interface{})["namespaceSelector"] = policyConf.NamespaceSelector
+	}
+}
+
+// buildPolicyTemplate generates single policy template by using objectTemplates with manifests.
+func buildPolicyTemplate(policyConf *types.PolicyConfig, objectTemplates *[]map[string]interface{}) *map[string]map[string]interface{} {
 	policyTemplate := map[string]map[string]interface{}{
 		"objectDefinition": {
 			"apiVersion": policyAPIVersion,
@@ -117,22 +160,14 @@ func getPolicyTemplates(policyConf *types.PolicyConfig) ([]map[string]map[string
 				"name": policyConf.Name,
 			},
 			"spec": map[string]interface{}{
-				"object-templates":  objectTemplates,
+				"object-templates":  *objectTemplates,
 				"remediationAction": policyConf.RemediationAction,
 				"severity":          policyConf.Severity,
 			},
 		},
 	}
 
-	if policyConf.NamespaceSelector.Exclude != nil || policyConf.NamespaceSelector.Include != nil {
-		policyTemplate["objectDefinition"]["spec"].(map[string]interface{})["namespaceSelector"] = policyConf.NamespaceSelector
-	}
-
-	policyTemplates := []map[string]map[string]interface{}{policyTemplate}
-	expandedPolicyTemplates := handleExpanders(manifests, policyConf)
-	policyTemplates = append(policyTemplates, expandedPolicyTemplates...)
-
-	return policyTemplates, nil
+	return &policyTemplate
 }
 
 // handleExpanders will go through all the enabled expanders and generate additional

internal/utils_test.go

@@ -64,13 +64,17 @@ data:
 		{Manifests: manifestFiles},
 		{Manifests: []types.Manifest{{Path: tmpDir}}},
 	}
+	// test ConsolidateManifests = true (default case)
+	// policyTemplates will have only one policyTemplate
+	// and two objTemplate under this policyTemplate
 	for _, test := range tests {
 		policyConf := types.PolicyConfig{
-			ComplianceType:    "musthave",
-			Manifests:         test.Manifests,
-			Name:              "policy-app-config",
-			RemediationAction: "inform",
-			Severity:          "low",
+			ComplianceType:       "musthave",
+			ConsolidateManifests: true,
+			Manifests:            test.Manifests,
+			Name:                 "policy-app-config",
+			RemediationAction:    "inform",
+			Severity:             "low",
 		}
 
 		policyTemplates, err := getPolicyTemplates(&policyConf)
@@ -106,6 +110,67 @@ data:
 		}
 		assertEqual(t, kind2, "ConfigMap")
 	}
+
+	// test ConsolidateManifests = false case
+	// policyTemplates will skip the consolidation and have two policyTemplate
+	// and each policyTemplate has only one objTemplate
+	for _, test := range tests {
+		policyConf := types.PolicyConfig{
+			ComplianceType:       "musthave",
+			ConsolidateManifests: false,
+			Manifests:            test.Manifests,
+			Name:                 "policy-app-config",
+			RemediationAction:    "inform",
+			Severity:             "low",
+		}
+
+		policyTemplates, err := getPolicyTemplates(&policyConf)
+		if err != nil {
+			t.Fatalf("Failed to get the policy templates: %v", err)
+		}
+		assertEqual(t, len(policyTemplates), 2)
+		policyTemplate1 := policyTemplates[0]
+		objdef1 := policyTemplate1["objectDefinition"]
+		assertEqual(t, objdef1["metadata"].(map[string]string)["name"], "policy-app-config")
+		spec1, ok := objdef1["spec"].(map[string]interface{})
+		if !ok {
+			t.Fatal("The spec field is an invalid format")
+		}
+		assertEqual(t, spec1["remediationAction"], "inform")
+		assertEqual(t, spec1["severity"], "low")
+		objTemplates1, ok := spec1["object-templates"].([]map[string]interface{})
+		if !ok {
+			t.Fatal("The object-templates field is an invalid format")
+		}
+		assertEqual(t, len(objTemplates1), 1)
+		assertEqual(t, objTemplates1[0]["complianceType"], "musthave")
+		kind1, ok := objTemplates1[0]["objectDefinition"].(map[string]interface{})["kind"]
+		if !ok {
+			t.Fatal("The objectDefinition field is an invalid format")
+		}
+		assertEqual(t, kind1, "ConfigMap")
+
+		policyTemplate2 := policyTemplates[1]
+		objdef2 := policyTemplate2["objectDefinition"]
+		assertEqual(t, objdef2["metadata"].(map[string]string)["name"], "policy-app-config")
+		spec2, ok := objdef2["spec"].(map[string]interface{})
+		if !ok {
+			t.Fatal("The spec field is an invalid format")
+		}
+		assertEqual(t, spec2["remediationAction"], "inform")
+		assertEqual(t, spec2["severity"], "low")
+		objTemplates2, ok := spec2["object-templates"].([]map[string]interface{})
+		if !ok {
+			t.Fatal("The object-templates field is an invalid format")
+		}
+		assertEqual(t, len(objTemplates2), 1)
+		assertEqual(t, objTemplates2[0]["complianceType"], "musthave")
+		kind2, ok := objTemplates2[0]["objectDefinition"].(map[string]interface{})["kind"]
+		if !ok {
+			t.Fatal("The objectDefinition field is an invalid format")
+		}
+		assertEqual(t, kind2, "ConfigMap")
+	}
 }
 
 func TestGetPolicyTemplatePatches(t *testing.T) {