Add policyset generation support (#33)

Implement logic to generate policysets. There are three ways to generate
1. use policyDefaults.policysets field. If specified, any policies
without override will be included in the policysets
2. use policies[*].policysets field to include current policy in the
policysets. If specified, it overrides the policyDefaults.policysets
3. use policysets field. This allows you specify any policies no matter
policies are generated by generator or pre-exsit on hub. This field also
allows you specify the description of the policyset.

Signed-off-by: Yu Cao <[email protected]>

Author: Yu Cao

docs/policygenerator-reference.yaml

@@ -60,6 +60,16 @@ policyDefaults:
   # annotation. This defaults to ["NIST SP 800-53"].
   standards:
     - "NIST SP 800-53"
+  # Optional. Array of policy sets that the policy will join. Policy set details can be defined
+  # in the policySets section. When a policy is part of a policy set, a placement binding will not be 
+  # generated for the policy since one is generated for the set. Set policies[*].generatePlacementWhenInSet 
+  # or policyDefaults.generatePlacementWhenInSet to override.
+  policySets: []
+  # Optional. When a policy is part of a policy set, by default the generator will not generate the placement
+  # for this policy since a placement is generated for the policy set. If a placement should still be generated, 
+  # set it to "true" so that the policy will be deployed with both policy placement and policy set placement. 
+  # This defaults to "false".
+  generatePlacementWhenInSet: false
 
 # Required. The list of policies to create along with overrides to either the default values or, if
 # set, the values given in policyDefaults.
@@ -125,3 +135,36 @@ policies:
     # Optional. (See policyDefaults.standards for description.)
     standards:
       - "NIST SP 800-53"
+    # Optional. (See policyDefaults.policySets for description.)
+    policySets: []
+    # Optional. (See policyDefaults.generatePlacementWhenInSet for description.)
+    generatePlacementWhenInSet: false
+
+# Optional. The list of policy sets to create. To include a policy in a policy set, use 
+# policies[*].policySets or policyDefaults.policySets or policySets.policies.
+policySets:
+    # Required. The name of the policy set to create.
+  - name: ""
+    # Optional. The description of the policy set to create.
+    description: ""
+    # Optional. The list of policies to be included in the policy set. If policies[*].policySets or 
+    # policyDefaults.policySets is also specified, the list is merged.
+    policies: []
+    # Optional. The placement configuration for the policy set. This defaults to a placement
+    # configuration that matches all clusters.
+    placement:
+      # To specify a placement rule, specify key:value pair cluster selectors. (See placementRulePath
+      # to specify an existing file instead.)
+      clusterSelectors: {}
+      # To specify a placement, specify key:value pair cluster label selectors. (See placementPath to
+      # specify an existing file instead.)
+      labelSelector: {}
+      # Optional. Specifying a name will consolidate placement rules that contain the same cluster
+      # selectors.
+      name: ""
+      # To reuse an existing placement manifest, specify the path here relative to the
+      # kustomization.yaml file. (See clusterSelectors to generate a new Placement instead.)
+      placementPath: ""
+      # To reuse an existing placement rule manifest, specify the path here relative to the
+      # kustomization.yaml file. (See clusterSelectors to generate a new PlacementRule instead.)
+      placementRulePath: ""

examples/policyGenerator.yaml

@@ -20,6 +20,8 @@ policyDefaults:
   remediationAction: inform
   severity: medium
   # standards: []
+  policySets: 
+    - policyset-config
 policies:
 - name: policy-app-config-aliens
   disabled: false
@@ -48,6 +50,17 @@ policies:
   disabled: true
   manifests:
     - path: input-kyverno/
+  policySets:
+    - policyset-kyverno
 - name: policy-require-ns-labels
   manifests:
-    - path: input-gatekeeper/
+    - path: input-gatekeeper/
+  policySets:
+    - policyset-gatekeeper
+policySets:
+- name: policyset-kyverno
+  description: this is a kyverno policy set.
+  policies:
+    - pre-exists-kyverno-policy
+- name: policyset-gatekeeper
+  description: this is a gatekeeper policy set.

internal/plugin.go

@@ -19,6 +19,7 @@ const (
 	configPolicyKind           = "ConfigurationPolicy"
 	policyAPIVersion           = "policy.open-cluster-management.io/v1"
 	policyKind                 = "Policy"
+	policySetKind              = "PolicySet"
 	placementBindingAPIVersion = "policy.open-cluster-management.io/v1"
 	placementBindingKind       = "PlacementBinding"
 	placementRuleAPIVersion    = "apps.open-cluster-management.io/v1"
@@ -37,8 +38,9 @@ type Plugin struct {
 	PlacementBindingDefaults struct {
 		Name string `json:"name,omitempty" yaml:"name,omitempty"`
 	} `json:"placementBindingDefaults,omitempty" yaml:"placementBindingDefaults,omitempty"`
-	PolicyDefaults types.PolicyDefaults `json:"policyDefaults,omitempty" yaml:"policyDefaults,omitempty"`
-	Policies       []types.PolicyConfig `json:"policies" yaml:"policies"`
+	PolicyDefaults types.PolicyDefaults    `json:"policyDefaults,omitempty" yaml:"policyDefaults,omitempty"`
+	Policies       []types.PolicyConfig    `json:"policies" yaml:"policies"`
+	PolicySets     []types.PolicySetConfig `json:"policySets" yaml:"policySets"`
 	// A set of all placement names that have been processed or generated
 	allPlcs map[string]bool
 	// The base of the directory tree to restrict all manifest files to be within
@@ -105,15 +107,26 @@ func (p *Plugin) Generate() ([]byte, error) {
 		}
 	}
 
+	for i := range p.PolicySets {
+		err := p.createPolicySet(&p.PolicySets[i])
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Keep track of which placement maps to which policy. This will be used to determine
 	// how many placement bindings are required since one binding per placement is required.
 	plcNameToPolicyIdxs := map[string][]int{}
 	for i := range p.Policies {
-		plcName, err := p.createPlacement(&p.Policies[i])
-		if err != nil {
-			return nil, err
+		// only generate placement when GeneratePlacementWhenInSet equals to true or policy is not
+		// part of any policy sets
+		if p.Policies[i].GeneratePlacementWhenInSet || len(p.Policies[i].PolicySets) == 0 {
+			plcName, err := p.createPlacement(&p.Policies[i])
+			if err != nil {
+				return nil, err
+			}
+			plcNameToPolicyIdxs[plcName] = append(plcNameToPolicyIdxs[plcName], i)
 		}
-		plcNameToPolicyIdxs[plcName] = append(plcNameToPolicyIdxs[plcName], i)
 	}
 
 	// Sort the keys of plcNameToPolicyIdxs so that the policy bindings are generated in a
@@ -201,11 +214,11 @@ func getPolicyBool(
 	return
 }
 
-// applyDefaults applies any missing defaults under Policy.PlacementBindingDefaults and
-// Policy.PolicyDefaults. It then applies the defaults and user provided defaults on each
-// policy entry if they are not overridden by the user. The input unmarshaledConfig is used
-// in situations where it is necessary to know if an explicit false is provided rather than
-// rely on the default Go value on the Plugin struct.
+// applyDefaults applies any missing defaults under Policy.PlacementBindingDefaults,
+// Policy.PolicyDefaults and PolicySets. It then applies the defaults and user provided
+// defaults on each policy and policyset entry if they are not overridden by the user. The
+// input unmarshaledConfig is used in situations where it is necessary to know if an explicit
+// false is provided rather than rely on the default Go value on the Plugin struct.
 func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 	if len(p.Policies) == 0 {
 		return
@@ -258,6 +271,25 @@ func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 	if p.PolicyDefaults.Standards == nil {
 		p.PolicyDefaults.Standards = defaults.Standards
 	}
+	// Generate temporary sets to later merge the policy sets declared in p.Policies[*] and p.PolicySets
+	plcsetToPlc := make(map[string]map[string]bool)
+	plcToPlcset := make(map[string]map[string]bool)
+
+	for _, plcset := range p.PolicySets {
+		if plcsetToPlc[plcset.Name] == nil {
+			plcsetToPlc[plcset.Name] = make(map[string]bool)
+		}
+
+		for _, plc := range plcset.Policies {
+			plcsetToPlc[plcset.Name][plc] = true
+
+			if plcToPlcset[plc] == nil {
+				plcToPlcset[plc] = make(map[string]bool)
+			}
+
+			plcToPlcset[plc][plcset.Name] = true
+		}
+	}
 
 	for i := range p.Policies {
 		policy := &p.Policies[i]
@@ -273,6 +305,18 @@ func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 			policy.Controls = p.PolicyDefaults.Controls
 		}
 
+		if policy.PolicySets == nil {
+			policy.PolicySets = p.PolicyDefaults.PolicySets
+		}
+
+		// GeneratePlacementWhenInSet default to false unless explicitly set in the config.
+		gpValue, setGp := getPolicyBool(unmarshaledConfig, i, "generatePlacementWhenInSet")
+		if setGp {
+			policy.GeneratePlacementWhenInSet = gpValue
+		} else {
+			policy.GeneratePlacementWhenInSet = p.PolicyDefaults.GeneratePlacementWhenInSet
+		}
+
 		// Policy expanders default to the policy default unless explicitly set.
 		// Gatekeeper policy expander policy override
 		igvValue, setIgv := getPolicyBool(unmarshaledConfig, i, "informGatekeeperPolicies")
@@ -342,6 +386,42 @@ func (p *Plugin) applyDefaults(unmarshaledConfig map[string]interface{}) {
 				policy.Manifests[i].ComplianceType = policy.ComplianceType
 			}
 		}
+
+		for _, plcsetInPlc := range policy.PolicySets {
+			if _, ok := plcsetToPlc[plcsetInPlc]; !ok {
+				newPlcset := types.PolicySetConfig{
+					Name: plcsetInPlc,
+				}
+				p.PolicySets = append(p.PolicySets, newPlcset)
+				plcsetToPlc[plcsetInPlc] = make(map[string]bool)
+			}
+			if plcToPlcset[policy.Name] == nil {
+				plcToPlcset[policy.Name] = make(map[string]bool)
+			}
+
+			plcToPlcset[policy.Name][plcsetInPlc] = true
+
+			plcsetToPlc[plcsetInPlc][policy.Name] = true
+		}
+
+		policy.PolicySets = make([]string, 0, len(plcToPlcset[policy.Name]))
+
+		for plcset := range plcToPlcset[policy.Name] {
+			policy.PolicySets = append(policy.PolicySets, plcset)
+		}
+	}
+
+	// Sync up the declared policy sets in p.Policies[*]
+	for i := range p.PolicySets {
+		plcset := &p.PolicySets[i]
+		plcset.Policies = make([]string, 0, len(plcsetToPlc[plcset.Name]))
+
+		for plc := range plcsetToPlc[plcset.Name] {
+			plcset.Policies = append(plcset.Policies, plc)
+		}
+
+		// Sort alphabetically to make it deterministic
+		sort.Strings(plcset.Policies)
 	}
 }
 
@@ -532,6 +612,36 @@ func (p *Plugin) createPolicy(policyConf *types.PolicyConfig) error {
 	return nil
 }
 
+// createPolicySet will generate the policyset based on the Policy Generator configuration.
+// The generated policyset is written to the plugin's output buffer. An error is returned if the
+// manifests specified in the configuration are invalid or can't be read.
+func (p *Plugin) createPolicySet(policySetConf *types.PolicySetConfig) error {
+	policyset := map[string]interface{}{
+		"apiVersion": policyAPIVersion,
+		"kind":       policySetKind,
+		"metadata": map[string]interface{}{
+			"name":      policySetConf.Name,
+			"namespace": p.PolicyDefaults.Namespace, // policyset should be generated in the same namespace of policy
+		},
+		"spec": map[string]interface{}{
+			"description": policySetConf.Description,
+			"policies":    policySetConf.Policies,
+		},
+	}
+
+	policysetYAML, err := yaml.Marshal(policyset)
+	if err != nil {
+		return fmt.Errorf(
+			"an unexpected error occurred when converting the policyset to YAML: %w", err,
+		)
+	}
+
+	p.outputBuffer.Write([]byte("---\n"))
+	p.outputBuffer.Write(policysetYAML)
+
+	return nil
+}
+
 // getPlcFromPath finds the placement manifest in the input manifest file. It will return the name
 // of the placement, the unmarshaled placement manifest, and an error. An error is returned if the
 // placement manifest cannot be found or is invalid.