Impl. securityLists conf opt and management modes (#180)

Adds loadbalancer.securityListManagementMode config option that
configures how the CCM should manage security lists. Depreciates
loadbalancer.disableSecurityListManagement in favour of
loadbalancer.securityListManagementMode: None.

Additionally adds loadbalancer.securityLists config option to allow
explicit configuration of the security lists managed by the CCM on a per
subnet basis.

Author: prydie

manifests/cloud-provider-example.yaml

@@ -17,21 +17,25 @@ compartment: ocid1.compartment.oc1..aaaaaaaa3um2atybwhder4qttfhgon4j3hcxgmsvnyvx
 vcn: ocid1.vcn.oc1..aaaaaaaask7mpk4mij3pnm6yvnntte25ffadxiivpokxevfxgtsu6ftkqhrq
 
 loadBalancer:
-  # disableSecurityListManagement disables the automatic creation of ingress
-  # rules for the node subnets and egress rules for the load balancers to the
-  # node subnets.
-  #
-  # If security list management is disabled, then it requires that the user
-  # has setup a rule that allows inbound traffic to the appropriate ports
-  # for kube proxy health port, node port ranges, and health check port ranges.
-  # E.g. 10.82.0.0/16 30000-32000
-  disableSecurityListManagement: false
-
   # subnet1 configures one of two subnets to which load balancers will be added.
   # OCI load balancers require two subnets to ensure high availability.
   subnet1: ocid1.subnet.oc1.phx.aaaaaaaasa53hlkzk6nzksqfccegk2qnkxmphkblst3riclzs4rhwg7rg57q
 
-  # subnet2 configures the second of two subnets to which load balancers will
-  # be added.  OCI load balancers require two subnets to ensure high
-  # availability.
+  # subnet2 configures the second of two subnets to which load balancers will be
+  # added. OCI load balancers require two subnets to ensure high availability.
   subnet2: ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq
+
+  # SecurityListManagementMode configures how security lists are managed by the CCM.
+  #   "All" (default): Manage all required security list rules for load balancer services.
+  #   "Frontend":      Manage only security list rules for ingress to the load
+  #                    balancer. Requires that the user has setup a rule that
+  #                    allows inbound traffic to the appropriate ports for kube
+  #                    proxy health port, node port ranges, and health check port ranges.
+  #                    E.g. 10.82.0.0/16 30000-32000.
+  #   "None":          Disables all security list management. Requires that the
+  #                    user has setup a rule that allows inbound traffic to the
+  #                    appropriate ports for kube proxy health port, node port
+  #                    ranges, and health check port ranges. E.g. 10.82.0.0/16 30000-32000.
+  #                    Additionally requires the user to mange rules to allow
+  #                    inbound traffic to load balancers.
+  securityListManagementMode: All

pkg/oci/ccm.go

@@ -30,6 +30,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	wait "k8s.io/apimachinery/pkg/util/wait"
 	informers "k8s.io/client-go/informers"
+	informersv1 "k8s.io/client-go/informers/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	listersv1 "k8s.io/client-go/listers/core/v1"
 	cache "k8s.io/client-go/tools/cache"
@@ -137,18 +138,17 @@ func (cp *CloudProvider) Initialize(clientBuilder controller.ControllerClientBui
 	}
 	cp.NodeLister = nodeInformer.Lister()
 
-	if cp.config.LoadBalancer.DisableSecurityListManagement {
-		cp.securityListManager = newSecurityListManagerNOOP()
-
-	} else {
-		serviceInformer := factory.Core().V1().Services()
+	var serviceInformer informersv1.ServiceInformer
+	if cp.config.LoadBalancer.SecurityListManagementMode != ManagementModeNone {
+		serviceInformer = factory.Core().V1().Services()
 		go serviceInformer.Informer().Run(wait.NeverStop)
 		glog.Info("Waiting for service informer cache to sync")
 		if !cache.WaitForCacheSync(wait.NeverStop, serviceInformer.Informer().HasSynced) {
 			utilruntime.HandleError(fmt.Errorf("Timed out waiting for service informer to sync"))
 		}
-		cp.securityListManager = newSecurityListManager(cp.client, serviceInformer.Lister())
+
 	}
+	cp.securityListManager = newSecurityListManager(cp.client, serviceInformer.Lister(), cp.config.LoadBalancer.SecurityLists, cp.config.LoadBalancer.SecurityListManagementMode)
 }
 
 // ProviderName returns the cloud-provider ID.

pkg/oci/config.go

@@ -50,8 +50,17 @@ type LoadBalancerConfig struct {
 	// E.g. 10.82.0.0/16 30000-32000
 	DisableSecurityListManagement bool `yaml:"disableSecurityListManagement"`
 
+	// SecurityListManagementMode defines how the CCM manages security lists
+	// when provisioning load balancers. Available modes are All, Frontend,
+	// and None.
+	SecurityListManagementMode string `yaml:"securityListManagementMode"`
+
 	Subnet1 string `yaml:"subnet1"`
 	Subnet2 string `yaml:"subnet2"`
+
+	// SecurityLists defines the Security List to mutate for each Subnet (
+	// both load balancer and worker).
+	SecurityLists map[string]string `yaml:"securityLists"`
 }
 
 // Config holds the OCI cloud-provider config passed to Kubernetes compontents
@@ -70,6 +79,13 @@ type Config struct {
 
 // Complete the config applying defaults / overrides.
 func (c *Config) Complete() {
+	if c.LoadBalancer.SecurityListManagementMode == "" {
+		c.LoadBalancer.SecurityListManagementMode = ManagementModeAll // default
+		if c.LoadBalancer.DisableSecurityListManagement {
+			glog.Warningf("cloud-provider config: \"loadBalancer.disableSecurityListManagement\" is DEPRECIATED and will be removed in a later release. Please set \"loadBalancer.SecurityListManagementMode: %s\".", ManagementModeNone)
+			c.LoadBalancer.SecurityListManagementMode = ManagementModeNone
+		}
+	}
 	if c.CompartmentID == "" && c.Auth.CompartmentID != "" {
 		glog.Warning("cloud-provider config: \"auth.compartment\" is DEPRECIATED and will be removed in a later release. Please set \"compartment\".")
 		c.CompartmentID = c.Auth.CompartmentID

pkg/oci/config_validate.go

@@ -55,6 +55,9 @@ func validateLoadBalancerConfig(c *LoadBalancerConfig, fldPath *field.Path) fiel
 	if c.Subnet2 == "" {
 		allErrs = append(allErrs, field.Required(fldPath.Child("subnet2"), ""))
 	}
+	if !IsValidSecurityListManagementMode(c.SecurityListManagementMode) {
+		allErrs = append(allErrs, field.Invalid(fldPath.Child("securityListManagementMode"), c.SecurityListManagementMode, "invalid security list management mode"))
+	}
 	return allErrs
 }
 

pkg/oci/config_validate_test.go

@@ -44,6 +44,24 @@ func TestValidateConfig(t *testing.T) {
 				},
 			},
 			errs: field.ErrorList{},
+		}, {
+			name: "valid_with_non_default_security_list_management_mode",
+			in: &Config{
+				Auth: AuthConfig{
+					Region:        "us-phoenix-1",
+					TenancyID:     "ocid1.tenancy.oc1..aaaaaaaatyn7scrtwtqedvgrxgr2xunzeo6uanvyhzxqblctwkrpisvke4kq",
+					CompartmentID: "ocid1.compartment.oc1..aaaaaaaa3um2atybwhder4qttfhgon4j3hcxgmsvnyvx4flfjyewkkwfzwnq",
+					UserID:        "ocid1.user.oc1..aaaaaaaai77mql2xerv7cn6wu3nhxang3y4jk56vo5bn5l5lysl34avnui3q",
+					PrivateKey:    "-----BEGIN RSA PRIVATE KEY----- (etc)",
+					Fingerprint:   "8c:bf:17:7b:5f:e0:7d:13:75:11:d6:39:0d:e2:84:74",
+				},
+				LoadBalancer: LoadBalancerConfig{
+					Subnet1:                    "ocid1.tenancy.oc1..aaaaaaaatyn7scrtwtqedvgrxgr2xunzeo6uanvyhzxqblctwkrpisvke4kq",
+					Subnet2:                    "ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq",
+					SecurityListManagementMode: ManagementModeFrontend,
+				},
+			},
+			errs: field.ErrorList{},
 		}, {
 			name: "missing_region",
 			in: &Config{
@@ -186,14 +204,40 @@ func TestValidateConfig(t *testing.T) {
 			errs: field.ErrorList{
 				&field.Error{Type: field.ErrorTypeRequired, Field: "loadBalancer.subnet2", BadValue: ""},
 			},
+		}, {
+			name: "invalid_security_list_management_mode",
+			in: &Config{
+				Auth: AuthConfig{
+					Region:        "us-phoenix-1",
+					TenancyID:     "ocid1.tenancy.oc1..aaaaaaaatyn7scrtwtqedvgrxgr2xunzeo6uanvyhzxqblctwkrpisvke4kq",
+					CompartmentID: "ocid1.compartment.oc1..aaaaaaaa3um2atybwhder4qttfhgon4j3hcxgmsvnyvx4flfjyewkkwfzwnq",
+					UserID:        "ocid1.user.oc1..aaaaaaaai77mql2xerv7cn6wu3nhxang3y4jk56vo5bn5l5lysl34avnui3q",
+					PrivateKey:    "-----BEGIN RSA PRIVATE KEY----- (etc)",
+					Fingerprint:   "8c:bf:17:7b:5f:e0:7d:13:75:11:d6:39:0d:e2:84:74",
+				},
+				LoadBalancer: LoadBalancerConfig{
+					Subnet1:                    "ocid1.tenancy.oc1..aaaaaaaatyn7scrtwtqedvgrxgr2xunzeo6uanvyhzxqblctwkrpisvke4kq",
+					Subnet2:                    "ocid1.subnet.oc1.phx.aaaaaaaahuxrgvs65iwdz7ekwgg3l5gyah7ww5klkwjcso74u3e4i64hvtvq",
+					SecurityListManagementMode: "invalid",
+				},
+			},
+			errs: field.ErrorList{
+				&field.Error{
+					Type:     field.ErrorTypeInvalid,
+					Field:    "loadBalancer.securityListManagementMode",
+					BadValue: "invalid",
+					Detail:   "invalid security list management mode",
+				},
+			},
 		},
 	}
 
 	for _, tt := range testCases {
 		t.Run(tt.name, func(t *testing.T) {
+			tt.in.Complete()
 			result := ValidateConfig(tt.in)
 			if !reflect.DeepEqual(result, tt.errs) {
-				t.Errorf("ValidateConfig(%#v)\n=> %#v\nExpected: %#v", tt.in, result, tt.errs)
+				t.Errorf("ValidateConfig(%#v)\n=> %q \nExpected: %q", tt.in, result, tt.errs)
 			}
 		})
 	}