Merge pull request #3688 from luthermonson/unmanaged-eniconfigs

Allow for Self-Managed VPC with a Secondary Subnet for Pods

Author: k8s-ci-robot

docs/book/src/topics/eks/pod-networking.md

@@ -49,6 +49,34 @@ spec:
   disableVPCCNI: false
 ```
 
+### Using Secondary CIDRs
+EKS allows users to assign a [secondary CIDR range](https://www.eksworkshop.com/beginner/160_advanced-networking/secondary_cidr/) for pods to be  assigned. Below are how to get CAPA to generate ENIConfigs in both the managed and unmanaged VPC configurations. 
+
+> Secondary CIDR functionality will not work unless you enable custom network config too.
+
+#### Managed (dynamic) VPC
+Default configuration for CAPA is to manage the VPC and all the subnets for you dynamically. It will create and delete them along with your cluster. In this method all you need to do is set a SecondaryCidrBlock to one of the allowed two IPv4 CIDR blocks: 100.64.0.0/10 and 198.19.0.0/16. CAPA will automatically generate subnets and ENIConfigs for you and the VPC CNI will do the rest.
+
+```yaml
+kind: AWSManagedControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+metadata:
+  name: "capi-managed-test-control-plane"
+spec:
+  secondaryCidrBlock: 100.64.0.0/10
+  vpcCni:
+    env:
+    - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
+      value: "true" 
+  
+```
+
+#### Unmanaged (static) VPC
+In an unmanaged VPC configuration CAPA will create no VPC or subnets and will instead assign the cluster pieces to the IDs you pass. In order to get ENIConfigs to generate you will need to add tags to the subnet you created and want to use as the secondary subnets for your pods. This is done through tagging the subnets with the following tag: `sigs.k8s.io/cluster-api-provider-aws/association=secondary`.
+
+> Setting `SecondaryCidrBlock` in this configuration will be ignored and no subnets are created.
+
+
 ## Using an alternative CNI
 
 There may be scenarios where you do not want to use the Amazon VPC CNI. EKS supports a number of alternative CNIs such as Calico, Cilium, and Weave Net (see [docs](https://docs.aws.amazon.com/eks/latest/userguide/alternate-cni-plugins.html) for full list).

pkg/cloud/services/awsnode/cni.go

@@ -80,13 +80,16 @@ func (s *Service) ReconcileCNI(ctx context.Context) error {
 		}
 	}
 
-	if s.scope.SecondaryCidrBlock() == nil {
+	secondarySubnets := s.secondarySubnets()
+	if len(secondarySubnets) == 0 {
 		if needsUpdate {
 			s.scope.Info("adding environment properties to vpc-cni", "cluster", klog.KRef(s.scope.Namespace(), s.scope.Name()))
 			if err = remoteClient.Update(ctx, &ds, &client.UpdateOptions{}); err != nil {
 				return err
 			}
 		}
+
+		// with no secondary subnets there is no need for eni configs
 		return nil
 	}
 
@@ -101,7 +104,7 @@ func (s *Service) ReconcileCNI(ctx context.Context) error {
 	}
 
 	s.scope.Info("for each subnet", "cluster", klog.KRef(s.scope.Namespace(), s.scope.Name()))
-	for _, subnet := range s.secondarySubnets() {
+	for _, subnet := range secondarySubnets {
 		var eniConfig amazoncni.ENIConfig
 		if err := remoteClient.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceSystem, Name: subnet.AvailabilityZone}, &eniConfig); err != nil {
 			if !apierrors.IsNotFound(err) {

pkg/cloud/services/awsnode/cni_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/gomega"
@@ -225,13 +226,30 @@ func TestReconcileCniVpcCniValues(t *testing.T) {
 						Name: "node",
 					},
 				},
+				subnets: infrav1.Subnets{
+					{
+						// we aren't testing reconcileSubnets where this extra conf is added so putting it in by hand
+						ID:        "sn-1234",
+						CidrBlock: "100.0.0.1/20",
+						Tags: infrav1.Tags{
+							infrav1.NameAWSSubnetAssociation: infrav1.SecondarySubnetTagValue,
+						},
+					},
+				},
 			}
 			s := NewService(m)
 
 			err := s.ReconcileCNI(context.Background())
 			g.Expect(err).NotTo(HaveOccurred())
-			g.Expect(mockClient.updateChain).NotTo(BeEmpty())
-			ds, ok := mockClient.updateChain[0].(*v1.DaemonSet)
+
+			g.Expect(mockClient.updateChain).NotTo(BeEmpty()) // 0: eniconfig 1: daemonset
+			eniconf, ok := mockClient.updateChain[0].(*v1alpha1.ENIConfig)
+			g.Expect(ok).To(BeTrue())
+			g.Expect(len(eniconf.Spec.SecurityGroups)).To(Equal(1))
+			g.Expect(eniconf.Spec.SecurityGroups[0]).To(Equal(m.securityGroups["node"].ID))
+			g.Expect(eniconf.Spec.Subnet).To(Equal(m.subnets[0].ID))
+
+			ds, ok := mockClient.updateChain[1].(*v1.DaemonSet)
 			g.Expect(ok).To(BeTrue())
 			g.Expect(ds.Spec.Template.Spec.Containers).NotTo(BeEmpty())
 			g.Expect(ds.Spec.Template.Spec.Containers[0].Env).To(ConsistOf(tc.consistsOf))

pkg/internal/cidr/cidr_test.go

@@ -34,20 +34,20 @@ func TestSplitIntoSubnetsIPv4(t *testing.T) {
 		{
 			// https://aws.amazon.com/about-aws/whats-new/2018/10/amazon-eks-now-supports-additional-vpc-cidr-blocks/
 			name:        "default secondary cidr block configuration with primary cidr",
-			cidrblock:   "100.64.0.0/16",
+			cidrblock:   "100.64.0.0/10",
 			subnetcount: 3,
 			expected: []*net.IPNet{
 				{
 					IP:   net.IPv4(100, 64, 0, 0).To4(),
-					Mask: net.IPv4Mask(255, 255, 192, 0),
+					Mask: net.IPv4Mask(255, 240, 0, 0),
 				},
 				{
-					IP:   net.IPv4(100, 64, 64, 0).To4(),
-					Mask: net.IPv4Mask(255, 255, 192, 0),
+					IP:   net.IPv4(100, 80, 0, 0).To4(),
+					Mask: net.IPv4Mask(255, 240, 0, 0),
 				},
 				{
-					IP:   net.IPv4(100, 64, 128, 0).To4(),
-					Mask: net.IPv4Mask(255, 255, 192, 0),
+					IP:   net.IPv4(100, 96, 0, 0).To4(),
+					Mask: net.IPv4Mask(255, 240, 0, 0),
 				},
 			},
 		},