|
| 1 | +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 |
| 2 | +From: Angus Lees < [email protected]> |
| 3 | +Date: Thu, 19 Nov 2020 17:34:07 +1100 |
| 4 | +Subject: [PATCH] --EKS-PATCH-- AWS: Include IPv6 addresses in NodeAddresses |
| 5 | + |
| 6 | +Description: |
| 7 | +* Modifies the in-tree cloud provider code to append ipv6 addresses to the Node object advertised by the kubelet. |
| 8 | + |
| 9 | +Upstream PR, Issue, KEP, etc. links: |
| 10 | +* Taken from commit https://github.com/anguslees/kubernetes/commit/f8ea814e2d459a900bfb5e6f613dbe521b31515b. |
| 11 | +* Some of these changes were proposed in PR #86918 (https://github.com/kubernetes/kubernetes/pull/86918), and the exact |
| 12 | +changes from this patch were put forth in PR #113114 (https://github.com/kubernetes/kubernetes/pull/113114). But both |
| 13 | +PRs were closed without being merged. See below for info about why they weren't. |
| 14 | + |
| 15 | +If this patch is based on an upstream commit, how (if at all) do this patch and the upstream source differ? |
| 16 | +* N/A |
| 17 | + |
| 18 | +If this patch's changes have not been added by upstream, why not? |
| 19 | +* In-tree cloud provider code, which this patch modifies, has been in in feature freeze since Kubernetes 1.21. Both of |
| 20 | +the upstream PRs (#86918 and #113114) that aimed to enable this functionality were not merged before the feature freeze |
| 21 | +began. Since this change is a feature and not a critical security fix, Kubernetes is not going to add it. See |
| 22 | +https://github.com/kubernetes/kubernetes/pull/113114#issuecomment-1282460346 |
| 23 | + |
| 24 | +Other patches related to this patch: |
| 25 | +* None |
| 26 | + |
| 27 | +Changes made to this patch after its initial creation and reasons for these changes: |
| 28 | +* None |
| 29 | + |
| 30 | +Kubernetes version this patch can be dropped: |
| 31 | +* Likely will need to be dropped or changed in v1.27, as AWS is removed from legacy-cloud-providers in this version. |
| 32 | +See https://github.com/kubernetes/kubernetes/pull/115838 |
| 33 | + |
| 34 | +Signed-off-by: Jyoti Mahapatra< [email protected]> |
| 35 | +--- |
| 36 | + .../k8s.io/legacy-cloud-providers/aws/aws.go | 53 +++++++++++++++++++ |
| 37 | + .../legacy-cloud-providers/aws/aws_fakes.go | 8 +++ |
| 38 | + .../legacy-cloud-providers/aws/aws_test.go | 24 ++++++--- |
| 39 | + 3 files changed, 79 insertions(+), 6 deletions(-) |
| 40 | + |
| 41 | +diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go |
| 42 | +index 2aceff30890..434df20cc2e 100644 |
| 43 | +--- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go |
| 44 | ++++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws.go |
| 45 | +@@ -24,6 +24,8 @@ import ( |
| 46 | + "errors" |
| 47 | + "fmt" |
| 48 | + "io" |
| 49 | ++ "net" |
| 50 | ++ "net/http" |
| 51 | + "path" |
| 52 | + "regexp" |
| 53 | + "sort" |
| 54 | +@@ -1497,6 +1499,17 @@ func (c *Cloud) HasClusterID() bool { |
| 55 | + return len(c.tagging.clusterID()) > 0 |
| 56 | + } |
| 57 | + |
| 58 | ++// isAWSNotFound returns true if the error was caused by an AWS API 404 response. |
| 59 | ++func isAWSNotFound(err error) bool { |
| 60 | ++ if err != nil { |
| 61 | ++ var aerr awserr.RequestFailure |
| 62 | ++ if errors.As(err, &aerr) { |
| 63 | ++ return aerr.StatusCode() == http.StatusNotFound |
| 64 | ++ } |
| 65 | ++ } |
| 66 | ++ return false |
| 67 | ++} |
| 68 | ++ |
| 69 | + // NodeAddresses is an implementation of Instances.NodeAddresses. |
| 70 | + func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.NodeAddress, error) { |
| 71 | + if c.selfAWSInstance.nodeName == name || len(name) == 0 { |
| 72 | +@@ -1551,6 +1564,27 @@ func (c *Cloud) NodeAddresses(ctx context.Context, name types.NodeName) ([]v1.No |
| 73 | + } |
| 74 | + } |
| 75 | + |
| 76 | ++ // IPv6. Ordered after IPv4 addresses, so legacy code can continue to just use the "first" address in a dual-stack cluster. |
| 77 | ++ for _, macID := range macIDs { |
| 78 | ++ ipPath := path.Join("network/interfaces/macs/", macID, "ipv6s") |
| 79 | ++ ips, err := c.metadata.GetMetadata(ipPath) |
| 80 | ++ if err != nil { |
| 81 | ++ if isAWSNotFound(err) { |
| 82 | ++ // No IPv6 configured. Not an error, just a disappointment. |
| 83 | ++ continue |
| 84 | ++ } |
| 85 | ++ return nil, fmt.Errorf("error querying AWS metadata for %q: %q", ipPath, err) |
| 86 | ++ } |
| 87 | ++ |
| 88 | ++ for _, ip := range strings.Split(ips, "\n") { |
| 89 | ++ if ip == "" { |
| 90 | ++ continue |
| 91 | ++ } |
| 92 | ++ // NB: "Internal" is actually about intra-cluster reachability, and not public vs private. |
| 93 | ++ addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip}) |
| 94 | ++ } |
| 95 | ++ } |
| 96 | ++ |
| 97 | + externalIP, err := c.metadata.GetMetadata("public-ipv4") |
| 98 | + if err != nil { |
| 99 | + //TODO: It would be nice to be able to determine the reason for the failure, |
| 100 | +@@ -1640,6 +1674,25 @@ func extractNodeAddresses(instance *ec2.Instance) ([]v1.NodeAddress, error) { |
| 101 | + } |
| 102 | + } |
| 103 | + |
| 104 | ++ // IPv6. Ordered after IPv4 addresses, so legacy code can continue to just use the "first" address. |
| 105 | ++ for _, networkInterface := range instance.NetworkInterfaces { |
| 106 | ++ // skip network interfaces that are not currently in use |
| 107 | ++ if aws.StringValue(networkInterface.Status) != ec2.NetworkInterfaceStatusInUse { |
| 108 | ++ continue |
| 109 | ++ } |
| 110 | ++ |
| 111 | ++ for _, addr6 := range networkInterface.Ipv6Addresses { |
| 112 | ++ if ipAddress := aws.StringValue(addr6.Ipv6Address); ipAddress != "" { |
| 113 | ++ ip := net.ParseIP(ipAddress) |
| 114 | ++ if ip == nil { |
| 115 | ++ return nil, fmt.Errorf("EC2 instance had invalid IPv6 address: %s (%q)", aws.StringValue(instance.InstanceId), ipAddress) |
| 116 | ++ } |
| 117 | ++ // NB: "Internal" is actually about intra-cluster reachability, and not public vs private. |
| 118 | ++ addresses = append(addresses, v1.NodeAddress{Type: v1.NodeInternalIP, Address: ip.String()}) |
| 119 | ++ } |
| 120 | ++ } |
| 121 | ++ } |
| 122 | ++ |
| 123 | + // TODO: Other IP addresses (multiple ips)? |
| 124 | + publicIPAddress := aws.StringValue(instance.PublicIpAddress) |
| 125 | + if publicIPAddress != "" { |
| 126 | +diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_fakes.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_fakes.go |
| 127 | +index c970a6aaa02..de39d4eb63b 100644 |
| 128 | +--- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_fakes.go |
| 129 | ++++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_fakes.go |
| 130 | +@@ -41,6 +41,7 @@ type FakeAWSServices struct { |
| 131 | + selfInstance *ec2.Instance |
| 132 | + networkInterfacesMacs []string |
| 133 | + networkInterfacesPrivateIPs [][]string |
| 134 | ++ networkInterfacesIPv6s [][]string |
| 135 | + networkInterfacesVpcIDs []string |
| 136 | + |
| 137 | + ec2 FakeEC2 |
| 138 | +@@ -376,6 +377,13 @@ func (m *FakeMetadata) GetMetadata(key string) (string, error) { |
| 139 | + } |
| 140 | + } |
| 141 | + } |
| 142 | ++ if len(keySplit) == 5 && keySplit[4] == "ipv6s" { |
| 143 | ++ for i, macElem := range m.aws.networkInterfacesMacs { |
| 144 | ++ if macParam == macElem { |
| 145 | ++ return strings.Join(m.aws.networkInterfacesIPv6s[i], "/\n"), nil |
| 146 | ++ } |
| 147 | ++ } |
| 148 | ++ } |
| 149 | + |
| 150 | + return "", nil |
| 151 | + } |
| 152 | +diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go |
| 153 | +index 8330df2919d..c379459b148 100644 |
| 154 | +--- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go |
| 155 | ++++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go |
| 156 | +@@ -590,7 +590,7 @@ func testHasNodeAddress(t *testing.T, addrs []v1.NodeAddress, addressType v1.Nod |
| 157 | + t.Errorf("Did not find expected address: %s:%s in %v", addressType, address, addrs) |
| 158 | + } |
| 159 | + |
| 160 | +-func makeInstance(num int, privateIP, publicIP, privateDNSName, publicDNSName string, setNetInterface bool) ec2.Instance { |
| 161 | ++func makeInstance(num int, privateIP, publicIP, ipv6IP, privateDNSName, publicDNSName string, setNetInterface bool) ec2.Instance { |
| 162 | + var tag ec2.Tag |
| 163 | + tag.Key = aws.String(TagNameKubernetesClusterLegacy) |
| 164 | + tag.Value = aws.String(TestClusterID) |
| 165 | +@@ -620,6 +620,14 @@ func makeInstance(num int, privateIP, publicIP, privateDNSName, publicDNSName st |
| 166 | + }, |
| 167 | + }, |
| 168 | + } |
| 169 | ++ |
| 170 | ++ if ipv6IP != "" { |
| 171 | ++ instance.NetworkInterfaces[0].Ipv6Addresses = []*ec2.InstanceIpv6Address{ |
| 172 | ++ { |
| 173 | ++ Ipv6Address: aws.String(ipv6IP), |
| 174 | ++ }, |
| 175 | ++ } |
| 176 | ++ } |
| 177 | + } |
| 178 | + return instance |
| 179 | + } |
| 180 | +@@ -627,9 +635,9 @@ func makeInstance(num int, privateIP, publicIP, privateDNSName, publicDNSName st |
| 181 | + func TestNodeAddresses(t *testing.T) { |
| 182 | + // Note instance0 and instance1 have the same name |
| 183 | + // (we test that this produces an error) |
| 184 | +- instance0 := makeInstance(0, "192.168.0.1", "1.2.3.4", "instance-same.ec2.internal", "instance-same.ec2.external", true) |
| 185 | +- instance1 := makeInstance(1, "192.168.0.2", "", "instance-same.ec2.internal", "", false) |
| 186 | +- instance2 := makeInstance(2, "192.168.0.1", "1.2.3.4", "instance-other.ec2.internal", "", false) |
| 187 | ++ instance0 := makeInstance(0, "192.168.0.1", "1.2.3.4", "2001:db8::1", "instance-same.ec2.internal", "instance-same.ec2.external", true) |
| 188 | ++ instance1 := makeInstance(1, "192.168.0.2", "", "", "instance-same.ec2.internal", "", false) |
| 189 | ++ instance2 := makeInstance(2, "192.168.0.1", "1.2.3.4", "", "instance-other.ec2.internal", "", false) |
| 190 | + instances := []*ec2.Instance{&instance0, &instance1, &instance2} |
| 191 | + |
| 192 | + aws1, _ := mockInstancesResp(&instance0, []*ec2.Instance{&instance0}) |
| 193 | +@@ -651,23 +659,25 @@ func TestNodeAddresses(t *testing.T) { |
| 194 | + if err3 != nil { |
| 195 | + t.Errorf("Should not error when instance found") |
| 196 | + } |
| 197 | +- if len(addrs3) != 5 { |
| 198 | ++ if len(addrs3) != 6 { |
| 199 | + t.Errorf("Should return exactly 5 NodeAddresses") |
| 200 | + } |
| 201 | + testHasNodeAddress(t, addrs3, v1.NodeInternalIP, "192.168.0.1") |
| 202 | + testHasNodeAddress(t, addrs3, v1.NodeExternalIP, "1.2.3.4") |
| 203 | ++ testHasNodeAddress(t, addrs3, v1.NodeInternalIP, "2001:db8::1") |
| 204 | + testHasNodeAddress(t, addrs3, v1.NodeExternalDNS, "instance-same.ec2.external") |
| 205 | + testHasNodeAddress(t, addrs3, v1.NodeInternalDNS, "instance-same.ec2.internal") |
| 206 | + testHasNodeAddress(t, addrs3, v1.NodeHostName, "instance-same.ec2.internal") |
| 207 | + } |
| 208 | + |
| 209 | + func TestNodeAddressesWithMetadata(t *testing.T) { |
| 210 | +- instance := makeInstance(0, "", "2.3.4.5", "instance.ec2.internal", "", false) |
| 211 | ++ instance := makeInstance(0, "", "2.3.4.5", "", "instance.ec2.internal", "", false) |
| 212 | + instances := []*ec2.Instance{&instance} |
| 213 | + awsCloud, awsServices := mockInstancesResp(&instance, instances) |
| 214 | + |
| 215 | + awsServices.networkInterfacesMacs = []string{"0a:77:89:f3:9c:f6", "0a:26:64:c4:6a:48"} |
| 216 | + awsServices.networkInterfacesPrivateIPs = [][]string{{"192.168.0.1"}, {"192.168.0.2"}} |
| 217 | ++ awsServices.networkInterfacesIPv6s = [][]string{{"2001:db8:1::1"}, {"2001:db8:1::2"}} |
| 218 | + addrs, err := awsCloud.NodeAddresses(context.TODO(), "") |
| 219 | + if err != nil { |
| 220 | + t.Errorf("unexpected error: %v", err) |
| 221 | +@@ -675,6 +685,8 @@ func TestNodeAddressesWithMetadata(t *testing.T) { |
| 222 | + testHasNodeAddress(t, addrs, v1.NodeInternalIP, "192.168.0.1") |
| 223 | + testHasNodeAddress(t, addrs, v1.NodeInternalIP, "192.168.0.2") |
| 224 | + testHasNodeAddress(t, addrs, v1.NodeExternalIP, "2.3.4.5") |
| 225 | ++ testHasNodeAddress(t, addrs, v1.NodeInternalIP, "2001:db8:1::1") |
| 226 | ++ testHasNodeAddress(t, addrs, v1.NodeInternalIP, "2001:db8:1::2") |
| 227 | + var index1, index2 int |
| 228 | + for i, addr := range addrs { |
| 229 | + if addr.Type == v1.NodeInternalIP && addr.Address == "192.168.0.1" { |
0 commit comments