Skip to content

Commit e84505e

Browse files
vchintalbryantbiggs
vchintal
and
bryantbiggs
authored
feat: Add example for Private EKS cluster endpoint access thru AWS PrivateLink (#1687)
Co-authored-by: Bryant Biggs <[email protected]>
1 parent c01ea01 commit e84505e

File tree

13 files changed

+803
-31
lines changed

13 files changed

+803
-31
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,3 +43,6 @@ terraform.rc
4343
# local env
4444
*.envrc
4545
*kube-config.yaml
46+
47+
# Generated files
48+
builds

‎docs/blueprints/aws-eks-privatelink.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
---
2+
title: AWS EKS PrivateLink
3+
---
4+
5+
{%
6+
include-markdown "../../examples/aws-eks-privatelink/README.md"
7+
%}

‎examples/fully-private-cluster/main.tf‎

Lines changed: 14 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -77,39 +77,22 @@ module "vpc" {
7777
tags = local.tags
7878
}
7979

80-
module "vpc_endpoints_sg" {
81-
source = "terraform-aws-modules/security-group/aws"
82-
version = "~> 5.0"
83-
84-
name = "${local.name}-vpc-endpoints"
85-
description = "Security group for VPC endpoint access"
86-
vpc_id = module.vpc.vpc_id
87-
88-
ingress_with_cidr_blocks = [
89-
{
90-
rule = "https-443-tcp"
91-
description = "VPC CIDR HTTPS"
92-
cidr_blocks = join(",", module.vpc.private_subnets_cidr_blocks)
93-
},
94-
]
95-
96-
egress_with_cidr_blocks = [
97-
{
98-
rule = "https-443-tcp"
99-
description = "All egress HTTPS"
100-
cidr_blocks = "0.0.0.0/0"
101-
},
102-
]
103-
104-
tags = local.tags
105-
}
106-
10780
module "vpc_endpoints" {
10881
source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
109-
version = "~> 5.0"
110-
111-
vpc_id = module.vpc.vpc_id
112-
security_group_ids = [module.vpc_endpoints_sg.security_group_id]
82+
version = "~> 5.1"
83+
84+
vpc_id = module.vpc.vpc_id
85+
86+
# Security group
87+
create_security_group = true
88+
security_group_name_prefix = "${local.name}-vpc-endpoints-"
89+
security_group_description = "VPC endpoint security group"
90+
security_group_rules = {
91+
ingress_https = {
92+
description = "HTTPS from VPC"
93+
cidr_blocks = [module.vpc.vpc_cidr_block]
94+
}
95+
}
11396

11497
endpoints = merge({
11598
s3 = {

‎examples/privatelink-access/README.md‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# Private EKS cluster access via AWS PrivateLink
2+
3+
This example demonstrates how to access a private EKS cluster using AWS PrivateLink.
4+
5+
Refer to the [documentation](https://docs.aws.amazon.com/vpc/latest/privatelink/concepts.html) for further details on `AWS PrivateLink`.
6+
7+
## Prerequisites:
8+
9+
Ensure that you have the following tools installed locally:
10+
11+
1. [aws cli](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html)
12+
2. [kubectl](https://Kubernetes.io/docs/tasks/tools/)
13+
3. [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli)
14+
15+
## Deploy
16+
17+
To provision this example, first deploy the Lambda function that responds to `CreateNetworkInterface` API calls. This needs to exist before the cluster is created so that it can respond to the ENIs created by the EKS control plane:
18+
19+
```sh
20+
terraform init
21+
terraform apply -target=module.create_eni_lambda -target=module.nlb
22+
```
23+
24+
Enter `yes` at command prompt to apply
25+
26+
Next, deploy the remaining resources:
27+
28+
```sh
29+
terraform apply
30+
```
31+
32+
Enter `yes` at command prompt to apply
33+
34+
## Validate
35+
36+
### Network Connectivity
37+
38+
1. An output `ssm_test` has been provided to aid in quickly testing the connectivity from the client EC2 instance to the private EKS cluster via AWS Privatelink. Copy the output value and paste it into your terminal to execute and check the connectivity. If configured correctly, the value returned should be `ok`.
39+
40+
```sh
41+
COMMAND_ID=$(aws ssm send-command --region us-west-2 --document-name "AWS-RunShellScript" \
42+
--parameters 'commands=["curl -ks https://0218D48323E3E7D404D98659F1D097DD.gr7.us-west-2.eks.amazonaws.com/readyz"]' \
43+
--targets "Key=instanceids,Values=i-0280cf604085f4a44" --query 'Command.CommandId' --output text)
44+
45+
aws ssm get-command-invocation --region us-west-2 --command-id $COMMAND_ID --instance-id i-0280cf604085f4a44 --query 'StandardOutputContent' --output text
46+
```
47+
48+
### Cluster Access
49+
50+
To test access to the cluster, you will need to execute Kubernetes API calls from within the private network to access the cluster. An EC2 instance has been deployed to simulate this scenario, where the EC2 is deployed into a "client" VPC. However, since the EKS cluster was created with your local IAM identity, the `aws-auth` ConfigMap will only have your local identity that is permitted to access the cluster. Since cluster's API endpoint is private, we cannot use Terraform to reach it to additional entries to the ConfigMap; we can only access the cluster from within the private network of the cluster's VPC or from the client VPC using AWS PrivateLink access.
51+
52+
:warning: The "client" EC2 instance provided and copying of AWS credentials to that instance are merely for demonstration purposes only. Please consider alternate methods of network access such as AWS Client VPN to provide more secure access.
53+
54+
Perform the following steps to access the cluster with `kubectl` from the provided "client" EC2 instance.
55+
56+
1. Execute the command below on your local machine to get temporary credentials that will be used on the "client" EC2 instance:
57+
58+
```sh
59+
aws sts get-session-token --duration-seconds 3600 --output yaml
60+
```
61+
62+
2. Start a new SSM session on the "client" EC2 instance using the provided `ssm_start_session` output value. Your terminal will now be connected to the "client" EC2 instance.
63+
64+
```sh
65+
ssm_start_session = "aws ssm start-session --region us-west-2 --target i-0280cf604085f4a44"
66+
```
67+
68+
3. Once logged in, export the following environment variables from the output of step 1. Note - the session credentials are only valid for 1 hour; you can adjust the session duration in the command provided in step 1:
69+
70+
```sh
71+
export AWS_ACCESS_KEY_ID=XXXX
72+
export AWS_SECRET_ACCESS_KEY=YYYY
73+
export AWS_SESSION_TOKEN=ZZZZ
74+
```
75+
76+
4. Update the local `~/.kube/config` file to enable access to the cluster:
77+
78+
```sh
79+
aws eks update-kubeconfig --region us-west-2 --name privatelink-access
80+
```
81+
82+
5. Test access by listing the pods running on the clsuter:
83+
84+
```sh
85+
sh-4.2$ kubectl get pods -A
86+
87+
# Output
88+
NAMESPACE NAME READY STATUS RESTARTS AGE
89+
kube-system aws-node-4f8g8 1/1 Running 0 1m
90+
kube-system coredns-6ff9c46cd8-59sqp 1/1 Running 0 1m
91+
kube-system coredns-6ff9c46cd8-svnpb 1/1 Running 0 2m
92+
kube-system kube-proxy-mm2zc 1/1 Running 0 1m
93+
```
94+
95+
## Destroy
96+
97+
Run the following command to destroy all the resources created by Terraform:
98+
99+
```sh
100+
terraform destroy --auto-approve
101+
```

‎examples/privatelink-access/client.tf‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# The resources defined in this file are only used for demonstrating private connectivity
2+
# They are not required for the solution
3+
4+
locals {
5+
client_name = "${local.name}-client"
6+
}
7+
8+
################################################################################
9+
# VPC
10+
################################################################################
11+
12+
module "client_vpc" {
13+
source = "terraform-aws-modules/vpc/aws"
14+
version = "~> 5.0"
15+
16+
name = local.client_name
17+
cidr = local.vpc_cidr
18+
19+
azs = local.azs
20+
private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
21+
public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k + 10)]
22+
23+
enable_nat_gateway = true
24+
single_nat_gateway = true
25+
26+
manage_default_network_acl = true
27+
default_network_acl_tags = { Name = "${local.client_name}-default" }
28+
manage_default_route_table = true
29+
default_route_table_tags = { Name = "${local.client_name}-default" }
30+
manage_default_security_group = true
31+
default_security_group_tags = { Name = "${local.client_name}-default" }
32+
33+
tags = local.tags
34+
}
35+
36+
################################################################################
37+
# EC2 Instance
38+
################################################################################
39+
40+
module "client_ec2_instance" {
41+
source = "terraform-aws-modules/ec2-instance/aws"
42+
43+
name = local.client_name
44+
45+
create_iam_instance_profile = true
46+
iam_role_policies = {
47+
AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
48+
}
49+
50+
vpc_security_group_ids = [module.client_security_group.security_group_id]
51+
subnet_id = element(module.client_vpc.private_subnets, 0)
52+
53+
user_data = <<-EOT
54+
#!/bin/bash
55+
56+
# Install kubectl
57+
curl -LO https://dl.k8s.io/release/v${module.eks.cluster_version}.0/bin/linux/amd64/kubectl
58+
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
59+
60+
# Remove default awscli which is v1 - we want latest v2
61+
yum remove awscli -y
62+
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
63+
unzip -qq awscliv2.zip
64+
./aws/install
65+
EOT
66+
67+
tags = local.tags
68+
}
69+
70+
module "client_security_group" {
71+
source = "terraform-aws-modules/security-group/aws"
72+
version = "~> 5.0"
73+
74+
name = local.client_name
75+
description = "Security group for SSM access to private cluster"
76+
vpc_id = module.client_vpc.vpc_id
77+
78+
egress_with_cidr_blocks = [
79+
{
80+
from_port = 0
81+
to_port = 0
82+
protocol = "-1"
83+
cidr_blocks = "0.0.0.0/0"
84+
},
85+
]
86+
87+
tags = local.tags
88+
}

‎examples/privatelink-access/eks.tf‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
################################################################################
2+
# EKS Cluster
3+
################################################################################
4+
5+
module "eks" {
6+
source = "terraform-aws-modules/eks/aws"
7+
version = "~> 19.15"
8+
9+
cluster_name = local.name
10+
cluster_version = "1.27"
11+
12+
cluster_addons = {
13+
coredns = {}
14+
kube-proxy = {}
15+
vpc-cni = {}
16+
}
17+
18+
cluster_security_group_additional_rules = {
19+
# Allow tcp/443 from the NLB IP addresses
20+
for ip_addr in data.dns_a_record_set.nlb.addrs : "nlb_ingress_${replace(ip_addr, ".", "")}" => {
21+
description = "Allow ingress from NLB"
22+
type = "ingress"
23+
from_port = 443
24+
to_port = 443
25+
protocol = "tcp"
26+
cidr_blocks = ["${ip_addr}/32"]
27+
}
28+
}
29+
30+
vpc_id = module.vpc.vpc_id
31+
subnet_ids = module.vpc.private_subnets
32+
33+
eks_managed_node_groups = {
34+
default = {
35+
instance_types = ["c5.large"]
36+
37+
min_size = 1
38+
max_size = 3
39+
desired_size = 1
40+
}
41+
}
42+
43+
tags = local.tags
44+
}
45+
46+
################################################################################
47+
# VPC
48+
################################################################################
49+
50+
module "vpc" {
51+
source = "terraform-aws-modules/vpc/aws"
52+
version = "~> 5.0"
53+
54+
name = local.name
55+
cidr = local.vpc_cidr
56+
57+
azs = local.azs
58+
private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
59+
60+
# Disable NAT gateway for fully private networking
61+
enable_nat_gateway = false
62+
63+
private_subnet_tags = {
64+
"kubernetes.io/role/internal-elb" = 1
65+
}
66+
67+
tags = local.tags
68+
}
69+
70+
# Explicitly create a Internet Gateway here in the Private EKS VPC as without an
71+
# internet gateway, a NLB cannot be created. Config option of create_igw = true
72+
# (default) did not work during the VPC creation as it requires public subnets
73+
# and the related routes that connect them to IGW
74+
resource "aws_internet_gateway" "igw" {
75+
vpc_id = module.vpc.vpc_id
76+
tags = local.tags
77+
}
78+
79+
################################################################################
80+
# VPC Endpoints
81+
################################################################################
82+
83+
module "vpc_endpoints" {
84+
source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints"
85+
version = "~> 5.1"
86+
87+
vpc_id = module.vpc.vpc_id
88+
89+
# Security group
90+
create_security_group = true
91+
security_group_name_prefix = "${local.name}-vpc-endpoints-"
92+
security_group_description = "VPC endpoint security group"
93+
security_group_rules = {
94+
ingress_https = {
95+
description = "HTTPS from VPC"
96+
cidr_blocks = [module.vpc.vpc_cidr_block]
97+
}
98+
}
99+
100+
endpoints = merge({
101+
s3 = {
102+
service = "s3"
103+
service_type = "Gateway"
104+
route_table_ids = module.vpc.private_route_table_ids
105+
tags = {
106+
Name = "${local.name}-s3"
107+
}
108+
}
109+
},
110+
{ for service in toset(["autoscaling", "ecr.api", "ecr.dkr", "ec2", "ec2messages", "elasticloadbalancing", "sts", "kms", "logs", "ssm", "ssmmessages"]) :
111+
replace(service, ".", "_") =>
112+
{
113+
service = service
114+
subnet_ids = module.vpc.private_subnets
115+
private_dns_enabled = true
116+
tags = { Name = "${local.name}-${service}" }
117+
}
118+
})
119+
120+
tags = local.tags
121+
}

0 commit comments

Comments
 (0)