Merge pull request openshift#7983 from mtulio/aws-byoip

openshift-merge-bot[bot] · web-flow · commit 9978285a5ca3 · 2024-02-26T19:18:45.000Z
SPLAT-1434: aws: add support of BYO public ipv4 pool
diff --git a/data/data/aws/bootstrap/main.tf b/data/data/aws/bootstrap/main.tf
@@ -249,3 +249,11 @@ resource "aws_security_group_rule" "bootstrap_journald_gateway" {
   from_port   = 19531
   to_port     = 19531
 }
+
+resource "aws_eip" "bootstrap" {
+  domain           = "vpc"
+  instance         = aws_instance.bootstrap.id
+  public_ipv4_pool = var.aws_public_ipv4_pool == "" ? null : var.aws_public_ipv4_pool
+
+  depends_on = [aws_instance.bootstrap]
+}
diff --git a/data/data/aws/cluster/main.tf b/data/data/aws/cluster/main.tf
@@ -109,6 +109,8 @@ module "vpc" {
   edge_parent_gw_map = var.aws_edge_parent_zones_index
   edge_zones_type    = var.aws_edge_zones_type
 
+  public_ipv4_pool = var.aws_public_ipv4_pool
+
   tags = local.tags
 }
 
diff --git a/data/data/aws/cluster/vpc/master-elb.tf b/data/data/aws/cluster/vpc/master-elb.tf
@@ -24,10 +24,18 @@ resource "aws_lb" "api_external" {
 
   name                             = "${var.cluster_id}-ext"
   load_balancer_type               = "network"
-  subnets                          = data.aws_subnet.public.*.id
   internal                         = false
   enable_cross_zone_load_balancing = true
 
+  dynamic "subnet_mapping" {
+    for_each = range(length(data.aws_subnet.public))
+
+    content {
+      subnet_id     = data.aws_subnet.public[subnet_mapping.key].id
+      allocation_id = aws_eip.api_nlb_public[subnet_mapping.key].id
+    }
+  }
+
   tags = merge(
     {
       "Name" = "${var.cluster_id}-ext"
@@ -38,7 +46,24 @@ resource "aws_lb" "api_external" {
   timeouts {
     create = "20m"
   }
+}
+
+resource "aws_eip" "api_nlb_public" {
+  count  = length(var.availability_zones)
+  domain = "vpc"
+
+  public_ipv4_pool = var.public_ipv4_pool == "" ? null : var.public_ipv4_pool
+
+  tags = merge(
+    {
+      "Name" = "${var.cluster_id}-eip-${var.availability_zones[count.index]}-lb-api"
+    },
+    var.tags,
+  )
 
+  # Terraform does not declare an explicit dependency towards the internet gateway.
+  # this can cause the internet gateway to be deleted/detached before the EIPs.
+  # https://github.com/coreos/tectonic-installer/issues/1017#issuecomment-307780549
   depends_on = [aws_internet_gateway.igw]
 }
 
diff --git a/data/data/aws/cluster/vpc/variables.tf b/data/data/aws/cluster/vpc/variables.tf
@@ -59,4 +59,9 @@ variable "public_subnets" {
 variable "private_subnets" {
   type        = list(string)
   description = "Existing private subnets into which the cluster should be installed."
+}
+
+variable "public_ipv4_pool" {
+  type        = string
+  description = "An Public IPv4 Pool"
 }
diff --git a/data/data/aws/cluster/vpc/vpc-public.tf b/data/data/aws/cluster/vpc/vpc-public.tf
@@ -95,6 +95,8 @@ resource "aws_eip" "nat_eip" {
   count = var.public_subnets == null ? length(var.availability_zones) : 0
   vpc   = true
 
+  public_ipv4_pool = var.public_ipv4_pool == "" ? null : var.public_ipv4_pool
+
   tags = merge(
     {
       "Name" = "${var.cluster_id}-eip-${var.availability_zones[count.index]}"
diff --git a/data/data/aws/variables-aws.tf b/data/data/aws/variables-aws.tf
@@ -226,4 +226,16 @@ a Public Route Table and the default route entry pointing to the carrier gateway
 
 Example: `{ "us-east-1-nyc-1a"=local-zone, "us-east-1-wl1-nyc-wlz-1"=wavelength-zone }`
 EOF
-}
+}
+
+variable "aws_public_ipv4_pool" {
+  type = string
+
+  description = <<EOF
+(optional) Indicates the installation process to use Public IPv4 address
+that you bring to your AWS account with BYOIP to create resources which consumes
+Elastic IPs when the publish strategy is External.
+EOF
+
+  default = ""
+}
diff --git a/data/data/install.openshift.io_installconfigs.yaml b/data/data/install.openshift.io_installconfigs.yaml
@@ -2287,6 +2287,11 @@ spec:
                       operators to include the specified user tags in the tags of
                       the AWS resources that the operators create.
                     type: boolean
+                  publicIpv4Pool:
+                    description: PublicIpv4Pool is an optional field that can be used
+                      to tell the installation process to use Public IPv4 address
+                      that you bring to your AWS account with BYOIP.
+                    type: string
                   region:
                     description: Region specifies the AWS region where the cluster
                       will be created.
diff --git a/docs/user/aws/custom_public_ipv4.md b/docs/user/aws/custom_public_ipv4.md
@@ -0,0 +1,38 @@
+# Install OpenShift on AWS using custom/owned Public IPv4 Pool
+
+Steps to create a cluster on AWS using Public IPv4 address pool
+that you bring to your AWS account with BYOIP.
+
+## Prerequisites
+
+- Public IPv4 Pool Provisioned in the Account
+- Total of ( (Zones*3 ) + 1) of Public IPv4 available in the pool, where: Zones is the total numbber of AWS zones used to deploy the OpenShift cluster.
+    - Example to query the IPv4 pools available in the account, which returns the  `TotalAvailableAddressCount`:
+```
+aws ec2 describe-public-ipv4-pools --region us-east-1
+```
+
+## Steps
+
+- Create the install config setting the field `platform.aws.publicIpv4PoolId`, and create the cluster:
+
+```yaml
+apiVersion: v1
+baseDomain: ${CLUSTER_BASE_DOMAIN}
+metadata:
+  name: ocp-byoip
+platform:
+  aws:
+    region: ${REGION}
+    publicIpv4Pool: ipv4pool-ec2-123456789abcde
+publish: External
+pullSecret: '...'
+sshKey: |
+  '...'
+```
+
+- Create the cluster
+
+```sh
+openshift-install create cluster
+```
diff --git a/pkg/asset/cluster/tfvars/tfvars.go b/pkg/asset/cluster/tfvars/tfvars.go
@@ -333,6 +333,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			Proxy:                     installConfig.Config.Proxy,
 			PreserveBootstrapIgnition: installConfig.Config.AWS.PreserveBootstrapIgnition,
 			MasterSecurityGroups:      securityGroups,
+			PublicIpv4Pool:            installConfig.Config.AWS.PublicIpv4Pool,
 		})
 		if err != nil {
 			return errors.Wrapf(err, "failed to get %s Terraform variables", platform)
diff --git a/pkg/asset/installconfig/aws/ec2.go b/pkg/asset/installconfig/aws/ec2.go
@@ -2,6 +2,7 @@ package aws
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -28,3 +29,24 @@ func DescribeSecurityGroups(ctx context.Context, session *session.Session, secur
 	}
 	return sgOutput.SecurityGroups, nil
 }
+
+// DescribePublicIpv4Pool returns the ec2 public IPv4 Pool attributes from the given ID.
+func DescribePublicIpv4Pool(ctx context.Context, session *session.Session, region string, poolID string) (*ec2.PublicIpv4Pool, error) {
+	client := ec2.New(session, aws.NewConfig().WithRegion(region))
+
+	cctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
+	defer cancel()
+
+	poolOutputs, err := client.DescribePublicIpv4PoolsWithContext(cctx, &ec2.DescribePublicIpv4PoolsInput{PoolIds: []*string{aws.String(poolID)}})
+	if err != nil {
+		return nil, err
+	}
+	if len(poolOutputs.PublicIpv4Pools) == 0 {
+		return nil, fmt.Errorf("public IPv4 Pool not found: %s", poolID)
+	}
+	// it should not happen
+	if len(poolOutputs.PublicIpv4Pools) > 1 {
+		return nil, fmt.Errorf("more than one Public IPv4 Pool: %s", poolID)
+	}
+	return poolOutputs.PublicIpv4Pools[0], nil
+}
diff --git a/pkg/asset/installconfig/aws/validation.go b/pkg/asset/installconfig/aws/validation.go
@@ -46,6 +46,7 @@ func Validate(ctx context.Context, meta *Metadata, config *types.InstallConfig)
 		return errors.New(field.Required(field.NewPath("platform", "aws"), "AWS validation requires an AWS platform configuration").Error())
 	}
 	allErrs = append(allErrs, validateAMI(ctx, config)...)
+	allErrs = append(allErrs, validatePublicIpv4Pool(ctx, meta, field.NewPath("platform", "aws", "publicIpv4PoolId"), config)...)
 	allErrs = append(allErrs, validatePlatform(ctx, meta, field.NewPath("platform", "aws"), config.Platform.AWS, config.Networking, config.Publish)...)
 
 	if config.ControlPlane != nil {
@@ -141,6 +142,47 @@ func validateAMI(ctx context.Context, config *types.InstallConfig) field.ErrorLi
 	return field.ErrorList{field.Required(field.NewPath("platform", "aws", "amiID"), "AMI must be provided")}
 }
 
+func validatePublicIpv4Pool(ctx context.Context, meta *Metadata, fldPath *field.Path, config *types.InstallConfig) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if config.Platform.AWS.PublicIpv4Pool == "" {
+		return nil
+	}
+	poolID := config.Platform.AWS.PublicIpv4Pool
+	if config.Publish != types.ExternalPublishingStrategy {
+		return append(allErrs, field.Invalid(fldPath, poolID, fmt.Errorf("publish strategy %s can't be used with custom Public IPv4 Pools", config.Publish).Error()))
+	}
+
+	// Pool validations
+	// Resources claiming Public IPv4 from Pool in regular 'External' installations:
+	// 1* for Bootsrtap
+	// N*Zones for NAT Gateways
+	// N*Zones for API LB
+	// N*Zones for Ingress LB
+	allzones, err := meta.AvailabilityZones(ctx)
+	if err != nil {
+		return append(allErrs, field.InternalError(fldPath, err))
+	}
+	totalPublicIPRequired := int64(1 + (len(allzones) * 3))
+
+	sess, err := meta.Session(ctx)
+	if err != nil {
+		return append(allErrs, field.Invalid(fldPath, nil, fmt.Sprintf("unable to start a session: %s", err.Error())))
+	}
+	publicIpv4Pool, err := DescribePublicIpv4Pool(ctx, sess, config.Platform.AWS.Region, poolID)
+	if err != nil {
+		return append(allErrs, field.Invalid(fldPath, poolID, err.Error()))
+	}
+
+	got := aws.Int64Value(publicIpv4Pool.TotalAvailableAddressCount)
+	if got < totalPublicIPRequired {
+		err = fmt.Errorf("required a minimum of %d Public IPv4 IPs available in the pool %s, got %d", totalPublicIPRequired, poolID, got)
+		return append(allErrs, field.InternalError(fldPath, err))
+	}
+
+	return nil
+}
+
 func validateSubnets(ctx context.Context, meta *Metadata, fldPath *field.Path, subnets []string, networking *types.Networking, publish types.PublishingStrategy) field.ErrorList {
 	allErrs := field.ErrorList{}
 	privateSubnets, err := meta.PrivateSubnets(ctx)
diff --git a/pkg/asset/installconfig/aws/validation_test.go b/pkg/asset/installconfig/aws/validation_test.go
@@ -807,6 +807,17 @@ func TestValidate(t *testing.T) {
 		publicSubnets:  validPublicSubnets(),
 		proxy:          "http://proxy.com",
 		expectErr:      `^\Qplatform.aws.serviceEndpoints[0].url: Invalid value: "http://test": Head "http://test": dial tcp: lookup test\E.*: no such host$`,
+	}, {
+		name: "invalid public ipv4 pool private installation",
+		installConfig: func() *types.InstallConfig {
+			c := validInstallConfig()
+			c.Publish = types.InternalPublishingStrategy
+			c.Platform.AWS.PublicIpv4Pool = "ipv4pool-ec2-123"
+			c.Platform.AWS.Subnets = []string{}
+			return c
+		}(),
+		availZones: validAvailZones(),
+		expectErr:  `^platform.aws.publicIpv4PoolId: Invalid value: "ipv4pool-ec2-123": publish strategy Internal can't be used with custom Public IPv4 Pools$`,
 	}}
 
 	for _, test := range tests {
diff --git a/pkg/explain/printer_test.go b/pkg/explain/printer_test.go
@@ -179,6 +179,9 @@ func Test_PrintFields(t *testing.T) {
     propagateUserTags <boolean>
       PropagateUserTags is a flag that directs in-cluster operators to include the specified user tags in the tags of the AWS resources that the operators create.
 
+    publicIpv4Pool <string>
+      PublicIpv4Pool is an optional field that can be used to tell the installation process to use Public IPv4 address that you bring to your AWS account with BYOIP.
+
     region <string> -required-
       Region specifies the AWS region where the cluster will be created.
 
diff --git a/pkg/tfvars/aws/aws.go b/pkg/tfvars/aws/aws.go
@@ -49,6 +49,7 @@ type Config struct {
 	BootstrapMetadataAuthentication string            `json:"aws_bootstrap_instance_metadata_authentication,omitempty"`
 	PreserveBootstrapIgnition       bool              `json:"aws_preserve_bootstrap_ignition"`
 	MasterSecurityGroups            []string          `json:"aws_master_security_groups,omitempty"`
+	PublicIpv4Pool                  string            `json:"aws_public_ipv4_pool"`
 }
 
 // TFVarsSources contains the parameters to be converted into Terraform variables
@@ -80,6 +81,8 @@ type TFVarsSources struct {
 	PreserveBootstrapIgnition bool
 
 	MasterSecurityGroups []string
+
+	PublicIpv4Pool string
 }
 
 // TFVars generates AWS-specific Terraform variables launching the cluster.
@@ -207,6 +210,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		WorkerIAMRoleName:         sources.WorkerIAMRoleName,
 		PreserveBootstrapIgnition: sources.PreserveBootstrapIgnition,
 		MasterSecurityGroups:      sources.MasterSecurityGroups,
+		PublicIpv4Pool:            sources.PublicIpv4Pool,
 	}
 
 	stubIgn, err := bootstrap.GenerateIgnitionShimWithCertBundleAndProxy(sources.IgnitionPresignedURL, sources.AdditionalTrustBundle, sources.Proxy)
diff --git a/pkg/types/aws/platform.go b/pkg/types/aws/platform.go
@@ -106,6 +106,11 @@ type Platform struct {
 	// during bootstrap destroy.
 	// +optional
 	PreserveBootstrapIgnition bool `json:"preserveBootstrapIgnition,omitempty"`
+
+	// PublicIpv4Pool is an optional field that can be used to tell the installation process to use
+	// Public IPv4 address that you bring to your AWS account with BYOIP.
+	// +optional
+	PublicIpv4Pool string `json:"publicIpv4Pool,omitempty"`
 }
 
 // ServiceEndpoint store the configuration for services to

Original file line number	Diff line number	Diff line change
`@@ -109,6 +109,8 @@ module "vpc" {`
`109`	`109`	`edge_parent_gw_map = var.aws_edge_parent_zones_index`
`110`	`110`	`edge_zones_type = var.aws_edge_zones_type`
`111`	`111`
	`112`	`+ public_ipv4_pool = var.aws_public_ipv4_pool`
	`113`	`+`
`112`	`114`	`tags = local.tags`
`113`	`115`	`}`
`114`	`116`
Original file line number	Diff line number	Diff line change
`@@ -95,6 +95,8 @@ resource "aws_eip" "nat_eip" {`
`95`	`95`	`count = var.public_subnets == null ? length(var.availability_zones) : 0`
`96`	`96`	`vpc = true`
`97`	`97`
	`98`	`+ public_ipv4_pool = var.public_ipv4_pool == "" ? null : var.public_ipv4_pool`
	`99`	`+`
`98`	`100`	`tags = merge(`
`99`	`101`	`{`
`100`	`102`	`"Name" = "${var.cluster_id}-eip-${var.availability_zones[count.index]}"`