Merge pull request openshift#7225 from patrickdillon/aws-shared-vpc

CORS-2613: AWS: Cross-account Shared VPC Support

Author: patrickdillon

data/data/aws/cluster/main.tf

@@ -70,9 +70,11 @@ module "dns" {
   cluster_id               = var.cluster_id
   tags                     = local.tags
   internal_zone            = var.aws_internal_zone
+  internal_zone_role       = var.aws_internal_zone_role
   vpc_id                   = module.vpc.vpc_id
   region                   = var.aws_region
   publish_strategy         = var.aws_publish_strategy
+  custom_endpoints         = var.custom_endpoints
 }
 
 module "vpc" {

data/data/aws/cluster/route53/base.tf

@@ -9,6 +9,27 @@ locals {
   use_alias = ! local.use_cname
 }
 
+provider "aws" {
+  alias = "private_hosted_zone"
+
+  assume_role {
+    role_arn = var.internal_zone_role
+  }
+
+  region = var.region
+
+  skip_region_validation = true
+
+  endpoints {
+    ec2     = lookup(var.custom_endpoints, "ec2", null)
+    elb     = lookup(var.custom_endpoints, "elasticloadbalancing", null)
+    iam     = lookup(var.custom_endpoints, "iam", null)
+    route53 = lookup(var.custom_endpoints, "route53", null)
+    s3      = lookup(var.custom_endpoints, "s3", null)
+    sts     = lookup(var.custom_endpoints, "sts", null)
+  }
+}
+
 data "aws_route53_zone" "public" {
   count = local.public_endpoints ? 1 : 0
 
@@ -18,6 +39,8 @@ data "aws_route53_zone" "public" {
 }
 
 data "aws_route53_zone" "int" {
+  provider = aws.private_hosted_zone
+
   zone_id = var.internal_zone == null ? aws_route53_zone.new_int[0].id : var.internal_zone
 }
 
@@ -54,7 +77,8 @@ resource "aws_route53_record" "api_external_alias" {
 }
 
 resource "aws_route53_record" "api_internal_alias" {
-  count = local.use_alias ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_alias ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -68,7 +92,8 @@ resource "aws_route53_record" "api_internal_alias" {
 }
 
 resource "aws_route53_record" "api_external_internal_zone_alias" {
-  count = local.use_alias ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_alias ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"
@@ -93,7 +118,8 @@ resource "aws_route53_record" "api_external_cname" {
 }
 
 resource "aws_route53_record" "api_internal_cname" {
-  count = local.use_cname ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_cname ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api-int.${var.cluster_domain}"
@@ -104,7 +130,8 @@ resource "aws_route53_record" "api_internal_cname" {
 }
 
 resource "aws_route53_record" "api_external_internal_zone_cname" {
-  count = local.use_cname ? 1 : 0
+  provider = aws.private_hosted_zone
+  count    = local.use_cname ? 1 : 0
 
   zone_id = data.aws_route53_zone.int.zone_id
   name    = "api.${var.cluster_domain}"

data/data/aws/cluster/route53/variables.tf

@@ -28,6 +28,12 @@ variable "internal_zone" {
   description = "An existing hosted zone (zone ID) to use for the internal API."
 }
 
+variable "internal_zone_role" {
+  type        = string
+  default     = null
+  description = "(optional) A role to assume when using an existing hosted zone from another account."
+}
+
 variable "api_external_lb_dns_name" {
   description = "External API's LB DNS name"
   type        = string
@@ -63,3 +69,16 @@ variable "region" {
   type = string
   description = "The target AWS region for the cluster."
 }
+
+variable "custom_endpoints" {
+  type = map(string)
+
+  description = <<EOF
+(optional) Custom AWS endpoints to override existing services.
+Check - https://www.terraform.io/docs/providers/aws/guides/custom-service-endpoints.html
+
+Example: `{ "key" = "value", "foo" = "bar" }`
+EOF
+
+  default = {}
+}

data/data/aws/variables-aws.tf

@@ -144,6 +144,13 @@ variable "aws_internal_zone" {
   description = "(optional) An existing hosted zone (zone ID) to use for the internal API."
 }
 
+variable "aws_internal_zone_role" {
+  type        = string
+  default     = null
+  description = "(optional) A role to assume when using an existing hosted zone from another account."
+}
+
+
 variable "aws_publish_strategy" {
   type        = string
   description = "The cluster publishing strategy, either Internal or External"

data/data/install.openshift.io_installconfigs.yaml

@@ -2267,6 +2267,14 @@ spec:
                       the subnets. Leave the hosted zone unset to have the installer
                       create the hosted zone on your behalf.
                     type: string
+                  hostedZoneRole:
+                    description: HostedZoneRole is the ARN of a role to be assumed
+                      when performing operations on the provided HostedZone. HostedZoneRole
+                      can be used in a shared VPC scenario when the private hosted
+                      zone belongs to a different account than the rest of the cluster
+                      resources. If HostedZoneRole is set, HostedZone must also be
+                      set.
+                    type: string
                   lbType:
                     description: "LBType is an optional field to specify a load balancer
                       type. \n When this field is specified, the default ingresscontroller

pkg/asset/cluster/aws/aws.go

@@ -11,6 +11,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/openshift/installer/pkg/asset/installconfig"
+	awsic "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	"github.com/openshift/installer/pkg/types"
 	awstypes "github.com/openshift/installer/pkg/types/aws"
 )
@@ -26,6 +27,7 @@ func Metadata(clusterID, infraID string, config *types.InstallConfig) *awstypes.
 		}},
 		ServiceEndpoints: config.AWS.ServiceEndpoints,
 		ClusterDomain:    config.ClusterDomain(),
+		HostedZoneRole:   config.AWS.HostedZoneRole,
 	}
 }
 
@@ -79,7 +81,8 @@ func tagSharedVPCResources(ctx context.Context, clusterID string, installConfig
 	}
 
 	if zone := installConfig.Config.AWS.HostedZone; zone != "" {
-		route53Client := route53.New(session)
+		r53cfg := awsic.GetR53ClientCfg(session, installConfig.Config.AWS.HostedZoneRole)
+		route53Client := route53.New(session, r53cfg)
 		if _, err := route53Client.ChangeTagsForResourceWithContext(ctx, &route53.ChangeTagsForResourceInput{
 			ResourceType: aws.String("hostedzone"),
 			ResourceId:   aws.String(zone),

pkg/asset/cluster/tfvars.go

@@ -292,6 +292,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			PrivateSubnets:        privateSubnets,
 			PublicSubnets:         publicSubnets,
 			InternalZone:          installConfig.Config.AWS.HostedZone,
+			InternalZoneRole:      installConfig.Config.AWS.HostedZoneRole,
 			Services:              installConfig.Config.AWS.ServiceEndpoints,
 			Publish:               installConfig.Config.Publish,
 			MasterConfigs:         masterConfigs,

pkg/asset/installconfig/aws/mock/awsroute53_generated.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	awss "github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/route53"
 	"github.com/pkg/errors"
@@ -17,10 +18,10 @@ import (
 
 // API represents the calls made to the API.
 type API interface {
-	GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput, error)
-	ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig) field.ErrorList
+	GetHostedZone(hostedZone string, cfg *aws.Config) (*route53.GetHostedZoneOutput, error)
+	ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig, cfg *aws.Config) field.ErrorList
 	GetBaseDomain(baseDomainName string) (*route53.HostedZone, error)
-	GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig) ([]string, error)
+	GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig, cfg *aws.Config) ([]string, error)
 }
 
 // Client makes calls to the AWS Route53 API.
@@ -37,9 +38,9 @@ func NewClient(ssn *awss.Session) *Client {
 }
 
 // GetHostedZone attempts to get the hosted zone from the AWS Route53 instance
-func (c *Client) GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput, error) {
+func (c *Client) GetHostedZone(hostedZone string, cfg *aws.Config) (*route53.GetHostedZoneOutput, error) {
 	// build a new Route53 instance from the same session that made it here
-	r53 := route53.New(c.ssn)
+	r53 := route53.New(c.ssn, cfg)
 
 	// validate that the hosted zone exists
 	hostedZoneOutput, err := r53.GetHostedZone(&route53.GetHostedZoneInput{Id: aws.String(hostedZone)})
@@ -50,10 +51,10 @@ func (c *Client) GetHostedZone(hostedZone string) (*route53.GetHostedZoneOutput,
 }
 
 // ValidateZoneRecords Attempts to validate each of the candidate HostedZones against the Config
-func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig) field.ErrorList {
+func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string, zonePath *field.Path, ic *types.InstallConfig, cfg *aws.Config) field.ErrorList {
 	allErrs := field.ErrorList{}
 
-	problematicRecords, err := c.GetSubDomainDNSRecords(zone, ic)
+	problematicRecords, err := c.GetSubDomainDNSRecords(zone, ic, cfg)
 	if err != nil {
 		allErrs = append(allErrs, field.InternalError(zonePath,
 			errors.Wrapf(err, "could not list record sets for domain %q", zoneName)))
@@ -72,15 +73,15 @@ func (c *Client) ValidateZoneRecords(zone *route53.HostedZone, zoneName string,
 
 // GetSubDomainDNSRecords Validates the hostedZone against the cluster domain, and ensures that the
 // cluster domain does not have a current record set for the hostedZone
-func (c *Client) GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig) ([]string, error) {
+func (c *Client) GetSubDomainDNSRecords(hostedZone *route53.HostedZone, ic *types.InstallConfig, cfg *aws.Config) ([]string, error) {
 	dottedClusterDomain := ic.ClusterDomain() + "."
 
 	// validate that the domain of the hosted zone is the cluster domain or a parent of the cluster domain
 	if !isHostedZoneDomainParentOfClusterDomain(hostedZone, dottedClusterDomain) {
 		return nil, errors.Errorf("hosted zone domain %q is not a parent of the cluster domain %q", *hostedZone.Name, dottedClusterDomain)
 	}
 
-	r53 := route53.New(c.ssn)
+	r53 := route53.New(c.ssn, cfg)
 
 	var problematicRecords []string
 	// validate that the hosted zone does not already have any record sets for the cluster domain
@@ -134,3 +135,14 @@ func (c *Client) GetBaseDomain(baseDomainName string) (*route53.HostedZone, erro
 	}
 	return baseDomainZone, nil
 }
+
+// GetR53ClientCfg creates a config for the route53 client by determining
+// whether it is needed to obtain STS assume role credentials.
+func GetR53ClientCfg(sess *awss.Session, roleARN string) *aws.Config {
+	if roleARN == "" {
+		return nil
+	}
+
+	creds := stscreds.NewCredentials(sess, roleARN)
+	return &aws.Config{Credentials: creds}
+}

pkg/asset/installconfig/aws/route53.go

@@ -385,21 +385,22 @@ func ValidateForProvisioning(client API, ic *types.InstallConfig, metadata *Meta
 	var zonePath *field.Path
 	var zone *route53.HostedZone
 
-	errors := field.ErrorList{}
 	allErrs := field.ErrorList{}
+	r53cfg := GetR53ClientCfg(metadata.session, ic.AWS.HostedZoneRole)
 
 	if ic.AWS.HostedZone != "" {
 		zoneName = ic.AWS.HostedZone
 		zonePath = field.NewPath("aws", "hostedZone")
-		zoneOutput, err := client.GetHostedZone(zoneName)
+		zoneOutput, err := client.GetHostedZone(zoneName, r53cfg)
 		if err != nil {
+			errMsg := errors.Wrapf(err, "unable to retrieve hosted zone").Error()
 			return field.ErrorList{
-				field.Invalid(zonePath, zoneName, "cannot find hosted zone"),
+				field.Invalid(zonePath, zoneName, errMsg),
 			}.ToAggregate()
 		}
 
-		if errors = validateHostedZone(zoneOutput, zonePath, zoneName, metadata); len(errors) > 0 {
-			allErrs = append(allErrs, errors...)
+		if errs := validateHostedZone(zoneOutput, zonePath, zoneName, metadata); len(errs) > 0 {
+			allErrs = append(allErrs, errs...)
 		}
 
 		zone = zoneOutput.HostedZone
@@ -416,8 +417,8 @@ func ValidateForProvisioning(client API, ic *types.InstallConfig, metadata *Meta
 		zone = baseDomainOutput
 	}
 
-	if errors = client.ValidateZoneRecords(zone, zoneName, zonePath, ic); len(errors) > 0 {
-		allErrs = append(allErrs, errors...)
+	if errs := client.ValidateZoneRecords(zone, zoneName, zonePath, ic, r53cfg); len(errs) > 0 {
+		allErrs = append(allErrs, errs...)
 	}
 
 	return allErrs.ToAggregate()