Merge pull request #8014 from barbacbd/CORS-3259

CORS-3259: GCP: Create DNS records and internal load balancer for CAPG Install

Author: openshift-merge-bot[bot]

pkg/infrastructure/gcp/clusterapi/clusterapi.go

@@ -2,8 +2,15 @@ package clusterapi
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/sirupsen/logrus"
+	capg "sigs.k8s.io/cluster-api-provider-gcp/api/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/openshift/installer/pkg/asset/manifests/capiutils"
 	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+	"github.com/openshift/installer/pkg/types"
 	gcptypes "github.com/openshift/installer/pkg/types/gcp"
 )
 
@@ -41,6 +48,50 @@ func (p Provider) Ignition(ctx context.Context, in clusterapi.IgnitionInput) ([]
 // is true, typically after load balancers have been provisioned. It can be used
 // to create DNS records.
 func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) error {
+	gcpCluster := &capg.GCPCluster{}
+	key := client.ObjectKey{
+		Name:      in.InfraID,
+		Namespace: capiutils.Namespace,
+	}
+	if err := in.Client.Get(ctx, key, gcpCluster); err != nil {
+		return fmt.Errorf("failed to get GCP cluster: %w", err)
+	}
+
+	// public load balancer is created by CAPG. The health check for this load balancer is also created by
+	// the CAPG.
+	apiIPAddress := gcpCluster.Spec.ControlPlaneEndpoint.Host
+	if apiIPAddress == "" && in.InstallConfig.Config.Publish == types.ExternalPublishingStrategy {
+		logrus.Debugf("publish strategy is set to external but api address is empty")
+	}
+
+	// Currently, the internal/private load balancer is not created by CAPG. The load balancer will be created
+	// by the installer for now
+	// TODO: remove the creation of the LB and health check here when supported by CAPG.
+	// https://github.com/kubernetes-sigs/cluster-api-provider-gcp/issues/903
+	// Create the public (optional) and private load balancer static ip addresses
+	// TODO: Do we then need to setup a subnet for internal load balancing ?
+	apiIntIPAddress, err := createInternalLBAddress(ctx, in)
+	if err != nil {
+		return fmt.Errorf("failed to create internal load balancer address: %w", err)
+	}
+
+	if in.InstallConfig.Config.GCP.UserProvisionedDNS != gcptypes.UserProvisionedDNSEnabled {
+		// Get the network from the GCP Cluster. The network is used to create the private managed zone.
+		if gcpCluster.Status.Network.SelfLink == nil {
+			return fmt.Errorf("failed to get GCP network: %w", err)
+		}
+
+		// Create the private zone if one does not exist
+		if err := createPrivateManagedZone(ctx, in.InstallConfig, in.InfraID, *gcpCluster.Status.Network.SelfLink); err != nil {
+			return fmt.Errorf("failed to create the private managed zone: %w", err)
+		}
+
+		// Create the public (optional) and private dns records
+		if err := createDNSRecords(ctx, in.InstallConfig, in.InfraID, apiIPAddress, apiIntIPAddress); err != nil {
+			return fmt.Errorf("failed to create DNS records: %w", err)
+		}
+	}
+
 	return nil
 }
 

pkg/infrastructure/gcp/clusterapi/dns.go

@@ -0,0 +1,175 @@
+package clusterapi
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"google.golang.org/api/dns/v1"
+
+	"github.com/openshift/installer/pkg/asset/installconfig"
+	gcpic "github.com/openshift/installer/pkg/asset/installconfig/gcp"
+	"github.com/openshift/installer/pkg/types"
+)
+
+var (
+	errNotFound = errors.New("not found")
+)
+
+func getDNSZoneName(ctx context.Context, ic *installconfig.InstallConfig, isPublic bool) (string, error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
+	defer cancel()
+
+	client, err := gcpic.NewClient(ctx)
+	if err != nil {
+		return "", fmt.Errorf("failed to create new client: %w", err)
+	}
+
+	cctx, ccancel := context.WithTimeout(ctx, time.Minute*1)
+	defer ccancel()
+
+	domain := ic.Config.ClusterDomain()
+	if isPublic {
+		domain = ic.Config.BaseDomain
+	}
+
+	zone, err := client.GetDNSZone(cctx, ic.Config.GCP.ProjectID, domain, isPublic)
+	if err != nil {
+		return "", fmt.Errorf("failed to get dns zone name: %w", err)
+	}
+
+	if zone != nil {
+		return zone.Name, nil
+	}
+
+	return "", errNotFound
+}
+
+type recordSet struct {
+	projectID string
+	zoneName  string
+	record    *dns.ResourceRecordSet
+}
+
+// createRecordSets will create a list of records that will be created during the install.
+func createRecordSets(ctx context.Context, ic *installconfig.InstallConfig, clusterID, apiIP, apiIntIP string) ([]recordSet, error) {
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
+	defer cancel()
+
+	// A shared VPC install allows a user to preconfigure a private zone. If there is a private zone found,
+	// use the existing private zone otherwise default to the installer private zone pattern below.
+	privateZoneName, err := getDNSZoneName(ctx, ic, false)
+	if err != nil {
+		if !errors.Is(err, errNotFound) {
+			return nil, fmt.Errorf("failed to find private zone: %w", err)
+		}
+		privateZoneName = fmt.Sprintf("%s-private-zone", clusterID)
+	}
+
+	records := []recordSet{
+		{
+			// api_internal
+			projectID: ic.Config.GCP.ProjectID,
+			zoneName:  privateZoneName,
+			record: &dns.ResourceRecordSet{
+				Name:    fmt.Sprintf("api-int.%s.", ic.Config.ClusterDomain()),
+				Type:    "A",
+				Ttl:     60,
+				Rrdatas: []string{apiIntIP},
+			},
+		},
+		{
+			// api_external_internal_zone
+			projectID: ic.Config.GCP.ProjectID,
+			zoneName:  privateZoneName,
+			record: &dns.ResourceRecordSet{
+				Name:    fmt.Sprintf("api.%s.", ic.Config.ClusterDomain()),
+				Type:    "A",
+				Ttl:     60,
+				Rrdatas: []string{apiIntIP},
+			},
+		},
+	}
+
+	if ic.Config.Publish == types.ExternalPublishingStrategy {
+		existingPublicZoneName, err := getDNSZoneName(ctx, ic, true)
+		if err != nil {
+			return nil, fmt.Errorf("failed to find a public zone: %w", err)
+		}
+
+		apiRecord := recordSet{
+			projectID: ic.Config.GCP.ProjectID,
+			zoneName:  existingPublicZoneName,
+			record: &dns.ResourceRecordSet{
+				Name:    fmt.Sprintf("api.%s.", ic.Config.ClusterDomain()),
+				Type:    "A",
+				Ttl:     60,
+				Rrdatas: []string{apiIP},
+			},
+		}
+		records = append(records, apiRecord)
+	}
+
+	return records, nil
+}
+
+// createDNSRecords will get the list of records to be created and execute their creation through the gcp dns api.
+func createDNSRecords(ctx context.Context, ic *installconfig.InstallConfig, clusterID, apiIP, apiIntIP string) error {
+	// TODO: use the opts for the service to restrict scopes see google.golang.org/api/option.WithScopes
+	dnsService, err := dns.NewService(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create the gcp dns service: %w", err)
+	}
+
+	records, err := createRecordSets(ctx, ic, clusterID, apiIP, apiIntIP)
+	if err != nil {
+		return err
+	}
+
+	// 1 minute timeout for each record
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*time.Duration(len(records)))
+	defer cancel()
+
+	for _, record := range records {
+		if _, err := dnsService.ResourceRecordSets.Create(record.projectID, record.zoneName, record.record).Context(ctx).Do(); err != nil {
+			return fmt.Errorf("failed to create record set %s: %w", record.record.Name, err)
+		}
+	}
+
+	return nil
+}
+
+// createPrivateManagedZone will create a private managed zone in the GCP project specified in the install config. The
+// private managed zone should only be created when one is not specified in the install config.
+func createPrivateManagedZone(ctx context.Context, ic *installconfig.InstallConfig, clusterID, network string) error {
+	// TODO: use the opts for the service to restrict scopes see google.golang.org/api/option.WithScopes
+	dnsService, err := dns.NewService(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to create the gcp dns service: %w", err)
+	}
+
+	managedZone := &dns.ManagedZone{
+		Name:        fmt.Sprintf("%s-private-zone", clusterID),
+		Description: resourceDescription,
+		DnsName:     fmt.Sprintf("%s.", ic.Config.ClusterDomain()),
+		Visibility:  "private",
+		Labels:      mergeLabels(ic, clusterID),
+		PrivateVisibilityConfig: &dns.ManagedZonePrivateVisibilityConfig{
+			Networks: []*dns.ManagedZonePrivateVisibilityConfigNetwork{
+				{
+					NetworkUrl: network,
+				},
+			},
+		},
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
+	defer cancel()
+
+	if _, err = dnsService.ManagedZones.Create(ic.Config.GCP.ProjectID, managedZone).Context(ctx).Do(); err != nil {
+		return fmt.Errorf("failed to create private managed zone: %w", err)
+	}
+
+	return nil
+}

pkg/infrastructure/gcp/clusterapi/gcp.go

@@ -0,0 +1,5 @@
+package clusterapi
+
+const (
+	resourceDescription = "Created By OpenShift Installer"
+)

pkg/infrastructure/gcp/clusterapi/labels.go

@@ -0,0 +1,17 @@
+package clusterapi
+
+import (
+	"fmt"
+
+	"github.com/openshift/installer/pkg/asset/installconfig"
+)
+
+func mergeLabels(ic *installconfig.InstallConfig, clusterID string) map[string]string {
+	labels := map[string]string{}
+	labels[fmt.Sprintf("kubernetes-io-cluster-%s", clusterID)] = "owned"
+	for _, label := range ic.Config.GCP.UserLabels {
+		labels[label.Key] = label.Value
+	}
+
+	return labels
+}

pkg/infrastructure/gcp/clusterapi/network.go

@@ -0,0 +1,93 @@
+package clusterapi
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"google.golang.org/api/compute/v1"
+
+	"github.com/openshift/installer/pkg/infrastructure/clusterapi"
+)
+
+func getAPIInternalResourceName(infraID string) string {
+	return fmt.Sprintf("%s-api-internal", infraID)
+}
+
+func getAPIAddressName(infraID string) string {
+	return fmt.Sprintf("%s-cluster-ip", infraID)
+}
+
+func getInternalLBAddress(ctx context.Context, project, region, name string) (string, error) {
+	service, err := NewComputeService()
+	if err != nil {
+		return "", err
+	}
+
+	addrOutput, err := service.Addresses.Get(project, region, name).Context(ctx).Do()
+	if err != nil {
+		return "", fmt.Errorf("failed to get compute address %s: %w", name, err)
+	}
+	return addrOutput.Address, nil
+}
+
+// createInternalLBAddress creates a static ip address for the internal load balancer.
+func createInternalLBAddress(ctx context.Context, in clusterapi.InfraReadyInput) (string, error) {
+	name := getAPIAddressName(in.InfraID)
+
+	// TODO: Find the self link for a subnet from the in.Client
+	// TODO: can we pick one returned from the service.Get() ?
+	subnetSelfLink := ""
+
+	// TODO: the subnet is only relevant for internal load balancer
+	addr := &compute.Address{
+		Name:        name,
+		AddressType: "INTERNAL",
+		Subnetwork:  subnetSelfLink,
+		Description: resourceDescription,
+		Labels:      mergeLabels(in.InstallConfig, in.InfraID),
+		Region:      in.InstallConfig.Config.GCP.Region,
+	}
+
+	service, err := NewComputeService()
+	if err != nil {
+		return "", err
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, time.Minute*1)
+	defer cancel()
+
+	op, err := service.Addresses.Insert(in.InstallConfig.Config.GCP.ProjectID, in.InstallConfig.Config.GCP.Region, addr).Context(ctx).Do()
+	if err != nil {
+		return "", fmt.Errorf("failed to create internal compute address: %w", err)
+	}
+
+	if err := WaitForOperationRegional(ctx, in.InstallConfig.Config.GCP.ProjectID, in.InstallConfig.Config.GCP.Region, op); err != nil {
+		return "", fmt.Errorf("failed to wait for compute address creation: %w", err)
+	}
+
+	ipAddress, err := getInternalLBAddress(ctx, in.InstallConfig.Config.GCP.ProjectID, in.InstallConfig.Config.GCP.Region, name)
+	if err != nil {
+		return "", fmt.Errorf("failed to get internal load balancer IP address: %w", err)
+	}
+
+	healthCheck := &compute.HealthCheck{
+		Name:               getAPIInternalResourceName(in.InfraID),
+		Description:        resourceDescription,
+		HealthyThreshold:   3,
+		UnhealthyThreshold: 3,
+		CheckIntervalSec:   2,
+		TimeoutSec:         2,
+		Type:               "HTTPS",
+		HttpsHealthCheck: &compute.HTTPSHealthCheck{
+			Port:        6443,
+			RequestPath: "/readyz",
+		},
+	}
+
+	if _, err := service.HealthChecks.Insert(in.InstallConfig.Config.GCP.ProjectID, healthCheck).Context(ctx).Do(); err != nil {
+		return "", fmt.Errorf("failed to create api-internal health check: %w", err)
+	}
+
+	return ipAddress, nil
+}