Add IPI installation on AWS dedicated hosts

Author: vr4manta

data/data/install.openshift.io_installconfigs.yaml

@@ -0,0 +1,47 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/sirupsen/logrus"
+)
+
+// Host holds metadata for a dedicated host.
+type Host struct {
+	ID   string
+	Zone string
+}
+
+// dedicatedHosts retrieves a list of dedicated hosts for the given region and
+// returns them in a map keyed by the host ID.
+func dedicatedHosts(ctx context.Context, session *session.Session, region string) (map[string]Host, error) {
+	hostsByID := map[string]Host{}
+
+	client := ec2.New(session, aws.NewConfig().WithRegion(region))
+	input := &ec2.DescribeHostsInput{}
+
+	if err := client.DescribeHostsPagesWithContext(ctx, input, func(page *ec2.DescribeHostsOutput, lastPage bool) bool {
+		for _, h := range page.Hosts {
+			id := aws.StringValue(h.HostId)
+			if id == "" {
+				// Skip entries lacking an ID (should not happen)
+				continue
+			}
+
+			logrus.Debugf("Found dedicatd host: %s", id)
+			hostsByID[id] = Host{
+				ID:   id,
+				Zone: aws.StringValue(h.AvailabilityZone),
+			}
+		}
+		return !lastPage
+	}); err != nil {
+		return nil, fmt.Errorf("fetching dedicated hosts: %w", err)
+	}
+
+	return hostsByID, nil
+}

pkg/asset/installconfig/aws/dedicatedhosts.go

@@ -27,6 +27,7 @@ type Metadata struct {
 	vpc               VPC
 	instanceTypes     map[string]InstanceType
 
+	Hosts           map[string]Host
 	Region          string                     `json:"region,omitempty"`
 	ProvidedSubnets []typesaws.Subnet          `json:"subnets,omitempty"`
 	Services        []typesaws.ServiceEndpoint `json:"services,omitempty"`
@@ -390,3 +391,23 @@ func (m *Metadata) InstanceTypes(ctx context.Context) (map[string]InstanceType,
 
 	return m.instanceTypes, nil
 }
+
+// DedicatedHosts retrieves all hosts available for use to verify against this installation for configured region.
+func (m *Metadata) DedicatedHosts(ctx context.Context) (map[string]Host, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if len(m.Hosts) == 0 {
+		awsSession, err := m.unlockedSession(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		m.Hosts, err = dedicatedHosts(ctx, awsSession, m.Region)
+		if err != nil {
+			return nil, fmt.Errorf("error listing dedicated hosts: %w", err)
+		}
+	}
+
+	return m.Hosts, nil
+}

pkg/asset/installconfig/aws/metadata.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"net/url"
+	"slices"
 	"sort"
 
 	ec2v2 "github.com/aws/aws-sdk-go-v2/service/ec2"
@@ -466,6 +467,8 @@ func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Pat
 		}
 	}
 
+	allErrs = append(allErrs, validateHostPlacement(ctx, meta, fldPath, pool)...)
+
 	return allErrs
 }
 
@@ -484,6 +487,54 @@ func translateEC2Arches(arches []string) sets.Set[string] {
 	return res
 }
 
+func validateHostPlacement(ctx context.Context, meta *Metadata, fldPath *field.Path, pool *awstypes.MachinePool) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if pool.HostPlacement == nil {
+		return allErrs
+	}
+
+	if pool.HostPlacement.Affinity != nil && *pool.HostPlacement.Affinity == awstypes.HostAffinityDedicatedHost {
+		placementPath := fldPath.Child("hostPlacement")
+		if pool.HostPlacement.DedicatedHost != nil {
+			configuredHosts := pool.HostPlacement.DedicatedHost
+
+			// Check to see if all configured hosts exist
+			foundHosts, err := meta.DedicatedHosts(ctx)
+			if err != nil {
+				allErrs = append(allErrs, field.InternalError(placementPath.Child("dedicatedHost"), err))
+			} else {
+				// Check the returned configured hosts to see if the dedicated hosts defined in install-config exists.
+				for idx, host := range configuredHosts {
+					dhPath := placementPath.Child("dedicatedHost").Index(idx)
+
+					// Is host in AWS?
+					foundHost, ok := foundHosts[host.ID]
+					if !ok {
+						errMsg := fmt.Sprintf("dedicated host %s not found", host.ID)
+						allErrs = append(allErrs, field.Invalid(dhPath, host, errMsg))
+						continue
+					}
+
+					// Is host valid for pools region and zone config?
+					if !slices.Contains(pool.Zones, foundHost.Zone) {
+						errMsg := fmt.Sprintf("dedicated host %s is not available in pool's zone list", host.ID)
+						allErrs = append(allErrs, field.Invalid(dhPath, host, errMsg))
+					}
+
+					// If user configured the zone for the dedicated host, let's check to make sure its correct
+					if host.Zone != "" && host.Zone != foundHost.Zone {
+						errMsg := fmt.Sprintf("dedicated host was configured with zone %v but expected zone %v", host.Zone, foundHost.Zone)
+						allErrs = append(allErrs, field.Invalid(dhPath.Child("zone"), host, errMsg))
+					}
+				}
+			}
+		}
+	}
+
+	return allErrs
+}
+
 func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/asset/installconfig/aws/validation.go

@@ -74,6 +74,7 @@ func TestValidate(t *testing.T) {
 		subnetsInVPC  *SubnetGroups
 		vpcTags       Tags
 		instanceTypes map[string]InstanceType
+		hosts         map[string]Host
 		proxy         string
 		publicOnly    bool
 		expectErr     string
@@ -1200,6 +1201,57 @@ func TestValidate(t *testing.T) {
 			},
 			expectErr: `^\Qplatform.aws.vpc.subnets: Forbidden: subnet subnet-valid-public-a1 is owned by other clusters [another-cluster] and cannot be used for new installations, another subnet must be created separately\E$`,
 		},
+		{
+			name: "valid dedicated host placement on compute",
+			installConfig: icBuild.build(
+				icBuild.withComputePlatformZones([]string{"a"}, true, 0),
+				icBuild.withComputeHostPlacement([]string{"h-1234567890abcdef0"}, 0),
+			),
+			availRegions: validAvailRegions(),
+			availZones:   validAvailZones(),
+			hosts: map[string]Host{
+				"h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"},
+			},
+		},
+		{
+			name: "invalid dedicated host not found",
+			installConfig: icBuild.build(
+				icBuild.withComputePlatformZones([]string{"a"}, true, 0),
+				icBuild.withComputeHostPlacement([]string{"h-aaaaaaaaaaaaaaaaa"}, 0),
+			),
+			availRegions: validAvailRegions(),
+			availZones:   validAvailZones(),
+			hosts: map[string]Host{
+				"h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"},
+			},
+			expectErr: "dedicated host h-aaaaaaaaaaaaaaaaa not found",
+		},
+		{
+			name: "invalid dedicated host zone not in pool zones",
+			installConfig: icBuild.build(
+				icBuild.withComputePlatformZones([]string{"a"}, true, 0),
+				icBuild.withComputeHostPlacement([]string{"h-bbbbbbbbbbbbbbbbb"}, 0),
+			),
+			availRegions: validAvailRegions(),
+			availZones:   validAvailZones(),
+			hosts: map[string]Host{
+				"h-bbbbbbbbbbbbbbbbb": {ID: "h-bbbbbbbbbbbbbbbbb", Zone: "b"},
+			},
+			expectErr: "is not available in pool's zone list",
+		},
+		{
+			name: "dedicated host placement on compute but for a zone that pool is not using",
+			installConfig: icBuild.build(
+				icBuild.withComputePlatformZones([]string{"b"}, true, 0),
+				icBuild.withComputeHostPlacementAndZone([]string{"h-1234567890abcdef0"}, "b", 0),
+			),
+			availRegions: validAvailRegions(),
+			availZones:   validAvailZones(),
+			hosts: map[string]Host{
+				"h-1234567890abcdef0": {ID: "h-1234567890abcdef0", Zone: "a"},
+			},
+			expectErr: "dedicated host was configured with zone b but expected zone a",
+		},
 	}
 
 	// Register mock http(s) responses for tests.
@@ -1232,6 +1284,7 @@ func TestValidate(t *testing.T) {
 					Tags: test.vpcTags,
 				},
 				instanceTypes:   test.instanceTypes,
+				Hosts:           test.hosts,
 				ProvidedSubnets: test.installConfig.Platform.AWS.VPC.Subnets,
 			}
 
@@ -1952,6 +2005,34 @@ func (icBuild icBuildForAWS) withComputePlatformZones(zones []string, overwrite
 	}
 }
 
+func (icBuild icBuildForAWS) withComputeHostPlacement(hostIDs []string, index int) icOption {
+	return func(ic *types.InstallConfig) {
+		aff := aws.HostAffinityDedicatedHost
+		dhs := make([]aws.DedicatedHost, 0, len(hostIDs))
+		for _, id := range hostIDs {
+			dhs = append(dhs, aws.DedicatedHost{ID: id})
+		}
+		ic.Compute[index].Platform.AWS.HostPlacement = &aws.HostPlacement{
+			Affinity:      &aff,
+			DedicatedHost: dhs,
+		}
+	}
+}
+
+func (icBuild icBuildForAWS) withComputeHostPlacementAndZone(hostIDs []string, zone string, index int) icOption {
+	return func(ic *types.InstallConfig) {
+		aff := aws.HostAffinityDedicatedHost
+		dhs := make([]aws.DedicatedHost, 0, len(hostIDs))
+		for _, id := range hostIDs {
+			dhs = append(dhs, aws.DedicatedHost{ID: id, Zone: zone})
+		}
+		ic.Compute[index].Platform.AWS.HostPlacement = &aws.HostPlacement{
+			Affinity:      &aff,
+			DedicatedHost: dhs,
+		}
+	}
+}
+
 func (icBuild icBuildForAWS) withControlPlanePlatformAMI(amiID string) icOption {
 	return func(ic *types.InstallConfig) {
 		ic.ControlPlane.Platform.AWS.AMIID = amiID

pkg/asset/installconfig/aws/validation_test.go

@@ -37,6 +37,7 @@ type machineProviderInput struct {
 	publicSubnet     bool
 	securityGroupIDs []string
 	cpuOptions       *awstypes.CPUOptions
+	dedicatedHost    string
 }
 
 // Machines returns a list of machines for a machinepool.
@@ -304,6 +305,15 @@ func provider(in *machineProviderInput) (*machineapi.AWSMachineProviderConfig, e
 		config.CPUOptions = &cpuOptions
 	}
 
+	if in.dedicatedHost != "" {
+		config.HostPlacement = &machineapi.HostPlacement{
+			Affinity: ptr.To(machineapi.HostAffinityDedicatedHost),
+			DedicatedHost: &machineapi.DedicatedHost{
+				ID: in.dedicatedHost,
+			},
+		}
+	}
+
 	return config, nil
 }
 
@@ -353,3 +363,18 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro
 	providerSpec := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig)
 	providerSpec.LoadBalancers = lbrefs
 }
+
+// DedicatedHost sets dedicated hosts for the specified zone.
+func DedicatedHost(hosts map[string]aws.Host, placement *awstypes.HostPlacement, zone string) string {
+	// If install-config has HostPlacements configured, lets check the DedicatedHosts to see if one matches our region & zone.
+	if placement != nil {
+		// We only support one host ID currently for an instance.  Need to also get host that matches the zone the machines will be put into.
+		for _, host := range placement.DedicatedHost {
+			hostDetails, found := hosts[host.ID]
+			if found && hostDetails.Zone == zone {
+				return hostDetails.ID
+			}
+		}
+	}
+	return ""
+}

pkg/asset/machines/aws/machines.go

@@ -25,6 +25,7 @@ type MachineSetInput struct {
 	Pool                     *types.MachinePool
 	Role                     string
 	UserDataSecret           string
+	Hosts                    map[string]icaws.Host
 }
 
 // MachineSets returns a list of machinesets for a machinepool.
@@ -87,6 +88,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID)
 		}
 
+		dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az)
+
 		provider, err := provider(&machineProviderInput{
 			clusterID:        in.ClusterID,
 			region:           in.InstallConfigPlatformAWS.Region,
@@ -103,12 +106,21 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			publicSubnet:     publicSubnet,
 			securityGroupIDs: in.Pool.Platform.AWS.AdditionalSecurityGroupIDs,
 			cpuOptions:       mpool.CPUOptions,
+			dedicatedHost:    dedicatedHost,
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create provider")
 		}
+
+		// If we are using any feature that is only available via CAPI, we must set the authoritativeAPI = ClusterAPI
+		authoritativeAPI := machineapi.MachineAuthorityMachineAPI
+		if isAuthoritativeClusterAPIRequired(provider) {
+			authoritativeAPI = machineapi.MachineAuthorityClusterAPI
+		}
+
 		name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az)
 		spec := machineapi.MachineSpec{
+			AuthoritativeAPI: authoritativeAPI,
 			ProviderSpec: machineapi.ProviderSpec{
 				Value: &runtime.RawExtension{Object: provider},
 			},
@@ -131,7 +143,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 			Spec: machineapi.MachineSetSpec{
-				Replicas: &replicas,
+				AuthoritativeAPI: authoritativeAPI,
+				Replicas:         &replicas,
 				Selector: metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						"machine.openshift.io/cluster-api-machineset": name,
@@ -152,8 +165,17 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 		}
+
 		machinesets = append(machinesets, mset)
 	}
 
 	return machinesets, nil
 }
+
+// isAuthoritativeClusterAPIRequired is called to determine if the machine spec should have the AuthoritativeAPI set to ClusterAPI.
+func isAuthoritativeClusterAPIRequired(provider *machineapi.AWSMachineProviderConfig) bool {
+	if provider.HostPlacement != nil && *provider.HostPlacement.Affinity != machineapi.HostAffinityAnyAvailable {
+		return true
+	}
+	return false
+}

pkg/asset/machines/aws/machinesets.go

@@ -534,6 +534,14 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				}
 			}
 
+			dHosts := map[string]icaws.Host{}
+			if pool.Platform.AWS.HostPlacement != nil {
+				dHosts, err = installConfig.AWS.DedicatedHosts(ctx)
+				if err != nil {
+					return fmt.Errorf("failed to retrieve dedicated hosts for compute pool: %w", err)
+				}
+			}
+
 			pool.Platform.AWS = &mpool
 			sets, err := aws.MachineSets(&aws.MachineSetInput{
 				ClusterID:                clusterID.InfraID,
@@ -544,6 +552,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				Pool:                     &pool,
 				Role:                     pool.Name,
 				UserDataSecret:           workerUserDataSecretName,
+				Hosts:                    dHosts,
 			})
 			if err != nil {
 				return errors.Wrap(err, "failed to create worker machine objects")