Add IPI installation on AWS dedicated hosts

Author: vr4manta

data/data/install.openshift.io_installconfigs.yaml

@@ -0,0 +1,48 @@
+package aws
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/sirupsen/logrus"
+)
+
+// Host holds metadata for a dedicated host.
+type Host struct {
+	ID   string
+	Name string
+	Zone string
+}
+
+// dedicatedHosts retrieves a list of dedicated hosts for the given region and
+// returns them in a map keyed by the host ID.
+func dedicatedHosts(ctx context.Context, session *session.Session, region string) (map[string]Host, error) {
+	hostsByID := map[string]Host{}
+
+	client := ec2.New(session, aws.NewConfig().WithRegion(region))
+	input := &ec2.DescribeHostsInput{}
+
+	if err := client.DescribeHostsPagesWithContext(ctx, input, func(page *ec2.DescribeHostsOutput, lastPage bool) bool {
+		for _, h := range page.Hosts {
+			id := aws.StringValue(h.HostId)
+			if id == "" {
+				// Skip entries lacking an ID (should not happen)
+				continue
+			}
+
+			logrus.Debugf("Found dedicatd host: %s", id)
+			hostsByID[id] = Host{
+				ID:   id,
+				Zone: aws.StringValue(h.AvailabilityZone),
+			}
+		}
+		return !lastPage
+	}); err != nil {
+		return nil, fmt.Errorf("fetching dedicated hosts: %w", err)
+	}
+
+	return hostsByID, nil
+}

pkg/asset/installconfig/aws/dedicatedhosts.go

@@ -27,6 +27,7 @@ type Metadata struct {
 	vpc               VPC
 	instanceTypes     map[string]InstanceType
 
+	Hosts           map[string]Host
 	Region          string                     `json:"region,omitempty"`
 	ProvidedSubnets []typesaws.Subnet          `json:"subnets,omitempty"`
 	Services        []typesaws.ServiceEndpoint `json:"services,omitempty"`
@@ -390,3 +391,23 @@ func (m *Metadata) InstanceTypes(ctx context.Context) (map[string]InstanceType,
 
 	return m.instanceTypes, nil
 }
+
+// DedicatedHosts retrieves all hosts available for use to verify against this installation for configured region.
+func (m *Metadata) DedicatedHosts(ctx context.Context) (map[string]Host, error) {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	if len(m.Hosts) == 0 {
+		awsSession, err := m.unlockedSession(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		m.Hosts, err = dedicatedHosts(ctx, awsSession, m.Region)
+		if err != nil {
+			return nil, fmt.Errorf("error listing dedicated hosts: %w", err)
+		}
+	}
+
+	return m.Hosts, nil
+}

pkg/asset/installconfig/aws/metadata.go

@@ -466,6 +466,8 @@ func validateMachinePool(ctx context.Context, meta *Metadata, fldPath *field.Pat
 		}
 	}
 
+	allErrs = append(allErrs, validateHostPlacement(ctx, meta, fldPath, pool)...)
+
 	return allErrs
 }
 
@@ -484,6 +486,36 @@ func translateEC2Arches(arches []string) sets.Set[string] {
 	return res
 }
 
+func validateHostPlacement(ctx context.Context, meta *Metadata, fldPath *field.Path, pool *awstypes.MachinePool) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if pool.HostPlacement == nil {
+		return allErrs
+	}
+
+	if pool.HostPlacement.Affinity != nil && *pool.HostPlacement.Affinity == awstypes.HostAffinityDedicatedHost {
+		placementPath := fldPath.Child("hostPlacement")
+		if pool.HostPlacement.DedicatedHost != nil {
+			configuredHosts := pool.HostPlacement.DedicatedHost
+			foundHosts, err := meta.DedicatedHosts(ctx)
+			if err != nil {
+				allErrs = append(allErrs, field.InternalError(placementPath.Child("dedicatedHost"), err))
+			} else {
+				// Check the returned configured hosts to see if the dedicated hosts defined in install-config exists.
+				for _, host := range configuredHosts {
+					_, ok := foundHosts[host.ID]
+					if !ok {
+						errMsg := fmt.Sprintf("dedicated host %s not found", host.ID)
+						allErrs = append(allErrs, field.Invalid(placementPath.Child("dedicatedHost"), pool.InstanceType, errMsg))
+					}
+				}
+			}
+		}
+	}
+
+	return allErrs
+}
+
 func validateSecurityGroupIDs(ctx context.Context, meta *Metadata, fldPath *field.Path, platform *awstypes.Platform, pool *awstypes.MachinePool) field.ErrorList {
 	allErrs := field.ErrorList{}
 

pkg/asset/installconfig/aws/validation.go

@@ -11,6 +11,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
 
 	v1 "github.com/openshift/api/config/v1"
 	machinev1 "github.com/openshift/api/machine/v1"
@@ -35,6 +36,7 @@ type machineProviderInput struct {
 	userTags         map[string]string
 	publicSubnet     bool
 	securityGroupIDs []string
+	dedicatedHost    string
 }
 
 // Machines returns a list of machines for a machinepool.
@@ -291,6 +293,15 @@ func provider(in *machineProviderInput) (*machineapi.AWSMachineProviderConfig, e
 		config.MetadataServiceOptions.Authentication = machineapi.MetadataServiceAuthentication(in.imds.Authentication)
 	}
 
+	if in.dedicatedHost != "" {
+		config.HostPlacement = &machineapi.HostPlacement{
+			Affinity: ptr.To(machineapi.HostAffinityDedicatedHost),
+			DedicatedHost: &machineapi.DedicatedHost{
+				ID: in.dedicatedHost,
+			},
+		}
+	}
+
 	return config, nil
 }
 
@@ -340,3 +351,18 @@ func ConfigMasters(machines []machineapi.Machine, controlPlane *machinev1.Contro
 	providerSpec := controlPlane.Spec.Template.OpenShiftMachineV1Beta1Machine.Spec.ProviderSpec.Value.Object.(*machineapi.AWSMachineProviderConfig)
 	providerSpec.LoadBalancers = lbrefs
 }
+
+// DedicatedHost sets dedicated hosts for the specified zone.
+func DedicatedHost(hosts map[string]aws.Host, placement *awstypes.HostPlacement, zone string) string {
+	// If install-config has HostPlacements configured, lets check the DedicatedHosts to see if one matches our region & zone.
+	if placement != nil {
+		// We only support one host ID currently for an instance.  Need to also get host that matches the zone the machines will be put into.
+		for _, host := range placement.DedicatedHost {
+			hostDetails, found := hosts[host.ID]
+			if found && hostDetails.Zone == zone {
+				return hostDetails.ID
+			}
+		}
+	}
+	return ""
+}

pkg/asset/machines/aws/machines.go

@@ -25,6 +25,7 @@ type MachineSetInput struct {
 	Pool                     *types.MachinePool
 	Role                     string
 	UserDataSecret           string
+	Hosts                    map[string]icaws.Host
 }
 
 // MachineSets returns a list of machinesets for a machinepool.
@@ -87,6 +88,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			instanceProfile = fmt.Sprintf("%s-worker-profile", in.ClusterID)
 		}
 
+		dedicatedHost := DedicatedHost(in.Hosts, mpool.HostPlacement, az)
+
 		provider, err := provider(&machineProviderInput{
 			clusterID:        in.ClusterID,
 			region:           in.InstallConfigPlatformAWS.Region,
@@ -102,12 +105,21 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 			userTags:         in.InstallConfigPlatformAWS.UserTags,
 			publicSubnet:     publicSubnet,
 			securityGroupIDs: in.Pool.Platform.AWS.AdditionalSecurityGroupIDs,
+			dedicatedHost:    dedicatedHost,
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create provider")
 		}
+
+		// If we are using any feature that is only available via CAPI, we must set the authoritativeAPI = ClusterAPI
+		authoritativeAPI := machineapi.MachineAuthorityMachineAPI
+		if isAuthoritativeClusterAPIRequired(provider) {
+			authoritativeAPI = machineapi.MachineAuthorityClusterAPI
+		}
+
 		name := fmt.Sprintf("%s-%s-%s", in.ClusterID, in.Pool.Name, az)
 		spec := machineapi.MachineSpec{
+			AuthoritativeAPI: authoritativeAPI,
 			ProviderSpec: machineapi.ProviderSpec{
 				Value: &runtime.RawExtension{Object: provider},
 			},
@@ -130,7 +142,8 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 			Spec: machineapi.MachineSetSpec{
-				Replicas: &replicas,
+				AuthoritativeAPI: authoritativeAPI,
+				Replicas:         &replicas,
 				Selector: metav1.LabelSelector{
 					MatchLabels: map[string]string{
 						"machine.openshift.io/cluster-api-machineset": name,
@@ -151,8 +164,17 @@ func MachineSets(in *MachineSetInput) ([]*machineapi.MachineSet, error) {
 				},
 			},
 		}
+
 		machinesets = append(machinesets, mset)
 	}
 
 	return machinesets, nil
 }
+
+// isAuthoritativeClusterAPIRequired is called to determine if the machine spec should have the AuthoritativeAPI set to ClusterAPI.
+func isAuthoritativeClusterAPIRequired(provider *machineapi.AWSMachineProviderConfig) bool {
+	if provider.HostPlacement != nil && *provider.HostPlacement.Affinity != machineapi.HostAffinityAnyAvailable {
+		return true
+	}
+	return false
+}

pkg/asset/machines/aws/machinesets.go

@@ -533,6 +533,13 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				}
 			}
 
+			// TODO: See if we can make changes mentioned in the review.
+			// Depending on how this func is called, sometimes AWS instance is not set.  So in this case, assume no dedicated hosts
+			dHosts := map[string]icaws.Host{}
+			if installConfig.AWS != nil {
+				dHosts = installConfig.AWS.Hosts
+			}
+
 			pool.Platform.AWS = &mpool
 			sets, err := aws.MachineSets(&aws.MachineSetInput{
 				ClusterID:                clusterID.InfraID,
@@ -543,6 +550,7 @@ func (w *Worker) Generate(ctx context.Context, dependencies asset.Parents) error
 				Pool:                     &pool,
 				Role:                     pool.Name,
 				UserDataSecret:           workerUserDataSecretName,
+				Hosts:                    dHosts,
 			})
 			if err != nil {
 				return errors.Wrap(err, "failed to create worker machine objects")

pkg/asset/machines/worker.go

@@ -48,6 +48,14 @@ type MachinePool struct {
 	// +kubebuilder:validation:MaxItems=10
 	// +optional
 	AdditionalSecurityGroupIDs []string `json:"additionalSecurityGroupIDs,omitempty"`
+
+	// hostPlacement configures placement on AWS Dedicated Hosts. This allows admins to assign instances to specific host
+	// for a variety of needs including for regulatory compliance, to leverage existing per-socket or per-core software licenses (BYOL),
+	// and to gain visibility and control over instance placement on a physical server.
+	// When omitted, the instance is not constrained to a dedicated host.
+	// +openshift:enable:FeatureGate=AWSDedicatedHosts
+	// +optional
+	HostPlacement *HostPlacement `json:"hostPlacement,omitempty"`
 }
 
 // Set sets the values from `required` to `a`.
@@ -96,6 +104,10 @@ func (a *MachinePool) Set(required *MachinePool) {
 	if len(required.AdditionalSecurityGroupIDs) > 0 {
 		a.AdditionalSecurityGroupIDs = required.AdditionalSecurityGroupIDs
 	}
+
+	if required.HostPlacement != nil {
+		a.HostPlacement = required.HostPlacement
+	}
 }
 
 // EC2RootVolume defines the storage for an ec2 instance.
@@ -135,3 +147,50 @@ type EC2Metadata struct {
 	// +optional
 	Authentication string `json:"authentication,omitempty"`
 }
+
+// HostPlacement is the type that will be used to configure the placement of AWS instances.
+// This can be configured for default placement (AnyAvailable) and dedicated hosts (DedicatedHost).
+// +kubebuilder:validation:XValidation:rule="has(self.affinity) && self.affinity == 'DedicatedHost' ?  has(self.dedicatedHost) : !has(self.dedicatedHost)",message="dedicatedHost is required when affinity is DedicatedHost, and forbidden otherwise"
+type HostPlacement struct {
+	// affinity specifies the affinity setting for the instance.
+	// Allowed values are AnyAvailable and DedicatedHost.
+	// When Affinity is set to DedicatedHost, an instance started onto a specific host always restarts on the same host if stopped. In this scenario, the `dedicatedHost` field must be set.
+	// When Affinity is set to AnyAvailable, and you stop and restart the instance, it can be restarted on any available host.
+	// +required
+	// +unionDiscriminator
+	Affinity *HostAffinity `json:"affinity,omitempty"`
+
+	// dedicatedHost specifies the exact host that an instance should be restarted on if stopped.
+	// dedicatedHost is required when 'affinity' is set to DedicatedHost, and forbidden otherwise.
+	// +optional
+	// +unionMember
+	DedicatedHost []DedicatedHost `json:"dedicatedHost,omitempty"`
+}
+
+// HostAffinity selects how an instance should be placed on AWS Dedicated Hosts.
+// +kubebuilder:validation:Enum:=DedicatedHost;AnyAvailable
+type HostAffinity string
+
+const (
+	// HostAffinityAnyAvailable lets the platform select any available dedicated host.
+	HostAffinityAnyAvailable HostAffinity = "AnyAvailable"
+
+	// HostAffinityDedicatedHost requires specifying a particular host via dedicatedHost.host.hostID.
+	HostAffinityDedicatedHost HostAffinity = "DedicatedHost"
+)
+
+// DedicatedHost represents the configuration for the usage of dedicated host.
+type DedicatedHost struct {
+	// id identifies the AWS Dedicated Host on which the instance must run.
+	// The value must start with "h-" followed by 17 lowercase hexadecimal characters (0-9 and a-f).
+	// Must be exactly 19 characters in length.
+	// +kubebuilder:validation:XValidation:rule="self.matches('^h-[0-9a-f]{17}$')",message="hostID must start with 'h-' followed by 17 lowercase hexadecimal characters (0-9 and a-f)"
+	// +kubebuilder:validation:MinLength=19
+	// +kubebuilder:validation:MaxLength=19
+	// +required
+	ID string `json:"id,omitempty"`
+
+	// zone is the availability zone that the dedicated host belongs to
+	// +optional
+	Zone string `json:"zone,omitempty"`
+}