CORS-2933: IBMCloud: Service endpoint override installconfig

Adding support to allow overrides of IBM Cloud service
endpoints in installconfig. This is a hard requirement for
disconnected cluster support, to allow use of non-default
endpoints for IBM Cloud services.

Related: https://issues.redhat.com/browse/CORS-2933

Author: cjschaef

data/data/install.openshift.io_installconfigs.yaml

@@ -3364,6 +3364,49 @@ spec:
                       resource group where the cluster should be installed. If empty,
                       a new resource group will be created for the cluster.
                     type: string
+                  serviceEndpoints:
+                    description: ServiceEndpoints is a list which contains custom
+                      endpoints to override default service endpoints of IBM Cloud
+                      Services. There must only be one ServiceEndpoint for a service
+                      (no duplicates).
+                    items:
+                      description: IBMCloudServiceEndpoint stores the configuration
+                        of a custom url to override existing defaults of IBM Cloud
+                        Services.
+                      properties:
+                        name:
+                          description: 'name is the name of the IBM Cloud service.
+                            Possible values are: CIS, COS, DNSServices, GlobalSearch,
+                            GlobalTagging, HyperProtect, IAM, KeyProtect, ResourceController,
+                            ResourceManager, or VPC. For example, the IBM Cloud Private
+                            IAM service could be configured with the service `name`
+                            of `IAM` and `url` of `https://private.iam.cloud.ibm.com`
+                            Whereas the IBM Cloud Private VPC service for US South
+                            (Dallas) could be configured with the service `name` of
+                            `VPC` and `url` of `https://us.south.private.iaas.cloud.ibm.com`'
+                          enum:
+                          - CIS
+                          - COS
+                          - DNSServices
+                          - GlobalSearch
+                          - GlobalTagging
+                          - HyperProtect
+                          - IAM
+                          - KeyProtect
+                          - ResourceController
+                          - ResourceManager
+                          - VPC
+                          type: string
+                        url:
+                          description: url is fully qualified URI with scheme https,
+                            that overrides the default generated endpoint for a client.
+                            This must be provided and cannot be empty.
+                          type: string
+                      required:
+                      - name
+                      - url
+                      type: object
+                    type: array
                   vpcName:
                     description: VPCName is the name of an already existing VPC to
                       be used during cluster creation.

pkg/asset/cluster/ibmcloud/ibmcloud.go

@@ -10,7 +10,8 @@ import (
 )
 
 // Metadata converts an install configuration to IBM Cloud metadata.
-func Metadata(infraID string, config *types.InstallConfig, meta *icibmcloud.Metadata) *ibmcloud.Metadata {
+func Metadata(infraID string, config *types.InstallConfig) *ibmcloud.Metadata {
+	meta := icibmcloud.NewMetadata(config)
 	accountID, _ := meta.AccountID(context.TODO())
 	cisCrn, _ := meta.CISInstanceCRN(context.TODO())
 	dnsInstance, _ := meta.DNSInstance(context.TODO())
@@ -40,6 +41,7 @@ func Metadata(infraID string, config *types.InstallConfig, meta *icibmcloud.Meta
 		DNSInstanceID:     dnsInstanceID,
 		Region:            config.Platform.IBMCloud.Region,
 		ResourceGroupName: config.Platform.IBMCloud.ClusterResourceGroupName(infraID),
+		ServiceEndpoints:  config.Platform.IBMCloud.ServiceEndpoints,
 		Subnets:           subnets,
 		VPC:               config.Platform.IBMCloud.GetVPCName(),
 	}

pkg/asset/cluster/metadata.go

@@ -89,7 +89,7 @@ func (m *Metadata) Generate(parents asset.Parents) (err error) {
 	case gcptypes.Name:
 		metadata.ClusterPlatformMetadata.GCP = gcp.Metadata(installConfig.Config)
 	case ibmcloudtypes.Name:
-		metadata.ClusterPlatformMetadata.IBMCloud = ibmcloud.Metadata(clusterID.InfraID, installConfig.Config, installConfig.IBMCloud)
+		metadata.ClusterPlatformMetadata.IBMCloud = ibmcloud.Metadata(clusterID.InfraID, installConfig.Config)
 	case baremetaltypes.Name:
 		metadata.ClusterPlatformMetadata.BareMetal = baremetal.Metadata(installConfig.Config)
 	case ovirttypes.Name:

pkg/asset/cluster/tfvars.go

@@ -34,6 +34,7 @@ import (
 	awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws"
 	aztypes "github.com/openshift/installer/pkg/asset/installconfig/azure"
 	gcpconfig "github.com/openshift/installer/pkg/asset/installconfig/gcp"
+	ibmcloudconfig "github.com/openshift/installer/pkg/asset/installconfig/ibmcloud"
 	ovirtconfig "github.com/openshift/installer/pkg/asset/installconfig/ovirt"
 	powervsconfig "github.com/openshift/installer/pkg/asset/installconfig/powervs"
 	vsphereconfig "github.com/openshift/installer/pkg/asset/installconfig/vsphere"
@@ -518,7 +519,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			Data:     data,
 		})
 	case ibmcloud.Name:
-		client, err := installConfig.IBMCloud.Client()
+		meta := ibmcloudconfig.NewMetadata(installConfig.Config)
+		client, err := meta.Client()
 		if err != nil {
 			return err
 		}
@@ -601,8 +603,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		vpcPermitted := false
 
 		if installConfig.Config.Publish == types.InternalPublishingStrategy {
-			// Get DNSInstanceCRN from InstallConfig metadata
-			dnsInstance, err := installConfig.IBMCloud.DNSInstance(ctx)
+			// Get DNSInstanceCRN from metadata
+			dnsInstance, err := meta.DNSInstance(ctx)
 			if err != nil {
 				return err
 			}
@@ -611,24 +613,46 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			}
 			// If the VPC already exists and the cluster is Private, check if the VPC is already a Permitted Network on DNS Instance
 			if preexistingVPC {
-				vpcPermitted, err = installConfig.IBMCloud.IsVPCPermittedNetwork(ctx, installConfig.Config.Platform.IBMCloud.VPCName)
+				vpcPermitted, err = meta.IsVPCPermittedNetwork(ctx, installConfig.Config.Platform.IBMCloud.VPCName)
 				if err != nil {
 					return err
 				}
 			}
 		} else {
-			// Get CISInstanceCRN from InstallConfig metadata
-			cisCRN, err = installConfig.IBMCloud.CISInstanceCRN(ctx)
+			// Get CISInstanceCRN from metadata
+			cisCRN, err = meta.CISInstanceCRN(ctx)
+			if err != nil {
+				return err
+			}
+		}
+
+		// NOTE(cjschaef): If one or more ServiceEndpoint's are supplied, attempt to build the Terraform endpoint_file_path
+		// https://registry.terraform.io/providers/IBM-Cloud/ibm/latest/docs/guides/custom-service-endpoints#file-structure-for-endpoints-file
+		var endpointsJSONFile string
+		if len(installConfig.Config.Platform.IBMCloud.ServiceEndpoints) > 0 {
+			endpointData, err := ibmcloudtfvars.CreateEndpointJSON(installConfig.Config.Platform.IBMCloud.ServiceEndpoints, installConfig.Config.Platform.IBMCloud.Region)
 			if err != nil {
 				return err
 			}
+			// While we should have already confirmed there are ServiceEndpoints, we can verify data did get created, requiring the JSON file gets created and passed along
+			if endpointData == nil {
+				return fmt.Errorf("failed to generate endpoint JSON with provided IBM Cloud ServiceEndpoints")
+			}
+
+			// Add endpoint JSON data to list of generated files for Terraform
+			t.FileList = append(t.FileList, &asset.File{
+				Filename: ibmcloudtfvars.IBMCloudEndpointJSONFileName,
+				Data:     endpointData,
+			})
+			endpointsJSONFile = ibmcloudtfvars.IBMCloudEndpointJSONFileName
 		}
 
 		data, err = ibmcloudtfvars.TFVars(
 			ibmcloudtfvars.TFVarsSources{
 				Auth:                     auth,
 				CISInstanceCRN:           cisCRN,
 				DNSInstanceID:            dnsID,
+				EndpointsJSONFile:        endpointsJSONFile,
 				ImageURL:                 string(*rhcosImage),
 				MasterConfigs:            masterConfigs,
 				MasterDedicatedHosts:     masterDedicatedHosts,