Merge pull request openshift#7547 from rna-afk/azure_public_and_private

CORS-2854: azure: Allow users to set visibility to components

Author: openshift-merge-bot[bot]

data/data/install.openshift.io_installconfigs.yaml

@@ -2182,6 +2182,28 @@ spec:
                 description: Deprecated name for NetworkType
                 type: string
             type: object
+          operatorPublishingStrategy:
+            description: OperatorPublishingStrategy controls the visibility of ingress
+              and apiserver. Defaults to public.
+            properties:
+              apiserver:
+                default: External
+                description: APIServer sets the visibility of the load balancers servicing
+                  the APIserver.
+                enum:
+                - ""
+                - External
+                - Internal
+                type: string
+              ingress:
+                default: External
+                description: Ingress sets the visibility of the created dns resources.
+                enum:
+                - ""
+                - External
+                - Internal
+                type: string
+            type: object
           platform:
             description: Platform is the configuration for the specific platform upon
               which to perform the installation.

pkg/asset/cluster/tfvars.go

@@ -402,6 +402,11 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			managedKeys.UserAssignedIdentityKey = installConfig.Config.Azure.CustomerManagedKey.UserAssignedIdentityKey
 		}
 
+		lbPrivate := false
+		if installConfig.Config.OperatorPublishingStrategy != nil {
+			lbPrivate = installConfig.Config.OperatorPublishingStrategy.APIServer == "Internal"
+		}
+
 		data, err := azuretfvars.TFVars(
 			azuretfvars.TFVarsSources{
 				Auth:                            auth,
@@ -423,6 +428,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				InfrastructureName:              clusterID.InfraID,
 				KeyVault:                        managedKeys.KeyVault,
 				UserAssignedIdentityKey:         managedKeys.UserAssignedIdentityKey,
+				LBPrivate:                       lbPrivate,
 			},
 		)
 		if err != nil {

pkg/asset/manifests/dns.go

@@ -135,7 +135,8 @@ func (d *DNS) Generate(dependencies asset.Parents) error {
 			return err
 		}
 
-		if installConfig.Config.Publish == types.ExternalPublishingStrategy {
+		if installConfig.Config.Publish == types.ExternalPublishingStrategy ||
+			(installConfig.Config.Publish == types.MixedPublishingStrategy && installConfig.Config.OperatorPublishingStrategy.Ingress != "Internal") {
 			//currently, this guesses the azure resource IDs from known parameter.
 			config.Spec.PublicZone = &configv1.DNSZone{
 				ID: dnsConfig.GetDNSZoneID(installConfig.Config.Azure.BaseDomainResourceGroupName, installConfig.Config.BaseDomain),

pkg/asset/manifests/ingress.go

@@ -136,6 +136,11 @@ func (ing *Ingress) generateClusterConfig(config *types.InstallConfig) ([]byte,
 
 func (ing *Ingress) generateDefaultIngressController(config *types.InstallConfig) ([]byte, error) {
 	switch config.Publish {
+	case types.MixedPublishingStrategy:
+		if config.OperatorPublishingStrategy.Ingress != "Internal" {
+			break
+		}
+		fallthrough
 	case types.InternalPublishingStrategy:
 		obj := &operatorv1.IngressController{
 			TypeMeta: metav1.TypeMeta{
@@ -156,9 +161,8 @@ func (ing *Ingress) generateDefaultIngressController(config *types.InstallConfig
 			},
 		}
 		return yaml.Marshal(obj)
-	default:
-		return nil, nil
 	}
+	return nil, nil
 }
 
 // Files returns the files generated by the asset.

pkg/explain/printer_test.go

@@ -84,6 +84,9 @@ func Test_PrintFields(t *testing.T) {
     networking <object>
       Networking is the configuration for the pod network provider in the cluster.
 
+    operatorPublishingStrategy <object>
+      OperatorPublishingStrategy controls the visibility of ingress and apiserver. Defaults to public.
+
     platform <object> -required-
       Platform is the configuration for the specific platform upon which to perform the installation.
 

pkg/tfvars/azure/azure.go

@@ -96,6 +96,7 @@ type TFVarsSources struct {
 	InfrastructureName              string
 	KeyVault                        azure.KeyVault
 	UserAssignedIdentityKey         string
+	LBPrivate                       bool
 }
 
 // TFVars generates Azure-specific Terraform variables launching the cluster.
@@ -170,7 +171,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		VolumeSize:                              masterConfig.OSDisk.DiskSizeGB,
 		ImageURL:                                sources.ImageURL,
 		ImageRelease:                            sources.ImageRelease,
-		Private:                                 sources.Publish == types.InternalPublishingStrategy,
+		Private:                                 sources.Publish == types.InternalPublishingStrategy || sources.LBPrivate,
 		OutboundType:                            string(sources.OutboundType),
 		ResourceGroupName:                       sources.ResourceGroupName,
 		BaseDomainResourceGroupName:             sources.BaseDomainResourceGroupName,

pkg/types/azure/validation/platform.go

@@ -61,7 +61,7 @@ func ValidatePlatform(p *azure.Platform, publish types.PublishingStrategy, fldPa
 	if p.Region == "" {
 		allErrs = append(allErrs, field.Required(fldPath.Child("region"), "region should be set to one of the supported Azure regions"))
 	}
-	if !p.IsARO() && publish == types.ExternalPublishingStrategy {
+	if !p.IsARO() && publish != types.InternalPublishingStrategy {
 		if p.BaseDomainResourceGroupName == "" {
 			allErrs = append(allErrs, field.Required(fldPath.Child("baseDomainResourceGroupName"), "baseDomainResourceGroupName is the resource group name where the azure dns zone is deployed"))
 		}

pkg/types/installconfig.go

@@ -71,6 +71,9 @@ const (
 	ExternalPublishingStrategy PublishingStrategy = "External"
 	// InternalPublishingStrategy exposes the endpoints for the cluster to the private network only.
 	InternalPublishingStrategy PublishingStrategy = "Internal"
+	// MixedPublishingStrategy allows for the api server and the ingress to be configured individually for exposure to
+	// private network or Internet.
+	MixedPublishingStrategy PublishingStrategy = "Mixed"
 )
 
 // PolicyType is for usage polices that are applied to additionalTrustBundle.
@@ -155,6 +158,9 @@ type InstallConfig struct {
 	// +optional
 	Publish PublishingStrategy `json:"publish,omitempty"`
 
+	// OperatorPublishingStrategy controls the visibility of ingress and apiserver. Defaults to public.
+	OperatorPublishingStrategy *OperatorPublishingStrategy `json:"operatorPublishingStrategy,omitempty"`
+
 	// FIPS configures https://www.nist.gov/itl/fips-general-information
 	//
 	// +kubebuilder:default=false
@@ -313,6 +319,22 @@ type Platform struct {
 	Nutanix *nutanix.Platform `json:"nutanix,omitempty"`
 }
 
+// OperatorPublishingStrategy is used to control the visibility of the components which can be used to have a mix of public
+// and private resources.
+type OperatorPublishingStrategy struct {
+	// Ingress sets the visibility of the created dns resources.
+	// +kubebuilder:validation:Enum="";External;Internal
+	// +kubebuilder:default=External
+	// +optional
+	Ingress string `json:"ingress,omitempty"`
+
+	// APIServer sets the visibility of the load balancers servicing the APIserver.
+	// +kubebuilder:validation:Enum="";External;Internal
+	// +kubebuilder:default=External
+	// +optional
+	APIServer string `json:"apiserver,omitempty"`
+}
+
 // Name returns a string representation of the platform (e.g. "aws" if
 // AWS is non-nil).  It returns an empty string if no platform is
 // configured.

pkg/types/validation/installconfig.go

@@ -156,6 +156,38 @@ func ValidateInstallConfig(c *types.InstallConfig, usingAgentMethod bool) field.
 		}
 	}
 
+	if c.Publish == types.MixedPublishingStrategy {
+		switch platformName := c.Platform.Name(); platformName {
+		case azure.Name:
+		default:
+			allErrs = append(allErrs, field.Invalid(field.NewPath("publish"), c.Publish, fmt.Sprintf("mixed publish strategy is not supported on %q platform", platformName)))
+		}
+		if c.OperatorPublishingStrategy == nil {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("publish"), c.Publish, "please specify the operator publishing strategy for mixed publish strategy"))
+		}
+	} else if c.OperatorPublishingStrategy != nil {
+		allErrs = append(allErrs, field.Invalid(field.NewPath("operatorPublishingStrategy"), c.Publish, "operator publishing strategy is only allowed with mixed publishing strategy installs"))
+	}
+
+	if c.OperatorPublishingStrategy != nil {
+		acceptedValues := sets.New[string]("Internal", "External")
+		if c.OperatorPublishingStrategy.APIServer == "" {
+			c.OperatorPublishingStrategy.APIServer = "External"
+		}
+		if c.OperatorPublishingStrategy.Ingress == "" {
+			c.OperatorPublishingStrategy.Ingress = "External"
+		}
+		if !acceptedValues.Has(c.OperatorPublishingStrategy.APIServer) {
+			allErrs = append(allErrs, field.NotSupported(field.NewPath("apiserver"), c.OperatorPublishingStrategy.APIServer, sets.List(acceptedValues)))
+		}
+		if !acceptedValues.Has(c.OperatorPublishingStrategy.Ingress) {
+			allErrs = append(allErrs, field.NotSupported(field.NewPath("ingress"), c.OperatorPublishingStrategy.Ingress, sets.List(acceptedValues)))
+		}
+		if c.OperatorPublishingStrategy.APIServer == "Internal" && c.OperatorPublishingStrategy.Ingress == "Internal" {
+			allErrs = append(allErrs, field.Invalid(field.NewPath("publish"), c.OperatorPublishingStrategy.APIServer, "cannot set both fields to internal in a mixed cluster, use publish internal instead"))
+		}
+	}
+
 	if c.Capabilities != nil {
 		if c.Capabilities.BaselineCapabilitySet == configv1.ClusterVersionCapabilitySetNone {
 			enabledCaps := sets.New[configv1.ClusterVersionCapability](c.Capabilities.AdditionalEnabledCapabilities...)
@@ -898,6 +930,7 @@ var (
 	validPublishingStrategies = map[types.PublishingStrategy]struct{}{
 		types.ExternalPublishingStrategy: {},
 		types.InternalPublishingStrategy: {},
+		types.MixedPublishingStrategy:    {},
 	}
 
 	validPublishingStrategyValues = func() []string {