Merge pull request openshift#7520 from rna-afk/azure_storage_encryption

CORS-2845: azure: Enable storage account encryption

Author: openshift-merge-bot[bot]

data/data/azure/bootstrap/main.tf

@@ -56,17 +56,16 @@ data "azurerm_storage_account_sas" "ignition" {
 }
 
 resource "azurerm_storage_container" "ignition" {
-  name                  = "ignition"
-  storage_account_name  = var.storage_account_name
-  container_access_type = "private"
+  name                 = "ignition"
+  storage_account_name = var.storage_account_name
 }
 
 resource "azurerm_storage_blob" "ignition" {
   name                   = "bootstrap.ign"
   source                 = var.ignition_bootstrap_file
   storage_account_name   = var.storage_account_name
   storage_container_name = azurerm_storage_container.ignition.name
-  type                   = "Block"
+  type                   = var.azure_keyvault_key_name != "" ? "Page" : "Block"
 }
 
 data "ignition_config" "redirect" {

data/data/azure/variables-azure.tf

@@ -278,3 +278,27 @@ variable "azure_master_virtualized_trusted_platform_module" {
   description = "Defines whether the instance should have vTPM enabled."
   default     = ""
 }
+
+variable "azure_keyvault_resource_group" {
+  type        = string
+  description = "Defines the resource group of the key vault used for storage account encryption."
+  default     = ""
+}
+
+variable "azure_keyvault_name" {
+  type        = string
+  description = "Defines the name of the key vault used for storage account encryption."
+  default     = ""
+}
+
+variable "azure_keyvault_key_name" {
+  type        = string
+  description = "Defines the key in the key vault used for storage account encryption."
+  default     = ""
+}
+
+variable "azure_user_assigned_identity_key" {
+  type        = string
+  description = "Defines the user identity key used for the encryption of storage account."
+  default     = ""
+}

data/data/azure/vnet/main.tf

@@ -43,15 +43,52 @@ data "azurerm_resource_group" "network" {
   name = var.azure_network_resource_group_name
 }
 
+data "azurerm_key_vault" "keyvault" {
+  count = var.azure_keyvault_name != "" ? 1 : 0
+
+  name                = var.azure_keyvault_name
+  resource_group_name = var.azure_keyvault_resource_group
+}
+
+data "azurerm_key_vault_key" "keyvault_key" {
+  count = var.azure_keyvault_name != "" ? 1 : 0
+
+  name         = var.azure_keyvault_key_name
+  key_vault_id = data.azurerm_key_vault.keyvault[0].id
+}
+
+data "azurerm_user_assigned_identity" "keyvault_identity" {
+  count = var.azure_keyvault_name != "" ? 1 : 0
+
+  resource_group_name = var.azure_keyvault_resource_group
+  name                = var.azure_user_assigned_identity_key
+}
+
 resource "azurerm_storage_account" "cluster" {
   name                            = "cluster${var.random_storage_account_suffix}"
   resource_group_name             = data.azurerm_resource_group.main.name
   location                        = var.azure_region
-  account_tier                    = "Standard"
+  account_tier                    = var.azure_keyvault_name != "" ? "Premium" : "Standard"
   account_replication_type        = "LRS"
   min_tls_version                 = contains(local.environments_with_min_tls_version, var.azure_environment) ? "TLS1_2" : null
-  allow_nested_items_to_be_public = false
+  allow_nested_items_to_be_public = var.azure_keyvault_name != "" ? true : false
   tags                            = var.azure_extra_tags
+
+  dynamic "customer_managed_key" {
+    for_each = var.azure_keyvault_name != "" ? [1] : []
+    content {
+      key_vault_key_id          = data.azurerm_key_vault_key.keyvault_key[0].id
+      user_assigned_identity_id = data.azurerm_user_assigned_identity.keyvault_identity[0].id
+    }
+  }
+
+  dynamic identity {
+    for_each = var.azure_keyvault_name != "" ? [1] : []
+    content {
+      type         = "UserAssigned"
+      identity_ids = [data.azurerm_user_assigned_identity.keyvault_identity[0].id]
+    }
+  }
 }
 
 resource "azurerm_user_assigned_identity" "main" {

data/data/install.openshift.io_installconfigs.yaml

@@ -2481,6 +2481,34 @@ spec:
                     description: ControlPlaneSubnet specifies an existing subnet for
                       use by the control plane nodes
                     type: string
+                  customerManagedKey:
+                    description: CustomerManagedKey has the keys needed to encrypt
+                      the storage account.
+                    properties:
+                      keyVault:
+                        description: KeyVault is the keyvault used for the customer
+                          created key required for encryption.
+                        properties:
+                          keyName:
+                            description: KeyName is the name of the key vault key.
+                            type: string
+                          name:
+                            description: Name is the name of the key vault.
+                            type: string
+                          resourceGroup:
+                            description: ResourceGroup defines the Azure resource
+                              group used by the key vault.
+                            type: string
+                        required:
+                        - keyName
+                        - name
+                        - resourceGroup
+                        type: object
+                      userAssignedIdentityKey:
+                        description: UserAssignedIdentityKey is the name of the user
+                          identity that has access to the managed key.
+                        type: string
+                    type: object
                   defaultMachinePlatform:
                     description: DefaultMachinePlatform is the default configuration
                       used when installing on Azure for machine pools which do not

pkg/asset/cluster/tfvars.go

@@ -182,6 +182,14 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		}
 	}
 
+	lengthBootstrapFile := int64(len(bootstrapIgn))
+	if installConfig.Config.Platform.Azure != nil && installConfig.Config.Platform.Azure.CustomerManagedKey != nil &&
+		installConfig.Config.Platform.Azure.CustomerManagedKey.UserAssignedIdentityKey != "" {
+		if lengthBootstrapFile%512 != 0 {
+			lengthBootstrapFile = (((lengthBootstrapFile / 512) + 1) * 512)
+		}
+	}
+
 	data, err := tfvars.TFVars(
 		clusterID.InfraID,
 		installConfig.Config.ClusterDomain(),
@@ -191,6 +199,7 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 		useIPv4,
 		useIPv6,
 		bootstrapIgn,
+		lengthBootstrapFile,
 		masterIgn,
 		masterCount,
 		mastersSchedulable,
@@ -383,6 +392,16 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 			bootstrapIgnStub = string(shim)
 		}
 
+		managedKeys := azure.CustomerManagedKey{}
+		if installConfig.Config.Azure.CustomerManagedKey != nil {
+			managedKeys.KeyVault = azure.KeyVault{
+				ResourceGroup: installConfig.Config.Azure.CustomerManagedKey.KeyVault.ResourceGroup,
+				Name:          installConfig.Config.Azure.CustomerManagedKey.KeyVault.Name,
+				KeyName:       installConfig.Config.Azure.CustomerManagedKey.KeyVault.KeyName,
+			}
+			managedKeys.UserAssignedIdentityKey = installConfig.Config.Azure.CustomerManagedKey.UserAssignedIdentityKey
+		}
+
 		data, err := azuretfvars.TFVars(
 			azuretfvars.TFVarsSources{
 				Auth:                            auth,
@@ -402,6 +421,8 @@ func (t *TerraformVariables) Generate(parents asset.Parents) error {
 				HyperVGeneration:                hyperVGeneration,
 				VMArchitecture:                  installConfig.Config.ControlPlane.Architecture,
 				InfrastructureName:              clusterID.InfraID,
+				KeyVault:                        managedKeys.KeyVault,
+				UserAssignedIdentityKey:         managedKeys.UserAssignedIdentityKey,
 			},
 		)
 		if err != nil {

pkg/explain/printer_test.go

@@ -213,6 +213,9 @@ func Test_PrintFields(t *testing.T) {
     controlPlaneSubnet <string>
       ControlPlaneSubnet specifies an existing subnet for use by the control plane nodes
 
+    customerManagedKey <object>
+      CustomerManagedKey has the keys needed to encrypt the storage account.
+
     defaultMachinePlatform <object>
       DefaultMachinePlatform is the default configuration used when installing on Azure for machine pools which do not define their own platform configuration.
 

pkg/tfvars/azure/azure.go

@@ -69,6 +69,10 @@ type config struct {
 	SecureVirtualMachineDiskEncryptionSetID string `json:"azure_master_secure_vm_disk_encryption_set_id,omitempty"`
 	SecureBoot                              string `json:"azure_master_secure_boot,omitempty"`
 	VirtualizedTrustedPlatformModule        string `json:"azure_master_virtualized_trusted_platform_module,omitempty"`
+	KeyVaultResourceGroup                   string `json:"azure_keyvault_resource_group,omitempty"`
+	KeyVaultName                            string `json:"azure_keyvault_name,omitempty"`
+	KeyVaultKeyName                         string `json:"azure_keyvault_key_name,omitempty"`
+	UserAssignedIdentity                    string `json:"azure_user_assigned_identity_key,omitempty"`
 }
 
 // TFVarsSources contains the parameters to be converted into Terraform variables
@@ -90,6 +94,8 @@ type TFVarsSources struct {
 	HyperVGeneration                string
 	VMArchitecture                  types.Architecture
 	InfrastructureName              string
+	KeyVault                        azure.KeyVault
+	UserAssignedIdentityKey         string
 }
 
 // TFVars generates Azure-specific Terraform variables launching the cluster.
@@ -187,6 +193,10 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		SecureVirtualMachineDiskEncryptionSetID: masterConfig.OSDisk.ManagedDisk.SecurityProfile.DiskEncryptionSet.ID,
 		SecureBoot:                              secureBoot,
 		VirtualizedTrustedPlatformModule:        virtualizedTrustedPlatformModule,
+		KeyVaultResourceGroup:                   sources.KeyVault.ResourceGroup,
+		KeyVaultName:                            sources.KeyVault.Name,
+		KeyVaultKeyName:                         sources.KeyVault.KeyName,
+		UserAssignedIdentity:                    sources.UserAssignedIdentityKey,
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/tfvars/tfvars.go

@@ -28,13 +28,21 @@ type Config struct {
 }
 
 // TFVars generates terraform.tfvar JSON for launching the cluster.
-func TFVars(clusterID string, clusterDomain string, baseDomain string, machineV4CIDRs []string, machineV6CIDRs []string, useIPv4, useIPv6 bool, bootstrapIgn string, masterIgn string, masterCount int, mastersSchedulable bool) ([]byte, error) {
+func TFVars(clusterID string, clusterDomain string, baseDomain string, machineV4CIDRs []string, machineV6CIDRs []string, useIPv4, useIPv6 bool, bootstrapIgn string, bootstrapIgnSize int64, masterIgn string, masterCount int, mastersSchedulable bool) ([]byte, error) {
 	f, err := os.CreateTemp("", "openshift-install-bootstrap-*.ign")
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create tmp file for bootstrap ignition")
 	}
 	defer f.Close()
 
+	// In azure, if the storage account is encrypted, page blob is used instead of block blob due to lack of support.
+	// Page blobs file size must be a multiple of 512, hence a little padding is needed to push the file.
+	// Finding the nearest size divisible by 512 and adding that padding to the file.
+	// Since the file is json type, padding at the end results in json parsing error at bootstrap ignition.
+	// Adding the paddding to just before the last } in the json file to bypass the parsing error.
+	padding := bootstrapIgnSize - int64(len(bootstrapIgn))
+	bootstrapIgn = bootstrapIgn[0:len(bootstrapIgn)-1] + strings.Repeat(" ", int(padding)) + string(bootstrapIgn[len(bootstrapIgn)-1])
+
 	if _, err := f.WriteString(bootstrapIgn); err != nil {
 		return nil, errors.Wrap(err, "failed to write bootstrap ignition")
 	}

pkg/types/azure/platform.go

@@ -97,6 +97,28 @@ type Platform struct {
 	// Resources created by the cluster itself may not include these tags.
 	// +optional
 	UserTags map[string]string `json:"userTags,omitempty"`
+
+	// CustomerManagedKey has the keys needed to encrypt the storage account.
+	CustomerManagedKey *CustomerManagedKey `json:"customerManagedKey,omitempty"`
+}
+
+// KeyVault defines an Azure Key Vault.
+type KeyVault struct {
+	// ResourceGroup defines the Azure resource group used by the key
+	// vault.
+	ResourceGroup string `json:"resourceGroup"`
+	// Name is the name of the key vault.
+	Name string `json:"name"`
+	// KeyName is the name of the key vault key.
+	KeyName string `json:"keyName"`
+}
+
+// CustomerManagedKey defines the customer managed key settings for encryption of the Azure storage account.
+type CustomerManagedKey struct {
+	// KeyVault is the keyvault used for the customer created key required for encryption.
+	KeyVault KeyVault `json:"keyVault,omitempty"`
+	// UserAssignedIdentityKey is the name of the user identity that has access to the managed key.
+	UserAssignedIdentityKey string `json:"userAssignedIdentityKey,omitempty"`
 }
 
 // CloudEnvironment is the name of the Azure cloud environment

pkg/types/azure/validation/platform.go

@@ -40,6 +40,15 @@ var (
 
 	// tagKeyPrefixRegex is for verifying that the tag value does not contain restricted prefixes.
 	tagKeyPrefixRegex = regexp.MustCompile(`^(?i)(name$|kubernetes\.io|openshift\.io|microsoft|azure|windows)`)
+
+	// keyVaultNameRegex is for verifying the name of the key vault used for storage account encryption.
+	keyVaultNameRegex = regexp.MustCompile(`^[A-Za-z][0-9A-Za-z-]{1,22}[A-Za-z0-9]$`)
+
+	// keyVaultKeyNameRegex is for verifying the key name of the key vault used for storage account encryption.
+	keyVaultKeyNameRegex = regexp.MustCompile(`^[0-9A-Za-z-]{1,127}$`)
+
+	// keyVaultUserAssignedIdentityRegex is for verifying the user assigned identity key used for storage account encryption.
+	keyVaultUserAssignedIdentityRegex = regexp.MustCompile(`^[0-9A-Za-z][0-9A-Za-z-_]{2,127}$`)
 )
 
 // maxUserTagLimit is the maximum userTags that can be configured as defined in openshift/api.
@@ -99,6 +108,10 @@ func ValidatePlatform(p *azure.Platform, publish types.PublishingStrategy, fldPa
 		}
 	}
 
+	if p.CustomerManagedKey != nil {
+		allErrs = append(allErrs, validateCustomerManagedKeys(p.CloudName, *p.CustomerManagedKey, fldPath.Child("customerManagedKey"))...)
+	}
+
 	// support for Azure user-defined tags made available through
 	// RFE-2017 is for AzurePublicCloud only.
 	if p.CloudName != azure.PublicCloud && len(p.UserTags) > 0 {
@@ -122,6 +135,41 @@ func ValidatePlatform(p *azure.Platform, publish types.PublishingStrategy, fldPa
 	return allErrs
 }
 
+// validateCustomerManagedKeys validates the key vault id.
+func validateCustomerManagedKeys(cloudName azure.CloudEnvironment, s azure.CustomerManagedKey, fldPath *field.Path) field.ErrorList {
+	var allErrs field.ErrorList
+
+	if cloudName == azure.StackCloud {
+		return append(allErrs, field.Invalid(fldPath, s.KeyVault.Name, "storage account encryption is not supported on this platform"))
+	}
+
+	if s.KeyVault.KeyName == "" {
+		allErrs = append(allErrs, field.Required(fldPath, "key vault key name is required for storage account encryption"))
+	} else if !keyVaultKeyNameRegex.MatchString(s.KeyVault.KeyName) {
+		allErrs = append(allErrs, field.Invalid(fldPath, s.KeyVault.KeyName, "invalid key name for encryption"))
+	}
+
+	if s.KeyVault.Name == "" {
+		allErrs = append(allErrs, field.Required(fldPath, "name of the key vault is required for storage account encryption"))
+	} else if !keyVaultNameRegex.MatchString(s.KeyVault.Name) || strings.Contains(s.KeyVault.Name, "--") {
+		allErrs = append(allErrs, field.Invalid(fldPath, s.KeyVault.Name, "invalid name for key vault for encryption"))
+	}
+
+	if s.KeyVault.ResourceGroup == "" {
+		allErrs = append(allErrs, field.Required(fldPath, "resource group of the key vault is required for storage account encryption"))
+	} else if !RxResourceGroup.MatchString(s.KeyVault.ResourceGroup) {
+		allErrs = append(allErrs, field.Invalid(fldPath, s.KeyVault.ResourceGroup, "invalid resource group for encryption"))
+	}
+
+	if s.UserAssignedIdentityKey == "" {
+		allErrs = append(allErrs, field.Required(fldPath, "user assigned identity key is required for storage account encryption"))
+	} else if !keyVaultUserAssignedIdentityRegex.MatchString(s.UserAssignedIdentityKey) {
+		allErrs = append(allErrs, field.Invalid(fldPath, s.UserAssignedIdentityKey, "invalid user assigned identity key for encryption"))
+	}
+
+	return allErrs
+}
+
 // validateUserTags verifies if configured number of UserTags is not more than
 // allowed limit and the tag keys and values are valid.
 func validateUserTags(tags map[string]string, fldPath *field.Path) field.ErrorList {