azure: add metadata info to resource group tags

When Hive runs the Installer destroy code, they don't always have all
the cluster metadata information the destroyer needs. Adding new fields
to the metadata is also a very painful process for them.

This PR adds some cluster metadata information to the cluster resource
group as tags, so that it can be discovered during destroy even if
azure-specific information is missing from the `metadata.json` file.

For now we are saving "region", "base domain resource group name", and
"network resource group name" but the mechanism will be useful if we
wish to add more data in the future without impacting Hive.

Author: r4f4

data/data/azure/variables-azure.tf

@@ -302,3 +302,9 @@ variable "azure_user_assigned_identity_key" {
   description = "Defines the user identity key used for the encryption of storage account."
   default     = ""
 }
+
+variable "azure_resource_group_metadata_tags" {
+  type        = map(string)
+  description = "Metadata Azure tags to be applied to the cluster resource group."
+  default     = {}
+}

data/data/azure/vnet/main.tf

@@ -28,7 +28,7 @@ resource "azurerm_resource_group" "main" {
 
   name     = "${var.cluster_id}-rg"
   location = var.azure_region
-  tags     = var.azure_extra_tags
+  tags     = merge(var.azure_extra_tags, var.azure_resource_group_metadata_tags)
 }
 
 data "azurerm_resource_group" "main" {

pkg/asset/cluster/azure/azure.go

@@ -124,6 +124,17 @@ func tagResourceGroup(ctx context.Context, clusterID string, installConfig *inst
 	tagKey, tagValue := ownedTag(clusterID)
 	group.Tags[tagKey] = tagValue
 	logrus.Debugf("Tagging resource group %s with %s: %s", installConfig.Config.Azure.ResourceGroupName, tagKey, *tagValue)
+
+	// Save metadata needed to destroy cluster into tags
+	config := installConfig.Config.Azure
+	group.Tags[azure.TagMetadataRegion] = to.StringPtr(config.Region)
+	if len(config.BaseDomainResourceGroupName) > 0 {
+		group.Tags[azure.TagMetadataBaseDomainRG] = to.StringPtr(config.BaseDomainResourceGroupName)
+	}
+	if len(config.NetworkResourceGroupName) > 0 {
+		group.Tags[azure.TagMetadataNetworkRG] = to.StringPtr(config.NetworkResourceGroupName)
+	}
+
 	_, err = client.Update(ctx, installConfig.Config.Azure.ResourceGroupName, resources.GroupPatchable{
 		Tags: group.Tags,
 	})

pkg/destroy/azure/azure.go

@@ -46,6 +46,7 @@ type ClusterUninstaller struct {
 	InfraID                     string
 	ResourceGroupName           string
 	BaseDomainResourceGroupName string
+	NetworkResourceGroupName    string
 
 	Logger logrus.FieldLogger
 
@@ -126,15 +127,10 @@ func New(logger logrus.FieldLogger, metadata *types.ClusterMetadata) (providers.
 		return nil, err
 	}
 
-	group := metadata.Azure.ResourceGroupName
-	if len(group) == 0 {
-		group = metadata.InfraID + "-rg"
-	}
-
 	return &ClusterUninstaller{
 		Session:                     session,
 		InfraID:                     metadata.InfraID,
-		ResourceGroupName:           group,
+		ResourceGroupName:           metadata.Azure.ResourceGroupName,
 		Logger:                      logger,
 		BaseDomainResourceGroupName: metadata.Azure.BaseDomainResourceGroupName,
 		CloudName:                   cloudName,
@@ -156,6 +152,38 @@ func (o *ClusterUninstaller) Run() (*types.ClusterQuota, error) {
 	waitCtx, cancel := context.WithTimeout(context.Background(), timeout)
 	defer cancel()
 
+	// Retrieve metadata from resource group tags, if available
+	filter := fmt.Sprintf("tagName eq 'kubernetes.io_cluster.%s' and tagValue eq 'owned'", o.InfraID)
+	groupPager, err := o.resourceGroupsClient.ListComplete(waitCtx, filter, to.Int32Ptr(1))
+	if err != nil {
+		return nil, fmt.Errorf("could not list resource groups: %w", err)
+	}
+
+	for ; groupPager.NotDone(); err = groupPager.NextWithContext(waitCtx) {
+		if err != nil {
+			o.Logger.Debugf("failed to advance to next resource group list page: %v", err)
+			continue
+		}
+		group := groupPager.Value()
+		if len(o.ResourceGroupName) == 0 {
+			o.ResourceGroupName = to.String(group.Name)
+			o.Logger.Debugf("found resource group name=%s from tags", o.ResourceGroupName)
+		}
+		if len(o.BaseDomainResourceGroupName) == 0 {
+			o.BaseDomainResourceGroupName = to.String(group.Tags[azure.TagMetadataBaseDomainRG])
+			o.Logger.Debugf("found base domain resource group name=%s from tags", o.BaseDomainResourceGroupName)
+		}
+		if len(o.NetworkResourceGroupName) == 0 {
+			o.NetworkResourceGroupName = to.String(group.Tags[azure.TagMetadataNetworkRG])
+			o.Logger.Debugf("found network resource group name=%s from tags", o.NetworkResourceGroupName)
+		}
+	}
+
+	if len(o.ResourceGroupName) == 0 {
+		o.ResourceGroupName = o.InfraID + "-rg"
+		o.Logger.Debugf("using default resource group name=%s", o.ResourceGroupName)
+	}
+
 	err = wait.PollUntilContextCancel(
 		waitCtx,
 		1*time.Second,

pkg/tfvars/azure/azure.go

@@ -65,14 +65,15 @@ type config struct {
 	UseMarketplaceImage                     bool              `json:"azure_use_marketplace_image"`
 	MarketplaceImageHasPlan                 bool              `json:"azure_marketplace_image_has_plan"`
 	OSImage                                 `json:",inline"`
-	SecurityEncryptionType                  string `json:"azure_master_security_encryption_type,omitempty"`
-	SecureVirtualMachineDiskEncryptionSetID string `json:"azure_master_secure_vm_disk_encryption_set_id,omitempty"`
-	SecureBoot                              string `json:"azure_master_secure_boot,omitempty"`
-	VirtualizedTrustedPlatformModule        string `json:"azure_master_virtualized_trusted_platform_module,omitempty"`
-	KeyVaultResourceGroup                   string `json:"azure_keyvault_resource_group,omitempty"`
-	KeyVaultName                            string `json:"azure_keyvault_name,omitempty"`
-	KeyVaultKeyName                         string `json:"azure_keyvault_key_name,omitempty"`
-	UserAssignedIdentity                    string `json:"azure_user_assigned_identity_key,omitempty"`
+	SecurityEncryptionType                  string            `json:"azure_master_security_encryption_type,omitempty"`
+	SecureVirtualMachineDiskEncryptionSetID string            `json:"azure_master_secure_vm_disk_encryption_set_id,omitempty"`
+	SecureBoot                              string            `json:"azure_master_secure_boot,omitempty"`
+	VirtualizedTrustedPlatformModule        string            `json:"azure_master_virtualized_trusted_platform_module,omitempty"`
+	KeyVaultResourceGroup                   string            `json:"azure_keyvault_resource_group,omitempty"`
+	KeyVaultName                            string            `json:"azure_keyvault_name,omitempty"`
+	KeyVaultKeyName                         string            `json:"azure_keyvault_key_name,omitempty"`
+	UserAssignedIdentity                    string            `json:"azure_user_assigned_identity_key,omitempty"`
+	ResourceGroupMetadataTags               map[string]string `json:"azure_resource_group_metadata_tags"`
 }
 
 // TFVarsSources contains the parameters to be converted into Terraform variables
@@ -157,6 +158,16 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		Version:   masterConfig.Image.Version,
 	}
 
+	// Metadata tags to be added to the resource group for the cluster destroy
+	metadataTags := map[string]string{}
+	metadataTags[azure.TagMetadataRegion] = region
+	if len(sources.BaseDomainResourceGroupName) > 0 {
+		metadataTags[azure.TagMetadataBaseDomainRG] = sources.BaseDomainResourceGroupName
+	}
+	if len(masterConfig.NetworkResourceGroup) > 0 {
+		metadataTags[azure.TagMetadataNetworkRG] = masterConfig.NetworkResourceGroup
+	}
+
 	cfg := &config{
 		Auth:                                    sources.Auth,
 		Environment:                             environment,
@@ -198,6 +209,7 @@ func TFVars(sources TFVarsSources) ([]byte, error) {
 		KeyVaultName:                            sources.KeyVault.Name,
 		KeyVaultKeyName:                         sources.KeyVault.KeyName,
 		UserAssignedIdentity:                    sources.UserAssignedIdentityKey,
+		ResourceGroupMetadataTags:               metadataTags,
 	}
 
 	return json.MarshalIndent(cfg, "", "  ")

pkg/types/azure/metadata.go

@@ -8,3 +8,10 @@ type Metadata struct {
 	ResourceGroupName           string           `json:"resourceGroupName"`
 	BaseDomainResourceGroupName string           `json:"baseDomainResourceGroupName"`
 }
+
+// Keys used to save Metadata information as tags.
+const (
+	TagMetadataRegion       = "openshift_region"
+	TagMetadataBaseDomainRG = "openshift_basedomainRG"
+	TagMetadataNetworkRG    = "openshift_networkRG"
+)