Make azsp tests more stable

nfx · nfx · commit 7b745c6010f4 · 2021-08-02T18:38:48.000+02:00
diff --git a/common/azure_auth.go b/common/azure_auth.go
@@ -89,7 +89,7 @@ func (aa *AzureAuth) getAzureEnvironment() (azure.Environment, error) {
 
 func (aa *AzureAuth) resourceID() string {
 	if aa.ResourceID != "" {
-		if aa.SubscriptionID == "" {
+		if aa.SubscriptionID == "" || aa.ResourceGroup == "" {
 			res, err := azure.ParseResourceID(aa.ResourceID)
 			if err != nil {
 				log.Printf("[ERROR] %s", err)
diff --git a/common/http.go b/common/http.go
@@ -25,6 +25,7 @@ var (
 		"com.databricks.backend.manager.util.UnknownWorkerEnvironmentException",
 		"does not have any associated worker environments",
 		"There is no worker environment with id",
+		"Unknown worker environment",
 		"ClusterNotReadyException",
 		"connection reset by peer",
 		"connection refused",
diff --git a/identity/acceptance/resource_group_member_test.go b/identity/acceptance/resource_group_member_test.go
@@ -11,7 +11,7 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestAccComplexValueResource(t *testing.T) {
+func TestAccGroupMemberResource(t *testing.T) {
 	acceptance.Test(t, []acceptance.Step{
 		{
 			Template: `
diff --git a/mws/resource_workspace_test.go b/mws/resource_workspace_test.go
@@ -121,22 +121,22 @@ func TestResourceWorkspaceCreateGcp(t *testing.T) {
 				Method:   "POST",
 				Resource: "/api/2.0/accounts/abc/workspaces",
 				// retreating to raw JSON, as certain fields don't work well together
-				ExpectedRequest: map[string]interface {}{
-					"account_id":"abc", 
-					"cloud":"gcp", 
-					"cloud_resource_bucket":map[string]interface {}{
-						"gcp":map[string]interface {}{
-							"project_id":"def",
+				ExpectedRequest: map[string]interface{}{
+					"account_id": "abc",
+					"cloud":      "gcp",
+					"cloud_resource_bucket": map[string]interface{}{
+						"gcp": map[string]interface{}{
+							"project_id": "def",
 						},
 					},
-					"location":"bcd", 
-					"workspace_name":"labdata",
+					"location":       "bcd",
+					"workspace_name": "labdata",
 				},
 				Response: Workspace{
 					WorkspaceID:    1234,
 					AccountID:      "abc",
 					DeploymentName: "900150983cd24fb0",
-					WorkspaceName: "labdata",
+					WorkspaceName:  "labdata",
 				},
 			},
 			{
@@ -148,7 +148,7 @@ func TestResourceWorkspaceCreateGcp(t *testing.T) {
 					WorkspaceID:     1234,
 					WorkspaceStatus: WorkspaceStatusRunning,
 					DeploymentName:  "900150983cd24fb0",
-					WorkspaceName: "labdata",
+					WorkspaceName:   "labdata",
 				},
 			},
 		},
diff --git a/scripts/modules/az-common/main.tf b/scripts/modules/az-common/main.tf
@@ -38,43 +38,47 @@ locals {
   }
 }
 
-resource "azurerm_resource_group" "example" {
+resource "azurerm_resource_group" "this" {
   name     = "${local.prefix}-rg"
   location = var.region
   tags     = local.tags
 }
 
 output "azure_region" {
-  value = azurerm_resource_group.example.location
+  value = azurerm_resource_group.this.location
 }
 
 output "test_resource_group" {
-  value = azurerm_resource_group.example.name
+  value = azurerm_resource_group.this.name
 }
 
-resource "azurerm_databricks_workspace" "example" {
+output "test_resource_group_id" {
+  value = azurerm_resource_group.this.id
+}
+
+resource "azurerm_databricks_workspace" "this" {
   name                        = "${local.prefix}-workspace"
-  resource_group_name         = azurerm_resource_group.example.name
-  location                    = azurerm_resource_group.example.location
+  resource_group_name         = azurerm_resource_group.this.name
+  location                    = azurerm_resource_group.this.location
   sku                         = "premium"
   managed_resource_group_name = "${local.prefix}-workspace-rg"
   tags                        = local.tags
 }
 
 output "databricks_azure_workspace_resource_id" {
   // The ID of the Databricks Workspace in the Azure management plane.
-  value = azurerm_databricks_workspace.example.id
+  value = azurerm_databricks_workspace.this.id
 }
 
 output "databricks_host" {
-  value = "https://${azurerm_databricks_workspace.example.workspace_url}/"
+  value = "https://${azurerm_databricks_workspace.this.workspace_url}/"
 }
 
 // ADLSv1 resource
 resource "azurerm_data_lake_store" "gen1" {
   name                = "${local.prefix}adlsv1"
-  resource_group_name = azurerm_resource_group.example.name
-  location            = azurerm_resource_group.example.location
+  resource_group_name = azurerm_resource_group.this.name
+  location            = azurerm_resource_group.this.location
 }
 
 output "test_data_lake_store_name" {
@@ -84,8 +88,8 @@ output "test_data_lake_store_name" {
 # Create container in storage acc and container for use by blob mount tests
 resource "azurerm_storage_account" "v2" {
   name                     = "${local.prefix}adlsv2"
-  resource_group_name      = azurerm_resource_group.example.name
-  location                 = azurerm_resource_group.example.location
+  resource_group_name      = azurerm_resource_group.this.name
+  location                 = azurerm_resource_group.this.location
   account_tier             = "Standard"
   account_replication_type = "LRS"
   account_kind             = "StorageV2"
@@ -111,7 +115,7 @@ output "test_storage_v2_wasbs" {
   value     = azurerm_storage_container.wasbs.name
 }
 
-data "azurerm_storage_account_blob_container_sas" "example" {
+data "azurerm_storage_account_blob_container_sas" "this" {
   connection_string = azurerm_storage_account.v2.primary_connection_string
   container_name    = azurerm_storage_container.wasbs.name
   https_only        = true
@@ -128,64 +132,63 @@ data "azurerm_storage_account_blob_container_sas" "example" {
 }
 
 output "test_storage_v2_wasbs_sas" {
-  value = data.azurerm_storage_account_blob_container_sas.example.sas
+  value = data.azurerm_storage_account_blob_container_sas.this.sas
   sensitive = true
 }
 
-resource "azurerm_storage_blob" "example" {
+resource "azurerm_storage_blob" "this" {
   name                   = "main.tf"
   storage_account_name   = azurerm_storage_account.v2.name
   storage_container_name = azurerm_storage_container.wasbs.name
   type                   = "Block"
   source                 = "${path.module}/main.tf"
 }
 
-resource "azurerm_key_vault" "example" {
+resource "azurerm_key_vault" "this" {
   name                     = "${local.prefix}-kv"
-  location                 = azurerm_resource_group.example.location
-  resource_group_name      = azurerm_resource_group.example.name
+  location                 = azurerm_resource_group.this.location
+  resource_group_name      = azurerm_resource_group.this.name
   tenant_id                = data.azurerm_client_config.current.tenant_id
   purge_protection_enabled = false
   sku_name                 = "standard"
   tags                     = local.tags
+}
 
-  access_policy {
-    tenant_id = data.azurerm_client_config.current.tenant_id
-    object_id = data.azurerm_client_config.current.object_id
-    secret_permissions = [
-      "delete", "get", "list", "set"
-    ]
-  }
+resource "azurerm_key_vault_access_policy" "this" {
+  key_vault_id       = azurerm_key_vault.this.id
+  tenant_id          = data.azurerm_client_config.current.tenant_id
+  object_id          = data.azurerm_client_config.current.object_id
+  secret_permissions = ["Delete", "Get", "List", "Set"]
 }
 
 output "test_key_vault_name" {
-  value = azurerm_key_vault.example.name
+  value = azurerm_key_vault.this.name
 }
 
 output "test_key_vault_resource_id" {
-  value = azurerm_key_vault.example.id
+  value = azurerm_key_vault.this.id
 }
 
 output "test_key_vault_dns_name" {
-  value = azurerm_key_vault.example.vault_uri
+  value = azurerm_key_vault.this.vault_uri
 }
 
 // "Key Vault Administrator" role required for SP
-resource "azurerm_key_vault_secret" "example" {
+resource "azurerm_key_vault_secret" "this" {
   name         = "answer"
   value        = "42"
-  key_vault_id = azurerm_key_vault.example.id
+  key_vault_id = azurerm_key_vault.this.id
   tags         = local.tags
 }
 
 output "test_key_vault_secret" {
-  value = azurerm_key_vault_secret.example.name
+  value = azurerm_key_vault_secret.this.name
 }
 
 output "test_key_vault_secret_value" {
   // this is for testing purposes only. 
   // must not be a practice for production.
-  value = azurerm_key_vault_secret.example.value
+  value = azurerm_key_vault_secret.this.value
   sensitive = true
 }
 
diff --git a/storage/adls_gen1_mount.go b/storage/adls_gen1_mount.go
@@ -26,11 +26,13 @@ func (m AzureADLSGen1Mount) Source() string {
 
 // Config ...
 func (m AzureADLSGen1Mount) Config(client *common.DatabricksClient) map[string]string {
+	aadEndpoint := client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint
 	return map[string]string{
 		m.PrefixType + ".oauth2.access.token.provider.type": "ClientCredential",
-		m.PrefixType + ".oauth2.client.id":                  m.ClientID,
-		m.PrefixType + ".oauth2.credential":                 fmt.Sprintf("{secrets/%s/%s}", m.SecretScope, m.SecretKey),
-		m.PrefixType + ".oauth2.refresh.url":                fmt.Sprintf("%s/%s/oauth2/token", client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint, m.TenantID),
+
+		m.PrefixType + ".oauth2.client.id":   m.ClientID,
+		m.PrefixType + ".oauth2.credential":  fmt.Sprintf("{secrets/%s/%s}", m.SecretScope, m.SecretKey),
+		m.PrefixType + ".oauth2.refresh.url": fmt.Sprintf("%s/%s/oauth2/token", aadEndpoint, m.TenantID),
 	}
 }
 
diff --git a/storage/adls_gen2_mount.go b/storage/adls_gen2_mount.go
@@ -27,12 +27,13 @@ func (m AzureADLSGen2Mount) Source() string {
 
 // Config returns mount configurations
 func (m AzureADLSGen2Mount) Config(client *common.DatabricksClient) map[string]string {
+	aadEndpoint := client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint
 	return map[string]string{
 		"fs.azure.account.auth.type":                          "OAuth",
 		"fs.azure.account.oauth.provider.type":                "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",
 		"fs.azure.account.oauth2.client.id":                   m.ClientID,
 		"fs.azure.account.oauth2.client.secret":               fmt.Sprintf("{secrets/%s/%s}", m.SecretScope, m.SecretKey),
-		"fs.azure.account.oauth2.client.endpoint":             fmt.Sprintf("%s/%s/oauth2/token", client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint, m.TenantID),
+		"fs.azure.account.oauth2.client.endpoint":             fmt.Sprintf("%s/%s/oauth2/token", aadEndpoint, m.TenantID),
 		"fs.azure.createRemoteFileSystemDuringInitialization": fmt.Sprintf("%t", m.InitializeFileSystem),
 	}
 }
diff --git a/workspace/acceptance/global_init_script_test.go b/workspace/acceptance/global_init_script_test.go
@@ -12,7 +12,7 @@ func TestAccGlobalInitScriptResource_Create(t *testing.T) {
 			Template: `
 			resource "databricks_global_init_script" "this" {
 				name = "init-{var.RANDOM}"
-				enabled = false
+				enabled = true
 				content_base64 = "ZWNobyBoZWxsbw=="
 			}`,
 		},

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@ import (`
`11`	`11`	`"github.com/stretchr/testify/assert"`
`12`	`12`	`)`
`13`	`13`
`14`		`-func TestAccComplexValueResource(t *testing.T) {`
	`14`	`+func TestAccGroupMemberResource(t *testing.T) {`
`15`	`15`	`acceptance.Test(t, []acceptance.Step{`
`16`	`16`	`{`
`17`	`17`	Template: `
Original file line number	Diff line number	Diff line change
`@@ -27,12 +27,13 @@ func (m AzureADLSGen2Mount) Source() string {`
`27`	`27`
`28`	`28`	`// Config returns mount configurations`
`29`	`29`	`func (m AzureADLSGen2Mount) Config(client *common.DatabricksClient) map[string]string {`
	`30`	`+ aadEndpoint := client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint`
`30`	`31`	`return map[string]string{`
`31`	`32`	`"fs.azure.account.auth.type": "OAuth",`
`32`	`33`	`"fs.azure.account.oauth.provider.type": "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider",`
`33`	`34`	`"fs.azure.account.oauth2.client.id": m.ClientID,`
`34`	`35`	`"fs.azure.account.oauth2.client.secret": fmt.Sprintf("{secrets/%s/%s}", m.SecretScope, m.SecretKey),`
`35`		`- "fs.azure.account.oauth2.client.endpoint": fmt.Sprintf("%s/%s/oauth2/token", client.AzureAuth.AzureEnvironment.ActiveDirectoryEndpoint, m.TenantID),`
	`36`	`+ "fs.azure.account.oauth2.client.endpoint": fmt.Sprintf("%s/%s/oauth2/token", aadEndpoint, m.TenantID),`
`36`	`37`	`"fs.azure.createRemoteFileSystemDuringInitialization": fmt.Sprintf("%t", m.InitializeFileSystem),`
`37`	`38`	`}`
`38`	`39`	`}`