Mahanifi/ml workspace module (#398)

mahhanifi · Mahsa Hanifi · web-flow · commit eaf1224b69c9 · 2020-07-31T13:33:54.000-05:00
* added mlw module with the associated test harness file

* updated the mlw module's version to 2.9

* removed the azurerm version from the version.tf file it was unnecessary

* updated the code with the requested changes on the PR comments

Co-authored-by: Mahsa Hanifi &lt;mahanifi@microsoft.com&gt;
diff --git a/infra/modules/providers/azure/ml-workspace/README.md b/infra/modules/providers/azure/ml-workspace/README.md
@@ -0,0 +1,35 @@
+# Module:  Azure Machine Learning Workspace
+
+This is terraform module can be used to provision an Azure ML Workspace with the following characteristics:
+
+- Ability to specify the existing resource group name in which the ML Workspace will be deployed.
+- Ability to specify the existing resource group location, so that the ML Workspace will be deployed in the same location.
+- Also gives ability to specify following settings for Azure ML Workspace based on the requirements:
+  - resource_group_name : Name of the existing resource group in which the ML Workspace will be created.
+  - application_insights_id : ID of the existing App Insight that is existing in the same resource group.
+  - key_vault_id : ID of the existing KeyVault that is existing in the same resource group.
+  - storage_account_id : ID of the storage account that is existing in the same resource group.
+
+Please click the [link](https://www.terraform.io/docs/providers/azurerm/r/machine_learning_workspace.html) to get additional details on settings in Terraform for Azure Machine Learning Workspace.
+
+## Usage
+
+### Module Definition
+
+ML Workspace Module : infra/modules/providers/azure/ml-workspace
+
+```terraform
+module "ml_workspace_walkthrough_app" {
+    source                      = "../../modules/providers/azure/ml-workspace"
+    name                        = "name of the workspace"
+    resource_group_name         = "name of the resource group"
+    application_insights_id     = "ID of the application insights"
+    key_vault_id                = "ID of the KeyVault"
+    storage_account_id          = "ID of the storage account"
+    sku_name                    = "Basic or Enterprise"
+}
+```
+
+## Outputs
+
+Once the deployments are completed successfully, the output will be generated as it is in the [output.tf file](./output.tf).
diff --git a/infra/modules/providers/azure/ml-workspace/main.tf b/infra/modules/providers/azure/ml-workspace/main.tf
@@ -0,0 +1,21 @@
+
+module "azure-provider" {
+  source = "../provider"
+}
+
+data "azurerm_resource_group" "ml_resource_group" {
+  name = var.resource_group_name
+}
+
+resource "azurerm_machine_learning_workspace" "mlworkspace" {
+  name                    = var.name
+  location                = data.azurerm_resource_group.ml_resource_group.location
+  resource_group_name     = var.resource_group_name
+  application_insights_id = var.application_insights_id
+  key_vault_id            = var.key_vault_id
+  storage_account_id      = var.storage_account_id
+  sku_name                = var.sku_name
+  identity {
+    type = "SystemAssigned" //This is the only supported type at this time
+  }
+}
diff --git a/infra/modules/providers/azure/ml-workspace/output.tf b/infra/modules/providers/azure/ml-workspace/output.tf
@@ -0,0 +1,9 @@
+output "id" {
+  description = "The azure machine learning workspace ID."
+  value       = azurerm_machine_learning_workspace.mlworkspace.id
+}
+
+output "mlw_identity_principal_id" {
+  description = "The ID of the principal(client) in Azure active directory"
+  value       = azurerm_machine_learning_workspace.mlworkspace.identity[0].principal_id
+}
diff --git a/infra/modules/providers/azure/ml-workspace/terraform.tfvars.template b/infra/modules/providers/azure/ml-workspace/terraform.tfvars.template
@@ -0,0 +1,6 @@
+resource_group_name     = ""
+application_insights_id = ""
+key_vault_id            = ""
+storage_account_id      = ""
+sku_name                = ""
+name                    = ""
diff --git a/infra/modules/providers/azure/ml-workspace/tests/.env.testing.template b/infra/modules/providers/azure/ml-workspace/tests/.env.testing.template
@@ -0,0 +1,2 @@
+RESOURCE_GROUP_NAME=""
+ARM_SUBSCRIPTION_ID=""
diff --git a/infra/modules/providers/azure/ml-workspace/tests/integration/mlworkspace_integration_test.go b/infra/modules/providers/azure/ml-workspace/tests/integration/mlworkspace_integration_test.go
@@ -0,0 +1,27 @@
+package integration
+
+import (
+	"os"
+	"testing"
+
+	"github.com/microsoft/cobalt/infra/modules/providers/azure/ml-workspace/tests"
+	"github.com/microsoft/terratest-abstraction/integration"
+)
+
+var subscription = os.Getenv("ARM_SUBSCRIPTION_ID")
+var resorurcegroupname = os.Getenv("RESOURCE_GROUP_NAME")
+
+func TestMLWorkspace(t *testing.T) {
+	testFixture := integration.IntegrationTestFixture{
+		GoTest:                t,
+		TfOptions:             tests.MLWorkspaceTFOptions,
+		ExpectedTfOutputCount: 1,
+		TfOutputAssertions: []integration.TerraformOutputValidation{
+			VerifyCreatedMLworkspace(subscription,
+				"resource_group_name",
+				"id",
+			),
+		},
+	}
+	integration.RunIntegrationTests(&testFixture)
+}
diff --git a/infra/modules/providers/azure/ml-workspace/tests/integration/mlworkspace_tests.go b/infra/modules/providers/azure/ml-workspace/tests/integration/mlworkspace_tests.go
@@ -0,0 +1,29 @@
+package integration
+
+import (
+	"testing"
+
+	"github.com/microsoft/cobalt/test-harness/terratest-extensions/modules/azure"
+	"github.com/microsoft/terratest-abstraction/integration"
+	"github.com/stretchr/testify/require"
+)
+
+// healthCheck - Asserts that the deployment was successful.
+func healthCheck(t *testing.T, provisionState *string) {
+	require.Equal(t, "Succeeded", *provisionState, "The deployment hasn't succeeded.")
+}
+
+// VerifyCreatedMLworkspace - validate the created ML Workspace
+func VerifyCreatedMLworkspace(subscriptionID, resourceGroupName, mlWorkSpaceOutputID string) func(goTest *testing.T, output integration.TerraformOutput) {
+	return func(goTest *testing.T, output integration.TerraformOutput) {
+		mlWorkspace := output[mlWorkSpaceOutputID].(string)
+		mlResourceGroup := output[resourceGroupName].(string)
+
+		mlWorkspaceNameFromAzure := azure.GetMLWorkspaceIDByResourceGroup(
+			goTest,
+			subscriptionID,
+			mlResourceGroup)
+
+		require.Equal(goTest, mlWorkspaceNameFromAzure, mlWorkspace, "The machine learning workspace does not exist")
+	}
+}
diff --git a/infra/modules/providers/azure/ml-workspace/tests/test.tfvars b/infra/modules/providers/azure/ml-workspace/tests/test.tfvars
@@ -0,0 +1,6 @@
+resource_group_name     = ""
+application_insights_id = ""
+key_vault_id            = ""
+storage_account_id      = ""
+sku_name                = ""
+name                    = ""
diff --git a/infra/modules/providers/azure/ml-workspace/tests/tf_options.go b/infra/modules/providers/azure/ml-workspace/tests/tf_options.go
@@ -0,0 +1,16 @@
+package tests
+
+import (
+	"os"
+
+	"github.com/gruntwork-io/terratest/modules/terraform"
+)
+
+// ResourceGroupName - The Resource Group Name
+var ResourceGroupName = os.Getenv("RESOURCE_GROUP_NAME")
+
+// MLWorkspaceTFOptions common terraform options used for unit and integration testing
+var MLWorkspaceTFOptions = &terraform.Options{
+	TerraformDir: "../../",
+	VarFiles:     []string{"./tests/test.tfvars"},
+}
diff --git a/infra/modules/providers/azure/ml-workspace/tests/unit/ml_workspace_unit_test.go b/infra/modules/providers/azure/ml-workspace/tests/unit/ml_workspace_unit_test.go
@@ -0,0 +1,44 @@
+package unit
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"github.com/gruntwork-io/terratest/modules/random"
+	tests "github.com/microsoft/cobalt/infra/modules/providers/azure/ml-workspace/tests"
+	"github.com/microsoft/terratest-abstraction/unit"
+)
+
+var resourceCount = 1
+var workspace = "sample-" + strings.ToLower(random.UniqueId())
+
+// helper function to parse blocks of JSON into a generic Go map
+func asMap(t *testing.T, jsonString string) map[string]interface{} {
+	var theMap map[string]interface{}
+	if err := json.Unmarshal([]byte(jsonString), &theMap); err != nil {
+		t.Fatal(err)
+	}
+	return theMap
+}
+
+func TestAzureMLWorkspaceDeployment_Unit(t *testing.T) {
+
+	expectedResult := map[string]interface{}{
+		"name":                "demo0012",
+		"resource_group_name": "azmltest",
+		"sku_name":            "Enterprise",
+	}
+	testFixture := unit.UnitTestFixture{
+		GoTest:                t,
+		TfOptions:             tests.MLWorkspaceTFOptions,
+		Workspace:             workspace,
+		PlanAssertions:        nil,
+		ExpectedResourceCount: resourceCount,
+		ExpectedResourceAttributeValues: unit.ResourceDescription{
+			"azurerm_machine_learning_workspace.mlworkspace": expectedResult,
+		},
+	}
+
+	unit.RunUnitTests(&testFixture)
+}
diff --git a/infra/modules/providers/azure/ml-workspace/variables.tf b/infra/modules/providers/azure/ml-workspace/variables.tf
@@ -0,0 +1,25 @@
+variable "resource_group_name" {
+  description = "Name of the Resource Group in which the Machine Learning Workspace should exist. Changing this forces a new resource to be created"
+  type        = string
+}
+variable "name" {
+  description = "Name of the Machine Learning Workspace. Changing this forces a new resource to be created"
+  type        = string
+}
+variable "application_insights_id" {
+  description = "Name of the Application Insights associated with this Machine Learning Workspace. Changing this forces a new resource to be created"
+  type        = string
+}
+variable "key_vault_id" {
+  description = "Name of key vault associated with this Machine Learning Workspace. Changing this forces a new resource to be created"
+  type        = string
+}
+variable "storage_account_id" {
+  description = "The name of the Storage Account associated with this Machine Learning Workspace. Changing this forces a new resource to be created"
+  type        = string
+}
+variable "sku_name" {
+  description = "SKU/edition of the Machine Learning Workspace"
+  type        = string
+  default     = "Enterprise"
+}
diff --git a/infra/modules/providers/azure/ml-workspace/versions.tf b/infra/modules/providers/azure/ml-workspace/versions.tf
@@ -0,0 +1,4 @@
+
+terraform {
+  required_version = ">= 0.12"
+}
diff --git a/test-harness/terratest-extensions/modules/azure/mlworkspace.go b/test-harness/terratest-extensions/modules/azure/mlworkspace.go
@@ -0,0 +1,54 @@
+package azure
+
+import (
+	"context"
+	"testing"
+
+	"github.com/Azure/azure-sdk-for-go/services/machinelearningservices/mgmt/2020-01-01/machinelearningservices"
+)
+
+func mlWorkSpaceClientE(subscriptionID string) (*machinelearningservices.WorkspacesClient, error) {
+	authorizer, err := DeploymentServicePrincipalAuthorizer()
+	if err != nil {
+		return nil, err
+	}
+	client := machinelearningservices.NewWorkspacesClient(subscriptionID)
+	client.Authorizer = authorizer
+	return &client, err
+}
+
+// GetMLWorkspaceIDByResourceGroup -
+func GetMLWorkspaceIDByResourceGroup(t *testing.T, subscriptionID, resourceGroupName string) string {
+	dfList, err := listMlWorkspaceByResourceGroupE(subscriptionID, resourceGroupName)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	result := ""
+	for _, workspace := range *dfList {
+		result = *workspace.ID
+		break
+	}
+	return result
+}
+
+func listMlWorkspaceByResourceGroupE(subscriptionID, resourceGroupName string) (*[]machinelearningservices.Workspace, error) {
+	dfClient, err := mlWorkSpaceClientE(subscriptionID)
+
+	if err != nil {
+		return nil, err
+	}
+	iteratorMlWorkspace, err := dfClient.ListByResourceGroupComplete(context.Background(), resourceGroupName, "")
+	if err != nil {
+		return nil, err
+	}
+	mlWorkspaceList := make([]machinelearningservices.Workspace, 0)
+	for iteratorMlWorkspace.NotDone() {
+		mlWorkspaceList = append(mlWorkspaceList, iteratorMlWorkspace.Value())
+		err = iteratorMlWorkspace.Next()
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &mlWorkspaceList, err
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+RESOURCE_GROUP_NAME=""`
	`2`	`+ARM_SUBSCRIPTION_ID=""`