Prepare 0.2.8 (#400)

* Pre-release fixing
* Added NAT to BYOVPC terraform module
* added instance profile locks
* Added sync block for instance profiles integration tests
* Fix #383 Cleaning up clusters that fail to start
* Added log delivery use case docs
* Fix #382 - ignore changes to deployment_name
* Fix test and lints
* Fix #382 by ignoring incoming prefix for deployment_name for databricks_mws_workspaces
* Improve documentation to fix #368
* fix linting issues

Co-authored-by: Serge Smertin <[email protected]>

Author: nfx

CHANGELOG.md

@@ -1,18 +1,17 @@
 # Version changelog
 
-## 0.2.9
-
-* Fixed restarting cluster on changes in cluster configuration aren't related to the cluster configuration ([issue #379](https://github.com/databrickslabs/terraform-provider-databricks/issues/379))
-
 ## 0.2.8
 
-* Added [databricks_workspace_conf](https://github.com/databrickslabs/terraform-provider-databricks/pull/398) resource
+* Added [databricks_workspace_conf](https://github.com/databrickslabs/terraform-provider-databricks/pull/398) resource.
 * Added [databricks_mws_log_delivery](https://github.com/databrickslabs/terraform-provider-databricks/pull/343) resource for billable usage & audit logs consumption.
 * Added [databricks_node_type](https://github.com/databrickslabs/terraform-provider-databricks/pull/376) data source for simpler selection of node types across AWS & Azure.
-* Added [Azure Key Vault support](https://github.com/databrickslabs/terraform-provider-databricks/pull/381) for databricks_secret_scope for Azure CLI authenticated users.
+* Added [Azure Key Vault support](https://github.com/databrickslabs/terraform-provider-databricks/pull/381) for `databricks_secret_scope` for Azure CLI authenticated users.
 * Added [is_pinned](https://github.com/databrickslabs/terraform-provider-databricks/pull/348) support for `databricks_cluster` resource.
-* Fixed [single node clusters](https://docs.databricks.com/clusters/single-node.html) support by allowing [`num_workers` to be `0`](https://github.com/databrickslabs/terraform-provider-databricks/pull/375).
+* Fixed restarting cluster on changes in cluster configuration aren't related to the cluster configuration ([issue #379](https://github.com/databrickslabs/terraform-provider-databricks/issues/379))
+* Fixed issue [#383](https://github.com/databrickslabs/terraform-provider-databricks/issues/383) by cleaning up clusters that fail to start.
+* Fixed issue [#382](https://github.com/databrickslabs/terraform-provider-databricks/issues/382) by ignoring any incoming changes to deployment name of `databricks_mws_workspaces`, as well as propagating the right error messages.
 * Internal: API for retrieval of the cluster events.
+* Increased code coverage to 71%.
 
 Updated dependency versions:
 
@@ -22,6 +21,9 @@ Updated dependency versions:
 * github.com/Azure/go-autorest/autorest/azure/cli v0.4.2
 * gopkg.in/ini.v1 1.62.0
 
+**Deprecations**
+* `network_error_messages` from `databricks_mws_workspaces` is deprecated and going to be removed in 0.3
+
 ## 0.2.7
 
 * Small fixes

access/resource_secret_scope.go

@@ -106,7 +106,7 @@ func (a SecretScopesAPI) Read(scopeName string) (SecretScope, error) {
 // ResourceSecretScope manages secret scopes
 func ResourceSecretScope() *schema.Resource {
 	s := internal.StructToSchema(SecretScope{}, func(s map[string]*schema.Schema) map[string]*schema.Schema {
-		// TODO: CustomDiffFunc for initial_manage_principal & importing
+		// TODO: DiffSuppressFunc for initial_manage_principal & importing
 		s["name"].ForceNew = true
 		s["initial_manage_principal"].ForceNew = true
 		s["keyvault_metadata"].ForceNew = true

common/version.go

@@ -2,35 +2,14 @@ package common
 
 import (
 	"fmt"
-	"os/exec"
-	"strings"
-	"sync"
 )
 
 var (
-	version    = ""
-	onceGitVer sync.Once
+	version = "0.2.8"
 )
 
-func versionFromGit() string {
-	out, err := exec.Command("git", "describe",
-		"--always", "--long").Output()
-	ee, ok := err.(*exec.ExitError)
-	if ok && !ee.Success() || err != nil {
-		return "dev"
-	}
-	return strings.TrimSpace(strings.ReplaceAll(string(out), "v", ""))
-}
-
 // Version returns version of provider
 func Version() string {
-	if version == "" {
-		onceGitVer.Do(func() {
-			// calling git once per process to know
-			// the version that integration tests are running
-			version = versionFromGit()
-		})
-	}
 	return version
 }
 

compute/clusters.go

@@ -32,6 +32,15 @@ func (a ClustersAPI) Create(cluster Cluster) (info ClusterInfo, err error) {
 		return
 	}
 	info, err = a.waitForClusterStatus(ci.ClusterID, ClusterStateRunning)
+	if err != nil {
+		// https://github.com/databrickslabs/terraform-provider-databricks/issues/383
+		log.Printf("[ERROR] Cleaning up created cluster, that failed to start: %s", err.Error())
+		deleteErr := a.PermanentDelete(ci.ClusterID)
+		if deleteErr != nil {
+			log.Printf("[ERROR] Failed : %s", deleteErr.Error())
+			err = deleteErr
+		}
+	}
 	return
 }
 

compute/clusters_test.go

@@ -596,8 +596,7 @@ func TestAzureAccNodeTypes(t *testing.T) {
 	clustersAPI := NewClustersAPI(common.CommonEnvironmentClient())
 	m := map[string]NodeTypeRequest{
 		"Standard_F4s":     {},
-		"Standard_L8s_v2":  {LocalDisk: true},
-		"Standard_NC6s_v3": {MinGPUs: 1},
+		"Standard_NC12":    {MinGPUs: 1},
 		"Standard_L32s_v2": {MinCores: 32, GBPerCore: 8},
 	}
 

compute/model.go

@@ -228,7 +228,7 @@ type Cluster struct {
 	ClusterName string `json:"cluster_name,omitempty"`
 
 	SparkVersion              string     `json:"spark_version"` // TODO: perhaps make a default
-	NumWorkers                int32      `json:"num_workers" tf:"group:size"`
+	NumWorkers                int32      `json:"num_workers,omitempty" tf:"group:size"`
 	Autoscale                 *AutoScale `json:"autoscale,omitempty" tf:"group:size"`
 	EnableElasticDisk         bool       `json:"enable_elastic_disk,omitempty" tf:"computed"`
 	EnableLocalDiskEncryption bool       `json:"enable_local_disk_encryption,omitempty"`
@@ -260,7 +260,7 @@ type ClusterList struct {
 
 // ClusterInfo contains the information when getting cluster info from the get request.
 type ClusterInfo struct {
-	NumWorkers                int32              `json:"num_workers"`
+	NumWorkers                int32              `json:"num_workers,omitempty"`
 	AutoScale                 *AutoScale         `json:"autoscale,omitempty"`
 	ClusterID                 string             `json:"cluster_id,omitempty"`
 	CreatorUserName           string             `json:"creator_user_name,omitempty"`

compute/resource_cluster.go

@@ -102,11 +102,6 @@ func resourceClusterSchema() map[string]*schema.Schema {
 			p.Sensitive = true
 		}
 		s["autotermination_minutes"].Default = 60
-		s["num_workers"] = &schema.Schema{
-			Type:     schema.TypeInt,
-			Optional: true,
-			Default:  0,
-		}
 		s["idempotency_token"].ForceNew = true
 		s["cluster_id"] = &schema.Schema{
 			Type:     schema.TypeString,

compute/resource_cluster_test.go

@@ -495,73 +495,6 @@ func TestResourceClusterCreate_WithLibraries(t *testing.T) {
 	assert.Equal(t, "abc", d.Id())
 }
 
-func TestResourceClusterCreate_WithoutNumWorkers(t *testing.T) {
-	d, err := qa.ResourceFixture{
-		Fixtures: []qa.HTTPFixture{
-			{
-				Method:   "POST",
-				Resource: "/api/2.0/clusters/create",
-				ExpectedRequest: Cluster{
-					NumWorkers:             0,
-					ClusterName:            "Single Node Cluster",
-					SparkVersion:           "7.1-scala12",
-					NodeTypeID:             "dev-tier-node",
-					AutoterminationMinutes: 120,
-				},
-				Response: ClusterInfo{
-					ClusterID: "abc",
-					State:     ClusterStateRunning,
-				},
-			},
-			{
-				Method:   "POST",
-				Resource: "/api/2.0/clusters/events",
-				ExpectedRequest: EventsRequest{
-					ClusterID:  "abc",
-					Limit:      1,
-					Order:      SortDescending,
-					EventTypes: []ClusterEventType{EvTypePinned, EvTypeUnpinned},
-				},
-				Response: EventsResponse{
-					Events:     []ClusterEvent{},
-					TotalCount: 0,
-				},
-			},
-			{
-				Method:       "GET",
-				ReuseRequest: true,
-				Resource:     "/api/2.0/clusters/get?cluster_id=abc",
-				Response: ClusterInfo{
-					ClusterID:              "abc",
-					ClusterName:            "Single Node Cluster",
-					SparkVersion:           "7.1-scala12",
-					NodeTypeID:             "dev-tier-node",
-					AutoterminationMinutes: 120,
-					State:                  ClusterStateRunning,
-				},
-			},
-			{
-				Method:   "GET",
-				Resource: "/api/2.0/libraries/cluster-status?cluster_id=abc",
-				Response: ClusterLibraryStatuses{
-					LibraryStatuses: []LibraryStatus{},
-				},
-			},
-		},
-		Create:   true,
-		Resource: ResourceCluster(),
-		State: map[string]interface{}{
-			"autotermination_minutes": 120,
-			"cluster_name":            "Single Node Cluster",
-			"spark_version":           "7.1-scala12",
-			"node_type_id":            "dev-tier-node",
-			"is_pinned":               false,
-		},
-	}.Apply(t)
-	assert.NoError(t, err, err)
-	assert.Equal(t, 0, d.Get("num_workers"))
-}
-
 func TestResourceClusterCreate_Error(t *testing.T) {
 	d, err := qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{

docs/resources/cluster.md

@@ -35,7 +35,7 @@ disable automatic termination. _It is highly recommended to have this setting pr
 * `idempotency_token` - (Optional) An optional token that can be used to guarantee the idempotency of cluster creation requests. If an active cluster with the provided token already exists, the request will not create a new cluster, but it will return the ID of the existing cluster instead. The existence of a cluster with the same token is not checked against terminated clusters. If you specify the idempotency token, upon failure you can retry until the request succeeds. Databricks will guarantee that exactly one cluster will be launched with that idempotency token. This token should have at most 64 characters.
 * `ssh_public_keys` - (Optional) SSH public key contents that will be added to each Spark node in this cluster. The corresponding private keys can be used to login with the user name ubuntu on port 2200. Up to 10 keys can be specified.
 * `spark_env_vars` - (Optional) Map with environment variable key-value pairs to fine tune Spark clusters. Key-value pair of the form (X,Y) are exported as is (i.e., export X='Y') while launching the driver and workers. To specify an additional set of SPARK_DAEMON_JAVA_OPTS, we recommend appending them to $SPARK_DAEMON_JAVA_OPTS as shown in the example below. This ensures that all default databricks managed environmental variables are included as well. 
-* `custom_tags` - (Optional) Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS instances and EBS volumes) with these tags in addition to `default_tags`.
+* `custom_tags` - (Optional) Additional tags for cluster resources. Databricks will tag all cluster resources (e.g., AWS EC2 instances and EBS volumes) with these tags in addition to `default_tags`.
 * `spark_conf` - (Optional) Map with key-value pairs to fine tune Spark clusters, where you can provide custom pSpark configuration properties](https://spark.apache.org/docs/latest/configuration.html) in a cluster configuration. You can also pass in a string of extra JVM options to the driver and the executors via `spark.driver.extraJavaOptions` and `spark.executor.extraJavaOptions` respectively. It is advised to keep all common configurations in [Cluster Policies](cluster_policy.md) to maintain control of the environments launched.
 * `is_pinned` - (Optional) boolean value specifying if cluster is pinned (not pinned by default).  If flag is set, then cluster is pinned using [corresponding API](https://docs.databricks.com/dev-tools/api/latest/clusters.html#pin).  You must be a Databricks administrator to use this.  The maximal number of the pinned clusters is also [limited to 20](https://docs.databricks.com/clusters/clusters-manage.html#pin-a-cluster), so task may fail if you have more than that.
 
@@ -67,10 +67,6 @@ resource "databricks_cluster" "shared_autoscaling" {
 
 When you [create a Databricks cluster](https://docs.databricks.com/clusters/configure.html#cluster-size-and-autoscaling), you can either provide a `num_workers` for the fixed size cluster or provide `min_workers` and/or `max_workers` for the cluster withing `autoscale` group. When you provide a fixed size cluster, Databricks ensures that your cluster has the specified number of workers. When you provide a range for the number of workers, Databricks chooses the appropriate number of workers required to run your job. This is referred to as autoscaling. With autoscaling, Databricks dynamically reallocates workers to account for the characteristics of your job. Certain parts of your pipeline may be more computationally demanding than others, and Databricks automatically adds additional workers during these phases of your job (and removes them when they’re no longer needed). It is advised to keep all common configurations in [Cluster Policies](cluster_policy.md) to maintain control of the environments launched.
 
-When using a [Single Node cluster](https://docs.databricks.com/clusters/single-node.html), `num_workers` needs to be `0`. In this case you can choose to explicitly add the argument or not, as it defaults to `0`.
-
-* `num_workers` - (Optional) Number of worker nodes that this cluster should have. A cluster has one Spark Driver and num_workers Executors for a total of num_workers + 1 Spark node. Set to `0` when not provided.
-
 `autoscale` optional configuration block supports the following:
 
 * `min_workers` - (Optional) The minimum number of workers to which the cluster can scale down when underutilized. It is also the initial number of workers the cluster will have after creation.

docs/resources/mws_log_delivery.md

@@ -83,7 +83,9 @@ resource "databricks_mws_log_delivery" "audit_logs" {
 
 ## Billable Usage
 
-CSV files with [static schema](https://docs.databricks.com/administration-guide/account-settings/usage.html) are delivered to `<delivery_path_prefix>/billable-usage/csv/`. Files are named `workspaceId=<workspace-id>-usageMonth=<month>.csv`, which are delivered daily by overwriting the month's CSV file for each workspace. 
+CSV files are delivered to `<delivery_path_prefix>/billable-usage/csv/` and are named `workspaceId=<workspace-id>-usageMonth=<month>.csv`, which are delivered daily by overwriting the month's CSV file for each workspace. Format of CSV file, as well as some usage examples, can be found [here](https://docs.databricks.com/administration-guide/account-settings/usage.html#download-usage-as-a-csv-file). 
+
+Common processing scenario is to apply [cost allocation tags](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html), that could be enforced by setting [custom_tags](cluster.md#custom_tags) on a cluster or through [cluster policy](cluster_policy.md). Report contains `clusterId` field, that could be joined with data from AWS [cost and usage reports](https://docs.aws.amazon.com/cur/latest/userguide/cur-create.html), that can be joined with `user:ClusterId` tag from AWS usage report.
 
 ```hcl
 resource "databricks_mws_log_delivery" "usage_logs" {