Exporter: improve exporting of databricks_cluster_policy resource (#2680)

This includes:

- Listing of all cluster policies, not only that are referenced in the jobs/clusters.
- Emit secret scopes when they are referenced in Spark Conf or environment variables

Also, fixes generation of non-existent secret scopes that happens when we emit
non-existent secret scope (for example, it's a dangling reference in cluster policy or old
job configuration).

This fixes #2664

Author: alexott

docs/guides/experimental-exporter.md

@@ -55,26 +55,27 @@ All arguments are optional and they tune what code is being generated.
 Services are just logical groups of resources used for filtering and organization in files written in `-directory`. All resources are globally sorted by their resource name, which technically allows you to use generated files for compliance purposes. Nevertheless, managing the entire Databricks workspace with Terraform is the preferred way. With the exception of notebooks and possibly libraries, which may have their own CI/CD processes.
 
 * `access` - [databricks_permissions](../resources/permissions.md), [databricks_instance_profile](../resources/instance_profile.md) and [databricks_ip_access_list](../resources/ip_access_list.md).
-* `compute` - **listing** [databricks_cluster](../resources/cluster.md). Includes [cluster policies](../resources/cluster_policy.md).
-* `directories` - **listing** [databricks_directory](../resources/directory.md)
-* `dlt` - **listing** [databricks_pipeline](../resources/pipeline.md)
+* `compute` - **listing** [databricks_cluster](../resources/cluster.md).
+* `directories` - **listing** [databricks_directory](../resources/directory.md).
+* `dlt` - **listing** [databricks_pipeline](../resources/pipeline.md).
 * `groups` - [databricks_group](../data-sources/group.md) with [membership](../resources/group_member.md) and [data access](../resources/group_instance_profile.md).
 * `jobs` - **listing** [databricks_job](../resources/job.md). Usually, there are more automated jobs than interactive clusters, so they get their own file in this tool's output.
 * `mlflow-webhooks` - **listing** [databricks_mlflow_webhook](../resources/mlflow_webhook.md).
 * `model-serving` - **listing** [databricks_model_serving](../resources/model_serving.md).
 * `mounts` - **listing** works only in combination with `-mounts` command-line option.
-* `notebooks` - **listing** [databricks_notebook](../resources/notebook.md) and [databricks_workspace_file](../resources/workspace_file.md)
+* `notebooks` - **listing** [databricks_notebook](../resources/notebook.md) and [databricks_workspace_file](../resources/workspace_file.md).
+* `policies` - **listing** [databricks_cluster_policy](../resources/cluster_policy).
 * `pools` - **listing** [instance pools](../resources/instance_pool.md).
-* `repos` - **listing** [databricks_repo](../resources/repo.md)
+* `repos` - **listing** [databricks_repo](../resources/repo.md).
 * `secrets` - **listing** [databricks_secret_scope](../resources/secret_scope.md) along with [keys](../resources/secret.md) and [ACLs](../resources/secret_acl.md). 
 * `sql-alerts` - **listing** [databricks_sql_alert](../resources/sql_alert.md).
-* `sql-dashboards` - **listing** [databricks_sql_dashboard](../resources/sql_dashboard.md) along with associated [databricks_sql_widget](../resources/sql_widget.md) and [databricks_sql_visualization](../resources/sql_visualization.md)
 * `sql-dashboards` - **listing** [databricks_sql_dashboard](../resources/sql_dashboard.md) along with associated [databricks_sql_widget](../resources/sql_widget.md) and [databricks_sql_visualization](../resources/sql_visualization.md).
-* `sql-endpoints` - **listing** [databricks_sql_endpoint](../resources/sql_endpoint.md) along with [databricks_sql_global_config](../resources/sql_global_config.md)
-* `sql-queries` - **listing** [databricks_sql_query](../resources/sql_query.md)
+* `sql-dashboards` - **listing** [databricks_sql_dashboard](../resources/sql_dashboard.md) along with associated [databricks_sql_widget](../resources/sql_widget.md) and [databricks_sql_visualization](../resources/sql_visualization.md).
+* `sql-endpoints` - **listing** [databricks_sql_endpoint](../resources/sql_endpoint.md) along with [databricks_sql_global_config](../resources/sql_global_config.md).
+* `sql-queries` - **listing** [databricks_sql_query](../resources/sql_query.md).
 * `storage` - any referenced [databricks_dbfs_file](../resources/dbfs_file.md) will be downloaded locally and properly arranged into terraform state.
-* `users` - [databricks_user](../resources/user.md) and [databricks_service_principal](../resources/service_principal.md) are written to their own file, simply because of their amount. If you use SCIM provisioning, the only use case for importing `users` service is to migrate workspaces.
-* `workspace` - [databricks_workspace_conf](../resources/workspace_conf.md) and [databricks_global_init_script](../resources/global_init_script.md)
+* `users` - [databricks_user](../resources/user.md) and [databricks_service_principal](../resources/service_principal.md) are written to their own file, simply because of their amount. If you use SCIM provisioning, the only use-case for importing `users` service is to migrate workspaces.
+* `workspace` - [databricks_workspace_conf](../resources/workspace_conf.md) and [databricks_global_init_script](../resources/global_init_script.md).
 
 ## Secrets
 

exporter/exporter_test.go

@@ -233,6 +233,13 @@ var emptyPipelines = qa.HTTPFixture{
 	Response:     pipelines.PipelineListResponse{},
 }
 
+var emptyClusterPolicies = qa.HTTPFixture{
+	Method:       "GET",
+	ReuseRequest: true,
+	Resource:     "/api/2.0/policies/clusters/list?",
+	Response:     compute.ListPoliciesResponse{},
+}
+
 var emptyMlflowWebhooks = qa.HTTPFixture{
 	Method:       "GET",
 	ReuseRequest: true,
@@ -354,6 +361,7 @@ func TestImportingUsersGroupsSecretScopes(t *testing.T) {
 			emptySqlQueries,
 			emptySqlAlerts,
 			emptyPipelines,
+			emptyClusterPolicies,
 			emptyWorkspaceConf,
 			allKnownWorkspaceConfs,
 			dummyWorkspaceConf,
@@ -555,6 +563,7 @@ func TestImportingNoResourcesError(t *testing.T) {
 			emptyMlflowWebhooks,
 			emptyWorkspaceConf,
 			emptyInstancePools,
+			emptyClusterPolicies,
 			dummyWorkspaceConf,
 			{
 				Method:   "GET",
@@ -751,6 +760,30 @@ func TestImportingClusters(t *testing.T) {
 				ReuseRequest: true,
 				Response:     getJSONObject("test-data/get-job-14-permissions.json"),
 			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/secrets/list?scope=some-kv-scope",
+				ReuseRequest: true,
+				Response:     getJSONObject("test-data/secret-scopes-list-scope-response.json"),
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/secrets/acls/list?scope=some-kv-scope",
+				ReuseRequest: true,
+				Response:     getJSONObject("test-data/secret-scopes-list-scope-acls-response.json"),
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/secrets/acls/get?principal=test%40test.com&scope=some-kv-scope",
+				ReuseRequest: true,
+				Response:     getJSONObject("test-data/secret-scopes-get-principal-response.json"),
+			},
+			{
+				Method:       "GET",
+				Resource:     "/api/2.0/secrets/scopes/list",
+				ReuseRequest: true,
+				Response:     getJSONObject("test-data/secret-scopes-response.json"),
+			},
 		},
 		func(ctx context.Context, client *common.DatabricksClient) {
 			os.Setenv("EXPORTER_PARALLELISM_databricks_cluster", "1")

exporter/importables.go

@@ -13,6 +13,7 @@ import (
 
 	"golang.org/x/exp/slices"
 
+	"github.com/databricks/databricks-sdk-go/service/compute"
 	"github.com/databricks/databricks-sdk-go/service/ml"
 	"github.com/databricks/databricks-sdk-go/service/settings"
 	"github.com/databricks/terraform-provider-databricks/clusters"
@@ -551,10 +552,38 @@ var resourcesMap map[string]importable = map[string]importable{
 		},
 	},
 	"databricks_cluster_policy": {
-		Service: "compute",
+		Service: "policies",
 		Name: func(ic *importContext, d *schema.ResourceData) string {
 			return d.Get("name").(string)
 		},
+		List: func(ic *importContext) error {
+			w, err := ic.Client.WorkspaceClient()
+			if err != nil {
+				return err
+			}
+			policies, err := w.ClusterPolicies.ListAll(ic.Context, compute.ListClusterPoliciesRequest{})
+			if err != nil {
+				return err
+			}
+			for offset, policy := range policies {
+				log.Printf("[INFO] Scanning %d:  %v", offset+1, policy)
+				if slices.Contains(predefinedClusterPolicies, policy.Name) {
+					continue
+				}
+				if !ic.MatchesName(policy.Name) {
+					log.Printf("[DEBUG] Policy %s doesn't match %s filter", policy.Name, ic.match)
+					continue
+				}
+				ic.Emit(&resource{
+					Resource: "databricks_cluster_policy",
+					ID:       policy.PolicyId,
+				})
+				if offset%10 == 0 {
+					log.Printf("[INFO] Scanned %d of %d cluster policies", offset+1, len(policies))
+				}
+			}
+			return nil
+		},
 		Import: func(ic *importContext, r *resource) error {
 			if ic.meAdmin {
 				ic.Emit(&resource{
@@ -573,7 +602,8 @@ var resourcesMap map[string]importable = map[string]importable{
 				defaultValue, dok := policy["defaultValue"]
 				typ := policy["type"]
 				if !vok && !dok {
-					log.Printf("[INFO] Skipping policy element as it doesn't have both value and defaultValue")
+					log.Printf("[DEBUG] Skipping policy element as it doesn't have both value and defaultValue. k='%v', policy='%v'",
+						k, policy)
 					continue
 				}
 				if k == "aws_attributes.instance_profile_arn" {
@@ -599,6 +629,15 @@ var resourcesMap map[string]importable = map[string]importable{
 						ID:       eitherString(value, defaultValue),
 					})
 				}
+				if typ == "fixed" && (strings.HasPrefix(k, "spark_conf.") || strings.HasPrefix(k, "spark_env_vars.")) {
+					either := eitherString(value, defaultValue)
+					if res := secretPathRegex.FindStringSubmatch(either); res != nil {
+						ic.Emit(&resource{
+							Resource: "databricks_secret_scope",
+							ID:       res[1],
+						})
+					}
+				}
 			}
 			policyName := r.Data.Get("name").(string)
 			if slices.Contains(predefinedClusterPolicies, policyName) {
@@ -946,6 +985,9 @@ var resourcesMap map[string]importable = map[string]importable{
 			}
 			return nil
 		},
+		Ignore: func(ic *importContext, r *resource) bool {
+			return r.Data.Get("name").(string) == ""
+		},
 	},
 	"databricks_secret": {
 		Service: "secrets",

exporter/importables_test.go

@@ -626,6 +626,61 @@ func TestSecretScopeListNoNameMatch(t *testing.T) {
 	})
 }
 
+func TestPoliciesListing(t *testing.T) {
+	qa.HTTPFixturesApply(t, []qa.HTTPFixture{
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/policies/clusters/list?",
+			Response: compute.ListPoliciesResponse{
+				Policies: []compute.Policy{
+					{
+						Name:     "Personal Compute",
+						PolicyId: "123",
+					},
+					{
+						Name:     "abcd",
+						PolicyId: "456",
+					},
+				},
+			},
+		},
+	}, func(ctx context.Context, client *common.DatabricksClient) {
+		ic := importContextForTest()
+		ic.Client = client
+		ic.Context = ctx
+		err := resourcesMap["databricks_cluster_policy"].List(ic)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(ic.testEmits))
+	})
+}
+
+func TestPoliciesListNoNameMatch(t *testing.T) {
+	qa.HTTPFixturesApply(t, []qa.HTTPFixture{
+		{
+			Method:   "GET",
+			Resource: "/api/2.0/policies/clusters/list?",
+			Response: compute.ListPoliciesResponse{
+				Policies: []compute.Policy{
+					{
+						Name: "Personal Compute",
+					},
+					{
+						Name: "abcd",
+					},
+				},
+			},
+		},
+	}, func(ctx context.Context, client *common.DatabricksClient) {
+		ic := importContextForTest()
+		ic.Client = client
+		ic.Context = ctx
+		ic.match = "bcd"
+		err := resourcesMap["databricks_cluster_policy"].List(ic)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(ic.testEmits))
+	})
+}
+
 func TestAwsS3MountProfile(t *testing.T) {
 	ic := importContextForTest()
 	ic.mounts = true

exporter/test-data/get-cluster-policy.json

@@ -1,6 +1,6 @@
 {
   "created_at_timestamp": 1606308550000,
-  "definition": "{\"aws_attributes.instance_profile_arn\":{\"hidden\":true,\"type\":\"fixed\",\"value\":\"arn:aws:iam::12345:instance-profile/shard-s3-access\"},\"instance_pool_id\":{\"hidden\":true,\"type\":\"fixed\",\"value\":\"pool1\"},\"autoscale.max_workers\":{\"defaultValue\":2,\"maxValue\":5,\"type\":\"range\"}}",
+  "definition": "{\"aws_attributes.instance_profile_arn\":{\"hidden\":true,\"type\":\"fixed\",\"value\":\"arn:aws:iam::12345:instance-profile/shard-s3-access\"},\"instance_pool_id\":{\"hidden\":true,\"type\":\"fixed\",\"value\":\"pool1\"},\"spark_conf.abc\":{\"hidden\":true,\"type\":\"fixed\",\"value\":\"{{secrets/some-kv-scope/secret}}\"},\"autoscale.max_workers\":{\"defaultValue\":2,\"maxValue\":5,\"type\":\"range\"}}",
   "name": "users cluster policy",
   "policy_id": "123"
 }