From 2fdc1925296a11db074fc5effbf0430fe5ad9124 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 16:08:52 +0530 Subject: [PATCH 1/9] [Feature] Unified provider support for jobs and cluster --- clusters/data_cluster.go | 1 + clusters/data_clusters.go | 1 + clusters/data_node_type.go | 1 + clusters/data_spark_version.go | 9 +- clusters/data_zones.go | 1 + clusters/resource_cluster.go | 13 ++- clusters/resource_library.go | 11 +- common/unified_provider.go | 81 +++++++++++++ jobs/data_jobs.go | 1 + jobs/data_jobs_acc_test.go | 99 ++++++++++++++++ jobs/job_test.go | 208 +++++++++++++++++++++++++++++++++ jobs/resource_job.go | 14 ++- 12 files changed, 426 insertions(+), 14 deletions(-) create mode 100644 common/unified_provider.go create mode 100644 jobs/data_jobs_acc_test.go diff --git a/clusters/data_cluster.go b/clusters/data_cluster.go index aee0503619..b114ac8034 100644 --- a/clusters/data_cluster.go +++ b/clusters/data_cluster.go @@ -11,6 +11,7 @@ import ( func DataSourceCluster() common.Resource { return common.WorkspaceData(func(ctx context.Context, data *struct { + common.Namespace Id string `json:"id,omitempty" tf:"computed"` ClusterId string `json:"cluster_id,omitempty" tf:"computed"` Name string `json:"cluster_name,omitempty" tf:"computed"` diff --git a/clusters/data_clusters.go b/clusters/data_clusters.go index 3fbf55c5b4..aa0579b07f 100644 --- a/clusters/data_clusters.go +++ b/clusters/data_clusters.go @@ -11,6 +11,7 @@ import ( func DataSourceClusters() common.Resource { return common.WorkspaceData(func(ctx context.Context, data *struct { + common.Namespace Id string `json:"id,omitempty" tf:"computed"` Ids []string `json:"ids,omitempty" tf:"computed,slice_set"` ClusterNameContains string `json:"cluster_name_contains,omitempty"` diff --git a/clusters/data_node_type.go b/clusters/data_node_type.go index 6777689ece..188a5e7462 100644 --- a/clusters/data_node_type.go +++ b/clusters/data_node_type.go @@ -12,6 +12,7 @@ import ( ) type NodeTypeRequest struct { + common.Namespace compute.NodeTypeRequest Arm bool `json:"arm,omitempty"` } diff --git a/clusters/data_spark_version.go b/clusters/data_spark_version.go index 6f2f0250ae..6a4fca806a 100644 --- a/clusters/data_spark_version.go +++ b/clusters/data_spark_version.go @@ -9,11 +9,16 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" ) +type sparkVersionRequestWrapper struct { + common.Namespace + compute.SparkVersionRequest +} + // DataSourceSparkVersion returns DBR version matching to the specification func DataSourceSparkVersion() common.Resource { - return common.WorkspaceDataWithCustomizeFunc(func(ctx context.Context, data *compute.SparkVersionRequest, w *databricks.WorkspaceClient) error { + return common.WorkspaceDataWithCustomizeFunc(func(ctx context.Context, data *sparkVersionRequestWrapper, w *databricks.WorkspaceClient) error { data.Id = "" - version, err := w.Clusters.SelectSparkVersion(ctx, *data) + version, err := w.Clusters.SelectSparkVersion(ctx, data.SparkVersionRequest) if err != nil { return err } diff --git a/clusters/data_zones.go b/clusters/data_zones.go index 888c6775a5..631c667f3c 100644 --- a/clusters/data_zones.go +++ b/clusters/data_zones.go @@ -10,6 +10,7 @@ import ( // DataSourceClusterZones ... func DataSourceClusterZones() common.Resource { return common.WorkspaceData(func(ctx context.Context, data *struct { + common.Namespace Id string `json:"id,omitempty" tf:"computed"` DefaultZone string `json:"default_zone,omitempty" tf:"computed"` Zones []string `json:"zones,omitempty" tf:"computed"` diff --git a/clusters/resource_cluster.go b/clusters/resource_cluster.go index a5347b198d..e7107f9e5b 100644 --- a/clusters/resource_cluster.go +++ b/clusters/resource_cluster.go @@ -31,6 +31,9 @@ const ( func ResourceCluster() common.Resource { return common.Resource{ + CustomizeDiff: func(ctx context.Context, d *schema.ResourceDiff) error { + return common.NamespaceCustomizeDiff(d) + }, Create: resourceClusterCreate, Read: resourceClusterRead, Update: resourceClusterUpdate, @@ -275,6 +278,7 @@ type LibraryWithAlias struct { type ClusterSpec struct { compute.ClusterSpec + common.Namespace LibraryWithAlias } @@ -323,6 +327,7 @@ func (ClusterSpec) CustomizeSchemaResourceSpecific(s *common.CustomizableSchema) } func (ClusterSpec) CustomizeSchema(s *common.CustomizableSchema) *common.CustomizableSchema { + common.NamespaceCustomizeSchema(s) s.SchemaPath("enable_elastic_disk").SetComputed() s.SchemaPath("enable_local_disk_encryption").SetComputed() s.SchemaPath("node_type_id").SetComputed().SetConflictsWith([]string{"driver_instance_pool_id", "instance_pool_id"}) @@ -388,7 +393,7 @@ func resourceClusterSchema() map[string]*schema.Schema { func resourceClusterCreate(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { start := time.Now() timeout := d.Timeout(schema.TimeoutCreate) - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -480,7 +485,7 @@ func setPinnedStatus(ctx context.Context, d *schema.ResourceData, clusterAPI com } func resourceClusterRead(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -532,7 +537,7 @@ func hasClusterConfigChanged(d *schema.ResourceData) bool { } func resourceClusterUpdate(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -697,7 +702,7 @@ func resourceClusterUpdate(ctx context.Context, d *schema.ResourceData, c *commo } func resourceClusterDelete(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } diff --git a/clusters/resource_library.go b/clusters/resource_library.go index 715e51bfea..e7b948a5c5 100644 --- a/clusters/resource_library.go +++ b/clusters/resource_library.go @@ -15,6 +15,7 @@ import ( type LibraryResource struct { compute.Library + common.Namespace } func (LibraryResource) CustomizeSchemaResourceSpecific(s *common.CustomizableSchema) *common.CustomizableSchema { @@ -28,6 +29,7 @@ func (LibraryResource) CustomizeSchemaResourceSpecific(s *common.CustomizableSch const EggDeprecationWarning = "The `egg` library type is deprecated. Please use `whl` or `pypi` instead." func (LibraryResource) CustomizeSchema(s *common.CustomizableSchema) *common.CustomizableSchema { + common.NamespaceCustomizeSchema(s) s.SchemaPath("egg").SetDeprecated(EggDeprecationWarning) return s } @@ -43,8 +45,11 @@ func ResourceLibrary() common.Resource { } return common.Resource{ Schema: libraySdkSchema, + CustomizeDiff: func(ctx context.Context, d *schema.ResourceDiff) error { + return common.NamespaceCustomizeDiff(d) + }, Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -74,7 +79,7 @@ func ResourceLibrary() common.Resource { }, Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { clusterID, libraryRep := parseId(d.Id()) - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -106,7 +111,7 @@ func ResourceLibrary() common.Resource { }, Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { clusterID, libraryRep := parseId(d.Id()) - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } diff --git a/common/unified_provider.go b/common/unified_provider.go new file mode 100644 index 0000000000..1ada74a392 --- /dev/null +++ b/common/unified_provider.go @@ -0,0 +1,81 @@ +package common + +import ( + "context" + "fmt" + "regexp" + + "github.com/databricks/databricks-sdk-go" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" +) + +const workspaceIDSchemaKey = "provider_config.0.workspace_id" + +type Namespace struct { + ProviderConfig *ProviderConfig `json:"provider_config,omitempty"` +} + +// ProviderConfig is used to store the provider configurations for unified terraform provider +// across resources onboarded to SDKv2. +type ProviderConfig struct { + WorkspaceID string `json:"workspace_id"` +} + +// workspaceIDValidateFunc is used to validate the workspace ID for the provider configuration +func workspaceIDValidateFunc() func(interface{}, string) ([]string, []error) { + return validation.All( + validation.StringIsNotEmpty, + validation.StringMatch(regexp.MustCompile(`^\d+$`), "workspace_id must be a valid integer"), + ) +} + +// NamespaceCustomizeSchema is used to customize the schema for the provider configuration +// for a single schema. +func NamespaceCustomizeSchema(s *CustomizableSchema) { + s.SchemaPath("provider_config", "workspace_id").SetValidateFunc(workspaceIDValidateFunc()) +} + +// NamespaceCustomizeSchemaMap is used to customize the schema for the provider configuration +// in a map of schemas. +func NamespaceCustomizeSchemaMap(m map[string]*schema.Schema) map[string]*schema.Schema { + if providerConfig, ok := m["provider_config"]; ok { + if elem, ok := providerConfig.Elem.(*schema.Resource); ok { + if workspaceID, ok := elem.Schema["workspace_id"]; ok { + workspaceID.ValidateFunc = workspaceIDValidateFunc() + } + } + } + return m +} + +// NamespaceCustomizeDiff is used to customize the diff for the provider configuration +// in a resource diff. +func NamespaceCustomizeDiff(d *schema.ResourceDiff) error { + // Force New + workspaceIDKey := workspaceIDSchemaKey + oldWorkspaceID, newWorkspaceID := d.GetChange(workspaceIDKey) + if oldWorkspaceID != "" && newWorkspaceID != "" && oldWorkspaceID != newWorkspaceID { + if err := d.ForceNew(workspaceIDKey); err != nil { + return err + } + } + + return nil +} + +// WorkspaceClientUnifiedProvider returns the WorkspaceClient for the workspace ID from the resource data +// This is used by resources and data sources that are developed over SDKv2. +func (c *DatabricksClient) WorkspaceClientUnifiedProvider(ctx context.Context, d *schema.ResourceData) (*databricks.WorkspaceClient, error) { + workspaceIDFromSchema := d.Get(workspaceIDSchemaKey) + // workspace_id does not exist in the schema + if workspaceIDFromSchema == nil { + return c.GetWorkspaceClientForUnifiedProvider(ctx, "") + } + var workspaceID string + workspaceID, ok := workspaceIDFromSchema.(string) + if !ok { + return nil, fmt.Errorf("workspace_id must be a string") + } + return c.GetWorkspaceClientForUnifiedProvider(ctx, workspaceID) +} diff --git a/jobs/data_jobs.go b/jobs/data_jobs.go index 14da68e1bc..91c2e3729f 100644 --- a/jobs/data_jobs.go +++ b/jobs/data_jobs.go @@ -21,6 +21,7 @@ func DataSourceJobs() common.Resource { Ids map[string]string `json:"ids,omitempty" tf:"computed"` NameFilter string `json:"job_name_contains,omitempty"` Key string `json:"key,omitempty" tf:"default:name"` + common.Namespace }, w *databricks.WorkspaceClient) error { iter := w.Jobs.List(ctx, jobs.ListJobsRequest{ExpandTasks: false, Limit: 100}) data.Ids = map[string]string{} diff --git a/jobs/data_jobs_acc_test.go b/jobs/data_jobs_acc_test.go new file mode 100644 index 0000000000..e478f4d7c4 --- /dev/null +++ b/jobs/data_jobs_acc_test.go @@ -0,0 +1,99 @@ +package jobs_test + +import ( + "context" + "fmt" + "regexp" + "strconv" + "testing" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/terraform-provider-databricks/internal/acceptance" + "github.com/hashicorp/terraform-plugin-testing/terraform" + "github.com/stretchr/testify/require" +) + +func TestAccDataSourcesJob_InvalidID(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: ` + data "databricks_jobs" "all" { + key = "id" + provider_config { + workspace_id = "invalid" + } + }`, + ExpectError: regexp.MustCompile(`failed to parse workspace_id.*invalid syntax`), + }) +} + +func TestAccDataSourcesJob_MismatchedID(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: ` + data "databricks_jobs" "all" { + key = "id" + provider_config { + workspace_id = "123" + } + }`, + ExpectError: regexp.MustCompile(`workspace_id mismatch.*please check the workspace_id provided in provider_config`), + }) +} + +func TestAccDataSourcesJob_EmptyID(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: ` + data "databricks_jobs" "all" { + key = "id" + provider_config { + workspace_id = "" + } + }`, + ExpectError: regexp.MustCompile(`expected "provider_config.0.workspace_id" to not be an empty string`), + }) +} + +func TestAccDataSourcesJob_EmptyBlock(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: ` + data "databricks_jobs" "all" { + key = "id" + provider_config { + } + }`, + ExpectError: regexp.MustCompile(`The argument "workspace_id" is required, but no definition was found.`), + }) +} + +func TestAccDataSourcesJob(t *testing.T) { + acceptance.LoadWorkspaceEnv(t) + ctx := context.Background() + w := databricks.Must(databricks.NewWorkspaceClient()) + workspaceID, err := w.CurrentWorkspaceID(ctx) + require.NoError(t, err) + workspaceIDStr := strconv.FormatInt(workspaceID, 10) + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: ` + data "databricks_jobs" "all" { + key = "id" + }`, + }, acceptance.Step{ + Template: fmt.Sprintf(` + data "databricks_jobs" "all" { + key = "id" + provider_config { + workspace_id = "%s" + } + }`, workspaceIDStr), + Check: func(s *terraform.State) error { + r, ok := s.RootModule().Resources["data.databricks_jobs.all"] + if !ok { + return fmt.Errorf("data not found in state") + } + id := r.Primary.Attributes["provider_config.0.workspace_id"] + if id != workspaceIDStr { + return fmt.Errorf("wrong workspace_id found: %v", r.Primary.Attributes) + } + return nil + }, + }) +} diff --git a/jobs/job_test.go b/jobs/job_test.go index 2812338cad..94b8f00479 100644 --- a/jobs/job_test.go +++ b/jobs/job_test.go @@ -3,6 +3,8 @@ package jobs_test import ( "context" "errors" + "fmt" + "regexp" "strconv" "testing" "time" @@ -12,9 +14,215 @@ import ( "github.com/databricks/terraform-provider-databricks/common" "github.com/databricks/terraform-provider-databricks/internal/acceptance" "github.com/databricks/terraform-provider-databricks/qa" + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-plugin-testing/plancheck" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func jobClusterTemplate(provider_config string) string { + return fmt.Sprintf(` + data "databricks_current_user" "me" {} + data "databricks_spark_version" "latest" {} + data "databricks_node_type" "smallest" { + local_disk = true + } + + resource "databricks_notebook" "this" { + path = "${data.databricks_current_user.me.home}/Terraform{var.RANDOM}" + language = "PYTHON" + content_base64 = base64encode(<<-EOT + # created from ${abspath(path.module)} + display(spark.range(10)) + EOT + ) + } + + resource "databricks_job" "this" { + name = "{var.RANDOM}" + + %s + + job_cluster { + job_cluster_key = "j" + new_cluster { + num_workers = 0 // Setting it to zero intentionally to cover edge case. + spark_version = data.databricks_spark_version.latest.id + node_type_id = data.databricks_node_type.smallest.id + custom_tags = { + "ResourceClass" = "SingleNode" + } + spark_conf = { + "spark.databricks.cluster.profile" : "singleNode" + "spark.master" : "local[*,4]" + } + } + } + + task { + task_key = "a" + + new_cluster { + num_workers = 1 + spark_version = data.databricks_spark_version.latest.id + node_type_id = data.databricks_node_type.smallest.id + } + + notebook_task { + notebook_path = databricks_notebook.this.path + } + } + + task { + task_key = "b" + + depends_on { + task_key = "a" + } + + new_cluster { + num_workers = 8 + spark_version = data.databricks_spark_version.latest.id + node_type_id = data.databricks_node_type.smallest.id + } + + notebook_task { + notebook_path = databricks_notebook.this.path + } + } + } +`, provider_config) +} + +func TestAccJobCluster_ProviderConfig_Invalid(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(` + provider_config { + workspace_id = "invalid" + } + `), + ExpectError: regexp.MustCompile(`workspace_id must be a valid integer`), + PlanOnly: true, + }) +} + +func TestAccJobCluster_ProviderConfig_Mismatched(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(` + provider_config { + workspace_id = "123" + } + `), + ExpectError: regexp.MustCompile(`workspace_id mismatch.*please check the workspace_id provided in provider_config`), + }) +} + +func TestAccJobCluster_ProviderConfig_Required(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(` + provider_config { + } + `), + ExpectError: regexp.MustCompile(`The argument "workspace_id" is required, but no definition was found.`), + }) +} + +func TestAccJobCluster_ProviderConfig_EmptyID(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(` + provider_config { + workspace_id = "" + } + `), + ExpectError: regexp.MustCompile(`expected "provider_config.0.workspace_id" to not be an empty string`), + }) +} + +func TestAccJobCluster_ProviderConfig_NotProvided(t *testing.T) { + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(""), + }) +} + +func TestAccJobCluster_ProviderConfig_Match(t *testing.T) { + acceptance.LoadWorkspaceEnv(t) + ctx := context.Background() + w := databricks.Must(databricks.NewWorkspaceClient()) + workspaceID, err := w.CurrentWorkspaceID(ctx) + require.NoError(t, err) + workspaceIDStr := strconv.FormatInt(workspaceID, 10) + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(""), + }, acceptance.Step{ + Template: jobClusterTemplate(fmt.Sprintf(` + provider_config { + workspace_id = "%s" + } + `, workspaceIDStr)), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("databricks_job.this", plancheck.ResourceActionUpdate), + }, + }, + }) +} + +func TestAccJobCluster_ProviderConfig_Recreate(t *testing.T) { + acceptance.LoadWorkspaceEnv(t) + ctx := context.Background() + w := databricks.Must(databricks.NewWorkspaceClient()) + workspaceID, err := w.CurrentWorkspaceID(ctx) + require.NoError(t, err) + workspaceIDStr := strconv.FormatInt(workspaceID, 10) + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(""), + }, acceptance.Step{ + Template: jobClusterTemplate(fmt.Sprintf(` + provider_config { + workspace_id = "%s" + } + `, workspaceIDStr)), + }, acceptance.Step{ + Template: jobClusterTemplate(` + provider_config { + workspace_id = "123" + } + `), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PostApplyPreRefresh: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("databricks_job.this", plancheck.ResourceActionDestroyBeforeCreate), + }, + }, + PlanOnly: true, + ExpectNonEmptyPlan: true, + }) +} + +func TestAccJobCluster_ProviderConfig_Remove(t *testing.T) { + acceptance.LoadWorkspaceEnv(t) + ctx := context.Background() + w := databricks.Must(databricks.NewWorkspaceClient()) + workspaceID, err := w.CurrentWorkspaceID(ctx) + require.NoError(t, err) + workspaceIDStr := strconv.FormatInt(workspaceID, 10) + acceptance.WorkspaceLevel(t, acceptance.Step{ + Template: jobClusterTemplate(""), + }, acceptance.Step{ + Template: jobClusterTemplate(fmt.Sprintf(` + provider_config { + workspace_id = "%s" + } + `, workspaceIDStr)), + }, acceptance.Step{ + Template: jobClusterTemplate(""), + ConfigPlanChecks: resource.ConfigPlanChecks{ + PreApply: []plancheck.PlanCheck{ + plancheck.ExpectResourceAction("databricks_job.this", plancheck.ResourceActionUpdate), + }, + }, + }) +} + func TestAccJobTasks(t *testing.T) { acceptance.WorkspaceLevel(t, acceptance.Step{ Template: ` diff --git a/jobs/resource_job.go b/jobs/resource_job.go index f0173c2546..825cdcf5b6 100644 --- a/jobs/resource_job.go +++ b/jobs/resource_job.go @@ -486,6 +486,7 @@ func (JobCreateStruct) CustomizeSchema(s *common.CustomizableSchema) *common.Cus type JobSettingsResource struct { jobs.JobSettings + common.Namespace // BEGIN Jobs API 2.0 ExistingClusterID string `json:"existing_cluster_id,omitempty" tf:"group:cluster_type"` @@ -510,6 +511,7 @@ func (JobSettingsResource) Aliases() map[string]map[string]string { } func (JobSettingsResource) CustomizeSchema(s *common.CustomizableSchema) *common.CustomizableSchema { + common.NamespaceCustomizeSchema(s) // Suppress diffs s.SchemaPath("email_notifications").SetSuppressDiff() s.SchemaPath("webhook_notifications").SetSuppressDiff() @@ -663,6 +665,8 @@ func (JobSettingsResource) CustomizeSchema(s *common.CustomizableSchema) *common // Technically this is required by the API, but marking it optional since we can infer it from the hostname. s.SchemaPath("git_source", "provider").SetOptional() + common.NamespaceCustomizeSchema(s) + return s } @@ -1094,14 +1098,14 @@ func ResourceJob() common.Resource { return fmt.Errorf("`control_run_state` must be specified only with `max_concurrent_runs = 1`") } } - return nil + return common.NamespaceCustomizeDiff(d) }, Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { var jsr JobSettingsResource common.DataToStructPointer(d, jobsGoSdkSchema, &jsr) if jsr.isMultiTask() { // Api 2.1 - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -1137,7 +1141,7 @@ func ResourceJob() common.Resource { common.DataToStructPointer(d, jobsGoSdkSchema, &jsr) if jsr.isMultiTask() { // Api 2.1 - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -1179,7 +1183,7 @@ func ResourceJob() common.Resource { if err != nil { return err } - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -1206,7 +1210,7 @@ func ResourceJob() common.Resource { }, Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { ctx = getReadCtx(ctx, d) - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } From f2f67516d03ba2879d7a8f723a4dad42a805bed0 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 16:10:53 +0530 Subject: [PATCH 2/9] - --- common/resource.go | 91 ++++++++++++++++++++++++++++++---------------- 1 file changed, 59 insertions(+), 32 deletions(-) diff --git a/common/resource.go b/common/resource.go index 69c5edd1b5..0abbdf376e 100644 --- a/common/resource.go +++ b/common/resource.go @@ -219,14 +219,19 @@ func MustCompileKeyRE(name string) *regexp.Regexp { // Deprecated: migrate to WorkspaceData func DataResource(sc any, read func(context.Context, any, *DatabricksClient) error) Resource { // TODO: migrate to go1.18 and get schema from second function argument?.. - s := StructToSchema(sc, func(m map[string]*schema.Schema) map[string]*schema.Schema { - return m - }) + s := StructToSchema(sc, NamespaceCustomizeSchemaMap) return Resource{ Schema: s, Read: func(ctx context.Context, d *schema.ResourceData, m *DatabricksClient) (err error) { ptr := reflect.New(reflect.ValueOf(sc).Type()) DataToReflectValue(d, s, ptr.Elem()) + if m.DatabricksClient != nil && !m.Config.IsAccountClient() { + w, err := m.WorkspaceClientUnifiedProvider(ctx, d) + if err != nil { + return err + } + m.SetWorkspaceClient(w) + } err = read(ctx, ptr.Interface(), m) if err != nil { err = nicerError(ctx, err, "read data") @@ -256,9 +261,13 @@ func DataResource(sc any, read func(context.Context, any, *DatabricksClient) err // ... // }) func WorkspaceData[T any](read func(context.Context, *T, *databricks.WorkspaceClient) error) Resource { - return genericDatabricksData((*DatabricksClient).WorkspaceClient, func(ctx context.Context, s struct{}, t *T, wc *databricks.WorkspaceClient) error { - return read(ctx, t, wc) - }, false, NoCustomize) + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (*databricks.WorkspaceClient, error) { + return client.WorkspaceClientUnifiedProvider(ctx, d) + }, + func(ctx context.Context, s T, t *T, wc *databricks.WorkspaceClient) error { + return read(ctx, t, wc) + }, false, NamespaceCustomizeSchemaMap) } // WorkspaceDataWithParams defines a data source that can be used to read data from the workspace API. @@ -293,14 +302,18 @@ func WorkspaceData[T any](read func(context.Context, *T, *databricks.WorkspaceCl // ... // }) func WorkspaceDataWithParams[T, P any](read func(context.Context, P, *databricks.WorkspaceClient) (*T, error)) Resource { - return genericDatabricksData((*DatabricksClient).WorkspaceClient, func(ctx context.Context, o P, s *T, w *databricks.WorkspaceClient) error { - res, err := read(ctx, o, w) - if err != nil { - return err - } - *s = *res - return nil - }, true, NoCustomize) + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (*databricks.WorkspaceClient, error) { + return client.WorkspaceClientUnifiedProvider(ctx, d) + }, + func(ctx context.Context, o P, s *T, w *databricks.WorkspaceClient) error { + res, err := read(ctx, o, w) + if err != nil { + return err + } + *s = *res + return nil + }, true, NoCustomize) } // WorkspaceDataWithCustomizeFunc defines a data source that can be used to read data from the workspace API. @@ -308,13 +321,16 @@ func WorkspaceDataWithParams[T, P any](read func(context.Context, P, *databricks // customizeSchemaFunc function. // // The additional argument is a function that will be called to customize the schema of the data source. - func WorkspaceDataWithCustomizeFunc[T any]( read func(context.Context, *T, *databricks.WorkspaceClient) error, customizeSchemaFunc func(map[string]*schema.Schema) map[string]*schema.Schema) Resource { - return genericDatabricksData((*DatabricksClient).WorkspaceClient, func(ctx context.Context, s struct{}, t *T, wc *databricks.WorkspaceClient) error { - return read(ctx, t, wc) - }, false, customizeSchemaFunc) + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (*databricks.WorkspaceClient, error) { + return client.WorkspaceClientUnifiedProvider(ctx, d) + }, + func(ctx context.Context, s struct{}, t *T, wc *databricks.WorkspaceClient) error { + return read(ctx, t, wc) + }, false, customizeSchemaFunc) } // AccountData is a generic way to define account data resources in Terraform provider. @@ -329,9 +345,13 @@ func WorkspaceDataWithCustomizeFunc[T any]( // ... // }) func AccountData[T any](read func(context.Context, *T, *databricks.AccountClient) error) Resource { - return genericDatabricksData((*DatabricksClient).AccountClient, func(ctx context.Context, s struct{}, t *T, ac *databricks.AccountClient) error { - return read(ctx, t, ac) - }, false, NoCustomize) + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (*databricks.AccountClient, error) { + return client.AccountClient() + }, + func(ctx context.Context, s struct{}, t *T, ac *databricks.AccountClient) error { + return read(ctx, t, ac) + }, false, NoCustomize) } // AccountDataWithParams defines a data source that can be used to read data from the account API. @@ -366,14 +386,18 @@ func AccountData[T any](read func(context.Context, *T, *databricks.AccountClient // ... // }) func AccountDataWithParams[T, P any](read func(context.Context, P, *databricks.AccountClient) (*T, error)) Resource { - return genericDatabricksData((*DatabricksClient).AccountClient, func(ctx context.Context, o P, s *T, a *databricks.AccountClient) error { - res, err := read(ctx, o, a) - if err != nil { - return err - } - *s = *res - return nil - }, true, NoCustomize) + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (*databricks.AccountClient, error) { + return client.AccountClient() + }, + func(ctx context.Context, o P, s *T, a *databricks.AccountClient) error { + res, err := read(ctx, o, a) + if err != nil { + return err + } + *s = *res + return nil + }, true, NoCustomize) } // genericDatabricksData is generic and common way to define both account and workspace data and calls their respective clients. @@ -382,7 +406,7 @@ func AccountDataWithParams[T, P any](read func(context.Context, P, *databricks.A // from OtherFields will be overlaid on top of the schema generated by SdkType. Otherwise, the schema generated by // SdkType will be used directly. func genericDatabricksData[T, P, C any]( - getClient func(*DatabricksClient) (C, error), + getClient func(*DatabricksClient, context.Context, *schema.ResourceData) (C, error), read func(context.Context, P, *T, C) error, hasOther bool, customizeSchemaFunc func(map[string]*schema.Schema) map[string]*schema.Schema) Resource { @@ -427,7 +451,7 @@ func genericDatabricksData[T, P, C any]( var other P DataToStructPointer(d, s, &other) DataToStructPointer(d, s, &dummy) - c, err := getClient(client) + c, err := getClient(client, ctx, d) if err != nil { return nicerError(ctx, err, "get client") } @@ -494,7 +518,10 @@ func AddAccountIdField(s map[string]*schema.Schema) map[string]*schema.Schema { // NoClientData is a generic way to define data resources in Terraform provider that doesn't require any client. // usage is similar to AccountData and WorkspaceData, but the read function doesn't take a client. func NoClientData[T any](read func(context.Context, *T) error) Resource { - return genericDatabricksData(func(*DatabricksClient) (any, error) { return nil, nil }, + return genericDatabricksData( + func(client *DatabricksClient, ctx context.Context, d *schema.ResourceData) (any, error) { + return nil, nil + }, func(ctx context.Context, s struct{}, t *T, ac any) error { return read(ctx, t) }, false, NoCustomize) From 8a8aeadb6f7d512feb21a11e1a539b77da7f3bc2 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 16:12:33 +0530 Subject: [PATCH 3/9] - --- common/resource.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/common/resource.go b/common/resource.go index 0abbdf376e..29fecfe2a0 100644 --- a/common/resource.go +++ b/common/resource.go @@ -219,19 +219,14 @@ func MustCompileKeyRE(name string) *regexp.Regexp { // Deprecated: migrate to WorkspaceData func DataResource(sc any, read func(context.Context, any, *DatabricksClient) error) Resource { // TODO: migrate to go1.18 and get schema from second function argument?.. - s := StructToSchema(sc, NamespaceCustomizeSchemaMap) + s := StructToSchema(sc, func(m map[string]*schema.Schema) map[string]*schema.Schema { + return m + }) return Resource{ Schema: s, Read: func(ctx context.Context, d *schema.ResourceData, m *DatabricksClient) (err error) { ptr := reflect.New(reflect.ValueOf(sc).Type()) DataToReflectValue(d, s, ptr.Elem()) - if m.DatabricksClient != nil && !m.Config.IsAccountClient() { - w, err := m.WorkspaceClientUnifiedProvider(ctx, d) - if err != nil { - return err - } - m.SetWorkspaceClient(w) - } err = read(ctx, ptr.Interface(), m) if err != nil { err = nicerError(ctx, err, "read data") From ed87b64f901cd8ce15176c16fe3007f8a75d7249 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 16:20:12 +0530 Subject: [PATCH 4/9] - --- jobs/job_test.go | 6 ------ jobs/resource_job.go | 1 - 2 files changed, 7 deletions(-) diff --git a/jobs/job_test.go b/jobs/job_test.go index 94b8f00479..654be68db9 100644 --- a/jobs/job_test.go +++ b/jobs/job_test.go @@ -138,12 +138,6 @@ func TestAccJobCluster_ProviderConfig_EmptyID(t *testing.T) { }) } -func TestAccJobCluster_ProviderConfig_NotProvided(t *testing.T) { - acceptance.WorkspaceLevel(t, acceptance.Step{ - Template: jobClusterTemplate(""), - }) -} - func TestAccJobCluster_ProviderConfig_Match(t *testing.T) { acceptance.LoadWorkspaceEnv(t) ctx := context.Background() diff --git a/jobs/resource_job.go b/jobs/resource_job.go index 825cdcf5b6..6a1af10137 100644 --- a/jobs/resource_job.go +++ b/jobs/resource_job.go @@ -511,7 +511,6 @@ func (JobSettingsResource) Aliases() map[string]map[string]string { } func (JobSettingsResource) CustomizeSchema(s *common.CustomizableSchema) *common.CustomizableSchema { - common.NamespaceCustomizeSchema(s) // Suppress diffs s.SchemaPath("email_notifications").SetSuppressDiff() s.SchemaPath("webhook_notifications").SetSuppressDiff() From c2fdf98fa24be5e27735cd51b5c868a412d60ad4 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 16:34:17 +0530 Subject: [PATCH 5/9] - --- clusters/data_node_type.go | 1 - common/unified_provider.go | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/clusters/data_node_type.go b/clusters/data_node_type.go index 188a5e7462..6777689ece 100644 --- a/clusters/data_node_type.go +++ b/clusters/data_node_type.go @@ -12,7 +12,6 @@ import ( ) type NodeTypeRequest struct { - common.Namespace compute.NodeTypeRequest Arm bool `json:"arm,omitempty"` } diff --git a/common/unified_provider.go b/common/unified_provider.go index 1ada74a392..ddcfd9e6e8 100644 --- a/common/unified_provider.go +++ b/common/unified_provider.go @@ -10,6 +10,7 @@ import ( "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" ) +// workspaceIDSchemaKey is the key for the workspace ID in schema const workspaceIDSchemaKey = "provider_config.0.workspace_id" type Namespace struct { @@ -68,7 +69,7 @@ func NamespaceCustomizeDiff(d *schema.ResourceDiff) error { // This is used by resources and data sources that are developed over SDKv2. func (c *DatabricksClient) WorkspaceClientUnifiedProvider(ctx context.Context, d *schema.ResourceData) (*databricks.WorkspaceClient, error) { workspaceIDFromSchema := d.Get(workspaceIDSchemaKey) - // workspace_id does not exist in the schema + // workspace_id does not exist in the resource data if workspaceIDFromSchema == nil { return c.GetWorkspaceClientForUnifiedProvider(ctx, "") } From df54b8c161ff8bdc913319764dbf2c8063399f7e Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 17:01:10 +0530 Subject: [PATCH 6/9] - --- common/unified_provider_test.go | 237 ++++++++++++++++++++++++++++++++ 1 file changed, 237 insertions(+) create mode 100644 common/unified_provider_test.go diff --git a/common/unified_provider_test.go b/common/unified_provider_test.go new file mode 100644 index 0000000000..09df1c2380 --- /dev/null +++ b/common/unified_provider_test.go @@ -0,0 +1,237 @@ +package common + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWorkspaceIDValidateFunc(t *testing.T) { + validateFunc := workspaceIDValidateFunc() + + testCases := []struct { + name string + input interface{} + expectError bool + }{ + { + name: "valid numeric workspace ID", + input: "123456789", + expectError: false, + }, + { + name: "valid single digit workspace ID", + input: "1", + expectError: false, + }, + { + name: "valid large workspace ID", + input: "999999999999999", + expectError: false, + }, + { + name: "invalid empty string", + input: "", + expectError: true, + }, + { + name: "invalid non-numeric string", + input: "abc123", + expectError: true, + }, + { + name: "invalid string with spaces", + input: "123 456", + expectError: true, + }, + { + name: "invalid string with special characters", + input: "123-456", + expectError: true, + }, + { + name: "invalid string with leading zero", + input: "0123", + expectError: false, // This is actually valid as it's still a numeric string + }, + { + name: "invalid negative number", + input: "-123", + expectError: true, + }, + { + name: "invalid decimal number", + input: "123.456", + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, errors := validateFunc(tc.input, "workspace_id") + if tc.expectError { + assert.NotEmpty(t, errors, "Expected validation errors but got none") + } else { + assert.Empty(t, errors, "Expected no validation errors but got: %v", errors) + } + }) + } +} + +func TestNamespaceCustomizeSchema(t *testing.T) { + // Create a test schema with provider_config structure + testSchema := map[string]*schema.Schema{ + "provider_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "workspace_id": { + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, + } + + // Apply the customization + NamespaceCustomizeSchema(&CustomizableSchema{ + Schema: &schema.Schema{ + Elem: &schema.Resource{ + Schema: testSchema, + }, + }, + }) + + // Verify that ValidateFunc was set + providerConfig := testSchema["provider_config"] + require.NotNil(t, providerConfig) + elem, ok := providerConfig.Elem.(*schema.Resource) + require.True(t, ok) + workspaceID := elem.Schema["workspace_id"] + require.NotNil(t, workspaceID) + assert.NotNil(t, workspaceID.ValidateFunc, "ValidateFunc should be set on workspace_id") + + // Test the validation function + _, errors := workspaceID.ValidateFunc("123456", "workspace_id") + assert.Empty(t, errors, "Valid workspace ID should not produce errors") + + _, errors = workspaceID.ValidateFunc("invalid", "workspace_id") + assert.NotEmpty(t, errors, "Invalid workspace ID should produce errors") +} + +func TestNamespaceCustomizeSchemaMap(t *testing.T) { + t.Run("with valid provider_config", func(t *testing.T) { + testSchema := map[string]*schema.Schema{ + "provider_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "workspace_id": { + Type: schema.TypeString, + Required: true, + }, + }, + }, + }, + "other_field": { + Type: schema.TypeString, + Optional: true, + }, + } + + result := NamespaceCustomizeSchemaMap(testSchema) + + // Verify that ValidateFunc was set + providerConfig := result["provider_config"] + require.NotNil(t, providerConfig) + elem, ok := providerConfig.Elem.(*schema.Resource) + require.True(t, ok) + workspaceID := elem.Schema["workspace_id"] + require.NotNil(t, workspaceID) + assert.NotNil(t, workspaceID.ValidateFunc, "ValidateFunc should be set on workspace_id") + + // Test the validation function + _, errors := workspaceID.ValidateFunc("123456", "workspace_id") + assert.Empty(t, errors, "Valid workspace ID should not produce errors") + + _, errors = workspaceID.ValidateFunc("invalid", "workspace_id") + assert.NotEmpty(t, errors, "Invalid workspace ID should produce errors") + }) + + t.Run("without provider_config", func(t *testing.T) { + testSchema := map[string]*schema.Schema{ + "other_field": { + Type: schema.TypeString, + Optional: true, + }, + } + + result := NamespaceCustomizeSchemaMap(testSchema) + assert.Equal(t, testSchema, result, "Schema should be returned unchanged when provider_config is not present") + }) +} + +func TestNamespaceCustomizeDiff(t *testing.T) { + testSchema := map[string]*schema.Schema{ + "provider_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "workspace_id": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "name": { + Type: schema.TypeString, + Required: true, + }, + } + + t.Run("handles nil ResourceDiff gracefully", func(t *testing.T) { + // Test that the function doesn't panic with nil input by checking the logic + // The actual CustomizeDiff is called by Terraform framework, so we just verify + // the function signature and basic error handling + + // Create a basic resource data to ensure schema is valid + d := schema.TestResourceDataRaw(t, testSchema, map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "123456", + }, + }, + }) + + // Verify the schema is properly set up + require.NotNil(t, d) + }) + + t.Run("validate workspace_id schema key constant", func(t *testing.T) { + // Verify the constant is correctly defined + assert.Equal(t, "provider_config.0.workspace_id", workspaceIDSchemaKey, + "workspaceIDSchemaKey constant should match the schema path") + }) + + t.Run("workspace_id in schema matches expected path", func(t *testing.T) { + // Verify that the schema path matches what we expect + providerConfig := testSchema["provider_config"] + require.NotNil(t, providerConfig) + elem, ok := providerConfig.Elem.(*schema.Resource) + require.True(t, ok) + workspaceID, exists := elem.Schema["workspace_id"] + require.True(t, exists, "workspace_id should exist in provider_config schema") + assert.Equal(t, schema.TypeString, workspaceID.Type, "workspace_id should be a string type") + }) +} From eea71cbfb49dcf663944a8e40c228c6513ca6e1e Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 17:45:17 +0530 Subject: [PATCH 7/9] - --- common/unified_provider_test.go | 287 +++++++++++++++++++++++++++++ docs/data-sources/cluster.md | 2 + docs/data-sources/clusters.md | 2 + docs/data-sources/jobs.md | 2 + docs/data-sources/spark_version.md | 2 + docs/data-sources/zones.md | 4 +- docs/resources/cluster.md | 2 + docs/resources/job.md | 2 + 8 files changed, 302 insertions(+), 1 deletion(-) diff --git a/common/unified_provider_test.go b/common/unified_provider_test.go index 09df1c2380..9e418bfa67 100644 --- a/common/unified_provider_test.go +++ b/common/unified_provider_test.go @@ -1,8 +1,12 @@ package common import ( + "context" "testing" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/client" + "github.com/databricks/databricks-sdk-go/config" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -235,3 +239,286 @@ func TestNamespaceCustomizeDiff(t *testing.T) { assert.Equal(t, schema.TypeString, workspaceID.Type, "workspace_id should be a string type") }) } + +func TestWorkspaceClientUnifiedProvider(t *testing.T) { + testSchema := map[string]*schema.Schema{ + "provider_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "workspace_id": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "name": { + Type: schema.TypeString, + Required: true, + }, + } + + ctx := context.Background() + + testCases := []struct { + name string + resourceData map[string]interface{} + cachedWorkspaceID int64 + isAccountLevel bool + accountID string + expectError bool + errorContains string + description string + }{ + { + name: "workspace_id not set - calls with empty string", + resourceData: map[string]interface{}{ + "name": "test", + }, + cachedWorkspaceID: 0, + isAccountLevel: false, + expectError: false, + description: "When provider_config is not set, should use cached workspace client", + }, + { + name: "workspace_id set to valid value", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "123456", + }, + }, + }, + cachedWorkspaceID: 123456, + isAccountLevel: false, + expectError: false, + description: "When workspace_id matches cached ID, should return workspace client", + }, + { + name: "workspace_id set to empty string", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "", + }, + }, + }, + cachedWorkspaceID: 0, + isAccountLevel: false, + expectError: false, + description: "When workspace_id is explicitly empty, should use cached workspace client", + }, + { + name: "workspace_id with different numeric value", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "789012", + }, + }, + }, + cachedWorkspaceID: 789012, + isAccountLevel: false, + expectError: false, + description: "Should handle different workspace IDs correctly", + }, + { + name: "account level provider without workspace_id - returns error", + resourceData: map[string]interface{}{ + "name": "test", + }, + cachedWorkspaceID: 0, + isAccountLevel: true, + accountID: "test-account-id", + expectError: true, + errorContains: "workspace_id", + description: "Account-level provider requires workspace_id to be set", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create resource data + d := schema.TestResourceDataRaw(t, testSchema, tc.resourceData) + + // Create a mock workspace client to be returned + mockWorkspaceClient := &databricks.WorkspaceClient{} + + // Create a DatabricksClient based on test case configuration + var dc *DatabricksClient + if tc.isAccountLevel { + // Create account-level provider + dc = &DatabricksClient{ + DatabricksClient: &client.DatabricksClient{ + Config: &config.Config{ + Host: "https://accounts.cloud.databricks.com", + AccountID: tc.accountID, + Token: "test-token", + }, + }, + } + } else { + // Create workspace-level provider + dc = &DatabricksClient{ + DatabricksClient: &client.DatabricksClient{ + Config: &config.Config{ + Host: "https://test.cloud.databricks.com", + Token: "test-token", + }, + }, + cachedWorkspaceClient: mockWorkspaceClient, + cachedWorkspaceID: tc.cachedWorkspaceID, + } + } + + // Call WorkspaceClientUnifiedProvider + result, err := dc.WorkspaceClientUnifiedProvider(ctx, d) + + // Verify results + if tc.expectError { + assert.Error(t, err, tc.description) + if tc.errorContains != "" { + assert.Contains(t, err.Error(), tc.errorContains) + } + } else { + assert.NoError(t, err, tc.description) + assert.NotNil(t, result) + assert.Equal(t, mockWorkspaceClient, result) + } + }) + } +} + +func TestWorkspaceIDExtractionLogic(t *testing.T) { + testSchema := map[string]*schema.Schema{ + "provider_config": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "workspace_id": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "name": { + Type: schema.TypeString, + Required: true, + }, + } + + testCases := []struct { + name string + resourceData map[string]interface{} + getKey string // optional: if set, use this key instead of workspaceIDSchemaKey + expectedValue string + expectedOk bool + shouldBeNil bool + description string + }{ + { + name: "provider_config not set - returns empty string", + resourceData: map[string]interface{}{ + "name": "test", + }, + expectedValue: "", + expectedOk: true, + shouldBeNil: false, + description: "d.Get returns empty string when provider_config is not set", + }, + { + name: "non-existent key returns nil", + resourceData: map[string]interface{}{ + "name": "test", + }, + getKey: "non_existent_key", + expectedValue: "", + expectedOk: false, + shouldBeNil: true, + description: "d.Get returns nil if the key doesn't exist in the schema", + }, + { + name: "workspace_id set to valid value", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "123456", + }, + }, + }, + expectedValue: "123456", + expectedOk: true, + shouldBeNil: false, + description: "d.Get returns the correct workspace_id when set", + }, + { + name: "workspace_id set to empty string", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "", + }, + }, + }, + expectedValue: "", + expectedOk: true, + shouldBeNil: false, + description: "d.Get returns empty string when explicitly set to empty", + }, + { + name: "workspace_id with numeric value", + resourceData: map[string]interface{}{ + "name": "test", + "provider_config": []interface{}{ + map[string]interface{}{ + "workspace_id": "999999999", + }, + }, + }, + expectedValue: "999999999", + expectedOk: true, + shouldBeNil: false, + description: "d.Get returns large numeric workspace_id as string", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create resource data with test case input + d := schema.TestResourceDataRaw(t, testSchema, tc.resourceData) + + // Test: workspaceIDFromSchema := d.Get(workspaceIDSchemaKey) + // Use custom key if specified, otherwise use workspaceIDSchemaKey + key := workspaceIDSchemaKey + if tc.getKey != "" { + key = tc.getKey + } + workspaceIDFromSchema := d.Get(key) + + // Test: if workspaceIDFromSchema == nil + if tc.shouldBeNil { + assert.Nil(t, workspaceIDFromSchema, tc.description) + } else { + assert.NotNil(t, workspaceIDFromSchema, tc.description) + } + + // Test: workspaceID, ok := workspaceIDFromSchema.(string) + workspaceID, ok := workspaceIDFromSchema.(string) + assert.Equal(t, tc.expectedOk, ok, "type assertion ok should match expected") + if ok { + assert.Equal(t, tc.expectedValue, workspaceID, tc.description) + } + }) + } +} diff --git a/docs/data-sources/cluster.md b/docs/data-sources/cluster.md index 922ef51465..a100a6de04 100644 --- a/docs/data-sources/cluster.md +++ b/docs/data-sources/cluster.md @@ -46,6 +46,8 @@ data "databricks_cluster" "my_cluster" { * `cluster_id` - (Required if `cluster_name` isn't specified) The id of the cluster. * `cluster_name` - (Required if `cluster_id` isn't specified) The exact name of the cluster to search. Can only be specified if there is exactly one cluster with the provided name. +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ## Attribute Reference diff --git a/docs/data-sources/clusters.md b/docs/data-sources/clusters.md index 095b9c0f6d..8146ebb488 100644 --- a/docs/data-sources/clusters.md +++ b/docs/data-sources/clusters.md @@ -58,6 +58,8 @@ data "databricks_clusters" "all_pinned_clusters" { * `cluster_name_contains` - (Optional) Only return [databricks_cluster](../resources/cluster.md#cluster_id) ids that match the given name string. * `filter_by` - (Optional) Filters to apply to the listed clusters. See [filter_by Configuration Block](#filter_by-configuration-block) below for details. +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ### filter_by Configuration Block diff --git a/docs/data-sources/jobs.md b/docs/data-sources/jobs.md index 8e69241ee1..5e7eeed725 100644 --- a/docs/data-sources/jobs.md +++ b/docs/data-sources/jobs.md @@ -62,6 +62,8 @@ resource "databricks_permissions" "everyone_can_view_all_jobs" { * `job_name_contains` - (Optional) Only return [databricks_job](../resources/job.md#) ids that match the given name string (case-insensitive). * `key` - (Optional) Attribute to use for keys in the returned map of [databricks_job](../resources/job.md#) ids by. Possible values are `name` (default) or `id`. Setting to `id` uses the job ID as the map key, allowing duplicate job names. +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ## Attribute Reference diff --git a/docs/data-sources/spark_version.md b/docs/data-sources/spark_version.md index 45cb1336da..3e0a370ec5 100644 --- a/docs/data-sources/spark_version.md +++ b/docs/data-sources/spark_version.md @@ -50,6 +50,8 @@ Data source allows you to pick groups by the following attributes: * `spark_version` - (string, optional) if we should limit the search only to runtimes that are based on specific Spark version. Default to empty string. It could be specified as `3`, or `3.0`, or full version, like, `3.0.1`. * `photon` - (boolean, optional) if we should limit the search only to Photon runtimes. Default to `false`. *Deprecated with DBR 14.0 release. Specify `runtime_engine=\"PHOTON\"` in the cluster configuration instead!* * `graviton` - (boolean, optional) if we should limit the search only to runtimes supporting AWS Graviton CPUs. Default to `false`. _Deprecated with DBR 14.0 release. DBR version compiled for Graviton will be automatically installed when nodes with Graviton CPUs are specified in the cluster configuration._ +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ## Attribute Reference diff --git a/docs/data-sources/zones.md b/docs/data-sources/zones.md index 6f1422238a..e79ff14c8b 100644 --- a/docs/data-sources/zones.md +++ b/docs/data-sources/zones.md @@ -15,7 +15,9 @@ data "databricks_zones" "zones" {} ## Argument Reference -There are no arguments to this data source and only attributes that are computed. +The following arguments are supported for this resource: +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ## Attribute Reference diff --git a/docs/resources/cluster.md b/docs/resources/cluster.md index d85e9e1916..6cb9472854 100644 --- a/docs/resources/cluster.md +++ b/docs/resources/cluster.md @@ -60,6 +60,8 @@ resource "databricks_cluster" "shared_autoscaling" { * `spark_conf` - (Optional) Map with key-value pairs to fine-tune Spark clusters, where you can provide custom [Spark configuration properties](https://spark.apache.org/docs/latest/configuration.html) in a cluster configuration. * `is_pinned` - (Optional) boolean value specifying if the cluster is pinned (not pinned by default). You must be a Databricks administrator to use this. The pinned clusters' maximum number is [limited to 100](https://docs.databricks.com/clusters/clusters-manage.html#pin-a-cluster), so `apply` may fail if you have more than that (this number may change over time, so check Databricks documentation for actual number). * `no_wait` - (Optional) If true, the provider will not wait for the cluster to reach `RUNNING` state when creating the cluster, allowing cluster creation and library installation to continue asynchronously. Defaults to false (the provider will wait for cluster creation and library installation to succeed). +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. The following example demonstrates how to create an autoscaling cluster with [Delta Cache](https://docs.databricks.com/delta/optimizations/delta-cache.html) enabled: diff --git a/docs/resources/job.md b/docs/resources/job.md index fc5e265f94..981900f86c 100644 --- a/docs/resources/job.md +++ b/docs/resources/job.md @@ -114,6 +114,8 @@ The resource supports the following arguments: * `performance_target` - (Optional) The performance mode on a serverless job. The performance target determines the level of compute performance or cost-efficiency for the run. Supported values are: * `PERFORMANCE_OPTIMIZED`: (default value) Prioritizes fast startup and execution times through rapid scaling and optimized cluster performance. * `STANDARD`: Enables cost-efficient execution of serverless workloads. +* `provider_config` - (Optional) Configure the provider for management through account provider. This block consists of the following fields: + * `workspace_id` - (Required) Workspace ID which the resource belongs to. This workspace must be part of the account which the provider is configured with. ### task Configuration Block From 10fd2b3a7d667ac5e7b6580458dc2d8dbae056e9 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 18:26:58 +0530 Subject: [PATCH 8/9] - --- jobs/data_jobs_acc_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jobs/data_jobs_acc_test.go b/jobs/data_jobs_acc_test.go index e478f4d7c4..3b767e4602 100644 --- a/jobs/data_jobs_acc_test.go +++ b/jobs/data_jobs_acc_test.go @@ -22,7 +22,8 @@ func TestAccDataSourcesJob_InvalidID(t *testing.T) { workspace_id = "invalid" } }`, - ExpectError: regexp.MustCompile(`failed to parse workspace_id.*invalid syntax`), + ExpectError: regexp.MustCompile(`workspace_id must be a valid integer`), + PlanOnly: true, }) } From ccd46dbcff14df6cd0232b168d1c0660286f66a7 Mon Sep 17 00:00:00 2001 From: Tanmay Rustagi Date: Fri, 24 Oct 2025 18:30:42 +0530 Subject: [PATCH 9/9] catalog --- catalog/resource_catalog.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/catalog/resource_catalog.go b/catalog/resource_catalog.go index db7f70cdc8..749d2d42ab 100644 --- a/catalog/resource_catalog.go +++ b/catalog/resource_catalog.go @@ -28,8 +28,13 @@ func ucDirectoryPathSlashAndEmptySuppressDiff(k, old, new string, d *schema.Reso return false } +type CatalogInfo struct { + catalog.CatalogInfo + common.Namespace +} + func ResourceCatalog() common.Resource { - catalogSchema := common.StructToSchema(catalog.CatalogInfo{}, + catalogSchema := common.StructToSchema(CatalogInfo{}, func(s map[string]*schema.Schema) map[string]*schema.Schema { s["force_destroy"] = &schema.Schema{ Type: schema.TypeBool, @@ -58,12 +63,13 @@ func ResourceCatalog() common.Resource { common.CustomizeSchemaPath(s, v).SetReadOnly() } common.CustomizeSchemaPath(s, "effective_predictive_optimization_flag").SetComputed().SetSuppressDiff() + common.NamespaceCustomizeSchemaMap(s) return s }) return common.Resource{ Schema: catalogSchema, Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -107,7 +113,7 @@ func ResourceCatalog() common.Resource { return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, bindings.BindingsSecurableTypeCatalog) }, Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -119,7 +125,7 @@ func ResourceCatalog() common.Resource { return common.StructToData(ci, catalogSchema, d) }, Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err } @@ -188,7 +194,7 @@ func ResourceCatalog() common.Resource { return bindings.AddCurrentWorkspaceBindings(ctx, d, w, ci.Name, bindings.BindingsSecurableTypeCatalog) }, Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error { - w, err := c.WorkspaceClient() + w, err := c.WorkspaceClientUnifiedProvider(ctx, d) if err != nil { return err }