From cad8a41e5e6878535c760db28ff6cf96c161283f Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 29 Sep 2025 15:28:10 +0200 Subject: [PATCH 01/10] direct: Added support for clusters deployment --- .../update-and-resize/databricks.yml.tmpl | 24 ++++ .../deploy/update-and-resize/hello_world.py | 1 + .../deploy/update-and-resize/out.test.toml | 5 + .../deploy/update-and-resize/output.txt | 125 ++++++++++++++++++ .../clusters/deploy/update-and-resize/script | 44 ++++++ .../deploy/update-and-resize/test.toml | 11 ++ libs/testserver/clusters.go | 119 +++++++++++++++++ libs/testserver/fake_workspace.go | 3 + libs/testserver/handlers.go | 26 ++++ 9 files changed, 358 insertions(+) create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/hello_world.py create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/script create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml create mode 100644 libs/testserver/clusters.go diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl new file mode 100644 index 0000000000..74c340a249 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: test-deploy-cluster-simple + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + test_cluster: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 2 + spark_conf: + "spark.executor.memory": "2g" + + jobs: + foo: + name: test-job-with-cluster-$UNIQUE_NAME + tasks: + - task_key: my_spark_python_task + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/hello_world.py b/acceptance/bundle/resources/clusters/deploy/update-and-resize/hello_world.py new file mode 100644 index 0000000000..f301245e24 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml new file mode 100644 index 0000000000..90061dedb1 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt new file mode 100644 index 0000000000..1a6f44a841 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -0,0 +1,125 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Cluster should exist after bundle deployment: +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 2 +} + +=== Changing num_workers should call update API on stopped cluster + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/edit", + "body": { + "autotermination_minutes": 60, + "cluster_id": "[UUID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 3, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } +} + +=== Cluster should have new num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 3 +} + +=== Starting the cluster +{ + "autotermination_minutes":60, + "cluster_id":"[UUID]", + "cluster_name":"test-cluster-[UNIQUE_NAME]", + "node_type_id":"[NODE_TYPE_ID]", + "num_workers":3, + "spark_conf": { + "spark.executor.memory":"2g" + }, + "spark_version":"13.3.x-snapshot-scala2.12", + "state":"RUNNING" +} + +=== Changing num_workers should call resize API on running cluster + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "cluster_id": "[UUID]", + "num_workers": 4 + } +} + +=== Cluster should have new num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 4 +} + +=== Changing num_workers and spark_conf should call update API + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/edit", + "body": { + "autotermination_minutes": 60, + "cluster_id": "[UUID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 5, + "spark_conf": { + "spark.executor.memory": "4g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } +} + +=== Cluster should have new num_workers and spark_conf +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 5, + "spark_conf": { + "spark.executor.memory": "4g" + } +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script new file mode 100644 index 0000000000..1cd8e640fe --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -0,0 +1,44 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Cluster should exist after bundle deployment:\n" +CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' + +title "Changing num_workers should call update API on stopped cluster\n" +update_file.py databricks.yml "num_workers: 2" "num_workers: 3" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have new num_workers\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' + +title "Starting the cluster\n" +$CLI clusters start "${CLUSTER_ID}" + +title "Changing num_workers should call resize API on running cluster\n" +update_file.py databricks.yml "num_workers: 3" "num_workers: 4" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have new num_workers\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' + +title "Changing num_workers and spark_conf should call update API\n" +update_file.py databricks.yml "num_workers: 4" "num_workers: 5" +update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have new num_workers and spark_conf\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,spark_conf}' diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml new file mode 100644 index 0000000000..964a45f5c1 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = false +RecordRequests = true + +Ignore = [ + "databricks.yml", +] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go new file mode 100644 index 0000000000..f471045b27 --- /dev/null +++ b/libs/testserver/clusters.go @@ -0,0 +1,119 @@ +package testserver + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/compute" + "github.com/google/uuid" +) + +func (s *FakeWorkspace) ClustersCreate(req Request) any { + var request compute.ClusterDetails + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + + defer s.LockUnlock()() + + clusterId := uuid.New().String() + request.ClusterId = clusterId + s.Clusters[clusterId] = request + + return Response{ + Body: compute.ClusterDetails{ + ClusterId: clusterId, + }, + } +} + +func (s *FakeWorkspace) ClustersResize(req Request) any { + var request compute.ResizeCluster + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + + defer s.LockUnlock()() + cluster, ok := s.Clusters[request.ClusterId] + if !ok { + return Response{StatusCode: 404} + } + + cluster.NumWorkers = request.NumWorkers + s.Clusters[request.ClusterId] = cluster + + return Response{} +} + +func (s *FakeWorkspace) ClustersEdit(req Request) any { + var request compute.ClusterDetails + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + + defer s.LockUnlock()() + _, ok := s.Clusters[request.ClusterId] + if !ok { + return Response{StatusCode: 404} + } + + s.Clusters[request.ClusterId] = request + return Response{} +} + +func (s *FakeWorkspace) ClustersGet(req Request, clusterId string) any { + defer s.LockUnlock()() + + cluster, ok := s.Clusters[clusterId] + if !ok { + return Response{StatusCode: 404} + } + + return Response{ + Body: cluster, + } +} + +func (s *FakeWorkspace) ClustersStart(req Request) any { + var request compute.StartCluster + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + defer s.LockUnlock()() + + cluster, ok := s.Clusters[request.ClusterId] + if !ok { + return Response{StatusCode: 404} + } + + cluster.State = compute.StateRunning + s.Clusters[request.ClusterId] = cluster + + return Response{} +} + +func (s *FakeWorkspace) ClustersPermanentDelete(req Request) any { + var request compute.PermanentDeleteCluster + if err := json.Unmarshal(req.Body, &request); err != nil { + return Response{ + StatusCode: 400, + Body: fmt.Sprintf("request parsing error: %s", err), + } + } + + defer s.LockUnlock()() + delete(s.Clusters, request.ClusterId) + return Response{} +} diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 9918cc1888..57c3595ab0 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -9,6 +9,7 @@ import ( "strings" "sync" + "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/database" "github.com/databricks/databricks-sdk-go/service/apps" @@ -75,6 +76,7 @@ type FakeWorkspace struct { Dashboards map[string]dashboards.Dashboard SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 + Clusters map[string]compute.ClusterDetails Acls map[string][]workspace.AclItem @@ -172,6 +174,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { DatabaseCatalogs: map[string]database.DatabaseCatalog{}, SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, Alerts: map[string]sql.AlertV2{}, + Clusters: map[string]compute.ClusterDetails{}, } } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a0ab2070eb..ebe0079e93 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -521,4 +521,30 @@ func AddDefaultHandlers(server *Server) { server.Handle("GET", "/api/2.0/permissions/jobs/{job_id}", func(req Request) any { return req.Workspace.JobsGetPermissions(req, req.Vars["job_id"]) }) + + // Clusters: + server.Handle("POST", "/api/2.1/clusters/resize", func(req Request) any { + return req.Workspace.ClustersResize(req) + }) + + server.Handle("POST", "/api/2.1/clusters/edit", func(req Request) any { + return req.Workspace.ClustersEdit(req) + }) + + server.Handle("GET", "/api/2.1/clusters/get", func(req Request) any { + clusterId := req.URL.Query().Get("cluster_id") + return req.Workspace.ClustersGet(req, clusterId) + }) + + server.Handle("POST", "/api/2.1/clusters/create", func(req Request) any { + return req.Workspace.ClustersCreate(req) + }) + + server.Handle("POST", "/api/2.1/clusters/start", func(req Request) any { + return req.Workspace.ClustersStart(req) + }) + + server.Handle("POST", "/api/2.1/clusters/permanent-delete", func(req Request) any { + return req.Workspace.ClustersPermanentDelete(req) + }) } From 5ef41fd9be1fcefb5c92f7d4b101ed805ea0a456 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 1 Oct 2025 16:45:56 +0200 Subject: [PATCH 02/10] fix lint --- libs/testserver/fake_workspace.go | 36 +++++++++++++++---------------- libs/testserver/handlers.go | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index e1caa7d817..5df470f79d 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -62,24 +62,24 @@ type FakeWorkspace struct { repoIdByPath map[string]int64 // normally, ids are not sequential, but we make them sequential for deterministic diff - nextJobId int64 - nextJobRunId int64 - Jobs map[int64]jobs.Job - JobRuns map[int64]jobs.Run - JobPermissions map[string][]jobs.JobAccessControlRequest - Pipelines map[string]pipelines.GetPipelineResponse - PipelineUpdates map[string]bool - Monitors map[string]catalog.MonitorInfo - Apps map[string]apps.App - Schemas map[string]catalog.SchemaInfo - SchemasGrants map[string][]catalog.PrivilegeAssignment - Volumes map[string]catalog.VolumeInfo - Dashboards map[string]dashboards.Dashboard - SqlWarehouses map[string]sql.GetWarehouseResponse - Alerts map[string]sql.AlertV2 - Experiments map[string]ml.GetExperimentResponse + nextJobId int64 + nextJobRunId int64 + Jobs map[int64]jobs.Job + JobRuns map[int64]jobs.Run + JobPermissions map[string][]jobs.JobAccessControlRequest + Pipelines map[string]pipelines.GetPipelineResponse + PipelineUpdates map[string]bool + Monitors map[string]catalog.MonitorInfo + Apps map[string]apps.App + Schemas map[string]catalog.SchemaInfo + SchemasGrants map[string][]catalog.PrivilegeAssignment + Volumes map[string]catalog.VolumeInfo + Dashboards map[string]dashboards.Dashboard + SqlWarehouses map[string]sql.GetWarehouseResponse + Alerts map[string]sql.AlertV2 + Experiments map[string]ml.GetExperimentResponse ModelRegistryModels map[string]ml.Model - Clusters map[string]compute.ClusterDetails + Clusters map[string]compute.ClusterDetails Acls map[string][]workspace.AclItem @@ -177,7 +177,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { DatabaseCatalogs: map[string]database.DatabaseCatalog{}, SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, Alerts: map[string]sql.AlertV2{}, - Experiments: map[string]ml.GetExperimentResponse{}, + Experiments: map[string]ml.GetExperimentResponse{}, ModelRegistryModels: map[string]ml.Model{}, Clusters: map[string]compute.ClusterDetails{}, } diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index a98d16e819..6bd1731010 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -546,7 +546,7 @@ func AddDefaultHandlers(server *Server) { server.Handle("POST", "/api/2.1/clusters/permanent-delete", func(req Request) any { return req.Workspace.ClustersPermanentDelete(req) - }) + }) // MLflow Experiments: server.Handle("GET", "/api/2.0/mlflow/experiments/get", func(req Request) any { From 9921abcc994469c4fa8826b2c8c9ee860b4df4c4 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 1 Oct 2025 18:08:50 +0200 Subject: [PATCH 03/10] implement direct --- .../deploy/update-and-resize/out.test.toml | 2 +- .../bundle/resources/clusters/test.toml | 1 - .../resourcemutator/resource_mutator.go | 4 + bundle/direct/apply.go | 21 ++ bundle/direct/bundle_plan.go | 33 ++-- bundle/direct/dresources/adapter.go | 45 ++++- bundle/direct/dresources/all.go | 1 + bundle/direct/dresources/cluster.go | 180 ++++++++++++++++++ 8 files changed, 269 insertions(+), 18 deletions(-) delete mode 100644 acceptance/bundle/resources/clusters/test.toml create mode 100644 bundle/direct/dresources/cluster.go diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml index 90061dedb1..e092fd5ed6 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.test.toml @@ -2,4 +2,4 @@ Local = true Cloud = false [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/resources/clusters/test.toml b/acceptance/bundle/resources/clusters/test.toml deleted file mode 100644 index e1ff579ead..0000000000 --- a/acceptance/bundle/resources/clusters/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # clusters are not implemented yet diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 98cbb8cb25..cf82177cae 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -92,6 +92,10 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Apps: {"resources.apps.*.description", ""}, + + // Clusters (same as terraform) + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go#L315 + {"resources.clusters.*.autotermination_minutes", 60}, } for _, defaultDef := range defaults { diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index f0dc0ad4c5..5baf3d00ce 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -54,6 +54,8 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, return d.Update(ctx, db, oldID, newState) case deployplan.ActionTypeUpdateWithID: return d.UpdateWithID(ctx, db, oldID, newState) + case deployplan.ActionTypeResize: + return d.Resize(ctx, db, oldID, newState) default: return fmt.Errorf("internal error: unexpected actionType: %#v", actionType) } @@ -186,6 +188,25 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, return nil } +func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, id string, newState any) error { + remoteState, err := d.Adapter.DoResize(ctx, id, newState) + if err != nil { + return fmt.Errorf("resizing id=%s: %w", id, err) + } + + err = d.SetRemoteState(remoteState) + if err != nil { + return err + } + + err = db.SaveState(d.ResourceKey, id, newState) + if err != nil { + return fmt.Errorf("saving state id=%s: %w", id, err) + } + + return nil +} + func typeConvert(destType reflect.Type, src any) (any, error) { raw, err := json.Marshal(src) if err != nil { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index f136631670..bc1edfa211 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -125,24 +125,12 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return false } - localAction, localChangeMap := convertChangesToTriggersMap(adapter, localDiff) - - if localAction == deployplan.ActionTypeRecreate { - entry.Action = localAction.String() - if len(localChangeMap) > 0 { - entry.Changes = &deployplan.Changes{ - Local: localChangeMap, - } - } - return true - } - remoteState, err := adapter.DoRefresh(ctx, dbentry.ID) if err != nil { if errors.Is(err, apierr.ErrResourceDoesNotExist) || errors.Is(err, apierr.ErrNotFound) { remoteState = nil } else { - logdiag.LogError(ctx, fmt.Errorf("%s: failed to read id=%q: %w (localAction=%q)", errorPrefix, dbentry.ID, err, localAction.String())) + logdiag.LogError(ctx, fmt.Errorf("%s: failed to read id=%q: %w", errorPrefix, dbentry.ID, err)) return false } } @@ -151,6 +139,17 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable entry.RemoteState = remoteState + localAction, localChangeMap := convertChangesToTriggersMap(ctx, adapter, localDiff, remoteState) + if localAction == deployplan.ActionTypeRecreate { + entry.Action = localAction.String() + if len(localChangeMap) > 0 { + entry.Changes = &deployplan.Changes{ + Local: localChangeMap, + } + } + return true + } + var remoteAction deployplan.ActionType var remoteChangeMap map[string]deployplan.Trigger @@ -219,12 +218,16 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return plan, nil } -func convertChangesToTriggersMap(adapter *dresources.Adapter, diff []structdiff.Change) (deployplan.ActionType, map[string]deployplan.Trigger) { +func convertChangesToTriggersMap(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { action := deployplan.ActionTypeSkip var m map[string]deployplan.Trigger for _, ch := range diff { - fieldAction := adapter.ClassifyByTriggers(ch) + fieldAction, err := adapter.ClassifyChange(ch, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify changes: %w", err)) + continue + } if fieldAction > action { action = fieldAction } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 2b9c0a0cfb..6cb4bfa9c0 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -66,6 +66,10 @@ type IResourceNoRefresh interface { // Example: func (r *ResourceVolume) DoUpdateWithID(ctx, id string, newState *catalog.CreateVolumeRequestContent) (string, error) DoUpdateWithID(ctx context.Context, id string, newState any) (string, error) + // [Optional] DoResize resizes the resource. Only supported by clusters + // Example: func (r *ResourceCluster) DoResize(ctx context.Context, id string, newState *compute.ClusterSpec) error + DoResize(ctx context.Context, id string, newState any) error + // [Optional] WaitAfterCreate waits for the resource to become ready after creation. // TODO: wait status should be persisted in the state. WaitAfterCreate(ctx context.Context, newState any) error @@ -92,6 +96,9 @@ type IResourceWithRefresh interface { // Optional: updates that may change ID. Returns new id and remote state when available. DoUpdateWithID(ctx context.Context, id string, newState any) (newID string, remoteState any, e error) + // [Optional] DoResize resizes the resource. Only supported by clusters + DoResize(ctx context.Context, id string, newState any) (remoteState any, e error) + // WaitAfterCreate waits for the resource to become ready after creation. WaitAfterCreate(ctx context.Context, newState any) (newRemoteState any, e error) @@ -115,6 +122,7 @@ type Adapter struct { waitAfterCreate *calladapt.BoundCaller waitAfterUpdate *calladapt.BoundCaller classifyChange *calladapt.BoundCaller + doResize *calladapt.BoundCaller fieldTriggers map[string]deployplan.ActionType } @@ -140,6 +148,7 @@ func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, err doCreate: nil, doUpdate: nil, doUpdateWithID: nil, + doResize: nil, waitAfterCreate: nil, waitAfterUpdate: nil, classifyChange: nil, @@ -230,6 +239,16 @@ func (a *Adapter) initMethods(resource any) error { return err } + a.classifyChange, err = calladapt.PrepareCall(resource, calladapt.TypeOf[IResourceNoRefresh](), "ClassifyChange") + if err != nil { + return err + } + + a.doResize, err = prepareCallFromTwoVariants(resource, "DoResize") + if err != nil { + return err + } + return nil } @@ -290,6 +309,13 @@ func (a *Adapter) validate() error { validations = append(validations, "DoUpdate remoteState return", a.doUpdate.OutTypes[0], remoteType) } + if a.doResize != nil { + validations = append(validations, "DoResize newState", a.doResize.InTypes[2], stateType) + if len(a.doUpdate.OutTypes) == 2 { + validations = append(validations, "DoResize remoteState return", a.doUpdate.OutTypes[0], remoteType) + } + } + if a.doUpdateWithID != nil { validations = append(validations, "DoUpdateWithID newState", a.doUpdateWithID.InTypes[2], stateType) if len(a.doUpdateWithID.OutTypes) == 3 { @@ -312,7 +338,7 @@ func (a *Adapter) validate() error { } if a.classifyChange != nil { - validations = append(validations, "ClassifyChange changes", a.classifyChange.InTypes[1], remoteType) + validations = append(validations, "ClassifyChange remoteState", a.classifyChange.InTypes[1], remoteType) } err = validateTypes(validations...) @@ -456,6 +482,23 @@ func (a *Adapter) DoUpdateWithID(ctx context.Context, oldID string, newState any } } +func (a *Adapter) DoResize(ctx context.Context, id string, newState any) (any, error) { + if a.doResize == nil { + return nil, errors.New("internal error: DoResize not found") + } + + outs, err := a.doResize.Call(ctx, id, newState) + if err != nil { + return nil, err + } + + if len(outs) == 1 { + return outs[0], nil + } else { + return nil, nil + } +} + // ClassifyByTriggers classifies a single using FieldTriggers. // Defaults to ActionTypeUpdate. func (a *Adapter) ClassifyByTriggers(change structdiff.Change) deployplan.ActionType { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index fcf68e3836..cec91e93f8 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -19,6 +19,7 @@ var SupportedResources = map[string]any{ "database_catalogs": (*ResourceDatabaseCatalog)(nil), "synced_database_tables": (*ResourceSyncedDatabaseTable)(nil), "alerts": (*ResourceAlert)(nil), + "clusters": (*ResourceCluster)(nil), } func InitAll(client *databricks.WorkspaceClient) (map[string]*Adapter, error) { diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go new file mode 100644 index 0000000000..a051a8a839 --- /dev/null +++ b/bundle/direct/dresources/cluster.go @@ -0,0 +1,180 @@ +package dresources + +import ( + "context" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/compute" +) + +type ResourceCluster struct { + client *databricks.WorkspaceClient +} + +func (r *ResourceCluster) New(client *databricks.WorkspaceClient) any { + return &ResourceCluster{ + client: client, + } +} + +func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.ClusterSpec { + return &input.ClusterSpec +} + +func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { + return &compute.ClusterSpec{ + Autoscale: input.Autoscale, + AutoterminationMinutes: input.AutoterminationMinutes, + AwsAttributes: input.AwsAttributes, + AzureAttributes: input.AzureAttributes, + ClusterLogConf: input.ClusterLogConf, + ClusterName: input.ClusterName, + CustomTags: input.CustomTags, + DataSecurityMode: input.DataSecurityMode, + DockerImage: input.DockerImage, + DriverInstancePoolId: input.DriverInstancePoolId, + DriverNodeTypeId: input.DriverNodeTypeId, + EnableElasticDisk: input.EnableElasticDisk, + EnableLocalDiskEncryption: input.EnableLocalDiskEncryption, + GcpAttributes: input.GcpAttributes, + InitScripts: input.InitScripts, + InstancePoolId: input.InstancePoolId, + IsSingleNode: input.IsSingleNode, + Kind: input.Kind, + NodeTypeId: input.NodeTypeId, + NumWorkers: input.NumWorkers, + PolicyId: input.PolicyId, + RemoteDiskThroughput: input.RemoteDiskThroughput, + RuntimeEngine: input.RuntimeEngine, + SingleUserName: input.SingleUserName, + SparkConf: input.SparkConf, + SparkEnvVars: input.SparkEnvVars, + SparkVersion: input.SparkVersion, + SshPublicKeys: input.SshPublicKeys, + TotalInitialRemoteDiskSize: input.TotalInitialRemoteDiskSize, + UseMlRuntime: input.UseMlRuntime, + WorkloadType: input.WorkloadType, + } +} + +func (r *ResourceCluster) DoRefresh(ctx context.Context, id string) (*compute.ClusterDetails, error) { + return r.client.Clusters.GetByClusterId(ctx, id) +} + +func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterSpec) (string, error) { + wait, err := r.client.Clusters.Create(ctx, makeCreateCluster(config)) + if err != nil { + return "", err + } + return wait.ClusterId, nil +} + +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec) error { + _, err := r.client.Clusters.Edit(ctx, makeEditCluster(id, config)) + if err != nil { + return err + } + return nil +} + +func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { + _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ + ClusterId: id, + NumWorkers: config.NumWorkers, + Autoscale: config.Autoscale, + }) + return err +} + +func (r *ResourceCluster) DoDelete(ctx context.Context, id string) error { + return r.client.Clusters.PermanentDeleteByClusterId(ctx, id) +} + +func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails) (deployplan.ActionType, error) { + // Always update if the cluster is not running. + if remoteState.State != compute.StateRunning { + return deployplan.ActionTypeUpdate, nil + } + + if change.Path.String() == "num_workers" { + return deployplan.ActionTypeResize, nil + } + + return deployplan.ActionTypeUpdate, nil +} + +func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { + return compute.CreateCluster{ + Autoscale: config.Autoscale, + AutoterminationMinutes: config.AutoterminationMinutes, + AwsAttributes: config.AwsAttributes, + AzureAttributes: config.AzureAttributes, + ClusterLogConf: config.ClusterLogConf, + ClusterName: config.ClusterName, + CustomTags: config.CustomTags, + DataSecurityMode: config.DataSecurityMode, + DockerImage: config.DockerImage, + DriverInstancePoolId: config.DriverInstancePoolId, + DriverNodeTypeId: config.DriverNodeTypeId, + EnableElasticDisk: config.EnableElasticDisk, + EnableLocalDiskEncryption: config.EnableLocalDiskEncryption, + GcpAttributes: config.GcpAttributes, + InitScripts: config.InitScripts, + InstancePoolId: config.InstancePoolId, + IsSingleNode: config.IsSingleNode, + Kind: config.Kind, + NodeTypeId: config.NodeTypeId, + NumWorkers: config.NumWorkers, + PolicyId: config.PolicyId, + RemoteDiskThroughput: config.RemoteDiskThroughput, + RuntimeEngine: config.RuntimeEngine, + SingleUserName: config.SingleUserName, + SparkConf: config.SparkConf, + SparkEnvVars: config.SparkEnvVars, + SparkVersion: config.SparkVersion, + SshPublicKeys: config.SshPublicKeys, + TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, + UseMlRuntime: config.UseMlRuntime, + WorkloadType: config.WorkloadType, + } +} + +func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster { + return compute.EditCluster{ + ClusterId: id, + Autoscale: config.Autoscale, + AutoterminationMinutes: config.AutoterminationMinutes, + AwsAttributes: config.AwsAttributes, + AzureAttributes: config.AzureAttributes, + ClusterLogConf: config.ClusterLogConf, + ClusterName: config.ClusterName, + CustomTags: config.CustomTags, + DataSecurityMode: config.DataSecurityMode, + DockerImage: config.DockerImage, + DriverInstancePoolId: config.DriverInstancePoolId, + DriverNodeTypeId: config.DriverNodeTypeId, + EnableElasticDisk: config.EnableElasticDisk, + EnableLocalDiskEncryption: config.EnableLocalDiskEncryption, + GcpAttributes: config.GcpAttributes, + InitScripts: config.InitScripts, + InstancePoolId: config.InstancePoolId, + IsSingleNode: config.IsSingleNode, + Kind: config.Kind, + NodeTypeId: config.NodeTypeId, + NumWorkers: config.NumWorkers, + PolicyId: config.PolicyId, + RemoteDiskThroughput: config.RemoteDiskThroughput, + RuntimeEngine: config.RuntimeEngine, + SingleUserName: config.SingleUserName, + SparkConf: config.SparkConf, + SparkEnvVars: config.SparkEnvVars, + SparkVersion: config.SparkVersion, + SshPublicKeys: config.SshPublicKeys, + TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, + UseMlRuntime: config.UseMlRuntime, + WorkloadType: config.WorkloadType, + } +} From 39ae957a8246e6b26f035aed46da254682ef9334 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 1 Oct 2025 18:41:47 +0200 Subject: [PATCH 04/10] udpate output --- .../bundle/override/clusters/output.txt | 2 + .../override/merge-string-map/output.txt | 1 + acceptance/bundle/refschema/out.fields.txt | 256 ++++++++++++++++++ .../clusters/deploy/simple/out.test.toml | 2 +- .../run/spark_python_task/out.test.toml | 2 +- .../empty_resources/empty_dict/output.txt | 4 +- .../empty_resources/with_grants/output.txt | 4 +- .../with_permissions/output.txt | 1 + bundle/direct/bundle_plan.go | 29 +- bundle/direct/dresources/adapter.go | 10 +- bundle/direct/dresources/all_test.go | 3 +- bundle/direct/dresources/cluster.go | 27 +- 12 files changed, 303 insertions(+), 38 deletions(-) diff --git a/acceptance/bundle/override/clusters/output.txt b/acceptance/bundle/override/clusters/output.txt index 7a35dfe256..1c26581d39 100644 --- a/acceptance/bundle/override/clusters/output.txt +++ b/acceptance/bundle/override/clusters/output.txt @@ -5,6 +5,7 @@ "max_workers": 7, "min_workers": 2 }, + "autotermination_minutes": 60, "cluster_name": "foo", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 2, @@ -20,6 +21,7 @@ "max_workers": 3, "min_workers": 1 }, + "autotermination_minutes": 60, "cluster_name": "foo-override", "node_type_id": "m5.xlarge", "num_workers": 3, diff --git a/acceptance/bundle/override/merge-string-map/output.txt b/acceptance/bundle/override/merge-string-map/output.txt index 20c3bcddc1..63a7a4d769 100644 --- a/acceptance/bundle/override/merge-string-map/output.txt +++ b/acceptance/bundle/override/merge-string-map/output.txt @@ -7,6 +7,7 @@ Warning: expected map, found string { "clusters": { "my_cluster": { + "autotermination_minutes": 60, "spark_version": "25" } } diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 92b66ab7ef..f2a85bdd9f 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -140,6 +140,262 @@ resources.apps.*.updater string ALL resources.apps.*.url string ALL resources.apps.*.user_api_scopes []string ALL resources.apps.*.user_api_scopes[*] string ALL +resources.clusters.*.apply_policy_default_values bool INPUT STATE +resources.clusters.*.autoscale *compute.AutoScale ALL +resources.clusters.*.autoscale.max_workers int ALL +resources.clusters.*.autoscale.min_workers int ALL +resources.clusters.*.autotermination_minutes int ALL +resources.clusters.*.aws_attributes *compute.AwsAttributes ALL +resources.clusters.*.aws_attributes.availability compute.AwsAvailability ALL +resources.clusters.*.aws_attributes.ebs_volume_count int ALL +resources.clusters.*.aws_attributes.ebs_volume_iops int ALL +resources.clusters.*.aws_attributes.ebs_volume_size int ALL +resources.clusters.*.aws_attributes.ebs_volume_throughput int ALL +resources.clusters.*.aws_attributes.ebs_volume_type compute.EbsVolumeType ALL +resources.clusters.*.aws_attributes.first_on_demand int ALL +resources.clusters.*.aws_attributes.instance_profile_arn string ALL +resources.clusters.*.aws_attributes.spot_bid_price_percent int ALL +resources.clusters.*.aws_attributes.zone_id string ALL +resources.clusters.*.azure_attributes *compute.AzureAttributes ALL +resources.clusters.*.azure_attributes.availability compute.AzureAvailability ALL +resources.clusters.*.azure_attributes.first_on_demand int ALL +resources.clusters.*.azure_attributes.log_analytics_info *compute.LogAnalyticsInfo ALL +resources.clusters.*.azure_attributes.log_analytics_info.log_analytics_primary_key string ALL +resources.clusters.*.azure_attributes.log_analytics_info.log_analytics_workspace_id string ALL +resources.clusters.*.azure_attributes.spot_bid_max_price float64 ALL +resources.clusters.*.cluster_cores float64 REMOTE +resources.clusters.*.cluster_id string REMOTE +resources.clusters.*.cluster_log_conf *compute.ClusterLogConf ALL +resources.clusters.*.cluster_log_conf.dbfs *compute.DbfsStorageInfo ALL +resources.clusters.*.cluster_log_conf.dbfs.destination string ALL +resources.clusters.*.cluster_log_conf.s3 *compute.S3StorageInfo ALL +resources.clusters.*.cluster_log_conf.s3.canned_acl string ALL +resources.clusters.*.cluster_log_conf.s3.destination string ALL +resources.clusters.*.cluster_log_conf.s3.enable_encryption bool ALL +resources.clusters.*.cluster_log_conf.s3.encryption_type string ALL +resources.clusters.*.cluster_log_conf.s3.endpoint string ALL +resources.clusters.*.cluster_log_conf.s3.kms_key string ALL +resources.clusters.*.cluster_log_conf.s3.region string ALL +resources.clusters.*.cluster_log_conf.volumes *compute.VolumesStorageInfo ALL +resources.clusters.*.cluster_log_conf.volumes.destination string ALL +resources.clusters.*.cluster_log_status *compute.LogSyncStatus REMOTE +resources.clusters.*.cluster_log_status.last_attempted int64 REMOTE +resources.clusters.*.cluster_log_status.last_exception string REMOTE +resources.clusters.*.cluster_memory_mb int64 REMOTE +resources.clusters.*.cluster_name string ALL +resources.clusters.*.cluster_source compute.ClusterSource REMOTE +resources.clusters.*.creator_user_name string REMOTE +resources.clusters.*.custom_tags map[string]string ALL +resources.clusters.*.custom_tags.* string ALL +resources.clusters.*.data_security_mode compute.DataSecurityMode ALL +resources.clusters.*.default_tags map[string]string REMOTE +resources.clusters.*.default_tags.* string REMOTE +resources.clusters.*.docker_image *compute.DockerImage ALL +resources.clusters.*.docker_image.basic_auth *compute.DockerBasicAuth ALL +resources.clusters.*.docker_image.basic_auth.password string ALL +resources.clusters.*.docker_image.basic_auth.username string ALL +resources.clusters.*.docker_image.url string ALL +resources.clusters.*.driver *compute.SparkNode REMOTE +resources.clusters.*.driver.host_private_ip string REMOTE +resources.clusters.*.driver.instance_id string REMOTE +resources.clusters.*.driver.node_aws_attributes *compute.SparkNodeAwsAttributes REMOTE +resources.clusters.*.driver.node_aws_attributes.is_spot bool REMOTE +resources.clusters.*.driver.node_id string REMOTE +resources.clusters.*.driver.private_ip string REMOTE +resources.clusters.*.driver.public_dns string REMOTE +resources.clusters.*.driver.start_timestamp int64 REMOTE +resources.clusters.*.driver_instance_pool_id string ALL +resources.clusters.*.driver_node_type_id string ALL +resources.clusters.*.enable_elastic_disk bool ALL +resources.clusters.*.enable_local_disk_encryption bool ALL +resources.clusters.*.executors []compute.SparkNode REMOTE +resources.clusters.*.executors[*] compute.SparkNode REMOTE +resources.clusters.*.executors[*].host_private_ip string REMOTE +resources.clusters.*.executors[*].instance_id string REMOTE +resources.clusters.*.executors[*].node_aws_attributes *compute.SparkNodeAwsAttributes REMOTE +resources.clusters.*.executors[*].node_aws_attributes.is_spot bool REMOTE +resources.clusters.*.executors[*].node_id string REMOTE +resources.clusters.*.executors[*].private_ip string REMOTE +resources.clusters.*.executors[*].public_dns string REMOTE +resources.clusters.*.executors[*].start_timestamp int64 REMOTE +resources.clusters.*.gcp_attributes *compute.GcpAttributes ALL +resources.clusters.*.gcp_attributes.availability compute.GcpAvailability ALL +resources.clusters.*.gcp_attributes.boot_disk_size int ALL +resources.clusters.*.gcp_attributes.first_on_demand int ALL +resources.clusters.*.gcp_attributes.google_service_account string ALL +resources.clusters.*.gcp_attributes.local_ssd_count int ALL +resources.clusters.*.gcp_attributes.use_preemptible_executors bool ALL +resources.clusters.*.gcp_attributes.zone_id string ALL +resources.clusters.*.id string INPUT +resources.clusters.*.init_scripts []compute.InitScriptInfo ALL +resources.clusters.*.init_scripts[*] compute.InitScriptInfo ALL +resources.clusters.*.init_scripts[*].abfss *compute.Adlsgen2Info ALL +resources.clusters.*.init_scripts[*].abfss.destination string ALL +resources.clusters.*.init_scripts[*].dbfs *compute.DbfsStorageInfo ALL +resources.clusters.*.init_scripts[*].dbfs.destination string ALL +resources.clusters.*.init_scripts[*].file *compute.LocalFileInfo ALL +resources.clusters.*.init_scripts[*].file.destination string ALL +resources.clusters.*.init_scripts[*].gcs *compute.GcsStorageInfo ALL +resources.clusters.*.init_scripts[*].gcs.destination string ALL +resources.clusters.*.init_scripts[*].s3 *compute.S3StorageInfo ALL +resources.clusters.*.init_scripts[*].s3.canned_acl string ALL +resources.clusters.*.init_scripts[*].s3.destination string ALL +resources.clusters.*.init_scripts[*].s3.enable_encryption bool ALL +resources.clusters.*.init_scripts[*].s3.encryption_type string ALL +resources.clusters.*.init_scripts[*].s3.endpoint string ALL +resources.clusters.*.init_scripts[*].s3.kms_key string ALL +resources.clusters.*.init_scripts[*].s3.region string ALL +resources.clusters.*.init_scripts[*].volumes *compute.VolumesStorageInfo ALL +resources.clusters.*.init_scripts[*].volumes.destination string ALL +resources.clusters.*.init_scripts[*].workspace *compute.WorkspaceStorageInfo ALL +resources.clusters.*.init_scripts[*].workspace.destination string ALL +resources.clusters.*.instance_pool_id string ALL +resources.clusters.*.is_single_node bool ALL +resources.clusters.*.jdbc_port int REMOTE +resources.clusters.*.kind compute.Kind ALL +resources.clusters.*.last_restarted_time int64 REMOTE +resources.clusters.*.last_state_loss_time int64 REMOTE +resources.clusters.*.lifecycle resources.Lifecycle INPUT +resources.clusters.*.lifecycle.prevent_destroy bool INPUT +resources.clusters.*.modified_status string INPUT +resources.clusters.*.node_type_id string ALL +resources.clusters.*.num_workers int ALL +resources.clusters.*.permissions []resources.ClusterPermission INPUT +resources.clusters.*.permissions[*] resources.ClusterPermission INPUT +resources.clusters.*.permissions[*].group_name string INPUT +resources.clusters.*.permissions[*].level resources.ClusterPermissionLevel INPUT +resources.clusters.*.permissions[*].service_principal_name string INPUT +resources.clusters.*.permissions[*].user_name string INPUT +resources.clusters.*.policy_id string ALL +resources.clusters.*.remote_disk_throughput int ALL +resources.clusters.*.runtime_engine compute.RuntimeEngine ALL +resources.clusters.*.single_user_name string ALL +resources.clusters.*.spark_conf map[string]string ALL +resources.clusters.*.spark_conf.* string ALL +resources.clusters.*.spark_context_id int64 REMOTE +resources.clusters.*.spark_env_vars map[string]string ALL +resources.clusters.*.spark_env_vars.* string ALL +resources.clusters.*.spark_version string ALL +resources.clusters.*.spec *compute.ClusterSpec REMOTE +resources.clusters.*.spec.apply_policy_default_values bool REMOTE +resources.clusters.*.spec.autoscale *compute.AutoScale REMOTE +resources.clusters.*.spec.autoscale.max_workers int REMOTE +resources.clusters.*.spec.autoscale.min_workers int REMOTE +resources.clusters.*.spec.autotermination_minutes int REMOTE +resources.clusters.*.spec.aws_attributes *compute.AwsAttributes REMOTE +resources.clusters.*.spec.aws_attributes.availability compute.AwsAvailability REMOTE +resources.clusters.*.spec.aws_attributes.ebs_volume_count int REMOTE +resources.clusters.*.spec.aws_attributes.ebs_volume_iops int REMOTE +resources.clusters.*.spec.aws_attributes.ebs_volume_size int REMOTE +resources.clusters.*.spec.aws_attributes.ebs_volume_throughput int REMOTE +resources.clusters.*.spec.aws_attributes.ebs_volume_type compute.EbsVolumeType REMOTE +resources.clusters.*.spec.aws_attributes.first_on_demand int REMOTE +resources.clusters.*.spec.aws_attributes.instance_profile_arn string REMOTE +resources.clusters.*.spec.aws_attributes.spot_bid_price_percent int REMOTE +resources.clusters.*.spec.aws_attributes.zone_id string REMOTE +resources.clusters.*.spec.azure_attributes *compute.AzureAttributes REMOTE +resources.clusters.*.spec.azure_attributes.availability compute.AzureAvailability REMOTE +resources.clusters.*.spec.azure_attributes.first_on_demand int REMOTE +resources.clusters.*.spec.azure_attributes.log_analytics_info *compute.LogAnalyticsInfo REMOTE +resources.clusters.*.spec.azure_attributes.log_analytics_info.log_analytics_primary_key string REMOTE +resources.clusters.*.spec.azure_attributes.log_analytics_info.log_analytics_workspace_id string REMOTE +resources.clusters.*.spec.azure_attributes.spot_bid_max_price float64 REMOTE +resources.clusters.*.spec.cluster_log_conf *compute.ClusterLogConf REMOTE +resources.clusters.*.spec.cluster_log_conf.dbfs *compute.DbfsStorageInfo REMOTE +resources.clusters.*.spec.cluster_log_conf.dbfs.destination string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3 *compute.S3StorageInfo REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.canned_acl string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.destination string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.enable_encryption bool REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.encryption_type string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.endpoint string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.kms_key string REMOTE +resources.clusters.*.spec.cluster_log_conf.s3.region string REMOTE +resources.clusters.*.spec.cluster_log_conf.volumes *compute.VolumesStorageInfo REMOTE +resources.clusters.*.spec.cluster_log_conf.volumes.destination string REMOTE +resources.clusters.*.spec.cluster_name string REMOTE +resources.clusters.*.spec.custom_tags map[string]string REMOTE +resources.clusters.*.spec.custom_tags.* string REMOTE +resources.clusters.*.spec.data_security_mode compute.DataSecurityMode REMOTE +resources.clusters.*.spec.docker_image *compute.DockerImage REMOTE +resources.clusters.*.spec.docker_image.basic_auth *compute.DockerBasicAuth REMOTE +resources.clusters.*.spec.docker_image.basic_auth.password string REMOTE +resources.clusters.*.spec.docker_image.basic_auth.username string REMOTE +resources.clusters.*.spec.docker_image.url string REMOTE +resources.clusters.*.spec.driver_instance_pool_id string REMOTE +resources.clusters.*.spec.driver_node_type_id string REMOTE +resources.clusters.*.spec.enable_elastic_disk bool REMOTE +resources.clusters.*.spec.enable_local_disk_encryption bool REMOTE +resources.clusters.*.spec.gcp_attributes *compute.GcpAttributes REMOTE +resources.clusters.*.spec.gcp_attributes.availability compute.GcpAvailability REMOTE +resources.clusters.*.spec.gcp_attributes.boot_disk_size int REMOTE +resources.clusters.*.spec.gcp_attributes.first_on_demand int REMOTE +resources.clusters.*.spec.gcp_attributes.google_service_account string REMOTE +resources.clusters.*.spec.gcp_attributes.local_ssd_count int REMOTE +resources.clusters.*.spec.gcp_attributes.use_preemptible_executors bool REMOTE +resources.clusters.*.spec.gcp_attributes.zone_id string REMOTE +resources.clusters.*.spec.init_scripts []compute.InitScriptInfo REMOTE +resources.clusters.*.spec.init_scripts[*] compute.InitScriptInfo REMOTE +resources.clusters.*.spec.init_scripts[*].abfss *compute.Adlsgen2Info REMOTE +resources.clusters.*.spec.init_scripts[*].abfss.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].dbfs *compute.DbfsStorageInfo REMOTE +resources.clusters.*.spec.init_scripts[*].dbfs.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].file *compute.LocalFileInfo REMOTE +resources.clusters.*.spec.init_scripts[*].file.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].gcs *compute.GcsStorageInfo REMOTE +resources.clusters.*.spec.init_scripts[*].gcs.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].s3 *compute.S3StorageInfo REMOTE +resources.clusters.*.spec.init_scripts[*].s3.canned_acl string REMOTE +resources.clusters.*.spec.init_scripts[*].s3.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].s3.enable_encryption bool REMOTE +resources.clusters.*.spec.init_scripts[*].s3.encryption_type string REMOTE +resources.clusters.*.spec.init_scripts[*].s3.endpoint string REMOTE +resources.clusters.*.spec.init_scripts[*].s3.kms_key string REMOTE +resources.clusters.*.spec.init_scripts[*].s3.region string REMOTE +resources.clusters.*.spec.init_scripts[*].volumes *compute.VolumesStorageInfo REMOTE +resources.clusters.*.spec.init_scripts[*].volumes.destination string REMOTE +resources.clusters.*.spec.init_scripts[*].workspace *compute.WorkspaceStorageInfo REMOTE +resources.clusters.*.spec.init_scripts[*].workspace.destination string REMOTE +resources.clusters.*.spec.instance_pool_id string REMOTE +resources.clusters.*.spec.is_single_node bool REMOTE +resources.clusters.*.spec.kind compute.Kind REMOTE +resources.clusters.*.spec.node_type_id string REMOTE +resources.clusters.*.spec.num_workers int REMOTE +resources.clusters.*.spec.policy_id string REMOTE +resources.clusters.*.spec.remote_disk_throughput int REMOTE +resources.clusters.*.spec.runtime_engine compute.RuntimeEngine REMOTE +resources.clusters.*.spec.single_user_name string REMOTE +resources.clusters.*.spec.spark_conf map[string]string REMOTE +resources.clusters.*.spec.spark_conf.* string REMOTE +resources.clusters.*.spec.spark_env_vars map[string]string REMOTE +resources.clusters.*.spec.spark_env_vars.* string REMOTE +resources.clusters.*.spec.spark_version string REMOTE +resources.clusters.*.spec.ssh_public_keys []string REMOTE +resources.clusters.*.spec.ssh_public_keys[*] string REMOTE +resources.clusters.*.spec.total_initial_remote_disk_size int REMOTE +resources.clusters.*.spec.use_ml_runtime bool REMOTE +resources.clusters.*.spec.workload_type *compute.WorkloadType REMOTE +resources.clusters.*.spec.workload_type.clients compute.ClientsTypes REMOTE +resources.clusters.*.spec.workload_type.clients.jobs bool REMOTE +resources.clusters.*.spec.workload_type.clients.notebooks bool REMOTE +resources.clusters.*.ssh_public_keys []string ALL +resources.clusters.*.ssh_public_keys[*] string ALL +resources.clusters.*.start_time int64 REMOTE +resources.clusters.*.state compute.State REMOTE +resources.clusters.*.state_message string REMOTE +resources.clusters.*.terminated_time int64 REMOTE +resources.clusters.*.termination_reason *compute.TerminationReason REMOTE +resources.clusters.*.termination_reason.code compute.TerminationReasonCode REMOTE +resources.clusters.*.termination_reason.parameters map[string]string REMOTE +resources.clusters.*.termination_reason.parameters.* string REMOTE +resources.clusters.*.termination_reason.type compute.TerminationReasonType REMOTE +resources.clusters.*.total_initial_remote_disk_size int ALL +resources.clusters.*.url string INPUT +resources.clusters.*.use_ml_runtime bool ALL +resources.clusters.*.workload_type *compute.WorkloadType ALL +resources.clusters.*.workload_type.clients compute.ClientsTypes ALL +resources.clusters.*.workload_type.clients.jobs bool ALL +resources.clusters.*.workload_type.clients.notebooks bool ALL resources.database_catalogs.*.create_database_if_not_exists bool ALL resources.database_catalogs.*.database_instance_name string ALL resources.database_catalogs.*.database_name string ALL diff --git a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml index 3cdb920b67..c3a1b55592 100644 --- a/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml +++ b/acceptance/bundle/resources/clusters/deploy/simple/out.test.toml @@ -2,4 +2,4 @@ Local = false Cloud = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml index 1ae7a3995d..601e190219 100644 --- a/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml +++ b/acceptance/bundle/resources/clusters/run/spark_python_task/out.test.toml @@ -3,4 +3,4 @@ Cloud = true CloudSlow = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt index d19111a5b9..f8de8d8c8f 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt +++ b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt @@ -134,7 +134,9 @@ Warning: required field "schema_name" is not set === resources.clusters.rname === { "clusters": { - "rname": {} + "rname": { + "autotermination_minutes": 60 + } } } diff --git a/acceptance/bundle/validate/empty_resources/with_grants/output.txt b/acceptance/bundle/validate/empty_resources/with_grants/output.txt index 3798d19e8a..bb761449cd 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_grants/output.txt @@ -163,7 +163,9 @@ Warning: unknown field: grants { "clusters": { - "rname": {} + "rname": { + "autotermination_minutes": 60 + } } } diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt index c8130b1792..6859b6e6ca 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt @@ -151,6 +151,7 @@ Warning: required field "schema_name" is not set { "clusters": { "rname": { + "autotermination_minutes": 60, "permissions": [] } } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index bc1edfa211..7405aba68b 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -135,11 +135,7 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d } } - // We have a choice whether to include remoteState or remoteStateComparable from below. - // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable - entry.RemoteState = remoteState - - localAction, localChangeMap := convertChangesToTriggersMap(ctx, adapter, localDiff, remoteState) + localAction, localChangeMap := convertChangesToTriggersMap(adapter, localDiff, remoteState) if localAction == deployplan.ActionTypeRecreate { entry.Action = localAction.String() if len(localChangeMap) > 0 { @@ -150,6 +146,10 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return true } + // We have a choice whether to include remoteState or remoteStateComparable from below. + // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable + entry.RemoteState = remoteState + var remoteAction deployplan.ActionType var remoteChangeMap map[string]deployplan.Trigger @@ -168,7 +168,7 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return false } - remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(ctx, adapter, remoteDiff, remoteState) + remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(adapter, remoteDiff, remoteState) } entry.Action = max(localAction, remoteAction).String() @@ -218,16 +218,12 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return plan, nil } -func convertChangesToTriggersMap(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { +func convertChangesToTriggersMap(adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { action := deployplan.ActionTypeSkip var m map[string]deployplan.Trigger for _, ch := range diff { - fieldAction, err := adapter.ClassifyChange(ch, remoteState) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify changes: %w", err)) - continue - } + fieldAction := adapter.ClassifyChange(ch, remoteState) if fieldAction > action { action = fieldAction } @@ -240,7 +236,7 @@ func convertChangesToTriggersMap(ctx context.Context, adapter *dresources.Adapte return action, m } -func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { +func interpretOldStateVsRemoteState(adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { action := deployplan.ActionTypeSkip m := make(map[string]deployplan.Trigger) @@ -255,12 +251,7 @@ func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Ada } continue } - fieldAction, err := adapter.ClassifyChange(ch, remoteState) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify changes: %w", err)) - continue - } - + fieldAction := adapter.ClassifyChange(ch, remoteState) if fieldAction > action { action = fieldAction } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 6cb4bfa9c0..340a6b1c9d 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -78,7 +78,7 @@ type IResourceNoRefresh interface { WaitAfterUpdate(ctx context.Context, newState any) error // [Optional] ClassifyChange classifies a set of changes using custom logic. - ClassifyChange(change structdiff.Change, remoteState any) (deployplan.ActionType, error) + ClassifyChange(change structdiff.Change, remoteState any) deployplan.ActionType } // IResourceWithRefresh is an alternative to IResourceNoRefresh but every method can return remoteState. @@ -553,19 +553,19 @@ func (a *Adapter) WaitAfterUpdate(ctx context.Context, newState any) (any, error } } -func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any) (deployplan.ActionType, error) { +func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any) deployplan.ActionType { // If ClassifyChange is not implemented, use FieldTriggers. if a.classifyChange == nil { - return a.ClassifyByTriggers(change), nil + return a.ClassifyByTriggers(change) } outs, err := a.classifyChange.Call(change, remoteState) if err != nil { - return deployplan.ActionTypeSkip, err + return deployplan.ActionTypeSkip } actionType := outs[0].(deployplan.ActionType) - return actionType, nil + return actionType } // prepareCallRequired prepares a call and ensures the method is found. diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 795785c284..eae52b87f1 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -226,12 +226,11 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W path, err := structpath.Parse("name") require.NoError(t, err) - _, err = adapter.ClassifyChange(structdiff.Change{ + _ = adapter.ClassifyChange(structdiff.Change{ Path: path, Old: nil, New: "mynewname", }, remote) - require.NoError(t, err) } // validateFields uses structwalk to generate all valid field paths and checks membership. diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index a051a8a839..caa4726e8c 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -25,7 +25,7 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.Cluste } func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { - return &compute.ClusterSpec{ + spec := &compute.ClusterSpec{ Autoscale: input.Autoscale, AutoterminationMinutes: input.AutoterminationMinutes, AwsAttributes: input.AwsAttributes, @@ -57,7 +57,12 @@ func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.Clu TotalInitialRemoteDiskSize: input.TotalInitialRemoteDiskSize, UseMlRuntime: input.UseMlRuntime, WorkloadType: input.WorkloadType, + ForceSendFields: filterFields[compute.ClusterSpec](input.ForceSendFields), } + if input.Spec != nil { + spec.ApplyPolicyDefaultValues = input.Spec.ApplyPolicyDefaultValues + } + return spec } func (r *ResourceCluster) DoRefresh(ctx context.Context, id string) (*compute.ClusterDetails, error) { @@ -82,9 +87,10 @@ func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compu func (r *ResourceCluster) DoResize(ctx context.Context, id string, config *compute.ClusterSpec) error { _, err := r.client.Clusters.Resize(ctx, compute.ResizeCluster{ - ClusterId: id, - NumWorkers: config.NumWorkers, - Autoscale: config.Autoscale, + ClusterId: id, + NumWorkers: config.NumWorkers, + Autoscale: config.Autoscale, + ForceSendFields: filterFields[compute.ResizeCluster](config.ForceSendFields), }) return err } @@ -93,27 +99,29 @@ func (r *ResourceCluster) DoDelete(ctx context.Context, id string) error { return r.client.Clusters.PermanentDeleteByClusterId(ctx, id) } -func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails) (deployplan.ActionType, error) { +func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails) deployplan.ActionType { // Always update if the cluster is not running. if remoteState.State != compute.StateRunning { - return deployplan.ActionTypeUpdate, nil + return deployplan.ActionTypeUpdate } if change.Path.String() == "num_workers" { - return deployplan.ActionTypeResize, nil + return deployplan.ActionTypeResize } - return deployplan.ActionTypeUpdate, nil + return deployplan.ActionTypeUpdate } func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { return compute.CreateCluster{ + ApplyPolicyDefaultValues: config.ApplyPolicyDefaultValues, Autoscale: config.Autoscale, AutoterminationMinutes: config.AutoterminationMinutes, AwsAttributes: config.AwsAttributes, AzureAttributes: config.AzureAttributes, ClusterLogConf: config.ClusterLogConf, ClusterName: config.ClusterName, + CloneFrom: nil, // Not supported by DABs CustomTags: config.CustomTags, DataSecurityMode: config.DataSecurityMode, DockerImage: config.DockerImage, @@ -139,12 +147,14 @@ func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, UseMlRuntime: config.UseMlRuntime, WorkloadType: config.WorkloadType, + ForceSendFields: filterFields[compute.CreateCluster](config.ForceSendFields), } } func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster { return compute.EditCluster{ ClusterId: id, + ApplyPolicyDefaultValues: config.ApplyPolicyDefaultValues, Autoscale: config.Autoscale, AutoterminationMinutes: config.AutoterminationMinutes, AwsAttributes: config.AwsAttributes, @@ -176,5 +186,6 @@ func makeEditCluster(id string, config *compute.ClusterSpec) compute.EditCluster TotalInitialRemoteDiskSize: config.TotalInitialRemoteDiskSize, UseMlRuntime: config.UseMlRuntime, WorkloadType: config.WorkloadType, + ForceSendFields: filterFields[compute.EditCluster](config.ForceSendFields), } } From b97b6fc5e18c682bbd0668e61d79b5e4fc543732 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 1 Oct 2025 19:09:46 +0200 Subject: [PATCH 05/10] fix lint --- bundle/direct/dresources/cluster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index caa4726e8c..4c1e633cbb 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -26,6 +26,7 @@ func (r *ResourceCluster) PrepareState(input *resources.Cluster) *compute.Cluste func (r *ResourceCluster) RemapState(input *compute.ClusterDetails) *compute.ClusterSpec { spec := &compute.ClusterSpec{ + ApplyPolicyDefaultValues: false, Autoscale: input.Autoscale, AutoterminationMinutes: input.AutoterminationMinutes, AwsAttributes: input.AwsAttributes, From f7fe812359c3e936fb7f4b6dbf264f454b02c91d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 12:01:26 +0200 Subject: [PATCH 06/10] autoscale resize --- bundle/direct/dresources/cluster.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 4c1e633cbb..b2bcdb1bdb 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -106,7 +106,8 @@ func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState * return deployplan.ActionTypeUpdate } - if change.Path.String() == "num_workers" { + changedPath := change.Path.String() + if changedPath == "num_workers" || changedPath == "autoscale" { return deployplan.ActionTypeResize } From d2d9d168a95b6b4bac28feca4c05c61cd0f49887 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Thu, 2 Oct 2025 12:59:42 +0200 Subject: [PATCH 07/10] fallback to classify by triggers --- bundle/direct/dresources/adapter.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 340a6b1c9d..ac967c927c 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -565,6 +565,10 @@ func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any) depl } actionType := outs[0].(deployplan.ActionType) + // If the action type is unset, use FieldTriggers. + if actionType == deployplan.ActionTypeUnset { + return a.ClassifyByTriggers(change) + } return actionType } From 4c1501b1f91ba7120ce6e8b44487f46ba232ffd2 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 6 Oct 2025 14:53:25 +0200 Subject: [PATCH 08/10] fixes --- .../interactive_cluster/out.test.toml | 2 +- .../interactive_cluster/test.toml | 1 - .../out.test.toml | 2 +- .../test.toml | 2 - .../interactive_single_user/out.test.toml | 2 +- .../interactive_single_user/test.toml | 1 - .../databricks.yml.tmpl | 22 +++ .../hello_world.py | 1 + .../update-and-resize-autoscale/out.test.toml | 5 + .../update-and-resize-autoscale/output.txt | 163 ++++++++++++++++++ .../deploy/update-and-resize-autoscale/script | 58 +++++++ .../update-and-resize-autoscale/test.toml | 11 ++ bundle/direct/apply.go | 7 +- bundle/direct/bundle_plan.go | 20 ++- bundle/direct/dresources/adapter.go | 36 ++-- bundle/direct/dresources/all_test.go | 3 +- bundle/direct/dresources/cluster.go | 11 +- libs/testserver/clusters.go | 1 + 18 files changed, 298 insertions(+), 50 deletions(-) delete mode 100644 acceptance/bundle/integration_whl/interactive_cluster/test.toml delete mode 100644 acceptance/bundle/integration_whl/interactive_single_user/test.toml create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/hello_world.py create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt create mode 100755 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml diff --git a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml index 1ae7a3995d..601e190219 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster/out.test.toml @@ -3,4 +3,4 @@ Cloud = true CloudSlow = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster/test.toml b/acceptance/bundle/integration_whl/interactive_cluster/test.toml deleted file mode 100644 index 4b60f66ec8..0000000000 --- a/acceptance/bundle/integration_whl/interactive_cluster/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # clusters resource diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml index f97d9ed887..e31a903bd2 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/out.test.toml @@ -3,5 +3,5 @@ Cloud = true CloudSlow = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] DATA_SECURITY_MODE = ["USER_ISOLATION", "SINGLE_USER"] diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml index 01ca313e09..35f395f3c4 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml @@ -1,5 +1,3 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # clusters not implemented - [EnvMatrix] DATA_SECURITY_MODE = [ "USER_ISOLATION", diff --git a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml index 1ae7a3995d..601e190219 100644 --- a/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml +++ b/acceptance/bundle/integration_whl/interactive_single_user/out.test.toml @@ -3,4 +3,4 @@ Cloud = true CloudSlow = true [EnvMatrix] - DATABRICKS_BUNDLE_ENGINE = ["terraform"] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/integration_whl/interactive_single_user/test.toml b/acceptance/bundle/integration_whl/interactive_single_user/test.toml deleted file mode 100644 index 4b60f66ec8..0000000000 --- a/acceptance/bundle/integration_whl/interactive_single_user/test.toml +++ /dev/null @@ -1 +0,0 @@ -EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"] # clusters resource diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl new file mode 100644 index 0000000000..929acfe1cd --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-deploy-cluster-autoscale + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +resources: + clusters: + test_cluster: + cluster_name: test-cluster-$UNIQUE_NAME + spark_version: $DEFAULT_SPARK_VERSION + node_type_id: $NODE_TYPE_ID + num_workers: 2 + + jobs: + foo: + name: test-job-with-cluster-$UNIQUE_NAME + tasks: + - task_key: my_spark_python_task + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + spark_python_task: + python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/hello_world.py b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/hello_world.py new file mode 100644 index 0000000000..f301245e24 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/hello_world.py @@ -0,0 +1 @@ +print("Hello World!") diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml new file mode 100644 index 0000000000..e092fd5ed6 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt new file mode 100644 index 0000000000..f2694cf0d4 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -0,0 +1,163 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Cluster should exist with num_workers after bundle deployment: +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 2, + "autoscale": null +} + +=== Adding autoscale section should call update API on stopped cluster + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/edit", + "body": { + "autoscale": { + "max_workers": 4, + "min_workers": 2 + }, + "autotermination_minutes": 60, + "cluster_id": "[UUID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } +} + +=== Cluster should have autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 4, + "min_workers": 2 + } +} + +=== Changing autoscale should call update API on stopped cluster + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/edit"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/edit", + "body": { + "autoscale": { + "max_workers": 5, + "min_workers": 3 + }, + "autotermination_minutes": 60, + "cluster_id": "[UUID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } +} + +=== Cluster should have new autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 5, + "min_workers": 3 + } +} + +=== Starting the cluster +{ + "autoscale": { + "max_workers":5, + "min_workers":3 + }, + "autotermination_minutes":60, + "cluster_id":"[UUID]", + "cluster_name":"test-cluster-[UNIQUE_NAME]", + "node_type_id":"[NODE_TYPE_ID]", + "spark_version":"13.3.x-snapshot-scala2.12", + "state":"RUNNING" +} + +=== Changing autoscale should call resize API on running cluster + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "cluster_id": "[UUID]" + } +} + +=== Cluster should have new autoscale +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": null, + "autoscale": { + "max_workers": 6, + "min_workers": 4 + } +} + +=== Removing autoscale section should call resize API + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> jq select(.method == "POST" and (.path | contains("/clusters/resize"))) out.requests.txt +{ + "method": "POST", + "path": "/api/2.1/clusters/resize", + "body": { + "cluster_id": "[UUID]", + "num_workers": 3 + } +} + +=== Cluster should have num_workers +{ + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "num_workers": 3, + "autoscale": null +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script new file mode 100755 index 0000000000..c5404e3a64 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -0,0 +1,58 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle deploy + +title "Cluster should exist with num_workers after bundle deployment:\n" +CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' + +title "Adding autoscale section should call update API on stopped cluster\n" +update_file.py databricks.yml " num_workers: 2" " autoscale: + min_workers: 2 + max_workers: 4" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have autoscale\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' + +title "Changing autoscale should call update API on stopped cluster\n" +update_file.py databricks.yml "min_workers: 2" "min_workers: 3" +update_file.py databricks.yml "max_workers: 4" "max_workers: 5" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have new autoscale\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' + +title "Starting the cluster\n" +$CLI clusters start "${CLUSTER_ID}" + +title "Changing autoscale should call resize API on running cluster\n" +update_file.py databricks.yml "min_workers: 3" "min_workers: 4" +update_file.py databricks.yml "max_workers: 5" "max_workers: 6" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have new autoscale\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' + +title "Removing autoscale section should call resize API\n" +update_file.py databricks.yml " autoscale: + min_workers: 4 + max_workers: 6" " num_workers: 3" +trace $CLI bundle deploy +trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt +rm out.requests.txt + +title "Cluster should have num_workers\n" +$CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml new file mode 100644 index 0000000000..964a45f5c1 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml @@ -0,0 +1,11 @@ +Local = true +Cloud = false +RecordRequests = true + +Ignore = [ + "databricks.yml", +] + +[[Repls]] +Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" +New = "[CLUSTER-ID]" diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 5baf3d00ce..f938ff8c93 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -189,16 +189,11 @@ func (d *DeploymentUnit) Delete(ctx context.Context, db *dstate.DeploymentState, } func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, id string, newState any) error { - remoteState, err := d.Adapter.DoResize(ctx, id, newState) + err := d.Adapter.DoResize(ctx, id, newState) if err != nil { return fmt.Errorf("resizing id=%s: %w", id, err) } - err = d.SetRemoteState(remoteState) - if err != nil { - return err - } - err = db.SaveState(d.ResourceKey, id, newState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 7405aba68b..ff03b2728f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -135,7 +135,7 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d } } - localAction, localChangeMap := convertChangesToTriggersMap(adapter, localDiff, remoteState) + localAction, localChangeMap := convertChangesToTriggersMap(ctx, adapter, localDiff, remoteState) if localAction == deployplan.ActionTypeRecreate { entry.Action = localAction.String() if len(localChangeMap) > 0 { @@ -168,7 +168,7 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return false } - remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(adapter, remoteDiff, remoteState) + remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(ctx, adapter, remoteDiff, remoteState) } entry.Action = max(localAction, remoteAction).String() @@ -218,12 +218,16 @@ func (b *DeploymentBundle) CalculatePlanForDeploy(ctx context.Context, client *d return plan, nil } -func convertChangesToTriggersMap(adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { +func convertChangesToTriggersMap(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { action := deployplan.ActionTypeSkip var m map[string]deployplan.Trigger for _, ch := range diff { - fieldAction := adapter.ClassifyChange(ch, remoteState) + fieldAction, err := adapter.ClassifyChange(ch, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify change: %w", err)) + continue + } if fieldAction > action { action = fieldAction } @@ -236,7 +240,7 @@ func convertChangesToTriggersMap(adapter *dresources.Adapter, diff []structdiff. return action, m } -func interpretOldStateVsRemoteState(adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { +func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.Trigger) { action := deployplan.ActionTypeSkip m := make(map[string]deployplan.Trigger) @@ -251,7 +255,11 @@ func interpretOldStateVsRemoteState(adapter *dresources.Adapter, diff []structdi } continue } - fieldAction := adapter.ClassifyChange(ch, remoteState) + fieldAction, err := adapter.ClassifyChange(ch, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify change: %w", err)) + continue + } if fieldAction > action { action = fieldAction } diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index ac967c927c..465eb51089 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -78,7 +78,7 @@ type IResourceNoRefresh interface { WaitAfterUpdate(ctx context.Context, newState any) error // [Optional] ClassifyChange classifies a set of changes using custom logic. - ClassifyChange(change structdiff.Change, remoteState any) deployplan.ActionType + ClassifyChange(change structdiff.Change, remoteState any) (deployplan.ActionType, error) } // IResourceWithRefresh is an alternative to IResourceNoRefresh but every method can return remoteState. @@ -96,9 +96,6 @@ type IResourceWithRefresh interface { // Optional: updates that may change ID. Returns new id and remote state when available. DoUpdateWithID(ctx context.Context, id string, newState any) (newID string, remoteState any, e error) - // [Optional] DoResize resizes the resource. Only supported by clusters - DoResize(ctx context.Context, id string, newState any) (remoteState any, e error) - // WaitAfterCreate waits for the resource to become ready after creation. WaitAfterCreate(ctx context.Context, newState any) (newRemoteState any, e error) @@ -244,7 +241,7 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.doResize, err = prepareCallFromTwoVariants(resource, "DoResize") + a.doResize, err = calladapt.PrepareCall(resource, calladapt.TypeOf[IResourceNoRefresh](), "DoResize") if err != nil { return err } @@ -311,9 +308,6 @@ func (a *Adapter) validate() error { if a.doResize != nil { validations = append(validations, "DoResize newState", a.doResize.InTypes[2], stateType) - if len(a.doUpdate.OutTypes) == 2 { - validations = append(validations, "DoResize remoteState return", a.doUpdate.OutTypes[0], remoteType) - } } if a.doUpdateWithID != nil { @@ -482,21 +476,13 @@ func (a *Adapter) DoUpdateWithID(ctx context.Context, oldID string, newState any } } -func (a *Adapter) DoResize(ctx context.Context, id string, newState any) (any, error) { +func (a *Adapter) DoResize(ctx context.Context, id string, newState any) error { if a.doResize == nil { - return nil, errors.New("internal error: DoResize not found") - } - - outs, err := a.doResize.Call(ctx, id, newState) - if err != nil { - return nil, err + return errors.New("internal error: DoResize not found") } - if len(outs) == 1 { - return outs[0], nil - } else { - return nil, nil - } + _, err := a.doResize.Call(ctx, id, newState) + return err } // ClassifyByTriggers classifies a single using FieldTriggers. @@ -553,23 +539,23 @@ func (a *Adapter) WaitAfterUpdate(ctx context.Context, newState any) (any, error } } -func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any) deployplan.ActionType { +func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any) (deployplan.ActionType, error) { // If ClassifyChange is not implemented, use FieldTriggers. if a.classifyChange == nil { - return a.ClassifyByTriggers(change) + return a.ClassifyByTriggers(change), nil } outs, err := a.classifyChange.Call(change, remoteState) if err != nil { - return deployplan.ActionTypeSkip + return deployplan.ActionTypeSkip, err } actionType := outs[0].(deployplan.ActionType) // If the action type is unset, use FieldTriggers. if actionType == deployplan.ActionTypeUnset { - return a.ClassifyByTriggers(change) + return a.ClassifyByTriggers(change), nil } - return actionType + return actionType, nil } // prepareCallRequired prepares a call and ensures the method is found. diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index eae52b87f1..795785c284 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -226,11 +226,12 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W path, err := structpath.Parse("name") require.NoError(t, err) - _ = adapter.ClassifyChange(structdiff.Change{ + _, err = adapter.ClassifyChange(structdiff.Change{ Path: path, Old: nil, New: "mynewname", }, remote) + require.NoError(t, err) } // validateFields uses structwalk to generate all valid field paths and checks membership. diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index b2bcdb1bdb..a3854ddb94 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "strings" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" @@ -100,18 +101,18 @@ func (r *ResourceCluster) DoDelete(ctx context.Context, id string) error { return r.client.Clusters.PermanentDeleteByClusterId(ctx, id) } -func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails) deployplan.ActionType { +func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails) (deployplan.ActionType, error) { // Always update if the cluster is not running. if remoteState.State != compute.StateRunning { - return deployplan.ActionTypeUpdate + return deployplan.ActionTypeUpdate, nil } changedPath := change.Path.String() - if changedPath == "num_workers" || changedPath == "autoscale" { - return deployplan.ActionTypeResize + if changedPath == "num_workers" || strings.HasPrefix(changedPath, "autoscale") { + return deployplan.ActionTypeResize, nil } - return deployplan.ActionTypeUpdate + return deployplan.ActionTypeUpdate, nil } func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { diff --git a/libs/testserver/clusters.go b/libs/testserver/clusters.go index f471045b27..e2caa1603e 100644 --- a/libs/testserver/clusters.go +++ b/libs/testserver/clusters.go @@ -46,6 +46,7 @@ func (s *FakeWorkspace) ClustersResize(req Request) any { } cluster.NumWorkers = request.NumWorkers + cluster.Autoscale = request.Autoscale s.Clusters[request.ClusterId] = cluster return Response{} From 676aebbe14f89f30a446fea29dd7a9a01c40d366 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 7 Oct 2025 13:01:10 +0200 Subject: [PATCH 09/10] remove permissions handler --- libs/testserver/handlers.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1f5acc004..4fa2cfc307 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -528,14 +528,6 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"]) }) - server.Handle("PUT", "/api/2.0/permissions/jobs/{job_id}", func(req Request) any { - return req.Workspace.JobsUpdatePermissions(req, req.Vars["job_id"]) - }) - - server.Handle("GET", "/api/2.0/permissions/jobs/{job_id}", func(req Request) any { - return req.Workspace.JobsGetPermissions(req, req.Vars["job_id"]) - }) - // Clusters: server.Handle("POST", "/api/2.1/clusters/resize", func(req Request) any { return req.Workspace.ClustersResize(req) From 24835883db8cdb50f6b65c18d350742fe16623df Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 7 Oct 2025 14:48:20 +0200 Subject: [PATCH 10/10] fixes --- .../databricks.yml.tmpl | 9 - .../out.plan_.direct-exp.json | 172 ++++++++++++++++++ .../out.plan_.direct-exp.txt | 15 ++ .../out.plan_.terraform.json | 35 ++++ .../out.plan_.terraform.txt | 15 ++ .../update-and-resize-autoscale/output.txt | 11 +- .../deploy/update-and-resize-autoscale/script | 11 ++ .../update-and-resize-autoscale/test.toml | 4 - .../update-and-resize/databricks.yml.tmpl | 9 - .../out.plan_.direct-exp.json | 135 ++++++++++++++ .../out.plan_.direct-exp.txt | 12 ++ .../out.plan_.terraform.json | 28 +++ .../update-and-resize/out.plan_.terraform.txt | 12 ++ .../deploy/update-and-resize/output.txt | 9 +- .../clusters/deploy/update-and-resize/script | 9 + .../deploy/update-and-resize/test.toml | 4 - cmd/bundle/plan.go | 4 +- 17 files changed, 455 insertions(+), 39 deletions(-) create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.json create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.txt create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.json create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.txt create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json create mode 100644 acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl index 929acfe1cd..f3d1cbc4a7 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/databricks.yml.tmpl @@ -11,12 +11,3 @@ resources: spark_version: $DEFAULT_SPARK_VERSION node_type_id: $NODE_TYPE_ID num_workers: 2 - - jobs: - foo: - name: test-job-with-cluster-$UNIQUE_NAME - tasks: - - task_key: my_spark_python_task - existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" - spark_python_task: - python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.json new file mode 100644 index 0000000000..79da574bca --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.json @@ -0,0 +1,172 @@ +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "create", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2, + "spark_version": "13.3.x-snapshot-scala2.12" + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update", + "new_state": { + "config": { + "autoscale": { + "max_workers": 4, + "min_workers": 2 + }, + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2, + "spark_version": "13.3.x-snapshot-scala2.12" + }, + "changes": { + "local": { + "autoscale": { + "action": "update" + }, + "num_workers": { + "action": "update" + } + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update", + "new_state": { + "config": { + "autoscale": { + "max_workers": 5, + "min_workers": 3 + }, + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autoscale": { + "max_workers": 4, + "min_workers": 2 + }, + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + }, + "changes": { + "local": { + "autoscale.max_workers": { + "action": "update" + }, + "autoscale.min_workers": { + "action": "update" + } + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "resize", + "new_state": { + "config": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autoscale": { + "max_workers": 5, + "min_workers": 3 + }, + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "local": { + "autoscale.max_workers": { + "action": "resize" + }, + "autoscale.min_workers": { + "action": "resize" + } + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "resize", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 3, + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autoscale": { + "max_workers": 6, + "min_workers": 4 + }, + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "local": { + "autoscale": { + "action": "resize" + }, + "num_workers": { + "action": "resize" + } + } + } + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.txt new file mode 100644 index 0000000000..0c010688c2 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct-exp.txt @@ -0,0 +1,15 @@ +create clusters.test_cluster + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +resize clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +resize clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json new file mode 100644 index 0000000000..ede4bf5217 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.json @@ -0,0 +1,35 @@ +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "create" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt new file mode 100644 index 0000000000..73b40dacde --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.terraform.txt @@ -0,0 +1,15 @@ +create clusters.test_cluster + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt index f2694cf0d4..2ae2661ba8 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/output.txt @@ -30,7 +30,7 @@ Deployment complete! "min_workers": 2 }, "autotermination_minutes": 60, - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", "node_type_id": "[NODE_TYPE_ID]", "spark_version": "13.3.x-snapshot-scala2.12" @@ -65,7 +65,7 @@ Deployment complete! "min_workers": 3 }, "autotermination_minutes": 60, - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", "node_type_id": "[NODE_TYPE_ID]", "spark_version": "13.3.x-snapshot-scala2.12" @@ -89,7 +89,7 @@ Deployment complete! "min_workers":3 }, "autotermination_minutes":60, - "cluster_id":"[UUID]", + "cluster_id":"[CLUSTER_ID]", "cluster_name":"test-cluster-[UNIQUE_NAME]", "node_type_id":"[NODE_TYPE_ID]", "spark_version":"13.3.x-snapshot-scala2.12", @@ -113,7 +113,7 @@ Deployment complete! "max_workers": 6, "min_workers": 4 }, - "cluster_id": "[UUID]" + "cluster_id": "[CLUSTER_ID]" } } @@ -140,7 +140,7 @@ Deployment complete! "method": "POST", "path": "/api/2.1/clusters/resize", "body": { - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "num_workers": 3 } } @@ -155,7 +155,6 @@ Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete cluster test_cluster - delete job foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script index c5404e3a64..01203fca55 100755 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/script @@ -6,16 +6,21 @@ cleanup() { } trap cleanup EXIT +$CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist with num_workers after bundle deployment:\n" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' title "Adding autoscale section should call update API on stopped cluster\n" update_file.py databricks.yml " num_workers: 2" " autoscale: min_workers: 2 max_workers: 4" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt rm out.requests.txt @@ -26,6 +31,8 @@ $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers,autoscale}' title "Changing autoscale should call update API on stopped cluster\n" update_file.py databricks.yml "min_workers: 2" "min_workers: 3" update_file.py databricks.yml "max_workers: 4" "max_workers: 5" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt rm out.requests.txt @@ -39,6 +46,8 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing autoscale should call resize API on running cluster\n" update_file.py databricks.yml "min_workers: 3" "min_workers: 4" update_file.py databricks.yml "max_workers: 5" "max_workers: 6" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -50,6 +59,8 @@ title "Removing autoscale section should call resize API\n" update_file.py databricks.yml " autoscale: min_workers: 4 max_workers: 6" " num_workers: 3" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml index 964a45f5c1..3aa4a19050 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/test.toml @@ -5,7 +5,3 @@ RecordRequests = true Ignore = [ "databricks.yml", ] - -[[Repls]] -Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" -New = "[CLUSTER-ID]" diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl index 74c340a249..c32af2055f 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/databricks.yml.tmpl @@ -13,12 +13,3 @@ resources: num_workers: 2 spark_conf: "spark.executor.memory": "2g" - - jobs: - foo: - name: test-job-with-cluster-$UNIQUE_NAME - tasks: - - task_key: my_spark_python_task - existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" - spark_python_task: - python_file: ./hello_world.py diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.json new file mode 100644 index 0000000000..612bbd6d0d --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.json @@ -0,0 +1,135 @@ +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "create", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 3, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 2, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + }, + "changes": { + "local": { + "num_workers": { + "action": "update" + } + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "resize", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 4, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 3, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "local": { + "num_workers": { + "action": "resize" + } + } + } + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update", + "new_state": { + "config": { + "autotermination_minutes": 60, + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 5, + "spark_conf": { + "spark.executor.memory": "4g" + }, + "spark_version": "13.3.x-snapshot-scala2.12" + } + }, + "remote_state": { + "autotermination_minutes": 60, + "cluster_id": "[CLUSTER_ID]", + "cluster_name": "test-cluster-[UNIQUE_NAME]", + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 4, + "spark_conf": { + "spark.executor.memory": "2g" + }, + "spark_version": "13.3.x-snapshot-scala2.12", + "state": "RUNNING" + }, + "changes": { + "local": { + "num_workers": { + "action": "resize" + }, + "spark_conf['spark.executor.memory']": { + "action": "update" + } + } + } + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.txt new file mode 100644 index 0000000000..70123004a2 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct-exp.txt @@ -0,0 +1,12 @@ +create clusters.test_cluster + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +resize clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json new file mode 100644 index 0000000000..0b0dc4dd9a --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.json @@ -0,0 +1,28 @@ +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "create" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} +{ + "plan": { + "resources.clusters.test_cluster": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt new file mode 100644 index 0000000000..d51cd2abf0 --- /dev/null +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.terraform.txt @@ -0,0 +1,12 @@ +create clusters.test_cluster + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +update clusters.test_cluster + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt index 1a6f44a841..38a9f6434a 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/output.txt @@ -25,7 +25,7 @@ Deployment complete! "path": "/api/2.1/clusters/edit", "body": { "autotermination_minutes": 60, - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 3, @@ -45,7 +45,7 @@ Deployment complete! === Starting the cluster { "autotermination_minutes":60, - "cluster_id":"[UUID]", + "cluster_id":"[CLUSTER_ID]", "cluster_name":"test-cluster-[UNIQUE_NAME]", "node_type_id":"[NODE_TYPE_ID]", "num_workers":3, @@ -69,7 +69,7 @@ Deployment complete! "method": "POST", "path": "/api/2.1/clusters/resize", "body": { - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "num_workers": 4 } } @@ -94,7 +94,7 @@ Deployment complete! "path": "/api/2.1/clusters/edit", "body": { "autotermination_minutes": 60, - "cluster_id": "[UUID]", + "cluster_id": "[CLUSTER_ID]", "cluster_name": "test-cluster-[UNIQUE_NAME]", "node_type_id": "[NODE_TYPE_ID]", "num_workers": 5, @@ -117,7 +117,6 @@ Deployment complete! >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete cluster test_cluster - delete job foo All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script index 1cd8e640fe..b1795f6992 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/script +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/script @@ -6,14 +6,19 @@ cleanup() { } trap cleanup EXIT +$CLI bundle plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan > out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy title "Cluster should exist after bundle deployment:\n" CLUSTER_ID=$($CLI bundle summary -o json | jq -r '.resources.clusters.test_cluster.id') +echo "$CLUSTER_ID:CLUSTER_ID" >> ACC_REPLS $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers should call update API on stopped cluster\n" update_file.py databricks.yml "num_workers: 2" "num_workers: 3" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt rm out.requests.txt @@ -26,6 +31,8 @@ $CLI clusters start "${CLUSTER_ID}" title "Changing num_workers should call resize API on running cluster\n" update_file.py databricks.yml "num_workers: 3" "num_workers: 4" +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/resize")))' out.requests.txt rm out.requests.txt @@ -36,6 +43,8 @@ $CLI clusters get "${CLUSTER_ID}" | jq '{cluster_name,num_workers}' title "Changing num_workers and spark_conf should call update API\n" update_file.py databricks.yml "num_workers: 4" "num_workers: 5" update_file.py databricks.yml '"spark.executor.memory": "2g"' '"spark.executor.memory": "4g"' +$CLI bundle plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.txt +$CLI bundle debug plan >> out.plan_.$DATABRICKS_BUNDLE_ENGINE.json trace $CLI bundle deploy trace jq 'select(.method == "POST" and (.path | contains("/clusters/edit")))' out.requests.txt rm out.requests.txt diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml b/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml index 964a45f5c1..3aa4a19050 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/test.toml @@ -5,7 +5,3 @@ RecordRequests = true Ignore = [ "databricks.yml", ] - -[[Repls]] -Old = "[0-9]{4}-[0-9]{6}-[0-9a-z]{8}" -New = "[CLUSTER-ID]" diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 5309909583..e3c949e885 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -77,11 +77,11 @@ It is useful for previewing changes before running 'bundle deploy'.`, switch change.ActionType { case deployplan.ActionTypeCreate: createCount++ - case deployplan.ActionTypeUpdate, deployplan.ActionTypeUpdateWithID: + case deployplan.ActionTypeUpdate, deployplan.ActionTypeUpdateWithID, deployplan.ActionTypeResize: updateCount++ case deployplan.ActionTypeDelete: deleteCount++ - case deployplan.ActionTypeRecreate, deployplan.ActionTypeResize: + case deployplan.ActionTypeRecreate: // A recreate counts as both a delete and a create deleteCount++ createCount++