Relax default num_workers=0 for job_clusters (#4080)

denik · web-flow · commit cfbf595de6a9 · 2025-12-04T11:04:00.000Z
## Why Terraform only adds it if neither autoscale nor num_workers are not set. It filters it out if autoscale is set. ## Tests New acceptance test that has different configurations and asserts that requests are the same. Difference also disappears in existing tests: in default-python recorded requests now have no difference wrt num_workers. In migrate/default-python num_workers no longer triggers drift post migration #4068
diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
@@ -130,7 +129,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
@@ -231,10 +230,6 @@
             "action": "skip",
             "reason": "server_side_default"
           },
-          "job_clusters[0].new_cluster.num_workers": {
-            "action": "update",
-            "old": 0
-          },
           "tasks[task_key='notebook_task'].libraries[0].whl": {
             "action": "update",
             "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
diff --git a/acceptance/bundle/migrate/default-python/out.state_after_migration.json b/acceptance/bundle/migrate/default-python/out.state_after_migration.json
@@ -21,7 +21,6 @@
        },
        "data_security_mode": "SINGLE_USER",
        "node_type_id": "[NODE_TYPE_ID]",
-       "num_workers": 0,
        "spark_version": "16.4.x-scala2.12"
       }
      }
diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt
@@ -82,10 +82,6 @@ Building python_artifact...
       "action": "skip",
       "reason": "server_side_default"
     },
-    "job_clusters[0].new_cluster.num_workers": {
-      "action": "update",
-      "old": 0
-    },
     "tasks[task_key='notebook_task'].libraries[0].whl": {
       "action": "update",
       "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
@@ -160,10 +156,6 @@ Building python_artifact...
       "action": "skip",
       "reason": "server_side_default"
     },
-    "job_clusters[0].new_cluster.num_workers": {
-      "action": "update",
-      "old": 0
-    },
     "tasks[task_key='notebook_task'].libraries[0].whl": {
       "action": "update",
       "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/job_update.json b/acceptance/bundle/resource_deps/jobs_update_remote/job_update.json
@@ -11,7 +11,6 @@
       {
         "job_cluster_key": "key",
         "new_cluster": {
-          "num_workers": 0,
           "spark_version": "13.3.x-scala2.12"
         }
       }
diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json
@@ -98,7 +98,6 @@
             {
               "job_cluster_key": "key",
               "new_cluster": {
-                "num_workers": 0,
                 "spark_version": "13.3.x-scala2.12"
               }
             }
@@ -125,6 +124,10 @@
             "action": "skip",
             "reason": "server_side_default"
           },
+          "job_clusters[0].new_cluster.num_workers": {
+            "action": "update",
+            "old": 0
+          },
           "timeout_seconds": {
             "action": "skip",
             "reason": "server_side_default"
diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/output.txt b/acceptance/bundle/resource_deps/jobs_update_remote/output.txt
@@ -107,7 +107,6 @@ Destroy complete!
         {
           "job_cluster_key": "key",
           "new_cluster": {
-            "num_workers": 0,
             "spark_version": "13.3.x-scala2.12"
           }
         }
diff --git a/acceptance/bundle/resources/jobs/create-error/output.txt b/acceptance/bundle/resources/jobs/create-error/output.txt
@@ -1,9 +1,5 @@
 
 >>> musterr [CLI] bundle deploy --force-lock
-Warning: required field "new_cluster" is not set
-  at resources.jobs.foo.job_clusters[0]
-  in databricks.yml:7:11
-
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Error: cannot create resources.jobs.foo: Shared job cluster feature is only supported in multi-task jobs. (400 INVALID_PARAMETER_VALUE)
diff --git a/acceptance/bundle/resources/jobs/num_workers/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/num_workers/databricks.yml.tmpl
@@ -0,0 +1,71 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    sample_job:
+      name: sample_job
+
+      trigger:
+        # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger
+        periodic:
+          interval: 1
+          unit: DAYS
+
+      tasks:
+        - task_key: notebook_task
+          notebook_task:
+            notebook_path: sample_notebook.py
+            source: WORKSPACE  # Without this, there is a different request between direct and terraform
+
+      job_clusters:
+        - job_cluster_key: job_cluster_autoscale
+          new_cluster:
+            spark_version: 16.4.x-scala2.12
+            node_type_id:  $NODE_TYPE_ID
+            data_security_mode: SINGLE_USER
+            autoscale:
+              min_workers: 1
+              max_workers: 4
+
+        # This config results in different request between terraform and direct:
+        # Terraform removes "num_workers: 0" and direct sends it as is.
+        # This is acceptable difference, users will get appropriate error message backend and can correct their config.
+        #
+        #- job_cluster_key: job_cluster_autoscale_num_workers0
+        #  new_cluster:
+        #    spark_version: 16.4.x-scala2.13
+        #    node_type_id:  $NODE_TYPE_ID
+        #    data_security_mode: SINGLE_USER
+        #    autoscale:
+        #      min_workers: 1
+        #      max_workers: 4
+        #    num_workers: 0
+
+        - job_cluster_key: job_cluster_autoscale_num_workers1
+          new_cluster:
+            spark_version: 16.4.x-scala2.14
+            node_type_id:  $NODE_TYPE_ID
+            data_security_mode: SINGLE_USER
+            autoscale:
+              min_workers: 1
+              max_workers: 4
+            num_workers: 1
+
+        - job_cluster_key: job_cluster_num_workers1
+          new_cluster:
+            spark_version: 16.4.x-scala2.15
+            node_type_id:  $NODE_TYPE_ID
+            data_security_mode: SINGLE_USER
+            num_workers: 1
+
+        - job_cluster_key: job_cluster_num_workers0
+          new_cluster:
+            spark_version: 16.4.x-scala2.16
+            node_type_id:  $NODE_TYPE_ID
+            data_security_mode: SINGLE_USER
+            num_workers: 0
+
+        - job_cluster_key: job_cluster_default
+          new_cluster:
+            spark_version: 16.4.x-scala2.17
diff --git a/acceptance/bundle/resources/jobs/num_workers/out.test.toml b/acceptance/bundle/resources/jobs/num_workers/out.test.toml
diff --git a/acceptance/bundle/resources/jobs/num_workers/output.txt b/acceptance/bundle/resources/jobs/num_workers/output.txt
@@ -0,0 +1,128 @@
+
+>>> [CLI] bundle deploy
+Warning: Single node cluster is not correctly configured
+  at resources.jobs.sample_job.job_clusters[3].new_cluster
+  in databricks.yml:64:13
+
+num_workers should be 0 only for single-node clusters. To create a
+valid single node cluster please ensure that the following properties
+are correctly set in the cluster specification:
+
+  spark_conf:
+    spark.databricks.cluster.profile: singleNode
+    spark.master: local[*]
+
+  custom_tags:
+    ResourceClass: SingleNode
+  
+
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests.py //jobs
+{
+  "method": "POST",
+  "path": "/api/2.2/jobs/create",
+  "body": {
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json"
+    },
+    "edit_mode": "UI_LOCKED",
+    "format": "MULTI_TASK",
+    "job_clusters": [
+      {
+        "job_cluster_key": "job_cluster_autoscale",
+        "new_cluster": {
+          "autoscale": {
+            "max_workers": 4,
+            "min_workers": 1
+          },
+          "data_security_mode": "SINGLE_USER",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "spark_version": "16.4.x-scala2.12"
+        }
+      },
+      {
+        "job_cluster_key": "job_cluster_autoscale_num_workers1",
+        "new_cluster": {
+          "autoscale": {
+            "max_workers": 4,
+            "min_workers": 1
+          },
+          "data_security_mode": "SINGLE_USER",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "num_workers": 1,
+          "spark_version": "16.4.x-scala2.14"
+        }
+      },
+      {
+        "job_cluster_key": "job_cluster_num_workers1",
+        "new_cluster": {
+          "data_security_mode": "SINGLE_USER",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "num_workers": 1,
+          "spark_version": "16.4.x-scala2.15"
+        }
+      },
+      {
+        "job_cluster_key": "job_cluster_num_workers0",
+        "new_cluster": {
+          "data_security_mode": "SINGLE_USER",
+          "node_type_id": "[NODE_TYPE_ID]",
+          "num_workers": 0,
+          "spark_version": "16.4.x-scala2.16"
+        }
+      },
+      {
+        "job_cluster_key": "job_cluster_default",
+        "new_cluster": {
+          "num_workers": 0,
+          "spark_version": "16.4.x-scala2.17"
+        }
+      }
+    ],
+    "max_concurrent_runs": 1,
+    "name": "sample_job",
+    "queue": {
+      "enabled": true
+    },
+    "tasks": [
+      {
+        "notebook_task": {
+          "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files/sample_notebook",
+          "source": "WORKSPACE"
+        },
+        "task_key": "notebook_task"
+      }
+    ],
+    "trigger": {
+      "pause_status": "UNPAUSED",
+      "periodic": {
+        "interval": 1,
+        "unit": "DAYS"
+      }
+    }
+  }
+}
+
+>>> [CLI] bundle plan
+Warning: Single node cluster is not correctly configured
+  at resources.jobs.sample_job.job_clusters[3].new_cluster
+  in databricks.yml:64:13
+
+num_workers should be 0 only for single-node clusters. To create a
+valid single node cluster please ensure that the following properties
+are correctly set in the cluster specification:
+
+  spark_conf:
+    spark.databricks.cluster.profile: singleNode
+    spark.master: local[*]
+
+  custom_tags:
+    ResourceClass: SingleNode
+  
+
+Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged
diff --git a/acceptance/bundle/resources/jobs/num_workers/sample_notebook.py b/acceptance/bundle/resources/jobs/num_workers/sample_notebook.py
@@ -0,0 +1 @@
+# Databricks notebook source
diff --git a/acceptance/bundle/resources/jobs/num_workers/script b/acceptance/bundle/resources/jobs/num_workers/script
@@ -0,0 +1,6 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+trace $CLI bundle deploy
+trace print_requests.py //jobs
+
+trace $CLI bundle plan
+rm out.requests.txt
diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
@@ -130,7 +129,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json
@@ -31,7 +31,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_dev.direct.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_prod.direct.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct.txt b/acceptance/bundle/templates/default-python/classic/out.requests.dev.direct.txt
@@ -106,7 +106,6 @@
           },
           "data_security_mode": "SINGLE_USER",
           "node_type_id": "[NODE_TYPE_ID]",
-          "num_workers": 0,
           "spark_version": "16.4.x-scala2.12"
         }
       }
diff --git a/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct.txt b/acceptance/bundle/templates/default-python/classic/out.requests.prod.direct.txt
@@ -109,7 +109,6 @@
           },
           "data_security_mode": "SINGLE_USER",
           "node_type_id": "[NODE_TYPE_ID]",
-          "num_workers": 0,
           "spark_version": "16.4.x-scala2.12"
         }
       }
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_dev.direct.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json b/acceptance/bundle/templates/default-python/integration_classic/out.plan_prod.direct.json
@@ -26,7 +26,6 @@
                 },
                 "data_security_mode": "SINGLE_USER",
                 "node_type_id": "[NODE_TYPE_ID]",
-                "num_workers": 0,
                 "spark_version": "16.4.x-scala2.12"
               }
             }
diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json
diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt
diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt
diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go
diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go

Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,6 @@`
`26`	`26`	`},`
`27`	`27`	`"data_security_mode": "SINGLE_USER",`
`28`	`28`	`"node_type_id": "[NODE_TYPE_ID]",`
`29`		`- "num_workers": 0,`
`30`	`29`	`"spark_version": "16.4.x-scala2.12"`
`31`	`30`	`}`
`32`	`31`	`}`
`@@ -130,7 +129,6 @@`
`130`	`129`	`},`
`131`	`130`	`"data_security_mode": "SINGLE_USER",`
`132`	`131`	`"node_type_id": "[NODE_TYPE_ID]",`
`133`		`- "num_workers": 0,`
`134`	`132`	`"spark_version": "16.4.x-scala2.12"`
`135`	`133`	`}`
`136`	`134`	`}`
Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,6 @@`
`21`	`21`	`},`
`22`	`22`	`"data_security_mode": "SINGLE_USER",`
`23`	`23`	`"node_type_id": "[NODE_TYPE_ID]",`
`24`		`- "num_workers": 0,`
`25`	`24`	`"spark_version": "16.4.x-scala2.12"`
`26`	`25`	`}`
`27`	`26`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,6 @@`
`11`	`11`	`{`
`12`	`12`	`"job_cluster_key": "key",`
`13`	`13`	`"new_cluster": {`
`14`		`- "num_workers": 0,`
`15`	`14`	`"spark_version": "13.3.x-scala2.12"`
`16`	`15`	`}`
`17`	`16`	`}`
Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,6 @@`
`98`	`98`	`{`
`99`	`99`	`"job_cluster_key": "key",`
`100`	`100`	`"new_cluster": {`
`101`		`- "num_workers": 0,`
`102`	`101`	`"spark_version": "13.3.x-scala2.12"`
`103`	`102`	`}`
`104`	`103`	`}`
`@@ -125,6 +124,10 @@`
`125`	`124`	`"action": "skip",`
`126`	`125`	`"reason": "server_side_default"`
`127`	`126`	`},`
	`127`	`+ "job_clusters[0].new_cluster.num_workers": {`
	`128`	`+ "action": "update",`
	`129`	`+ "old": 0`
	`130`	`+ },`
`128`	`131`	`"timeout_seconds": {`
`129`	`132`	`"action": "skip",`
`130`	`133`	`"reason": "server_side_default"`
Original file line number	Diff line number	Diff line change
`@@ -107,7 +107,6 @@ Destroy complete!`
`107`	`107`	`{`
`108`	`108`	`"job_cluster_key": "key",`
`109`	`109`	`"new_cluster": {`
`110`		`- "num_workers": 0,`
`111`	`110`	`"spark_version": "13.3.x-scala2.12"`
`112`	`111`	`}`
`113`	`112`	`}`