diff --git a/acceptance/bundle/resources/jobs/big_id/databricks.yml b/acceptance/bundle/resources/jobs/big_id/databricks.yml new file mode 100644 index 0000000000..69ce43f0d3 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/databricks.yml @@ -0,0 +1,20 @@ +# Note, I'm not testing jobs here. I'm testing how encoder/decoders work with int64, job_id is just a field that came up. +resources: + jobs: + foo: + tasks: + - task_key: max(int64) + run_job_task: + job_id: 9223372036854775807 + - task_key: max(int64) - 1 + run_job_task: + job_id: 9223372036854775806 + - task_key: max(int64) - 2 + run_job_task: + job_id: 9223372036854775805 + - task_key: min(int64) + run_job_task: + job_id: -9223372036854775808 + - task_key: min(int64) - 4 + run_job_task: + job_id: -9223372036854775804 diff --git a/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json new file mode 100644 index 0000000000..fcbaed0c22 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/out.plan.direct.json @@ -0,0 +1,56 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.foo": { + "action": "create", + "new_state": { + "value": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max(int64)" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max(int64) - 1" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_2] + }, + "task_key": "max(int64) - 2" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min(int64)" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64_MINUS_4] + }, + "task_key": "min(int64) - 4" + } + ] + } + } + } + } +} diff --git a/acceptance/bundle/resources/jobs/big_id/out.state.direct.json b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json new file mode 100644 index 0000000000..4a0e000125 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/out.state.direct.json @@ -0,0 +1,56 @@ +{ + "state_version": 1, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 2, + "state": { + "resources.jobs.foo": { + "__id__": "[NUMID]", + "state": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max(int64)" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max(int64) - 1" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_2] + }, + "task_key": "max(int64) - 2" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min(int64)" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64_MINUS_4] + }, + "task_key": "min(int64) - 4" + } + ] + } + } + } +} diff --git a/acceptance/bundle/resources/jobs/big_id/out.test.toml b/acceptance/bundle/resources/jobs/big_id/out.test.toml new file mode 100644 index 0000000000..54146af564 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/jobs/big_id/out.validate.json b/acceptance/bundle/resources/jobs/big_id/out.validate.json new file mode 100644 index 0000000000..604a3647ab --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/out.validate.json @@ -0,0 +1,49 @@ +{ + "jobs": { + "foo": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max(int64)" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max(int64) - 1" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_2] + }, + "task_key": "max(int64) - 2" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min(int64)" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64_MINUS_4] + }, + "task_key": "min(int64) - 4" + } + ] + } + } +} diff --git a/acceptance/bundle/resources/jobs/big_id/output.txt b/acceptance/bundle/resources/jobs/big_id/output.txt new file mode 100644 index 0000000000..82ec469ca8 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/output.txt @@ -0,0 +1,80 @@ + +>>> [CLI] bundle validate -o json + +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/create", + "body": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", + "queue": { + "enabled": true + }, + "tasks": [ + { + "run_job_task": { + "job_id": [MAX_INT_64] + }, + "task_key": "max(int64)" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_1] + }, + "task_key": "max(int64) - 1" + }, + { + "run_job_task": { + "job_id": [MAX_INT_64_MINUS_2] + }, + "task_key": "max(int64) - 2" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64] + }, + "task_key": "min(int64)" + }, + { + "run_job_task": { + "job_id": [MIN_INT_64_MINUS_4] + }, + "task_key": "min(int64) - 4" + } + ] + } +} + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.jobs.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! + +>>> print_requests.py //jobs +{ + "method": "POST", + "path": "/api/2.2/jobs/delete", + "body": { + "job_id": [NUMID] + } +} diff --git a/acceptance/bundle/resources/jobs/big_id/script b/acceptance/bundle/resources/jobs/big_id/script new file mode 100644 index 0000000000..ff880e6652 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/script @@ -0,0 +1,8 @@ +trace $CLI bundle validate -o json | jq .resources > out.validate.json +trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json +$CLI bundle deploy +trace print_requests.py //jobs +print_state.py > out.state.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan | contains.py '0 to add, 0 to change, 0 to delete' +trace $CLI bundle destroy --auto-approve +trace print_requests.py //jobs diff --git a/acceptance/bundle/resources/jobs/big_id/test.toml b/acceptance/bundle/resources/jobs/big_id/test.toml new file mode 100644 index 0000000000..b0147e8645 --- /dev/null +++ b/acceptance/bundle/resources/jobs/big_id/test.toml @@ -0,0 +1,23 @@ +# terraform fails with: +# panic: Error reading level state: strconv.ParseInt: parsing "[NUMID]": value out of range +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ['direct'] + +[[Repls]] +Old = '9223372036854775807' +New = '[MAX_INT_64]' + +[[Repls]] +Old = '9223372036854775806' +New = '[MAX_INT_64_MINUS_1]' + +[[Repls]] +Old = '9223372036854775805' +New = '[MAX_INT_64_MINUS_2]' + +[[Repls]] +Old = '-9223372036854775808' +New = '[MIN_INT_64]' + +[[Repls]] +Old = '-9223372036854775804' +New = '[MIN_INT_64_MINUS_4]' diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 99cbd66c1b..f8eb5c9fc2 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -1,7 +1,6 @@ package direct import ( - "bytes" "context" "encoding/json" "errors" @@ -210,15 +209,9 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return nil } -func typeConvert(destType reflect.Type, src any) (any, error) { - raw, err := json.Marshal(src) - if err != nil { - return nil, fmt.Errorf("marshalling: %w", err) - } - +func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { destPtr := reflect.New(destType).Interface() - dec := json.NewDecoder(bytes.NewReader(raw)) - err = dec.Decode(destPtr) + err := json.Unmarshal(raw, destPtr) if err != nil { return nil, fmt.Errorf("unmarshalling into %s: %w", destType, err) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index d8031540d5..3daca7de2a 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -133,7 +133,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - savedState, err := typeConvert(adapter.StateType(), dbentry.State) + savedState, err := parseState(adapter.StateType(), dbentry.State) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: interpreting state: %w", errorPrefix, err)) return false diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 454e3b4008..ce8b6adbec 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -5,9 +5,9 @@ import ( "encoding/json" "fmt" "os" + "strings" "sync" - "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/statemgmt/resourcestate" "github.com/databricks/cli/internal/build" "github.com/google/uuid" @@ -30,8 +30,8 @@ type Database struct { } type ResourceEntry struct { - ID string `json:"__id__"` - State any `json:"state"` + ID string `json:"__id__"` + State json.RawMessage `json:"state"` } func NewDatabase() Database { @@ -57,9 +57,14 @@ func (db *DeploymentState) SaveState(key, newID string, state any) error { db.Data.State = make(map[string]ResourceEntry) } + jsonMessage, err := json.MarshalIndent(state, " ", " ") + if err != nil { + return err + } + db.Data.State[key] = ResourceEntry{ ID: newID, - State: state, + State: json.RawMessage(jsonMessage), } return nil @@ -140,21 +145,16 @@ func (db *DeploymentState) AssertOpened() { func (db *DeploymentState) ExportState(ctx context.Context) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range db.Data.State { - // Extract etag for dashboards. var etag string - switch dashboard := entry.State.(type) { - // Dashboard state has type map[string]any during bundle deployment. + // Extract etag for dashboards. // covered by test case: bundle/deploy/dashboard/detect-change - case map[string]any: - v, ok := dashboard["etag"].(string) - if ok { - etag = v + if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + var holder struct { + Etag string `json:"etag"` + } + if err := json.Unmarshal(entry.State, &holder); err == nil { + etag = holder.Etag } - - // Dashboard state has type *resources.DashboardConfig during bundle generation. - // covered by test case: bundle/deploy/dashboard/generate_inplace - case *resources.DashboardConfig: - etag = dashboard.Etag } result[key] = resourcestate.ResourceState{ diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 706a680ea0..07fceab214 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -2,6 +2,7 @@ package deployment import ( "bytes" + "encoding/json" "errors" "fmt" "os" @@ -177,7 +178,10 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va state := make(map[string]dstate.ResourceEntry) for key, resourceEntry := range terraformResources { - state[key] = dstate.ResourceEntry{ID: resourceEntry.ID} + state[key] = dstate.ResourceEntry{ + ID: resourceEntry.ID, + State: json.RawMessage("{}"), + } if resourceEntry.ETag != "" { // dashboard: etags[key] = resourceEntry.ETag