Add support for lakebase catalog (#3436)

anton-107 · web-flow · commit 6430e79160a1 · 2025-08-20T11:58:39.000Z
## Changes
&lt;!-- Brief summary of your changes that is easy to understand --&gt;

Added support for creating Lakebase database catalogs in DABs

## Why
&lt;!-- Why are these changes needed? Provide the context that the reviewer
might be missing.
For example, were there any decisions behind the change that are not
reflected in the code itself? --&gt;

This change allows users to define database catalogs as part of their
assets bundle:

```
...
resources:
  ...
  database_instances:
    my_database_instance:
      name: my_database_instance
      capacity: CU_1
  database_catalogs:
    my_catalog:
      database_instance_name: ${resources.database_instances.my_instance.name}
      database_name: "my_database"
      name: my_catalog_$UNIQUE_NAME
      create_database_if_not_exists: true
```

## Tests
&lt;!-- How have you tested the changes? --&gt;
Added acceptance tests for database instances deployments

&lt;!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. --&gt;
diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md
@@ -12,6 +12,7 @@
 
 ### Bundles
 * Add support for Lakebase database instances in DABs ([#3283](https://github.com/databricks/cli/pull/3283))
+* Add support for Lakebase database catalogs in DABs ([#3436](https://github.com/databricks/cli/pull/3436))
 * Improve error message for SDK/API errors ([#3379](https://github.com/databricks/cli/pull/3379))
 
 ### API Changes
diff --git a/acceptance/bundle/deploy/lakebase/database-catalog/databricks.yml.tmpl b/acceptance/bundle/deploy/lakebase/database-catalog/databricks.yml.tmpl
@@ -0,0 +1,14 @@
+bundle:
+  name: deploy-lakebase-catalog-$UNIQUE_NAME
+
+resources:
+  database_instances:
+    my_instance:
+      name: test-database-instance-$UNIQUE_NAME
+      capacity: CU_1
+  database_catalogs:
+    my_catalog:
+      database_instance_name: ${resources.database_instances.my_instance.name}
+      database_name: "my_database"
+      name: my_catalog_$UNIQUE_NAME
+      create_database_if_not_exists: true
diff --git a/acceptance/bundle/deploy/lakebase/database-catalog/out.test.toml b/acceptance/bundle/deploy/lakebase/database-catalog/out.test.toml
@@ -0,0 +1,9 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true
+
+[CloudEnvs]
+  gcp = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
diff --git a/acceptance/bundle/deploy/lakebase/database-catalog/output.txt b/acceptance/bundle/deploy/lakebase/database-catalog/output.txt
@@ -0,0 +1,68 @@
+
+>>> [CLI] bundle validate
+Name: deploy-lakebase-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-catalog-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle summary
+Name: deploy-lakebase-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-catalog-[UNIQUE_NAME]/default
+Resources:
+  Database catalogs:
+    my_catalog:
+      Name: my_catalog_[UNIQUE_NAME]
+  Database instances:
+    my_instance:
+      Name: test-database-instance-[UNIQUE_NAME]
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-catalog-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] database get-database-instance test-database-instance-[UNIQUE_NAME]
+{
+  "capacity": "CU_1",
+  "creation_time": "[TIMESTAMP]Z",
+  "creator": "[USERNAME]",
+  "effective_enable_readable_secondaries": false,
+  "effective_node_count": 1,
+  "effective_retention_window_in_days": 7,
+  "effective_stopped": false,
+  "name": "test-database-instance-[UNIQUE_NAME]",
+  "pg_version": "PG_VERSION_16",
+  "state": "AVAILABLE",
+  "uid": "[UUID]"
+}
+
+>>> [CLI] bundle summary
+Name: deploy-lakebase-catalog-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-catalog-[UNIQUE_NAME]/default
+Resources:
+  Database catalogs:
+    my_catalog:
+      Name: my_catalog_[UNIQUE_NAME]
+  Database instances:
+    my_instance:
+      Name: test-database-instance-[UNIQUE_NAME]
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete database_catalog my_catalog
+  delete database_instance my_instance
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-catalog-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!
diff --git a/acceptance/bundle/deploy/lakebase/database-catalog/script b/acceptance/bundle/deploy/lakebase/database-catalog/script
@@ -0,0 +1,15 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+trace $CLI bundle validate
+
+# URL is excluded because some cloud envs add an ?0= query param and some don't
+trace $CLI bundle summary | grep -v "URL"
+trace $CLI bundle deploy
+# _dns fields are excluded since they differ between cloud envs
+trace $CLI database get-database-instance "test-database-instance-${UNIQUE_NAME}" | jq 'del(.read_only_dns, .read_write_dns)'
+trace $CLI bundle summary | grep -v "URL"
diff --git a/acceptance/bundle/deploy/lakebase/database-catalog/test.toml b/acceptance/bundle/deploy/lakebase/database-catalog/test.toml
@@ -0,0 +1,7 @@
+Local = true
+Cloud = true
+
+RecordRequests = false
+
+[EnvMatrix]
+DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
diff --git a/acceptance/bundle/deploy/lakebase/database-instance/single-instance/test.toml b/acceptance/bundle/deploy/lakebase/database-instance/single-instance/test.toml
@@ -1,10 +1,6 @@
 Local = true
 Cloud = true
 
-RecordRequests = false
-RequiresUnityCatalog = true
-CloudEnvs.gcp = false # lakebase is not available in GCP as of August 2025
-
 Ignore = [
     "databricks.yml",
 ]
diff --git a/acceptance/bundle/deploy/lakebase/test.toml b/acceptance/bundle/deploy/lakebase/test.toml
@@ -0,0 +1,2 @@
+RequiresUnityCatalog = true
+CloudEnvs.gcp = false # lakebase is not available in GCP as of August 2025
diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go
@@ -514,4 +514,16 @@ func addDefaultHandlers(server *testserver.Server) {
 	server.Handle("DELETE", "/api/2.0/database/instances/{name}", func(req testserver.Request) any {
 		return testserver.DatabaseInstanceMapDelete(req)
 	})
+
+	server.Handle("POST", "/api/2.0/database/catalogs", func(req testserver.Request) any {
+		return req.Workspace.DatabaseCatalogCreate(req)
+	})
+
+	server.Handle("GET", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any {
+		return testserver.MapGet(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"])
+	})
+
+	server.Handle("DELETE", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any {
+		return testserver.MapDelete(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"])
+	})
 }
diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go
@@ -15,7 +15,7 @@ import (
 	"github.com/databricks/cli/libs/dyn/convert"
 )
 
-var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models"}
+var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs"}
 
 var (
 	allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN}
diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go
@@ -176,6 +176,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 						},
 					},
 				},
+				DatabaseCatalogs: map[string]*resources.DatabaseCatalog{
+					"database_catalog1": {
+						DatabaseCatalog: database.DatabaseCatalog{
+							Name: "database_catalog1",
+						},
+					},
+				},
 			},
 		},
 		SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
@@ -344,7 +351,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 				resourceType := resources.Type().Field(i).Name
 
 				// Skip resources that are not renamed
-				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" {
+				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" {
 					continue
 				}
 
diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go
@@ -35,6 +35,7 @@ func allResourceTypes(t *testing.T) []string {
 		"apps",
 		"clusters",
 		"dashboards",
+		"database_catalogs",
 		"database_instances",
 		"experiments",
 		"jobs",
@@ -140,6 +141,7 @@ func TestRunAsWorksForAllowedResources(t *testing.T) {
 // they are not on the allow list below.
 var allowList = []string{
 	"clusters",
+	"database_catalogs",
 	"database_instances",
 	"jobs",
 	"models",
diff --git a/bundle/config/resources.go b/bundle/config/resources.go
@@ -27,6 +27,7 @@ type Resources struct {
 	SecretScopes          map[string]*resources.SecretScope          `json:"secret_scopes,omitempty"`
 	SqlWarehouses         map[string]*resources.SqlWarehouse         `json:"sql_warehouses,omitempty"`
 	DatabaseInstances     map[string]*resources.DatabaseInstance     `json:"database_instances,omitempty"`
+	DatabaseCatalogs      map[string]*resources.DatabaseCatalog      `json:"database_catalogs,omitempty"`
 }
 
 type ConfigResource interface {
@@ -92,6 +93,7 @@ func (r *Resources) AllResources() []ResourceGroup {
 		collectResourceMap(descriptions["secret_scopes"], r.SecretScopes),
 		collectResourceMap(descriptions["sql_warehouses"], r.SqlWarehouses),
 		collectResourceMap(descriptions["database_instances"], r.DatabaseInstances),
+		collectResourceMap(descriptions["database_catalogs"], r.DatabaseCatalogs),
 	}
 }
 
@@ -181,6 +183,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 		}
 	}
 
+	for k := range r.DatabaseCatalogs {
+		if k == key {
+			found = append(found, r.DatabaseCatalogs[k])
+		}
+	}
+
 	if len(found) == 0 {
 		return nil, fmt.Errorf("no such resource: %s", key)
 	}
@@ -214,5 +222,6 @@ func SupportedResources() map[string]resources.ResourceDescription {
 		"secret_scopes":           (&resources.SecretScope{}).ResourceDescription(),
 		"sql_warehouses":          (&resources.SqlWarehouse{}).ResourceDescription(),
 		"database_instances":      (&resources.DatabaseInstance{}).ResourceDescription(),
+		"database_catalogs":       (&resources.DatabaseCatalog{}).ResourceDescription(),
 	}
 }
diff --git a/bundle/config/resources/database_catalog.go b/bundle/config/resources/database_catalog.go
@@ -0,0 +1,53 @@
+package resources
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/databricks/cli/libs/log"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/database"
+)
+
+type DatabaseCatalog struct {
+	ID             string         `json:"id,omitempty" bundle:"readonly"`
+	URL            string         `json:"url,omitempty" bundle:"internal"`
+	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
+
+	database.DatabaseCatalog
+}
+
+func (d *DatabaseCatalog) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) {
+	_, err := w.Database.GetDatabaseCatalog(ctx, database.GetDatabaseCatalogRequest{Name: name})
+	if err != nil {
+		log.Debugf(ctx, "database Catalog %s does not exist", name)
+		return false, err
+	}
+	return true, nil
+}
+
+func (d *DatabaseCatalog) ResourceDescription() ResourceDescription {
+	return ResourceDescription{
+		SingularName:  "database_catalog",
+		PluralName:    "database_catalogs",
+		SingularTitle: "Database catalog",
+		PluralTitle:   "Database catalogs",
+	}
+}
+
+func (d *DatabaseCatalog) GetName() string {
+	return d.Name
+}
+
+func (d *DatabaseCatalog) GetURL() string {
+	return d.URL
+}
+
+func (d *DatabaseCatalog) InitializeURL(baseURL url.URL) {
+	if d.Name == "" {
+		return
+	}
+	baseURL.Path = "explore/data/" + d.Name
+	d.URL = baseURL.String()
+}
diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go
@@ -185,6 +185,11 @@ func TestResourcesBindSupport(t *testing.T) {
 				DatabaseInstance: database.DatabaseInstance{},
 			},
 		},
+		DatabaseCatalogs: map[string]*resources.DatabaseCatalog{
+			"my_database_catalog": {
+				DatabaseCatalog: database.DatabaseCatalog{},
+			},
+		},
 	}
 	unbindableResources := map[string]bool{"model": true}
 
@@ -206,6 +211,7 @@ func TestResourcesBindSupport(t *testing.T) {
 	}, nil)
 	m.GetMockWarehousesAPI().EXPECT().GetById(mock.Anything, mock.Anything).Return(nil, nil)
 	m.GetMockDatabaseAPI().EXPECT().GetDatabaseInstance(mock.Anything, mock.Anything).Return(nil, nil)
+	m.GetMockDatabaseAPI().EXPECT().GetDatabaseCatalog(mock.Anything, mock.Anything).Return(nil, nil)
 
 	allResources := supportedResources.AllResources()
 	for _, group := range allResources {
diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go
@@ -117,6 +117,7 @@ var GroupToTerraformName = map[string]string{
 	"secret_scopes":           "databricks_secret_scope",
 	"sql_warehouses":          "databricks_sql_endpoint",
 	"database_instances":      "databricks_database_instance",
+	"database_catalogs":       "databricks_database_database_catalog",
 }
 
 var TerraformToGroupName = func() map[string]string {
diff --git a/bundle/deploy/terraform/tfdyn/convert_database_catalog.go b/bundle/deploy/terraform/tfdyn/convert_database_catalog.go
@@ -0,0 +1,28 @@
+package tfdyn
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/service/database"
+)
+
+type databaseCatalogConverter struct{}
+
+func (d databaseCatalogConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
+	// Normalize the output value to the target schema.
+	vout, diags := convert.Normalize(database.DatabaseCatalog{}, vin)
+	for _, diag := range diags {
+		log.Debugf(ctx, "database Catalog normalization diagnostic: %s", diag.Summary)
+	}
+	out.DatabaseDatabaseCatalog[key] = vout.AsAny()
+
+	return nil
+}
+
+func init() {
+	registerConverter("database_catalogs", databaseCatalogConverter{})
+}
diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml
@@ -169,6 +169,9 @@ github.com/databricks/cli/bundle/config.Resources:
       The dashboard definitions for the bundle, where each key is the name of the dashboard.
     "markdown_description": |-
       The dashboard definitions for the bundle, where each key is the name of the dashboard. See [\_](/dev-tools/bundles/resources.md#dashboards).
+  "database_catalogs":
+    "description": |-
+      PLACEHOLDER
   "database_instances":
     "description": |-
       PLACEHOLDER
@@ -472,6 +475,25 @@ github.com/databricks/cli/bundle/config/resources.DashboardPermission:
   "user_name":
     "description": |-
       PLACEHOLDER
+github.com/databricks/cli/bundle/config/resources.DatabaseCatalog:
+  "create_database_if_not_exists":
+    "description": |-
+      PLACEHOLDER
+  "database_instance_name":
+    "description": |-
+      PLACEHOLDER
+  "database_name":
+    "description": |-
+      PLACEHOLDER
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "permissions":
+    "description": |-
+      PLACEHOLDER
+  "uid":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.DatabaseInstance:
   "capacity":
     "description": |-
diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json
diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go
diff --git a/libs/testserver/database_catalogs.go b/libs/testserver/database_catalogs.go
diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+RequiresUnityCatalog = true`
	`2`	`+CloudEnvs.gcp = false # lakebase is not available in GCP as of August 2025`
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@ import (`
`15`	`15`	`"github.com/databricks/cli/libs/dyn/convert"`
`16`	`16`	`)`
`17`	`17`
`18`		`-var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models"}`
	`18`	`+var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs"}`
`19`	`19`
`20`	`20`	`var (`
`21`	`21`	`allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN}`