Skip to content

Commit 779e84f

Browse files
anton-107anton-107
authored
Add support for Lakebase synced database tables (#3467)
## Changes <!-- Brief summary of your changes that is easy to understand --> Added support for creating Lakebase synced database tables in DABs ## Why <!-- Why are these changes needed? Provide the context that the reviewer might be missing. For example, were there any decisions behind the change that are not reflected in the code itself? --> This change allows users to define synced database tables as part of their assets bundle: ``` ... resources: ... database_catalogs: my_catalog: database_instance_name: my-instance database_name: "my_database" name: my_catalog create_database_if_not_exists: true synced_database_tables: my_synced_table: name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_destination_table database_instance_name: ${resources.database_catalogs.my_catalog.database_instance_name} logical_database_name: ${resources.database_catalogs.my_catalog.database_name} spec: source_table_full_name: "my_source_table" scheduling_policy: SNAPSHOT primary_key_columns: - my_pk_column ``` ## Tests <!-- How have you tested the changes? --> Added new acceptance test <!-- If your PR needs to be included in the release notes for next release, add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->
1 parent 0b84364 commit 779e84f

File tree

21 files changed

+684
-21
lines changed

21 files changed

+684
-21
lines changed

‎NEXT_CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ See more details here: ([#3225](https://github.com/databricks/cli/pull/3225))
1616

1717
### Bundles
1818
* [Breaking Change] Remove deprecated path fallback mechanism for jobs and pipelines ([#3225](https://github.com/databricks/cli/pull/3225))
19+
* Add support for Lakebase synced database tables in DABs ([#3467](https://github.com/databricks/cli/pull/3467))
1920
* Rename Delta Live Tables to Lakeflow Declarative Pipelines in the default-python template ([#3476](https://github.com/databricks/cli/pull/3476)).
2021

2122
### API Changes

‎acceptance/bundle/deploy/lakebase/synced-database-table/databricks.yml.tmpl‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
bundle:
2+
name: deploy-lakebase-synced-table-$UNIQUE_NAME
3+
4+
resources:
5+
database_instances:
6+
my_instance:
7+
name: test-database-instance-$UNIQUE_NAME
8+
capacity: CU_1
9+
database_catalogs:
10+
my_catalog:
11+
database_instance_name: ${resources.database_instances.my_instance.name}
12+
database_name: my_database
13+
name: my_catalog_$UNIQUE_NAME
14+
create_database_if_not_exists: true
15+
synced_database_tables:
16+
my_synced_table:
17+
name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table
18+
database_instance_name: ${resources.database_instances.my_instance.name}
19+
logical_database_name: ${resources.database_catalogs.my_catalog.database_name}
20+
spec:
21+
source_table_full_name: "samples.nyctaxi.trips"
22+
scheduling_policy: SNAPSHOT
23+
primary_key_columns:
24+
- tpep_pickup_datetime

‎acceptance/bundle/deploy/lakebase/synced-database-table/out.test.toml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
Local = true
2+
Cloud = true
3+
RequiresUnityCatalog = true
4+
5+
[CloudEnvs]
6+
gcp = false
7+
8+
[EnvMatrix]
9+
DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

‎acceptance/bundle/deploy/lakebase/synced-database-table/output.txt‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
2+
>>> [CLI] bundle validate
3+
Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
4+
Target: default
5+
Workspace:
6+
User: [USERNAME]
7+
Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
8+
9+
Validation OK!
10+
11+
>>> [CLI] bundle summary
12+
Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
13+
Target: default
14+
Workspace:
15+
User: [USERNAME]
16+
Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
17+
Resources:
18+
Database catalogs:
19+
my_catalog:
20+
Name: my_catalog_[UNIQUE_NAME]
21+
URL: [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME]
22+
Database instances:
23+
my_instance:
24+
Name: test-database-instance-[UNIQUE_NAME]
25+
URL: (not deployed)
26+
Synced database tables:
27+
my_synced_table:
28+
Name: ${databricks_database_database_catalog.my_catalog.name}.${databricks_database_database_catalog.my_catalog.database_name}.my_synced_table
29+
URL: [DATABRICKS_URL]/explore/data/$%7Bdatabricks_database_database_catalog.my_catalog.name%7D.$%7Bdatabricks_database_database_catalog.my_catalog.database_name%7D.my_synced_table
30+
31+
>>> [CLI] bundle deploy
32+
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default/files...
33+
Deploying resources...
34+
Updating deployment state...
35+
Deployment complete!
36+
37+
>>> [CLI] bundle summary
38+
Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
39+
Target: default
40+
Workspace:
41+
User: [USERNAME]
42+
Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
43+
Resources:
44+
Database catalogs:
45+
my_catalog:
46+
Name: my_catalog_[UNIQUE_NAME]
47+
URL: [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME]
48+
Database instances:
49+
my_instance:
50+
Name: test-database-instance-[UNIQUE_NAME]
51+
URL: [DATABRICKS_URL]/compute/database-instances/test-database-instance-[UNIQUE_NAME]
52+
Synced database tables:
53+
my_synced_table:
54+
Name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table
55+
URL: [DATABRICKS_URL]/explore/data/$%7Bresources.database_catalogs.my_catalog.name%7D.$%7Bresources.database_catalogs.my_catalog.database_name%7D.my_synced_table
56+
57+
>>> [CLI] bundle destroy --auto-approve
58+
The following resources will be deleted:
59+
delete database_catalog my_catalog
60+
delete database_instance my_instance
61+
delete synced_database_table my_synced_table
62+
63+
All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
64+
65+
Deleting files...
66+
Destroy complete!

‎acceptance/bundle/deploy/lakebase/synced-database-table/script‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
envsubst < databricks.yml.tmpl > databricks.yml
2+
3+
cleanup() {
4+
trace $CLI bundle destroy --auto-approve
5+
}
6+
trap cleanup EXIT
7+
8+
trace $CLI bundle validate
9+
trace $CLI bundle summary
10+
trace $CLI bundle deploy
11+
trace $CLI bundle summary

‎acceptance/bundle/deploy/lakebase/synced-database-table/test.toml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Local = true
2+
Cloud = true
3+
4+
RecordRequests = false
5+
6+
[EnvMatrix]
7+
DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
8+
9+
[[Repls]]
10+
# clean up ?o=<num> suffix after URL since not all workspaces have that
11+
Old = '\?o=\[(NUMID|ALPHANUMID)\]'
12+
New = ''
13+
Order = 1000

‎acceptance/internal/handlers.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -526,4 +526,16 @@ func addDefaultHandlers(server *testserver.Server) {
526526
server.Handle("DELETE", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any {
527527
return testserver.MapDelete(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"])
528528
})
529+
530+
server.Handle("POST", "/api/2.0/database/synced_tables", func(req testserver.Request) any {
531+
return req.Workspace.SyncedDatabaseTableCreate(req)
532+
})
533+
534+
server.Handle("GET", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any {
535+
return testserver.MapGet(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"])
536+
})
537+
538+
server.Handle("DELETE", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any {
539+
return testserver.MapDelete(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"])
540+
})
529541
}

‎bundle/config/mutator/resourcemutator/apply_bundle_permissions.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import (
1515
"github.com/databricks/cli/libs/dyn/convert"
1616
)
1717

18-
var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs"}
18+
var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs", "synced_database_tables"}
1919

2020
var (
2121
allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN}

‎bundle/config/mutator/resourcemutator/apply_target_mode_test.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
183183
},
184184
},
185185
},
186+
SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{
187+
"synced_database_table1": {
188+
SyncedDatabaseTable: database.SyncedDatabaseTable{
189+
Name: "synced_database_table1",
190+
},
191+
},
192+
},
186193
},
187194
},
188195
SyncRoot: vfs.MustNew("/Users/[email protected]"),
@@ -351,7 +358,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
351358
resourceType := resources.Type().Field(i).Name
352359

353360
// Skip resources that are not renamed
354-
if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" {
361+
if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" || resourceType == "SyncedDatabaseTables" {
355362
continue
356363
}
357364

‎bundle/config/mutator/resourcemutator/run_as_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ func allResourceTypes(t *testing.T) []string {
4747
"schemas",
4848
"secret_scopes",
4949
"sql_warehouses",
50+
"synced_database_tables",
5051
"volumes",
5152
},
5253
resourceTypes,
@@ -143,6 +144,7 @@ var allowList = []string{
143144
"clusters",
144145
"database_catalogs",
145146
"database_instances",
147+
"synced_database_tables",
146148
"jobs",
147149
"models",
148150
"registered_models",

0 commit comments

Comments
 (0)