Added secrets resource support to DABs

acceptance/bundle/resources/secret_scopes/test.toml

@@ -2,7 +2,8 @@ Cloud = true
 Local = true
 RecordRequests = false
 
-EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]  # secret scopes not implemented
+# direct mode doesn't support secret scope permissions yet
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]
 
 Ignore = [
     "databricks.yml",

acceptance/bundle/resources/secrets/databricks.yml.tmpl

@@ -0,0 +1,7 @@
+bundle:
+  name: deploy-secrets-test-$UNIQUE_NAME
+
+resources:
+  secret_scopes:
+    test_scope:
+      name: $SECRET_SCOPE_NAME

acceptance/bundle/resources/secrets/output.txt

@@ -0,0 +1,49 @@
+
+=== Deploy the secret scope first
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-secrets-test-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Verify the secret scope was created
+>>> [CLI] secrets list-scopes -o json
+{
+  "backend_type": "DATABRICKS",
+  "name": "secrets-test-[UUID]"
+}
+
+=== Create a secret via CLI
+>>> [CLI] secrets put-secret secrets-test-[UUID] my-secret-key --string-value my-secret-value
+
+=== Verify secret was created
+>>> [CLI] secrets get-secret secrets-test-[UUID] my-secret-key
+{
+  "key":"my-secret-key",
+  "value":"bXktc2VjcmV0LXZhbHVl"
+}
+
+=== Now add a secret to the bundle
+=== Deploy bundle with secret (scope already exists)
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-secrets-test-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Verify bundle secret was created
+>>> [CLI] secrets get-secret secrets-test-[UUID] bundle_secret
+{
+  "key":"bundle_secret",
+  "value":"YnVuZGxlLXNlY3JldC12YWx1ZQ=="
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.secret_scopes.test_scope
+  delete resources.secrets.secret1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-secrets-test-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/secrets/script

@@ -0,0 +1,47 @@
+SECRET_SCOPE_NAME="secrets-test-$(uuid)"
+if [ -z "$CLOUD_ENV" ]; then
+    SECRET_SCOPE_NAME="secrets-test-6260d50f-e8ff-4905-8f28-812345678903"   # use hard-coded uuid when running locally
+fi
+export SECRET_SCOPE_NAME
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+}
+trap cleanup EXIT
+
+title "Deploy the secret scope first"
+trace $CLI bundle deploy
+
+title "Verify the secret scope was created"
+trace $CLI secrets list-scopes -o json | jq --arg name "${SECRET_SCOPE_NAME}" '.[] | select(.name == $name)'
+
+title "Create a secret via CLI"
+trace $CLI secrets put-secret ${SECRET_SCOPE_NAME} my-secret-key --string-value "my-secret-value"
+
+title "Verify secret was created"
+trace $CLI secrets get-secret ${SECRET_SCOPE_NAME} my-secret-key
+
+title "Now add a secret to the bundle"
+cat > databricks.yml <<EOF
+bundle:
+  name: deploy-secrets-test-${UNIQUE_NAME}
+
+resources:
+  secret_scopes:
+    test_scope:
+      name: ${SECRET_SCOPE_NAME}
+
+  secrets:
+    secret1:
+      scope: ${SECRET_SCOPE_NAME}
+      key: bundle_secret
+      string_value: "bundle-secret-value"
+EOF
+
+title "Deploy bundle with secret (scope already exists)"
+trace $CLI bundle deploy
+
+title "Verify bundle secret was created"
+trace $CLI secrets get-secret ${SECRET_SCOPE_NAME} bundle_secret

acceptance/bundle/resources/secrets/test.toml

@@ -0,0 +1,10 @@
+Cloud = true
+Local = true
+RecordRequests = false
+
+# Use FakeWorkspace handlers for secrets which properly maintain state
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]
+
+Ignore = [
+    "databricks.yml",
+]

bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go

@@ -25,6 +25,7 @@ var unsupportedResources = []string{
 	"registered_models",
 	"database_catalogs",
 	"synced_database_tables",
+	"secrets",
 }
 
 func TestApplyBundlePermissions(t *testing.T) {

bundle/config/mutator/resourcemutator/apply_target_mode_test.go

@@ -162,6 +162,12 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 						Name: "secretScope1",
 					},
 				},
+				Secrets: map[string]*resources.Secret{
+					"secret1": {
+						Scope: "secretScope1",
+						Key:   "secret1",
+					},
+				},
 				SqlWarehouses: map[string]*resources.SqlWarehouse{
 					"sql_warehouse1": {
 						CreateWarehouseRequest: sql.CreateWarehouseRequest{
@@ -365,7 +371,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 				resourceType := resources.Type().Field(i).Name
 
 				// Skip resources that are not renamed
-				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" || resourceType == "SyncedDatabaseTables" {
+				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "Secrets" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" || resourceType == "SyncedDatabaseTables" {
 					continue
 				}
 

bundle/config/mutator/resourcemutator/run_as_test.go

@@ -47,6 +47,7 @@ func allResourceTypes(t *testing.T) []string {
 		"registered_models",
 		"schemas",
 		"secret_scopes",
+		"secrets",
 		"sql_warehouses",
 		"synced_database_tables",
 		"volumes",
@@ -156,6 +157,7 @@ var allowList = []string{
 	"experiments",
 	"schemas",
 	"secret_scopes",
+	"secrets",
 	"sql_warehouses",
 	"volumes",
 }

bundle/config/resources.go

@@ -25,6 +25,7 @@ type Resources struct {
 	Dashboards            map[string]*resources.Dashboard            `json:"dashboards,omitempty"`
 	Apps                  map[string]*resources.App                  `json:"apps,omitempty"`
 	SecretScopes          map[string]*resources.SecretScope          `json:"secret_scopes,omitempty"`
+	Secrets               map[string]*resources.Secret               `json:"secrets,omitempty"`
 	// Alerts                map[string]*resources.Alert                `json:"alerts,omitempty"`
 	SqlWarehouses        map[string]*resources.SqlWarehouse        `json:"sql_warehouses,omitempty"`
 	DatabaseInstances    map[string]*resources.DatabaseInstance    `json:"database_instances,omitempty"`
@@ -94,6 +95,7 @@ func (r *Resources) AllResources() []ResourceGroup {
 		collectResourceMap(descriptions["apps"], r.Apps),
 		// collectResourceMap(descriptions["alerts"], r.Alerts),
 		collectResourceMap(descriptions["secret_scopes"], r.SecretScopes),
+		collectResourceMap(descriptions["secrets"], r.Secrets),
 		collectResourceMap(descriptions["sql_warehouses"], r.SqlWarehouses),
 		collectResourceMap(descriptions["database_instances"], r.DatabaseInstances),
 		collectResourceMap(descriptions["database_catalogs"], r.DatabaseCatalogs),
@@ -175,6 +177,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 		}
 	}
 
+	for k := range r.Secrets {
+		if k == key {
+			found = append(found, r.Secrets[k])
+		}
+	}
+
 	// for k := range r.Alerts {
 	// 	if k == key {
 	// 		found = append(found, r.Alerts[k])
@@ -236,6 +244,7 @@ func SupportedResources() map[string]resources.ResourceDescription {
 		"volumes":                 (&resources.Volume{}).ResourceDescription(),
 		"apps":                    (&resources.App{}).ResourceDescription(),
 		"secret_scopes":           (&resources.SecretScope{}).ResourceDescription(),
+		"secrets":                 (&resources.Secret{}).ResourceDescription(),
 		// "alerts":                  (&resources.Alert{}).ResourceDescription(),
 		"sql_warehouses":         (&resources.SqlWarehouse{}).ResourceDescription(),
 		"database_instances":     (&resources.DatabaseInstance{}).ResourceDescription(),

bundle/config/resources/secret.go

@@ -0,0 +1,78 @@
+package resources
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/apierr"
+	"github.com/databricks/databricks-sdk-go/marshal"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+)
+
+type Secret struct {
+	BaseResource
+
+	// The name of the secret scope containing the secret.
+	Scope string `json:"scope"`
+
+	// A unique name to identify the secret.
+	Key string `json:"key"`
+
+	// The string value of the secret. Only one of string_value or bytes_value can be specified.
+	StringValue string `json:"string_value,omitempty"`
+
+	// The base64-encoded bytes value of the secret. Only one of string_value or bytes_value can be specified.
+	BytesValue string `json:"bytes_value,omitempty"`
+}
+
+func (s *Secret) UnmarshalJSON(b []byte) error {
+	return marshal.Unmarshal(b, s)
+}
+
+func (s Secret) MarshalJSON() ([]byte, error) {
+	return marshal.Marshal(s)
+}
+
+func (s Secret) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
+	// Secrets don't have a direct "get" API - we can check if it exists by trying to get its metadata
+	// The GetSecret API returns the secret metadata (not the value)
+	_, err := w.Secrets.GetSecret(ctx, workspace.GetSecretRequest{
+		Scope: s.Scope,
+		Key:   s.Key,
+	})
+	if err != nil {
+		if apierr.IsMissing(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func (s Secret) ResourceDescription() ResourceDescription {
+	return ResourceDescription{
+		SingularName:  "secret",
+		PluralName:    "secrets",
+		SingularTitle: "Secret",
+		PluralTitle:   "Secrets",
+	}
+}
+
+func (s Secret) GetName() string {
+	if s.ID != "" {
+		return s.ID
+	}
+	// Return a composite name of scope/key
+	return fmt.Sprintf("%s/%s", s.Scope, s.Key)
+}
+
+func (s Secret) GetURL() string {
+	// Secrets do not have a URL in the Databricks UI
+	return ""
+}
+
+func (s Secret) InitializeURL(_ url.URL) {
+	// Secrets do not have a URL
+}

bundle/config/resources/secret_scope.go

@@ -61,7 +61,7 @@ func (s SecretScope) Exists(ctx context.Context, w *databricks.WorkspaceClient,
 	// The indirect methods are not semantically ideal for simple existence checks, so we use the list API here
 	scopes, err := w.Secrets.ListScopesAll(ctx)
 	if err != nil {
-		return false, nil
+		return false, err
 	}
 
 	for _, scope := range scopes {

bundle/config/resources/secret_scope_test.go

@@ -0,0 +1,67 @@
+package resources
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/databricks-sdk-go/experimental/mocks"
+	"github.com/databricks/databricks-sdk-go/service/workspace"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSecretScopeExists(t *testing.T) {
+	ctx := context.Background()
+
+	m := mocks.NewMockWorkspaceClient(t)
+	m.GetMockSecretsAPI().On("ListScopesAll", mock.Anything).Return([]workspace.SecretScope{
+		{Name: "my_scope"},
+		{Name: "other_scope"},
+	}, nil)
+
+	s := &SecretScope{
+		Name: "my_scope",
+	}
+	exists, err := s.Exists(ctx, m.WorkspaceClient, "my_scope")
+
+	require.NoError(t, err)
+	assert.True(t, exists)
+}
+
+func TestSecretScopeNotFound(t *testing.T) {
+	ctx := context.Background()
+
+	m := mocks.NewMockWorkspaceClient(t)
+	m.GetMockSecretsAPI().On("ListScopesAll", mock.Anything).Return([]workspace.SecretScope{
+		{Name: "other_scope"},
+	}, nil)
+
+	s := &SecretScope{
+		Name: "my_scope",
+	}
+	exists, err := s.Exists(ctx, m.WorkspaceClient, "my_scope")
+
+	require.NoError(t, err)
+	assert.False(t, exists)
+}
+
+func TestSecretScopeGetName(t *testing.T) {
+	s := &SecretScope{
+		Name: "my_scope",
+	}
+	assert.Equal(t, "my_scope", s.GetName())
+
+	s.ID = "custom_id"
+	assert.Equal(t, "custom_id", s.GetName())
+}
+
+func TestSecretScopeResourceDescription(t *testing.T) {
+	s := &SecretScope{}
+	desc := s.ResourceDescription()
+
+	assert.Equal(t, "secret_scope", desc.SingularName)
+	assert.Equal(t, "secret_scopes", desc.PluralName)
+	assert.Equal(t, "Secret Scope", desc.SingularTitle)
+	assert.Equal(t, "Secret Scopes", desc.PluralTitle)
+}