direct: Add support for secret scopes

acceptance/bundle/refschema/out.fields.txt

@@ -2978,6 +2978,35 @@ resources.schemas.*.grants.grants[*].principal	string	ALL
 resources.schemas.*.grants.grants[*].privileges	[]catalog.Privilege	ALL
 resources.schemas.*.grants.grants[*].privileges[*]	catalog.Privilege	ALL
 resources.schemas.*.grants.securable_type	string	ALL
+resources.secret_scopes.*.backend_azure_keyvault	*workspace.AzureKeyVaultSecretScopeMetadata	STATE
+resources.secret_scopes.*.backend_azure_keyvault.dns_name	string	STATE
+resources.secret_scopes.*.backend_azure_keyvault.resource_id	string	STATE
+resources.secret_scopes.*.backend_type	workspace.ScopeBackendType	INPUT	REMOTE
+resources.secret_scopes.*.id	string	INPUT
+resources.secret_scopes.*.initial_manage_principal	string	STATE
+resources.secret_scopes.*.keyvault_metadata	*workspace.AzureKeyVaultSecretScopeMetadata	INPUT	REMOTE
+resources.secret_scopes.*.keyvault_metadata.dns_name	string	INPUT	REMOTE
+resources.secret_scopes.*.keyvault_metadata.resource_id	string	INPUT	REMOTE
+resources.secret_scopes.*.lifecycle	resources.Lifecycle	INPUT
+resources.secret_scopes.*.lifecycle.prevent_destroy	bool	INPUT
+resources.secret_scopes.*.modified_status	string	INPUT
+resources.secret_scopes.*.name	string	INPUT	REMOTE
+resources.secret_scopes.*.permissions	[]resources.SecretScopePermission	INPUT
+resources.secret_scopes.*.permissions[*]	resources.SecretScopePermission	INPUT
+resources.secret_scopes.*.permissions[*].group_name	string	INPUT
+resources.secret_scopes.*.permissions[*].level	resources.SecretScopePermissionLevel	INPUT
+resources.secret_scopes.*.permissions[*].service_principal_name	string	INPUT
+resources.secret_scopes.*.permissions[*].user_name	string	INPUT
+resources.secret_scopes.*.scope	string	STATE
+resources.secret_scopes.*.scope_backend_type	workspace.ScopeBackendType	STATE
+resources.secret_scopes.*.url	string	INPUT
+resources.secret_scopes.*.permissions.acls	[]resources.SecretScopePermission	ALL
+resources.secret_scopes.*.permissions.acls[*]	resources.SecretScopePermission	ALL
+resources.secret_scopes.*.permissions.acls[*].group_name	string	ALL
+resources.secret_scopes.*.permissions.acls[*].level	resources.SecretScopePermissionLevel	ALL
+resources.secret_scopes.*.permissions.acls[*].service_principal_name	string	ALL
+resources.secret_scopes.*.permissions.acls[*].user_name	string	ALL
+resources.secret_scopes.*.permissions.scope_name	string	ALL
 resources.sql_warehouses.*.auto_stop_mins	int	ALL
 resources.sql_warehouses.*.channel	*sql.Channel	ALL
 resources.sql_warehouses.*.channel.dbsql_version	string	ALL

acceptance/bundle/resources/secret_scopes/backend-type/output.txt

@@ -5,12 +5,16 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> jq -s .[] | select(.path=="/api/2.0/secrets/scopes/create") | .body out.requests.txt
+>>> print_requests.py //secrets
 {
-  "backend_azure_keyvault": {
-    "dns_name": "my_azure_keyvault_dns_name",
-    "resource_id": "my_azure_keyvault_id"
-  },
-  "scope": "test-secrets-azure-backend",
-  "scope_backend_type": "AZURE_KEYVAULT"
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/create",
+  "body": {
+    "backend_azure_keyvault": {
+      "dns_name": "my_azure_keyvault_dns_name",
+      "resource_id": "my_azure_keyvault_id"
+    },
+    "scope": "test-secrets-azure-backend",
+    "scope_backend_type": "AZURE_KEYVAULT"
+  }
 }

acceptance/bundle/resources/secret_scopes/backend-type/script

@@ -1,3 +1,2 @@
 trace $CLI bundle deploy
-trace jq -s '.[] | select(.path=="/api/2.0/secrets/scopes/create") | .body' out.requests.txt
-rm out.requests.txt
+trace print_requests.py //secrets

acceptance/bundle/resources/secret_scopes/databricks.yml.tmpl → acceptance/bundle/resources/secret_scopes/basic/databricks.yml.tmpl

@@ -1,13 +1,11 @@
 bundle:
-  name: deploy-secret-scope-test-$UNIQUE_NAME
+  name: secret-scope-basic-$UNIQUE_NAME
 
 resources:
   secret_scopes:
-    secret_scope1:
+    my_scope:
       name: $SECRET_SCOPE_NAME
       backend_type: "DATABRICKS"
       permissions:
-        - user_name: admins
+        - user_name: [email protected]
           level: WRITE
-        - user_name: users
-          level: READ

acceptance/bundle/resources/secret_scopes/basic/out.second-requests.direct.txt

@@ -0,0 +1,33 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/delete",
+  "body": {
+    "scope": "test-scope-[UNIQUE_NAME]-1"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/create",
+  "body": {
+    "scope": "test-scope-[UNIQUE_NAME]-2",
+    "scope_backend_type": "DATABRICKS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/acls/put",
+  "body": {
+    "permission": "WRITE",
+    "principal": "[email protected]",
+    "scope": "test-scope-[UNIQUE_NAME]-2"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/put",
+  "body": {
+    "key": "another-key",
+    "scope": "test-scope-[UNIQUE_NAME]-2",
+    "string_value": "another-secret-value"
+  }
+}

acceptance/bundle/resources/secret_scopes/basic/out.second-requests.terraform.txt

@@ -0,0 +1,41 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/acls/delete",
+  "body": {
+    "principal": "[email protected]",
+    "scope": "test-scope-[UNIQUE_NAME]-1"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/delete",
+  "body": {
+    "scope": "test-scope-[UNIQUE_NAME]-1"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/create",
+  "body": {
+    "scope": "test-scope-[UNIQUE_NAME]-2",
+    "scope_backend_type": "DATABRICKS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/acls/put",
+  "body": {
+    "permission": "WRITE",
+    "principal": "[email protected]",
+    "scope": "test-scope-[UNIQUE_NAME]-2"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/put",
+  "body": {
+    "key": "another-key",
+    "scope": "test-scope-[UNIQUE_NAME]-2",
+    "string_value": "another-secret-value"
+  }
+}

acceptance/bundle/resources/secret_scopes/basic/output.txt

@@ -0,0 +1,93 @@
+
+=== create the secret scope
+>>> [CLI] bundle plan
+create secret_scopes.my_scope
+
+Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/secret-scope-basic-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] secrets list-scopes -o json
+{
+  "backend_type": "DATABRICKS",
+  "name": "test-scope-[UNIQUE_NAME]-1"
+}
+
+=== put and get secret in first scope
+>>> [CLI] secrets put-secret test-scope-[UNIQUE_NAME]-1 my-key --string-value my-secret-value
+
+>>> [CLI] secrets get-secret test-scope-[UNIQUE_NAME]-1 my-key
+{
+  "key":"my-key",
+  "value":"bXktc2VjcmV0LXZhbHVl"
+}
+
+>>> print_requests.py //secrets
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/scopes/create",
+  "body": {
+    "scope": "test-scope-[UNIQUE_NAME]-1",
+    "scope_backend_type": "DATABRICKS"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/acls/put",
+  "body": {
+    "permission": "WRITE",
+    "principal": "[email protected]",
+    "scope": "test-scope-[UNIQUE_NAME]-1"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/secrets/put",
+  "body": {
+    "key": "my-key",
+    "scope": "test-scope-[UNIQUE_NAME]-1",
+    "string_value": "my-secret-value"
+  }
+}
+
+=== update the name of the scope (should recreate)
+>>> [CLI] bundle plan
+recreate secret_scopes.my_scope
+
+Plan: 1 to add, 0 to change, 1 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/secret-scope-basic-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] secrets list-scopes -o json
+{
+  "backend_type": "DATABRICKS",
+  "name": "test-scope-[UNIQUE_NAME]-2"
+}
+
+=== put and get secret in recreated scope
+>>> [CLI] secrets put-secret test-scope-[UNIQUE_NAME]-2 another-key --string-value another-secret-value
+
+>>> [CLI] secrets get-secret test-scope-[UNIQUE_NAME]-2 another-key
+{
+  "key":"another-key",
+  "value":"YW5vdGhlci1zZWNyZXQtdmFsdWU="
+}
+
+>>> print_requests.py //secrets
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete secret_scope my_scope
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/secret-scope-basic-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/secret_scopes/basic/script

@@ -0,0 +1,39 @@
+export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-1"
+
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm out.requests.txt
+}
+trap cleanup EXIT
+
+title "create the secret scope"
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name')
+trace $CLI secrets list-scopes -o json | jq ".[] | select(.name == \"$scope_name\")"
+
+title "put and get secret in first scope"
+trace $CLI secrets put-secret $scope_name my-key --string-value "my-secret-value"
+trace $CLI secrets get-secret $scope_name my-key
+
+trace print_requests.py //secrets
+
+title "update the name of the scope (should recreate)"
+export SECRET_SCOPE_NAME="test-scope-$UNIQUE_NAME-2"
+envsubst < databricks.yml.tmpl > databricks.yml
+
+trace $CLI bundle plan
+trace $CLI bundle deploy
+
+scope_name=$($CLI bundle summary --output json | jq -r '.resources.secret_scopes.my_scope.name')
+trace $CLI secrets list-scopes -o json | jq ".[] | select(.name == \"$scope_name\")"
+
+title "put and get secret in recreated scope"
+trace $CLI secrets put-secret $scope_name another-key --string-value "another-secret-value"
+trace $CLI secrets get-secret $scope_name another-key
+
+# Capture API requests for verification. Terraform cleans up ACLs before deleting the scope, but direct does not, hence the difference in requests.
+trace print_requests.py //secrets > out.second-requests.$DATABRICKS_BUNDLE_ENGINE.txt

acceptance/bundle/resources/secret_scopes/basic/test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+Local = true
+RecordRequests = true