Add databricks_workspace_file resource (#2266)

* Add `databricks_workspace_file` resource

With recent release of the workspace file support it's now possible to store arbitrary
files in the workspace, not only notebooks.  It's also recommended way to store init
scripts to avoid security problems associated with storing init scripts on DBFS.

* Address review comments

* swtich to SDK

* One more fix

Author: alexott

docs/resources/notebook.md

@@ -63,7 +63,7 @@ In addition to all arguments above, the following attributes are exported:
 
 ## Access Control
 
-* [databricks_permissions](permissions.md#Notebook-usage) can control which groups or individual users can access notebooks or folders.
+* [databricks_permissions](permissions.md#notebook-usage) can control which groups or individual users can access notebooks or folders.
 
 ## Import
 

docs/resources/permissions.md

@@ -320,6 +320,45 @@ resource "databricks_permissions" "notebook_usage" {
 }
 ```
 
+## Workspace file usage
+
+Valid permission levels for [databricks_workspace_file](workspace_file.md) are: `CAN_READ`, `CAN_RUN`, `CAN_EDIT`, and `CAN_MANAGE`.
+
+```hcl
+resource "databricks_group" "auto" {
+  display_name = "Automation"
+}
+
+resource "databricks_group" "eng" {
+  display_name = "Engineering"
+}
+
+resource "databricks_workspace_file" "this" {
+  content_base64 = base64encode("print('Hello World')")
+  path           = "/Production/ETL/Features.py"
+}
+
+resource "databricks_permissions" "workspace_file_usage" {
+  workspace_file_path = databricks_workspace_file.this.path
+
+  access_control {
+    group_name       = "users"
+    permission_level = "CAN_READ"
+  }
+
+  access_control {
+    group_name       = databricks_group.auto.display_name
+    permission_level = "CAN_RUN"
+  }
+
+  access_control {
+    group_name       = databricks_group.eng.display_name
+    permission_level = "CAN_EDIT"
+  }
+}
+```
+
+
 ## Folder usage
 
 Valid [permission levels](https://docs.databricks.com/security/access-control/workspace-acl.html#folder-permissions) for folders of [databricks_directory](directory.md) are: `CAN_READ`, `CAN_RUN`, `CAN_EDIT`, and `CAN_MANAGE`. Notebooks and experiments in a folder inherit all permissions settings of that folder. For example, a user (or service principal) that has `CAN_RUN` permission on a folder has `CAN_RUN` permission on the notebooks in that folder.

docs/resources/workspace_file.md

@@ -0,0 +1,79 @@
+---
+subcategory: "Workspace"
+---
+# databricks_workspace_file Resource
+
+This resource allows you to manage [Databricks Workspace Files](https://docs.databricks.com/files/workspace.html).
+
+## Example Usage
+
+You can declare Terraform-managed workspace file by specifying `source` attribute of corresponding local file.
+
+```hcl
+data "databricks_current_user" "me" {
+}
+
+resource "databricks_workspace_file" "module" {
+  source = "${path.module}/module.py"
+  path   = "${data.databricks_current_user.me.home}/AA/BB/CC"
+}
+```
+
+You can also create a managed workspace file with inline sources through `content_base64`  attribute.
+
+```hcl
+resource "databricks_workspace_file" "init_script" {
+  content_base64 = base64encode(<<-EOT
+    #!/bin/bash
+    echo "Hello World"
+    EOT
+  )
+  path = "/Shared/init-script.sh"
+}
+```
+
+## Argument Reference
+
+-> **Note** Files in Databricks workspace would only be changed, if Terraform stage did change. This means that any manual changes to managed workspace files won't be overwritten by Terraform, if there's no local change to file sources. Workspace files are identified by their path, so changing file's name manually on the workspace and then applying Terraform state would result in creation of workspace file from Terraform state.
+
+The size of a workspace file source code must not exceed a few megabytes. The following arguments are supported:
+
+* `path` -  (Required) The absolute path of the workspace file, beginning with "/", e.g. "/Demo".
+* `source` - Path to file on local filesystem. Conflicts with `content_base64`.
+* `content_base64` - The base64-encoded file content. Conflicts with `source`. Use of `content_base64` is discouraged, as it's increasing memory footprint of Terraform state and should only be used in exceptional circumstances, like creating a workspace file with configuration properties for a data pipeline.
+
+## Attribute Reference
+
+In addition to all arguments above, the following attributes are exported:
+
+* `id` -  Path of workspace file
+* `url` - Routable URL of the workspace file
+* `object_id` -  Unique identifier for a workspace file
+
+## Access Control
+
+* [databricks_permissions](permissions.md#workspace-file-usage) can control which groups or individual users can access workspace file.
+
+## Import
+
+The workspace file resource can be imported using workspace file path
+
+```bash
+$ terraform import databricks_workspace_file.this /path/to/file
+```
+
+## Related Resources
+
+The following resources are often used in the same context:
+
+* [End to end workspace management](../guides/workspace-management.md) guide.
+* [databricks_cluster](cluster.md) to create [Databricks Clusters](https://docs.databricks.com/clusters/index.html).
+* [databricks_directory](directory.md) to manage directories in [Databricks Workpace](https://docs.databricks.com/workspace/workspace-objects.html).
+* [databricks_job](job.md) to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a [databricks_cluster](cluster.md).
+* [databricks_pipeline](pipeline.md) to deploy [Delta Live Tables](https://docs.databricks.com/data-engineering/delta-live-tables/index.html).
+* [databricks_repo](repo.md) to manage [Databricks Repos](https://docs.databricks.com/repos.html).
+* [databricks_secret](secret.md) to manage [secrets](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) in Databricks workspace.
+* [databricks_secret_acl](secret_acl.md) to manage access to [secrets](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) in Databricks workspace.
+* [databricks_secret_scope](secret_scope.md) to create [secret scopes](https://docs.databricks.com/security/secrets/index.html#secrets-user-guide) in Databricks workspace.
+* [databricks_user](user.md) to [manage users](https://docs.databricks.com/administration-guide/users-groups/users.html), that could be added to [databricks_group](group.md) within the workspace.
+* [databricks_user](../data-sources/user.md) data to retrieve information about [databricks_user](user.md).

internal/acceptance/workspace_file_test.go

@@ -0,0 +1,33 @@
+package acceptance
+
+import (
+	"testing"
+)
+
+func TestAccWorkspaceFile(t *testing.T) {
+	workspaceLevel(t, step{
+		Template: `resource "databricks_workspace_file" "this" {
+			source = "{var.CWD}/../../storage/testdata/tf-test-python.py"
+			path = "/Shared/provider-test/xx_{var.RANDOM}"
+		}`,
+	}, step{
+		Template: `resource "databricks_workspace_file" "this" {
+			source = "{var.CWD}/../../storage/testdata/tf-test-python.py"
+			path = "/Shared/provider-test/xx_{var.RANDOM}_renamed"
+		}`,
+	})
+}
+
+func TestAccWorkspaceFileBase64(t *testing.T) {
+	workspaceLevel(t, step{
+		Template: `resource "databricks_workspace_file" "this2" {
+			"content_base64": "YWJjCg==",
+			path = "/Shared/provider-test/xx2_{var.RANDOM}"
+		}`,
+	}, step{
+		Template: `resource "databricks_workspace_file" "this2" {
+			"content_base64": "YWJjCg==",
+			path = "/Shared/provider-test/xx2_{var.RANDOM}_renamed"
+		}`,
+	})
+}

permissions/resource_permissions.go

@@ -296,6 +296,8 @@ func permissionsResourceIDFields() []permissionsIDFieldMapping {
 		{"notebook_path", "notebook", "notebooks", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
 		{"directory_id", "directory", "directories", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, SIMPLE},
 		{"directory_path", "directory", "directories", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
+		{"workspace_file_id", "file", "files", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, SIMPLE},
+		{"workspace_file_path", "file", "files", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
 		{"repo_id", "repo", "repos", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, SIMPLE},
 		{"repo_path", "repo", "repos", []string{"CAN_READ", "CAN_RUN", "CAN_EDIT", "CAN_MANAGE"}, PATH},
 		{"authorization", "tokens", "authorization", []string{"CAN_USE"}, SIMPLE},
@@ -336,7 +338,12 @@ func (oa *ObjectACL) ToPermissionsEntity(d *schema.ResourceData, me string) (Per
 			continue
 		}
 		entity.ObjectType = mapping.objectType
-		pathVariant := d.Get(mapping.objectType + "_path")
+		var pathVariant any
+		if mapping.objectType == "file" {
+			pathVariant = d.Get("workspace_file_path")
+		} else {
+			pathVariant = d.Get(mapping.objectType + "_path")
+		}
 		if pathVariant != nil && pathVariant.(string) != "" {
 			// we're not importing and it's a path... it's set, so let's not re-set it
 			return entity, nil

permissions/resource_permissions_test.go

@@ -973,6 +973,80 @@ func TestResourcePermissionsCreate_NotebookPath(t *testing.T) {
 	assert.Equal(t, "CAN_READ", firstElem["permission_level"])
 }
 
+func TestResourcePermissionsCreate_WorkspaceFilePath(t *testing.T) {
+	d, err := qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			me,
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.0/workspace/get-status?path=%2FDevelopment%2FInit",
+				Response: workspace.ObjectStatus{
+					ObjectID:   988765,
+					ObjectType: workspace.File,
+				},
+			},
+			{
+				Method:   http.MethodPut,
+				Resource: "/api/2.0/permissions/files/988765",
+				ExpectedRequest: AccessControlChangeList{
+					AccessControlList: []AccessControlChange{
+						{
+							UserName:        TestingUser,
+							PermissionLevel: "CAN_READ",
+						},
+					},
+				},
+			},
+			{
+				Method:   http.MethodGet,
+				Resource: "/api/2.0/permissions/files/988765",
+				Response: ObjectACL{
+					ObjectID:   "/files/988765",
+					ObjectType: "file",
+					AccessControlList: []AccessControl{
+						{
+							UserName: TestingUser,
+							AllPermissions: []Permission{
+								{
+									PermissionLevel: "CAN_READ",
+									Inherited:       false,
+								},
+							},
+						},
+						{
+							UserName: TestingAdminUser,
+							AllPermissions: []Permission{
+								{
+									PermissionLevel: "CAN_MANAGE",
+									Inherited:       false,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		Resource: ResourcePermissions(),
+		State: map[string]any{
+			"workspace_file_path": "/Development/Init",
+			"access_control": []any{
+				map[string]any{
+					"user_name":        TestingUser,
+					"permission_level": "CAN_READ",
+				},
+			},
+		},
+		Create: true,
+	}.Apply(t)
+
+	assert.NoError(t, err)
+	ac := d.Get("access_control").(*schema.Set)
+	require.Equal(t, 1, len(ac.List()))
+	firstElem := ac.List()[0].(map[string]any)
+	assert.Equal(t, TestingUser, firstElem["user_name"])
+	assert.Equal(t, "CAN_READ", firstElem["permission_level"])
+}
+
 func TestResourcePermissionsCreate_error(t *testing.T) {
 	qa.ResourceFixture{
 		Fixtures: []qa.HTTPFixture{

provider/provider.go

@@ -154,6 +154,7 @@ func DatabricksProvider() *schema.Provider {
 			"databricks_user_instance_profile":       aws.ResourceUserInstanceProfile(),
 			"databricks_user_role":                   aws.ResourceUserRole(),
 			"databricks_workspace_conf":              workspace.ResourceWorkspaceConf(),
+			"databricks_workspace_file":              workspace.ResourceWorkspaceFile(),
 		},
 		Schema: providerSchema(),
 	}

workspace/data_notebook_paths.go

@@ -20,10 +20,12 @@ func DataSourceNotebookPaths() *schema.Resource {
 			d.SetId(path)
 			var notebookPathList []map[string]string
 			for _, v := range notebookList {
-				notebookPathMap := map[string]string{}
-				notebookPathMap["path"] = v.Path
-				notebookPathMap["language"] = string(v.Language)
-				notebookPathList = append(notebookPathList, notebookPathMap)
+				if v.ObjectType == Notebook {
+					notebookPathMap := map[string]string{}
+					notebookPathMap["path"] = v.Path
+					notebookPathMap["language"] = string(v.Language)
+					notebookPathList = append(notebookPathList, notebookPathMap)
+				}
 			}
 			// nolint
 			d.Set("notebook_path_list", notebookPathList)

workspace/resource_notebook.go

@@ -16,12 +16,14 @@ import (
 // ...
 const (
 	Notebook  string = "NOTEBOOK"
+	File      string = "FILE"
 	Directory string = "DIRECTORY"
 	Scala     string = "SCALA"
 	Python    string = "PYTHON"
 	SQL       string = "SQL"
 	R         string = "R"
 	Jupyter   string = "JUPYTER"
+	Auto      string = "AUTO"
 )
 
 type notebookLanguageFormat struct {
@@ -149,7 +151,7 @@ func (a NotebooksAPI) recursiveAddPaths(path string, pathList *[]ObjectStatus) e
 		return err
 	}
 	for _, v := range notebookInfoList {
-		if v.ObjectType == Notebook {
+		if v.ObjectType == Notebook || v.ObjectType == File {
 			*pathList = append(*pathList, v)
 		} else if v.ObjectType == Directory {
 			err := a.recursiveAddPaths(v.Path, pathList)
@@ -254,10 +256,9 @@ func ResourceNotebook() *schema.Resource {
 			Deprecated: "Always is a notebook",
 		},
 		"object_id": {
-			Type:       schema.TypeInt,
-			Optional:   true,
-			Computed:   true,
-			Deprecated: "Use id argument to retrieve object id",
+			Type:     schema.TypeInt,
+			Optional: true,
+			Computed: true,
 		},
 	})
 	s["content_base64"].RequiredWith = []string{"language"}