Added databricks_jobs data resource (#1138)

Retrieves a list of databricks_job ids, that were created by Terraform or manually, so that special handling could be applied.

## Example Usage

Granting view [databricks_permissions](../resources/permissions.md) to all [databricks_job](../resources/job.md) within the workspace:

```hcl
data "databricks_jobs" "this" {}

resource "databricks_permissions" "everyone_can_view_all_jobs" {
  for_each = data.databricks_jobs.this.ids
  job_id   = each.value

  access_control {
    group_name       = "users"
    permission_level = "CAN_VIEW"
  }
}
```

Getting ID of specific [databricks_job](../resources/job.md) by name:

```hcl
data "databricks_jobs" "this" {}

output "x" {
  value     = "ID of `x` job is ${data.databricks_jobs.this.ids["x"]}"
  sensitive = false
}
```

Author: nfx

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 0.5.1
 
 * Added an extended documentation from provisioning AWS PrivateLink workspace ([#1084](https://github.com/databrickslabs/terraform-provider-databricks/pull/1084)).
+* Added `databricks_jobs` data resource to get a map of all job names and their ids ([#1138](https://github.com/databrickslabs/terraform-provider-databricks/pull/1138)).
 
 ## 0.5.0
 

common/reflect_resource.go

@@ -171,6 +171,11 @@ func diffSuppressor(zero string) func(k, old, new string, d *schema.ResourceData
 func typeToSchema(v reflect.Value, t reflect.Type, path []string) map[string]*schema.Schema {
 	scm := map[string]*schema.Schema{}
 	rk := v.Kind()
+	if rk == reflect.Ptr {
+		v = v.Elem()
+		t = v.Type()
+		rk = v.Kind()
+	}
 	if rk != reflect.Struct {
 		panic(fmt.Errorf("Schema value of Struct is expected, but got %s: %#v", reflectKind(rk), v))
 	}
@@ -384,7 +389,11 @@ func isValueNilOrEmpty(valueField *reflect.Value, fieldPath string) bool {
 
 // StructToData reads result using schema onto resource data
 func StructToData(result interface{}, s map[string]*schema.Schema, d *schema.ResourceData) error {
-	return iterFields(reflect.ValueOf(result), []string{}, s, func(
+	v := reflect.ValueOf(result)
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	return iterFields(v, []string{}, s, func(
 		fieldSchema *schema.Schema, path []string, valueField *reflect.Value) error {
 		fieldValue := valueField.Interface()
 		if fieldValue == nil {

common/reflect_resource_test.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"context"
 	"fmt"
 	"reflect"
 	"testing"
@@ -579,3 +580,32 @@ func TestDataToReflectValueBypass(t *testing.T) {
 	err := DataToReflectValue(nil, &schema.Resource{Schema: map[string]*schema.Schema{}}, reflect.ValueOf(0))
 	assert.EqualError(t, err, "value of Struct is expected, but got Int: 0")
 }
+
+func TestDataResource(t *testing.T) {
+	r := func() *schema.Resource {
+		var dto struct {
+			In string `json:"in"`
+			Out string `json:"out,omitempty" tf:"computed"`
+		}
+		return DataResource(&dto, func(ctx context.Context, c *DatabricksClient) error {
+			dto.Out = "out: " + dto.In
+			if dto.In == "fail" {
+				return fmt.Errorf("happens")
+			}
+			return nil
+		})
+	}()
+	d := r.TestResourceData()
+	d.Set("in", "test")
+
+	diags := r.ReadContext(context.Background(), d, nil)
+	assert.Len(t, diags, 1)
+
+	diags = r.ReadContext(context.Background(), d, &DatabricksClient{})
+	assert.Len(t, diags, 0)
+	assert.Equal(t, "out: test", d.Get("out"))
+
+	d.Set("in", "fail")
+	diags = r.ReadContext(context.Background(), d, &DatabricksClient{})
+	assert.Len(t, diags, 1)
+}

common/resource.go

@@ -167,3 +167,27 @@ func makeEmptyBlockSuppressFunc(name string) func(k, old, new string, d *schema.
 		return false
 	}
 }
+
+func DataResource(e interface{}, read func(context.Context, *DatabricksClient) error) *schema.Resource {
+	s := StructToSchema(e, func(m map[string]*schema.Schema) map[string]*schema.Schema { return m })
+	return &schema.Resource{
+		Schema: s,
+		ReadContext: func(ctx context.Context, d *schema.ResourceData, m interface{}) (diags diag.Diagnostics) {
+			defer func() {
+				// using recoverable() would cause more complex rewrapping of DataToStructPointer & StructToData
+				if panic := recover(); panic != nil {
+					diags = diag.Errorf("panic: %v", panic)
+				}
+			}()
+			DataToStructPointer(d, s, e)
+			err := read(ctx, m.(*DatabricksClient))
+			if err != nil {
+				err = nicerError(ctx, err, "read data")
+				diags = diag.FromErr(err)
+			}
+			StructToData(e, s, d)
+			d.SetId("_")
+			return
+		},
+	}
+}

docs/data-sources/jobs.md

@@ -0,0 +1,51 @@
+---
+subcategory: "Compute"
+---
+# databricks_jobs Data Source
+
+-> **Note** If you have a fully automated setup with workspaces created by [databricks_mws_workspaces](../resources/mws_workspaces.md) or [azurerm_databricks_workspace](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/databricks_workspace), please make sure to add [depends_on attribute](../index.md#data-resources-and-authentication-is-not-configured-errors) in order to prevent _authentication is not configured for provider_ errors.
+
+Retrieves a list of [databricks_job](../resources/job.md) ids, that were created by Terraform or manually, so that special handling could be applied.
+
+-> **Note** Data resource will error in case of jobs with duplicate names.
+
+## Example Usage
+
+Granting view [databricks_permissions](../resources/permissions.md) to all [databricks_job](../resources/job.md) within the workspace:
+
+```hcl
+data "databricks_jobs" "this" {}
+
+resource "databricks_permissions" "everyone_can_view_all_jobs" {
+  for_each = data.databricks_jobs.this.ids
+  job_id   = each.value
+
+  access_control {
+    group_name       = "users"
+    permission_level = "CAN_VIEW"
+  }
+}
+```
+
+Getting ID of specific [databricks_job](../resources/job.md) by name:
+
+```hcl
+data "databricks_jobs" "this" {}
+
+output "x" {
+  value     = "ID of `x` job is ${data.databricks_jobs.this.ids["x"]}"
+  sensitive = false
+}
+```
+
+## Attribute Reference
+
+This data source exports the following attributes:
+
+* `ids` - map of [databricks_job](../resources/job.md) names to ids
+
+## Related Resources
+
+The following resources are used in the same context:
+
+* [databricks_job](../resources/job.md) to manage [Databricks Jobs](https://docs.databricks.com/jobs.html) to run non-interactive code in a [databricks_cluster](../resources/cluster.md).

docs/resources/job.md

@@ -221,6 +221,7 @@ The following resources are often used in the same context:
 * [databricks_global_init_script](global_init_script.md) to manage [global init scripts](https://docs.databricks.com/clusters/init-scripts.html#global-init-scripts), which are run on all [databricks_cluster](cluster.md#init_scripts) and [databricks_job](job.md#new_cluster).
 * [databricks_instance_pool](instance_pool.md) to manage [instance pools](https://docs.databricks.com/clusters/instance-pools/index.html) to reduce [cluster](cluster.md) start and auto-scaling times by maintaining a set of idle, ready-to-use instances.
 * [databricks_instance_profile](instance_profile.md) to manage AWS EC2 instance profiles that users can launch [databricks_cluster](cluster.md) and access data, like [databricks_mount](mount.md).
+* [databricks_jobs] data to get all jobs and their names from a workspace.
 * [databricks_library](library.md) to install a [library](https://docs.databricks.com/libraries/index.html) on [databricks_cluster](cluster.md).
 * [databricks_node_type](../data-sources/node_type.md) data to get the smallest node type for [databricks_cluster](cluster.md) that fits search criteria, like amount of RAM or number of cores.
 * [databricks_notebook](notebook.md) to manage [Databricks Notebooks](https://docs.databricks.com/notebooks/index.html).

jobs/data_jobs.go

@@ -0,0 +1,32 @@
+package jobs
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/databrickslabs/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+func DataSourceJobs() *schema.Resource {
+	var response struct {
+		Ids map[string]string `json:"ids,omitempty" tf:"computed"`
+	}
+	return common.DataResource(&response, func(ctx context.Context, c *common.DatabricksClient) error {
+		jobsAPI := NewJobsAPI(ctx, c)
+		list, err := jobsAPI.List()
+		if err != nil {
+			return err
+		}
+		response.Ids = map[string]string{}
+		for _, v := range list.Jobs {
+			name := v.Settings.Name
+			_, duplicateName := response.Ids[name]
+			if duplicateName {
+				return fmt.Errorf("duplicate job name detected: %s", name)
+			}
+			response.Ids[name] = v.ID()
+		}
+		return nil
+	})
+}

jobs/data_jobs_test.go

@@ -0,0 +1,43 @@
+package jobs
+
+import (
+	"testing"
+
+	"github.com/databrickslabs/terraform-provider-databricks/qa"
+)
+
+func TestJobsData(t *testing.T) {
+	qa.ResourceFixture{
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.0/jobs/list",
+				Response: JobList{
+					Jobs: []Job{
+						{
+							JobID: 123,
+							Settings: &JobSettings{
+								Name: "First",
+							},
+						},
+						{
+							JobID: 234,
+							Settings: &JobSettings{
+								Name: "Second",
+							},
+						},
+					},
+				},
+			},
+		},
+		Resource:    DataSourceJobs(),
+		Read:        true,
+		NonWritable: true,
+		ID:          "_",
+	}.ApplyAndExpectData(t, map[string]interface{}{
+		"ids": map[string]interface{}{
+			"First":  "123",
+			"Second": "234",
+		},
+	})
+}

provider/provider.go

@@ -44,6 +44,7 @@ func DatabricksProvider() *schema.Provider {
 			"databricks_dbfs_file":               storage.DataSourceDBFSFile(),
 			"databricks_dbfs_file_paths":         storage.DataSourceDBFSFilePaths(),
 			"databricks_group":                   scim.DataSourceGroup(),
+			"databricks_jobs":                    jobs.DataSourceJobs(),
 			"databricks_node_type":               clusters.DataSourceNodeType(),
 			"databricks_notebook":                workspace.DataSourceNotebook(),
 			"databricks_notebook_paths":          workspace.DataSourceNotebookPaths(),

qa/testing.go

@@ -261,7 +261,7 @@ func (f ResourceFixture) ApplyNoError(t *testing.T) {
 // ApplyAndExpectData is a convenience method for tests that doesn't expect error, but want to check data
 func (f ResourceFixture) ApplyAndExpectData(t *testing.T, data map[string]interface{}) {
 	d, err := f.Apply(t)
-	assert.NoError(t, err, err)
+	require.NoError(t, err, err)
 	for k, expected := range data {
 		if k == "id" {
 			assert.Equal(t, expected, d.Id())