[Fix] Populate partitions when reading databricks_sql_table (#4486)

## Changes
- UC returns `partition_index` in the column struct, so we can calculate
`partitions` value based on that
- Resolves #3980
## Tests
<!-- 
How is this tested? Please see the checklist below and also describe any
other relevant tests
-->

- [x] `make test` run locally
- [x] relevant change in `docs/` folder
- [x] using Go SDK

---------

Co-authored-by: Alex Ott <[email protected]>

Author: nkvuong

NEXT_CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Bug Fixes
 
+ * Populate `partitions` when reading `databricks_sql_table` ([#4486](https://github.com/databricks/terraform-provider-databricks/pull/4486)).
  * Fix configuration drift when configuring `databricks_connection` to builtin Hive Metastore ([#4505](https://github.com/databricks/terraform-provider-databricks/pull/4505)).
  * Only allow `authorized_paths` to be updated in the `options` field of `databricks_catalog` ([#4517](https://github.com/databricks/terraform-provider-databricks/pull/4517)).
  * Mark `default_catalog_name` attribute in `databricks_metastore_assignment` as deprecated ([#4522](https://github.com/databricks/terraform-provider-databricks/pull/4522))

catalog/resource_sql_table.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"reflect"
 	"slices"
+	"sort"
 	"strings"
 	"time"
 
@@ -668,13 +669,42 @@ func ResourceSqlTable() common.Resource {
 			if err != nil {
 				return err
 			}
+			w, err := c.WorkspaceClient()
+			if err != nil {
+				return err
+			}
+			partitionInfo, err := w.Tables.GetByFullName(ctx, d.Id())
+			if err != nil {
+				return err
+			}
+			partitionIndexes := map[int]string{}
 			for i := range ti.ColumnInfos {
 				c := &ti.ColumnInfos[i]
 				c.Identity, err = reconstructIdentity(c)
 				if err != nil {
 					return err
 				}
 			}
+
+			for i := range partitionInfo.Columns {
+				c := &partitionInfo.Columns[i]
+				if slices.Contains(c.ForceSendFields, "PartitionIndex") {
+					partitionIndexes[c.PartitionIndex] = c.Name
+				}
+			}
+			indexes := []int{}
+			partitions := []string{}
+
+			for index := range partitionIndexes {
+				indexes = append(indexes, index)
+			}
+			sort.Ints(indexes)
+
+			for _, p := range indexes {
+				partitions = append(partitions, partitionIndexes[p])
+			}
+
+			d.Set("partitions", partitions)
 			return common.StructToData(ti, tableSchema, d)
 		},
 		Update: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {

catalog/resource_sql_table_test.go

@@ -229,6 +229,11 @@ func TestResourceSqlTableCreateTable(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 		}, useExistingClusterForSql...),
 		Create:   true,
 		Resource: ResourceSqlTable(),
@@ -304,6 +309,11 @@ func TestResourceSqlTableCreateTableWithOwner(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 		}, useExistingClusterForSql...),
 		Create:   true,
 		Resource: ResourceSqlTable(),
@@ -426,6 +436,11 @@ func TestResourceSqlTableUpdateTable(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -527,6 +542,11 @@ func TestResourceSqlTableUpdateTableAndOwner(t *testing.T) {
 					Owner: "old group",
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -623,6 +643,11 @@ func TestResourceSqlTableUpdateTableClusterKeys(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -713,6 +738,11 @@ func TestResourceSqlTableUpdateView(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -822,6 +852,11 @@ func TestResourceSqlTableUpdateView_Definition(t *testing.T) {
 					ViewDefinition: "SELECT * FROM main.foo.bar2",
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.barview?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -877,6 +912,11 @@ func TestResourceSqlTableUpdateView_Comments(t *testing.T) {
 					Comment:        "to be changed (requires new)",
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.barview?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -969,6 +1009,11 @@ func resourceSqlTableUpdateColumnHelper(t *testing.T, testMetaData resourceSqlTa
 					ColumnInfos:           testMetaData.oldColumns,
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 			{
 				Method:   "POST",
 				Resource: "/api/2.0/clusters/start",
@@ -1357,6 +1402,11 @@ func TestResourceSqlTableCreateTable_ExistingSQLWarehouse(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 		},
 		Create:   true,
 		Resource: ResourceSqlTable(),
@@ -1450,6 +1500,11 @@ func TestResourceSqlTableCreateTableWithIdentityColumn_ExistingSQLWarehouse(t *t
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 		},
 		Create:   true,
 		Resource: ResourceSqlTable(),
@@ -1516,6 +1571,99 @@ func TestResourceSqlTableReadTableWithIdentityColumn_ExistingSQLWarehouse(t *tes
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
+		},
+		ID:       "main.foo.bar",
+		Read:     true,
+		Resource: ResourceSqlTable(),
+	}.ApplyAndExpectData(t, map[string]any{
+		"column.0.identity": "always",
+		"column.1.identity": "",
+		"column.2.identity": "default",
+	})
+}
+
+func TestResourceSqlTableReadTableWithPartitionColumn_ExistingSQLWarehouse(t *testing.T) {
+	qa.ResourceFixture{
+		CommandMock: func(commandStr string) common.CommandResults {
+			return common.CommandResults{
+				ResultType: "",
+				Data:       nil,
+			}
+		},
+		HCL: `
+		name               = "bar"
+		catalog_name       = "main"
+		schema_name        = "foo"
+		table_type         = "MANAGED"
+		data_source_format = "DELTA"
+		storage_location   = "abfss://container@account/somepath"
+		warehouse_id       = "existingwarehouse"
+	  
+	
+		comment = "this table is managed by terraform"
+		`,
+		Fixtures: []qa.HTTPFixture{
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar",
+				Response: SqlTableInfo{
+					Name:                  "bar",
+					CatalogName:           "main",
+					SchemaName:            "foo",
+					TableType:             "EXTERNAL",
+					DataSourceFormat:      "DELTA",
+					StorageLocation:       "s3://ext-main/foo/bar1",
+					StorageCredentialName: "somecred",
+					Comment:               "terraform managed",
+					Properties: map[string]string{
+						"one":   "two",
+						"three": "four",
+					},
+					ColumnInfos: []SqlColumnInfo{
+						{
+							Name:     "id",
+							Type:     "bigint",
+							TypeJson: "{\"type\":\"bigint\",\"nullable\":true, \"metadata\":{\"delta.identity.start\":1,\"delta.identity.allowExplicitInsert\":false}}",
+						},
+						{
+							Name:    "name",
+							Type:    "string",
+							Comment: "name of thing",
+						},
+						{
+							Name:     "number",
+							Type:     "bigint",
+							TypeJson: "{\"type\":\"bigint\",\"nullable\":true, \"metadata\":{\"delta.identity.start\":1,\"delta.identity.allowExplicitInsert\":true}}",
+						},
+					},
+				},
+			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{
+					Columns: []catalog.ColumnInfo{
+						{
+							Name:            "id",
+							PartitionIndex:  1,
+							ForceSendFields: []string{"PartitionIndex"},
+						},
+						{
+							Name:            "name",
+							PartitionIndex:  0,
+							ForceSendFields: []string{"PartitionIndex"},
+						},
+						{
+							Name: "number",
+						},
+					},
+				},
+			},
 		},
 		ID:       "main.foo.bar",
 		Read:     true,
@@ -1524,6 +1672,7 @@ func TestResourceSqlTableReadTableWithIdentityColumn_ExistingSQLWarehouse(t *tes
 		"column.0.identity": "always",
 		"column.1.identity": "",
 		"column.2.identity": "default",
+		"partitions":        []any{"name", "id"},
 	})
 }
 
@@ -1587,6 +1736,11 @@ func TestResourceSqlTableCreateTable_OnlyManagedProperties(t *testing.T) {
 					},
 				},
 			},
+			{
+				Method:   "GET",
+				Resource: "/api/2.1/unity-catalog/tables/main.foo.bar?",
+				Response: catalog.TableInfo{},
+			},
 		},
 		Create:   true,
 		Resource: ResourceSqlTable(),

docs/resources/sql_table.md

@@ -9,7 +9,7 @@ A `databricks_sql_table` is contained within [databricks_schema](schema.md), and
 
 This resource creates and updates the Unity Catalog table/view by executing the necessary SQL queries on a special auto-terminating cluster it would create for this operation. You could also specify a SQL warehouse or cluster for the queries to be executed on.
 
-~> This resource doesn't handle complex cases of schema evolution due to the limitations of Terraform itself.  If you need to implement schema evolution it's recommended to use specialized tools, such as, [Luquibase](https://medium.com/dbsql-sme-engineering/advanced-schema-management-on-databricks-with-liquibase-1900e9f7b9c0) and [Flyway](https://medium.com/dbsql-sme-engineering/databricks-schema-management-with-flyway-527c4a9f5d67).
+~> This resource doesn't handle complex cases of schema evolution due to the limitations of Terraform itself.  If you need to implement schema evolution it's recommended to use specialized tools, such as, [Liquibase](https://medium.com/dbsql-sme-engineering/advanced-schema-management-on-databricks-with-liquibase-1900e9f7b9c0) and [Flyway](https://medium.com/dbsql-sme-engineering/databricks-schema-management-with-flyway-527c4a9f5d67).
 
 ## Example Usage
 
@@ -162,15 +162,15 @@ The following arguments are supported:
 * `storage_location` - (Optional) URL of storage location for Table data (required for EXTERNAL Tables). Not supported for `VIEW` or `MANAGED` table_type.
 * `data_source_format` - (Optional) External tables are supported in multiple data source formats. The string constants identifying these formats are `DELTA`, `CSV`, `JSON`, `AVRO`, `PARQUET`, `ORC`, and `TEXT`. Change forces the creation of a new resource. Not supported for `MANAGED` tables or `VIEW`.
 * `view_definition` - (Optional) SQL text defining the view (for `table_type == "VIEW"`). Not supported for `MANAGED` or `EXTERNAL` table_type.
-* `cluster_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a cluster_id is specified, it will be used to execute SQL commands to manage this table. If empty, a cluster will be created automatically with the name `terraform-sql-table`.
+* `cluster_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a cluster_id is specified, it will be used to execute SQL commands to manage this table. If empty, a cluster will be created automatically with the name `terraform-sql-table`. Conflicts with `warehouse_id`.
 * `warehouse_id` - (Optional) All table CRUD operations must be executed on a running cluster or SQL warehouse. If a `warehouse_id` is specified, that SQL warehouse will be used to execute SQL commands to manage this table. Conflicts with `cluster_id`.
 * `cluster_keys` - (Optional) a subset of columns to liquid cluster the table by. Conflicts with `partitions`.
+* `partitions` - (Optional) a subset of columns to partition the table by. Change forces the creation of a new resource. Conflicts with `cluster_keys`.
 * `storage_credential_name` - (Optional) For EXTERNAL Tables only: the name of storage credential to use. Change forces the creation of a new resource.
-* `owner` - (Optional) User name/group name/sp application_id of the schema owner.
+* `owner` - (Optional) User name/group name/sp application_id of the table owner.
 * `comment` - (Optional) User-supplied free-form text. Changing the comment is not currently supported on the `VIEW` table type.
 * `options` - (Optional) Map of user defined table options. Change forces creation of a new resource.
 * `properties` - (Optional) A map of table properties.
-* `partitions` - (Optional) a subset of columns to partition the table by. Change forces the creation of a new resource. Conflicts with `cluster_keys`. Change forces creation of a new resource.
 
 ### `column` configuration block
 
@@ -191,7 +191,7 @@ In addition to all the arguments above, the following attributes are exported:
 
 ## Import
 
-This resource can be imported by its full name:
+This resource can be imported by its full name.
 
 ```bash
 terraform import databricks_sql_table.this <catalog_name>.<schema_name>.<name>
@@ -200,24 +200,26 @@ terraform import databricks_sql_table.this <catalog_name>.<schema_name>.<name>
 ## Migration from `databricks_table`
 
 The `databricks_table` resource has been deprecated in favor of `databricks_sql_table`. To migrate from `databricks_table` to `databricks_sql_table`:
+
 1. Define a `databricks_sql_table` resource with arguments corresponding to `databricks_table`.
 2. Add a `removed` block to remove the `databricks_table` resource without deleting the existing table by using the `lifecycle` block. If you're using Terraform version below v1.7.0, you will need to use the `terraform state rm` command instead.
 3. Add an `import` block to add the `databricks_sql_table` resource, corresponding to the existing table. If you're using Terraform version below v1.5.0, you will need to use `terraform import` command instead.
 
 For example, suppose we have the following `databricks_table` resource:
+
 ```hcl
 resource "databricks_table" "this" {
-  catalog_name = "catalog"
-  schema_name = "schema"
-  name = "table"
-  table_type = "MANAGED"
+  catalog_name       = "catalog"
+  schema_name        = "schema"
+  name               = "table"
+  table_type         = "MANAGED"
   data_source_format = "DELTA"
   column {
-    name = "col1"
+    name      = "col1"
     type_name = "STRING"
     type_json = "{\"type\":\"STRING\"}"
-    comment = "comment"
-    nullable = true
+    comment   = "comment"
+    nullable  = true
   }
   comment = "comment"
   properties = {
@@ -227,6 +229,7 @@ resource "databricks_table" "this" {
 ```
 
 The migration would look like this:
+
 ```hcl
 # Leave this resource definition as-is.
 resource "databricks_table" "this" { ... }