direct: Support permissions (#3724)

## Changes
Implement permissions support for all resources that have it.

Note, $resources references in permissions are not supported yet.

## Tests
New and existing acceptance tests.

Author: denik

acceptance/bundle/deploy/mlops-stacks/out.test.toml

@@ -7,6 +7,14 @@ Ignore = [
     "config.json"
 ]
 
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform"]
+# On direct, this fails with
+#        +Endpoint: PUT [DATABRICKS_URL]/api/2.0/permissions/registered-models/%5Bdev%20[USERNAME]%5D%20dev-project_name_[UNIQUE_NAME]-model
+#        +HTTP Status: 400 Bad Request
+#        +API error_code: INVALID_PARAMETER_VALUE
+#        +API message: '[dev [USERNAME]] dev-project_name_[UNIQUE_NAME]-model' is not a valid registered model ID.
+
+
 [[Repls]]
 Old = "aws|azure|gcp"
 New = "[CLOUD_ENV_BASE]"

acceptance/bundle/deploy/mlops-stacks/test.toml

@@ -40,7 +40,7 @@ resources.alerts.*.parent_path	string	ALL
 resources.alerts.*.permissions	[]resources.AlertPermission	INPUT
 resources.alerts.*.permissions[*]	resources.AlertPermission	INPUT
 resources.alerts.*.permissions[*].group_name	string	INPUT
-resources.alerts.*.permissions[*].level	string	INPUT
+resources.alerts.*.permissions[*].level	resources.AlertPermissionLevel	INPUT
 resources.alerts.*.permissions[*].service_principal_name	string	INPUT
 resources.alerts.*.permissions[*].user_name	string	INPUT
 resources.alerts.*.query_text	string	ALL
@@ -145,6 +145,13 @@ resources.apps.*.updater	string	ALL
 resources.apps.*.url	string	ALL
 resources.apps.*.user_api_scopes	[]string	ALL
 resources.apps.*.user_api_scopes[*]	string	ALL
+resources.apps.*.permissions.object_id	string	ALL
+resources.apps.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.apps.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.apps.*.permissions.permissions[*].group_name	string	ALL
+resources.apps.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.apps.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.apps.*.permissions.permissions[*].user_name	string	ALL
 resources.clusters.*.apply_policy_default_values	bool	INPUT	STATE
 resources.clusters.*.autoscale	*compute.AutoScale	ALL
 resources.clusters.*.autoscale.max_workers	int	ALL
@@ -401,6 +408,13 @@ resources.clusters.*.workload_type	*compute.WorkloadType	ALL
 resources.clusters.*.workload_type.clients	compute.ClientsTypes	ALL
 resources.clusters.*.workload_type.clients.jobs	bool	ALL
 resources.clusters.*.workload_type.clients.notebooks	bool	ALL
+resources.clusters.*.permissions.object_id	string	ALL
+resources.clusters.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.clusters.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.clusters.*.permissions.permissions[*].group_name	string	ALL
+resources.clusters.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.clusters.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.clusters.*.permissions.permissions[*].user_name	string	ALL
 resources.database_catalogs.*.create_database_if_not_exists	bool	ALL
 resources.database_catalogs.*.database_instance_name	string	ALL
 resources.database_catalogs.*.database_name	string	ALL
@@ -465,6 +479,13 @@ resources.database_instances.*.stopped	bool	ALL
 resources.database_instances.*.uid	string	ALL
 resources.database_instances.*.url	string	INPUT
 resources.database_instances.*.usage_policy_id	string	ALL
+resources.database_instances.*.permissions.object_id	string	ALL
+resources.database_instances.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.database_instances.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.database_instances.*.permissions.permissions[*].group_name	string	ALL
+resources.database_instances.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.database_instances.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.database_instances.*.permissions.permissions[*].user_name	string	ALL
 resources.experiments.*.artifact_location	string	ALL
 resources.experiments.*.creation_time	int64	REMOTE
 resources.experiments.*.experiment_id	string	REMOTE
@@ -486,6 +507,13 @@ resources.experiments.*.tags[*]	ml.ExperimentTag	ALL
 resources.experiments.*.tags[*].key	string	ALL
 resources.experiments.*.tags[*].value	string	ALL
 resources.experiments.*.url	string	INPUT
+resources.experiments.*.permissions.object_id	string	ALL
+resources.experiments.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.experiments.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.experiments.*.permissions.permissions[*].group_name	string	ALL
+resources.experiments.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.experiments.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.experiments.*.permissions.permissions[*].user_name	string	ALL
 resources.jobs.*.budget_policy_id	string	INPUT	STATE
 resources.jobs.*.continuous	*jobs.Continuous	INPUT	STATE
 resources.jobs.*.continuous.pause_status	jobs.PauseStatus	INPUT	STATE
@@ -2205,6 +2233,13 @@ resources.jobs.*.webhook_notifications.on_streaming_backlog_exceeded[*].id	strin
 resources.jobs.*.webhook_notifications.on_success	[]jobs.Webhook	INPUT	STATE
 resources.jobs.*.webhook_notifications.on_success[*]	jobs.Webhook	INPUT	STATE
 resources.jobs.*.webhook_notifications.on_success[*].id	string	INPUT	STATE
+resources.jobs.*.permissions.object_id	string	ALL
+resources.jobs.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.jobs.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.jobs.*.permissions.permissions[*].group_name	string	ALL
+resources.jobs.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.jobs.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.jobs.*.permissions.permissions[*].user_name	string	ALL
 resources.models.*.creation_timestamp	int64	REMOTE
 resources.models.*.description	string	ALL
 resources.models.*.id	string	INPUT	REMOTE
@@ -2244,6 +2279,13 @@ resources.models.*.tags[*].key	string	ALL
 resources.models.*.tags[*].value	string	ALL
 resources.models.*.url	string	INPUT
 resources.models.*.user_id	string	REMOTE
+resources.models.*.permissions.object_id	string	ALL
+resources.models.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.models.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.models.*.permissions.permissions[*].group_name	string	ALL
+resources.models.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.models.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.models.*.permissions.permissions[*].user_name	string	ALL
 resources.pipelines.*.allow_duplicate_names	bool	INPUT	STATE
 resources.pipelines.*.budget_policy_id	string	INPUT	STATE
 resources.pipelines.*.catalog	string	INPUT	STATE
@@ -2838,6 +2880,13 @@ resources.pipelines.*.trigger.cron.quartz_cron_schedule	string	INPUT	STATE
 resources.pipelines.*.trigger.cron.timezone_id	string	INPUT	STATE
 resources.pipelines.*.trigger.manual	*pipelines.ManualTrigger	INPUT	STATE
 resources.pipelines.*.url	string	INPUT
+resources.pipelines.*.permissions.object_id	string	ALL
+resources.pipelines.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.pipelines.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.pipelines.*.permissions.permissions[*].group_name	string	ALL
+resources.pipelines.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.pipelines.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.pipelines.*.permissions.permissions[*].user_name	string	ALL
 resources.registered_models.*.aliases	[]catalog.RegisteredModelAlias	ALL
 resources.registered_models.*.aliases[*]	catalog.RegisteredModelAlias	ALL
 resources.registered_models.*.aliases[*].alias_name	string	ALL
@@ -2951,6 +3000,13 @@ resources.sql_warehouses.*.tags.custom_tags[*].value	string	ALL
 resources.sql_warehouses.*.url	string	INPUT
 resources.sql_warehouses.*.warehouse_type	sql.CreateWarehouseRequestWarehouseType	INPUT	STATE
 resources.sql_warehouses.*.warehouse_type	sql.GetWarehouseResponseWarehouseType	REMOTE
+resources.sql_warehouses.*.permissions.object_id	string	ALL
+resources.sql_warehouses.*.permissions.permissions	[]iam.AccessControlRequest	ALL
+resources.sql_warehouses.*.permissions.permissions[*]	iam.AccessControlRequest	ALL
+resources.sql_warehouses.*.permissions.permissions[*].group_name	string	ALL
+resources.sql_warehouses.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
+resources.sql_warehouses.*.permissions.permissions[*].service_principal_name	string	ALL
+resources.sql_warehouses.*.permissions.permissions[*].user_name	string	ALL
 resources.synced_database_tables.*.data_synchronization_status	*database.SyncedTableStatus	ALL
 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status	*database.SyncedTableContinuousUpdateStatus	ALL
 resources.synced_database_tables.*.data_synchronization_status.continuous_update_status.initial_pipeline_sync_progress	*database.SyncedTablePipelineProgress	ALL

acceptance/bundle/refschema/out.fields.txt

@@ -1,8 +1,2 @@
 Cloud = false
 Local = false
-
-# Permissions with alerts does not work with direct deployment yet.
-# So we skip this test on direct-exp until we have permissions support in
-# direct deployment.
-[EnvMatrix]
-DATABRICKS_BUNDLE_ENGINE = ["terraform"]

acceptance/bundle/resources/alerts/basic/test.toml

@@ -5,7 +5,3 @@ RecordRequests = false
 [[Repls]]
 Old = '\d{3,}'
 New = "[NUMID]"
-
-# Test both terraform and direct deployment engines
-[EnvMatrix]
-DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct-exp"]

acceptance/bundle/resources/experiments/basic/test.toml

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""
+Analyze all requests recorded in subtests to highlight differences between direct and terraform.
+"""
+
+import os
+import re
+import json
+import sys
+from pathlib import Path
+from difflib import unified_diff
+
+
+def read_json_many(file):
+    with open(file) as fobj:
+        s = fobj.read()
+
+    # fix invalid json due to replacement: x: [NUMID]  -->  x: "[NUMID]"
+    s = re.sub(r": \[(.*?)\]", r': "[\1]"', s)
+
+    result = []
+
+    try:
+        dec = json.JSONDecoder()
+        pos = 0
+        n = len(s)
+        while True:
+            # skip whitespace between objects
+            while pos < n and s[pos].isspace():
+                pos += 1
+            if pos >= n:
+                break
+            obj, idx = dec.raw_decode(s, pos)
+            result.append(obj)
+            pos = idx
+
+    except Exception as ex:
+        sys.exit(f"Failed to parse {file}: {ex}")
+
+    return result
+
+
+def normalize_acls(data):
+    """Recursively normalize ACLs in the data structure by sorting them."""
+    if isinstance(data, dict):
+        result = {}
+        for key, value in data.items():
+            if key == "access_control_list" and isinstance(value, list):
+                # Sort ACLs by all fields to normalize order
+                result[key] = sorted(value, key=lambda x: json.dumps(x, sort_keys=True))
+            else:
+                result[key] = normalize_acls(value)
+        return result
+    elif isinstance(data, list):
+        return [normalize_acls(item) for item in data]
+    else:
+        return data
+
+
+def compare_files(file1, file2):
+    """Compare two JSON files and return comparison result."""
+    data1 = read_json_many(file1)
+    data2 = read_json_many(file2)
+
+    if data1 == data2:
+        return "EXACT", ""
+
+    normalized1 = normalize_acls(data1)
+    normalized2 = normalize_acls(data2)
+
+    if normalized1 == normalized2:
+        return "MATCH", ""
+
+    json1_str = json.dumps(normalized1, indent=2, sort_keys=True)
+    json2_str = json.dumps(normalized2, indent=2, sort_keys=True)
+
+    diff_lines = list(
+        unified_diff(
+            json1_str.splitlines(keepends=True),
+            json2_str.splitlines(keepends=True),
+            fromfile=str(file1),
+            tofile=str(file2),
+            n=3,
+        )
+    )
+
+    return "DIFF ", "\n" + to_slash("".join(diff_lines).rstrip())
+
+
+def to_slash(x):
+    return str(x).replace("\\", "/")
+
+
+def main():
+    current_dir = Path(".")
+
+    direct_files = list(current_dir.glob("**/*.direct-exp.json"))
+
+    for direct_file in sorted(direct_files):
+        if direct_file.name.startswith("out.plan"):
+            # expected difference
+            continue
+
+        terraform_file = direct_file.parent / direct_file.name.replace(".direct-exp.", ".terraform.")
+
+        fname = to_slash(direct_file)
+
+        if terraform_file.exists():
+            result, diff = compare_files(direct_file, terraform_file)
+            print(result + " " + fname + diff)
+        else:
+            print(f"ERROR {fname}: Missing terraform file {to_slash(terraform_file)}")
+
+
+if __name__ == "__main__":
+    main()

acceptance/bundle/resources/permissions/analyze_requests.py

@@ -8,6 +8,41 @@
           "name": "foo"
         }
       }
+    },
+    "resources.apps.foo.permissions": {
+      "depends_on": [
+        {
+          "node": "resources.apps.foo",
+          "label": "${resources.apps.foo.id}"
+        }
+      ],
+      "action": "create",
+      "new_state": {
+        "config": {
+          "object_id": "",
+          "permissions": [
+            {
+              "permission_level": "CAN_USE",
+              "user_name": "[email protected]"
+            },
+            {
+              "group_name": "data-team",
+              "permission_level": "CAN_MANAGE"
+            },
+            {
+              "permission_level": "CAN_MANAGE",
+              "service_principal_name": "[UUID]"
+            },
+            {
+              "permission_level": "CAN_MANAGE",
+              "user_name": "[USERNAME]"
+            }
+          ]
+        },
+        "vars": {
+          "object_id": "/apps/${resources.apps.foo.id}"
+        }
+      }
     }
   }
 }

acceptance/bundle/resources/permissions/apps/current_can_manage/out.plan.direct-exp.json

@@ -0,0 +1,24 @@
+{
+  "method": "PUT",
+  "path": "/api/2.0/permissions/apps/foo",
+  "body": {
+    "access_control_list": [
+      {
+        "permission_level": "CAN_USE",
+        "user_name": "[email protected]"
+      },
+      {
+        "group_name": "data-team",
+        "permission_level": "CAN_MANAGE"
+      },
+      {
+        "permission_level": "CAN_MANAGE",
+        "service_principal_name": "[UUID]"
+      },
+      {
+        "permission_level": "CAN_MANAGE",
+        "user_name": "[USERNAME]"
+      }
+    ]
+  }
+}

acceptance/bundle/resources/permissions/apps/current_can_manage/out.requests.deploy.direct-exp.json

@@ -8,6 +8,41 @@
           "name": "foo"
         }
       }
+    },
+    "resources.apps.foo.permissions": {
+      "depends_on": [
+        {
+          "node": "resources.apps.foo",
+          "label": "${resources.apps.foo.id}"
+        }
+      ],
+      "action": "create",
+      "new_state": {
+        "config": {
+          "object_id": "",
+          "permissions": [
+            {
+              "permission_level": "CAN_USE",
+              "user_name": "[email protected]"
+            },
+            {
+              "group_name": "data-team",
+              "permission_level": "CAN_MANAGE"
+            },
+            {
+              "permission_level": "CAN_MANAGE",
+              "service_principal_name": "[UUID]"
+            },
+            {
+              "permission_level": "CAN_MANAGE",
+              "user_name": "[USERNAME]"
+            }
+          ]
+        },
+        "vars": {
+          "object_id": "/apps/${resources.apps.foo.id}"
+        }
+      }
     }
   }
 }