Add depends_on chain for secret scope ACLs to fix flaky permission removal

The secret scope permissions test fails ~33% of the time when using
the Terraform bundle engine. The root cause is a backend API race
condition where parallel ACL modifications return inconsistent results.

The direct bundle engine already works around this by sequentializing
ACL operations (see #3886):

    // Set ACLs. The service returns inconsistent results for parallel
    // API calls. That's why we do them sequentially here to maintain
    // correctness.

The Terraform provider has a similar workaround for creates via
robustPutACL with retry/verification (terraform-provider-databricks#4885,
issue #4195), but deletions have no such protection.

This change adds a depends_on chain between ACL resources, forcing
Terraform to execute them sequentially:

    ACL_0 → depends_on: [scope]
    ACL_1 → depends_on: [scope, ACL_0]
    ACL_2 → depends_on: [scope, ACL_1]

This avoids triggering the backend race condition without requiring
changes to the Terraform provider.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>

Author: pietern

bundle/deploy/terraform/tfdyn/convert_secret_scope.go

@@ -19,6 +19,8 @@ type resourceSecretAcl struct {
 type secretScopeConverter struct{}
 
 func convertPermissionsSecretScope(key, scopeName string, permissions []dyn.Value, out *schema.Resources) {
+	var previousAclKey string
+
 	for idx, permission := range permissions {
 		level, _ := permission.Get("level").AsString()
 		userName, _ := permission.Get("user_name").AsString()
@@ -34,17 +36,27 @@ func convertPermissionsSecretScope(key, scopeName string, permissions []dyn.Valu
 			principal = servicePrincipalName
 		}
 
+		// Build depends_on list - always depend on scope, plus previous ACL if exists.
+		// This forces Terraform to execute ACL operations sequentially, avoiding
+		// a backend race condition where parallel ACL modifications return inconsistent results.
+		dependsOn := []string{"databricks_secret_scope." + key}
+		if previousAclKey != "" {
+			dependsOn = append(dependsOn, "databricks_secret_acl."+previousAclKey)
+		}
+
 		acl := &resourceSecretAcl{
 			ResourceSecretAcl: schema.ResourceSecretAcl{
 				Permission: level,
 				Principal:  principal,
 				Scope:      scopeName,
 			},
-			DependsOn: []string{"databricks_secret_scope." + key},
+			DependsOn: dependsOn,
 		}
 
 		aclKey := fmt.Sprintf("secret_acl_%s_%d", key, idx)
 		out.SecretAcl[aclKey] = acl
+
+		previousAclKey = aclKey
 	}
 }
 

bundle/deploy/terraform/tfdyn/convert_secret_scope_test.go

@@ -0,0 +1,102 @@
+package tfdyn
+
+import (
+	"context"
+	"testing"
+
+	"github.com/databricks/cli/bundle/config/resources"
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConvertSecretScopeWithPermissions(t *testing.T) {
+	src := resources.SecretScope{
+		Name: "my_scope",
+		Permissions: []resources.SecretScopePermission{
+			{UserName: "[email protected]", Level: resources.SecretScopePermissionLevelWrite},
+			{GroupName: "data-team", Level: resources.SecretScopePermissionLevelRead},
+			{ServicePrincipalName: "sp-uuid", Level: resources.SecretScopePermissionLevelManage},
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = secretScopeConverter{}.Convert(ctx, "my_scope", vin, out)
+	require.NoError(t, err)
+
+	// Verify ACL count
+	assert.Len(t, out.SecretAcl, 3)
+
+	// Verify first ACL depends only on scope
+	acl0 := out.SecretAcl["secret_acl_my_scope_0"].(*resourceSecretAcl)
+	assert.Equal(t, "[email protected]", acl0.Principal)
+	assert.Equal(t, "WRITE", acl0.Permission)
+	assert.Equal(t, []string{"databricks_secret_scope.my_scope"}, acl0.DependsOn)
+
+	// Verify second ACL depends on scope + first ACL (sequential execution)
+	acl1 := out.SecretAcl["secret_acl_my_scope_1"].(*resourceSecretAcl)
+	assert.Equal(t, "data-team", acl1.Principal)
+	assert.Equal(t, "READ", acl1.Permission)
+	assert.Equal(t, []string{
+		"databricks_secret_scope.my_scope",
+		"databricks_secret_acl.secret_acl_my_scope_0",
+	}, acl1.DependsOn)
+
+	// Verify third ACL depends on scope + second ACL (sequential execution)
+	acl2 := out.SecretAcl["secret_acl_my_scope_2"].(*resourceSecretAcl)
+	assert.Equal(t, "sp-uuid", acl2.Principal)
+	assert.Equal(t, "MANAGE", acl2.Permission)
+	assert.Equal(t, []string{
+		"databricks_secret_scope.my_scope",
+		"databricks_secret_acl.secret_acl_my_scope_1",
+	}, acl2.DependsOn)
+}
+
+func TestConvertSecretScopeSinglePermission(t *testing.T) {
+	src := resources.SecretScope{
+		Name: "single_scope",
+		Permissions: []resources.SecretScopePermission{
+			{UserName: "[email protected]", Level: resources.SecretScopePermissionLevelManage},
+		},
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = secretScopeConverter{}.Convert(ctx, "single_scope", vin, out)
+	require.NoError(t, err)
+
+	// Single ACL should only depend on scope (no chaining needed)
+	assert.Len(t, out.SecretAcl, 1)
+	acl := out.SecretAcl["secret_acl_single_scope_0"].(*resourceSecretAcl)
+	assert.Equal(t, "[email protected]", acl.Principal)
+	assert.Equal(t, []string{"databricks_secret_scope.single_scope"}, acl.DependsOn)
+}
+
+func TestConvertSecretScopeNoPermissions(t *testing.T) {
+	src := resources.SecretScope{
+		Name: "no_permissions_scope",
+	}
+
+	vin, err := convert.FromTyped(src, dyn.NilValue)
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	out := schema.NewResources()
+	err = secretScopeConverter{}.Convert(ctx, "no_permissions_scope", vin, out)
+	require.NoError(t, err)
+
+	// No ACLs should be created
+	assert.Len(t, out.SecretAcl, 0)
+
+	// But the scope should still be created
+	assert.Contains(t, out.SecretScope, "no_permissions_scope")
+}