Added experimental authentication methods (#1660)

Author: nfx

common/client.go

@@ -42,6 +42,10 @@ type DatabricksClient struct {
 	Username string `name:"username" env:"DATABRICKS_USERNAME" auth:"password"`
 	Password string `name:"password" env:"DATABRICKS_PASSWORD" auth:"password,sensitive"`
 
+	ClientID     string `name:"client_id" env:"DATABRICKS_CLIENT_ID" auth:"oauth"`
+	ClientSecret string `name:"client_secret" env:"DATABRICKS_CLIENT_SECRET" auth:"oauth,sensitive"`
+	TokenEndpoint string `name:"token_endpoint" env:"DATABRICKS_TOKEN_ENDPOINT" auth:"oauth"`
+
 	// Databricks Account ID for Accounts API. This field is used in dependencies.
 	AccountID string `name:"account_id" env:"DATABRICKS_ACCOUNT_ID"`
 
@@ -251,6 +255,7 @@ func (c *DatabricksClient) Authenticate(ctx context.Context) error {
 	providers := []auth{
 		{c.configureWithPat, "pat"},
 		{c.configureWithBasicAuth, "basic"},
+		{c.configureWithOAuthM2M, "oauth-m2m"},
 		{c.configureWithAzureClientSecret, "azure-client-secret"},
 		{c.configureWithAzureManagedIdentity, "azure-msi"},
 		{c.configureWithAzureCLI, "azure-cli"},
@@ -526,6 +531,8 @@ func (c *DatabricksClient) ClientForHost(ctx context.Context, url string) (*Data
 		Username:             c.Username,
 		Password:             c.Password,
 		Token:                c.Token,
+		ClientID:             c.ClientID,
+		ClientSecret:         c.ClientSecret,
 		GoogleServiceAccount: c.GoogleServiceAccount,
 		GoogleCredentials:    c.GoogleCredentials,
 		AzurermEnvironment:   c.AzurermEnvironment,

common/client_test.go

@@ -154,7 +154,7 @@ func TestDatabricksClient_FormatURL(t *testing.T) {
 
 func TestClientAttributes(t *testing.T) {
 	ca := ClientAttributes()
-	assert.Len(t, ca, 22)
+	assert.Len(t, ca, 25)
 }
 
 func TestDatabricksClient_Authenticate(t *testing.T) {

common/gcp.go

@@ -122,10 +122,10 @@ func (c *DatabricksClient) configureWithGoogleForWorkspace(ctx context.Context)
 	if err != nil {
 		return nil, err
 	}
-	return newOidcAuthorizerForWorkspace(oidcSource), nil
+	return newOidcAuthorizerWithJustBearer(oidcSource), nil
 }
 
-func newOidcAuthorizerForWorkspace(oidcSource oauth2.TokenSource) func(r *http.Request) error {
+func newOidcAuthorizerWithJustBearer(oidcSource oauth2.TokenSource) func(r *http.Request) error {
 	return func(r *http.Request) error {
 		oidc, err := oidcSource.Token()
 		if err != nil {

common/gcp_test.go

@@ -162,7 +162,7 @@ func TestNewOidcAuthorizerForWorkspace(t *testing.T) {
 		AccessToken: "abc",
 		TokenType:   "Bearer",
 	}
-	auth := newOidcAuthorizerForWorkspace(
+	auth := newOidcAuthorizerWithJustBearer(
 		oauth2.StaticTokenSource(&token))
 	request := httptest.NewRequest("GET", "http://localhost", nil)
 	err := auth(request)

common/http.go

@@ -385,6 +385,10 @@ func (c *DatabricksClient) recursiveMask(requestMap map[string]any) any {
 			requestMap[k] = "**REDACTED**"
 			continue
 		}
+		if k == "secret" {
+			requestMap[k] = "**REDACTED**"
+			continue
+		}
 		if k == "content" {
 			requestMap[k] = "**REDACTED**"
 			continue

common/m2m.go

@@ -0,0 +1,74 @@
+package common
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/clientcredentials"
+)
+
+type oauthAuthorizationServer struct {
+	AuthorizationEndpoint string `json:"authorization_endpoint"`
+	TokenEndpoint         string `json:"token_endpoint"`
+}
+
+var errNotAvailable = errors.New("not available")
+
+func (c *DatabricksClient) getOAuthEndpoints() (*oauthAuthorizationServer, error) {
+	err := c.fixHost()
+	if err != nil {
+		return nil, fmt.Errorf("host: %w", err)
+	}
+	oidc := fmt.Sprintf("%s/oidc/.well-known/oauth-authorization-server", c.Host)
+	oidcResponse, err := http.Get(oidc)
+	if err != nil {
+		return nil, errNotAvailable
+	}
+	if oidcResponse.Body == nil {
+		return nil, fmt.Errorf("fetch .well-known: empty body")
+	}
+	defer oidcResponse.Body.Close()
+	raw, err := io.ReadAll(oidcResponse.Body)
+	if err != nil {
+		return nil, fmt.Errorf("read .well-known: %w", err)
+	}
+	var oauthEndpoints oauthAuthorizationServer
+	err = json.Unmarshal(raw, &oauthEndpoints)
+	if err != nil {
+		return nil, fmt.Errorf("parse .well-known: %w", err)
+	}
+	return &oauthEndpoints, nil
+}
+
+func (c *DatabricksClient) configureWithOAuthM2M(
+	ctx context.Context) (func(r *http.Request) error, error) {
+	if !c.IsAws() || c.ClientID == "" || c.ClientSecret == "" || c.Host == "" {
+		return nil, nil
+	}
+	// workaround for accounts endpoint not having yet a well-known OIDC alias
+	if c.TokenEndpoint == "" {
+		endpoints, err := c.getOAuthEndpoints()
+		if err == errNotAvailable {
+			return nil, nil
+		}
+		if err != nil {
+			return nil, fmt.Errorf("databricks oauth: %w", err)
+		}
+		c.TokenEndpoint = endpoints.TokenEndpoint
+	}
+	log.Printf("[INFO] Generating Databricks OAuth token for Service Principal (%s)", c.ClientID)
+	ts := (&clientcredentials.Config{
+		ClientID:     c.ClientID,
+		ClientSecret: c.ClientSecret,
+		AuthStyle:    oauth2.AuthStyleInHeader,
+		TokenURL:     c.TokenEndpoint,
+		Scopes:       []string{"all-apis"},
+	}).TokenSource(ctx)
+	return newOidcAuthorizerWithJustBearer(ts), nil
+}

common/m2m_test.go

@@ -0,0 +1,50 @@
+package common
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestConfigureWithOAuthM2M(t *testing.T) {
+	defer CleanupEnvironment()()
+	cnt := []int{0}
+	server := httptest.NewServer(http.HandlerFunc(
+		func(rw http.ResponseWriter, req *http.Request) {
+			if req.RequestURI ==
+				"/oidc/.well-known/oauth-authorization-server" {
+				_, err := rw.Write([]byte(
+					`{"token_endpoint": "http://localhost/oauth/token"}`))
+				assert.NoError(t, err)
+				cnt[0]++
+				return
+			}
+			assert.Fail(t, fmt.Sprintf("Received unexpected call: %s %s",
+				req.Method, req.RequestURI))
+		}))
+	defer server.Close()
+
+	c := &DatabricksClient{
+		Host:         server.URL,
+		ClientID:     "abc",
+		ClientSecret: "bcd",
+	}
+	_, err := c.configureWithOAuthM2M(context.Background())
+	assert.NoError(t, err)
+	assert.Equal(t, 1, cnt[0])
+}
+
+func TestConfigureWithOAuthOIDCUnavailableSkips(t *testing.T) {
+	defer CleanupEnvironment()()
+	c := &DatabricksClient{
+		Host:         "http://localhost:22",
+		ClientID:     "abc",
+		ClientSecret: "bcd",
+	}
+	_, err := c.configureWithOAuthM2M(context.Background())
+	assert.NoError(t, err)
+}

provider/provider.go

@@ -116,6 +116,7 @@ func DatabricksProvider() *schema.Provider {
 			"databricks_secret_acl":                  secrets.ResourceSecretACL(),
 			"databricks_service_principal":           scim.ResourceServicePrincipal(),
 			"databricks_service_principal_role":      aws.ResourceServicePrincipalRole(),
+			"databricks_service_principal_secret":    tokens.ResourceServicePrincipalSecret(),
 			"databricks_sql_dashboard":               sql.ResourceSqlDashboard(),
 			"databricks_sql_endpoint":                sql.ResourceSqlEndpoint(),
 			"databricks_sql_global_config":           sql.ResourceSqlGlobalConfig(),

tokens/resource_service_principal_secret.go

@@ -0,0 +1,121 @@
+package tokens
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/databricks/terraform-provider-databricks/common"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+)
+
+type ServicePrincipalSecret struct {
+	ID     string `json:"id,omitempty"`
+	Secret string `json:"secret,omitempty" tf:"computed,sensitive"`
+	Status string `json:"status,omitempty" tf:"computed"`
+}
+
+type ListServicePrincipalSecrets struct {
+	Secrets []ServicePrincipalSecret `json:"secrets"`
+}
+
+// NewServicePrincipalSecretAPI creates ServicePrincipalSecretAPI instance from provider meta
+func NewServicePrincipalSecretAPI(ctx context.Context, m any) ServicePrincipalSecretAPI {
+	return ServicePrincipalSecretAPI{m.(*common.DatabricksClient), ctx}
+}
+
+// ServicePrincipalSecretAPI exposes the API to create client secrets
+type ServicePrincipalSecretAPI struct {
+	client  *common.DatabricksClient
+	context context.Context
+}
+
+func (a ServicePrincipalSecretAPI) createServicePrincipalSecret(spnID string) (secret *ServicePrincipalSecret, err error) {
+	path := fmt.Sprintf("/accounts/%s/servicePrincipals/%s/credentials/secrets", a.client.AccountID, spnID)
+	err = a.client.Post(a.context, path, map[string]any{}, &secret)
+	return
+}
+
+func (a ServicePrincipalSecretAPI) listServicePrincipalSecrets(spnID string) (secrets ListServicePrincipalSecrets, err error) {
+	path := fmt.Sprintf("/accounts/%s/servicePrincipals/%s/credentials/secrets", a.client.AccountID, spnID)
+	err = a.client.Get(a.context, path, nil, &secrets)
+	return
+}
+
+func (a ServicePrincipalSecretAPI) deleteServicePrincipalSecret(spnID, secretID string) error { // FIXME
+	path := fmt.Sprintf("/accounts/%s/servicePrincipals/%s/credentials/secrets/%s", a.client.AccountID, spnID, secretID)
+	return a.client.Delete(a.context, path, nil)
+}
+
+func ResourceServicePrincipalSecret() *schema.Resource {
+	spnSecretSchema := common.StructToSchema(ServicePrincipalSecret{},
+		func(m map[string]*schema.Schema) map[string]*schema.Schema {
+			m["id"].Computed = true
+			m["service_principal_id"] = &schema.Schema{
+				Type:     schema.TypeString,
+				ForceNew: true,
+				Required: true,
+			}
+			return m
+		})
+	return common.Resource{
+		Schema: spnSecretSchema,
+		Create: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			if c.AccountID == "" {
+				return errors.New("must have `account_id` on provider")
+			}
+			idSeen := map[string]bool{}
+			api := NewServicePrincipalSecretAPI(ctx, c)
+			spnID := d.Get("service_principal_id").(string)
+			secrets, err := api.listServicePrincipalSecrets(spnID)
+			if err != nil {
+				return err
+			}
+			for _, v := range secrets.Secrets {
+				idSeen[v.ID] = true
+			}
+			secret, err := api.createServicePrincipalSecret(spnID)
+			if err != nil {
+				return err
+			}
+			secrets, err = api.listServicePrincipalSecrets(spnID)
+			if err != nil {
+				return err
+			}
+			// ugly hack because rpc does not return ID of created secret
+			for _, v := range secrets.Secrets {
+				if len(idSeen) > 0 && idSeen[v.ID] {
+					continue
+				}
+				d.SetId(v.ID)
+			}
+			return d.Set("secret", secret.Secret)
+		},
+		Read: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			if c.AccountID == "" {
+				return errors.New("must have `account_id` on provider")
+			}
+			api := NewServicePrincipalSecretAPI(ctx, c)
+			spnID := d.Get("service_principal_id").(string)
+			secrets, err := api.listServicePrincipalSecrets(spnID)
+			if err != nil {
+				return err
+			}
+			for _, v := range secrets.Secrets {
+				if v.ID != d.Id() {
+					continue
+				}
+				return d.Set("status", v.Status)
+			}
+			return common.NotFound("client secret not found")
+		},
+		Delete: func(ctx context.Context, d *schema.ResourceData, c *common.DatabricksClient) error {
+			if c.AccountID == "" {
+				return errors.New("must have `account_id` on provider")
+			}
+			api := NewServicePrincipalSecretAPI(ctx, c)
+			spnID := d.Get("service_principal_id").(string)
+			return api.deleteServicePrincipalSecret(spnID, d.Id())
+		},
+	}.ToResource()
+}