Use user provided scopes to all OAuth authentication flow. (#495)

## What changes are proposed in this pull request?

This PR updates OAuth related flows to use the scopes user provided
scopes if any, or default to `all-apis` otherwize.

## How is this tested?

Unit and integration tests.

Author: renaudhartert-db

NEXT_CHANGELOG.md

@@ -6,6 +6,9 @@
 
 ### Bug Fixes
 
+* User provided scopes are now properly propagated in OAuth flows.
+* [Warning] Correctly defaults to scope `all-apis` (instead of `clusters sql`) in U2M if no scopes are provided by the users. This change aligns the Java SDK logic with the Python and Go SDK logic.
+
 ### Documentation
 
 ### Internal Changes

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java

@@ -351,7 +351,16 @@ public DatabricksConfig setOAuthRedirectUrl(String redirectUrl) {
     return this;
   }
 
+  /**
+   * Returns the scopes to use for the current configuration. If no scopes were provided, returns
+   * the default "all-apis" scope.
+   *
+   * @return The scopes to use for the current configuration
+   */
   public List<String> getScopes() {
+    if (scopes == null || scopes.isEmpty()) {
+      return Arrays.asList("all-apis");
+    }
     return scopes;
   }
 

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DefaultCredentialsProvider.java

@@ -139,6 +139,7 @@ private void addOIDCCredentialsProviders(DatabricksConfig config) {
                   config.getHttpClient())
               .audience(config.getTokenAudience())
               .accountId(config.isAccountClient() ? config.getAccountId() : null)
+              .scopes(config.getScopes())
               .build();
 
       providers.add(new TokenSourceCredentialsProvider(oauthTokenSource, namedIdTokenSource.name));

databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/DatabricksOAuthTokenSource.java

@@ -4,7 +4,9 @@
 import com.databricks.sdk.core.http.HttpClient;
 import com.google.common.base.Strings;
 import java.time.Instant;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import org.slf4j.Logger;
@@ -31,10 +33,11 @@ public class DatabricksOAuthTokenSource implements TokenSource {
   private final IDTokenSource idTokenSource;
   /** HTTP client for making token exchange requests. */
   private final HttpClient httpClient;
+  /** Scopes to request during token exchange. */
+  private final List<String> scopes;
 
   private static final String GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange";
   private static final String SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:jwt";
-  private static final String SCOPE = "all-apis";
   private static final String GRANT_TYPE_PARAM = "grant_type";
   private static final String SUBJECT_TOKEN_PARAM = "subject_token";
   private static final String SUBJECT_TOKEN_TYPE_PARAM = "subject_token_type";
@@ -49,6 +52,7 @@ private DatabricksOAuthTokenSource(Builder builder) {
     this.audience = builder.audience;
     this.idTokenSource = builder.idTokenSource;
     this.httpClient = builder.httpClient;
+    this.scopes = builder.scopes == null ? Arrays.asList() : builder.scopes;
   }
 
   /**
@@ -63,6 +67,7 @@ public static class Builder {
     private final HttpClient httpClient;
     private String accountId;
     private String audience;
+    private List<String> scopes;
 
     /**
      * Creates a new Builder with required parameters.
@@ -108,6 +113,17 @@ public Builder audience(String audience) {
       return this;
     }
 
+    /**
+     * Sets the scopes to request during token exchange.
+     *
+     * @param scopes The scopes to request.
+     * @return This builder instance.
+     */
+    public Builder scopes(List<String> scopes) {
+      this.scopes = scopes;
+      return this;
+    }
+
     /**
      * Builds a new DatabricksOAuthTokenSource instance.
      *
@@ -149,7 +165,7 @@ public Token getToken() {
     params.put(GRANT_TYPE_PARAM, GRANT_TYPE);
     params.put(SUBJECT_TOKEN_PARAM, idToken.getValue());
     params.put(SUBJECT_TOKEN_TYPE_PARAM, SUBJECT_TOKEN_TYPE);
-    params.put(SCOPE_PARAM, SCOPE);
+    params.put(SCOPE_PARAM, String.join(" ", scopes));
     params.put(CLIENT_ID_PARAM, clientId);
 
     OAuthResponse response;

databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProvider.java

@@ -5,8 +5,11 @@
 import com.databricks.sdk.core.DatabricksException;
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -106,6 +109,17 @@ CachedTokenSource performBrowserAuth(
       DatabricksConfig config, String clientId, String clientSecret, TokenCache tokenCache)
       throws IOException {
     LOGGER.debug("Performing browser authentication");
+
+    // Get user-provided scopes and add required default scopes.
+    Set<String> scopes = new HashSet<>(config.getScopes());
+
+    // Needed to request a refresh token.
+    scopes.add("offline_access");
+
+    if (config.isAzure()) {
+      scopes.add(config.getEffectiveAzureLoginAppId() + "/user_impersonation");
+    }
+
     OAuthClient client =
         new OAuthClient.Builder()
             .withHttpClient(config.getHttpClient())
@@ -114,7 +128,7 @@ CachedTokenSource performBrowserAuth(
             .withHost(config.getHost())
             .withAccountId(config.getAccountId())
             .withRedirectUrl(config.getEffectiveOAuthRedirectUrl())
-            .withScopes(config.getScopes())
+            .withScopes(new ArrayList<>(scopes))
             .build();
     Consent consent = client.initiateConsent();
 

databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/GithubOidcCredentialsProvider.java

@@ -6,7 +6,6 @@
 import com.databricks.sdk.core.HeaderFactory;
 import com.google.common.collect.ImmutableMap;
 import java.io.IOException;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -47,7 +46,7 @@ public HeaderFactory configure(DatabricksConfig config) throws DatabricksExcepti
             .withHttpClient(config.getHttpClient())
             .withClientId(config.getClientId())
             .withTokenUrl(endpointUrl)
-            .withScopes(Collections.singletonList("all-apis"))
+            .withScopes(config.getScopes())
             .withAuthParameterPosition(AuthParameterPosition.HEADER)
             .withEndpointParametersSupplier(
                 () ->

databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/OAuthClient.java

@@ -109,17 +109,7 @@ private OAuthClient(Builder b) throws IOException {
     this.isAzure = config.isAzure();
     this.tokenUrl = oidc.getTokenEndpoint();
     this.authUrl = oidc.getAuthorizationEndpoint();
-
-    List<String> scopes = b.scopes;
-    if (scopes == null) {
-      scopes = Arrays.asList("offline_access", "clusters", "sql");
-    }
-    if (config.isAzure()) {
-      scopes =
-          Arrays.asList(
-              config.getEffectiveAzureLoginAppId() + "/user_impersonation", "offline_access");
-    }
-    this.scopes = scopes;
+    this.scopes = b.scopes;
   }
 
   public String getHost() {

databricks-sdk-java/src/main/java/com/databricks/sdk/core/oauth/OAuthM2MServicePrincipalCredentialsProvider.java

@@ -2,7 +2,6 @@
 
 import com.databricks.sdk.core.*;
 import java.io.IOException;
-import java.util.Collections;
 
 /**
  * Adds refreshed Databricks machine-to-machine OAuth Bearer token to every request, if
@@ -33,7 +32,7 @@ public OAuthHeaderFactory configure(DatabricksConfig config) {
               .withClientId(config.getClientId())
               .withClientSecret(config.getClientSecret())
               .withTokenUrl(jsonResponse.getTokenEndpoint())
-              .withScopes(Collections.singletonList("all-apis"))
+              .withScopes(config.getScopes())
               .withAuthParameterPosition(AuthParameterPosition.HEADER)
               .build();
 

databricks-sdk-java/src/test/java/com/databricks/sdk/core/oauth/ExternalBrowserCredentialsProviderTest.java

@@ -62,7 +62,7 @@ void clientAndConsentTest() throws IOException {
       assertTrue(authUrl.contains("response_type=code"));
       assertTrue(authUrl.contains("client_id=test-client-id"));
       assertTrue(authUrl.contains("redirect_uri=http://localhost:8080/callback"));
-      assertTrue(authUrl.contains("scope=offline_access%20clusters%20sql"));
+      assertTrue(authUrl.contains("scope=all-apis"));
     }
   }