[PECO-1542] Add a way to provide proxy details to SDK (#271)

## Changes
<!-- Summary of your changes that are easy to understand -->
Duplicate of #242. Re-created to sign commits. (This PR adds a way to
provide proxy configs in the SDK directly, ability to pick up the system
properties and authentication for proxy via basic or negotiate-kerberos
schemes)

## Tests
<!-- How is this tested? -->
Documented in #242 (These changes were tested in an environment
consisting of a proxy server with kerberos authentication hosted on
Ubuntu and Windows VMs. The detailed testing details are documented
here:
https://docs.google.com/document/d/1Zxlfx-R_JytFqMZfQMBfSQ8mj4W-aHa60vw4KdjUkEc/edit)

Author: vikrantpuppala

databricks-sdk-java/src/main/java/com/databricks/sdk/core/ConfigAttributeAccessor.java

@@ -47,6 +47,10 @@ public void setValueOnConfig(DatabricksConfig cfg, String value) throws IllegalA
         field.set(cfg, Integer.parseInt(value));
       } else if (field.getType() == boolean.class) {
         field.set(cfg, Boolean.parseBoolean(value));
+      } else if (field.getType() == ProxyConfig.ProxyAuthType.class) {
+        if (value != null) {
+          field.set(cfg, ProxyConfig.ProxyAuthType.valueOf(value));
+        }
       }
       field.setAccessible(false);
     }

databricks-sdk-java/src/main/java/com/databricks/sdk/core/DatabricksConfig.java

@@ -114,6 +114,24 @@ public class DatabricksConfig {
   @ConfigAttribute(env = "DATABRICKS_RATE_LIMIT")
   private Integer rateLimit;
 
+  @ConfigAttribute(env = "PROXY_HOST")
+  private String proxyHost;
+
+  @ConfigAttribute(env = "PROXY_PORT")
+  private Integer proxyPort;
+
+  @ConfigAttribute(env = "PROXY_USERNAME")
+  private String proxyUsername;
+
+  @ConfigAttribute(env = "PROXY_PASSWORD")
+  private String proxyPassword;
+
+  @ConfigAttribute(env = "PROXY_AUTH_TYPE")
+  private ProxyConfig.ProxyAuthType proxyAuthType;
+
+  @ConfigAttribute(env = "USE_SYSTEM_PROPERTIES_HTTP")
+  private Boolean useSystemPropertiesHttp;
+
   private volatile boolean resolved;
   private HeaderFactory headerFactory;
 
@@ -156,12 +174,8 @@ private void initHttp() {
     if (httpClient != null) {
       return;
     }
-    int timeout = 300;
-    if (httpTimeoutSeconds != null) {
-      timeout = httpTimeoutSeconds;
-    }
     // eventually it'll get decoupled from config.
-    httpClient = new CommonsHttpClient(timeout);
+    httpClient = new CommonsHttpClient(this);
   }
 
   public synchronized Map<String, String> authenticate() throws DatabricksException {
@@ -462,6 +476,60 @@ public DatabricksConfig setHttpClient(HttpClient httpClient) {
     return this;
   }
 
+  public String getProxyHost() {
+    return proxyHost;
+  }
+
+  public DatabricksConfig setProxyHost(String proxyHost) {
+    this.proxyHost = proxyHost;
+    return this;
+  }
+
+  public Integer getProxyPort() {
+    return proxyPort;
+  }
+
+  public DatabricksConfig setProxyPort(Integer proxyPort) {
+    this.proxyPort = proxyPort;
+    return this;
+  }
+
+  public String getProxyUsername() {
+    return proxyUsername;
+  }
+
+  public DatabricksConfig setProxyUsername(String proxyUsername) {
+    this.proxyUsername = proxyUsername;
+    return this;
+  }
+
+  public String getProxyPassword() {
+    return proxyPassword;
+  }
+
+  public DatabricksConfig setProxyPassword(String proxyPassword) {
+    this.proxyPassword = proxyPassword;
+    return this;
+  }
+
+  public ProxyConfig.ProxyAuthType getProxyAuthType() {
+    return proxyAuthType;
+  }
+
+  public DatabricksConfig setProxyAuthType(ProxyConfig.ProxyAuthType proxyAuthType) {
+    this.proxyAuthType = proxyAuthType;
+    return this;
+  }
+
+  public Boolean getUseSystemPropertiesHttp() {
+    return useSystemPropertiesHttp;
+  }
+
+  public DatabricksConfig setUseSystemPropertiesHttp(Boolean useSystemPropertiesHttp) {
+    this.useSystemPropertiesHttp = useSystemPropertiesHttp;
+    return this;
+  }
+
   public boolean isAzure() {
     return this.getDatabricksEnvironment().getCloud() == Cloud.AZURE;
   }

databricks-sdk-java/src/main/java/com/databricks/sdk/core/ProxyConfig.java

@@ -0,0 +1,81 @@
+package com.databricks.sdk.core;
+
+public class ProxyConfig {
+  private String host;
+  private Integer port;
+  private String username;
+  private String password;
+  private ProxyAuthType proxyAuthType;
+  private Boolean useSystemProperties;
+
+  public enum ProxyAuthType {
+    // Currently we only support BASIC and SPNEGO
+    NONE,
+    BASIC,
+    // We only support kerberos for negotiate
+    SPNEGO
+  }
+
+  public ProxyConfig(DatabricksConfig config) {
+    this.host = config.getProxyHost();
+    this.port = config.getProxyPort();
+    this.username = config.getProxyUsername();
+    this.password = config.getProxyPassword();
+    this.proxyAuthType = config.getProxyAuthType();
+    this.useSystemProperties = config.getUseSystemPropertiesHttp();
+  }
+
+  public String getHost() {
+    return host;
+  }
+
+  public ProxyConfig setHost(String host) {
+    this.host = host;
+    return this;
+  }
+
+  public Integer getPort() {
+    return port;
+  }
+
+  public ProxyConfig setPort(Integer port) {
+    this.port = port;
+    return this;
+  }
+
+  public String getUsername() {
+    return username;
+  }
+
+  public ProxyConfig setUsername(String username) {
+    this.username = username;
+    return this;
+  }
+
+  public String getPassword() {
+    return password;
+  }
+
+  public ProxyConfig setPassword(String password) {
+    this.password = password;
+    return this;
+  }
+
+  public ProxyAuthType getProxyAuthType() {
+    return proxyAuthType;
+  }
+
+  public ProxyConfig setProxyAuthType(ProxyAuthType proxyAuthType) {
+    this.proxyAuthType = proxyAuthType;
+    return this;
+  }
+
+  public Boolean getUseSystemProperties() {
+    return useSystemProperties;
+  }
+
+  public ProxyConfig setUseSystemProperties(Boolean useSystemProperties) {
+    this.useSystemProperties = useSystemProperties;
+    return this;
+  }
+}

databricks-sdk-java/src/main/java/com/databricks/sdk/core/commons/CommonsHttpClient.java

@@ -2,11 +2,14 @@
 
 import static org.apache.http.entity.ContentType.APPLICATION_JSON;
 
+import com.databricks.sdk.core.DatabricksConfig;
 import com.databricks.sdk.core.DatabricksException;
+import com.databricks.sdk.core.ProxyConfig;
 import com.databricks.sdk.core.http.HttpClient;
 import com.databricks.sdk.core.http.Request;
 import com.databricks.sdk.core.http.Response;
 import com.databricks.sdk.core.utils.CustomCloseInputStream;
+import com.databricks.sdk.core.utils.ProxyUtils;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
@@ -41,6 +44,20 @@ public CommonsHttpClient(int timeoutSeconds) {
     hc = makeClosableHttpClient();
   }
 
+  public CommonsHttpClient(DatabricksConfig databricksConfig) {
+    this(
+        databricksConfig.getHttpTimeoutSeconds() == null
+            ? 300
+            : databricksConfig.getHttpTimeoutSeconds(),
+        new ProxyConfig(databricksConfig));
+  }
+
+  public CommonsHttpClient(int timeoutSeconds, ProxyConfig proxyConfig) {
+    timeout = timeoutSeconds * 1000;
+    connectionManager.setMaxTotal(100);
+    hc = makeClosableHttpClient(proxyConfig);
+  }
+
   private RequestConfig makeRequestConfig() {
     return RequestConfig.custom()
         .setConnectionRequestTimeout(timeout)
@@ -53,10 +70,18 @@ private CloseableHttpClient makeClosableHttpClient() {
     return HttpClientBuilder.create()
         .setConnectionManager(connectionManager)
         .setDefaultRequestConfig(makeRequestConfig())
-        .useSystemProperties()
         .build();
   }
 
+  private CloseableHttpClient makeClosableHttpClient(ProxyConfig proxyConfig) {
+    HttpClientBuilder builder =
+        HttpClientBuilder.create()
+            .setConnectionManager(connectionManager)
+            .setDefaultRequestConfig(makeRequestConfig());
+    ProxyUtils.setupProxy(proxyConfig, builder);
+    return builder.build();
+  }
+
   @Override
   public Response execute(Request in) throws IOException {
     HttpUriRequest request = transformRequest(in);

databricks-sdk-java/src/main/java/com/databricks/sdk/core/utils/ProxyUtils.java

@@ -0,0 +1,143 @@
+package com.databricks.sdk.core.utils;
+
+import com.databricks.sdk.core.DatabricksException;
+import com.databricks.sdk.core.ProxyConfig;
+import java.security.Principal;
+import org.apache.http.HttpHost;
+import org.apache.http.auth.AuthSchemeProvider;
+import org.apache.http.auth.AuthScope;
+import org.apache.http.auth.Credentials;
+import org.apache.http.auth.UsernamePasswordCredentials;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.AuthSchemes;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.impl.auth.SPNegoSchemeFactory;
+import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.ProxyAuthenticationStrategy;
+
+/**
+ * This class is used to setup the proxy configs for the http client. This includes setting up the
+ * proxy host, port, and authentication.
+ */
+public class ProxyUtils {
+
+  /**
+   * Setup the proxy configuration in the http client builder.
+   *
+   * @param config the proxy configuration
+   * @param builder the http client builder
+   */
+  public static void setupProxy(ProxyConfig config, HttpClientBuilder builder) {
+    String proxyHost = null;
+    Integer proxyPort = null;
+    String proxyUser = null;
+    String proxyPassword = null;
+    if (config.getUseSystemProperties() != null && config.getUseSystemProperties()) {
+      builder.useSystemProperties();
+      String protocol = System.getProperty("https.proxyHost") != null ? "https" : "http";
+      proxyHost = System.getProperty(protocol + ".proxyHost");
+      proxyPort = Integer.parseInt(System.getProperty(protocol + ".proxyPort"));
+      proxyUser = System.getProperty(protocol + ".proxyUser");
+      proxyPassword = System.getProperty(protocol + ".proxyPassword");
+    }
+    // Override system properties if proxy configuration is explicitly set
+    if (config.getHost() != null) {
+      proxyHost = config.getHost();
+      proxyPort = config.getPort();
+      proxyUser = config.getUsername();
+      proxyPassword = config.getPassword();
+      builder.setProxy(new HttpHost(proxyHost, proxyPort));
+    }
+    setupProxyAuth(
+        proxyHost, proxyPort, config.getProxyAuthType(), proxyUser, proxyPassword, builder);
+  }
+
+  /**
+   * This method sets up the proxy authentication in the http client builder.
+   *
+   * @param proxyHost the proxy host
+   * @param proxyPort the proxy port
+   * @param proxyAuthType the proxy authentication type
+   * @param proxyUser the proxy user
+   * @param proxyPassword the proxy password
+   * @param builder the http client builder
+   */
+  public static void setupProxyAuth(
+      String proxyHost,
+      Integer proxyPort,
+      ProxyConfig.ProxyAuthType proxyAuthType,
+      String proxyUser,
+      String proxyPassword,
+      HttpClientBuilder builder) {
+    if (proxyAuthType == null) {
+      return;
+    }
+    AuthScope authScope = new AuthScope(proxyHost, proxyPort);
+    switch (proxyAuthType) {
+      case NONE:
+        break;
+      case BASIC:
+        setupBasicProxyAuth(builder, authScope, proxyUser, proxyPassword);
+        break;
+      case SPNEGO:
+        setupNegotiateProxyAuth(builder, authScope);
+        break;
+      default:
+        throw new DatabricksException("Unknown proxy auth type: " + proxyAuthType);
+    }
+  }
+
+  /**
+   * This method sets up the proxy authentication using the negotiate mechanism in the http client
+   * builder.
+   *
+   * @param builder the http client builder
+   * @param authScope the authentication scope
+   */
+  public static void setupNegotiateProxyAuth(HttpClientBuilder builder, AuthScope authScope) {
+    // We only support kerberos for negotiate as of now
+    System.setProperty("javax.security.auth.useSubjectCredsOnly", "false");
+    // "java.security.krb5.conf" system property needs to be set if krb5.conf is not in the default
+    // location
+    // Use "sun.security.krb5.debug" and "sun.security.jgss.debug" system properties for debugging
+    Credentials useJaasCreds =
+        new Credentials() {
+          public String getPassword() {
+            return null;
+          }
+
+          public Principal getUserPrincipal() {
+            return null;
+          }
+        };
+
+    CredentialsProvider credsProvider = new BasicCredentialsProvider();
+    credsProvider.setCredentials(authScope, useJaasCreds);
+    builder
+        .setDefaultCredentialsProvider(credsProvider)
+        .setDefaultAuthSchemeRegistry(
+            RegistryBuilder.<AuthSchemeProvider>create()
+                .register(AuthSchemes.SPNEGO, new SPNegoSchemeFactory(true))
+                .build());
+  }
+
+  /**
+   * This method sets up the proxy authentication using the basic mechanism credentials provided
+   * into the http client builder.
+   *
+   * @param builder the http client builder
+   * @param authScope the authentication scope
+   * @param proxyUser the proxy user
+   * @param proxyPassword the proxy password
+   */
+  public static void setupBasicProxyAuth(
+      HttpClientBuilder builder, AuthScope authScope, String proxyUser, String proxyPassword) {
+    CredentialsProvider credsProvider = new BasicCredentialsProvider();
+    credsProvider.setCredentials(
+        authScope, new UsernamePasswordCredentials(proxyUser, proxyPassword));
+    builder
+        .setDefaultCredentialsProvider(credsProvider)
+        .setProxyAuthenticationStrategy(new ProxyAuthenticationStrategy());
+  }
+}