Merge branch 'main' into allow-v1-and-v2

Author: Paultagoras

CHANGELOG.md

@@ -1,3 +1,8 @@
+## Latest
+
+### New Features
+- Added basic auth support for proxies. Now you can specify username/password when connecting via a proxy that requires it with HttpURLConnection and Apache HttpClient.
+
 ## 0.7.1-patch1
 
 ### Bug Fixes

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java

@@ -245,6 +245,8 @@ public static Map<ClickHouseOption, Serializable> toClientOptions(Map<?, ?> prop
     private final ClickHouseProxyType proxyType;
     private final String proxyHost;
     private final int proxyPort;
+    private final String proxyUserName;
+    private final char[] proxyPassword;
     // client specific options
     private final Map<ClickHouseOption, Serializable> options;
     private final ClickHouseCredentials credentials;
@@ -383,6 +385,8 @@ public ClickHouseConfig(Map<ClickHouseOption, Serializable> options, ClickHouseC
         this.proxyType = getOption(ClickHouseClientOption.PROXY_TYPE, ClickHouseProxyType.class);
         this.proxyHost = getStrOption(ClickHouseClientOption.PROXY_HOST);
         this.proxyPort = getIntOption(ClickHouseClientOption.PROXY_PORT);
+        this.proxyUserName = getStrOption(ClickHouseClientOption.PROXY_USERNAME);
+        this.proxyPassword = getStrOption(ClickHouseClientOption.PROXY_PASSWORD).toCharArray();
     }
 
     @Override
@@ -674,14 +678,6 @@ public boolean isUseObjectsInArray() {
         return useObjectsInArray;
     }
 
-    /**
-     * Checks whether no proxy is used or not.
-     *
-     * @return true if no proxy is used; false otherwise
-     * @deprecated will be dropped in 0.5, please use {@link #getProxyType()}
-     *             instead
-     */
-
     public ClickHouseProxyType getProxyType() {
         return proxyType;
     }
@@ -694,6 +690,14 @@ public int getProxyPort() {
         return proxyPort;
     }
 
+    public String getProxyUserName() {
+        return proxyUserName;
+    }
+
+    public char[] getProxyPassword() {
+        return proxyPassword;
+    }
+
     public boolean isUseServerTimeZone() {
         return useServerTimeZone;
     }

clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java

@@ -402,7 +402,15 @@ public enum ClickHouseClientOption implements ClickHouseOption {
     /**
      * Set ClickHouse proxy port.
      */
-    PROXY_PORT("proxy_port", -1, "Set ClickHouse server proxy hostname."),
+    PROXY_PORT("proxy_port", -1, "Set ClickHouse server proxy port."),
+    /**
+     * Set Clickhouse proxy username.
+     */
+    PROXY_USERNAME("proxy_username", "", "Set ClickHouse server proxy username."),
+    /**
+     * Set ClickHouse proxy password.
+     */
+    PROXY_PASSWORD("proxy_password", "", "Set ClickHouse server proxy password."),
     /**
      * Whether to use server time zone.
      */

clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java

@@ -19,9 +19,12 @@
 import com.clickhouse.data.ClickHouseUtils;
 import com.clickhouse.logging.Logger;
 import com.clickhouse.logging.LoggerFactory;
+import org.apache.hc.client5.http.auth.AuthScope;
+import org.apache.hc.client5.http.auth.UsernamePasswordCredentials;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
 import org.apache.hc.client5.http.classic.methods.HttpPost;
 import org.apache.hc.client5.http.config.ConnectionConfig;
+import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpClient;
 import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse;
 import org.apache.hc.client5.http.impl.classic.HttpClientBuilder;
@@ -121,6 +124,14 @@ private CloseableHttpClient newConnection(ClickHouseConfig c) throws IOException
         }
         if (c.getProxyType() == ClickHouseProxyType.HTTP) {
             builder.setProxy(new HttpHost(c.getProxyHost(), c.getProxyPort()));
+
+            if (c.getProxyUserName() != null && c.getProxyUserName() != "") {
+                AuthScope authScope = new AuthScope(c.getProxyHost(), c.getProxyPort());
+                UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(c.getProxyUserName(), c.getProxyPassword());
+                BasicCredentialsProvider credsProvider = new BasicCredentialsProvider();
+                credsProvider.setCredentials(authScope, credentials);
+                builder.setDefaultCredentialsProvider(credsProvider);
+            }
         }
         return builder.build();
     }

clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java

@@ -287,6 +287,19 @@ protected static Proxy getProxy(ClickHouseConfig config) {
         return proxy;
     }
 
+    protected static String getProxyAuth(ClickHouseConfig config) {
+        String authHeader = null;
+        if (config.getProxyType() == ClickHouseProxyType.HTTP) {
+            String userName = config.getProxyUserName();
+            if (!ClickHouseChecker.isNullOrEmpty(userName)) {
+                String auth = userName + ":" + new String(config.getProxyPassword());
+                byte[] encodedAuth = Base64.getEncoder().encode(auth.getBytes(StandardCharsets.UTF_8));
+                authHeader = "Basic " + new String(encodedAuth);
+            }
+        }
+        return authHeader;
+    }
+
     protected static String parseErrorFromException(String errorCode, String serverName, IOException e, byte[] bytes) {
         log.debug("Failed to read error message[code=%s] from server [%s] due to: %s", errorCode, serverName,
                 e.getMessage());

clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java

@@ -111,6 +111,10 @@ private HttpURLConnection newConnection(String url, boolean post) throws IOExcep
         if (proxy != null) {
             log.debug("using proxy type [%s] address [%s]", proxy.type().name(), proxy.address().toString());
             newConn = (HttpURLConnection) new URL(url).openConnection(proxy);
+            String authString = getProxyAuth(c);
+            if (authString != null) {
+                newConn.setRequestProperty("Proxy-Authorization", authString);
+            }
         } else {
             newConn = (HttpURLConnection) new URL(url).openConnection();
         }

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -1903,7 +1903,7 @@ public ClickHouseBinaryFormatReader newBinaryFormatReader(QueryResponse response
                         byteBufferPool);
                 break;
             default:
-                throw new IllegalArgumentException("Unsupported format: " + response.getFormat());
+                throw new IllegalArgumentException("Binary readers doesn't support format: " + response.getFormat());
         }
         return reader;
     }

client-v2/src/main/java/com/clickhouse/client/api/ClientSettings.java

@@ -39,4 +39,16 @@ public static List<String> valuesFromCommaSeparated(String value) {
     public static final String SETTING_LOG_COMMENT = SERVER_SETTING_PREFIX + "log_comment";
 
     public static final String HTTP_USE_BASIC_AUTH = "http_use_basic_auth";
+
+    // -- Experimental features --
+
+    /**
+     * Server will expect a string in JSON format and parse it into a JSON object.
+     */
+    public static final String INPUT_FORMAT_BINARY_READ_JSON_AS_STRING = "input_format_binary_read_json_as_string";
+
+    /**
+     * Server will return a JSON object as a string.
+     */
+    public static final String OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING = "output_format_binary_write_json_as_string";
 }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java

@@ -1,7 +1,9 @@
 package com.clickhouse.client.api.data_formats.internal;
 
 import com.clickhouse.client.api.ClientException;
+import com.clickhouse.client.api.ClientSettings;
 import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
+import com.clickhouse.client.api.internal.MapUtils;
 import com.clickhouse.client.api.metadata.TableSchema;
 import com.clickhouse.client.api.query.NullValueException;
 import com.clickhouse.client.api.query.POJOSetter;
@@ -32,8 +34,10 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TimeZone;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
@@ -71,7 +75,9 @@ protected AbstractBinaryFormatReader(InputStream inputStream, QuerySettings quer
         if (timeZone == null) {
             throw new ClientException("Time zone is not set. (useServerTimezone:" + useServerTimeZone + ")");
         }
-        this.binaryStreamReader = new BinaryStreamReader(inputStream, timeZone, LOG, byteBufferAllocator);
+        boolean jsonAsString = MapUtils.getFlag(this.settings,
+                ClientSettings.SERVER_SETTING_PREFIX + ClientSettings.OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING, false);
+        this.binaryStreamReader = new BinaryStreamReader(inputStream, timeZone, LOG, byteBufferAllocator, jsonAsString);
         if (schema != null) {
             setSchema(schema);
         }

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

@@ -45,16 +45,7 @@ public class BinaryStreamReader {
 
     private final ByteBufferAllocator bufferAllocator;
 
-    /**
-     * Creates a BinaryStreamReader instance that will use {@link DefaultByteBufferAllocator} to allocate buffers.
-     *
-     * @param input - source of raw data in a suitable format
-     * @param timeZone - timezone to use for date and datetime values
-     * @param log - logger
-     */
-    BinaryStreamReader(InputStream input, TimeZone timeZone, Logger log) {
-        this(input, timeZone, log, new DefaultByteBufferAllocator());
-    }
+    private final boolean jsonAsString;
 
     /**
      * Createa a BinaryStreamReader instance that will use the provided buffer allocator.
@@ -64,11 +55,12 @@ public class BinaryStreamReader {
      * @param log - logger
      * @param bufferAllocator - byte buffer allocator
      */
-    BinaryStreamReader(InputStream input, TimeZone timeZone, Logger log, ByteBufferAllocator bufferAllocator) {
+    BinaryStreamReader(InputStream input, TimeZone timeZone, Logger log, ByteBufferAllocator bufferAllocator, boolean jsonAsString) {
         this.log = log == null ? NOPLogger.NOP_LOGGER : log;
         this.timeZone = timeZone;
         this.input = input;
         this.bufferAllocator = bufferAllocator;
+        this.jsonAsString = jsonAsString;
     }
 
     /**
@@ -203,8 +195,13 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
                 case Ring:
                     return (T) readGeoRing();
 
-//                case JSON: // obsolete https://clickhouse.com/docs/en/sql-reference/data-types/json#displaying-json-column
-//                case Object:
+                case JSON: // experimental https://clickhouse.com/docs/en/sql-reference/data-types/newjson
+                    if (jsonAsString) {
+                        return (T) readString(input);
+                    } else {
+                        throw new RuntimeException("Reading JSON from binary is not implemented yet");
+                    }
+//                case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
                 case Array:
                     return convertArray(readArray(column), typeHint);
                 case Map: