Merge branch 'main' into datagrip-issues

Author: Paultagoras

CHANGELOG.md

@@ -3,6 +3,9 @@
 ### New Features
 - Added basic auth support for proxies. Now you can specify username/password when connecting via a proxy that requires it with HttpURLConnection and Apache HttpClient.
 
+### Bug Fixes
+- Fix for retrying on `ConnectTimeoutException`
+
 ## 0.7.1-patch1
 
 ### Bug Fixes

client-v2/src/main/java/com/clickhouse/client/api/Client.java

@@ -63,6 +63,7 @@
 import java.io.InputStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.net.ConnectException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
@@ -209,6 +210,10 @@ public void close() {
         if (oldClient != null) {
             oldClient.close();
         }
+
+        if (httpClientHelper != null) {
+            httpClientHelper.close();
+        }
     }
 
     public static class Builder {
@@ -290,7 +295,7 @@ public Builder addEndpoint(String endpoint) {
                     throw new IllegalArgumentException("Only HTTP and HTTPS protocols are supported");
                 }
             } catch (java.net.MalformedURLException e) {
-                throw new IllegalArgumentException("Endpoint should be a valid URL string", e);
+                throw new IllegalArgumentException("Endpoint should be a valid URL string, but was " + endpoint, e);
             }
             return this;
         }
@@ -940,6 +945,16 @@ public Builder setClientName(String clientName) {
             return this;
         }
 
+        /**
+         * Sets client options from provided map. Values are copied as is
+         * @param options - map of client options
+         * @return same instance of the builder
+         */
+        public Builder setOptions(Map<String, String> options) {
+            this.configuration.putAll(options);
+            return this;
+        }
+
         public Client build() {
             setDefaults();
 
@@ -950,7 +965,7 @@ public Client build() {
             // check if username and password are empty. so can not initiate client?
             if (!this.configuration.containsKey("access_token") &&
                 (!this.configuration.containsKey("user") || !this.configuration.containsKey("password")) &&
-                !MapUtils.getFlag(this.configuration, "ssl_authentication")) {
+                !MapUtils.getFlag(this.configuration, "ssl_authentication", false)) {
                 throw new IllegalArgumentException("Username and password (or access token, or SSL authentication) are required");
             }
 
@@ -1394,7 +1409,7 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
                         metrics.operationComplete();
                         metrics.setQueryId(queryId);
                         return new InsertResponse(metrics);
-                    } catch ( NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException e) {
+                    } catch (NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException | ConnectException e) {
                         lastException = httpClientHelper.wrapException("Insert request initiation failed", e);
                         if (httpClientHelper.shouldRetry(e, finalSettings.getAllSettings())) {
                             LOG.warn("Retrying", e);
@@ -1522,7 +1537,7 @@ public CompletableFuture<InsertResponse> insert(String tableName,
                         metrics.operationComplete();
                         metrics.setQueryId(queryId);
                         return new InsertResponse(metrics);
-                    } catch ( NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException e) {
+                    } catch (NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException | ConnectException e) {
                         lastException = httpClientHelper.wrapException("Insert request initiation failed", e);
                         if (httpClientHelper.shouldRetry(e, finalSettings.getAllSettings())) {
                             LOG.warn("Retrying", e);
@@ -1688,7 +1703,7 @@ public CompletableFuture<QueryResponse> query(String sqlQuery, Map<String, Objec
 
                         return new QueryResponse(httpResponse, finalSettings.getFormat(), finalSettings, metrics);
 
-                    } catch ( NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException e) {
+                    } catch (NoHttpResponseException | ConnectionRequestTimeoutException | ConnectTimeoutException | ConnectException e) {
                         lastException = httpClientHelper.wrapException("Query request initiation failed", e);
                         if (httpClientHelper.shouldRetry(e, finalSettings.getAllSettings())) {
                             LOG.warn("Retrying.", e);
@@ -2105,6 +2120,10 @@ public Set<String> getEndpoints() {
         return Collections.unmodifiableSet(endpoints);
     }
 
+    public String getUser() {
+        return this.configuration.get(ClientConfigProperties.USER.getKey());
+    }
+
     /**
      * Sets list of DB roles that should be applied to each query.
      *

client-v2/src/main/java/com/clickhouse/client/api/ClientConfigProperties.java

@@ -17,14 +17,14 @@ public enum ClientConfigProperties {
 
     HTTP_USE_BASIC_AUTH("http_use_basic_auth"),
 
-    USER("user"),
+    USER("user", "default"),
 
-    PASSWORD("password"),
+    PASSWORD("password", ""),
 
     /**
      * Maximum number of active connection in internal connection pool.
      */
-    HTTP_MAX_OPEN_CONNECTIONS("max_open_connections"),
+    HTTP_MAX_OPEN_CONNECTIONS("max_open_connections", "10"),
 
     /**
      * HTTP keep-alive timeout override.
@@ -59,7 +59,7 @@ public enum ClientConfigProperties {
 
     SOCKET_LINGER_OPT("socket_linger"),
 
-    DATABASE("database"),
+    DATABASE("database", "default"),
 
     COMPRESS_SERVER_RESPONSE("compress"), // actually a server setting, but has client effect too
 
@@ -118,14 +118,37 @@ public enum ClientConfigProperties {
 
     private String key;
 
+    private String defaultValue;
+
+    private List<String> choices;
+
+
     ClientConfigProperties(String key) {
+        this(key, null, Collections.emptyList());
+    }
+
+    ClientConfigProperties(String key, String defaultValue) {
+        this(key, defaultValue, Collections.emptyList());
+    }
+
+    ClientConfigProperties(String key, String defaultValue, List<String> choices) {
         this.key = key;
+        this.defaultValue = defaultValue;
+        this.choices = Collections.unmodifiableList(choices);
     }
 
     public String getKey() {
         return key;
     }
 
+    public List<String> getChoices() {
+        return choices;
+    }
+
+    public String getDefaultValue() {
+        return defaultValue;
+    }
+
     public static final String HTTP_HEADER_PREFIX = "http_header_";
 
     public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_";

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java

@@ -251,6 +251,7 @@ protected void setSchema(TableSchema schema) {
                 case Decimal128:
                 case Decimal256:
                 case Bool:
+                case String:
                     this.convertions[i] = NumberConverter.NUMBER_CONVERTERS;
                     break;
                 default:
@@ -518,6 +519,7 @@ public boolean hasValue(int colIndex) {
 
     @Override
     public boolean hasValue(String colName) {
+        getSchema().getColumnByName(colName);
         return currentRecord.containsKey(colName);
     }
 

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/MapBackedRecord.java

@@ -81,8 +81,8 @@ private <T> T readNumberValue(String colName, NumberConverter.NumberType targetT
             return (T) converter.apply(value);
         } else {
             String columnTypeName = schema.getColumnByName(colName).getDataType().name();
-            throw new ClientException("Column " + colName + " " + columnTypeName +
-                    " cannot be converted to " + targetType.getTypeName());
+            throw new ClientException("Column '" + colName + "' of type " + columnTypeName +
+                    " cannot be converted to '" + targetType.getTypeName() + "' value");
         }
     }
 
@@ -238,7 +238,12 @@ public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(String colName) {
 
     @Override
     public <T> List<T> getList(String colName) {
-        return getList(schema.nameToIndex(colName));
+        Object value = readValue(colName);
+        if (value instanceof BinaryStreamReader.ArrayValue) {
+            return ((BinaryStreamReader.ArrayValue) value).asList();
+        } else {
+            throw new ClientException("Column is not of array type");
+        }
     }
 
 
@@ -388,12 +393,7 @@ public ClickHouseGeoMultiPolygonValue getGeoMultiPolygon(int index) {
 
     @Override
     public <T> List<T> getList(int index) {
-        Object value = readValue(index);
-        if (value instanceof BinaryStreamReader.ArrayValue) {
-            return ((BinaryStreamReader.ArrayValue) value).asList();
-        } else {
-            throw new ClientException("Column is not of array type");
-        }
+        return getList(schema.indexToName(index));
     }
 
     @Override

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/NumberConverter.java

@@ -153,7 +153,6 @@ public static BigDecimal toBigDecimal(Object value) {
         }
     }
 
-
     static Map<NumberType, Function<Object, ?>> getNumberConverters() {
         Map<NumberType, Function<Object, ?>> converters = new HashMap<>();
         converters.put(NumberType.Byte, NumberConverter::toByte);

client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java

@@ -42,6 +42,7 @@
 import org.apache.hc.core5.http.impl.io.DefaultHttpResponseParserFactory;
 import org.apache.hc.core5.http.io.SocketConfig;
 import org.apache.hc.core5.http.io.entity.EntityTemplate;
+import org.apache.hc.core5.io.CloseMode;
 import org.apache.hc.core5.io.IOCallback;
 import org.apache.hc.core5.net.URIBuilder;
 import org.apache.hc.core5.pool.PoolConcurrencyPolicy;
@@ -582,7 +583,7 @@ public boolean shouldRetry(Exception ex, Map<String, Object> requestSettings) {
             return retryCauses.contains(ClientFaultCause.NoHttpResponse);
         }
 
-        if (ex instanceof ConnectException) {
+        if (ex instanceof ConnectException || ex instanceof ConnectTimeoutException) {
             return retryCauses.contains(ClientFaultCause.ConnectTimeout);
         }
 
@@ -597,6 +598,7 @@ public boolean shouldRetry(Exception ex, Map<String, Object> requestSettings) {
     // ClientException will be also wrapped
     public ClientException wrapException(String message, Exception cause) {
         if (cause instanceof ConnectionRequestTimeoutException ||
+                cause instanceof NoHttpResponseException ||
                 cause instanceof ConnectTimeoutException ||
                 cause instanceof ConnectException) {
             return new ConnectionInitiationException(message, cause);
@@ -639,4 +641,8 @@ public static Map<String, String> parseUrlParameters(URL url) {
 
         return params;
     }
+
+    public void close() {
+        httpClient.close(CloseMode.IMMEDIATE);
+    }
 }

client-v2/src/main/java/com/clickhouse/client/api/internal/ServerSettings.java

@@ -22,4 +22,14 @@ public final class ServerSettings {
      */
     public static final String OUTPUT_FORMAT_BINARY_WRITE_JSON_AS_STRING = "output_format_binary_write_json_as_string";
 
+    /**
+     * Limit number of rows in a result set
+     */
+    public static final String MAX_RESULT_ROWS = "max_result_rows";
+
+    /**
+     * Defines server response if result set exceeded a limit set by {@code max_result_rows}.
+     * Possible values are 'throw' or 'break'. Default is 'throw'
+     */
+    public static final String RESULT_OVERFLOW_MODE = "result_overflow_mode";
 }

client-v2/src/main/java/com/clickhouse/client/api/metadata/NoSuchColumnException.java

@@ -0,0 +1,10 @@
+package com.clickhouse.client.api.metadata;
+
+import com.clickhouse.client.api.ClientException;
+
+public class NoSuchColumnException extends ClientException {
+
+    public NoSuchColumnException(String message) {
+        super(message);
+    }
+}

client-v2/src/main/java/com/clickhouse/client/api/metadata/TableSchema.java

@@ -1,5 +1,6 @@
 package com.clickhouse.client.api.metadata;
 
+import com.clickhouse.client.api.ClientException;
 import com.clickhouse.data.ClickHouseColumn;
 
 import java.util.ArrayList;
@@ -87,20 +88,22 @@ public void addColumn(String name, String type, String defaultType) {
     }
 
     public ClickHouseColumn getColumnByName(String name) {
-        for (ClickHouseColumn column : columns) {
-            if (column.getColumnName().equalsIgnoreCase(name)) {
-                return column;//TODO: Try to deep clone the column rather than reference pass
-            }
-        }
-
-        return null;
+        return columns.get(nameToIndex(name));
     }
 
     public String indexToName(int index) {
-        return columns.get(index).getColumnName();
+        try {
+            return columns.get(index).getColumnName();
+        } catch (IndexOutOfBoundsException e) {
+            throw new NoSuchColumnException("Result has no column with index = " + index);
+        }
     }
 
     public int nameToIndex(String name) {
+        Integer index = colIndex.get(name);
+        if (index == null) {
+            throw new NoSuchColumnException("Result has no column with name '" + name + "'");
+        }
         return colIndex.get(name).intValue();
     }