Merge pull request #815 from zhicwu/fix-precison

Fix incorrect precision and scale from DatabaseMetaData.getColumns

Author: zhicwu

README.md

@@ -99,7 +99,7 @@ Java 8 or higher is required in order to use Java client([clickhouse-client](htt
     <groupId>com.clickhouse</groupId>
     <!-- or clickhouse-grpc-client if you prefer gRPC -->
     <artifactId>clickhouse-http-client</artifactId>
-    <version>0.3.2-patch2</version>
+    <version>0.3.2-patch3</version>
 </dependency>
 ```
 
@@ -135,7 +135,7 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol);
     <!-- will stop using ru.yandex.clickhouse starting from 0.4.0  -->
     <groupId>com.clickhouse</groupId>
     <artifactId>clickhouse-jdbc</artifactId>
-    <version>0.3.2-patch2</version>
+    <version>0.3.2-patch3</version>
     <!-- below is only needed when all you want is a shaded jar -->
     <classifier>http</classifier>
     <exclusions>

clickhouse-client/README.md

@@ -9,7 +9,7 @@ Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so
 <dependency>
     <groupId>com.clickhouse</groupId>
     <artifactId>clickhouse-http-client</artifactId>
-    <version>0.3.2-patch1</version>
+    <version>0.3.2-patch3</version>
 </dependency>
 ```
 

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseColumn.java

@@ -45,6 +45,7 @@ public final class ClickHouseColumn implements Serializable {
     private static ClickHouseColumn update(ClickHouseColumn column) {
         column.enumConstants = ClickHouseEnum.EMPTY;
         int size = column.parameters.size();
+        column.precision = column.dataType.getMaxPrecision();
         switch (column.dataType) {
             case Array:
                 column.arrayLevel = 1;

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseDataType.java

@@ -44,15 +44,15 @@ public enum ClickHouseDataType {
     UInt32(Long.class, false, true, false, 4, 10, 0, 0, 0, "INT UNSIGNED", "INTEGER UNSIGNED", "MEDIUMINT UNSIGNED"),
     UInt64(Long.class, false, true, false, 8, 20, 0, 0, 0, "BIGINT UNSIGNED"),
     UInt128(BigInteger.class, false, true, false, 16, 39, 0, 0, 0),
-    UInt256(BigInteger.class, false, true, false, 32, 78, 0, 0, 0), Int8(Byte.class, false, true, true, 1, 3, 0, 0, 0,
-            "BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
+    UInt256(BigInteger.class, false, true, false, 32, 78, 0, 0, 0),
+    Int8(Byte.class, false, true, true, 1, 3, 0, 0, 0, "BYTE", "INT1", "INT1 SIGNED", "TINYINT", "TINYINT SIGNED"),
     Int16(Short.class, false, true, true, 2, 5, 0, 0, 0, "SMALLINT", "SMALLINT SIGNED"),
     Int32(Integer.class, false, true, true, 4, 10, 0, 0, 0, "INT", "INTEGER", "MEDIUMINT", "INT SIGNED",
             "INTEGER SIGNED", "MEDIUMINT SIGNED"),
     Int64(Long.class, false, true, true, 8, 19, 0, 0, 0, "BIGINT", "BIGINT SIGNED"),
     Int128(BigInteger.class, false, true, true, 16, 39, 0, 0, 0),
     Int256(BigInteger.class, false, true, true, 32, 77, 0, 0, 0),
-    Bool(Boolean.class, false, false, true, 1, 3, 0, 0, 0, "BOOLEAN"),
+    Bool(Boolean.class, false, false, true, 1, 1, 0, 0, 0, "BOOLEAN"),
     Date(LocalDate.class, false, false, false, 2, 10, 0, 0, 0),
     Date32(LocalDate.class, false, false, false, 4, 10, 0, 0, 0),
     DateTime(LocalDateTime.class, true, false, false, 0, 29, 0, 0, 9, "TIMESTAMP"),
@@ -63,22 +63,28 @@ public enum ClickHouseDataType {
     Decimal64(BigDecimal.class, true, false, true, 8, 18, 18, 0, 18),
     Decimal128(BigDecimal.class, true, false, true, 16, 38, 38, 0, 38),
     Decimal256(BigDecimal.class, true, false, true, 32, 76, 20, 0, 76),
-    UUID(UUID.class, false, true, false, 16, 69, 0, 0, 0), Enum(String.class, true, true, false, 1, 0, 0, 0, 0),
-    Enum8(String.class, true, true, false, 1, 0, 0, 0, 0), Enum16(String.class, true, true, false, 2, 0, 0, 0, 0),
+    UUID(UUID.class, false, true, false, 16, 69, 0, 0, 0),
+    @Deprecated
+    Enum(String.class, true, true, false, 1, 0, 0, 0, 0),
+    Enum8(String.class, true, true, false, 1, 0, 0, 0, 0), // "ENUM"),
+    Enum16(String.class, true, true, false, 2, 0, 0, 0, 0),
     Float32(Float.class, false, true, true, 4, 12, 0, 0, 38, "FLOAT", "REAL", "SINGLE"),
     Float64(Double.class, false, true, true, 16, 22, 0, 0, 308, "DOUBLE", "DOUBLE PRECISION"),
-    IPv4(Inet4Address.class, false, true, false, 4, 0, 0, 0, 0, "INET4"),
-    IPv6(Inet6Address.class, false, true, false, 16, 0, 0, 0, 0, "INET6"),
+    IPv4(Inet4Address.class, false, true, false, 4, 10, 0, 0, 0, "INET4"),
+    IPv6(Inet6Address.class, false, true, false, 16, 39, 0, 0, 0, "INET6"),
     FixedString(String.class, true, true, false, 0, 0, 0, 0, 0, "BINARY"),
     String(String.class, false, true, false, 0, 0, 0, 0, 0, "BINARY LARGE OBJECT", "BINARY VARYING", "BLOB", "BYTEA",
-            "CHAR", "CHAR LARGE OBJECT", "CHAR VARYING", "CHARACTER", "CHARACTER LARGE OBJECT", "CHARACTER VARYING",
-            "CLOB", "LONGBLOB", "LONGTEXT", "MEDIUMBLOB", "MEDIUMTEXT", "NATIONAL CHAR", "NATIONAL CHAR VARYING",
-            "NATIONAL CHARACTER", "NATIONAL CHARACTER LARGE OBJECT", "NATIONAL CHARACTER VARYING", "NCHAR",
-            "NCHAR LARGE OBJECT", "NCHAR VARYING", "NVARCHAR", "TEXT", "TINYBLOB", "TINYTEXT", "VARCHAR", "VARCHAR2"),
+            "CHAR", "CHARACTER", "CHARACTER LARGE OBJECT", "CHARACTER VARYING", "CHAR LARGE OBJECT", "CHAR VARYING",
+            "CLOB", "LONGBLOB", "LONGTEXT", "MEDIUMBLOB", "MEDIUMTEXT", "NATIONAL CHAR", "NATIONAL CHARACTER",
+            "NATIONAL CHARACTER LARGE OBJECT", "NATIONAL CHARACTER VARYING", "NATIONAL CHAR VARYING", "NCHAR",
+            "NCHAR LARGE OBJECT", "NCHAR VARYING", "NVARCHAR", "TEXT", "TINYBLOB", "TINYTEXT", "VARBINARY", "VARCHAR",
+            "VARCHAR2"),
     AggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0), // implementation-defined intermediate state
     SimpleAggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0),
-    Array(Object.class, true, true, false, 0, 0, 0, 0, 0), Map(Map.class, true, true, false, 0, 0, 0, 0, 0),
-    Nested(Object.class, true, true, false, 0, 0, 0, 0, 0), Tuple(List.class, true, true, false, 0, 0, 0, 0, 0),
+    Array(Object.class, true, true, false, 0, 0, 0, 0, 0),
+    Map(Map.class, true, true, false, 0, 0, 0, 0, 0),
+    Nested(Object.class, true, true, false, 0, 0, 0, 0, 0),
+    Tuple(List.class, true, true, false, 0, 0, 0, 0, 0),
     Point(Object.class, false, true, true, 33, 0, 0, 0, 0), // same as Tuple(Float64, Float64)
     Polygon(Object.class, false, true, true, 0, 0, 0, 0, 0), // same as Array(Ring)
     MultiPolygon(Object.class, false, true, true, 0, 0, 0, 0, 0), // same as Array(Polygon)

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java

@@ -298,7 +298,6 @@ public byte[] readBytes(int length) throws IOException {
             ensureOpen();
 
             byte[] bytes = new byte[length];
-            int offset = 0;
             int counter = 0;
             while (counter < length) {
                 if (position >= limit && updateBuffer() < 0) {
@@ -314,9 +313,8 @@ public byte[] readBytes(int length) throws IOException {
                 }
 
                 int size = Math.min(limit - position, length - counter);
-                System.arraycopy(buffer, position, bytes, offset, size);
+                System.arraycopy(buffer, position, bytes, counter, size);
                 position += size;
-                offset += size;
                 counter += size;
             }
 

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValue.java

@@ -538,6 +538,55 @@ default <T, E extends Enum<E>> T asObject(Class<T> clazz) {
         }
     }
 
+    /**
+     * Gets binary value as byte array.
+     *
+     * @return byte array which could be null
+     */
+    default byte[] asBinary() {
+        return asBinary(0, null);
+    }
+
+    /**
+     * Gets binary value as fixed length byte array.
+     *
+     * @param length byte length of value, 0 or negative number means no limit
+     * @return byte array which could be null
+     */
+    default byte[] asBinary(int length) {
+        return asBinary(length, null);
+    }
+
+    /**
+     * Gets binary value as byte array.
+     *
+     * @param charset charset, null is same as default(UTF-8)
+     * @return byte array which could be null
+     */
+    default byte[] asBinary(Charset charset) {
+        return asBinary(0, charset);
+    }
+
+    /**
+     * Gets binary value as byte array.
+     *
+     * @param length  byte length of value, 0 or negative number means no limit
+     * @param charset charset, null is same as default(UTF-8)
+     * @return byte array which could be null
+     */
+    default byte[] asBinary(int length, Charset charset) {
+        if (isNullOrEmpty()) {
+            return null;
+        }
+
+        byte[] bytes = asString().getBytes(charset == null ? StandardCharsets.UTF_8 : charset);
+        if (length > 0) {
+            ClickHouseChecker.notWithDifferentLength(bytes, length);
+        }
+
+        return bytes;
+    }
+
     /**
      * Gets value as unbounded string, using default charset(usually UTF-8).
      *

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseValues.java

@@ -8,6 +8,7 @@
 import java.net.Inet6Address;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
+import java.nio.charset.StandardCharsets;
 import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
@@ -76,7 +77,9 @@ public final class ClickHouseValues {
     public static final long[] EMPTY_LONG_ARRAY = new long[0];
     public static final float[] EMPTY_FLOAT_ARRAY = new float[0];
     public static final double[] EMPTY_DOUBLE_ARRAY = new double[0];
+
     public static final String EMPTY_ARRAY_EXPR = "[]";
+    public static final String EMPTY_STRING_EXPR = "''";
 
     public static final BigDecimal NANOS = new BigDecimal(BigInteger.TEN.pow(9));
 
@@ -134,6 +137,10 @@ public final class ClickHouseValues {
 
     public static final String TYPE_CLASS = "Class";
 
+    private static final byte[] HEX_ARRAY = "0123456789ABCDEF".getBytes(StandardCharsets.US_ASCII);
+    private static final byte[] UNHEX_PREFIX = "unhex('".getBytes(StandardCharsets.US_ASCII);
+    private static final byte[] UNHEX_SUFFIX = "')".getBytes(StandardCharsets.US_ASCII);
+
     /**
      * Converts IP address to big integer.
      *
@@ -180,6 +187,52 @@ public static BigInteger convertToBigInteger(UUID value) {
         return low.add(high.multiply(BIGINT_HL_BOUNDARY));
     }
 
+    /**
+     * Converts given byte array to string in hexadecimal format.
+     *
+     * @param bytes byte array
+     * @return non-null string
+     */
+    public static String convertToHexString(byte[] bytes) {
+        int len = bytes != null ? bytes.length : 0;
+        if (len == 0) {
+            return "";
+        }
+
+        byte[] hexChars = new byte[len * 2];
+        for (int i = 0; i < len; i++) {
+            int v = bytes[i] & 0xFF;
+            int j = i * 2;
+            hexChars[j] = HEX_ARRAY[v >>> 4];
+            hexChars[j + 1] = HEX_ARRAY[v & 0x0F];
+        }
+        return new String(hexChars, StandardCharsets.UTF_8);
+    }
+
+    /**
+     * Converts given byte array to unhex() expression.
+     *
+     * @param bytes byte array
+     * @return non-null expression
+     */
+    public static String convertToUnhexExpression(byte[] bytes) {
+        int len = bytes != null ? bytes.length : 0;
+        if (len == 0) {
+            return EMPTY_STRING_EXPR;
+        }
+
+        int offset = UNHEX_PREFIX.length;
+        byte[] hexChars = new byte[len * 2 + offset + UNHEX_SUFFIX.length];
+        System.arraycopy(UNHEX_PREFIX, 0, hexChars, 0, offset);
+        System.arraycopy(UNHEX_SUFFIX, 0, hexChars, hexChars.length - UNHEX_SUFFIX.length, UNHEX_SUFFIX.length);
+        for (int i = 0; i < len; i++) {
+            int v = bytes[i] & 0xFF;
+            hexChars[offset++] = HEX_ARRAY[v >>> 4];
+            hexChars[offset++] = HEX_ARRAY[v & 0x0F];
+        }
+        return new String(hexChars, StandardCharsets.UTF_8);
+    }
+
     /**
      * Converts big decimal to instant.
      *

clickhouse-client/src/main/java/com/clickhouse/client/data/BinaryStreamUtils.java

@@ -1649,6 +1649,19 @@ public static void writeString(OutputStream output, String value, Charset charse
         output.write(bytes);
     }
 
+    /**
+     * Writes a binary string to given output stream.
+     *
+     * @param output non-null output stream
+     * @param value  non-null byte array
+     * @throws IOException when failed to write value to output stream or reached
+     *                     end of the stream
+     */
+    public static void writeString(OutputStream output, byte[] value) throws IOException {
+        writeVarInt(output, value.length);
+        output.write(value);
+    }
+
     /**
      * Read varint from given input stream.
      *

clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseRowBinaryProcessor.java

@@ -331,13 +331,11 @@ private void buildMappingsForDataTypes() {
 
             // string and uuid
             buildMappings(deserializers, serializers,
-                    (r, f, c, i) -> ClickHouseStringValue.of(r, BinaryStreamUtils.readFixedString(i, c.getPrecision())),
-                    (v, f, c, o) -> BinaryStreamUtils.writeFixedString(o, v.asString(c.getPrecision()),
-                            c.getPrecision()),
-                    ClickHouseDataType.FixedString);
+                    (r, f, c, i) -> ClickHouseStringValue.of(r, i.readBytes(c.getPrecision())),
+                    (v, f, c, o) -> o.write(v.asBinary(c.getPrecision())), ClickHouseDataType.FixedString);
             buildMappings(deserializers, serializers,
-                    (r, f, c, i) -> ClickHouseStringValue.of(r, i.readUnicodeString()),
-                    (v, f, c, o) -> BinaryStreamUtils.writeString(o, v.asString()), ClickHouseDataType.String);
+                    (r, f, c, i) -> ClickHouseStringValue.of(r, i.readBytes(i.readVarInt())),
+                    (v, f, c, o) -> BinaryStreamUtils.writeString(o, v.asBinary()), ClickHouseDataType.String);
             buildMappings(deserializers, serializers,
                     (r, f, c, i) -> ClickHouseUuidValue.of(r, BinaryStreamUtils.readUuid(i)),
                     (v, f, c, o) -> BinaryStreamUtils.writeUuid(o, v.asUuid()), ClickHouseDataType.UUID);