implemented reading JSON columns with different definitions

Author: chernser

clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java

@@ -37,11 +37,9 @@
 
 import java.io.Serializable;
 import java.lang.reflect.Array;
-import java.math.BigInteger;
 import java.time.OffsetDateTime;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -507,19 +505,19 @@ protected static int readColumn(String args, int startIndex, int len, String nam
             }
         } else if (args.startsWith(KEYWORD_JSON, i)) {
             int index = args.indexOf('(', i + KEYWORD_JSON.length());
-            if (index < i) {
-                throw new IllegalArgumentException(ERROR_MISSING_NESTED_TYPE);
-            }
-            i = ClickHouseUtils.skipBrackets(args, index, len, '(');
-            String originalTypeName = args.substring(startIndex, i);
-            List<ClickHouseColumn> nestedColumns = parse(args.substring(index + 1, i - 1));
-            if (nestedColumns.isEmpty()) {
-                throw new IllegalArgumentException("Nested should have at least one nested column");
+            if (index > i) {
+                i = ClickHouseUtils.skipBrackets(args, index, len, '(');
+                String originalTypeName = args.substring(startIndex, i);
+                List<String> params = new ArrayList<>();
+                List<ClickHouseColumn> nestedColumns = new ArrayList<>();
+                List<String> parameters = new ArrayList<>();
+                parseJSONColumn(args.substring(index + 1, i - 1), nestedColumns, parameters);
+                nestedColumns.sort(Comparator.comparing(o -> o.getDataType().name()));
+                column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality,
+                        parameters, nestedColumns);
+                fixedLength = false;
+                estimatedLength++;
             }
-            column = new ClickHouseColumn(ClickHouseDataType.JSON, name, originalTypeName, nullable, lowCardinality,
-                    null, nestedColumns);
-            fixedLength = false;
-            estimatedLength++;
         }
 
         if (column == null) {
@@ -674,6 +672,48 @@ public static List<ClickHouseColumn> parse(String args) {
         return Collections.unmodifiableList(c);
     }
 
+    public static final String JSON_MAX_PATHS_PARAM = "max_dynamic_paths";
+    public static final String JSON_MAX_DYN_TYPES_PARAM = "max_dynamic_types";
+    public static final String JSON_SKIP_MARKER = "SKIP";
+
+    public static void parseJSONColumn(String args, List<ClickHouseColumn> nestedColumns, List<String> parameters) {
+        if (args == null || args.isEmpty()) {
+            return;
+        }
+
+        String name = null;
+        ClickHouseColumn column = null;
+        StringBuilder builder = new StringBuilder();
+        for (int i = 0, len = args.length(); i < len; i++) {
+            char ch = args.charAt(i);
+            if (Character.isWhitespace(ch)) {
+                continue;
+            }
+
+            if (name == null) { // column name
+                i = ClickHouseUtils.readNameOrQuotedString(args, i, len, builder) - 1;
+                name = builder.toString();
+                if (name.startsWith(JSON_SKIP_MARKER)) {
+                    name = null; // skip parameters
+                    i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                } else if ( name.startsWith(JSON_MAX_PATHS_PARAM) || name.startsWith(JSON_MAX_DYN_TYPES_PARAM)) {
+                    parameters.add(name);
+                    name = null;
+                    i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                }
+                builder.setLength(0);
+            } else if (column == null) { // now type
+                LinkedList<ClickHouseColumn> colList = new LinkedList<>();
+                i = readColumn(args, i, len, name, colList) - 1;
+                nestedColumns.add(column = colList.getFirst());
+            } else { // prepare for next column
+                i = ClickHouseUtils.skipContentsUntil(args, i, len, ',') - 1;
+                name = null;
+                column = null;
+            }
+        }
+    }
+
     public ClickHouseColumn(ClickHouseDataType dataType, String columnName, String originalTypeName, boolean nullable,
             boolean lowCardinality, List<String> parameters, List<ClickHouseColumn> nestedColumns) {
         this(dataType, columnName, originalTypeName, nullable, lowCardinality, parameters, nestedColumns, ClickHouseEnum.EMPTY);

client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java

@@ -35,6 +35,8 @@
 import java.util.Set;
 import java.util.TimeZone;
 import java.util.UUID;
+import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * This class is not thread safe and should not be shared between multiple threads.
@@ -226,7 +228,7 @@ public <T> T readValue(ClickHouseColumn column, Class<?> typeHint) throws IOExce
                     if (jsonAsString) {
                         return (T) readString(input);
                     } else {
-                        return (T) readJsonData(input);
+                        return (T) readJsonData(input, actualColumn);
                     }
 //                case Object: // deprecated https://clickhouse.com/docs/en/sql-reference/data-types/object-data-type
                 case Array:
@@ -1192,16 +1194,20 @@ private ClickHouseColumn readDynamicData() throws IOException {
 
     private static final ClickHouseColumn JSON_PLACEHOLDER_COL = ClickHouseColumn.parse("v Dynamic").get(0);
 
-    private Map<String, Object> readJsonData(InputStream input) throws IOException {
+    private Map<String, Object> readJsonData(InputStream input, ClickHouseColumn column) throws IOException {
         int numOfPaths = readVarInt(input);
         if (numOfPaths == 0) {
             return Collections.emptyMap();
         }
 
         Map<String, Object> obj = new HashMap<>();
+
+        final int predefinedPaths = column.getNestedColumns().size();
+        final List<ClickHouseColumn> predefinedColumns = column.getNestedColumns();
         for (int i = 0; i < numOfPaths; i++) {
             String path = readString(input);
-            Object value = readValue(JSON_PLACEHOLDER_COL);
+            ClickHouseColumn dataColumn = i < predefinedPaths ? predefinedColumns.get(i) : JSON_PLACEHOLDER_COL;
+            Object value = readValue(dataColumn);
             obj.put(path, value);
         }
         return obj;

client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java

@@ -904,8 +904,12 @@ public Object[][] testJSONBinaryFormat_dp() {
 
         return new Object[][] {
                 {"JSON"},
-//                {"JSON(a Int32, d String)"},
+                {"JSON()"},
                 {"JSON(stat.name String, count Int32)"},
+                {"JSON(stat.name String, `comments` String)"},
+                {"JSON(max_dynamic_paths=3, stat.name String, SKIP alt_count)"},
+                {"JSON(max_dynamic_paths=3, stat.name String, SKIP REGEXP '^-.*')"},
+                {"JSON(max_dynamic_paths=3,SKIP REGEXP '^-.*',SKIP ff,   flags Array(Array(Array(Int8))), SKIP alt_count)"},
         };
     }