Merge pull request #1696 from ClickHouse/add-nested-serializer

Paultagoras · web-flow · commit a593b526fb3d · 2024-06-20T04:56:50.000-04:00
Updating Nested type code and fixing proxy test
diff --git a/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java b/clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java
@@ -589,7 +589,7 @@ private ClickHouseColumn(ClickHouseDataType dataType, String columnName, String
         this.originalTypeName = originalTypeName == null ? dataType.name() : originalTypeName;
         this.nullable = nullable;
         this.lowCardinality = lowCardinality;
-        this.hasDefault = originalTypeName != null && originalTypeName.toUpperCase().contains("DEFAULT");
+        this.hasDefault = false;
 
         if (parameters == null || parameters.isEmpty()) {
             this.parameters = Collections.emptyList();
@@ -734,6 +734,10 @@ public boolean hasDefault() {
         return hasDefault;
     }
 
+    public void setHasDefault(boolean hasDefault) {
+        this.hasDefault = hasDefault;
+    }
+
     public boolean isLowCardinality() {
         return !lowCardinalityDisabled && lowCardinality;
     }
diff --git a/client-v2/src/main/java/com/clickhouse/client/api/Client.java b/client-v2/src/main/java/com/clickhouse/client/api/Client.java
@@ -1,11 +1,7 @@
 package com.clickhouse.client.api;
 
 import com.clickhouse.client.ClickHouseClient;
-import com.clickhouse.client.ClickHouseClientBuilder;
-import com.clickhouse.client.ClickHouseConfig;
 import com.clickhouse.client.ClickHouseNode;
-import com.clickhouse.client.ClickHouseNodeSelector;
-import com.clickhouse.client.ClickHouseProtocol;
 import com.clickhouse.client.ClickHouseRequest;
 import com.clickhouse.client.ClickHouseResponse;
 import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
@@ -31,7 +27,6 @@
 import com.clickhouse.client.api.query.QuerySettings;
 import com.clickhouse.client.api.query.Records;
 import com.clickhouse.client.config.ClickHouseClientOption;
-import com.clickhouse.config.ClickHouseOption;
 import com.clickhouse.data.ClickHouseColumn;
 import com.clickhouse.data.ClickHouseDataStreamFactory;
 import com.clickhouse.data.ClickHouseFormat;
@@ -44,7 +39,6 @@
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.Serializable;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.net.URL;
@@ -465,7 +459,7 @@ public void register(Class<?> clazz, TableSchema schema) {
         this.getterMethods.put(clazz, getterMethods);//Store the getter methods for later use
 
         for (ClickHouseColumn column : schema.getColumns()) {
-            String columnName = column.getColumnName().toLowerCase().replace("_", "");
+            String columnName = column.getColumnName().toLowerCase().replace("_", "").replace(".","");
             serializers.add((obj, stream) -> {
                 if (!getterMethods.containsKey(columnName)) {
                     LOG.warn("No getter method found for column: {}", columnName);
diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/SerializerUtils.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/SerializerUtils.java
@@ -36,9 +36,6 @@ public static void serializeData(OutputStream stream, Object value, ClickHouseCo
             case Map:
                 serializeMapData(stream, value, column);
                 break;
-            case Nested:
-                LOG.error("Nested data type is not supported.");
-                break;
             default:
                 serializePrimitiveData(stream, value, column);
                 break;
diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java
@@ -22,7 +22,7 @@ public TableSchema createFromBinaryResponse(ClickHouseResponse response, String
             try {
                 p.clear();
                 p.load(new StringReader(values));
-                schema.addColumn(p.getProperty("name"), p.getProperty("type"));
+                schema.addColumn(p.getProperty("name"), p.getProperty("type"), p.getProperty("default_type"));
             } catch ( IOException e) {
                 throw new RuntimeException("Failed to parse table schema", e);
             }
diff --git a/client-v2/src/main/java/com/clickhouse/client/api/metadata/TableSchema.java b/client-v2/src/main/java/com/clickhouse/client/api/metadata/TableSchema.java
@@ -58,10 +58,16 @@ public boolean hasDefaults() {
     }
 
     public void addColumn(String name, String type) {
-        columns.add(ClickHouseColumn.of(name, type));
-        if (type.toUpperCase().contains("DEFAULT")) {
+        addColumn(name, type, "");
+    }
+    public void addColumn(String name, String type, String defaultType) {
+        ClickHouseColumn column = ClickHouseColumn.of(name, type);
+        if (defaultType.toUpperCase().contains("DEFAULT")) {
             hasDefaults = true;
+            column.setHasDefault(true);
         }
+        columns.add(column);
+
         Map<String, Object> columnMetadata = metadata.computeIfAbsent(name, k -> new HashMap<>());
             columnMetadata.put("type", type);
         colIndex.put(name, columns.size() - 1);
@@ -84,5 +90,17 @@ public String indexToName(int index) {
     public int nameToIndex(String name) {
         return colIndex.get(name).intValue();
     }
+
+    @Override
+    public String toString() {
+        return "TableSchema{" +
+                "tableName='" + tableName + '\'' +
+                ", databaseName='" + databaseName + '\'' +
+                ", columns=" + columns +
+                ", metadata=" + metadata +
+                ", colIndex=" + colIndex +
+                ", hasDefaults=" + hasDefaults +
+                '}';
+    }
 }
 
diff --git a/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java b/client-v2/src/test/java/com/clickhouse/client/ProxyTests.java
@@ -25,6 +25,7 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertThrows;
 import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
 
 public class ProxyTests extends BaseIntegrationTest{
     private Client client;
@@ -69,14 +70,14 @@ private void createTable(String tableQuery) throws ClickHouseException {
     }
 
 
-    @Test(groups = { "integration" }, enabled = false)
+    @Test(groups = { "integration" }, enabled = true)
     public void simpleProxyTest() throws Exception {
-        String tableName = "simple_pojo_proxy_table";
+        String tableName = "simple_pojo_enable_proxy_table";
         String createSQL = SamplePOJO.generateTableCreateSQL(tableName);
         System.out.println(createSQL);
         createTable(createSQL);
 
-        client.register(SamplePOJO.class, SamplePOJO.generateTableSchema(tableName));
+        client.register(SamplePOJO.class, client.getTableSchema(tableName, "default"));
         List<Object> simplePOJOs = new ArrayList<>();
 
         for (int i = 0; i < 1000; i++) {
@@ -94,24 +95,23 @@ public void simpleProxyTest() throws Exception {
 
     @Test(groups = { "integration" }, enabled = true)
     public void simpleDisabledProxyTest() throws Exception {
-        String tableName = "simple_pojo_proxy_table";
+        String tableName = "simple_pojo_disable_proxy_table";
         String createSQL = SamplePOJO.generateTableCreateSQL(tableName);
         System.out.println(createSQL);
         createTable(createSQL);
 
-        client.register(SamplePOJO.class, SamplePOJO.generateTableSchema(tableName));
+        client.register(SamplePOJO.class, client.getTableSchema(tableName, "default"));
         List<Object> simplePOJOs = new ArrayList<>();
 
         for (int i = 0; i < 1000; i++) {
             simplePOJOs.add(new SamplePOJO());
         }
-        //proxy.disable();
+        proxy.disable();
         try {
             InsertResponse response = client.insert(tableName, simplePOJOs).get(120, TimeUnit.SECONDS);
+            fail("Should have thrown exception.");
         } catch (Exception e) {
-            //FOR NOW THIS WILL FAIL
-            //IF IT SUCCEEDS (meaning the test fails), YAY PROXY WORKS WE SHOULD REIMPLEMENT THE TESTS!
-            assertTrue(e.getMessage().contains("Operation has likely timed out."));
+            assertTrue(e instanceof ClientException);
         }
     }
 }
diff --git a/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java b/client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java
@@ -11,6 +11,7 @@
 import com.clickhouse.client.api.enums.Protocol;
 import com.clickhouse.client.api.insert.InsertResponse;
 import com.clickhouse.client.api.insert.InsertSettings;
+import com.clickhouse.client.api.metadata.TableSchema;
 import com.clickhouse.client.api.metrics.ClientMetrics;
 import com.clickhouse.client.api.metrics.OperationMetrics;
 import com.clickhouse.client.api.metrics.ServerMetrics;
@@ -58,8 +59,7 @@ public void insertSimplePOJOs() throws Exception {
         String createSQL = SamplePOJO.generateTableCreateSQL(tableName);
         System.out.println(createSQL);
         createTable(createSQL);
-
-        client.register(SamplePOJO.class, SamplePOJO.generateTableSchema(tableName));
+        client.register(SamplePOJO.class, client.getTableSchema(tableName, "default"));
         List<Object> simplePOJOs = new ArrayList<>();
 
         for (int i = 0; i < 1000; i++) {
diff --git a/client-v2/src/test/java/com/clickhouse/client/insert/SamplePOJO.java b/client-v2/src/test/java/com/clickhouse/client/insert/SamplePOJO.java

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ public TableSchema createFromBinaryResponse(ClickHouseResponse response, String`
`22`	`22`	`try {`
`23`	`23`	`p.clear();`
`24`	`24`	`p.load(new StringReader(values));`
`25`		`- schema.addColumn(p.getProperty("name"), p.getProperty("type"));`
	`25`	`+ schema.addColumn(p.getProperty("name"), p.getProperty("type"), p.getProperty("default_type"));`
`26`	`26`	`} catch ( IOException e) {`
`27`	`27`	`throw new RuntimeException("Failed to parse table schema", e);`
`28`	`28`	`}`