Skip to content

Commit a3f80b6

Browse files
mzitnikmzitnik
authored
Issue 2118 (#2320)
* Solves issue #2118 * Added support for RowBinaryFormatWriter * Add testing for ALIAS and EPHEMERAL as well
1 parent 48cc994 commit a3f80b6

File tree

6 files changed

+151
-1
lines changed

6 files changed

+151
-1
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,21 @@ public final class ClickHouseColumn implements Serializable {
106106
private Map<Class<?>, Integer> mapKeyToVariantOrdNumMap;
107107
private Map<Class<?>, Integer> mapValueToVariantOrdNumMap;
108108

109+
public enum DefaultValue {
110+
DEFAULT("Default"),
111+
MATERIALIZED("MATERIALIZED"),
112+
EPHEMERAL("EPHEMERAL"),
113+
ALIAS("ALIAS");
114+
115+
public final String defaultValue;
116+
117+
DefaultValue(String defaultValue) {
118+
this.defaultValue = defaultValue;
119+
}
120+
}
121+
122+
private DefaultValue defaultValue;
123+
private String defaultExpression;
109124

110125
private static ClickHouseColumn update(ClickHouseColumn column) {
111126
column.enumConstants = ClickHouseEnum.EMPTY;
@@ -853,6 +868,14 @@ public void setHasDefault(boolean hasDefault) {
853868
this.hasDefault = hasDefault;
854869
}
855870

871+
public void setDefaultValue(DefaultValue defaultValue) { this.defaultValue = defaultValue; }
872+
873+
public DefaultValue getDefaultValue() { return defaultValue; }
874+
875+
public void setDefaultExpression(String defaultExpression) { this.defaultExpression = defaultExpression; }
876+
877+
public String getDefaultExpression() { return defaultExpression; }
878+
856879
public boolean isLowCardinality() {
857880
return !lowCardinalityDisabled && lowCardinality;
858881
}

‎client-v2/src/main/java/com/clickhouse/client/api/Client.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,6 +1340,9 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
13401340
.getOrDefault(tableName, Collections.emptyMap());
13411341
List<POJOSerializer> serializersForTable = new ArrayList<>();
13421342
for (ClickHouseColumn column : tableSchema.getColumns()) {
1343+
if (column.hasDefault() && column.getDefaultValue() != ClickHouseColumn.DefaultValue.DEFAULT ) {
1344+
continue;
1345+
}
13431346
POJOSerializer serializer = classSerializers.get(column.getColumnName());
13441347
if (serializer == null) {
13451348
throw new IllegalArgumentException("No serializer found for column '" + column.getColumnName() + "'. Did you forget to register it?");

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/RowBinaryFormatWriter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ public void commitRow() throws IOException {
5454
List<ClickHouseColumn> columnList = tableSchema.getColumns();
5555
for (int i = 0; i < row.length; i++) {
5656
ClickHouseColumn column = columnList.get(i);
57-
57+
// here we skip if we have a default value that is MATERIALIZED or ALIAS or ...
58+
if (column.hasDefault() && column.getDefaultValue() != ClickHouseColumn.DefaultValue.DEFAULT)
59+
continue;
5860
if (RowBinaryFormatSerializer.writeValuePreamble(out, defaultSupport, column, row[i])) {
5961
SerializerUtils.serializeData(out, row[i], column);
6062
}

‎client-v2/src/main/java/com/clickhouse/client/api/internal/TableSchemaParser.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,13 @@ public static TableSchema readTSKV(InputStream content, String table, String sql
2525
p.load(new StringReader(line.replaceAll("\t", "\n")));
2626
ClickHouseColumn column = ClickHouseColumn.of(p.getProperty("name"), p.getProperty("type"));
2727
String defaultType = p.getProperty("default_type");
28+
String defaultExpression = p.getProperty("default_expression");
2829
column.setHasDefault(defaultType != null && !defaultType.isEmpty());
30+
if ( column.hasDefault() ) {
31+
column.setDefaultValue(ClickHouseColumn.DefaultValue.valueOf(defaultType));
32+
if ( defaultExpression != null && !defaultExpression.isEmpty() )
33+
column.setDefaultExpression(defaultExpression);
34+
}
2935
columns.add(column);
3036
}
3137
}

‎client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,54 @@ public void testAdvancedWriter() throws Exception {
554554
}
555555
}
556556

557+
@Test
558+
public void testWriterWithMaterialize() throws Exception {
559+
String tableName = "table_name_with_materialize";
560+
String tableCreate = "CREATE TABLE \"" + tableName + "\" " +
561+
" (name String, " +
562+
" v1 Float32, " +
563+
" v2 Float32, " +
564+
" attrs Nullable(String), " +
565+
" corrected_time DateTime('UTC') DEFAULT now()," +
566+
" special_attr Nullable(Int8) DEFAULT -1," +
567+
" name_lower String MATERIALIZED lower(name)," +
568+
" name_lower_alias String ALIAS lower(name)," +
569+
" unhexed String EPHEMERAL," +
570+
" hexed FixedString(4) DEFAULT unhex(unhexed)" +
571+
" ) Engine = MergeTree ORDER by (name)";
572+
573+
initTable(tableName, tableCreate);
574+
575+
ZonedDateTime correctedTime = Instant.now().atZone(ZoneId.of("UTC"));
576+
Object[][] rows = new Object[][] {
577+
{"foo1", 0.3f, 0.6f, "a=1,b=2,c=5", correctedTime, 10, "Z��"},
578+
{"foo2", 0.6f, 0.1f, "a=1,b=2,c=5", correctedTime, null, "Z��"},
579+
{"foo3", 0.7f, 0.4f, "a=1,b=2,c=5", null, null, "Z��"},
580+
{"foo4", 0.8f, 0.5f, null, null, null, "Z��"},
581+
};
582+
583+
TableSchema schema = client.getTableSchema(tableName);
584+
585+
ClickHouseFormat format = ClickHouseFormat.RowBinaryWithDefaults;
586+
try (InsertResponse response = client.insert(tableName, out -> {
587+
RowBinaryFormatWriter w = new RowBinaryFormatWriter(out, schema, format);
588+
for (Object[] row : rows) {
589+
for (int i = 0; i < row.length; i++) {
590+
w.setValue(i + 1, row[i]);
591+
}
592+
w.commitRow();
593+
}
594+
}, format, new InsertSettings()).get()) {
595+
System.out.println("Rows written: " + response.getWrittenRows());
596+
}
597+
598+
List<GenericRecord> records = client.queryAll("SELECT * FROM \"" + tableName + "\"" );
599+
600+
for (GenericRecord record : records) {
601+
System.out.println("> " + record.getString(1) + ", " + record.getFloat(2) + ", " + record.getFloat(3));
602+
}
603+
}
604+
557605
@Test
558606
public void testCollectionInsert() throws Exception {
559607
String tableName = "very_long_table_name_with_uuid_" + UUID.randomUUID().toString().replace('-', '_');
@@ -705,6 +753,33 @@ public void testPOJOWithDynamicType() throws Exception {
705753
}
706754
}
707755

756+
@Test(groups = { "integration" }, enabled = true)
757+
public void insertSimplePOJOsWithMaterializeColumn() throws Exception {
758+
String tableName = "simple_pojo_table_with_materialize_column";
759+
String createSQL = SimplePOJO.generateTableCreateSQL(tableName);
760+
String uuid = UUID.randomUUID().toString();
761+
762+
initTable(tableName, createSQL);
763+
764+
client.register(SimplePOJO.class, client.getTableSchema(tableName));
765+
List<Object> simplePOJOs = new ArrayList<>();
766+
767+
for (int i = 0; i < 1000; i++) {
768+
simplePOJOs.add(new SimplePOJO());
769+
}
770+
settings.setQueryId(uuid);
771+
InsertResponse response = client.insert(tableName, simplePOJOs, settings).get(EXECUTE_CMD_TIMEOUT, TimeUnit.SECONDS);
772+
773+
OperationMetrics metrics = response.getMetrics();
774+
assertEquals(simplePOJOs.size(), metrics.getMetric(ServerMetrics.NUM_ROWS_WRITTEN).getLong());
775+
assertEquals(simplePOJOs.size(), response.getWrittenRows());
776+
assertTrue(metrics.getMetric(ClientMetrics.OP_DURATION).getLong() > 0);
777+
assertTrue(metrics.getMetric(ClientMetrics.OP_SERIALIZATION).getLong() > 0);
778+
assertEquals(metrics.getQueryId(), uuid);
779+
assertEquals(response.getQueryId(), uuid);
780+
}
781+
782+
708783
protected void initTable(String tableName, String createTableSQL) throws Exception {
709784
initTable(tableName, createTableSQL, new CommandSettings());
710785
}

‎client-v2/src/test/java/com/clickhouse/client/insert/SimplePOJO.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package com.clickhouse.client.insert;
2+
3+
import com.clickhouse.client.ClientTests;
4+
import lombok.Getter;
5+
import lombok.Setter;
6+
import org.apache.commons.lang3.RandomStringUtils;
7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
9+
10+
import java.util.Random;
11+
12+
@Getter
13+
@Setter
14+
public class SimplePOJO {
15+
16+
private static final Logger LOGGER = LoggerFactory.getLogger(SimplePOJO.class);
17+
private int int32;
18+
private String str;
19+
private String hexed;
20+
21+
public SimplePOJO() {
22+
long seed = System.currentTimeMillis();
23+
final Random random = new Random(seed);
24+
this.int32 = random.nextInt();
25+
this.str = RandomStringUtils.randomAlphabetic(1, 256);
26+
this.hexed = RandomStringUtils.randomAlphanumeric(4);
27+
}
28+
29+
public static String generateTableCreateSQL(String tableName) {
30+
return "CREATE TABLE " + tableName + " (" +
31+
"int32 Int32, " +
32+
"str String, " +
33+
"int64 Int64 MATERIALIZED abs(toInt64(int32)), " +
34+
"str_lower String ALIAS lower(str), " +
35+
"unhexed String EPHEMERAL, " +
36+
"hexed FixedString(4) DEFAULT unhex(unhexed), " +
37+
") ENGINE = MergeTree ORDER BY ()";
38+
}
39+
40+
}
41+

0 commit comments

Comments
 (0)