Skip to content

Commit b624c2d

Browse files
chernserchernser
committed
Merge branch 'main' into jpa_example
2 parents a66b3b0 + da90573 commit b624c2d

File tree

31 files changed

+594
-541
lines changed

31 files changed

+594
-541
lines changed

‎CHANGELOG.md‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,18 @@
1-
## Latest
1+
## 0.8.3
22

3-
## 0.8.2
3+
### Improvements
4+
- [client-v2] Support for native LZ4 compression (https://github.com/ClickHouse/clickhouse-java/issues/2274)
45

5-
### New Features
6+
### Bug Fixes
7+
- [jdbc-v2] Fixed several issues with reading database metadata in JDBC driver. (https://github.com/ClickHouse/clickhouse-java/issues/2282)
8+
- [jdbc-v2] Fixed settings client name in JDBC. (https://github.com/ClickHouse/clickhouse-java/issues/2233)
9+
- [client-v2] Fixed reading data from columns defined as `Nullable(FixedString(N))`. (https://github.com/ClickHouse/clickhouse-java/issues/2218)
10+
- [jdbc-v2] Fixed SQL parser failure to parse SQL statement with comments (https://github.com/ClickHouse/clickhouse-java/issues/2217)
11+
- [client-v2] Fixed issue with excessive logging (https://github.com/ClickHouse/clickhouse-java/issues/2201)
12+
- [jdbc-v2] Fixed handling IP addresses (https://github.com/ClickHouse/clickhouse-java/issues/2140)
13+
- [jdbc] - Fixed missing LZ4 dependency in shaded package (https://github.com/ClickHouse/clickhouse-java/issues/2275)
14+
15+
## 0.8.2
616

717
### Bug Fixes
818
- [jdbc-v2] - Significantly improved performance of JDBC inserts. (https://github.com/ClickHouse/clickhouse-java/pull/2165)

‎clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -532,7 +532,17 @@ public static String readVersionFromResource(String resourceFilePath) {
532532
tmpVersion = tmp;
533533
}
534534
} catch (Exception e) {
535-
// ignore
535+
try(InputStream in = ClickHouseClientOption.class.getClassLoader().getResourceAsStream(resourceFilePath)) {
536+
Properties p = new Properties();
537+
p.load(in);
538+
539+
String tmp = p.getProperty("version");
540+
if (tmp != null && !tmp.isEmpty() && !tmp.equals("${revision}")) {
541+
tmpVersion = tmp;
542+
}
543+
} catch (Exception ee) {
544+
// ignore
545+
}
536546
}
537547
return tmpVersion;
538548
}

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ private static ClickHouseColumn update(ClickHouseColumn column) {
129129
case Bool:
130130
column.template = ClickHouseBoolValue.ofNull();
131131
break;
132+
case Enum:
132133
case Enum8:
133134
case Enum16:
134135
column.template = ClickHouseEnumValue
@@ -746,7 +747,7 @@ public boolean isArray() {
746747
}
747748

748749
public boolean isEnum() {
749-
return dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16;
750+
return dataType == ClickHouseDataType.Enum8 || dataType == ClickHouseDataType.Enum16 || dataType == ClickHouseDataType.Enum;
750751
}
751752

752753
public boolean isFixedLength() {

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseDataType.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public enum ClickHouseDataType {
5555
DateTime(LocalDateTime.class, true, false, false, 0, 29, 0, 0, 9, false, 0x11, "TIMESTAMP"),
5656
DateTime32(LocalDateTime.class, true, false, false, 4, 19, 0, 0, 0, false, 0x12),
5757
DateTime64(LocalDateTime.class, true, false, false, 8, 29, 3, 0, 9, false, 0x14), // we always write timezone as argument
58+
Enum(String.class, true, true, false, 0, 0, 0, 0, 0, false),
5859
Enum8(String.class, true, true, false, 1, 0, 0, 0, 0, false, 0x17, "ENUM"),
5960
Enum16(String.class, true, true, false, 2, 0, 0, 0, 0, false, 0x18),
6061
FixedString(String.class, true, true, false, 0, 0, 0, 0, 0, false, 0x16, "BINARY"),
@@ -90,6 +91,8 @@ public enum ClickHouseDataType {
9091
Decimal64(BigDecimal.class, true, false, true, 8, 18, 18, 0, 18, false, 0x1A),
9192
Decimal128(BigDecimal.class, true, false, true, 16, 38, 38, 0, 38, false, 0x1B),
9293
Decimal256(BigDecimal.class, true, false, true, 32, 76, 20, 0, 76, false, 0x1C),
94+
95+
BFloat16(Float.class, false, true, true, 2, 3, 0, 0, 16, false, 0x31),
9396
Float32(Float.class, false, true, true, 4, 12, 0, 0, 38, false, 0x0D, "FLOAT", "REAL", "SINGLE"),
9497
Float64(Double.class, false, true, true, 8, 22, 0, 0, 308, false, 0x0E, "DOUBLE", "DOUBLE PRECISION"),
9598
IPv4(Inet4Address.class, false, true, false, 4, 10, 0, 0, 0, false, 0x28, "INET4"),
@@ -99,10 +102,13 @@ public enum ClickHouseDataType {
99102
Polygon(Object.class, false, true, true, 0, 0, 0, 0, 0, true, 0x2C), // same as Array(Ring)
100103
MultiPolygon(Object.class, false, true, true, 0, 0, 0, 0, 0, true, 0x2C), // same as Array(Polygon)
101104
Ring(Object.class, false, true, true, 0, 0, 0, 0, 0, true, 0x2C), // same as Array(Point)
105+
LineString( Object.class, false, true, true, 0, 0, 0, 0, 0, true, 0x2C), // same as Array(Point)
106+
MultiLineString(Object.class, false, true, true, 0, 0, 0, 0, 0, true, 0x2C), // same as Array(Ring)
107+
102108
JSON(Object.class, false, false, false, 0, 0, 0, 0, 0, true, 0x30),
103109
@Deprecated
104110
Object(Object.class, true, true, false, 0, 0, 0, 0, 0, true),
105-
String(String.class, false, false, false, 0, 0, 0, 0, 0, false, 0x15, "BINARY LARGE OBJECT", "BINARY VARYING", "BLOB",
111+
String(String.class, false, true, false, 0, 0, 0, 0, 0, false, 0x15, "BINARY LARGE OBJECT", "BINARY VARYING", "BLOB",
106112
"BYTEA", "CHAR", "CHAR LARGE OBJECT", "CHAR VARYING", "CHARACTER", "CHARACTER LARGE OBJECT",
107113
"CHARACTER VARYING", "CLOB", "GEOMETRY", "LONGBLOB", "LONGTEXT", "MEDIUMBLOB", "MEDIUMTEXT",
108114
"NATIONAL CHAR", "NATIONAL CHAR VARYING", "NATIONAL CHARACTER", "NATIONAL CHARACTER LARGE OBJECT",
@@ -113,6 +119,8 @@ public enum ClickHouseDataType {
113119
Nested(Object.class, true, true, false, 0, 0, 0, 0, 0, true, 0x2F),
114120
Tuple(List.class, true, true, false, 0, 0, 0, 0, 0, true, 0x1F),
115121
Nothing(Object.class, false, true, false, 0, 0, 0, 0, 0, true, 0x00),
122+
LowCardinality(Object.class, true, true, false, 0, 0, 0, 0, 0, true, 0x26),
123+
Nullable( Object.class, true, true, false, 0, 0, 0, 0, 0, true, 0x23),
116124
SimpleAggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0, false, 0x2E),
117125
// implementation-defined intermediate state
118126
AggregateFunction(String.class, true, true, false, 0, 0, 0, 0, 0, true),
@@ -364,7 +372,7 @@ public byte getTag() {
364372
* @return true if the type name is an alias; false otherwise
365373
*/
366374
public static boolean isAlias(String typeName) {
367-
return typeName != null && !typeName.isEmpty() && allAliases.contains(typeName.trim().toUpperCase());
375+
return typeName != null && !typeName.isEmpty() && allAliases.contains(typeName.trim());
368376
}
369377

370378
/**

‎clickhouse-data/src/test/java/com/clickhouse/data/ClickHouseColumnTest.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,9 @@ public void testEnum(String typeName) {
253253
Assert.assertEquals(column.getEnumConstants().name(10), "Query'Finish");
254254
Assert.assertEquals(column.getEnumConstants().value("Query'Start"), 1);
255255
Assert.assertEquals(column.getEnumConstants().value("Query'Finish"), 10);
256-
Assert.assertTrue(column.isFixedLength(), "Should have fixed length in byte");
256+
if (column.getDataType() != ClickHouseDataType.Enum) { // virtual type
257+
Assert.assertTrue(column.isFixedLength(), "Should have fixed length in byte");
258+
}
257259
Assert.assertEquals(column.getEstimatedLength(), column.getDataType().getByteLength());
258260
}
259261

@@ -417,7 +419,8 @@ public boolean isWidenUnsignedTypes() {
417419
for (ClickHouseDataType type : ClickHouseDataType.values()) {
418420
// skip advanced types
419421
if (type.isNested() || type == ClickHouseDataType.AggregateFunction
420-
|| type == ClickHouseDataType.SimpleAggregateFunction) {
422+
|| type == ClickHouseDataType.SimpleAggregateFunction || type == ClickHouseDataType.Enum
423+
|| type == ClickHouseDataType.Nullable || type == ClickHouseDataType.BFloat16) {
421424
continue;
422425
}
423426

‎clickhouse-jdbc/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -420,6 +420,7 @@
420420
<include>org.apache.httpcomponents.core5:httpcore5</include>
421421
<include>org.apache.httpcomponents.core5:httpcore5-h2</include>
422422
<include>com.clickhouse:jdbc-v2</include>
423+
<include>org.apache.commons:commons-compress</include>
423424
<include>org.lz4:lz4-java</include>
424425
</includes>
425426
</artifactSet>

‎client-v2/src/main/java/com/clickhouse/client/api/Client.java‎

Lines changed: 64 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1425,9 +1425,23 @@ public CompletableFuture<InsertResponse> insert(String tableName, List<?> data,
14251425
* @return {@code CompletableFuture<InsertResponse>} - a promise to insert response
14261426
*/
14271427
public CompletableFuture<InsertResponse> insert(String tableName, InputStream data, ClickHouseFormat format) {
1428-
return insert(tableName, data, format, new InsertSettings());
1428+
return insert(tableName, Collections.emptyList(), data, format, new InsertSettings());
14291429
}
14301430

1431+
/**
1432+
* <p>Sends write request to database. Input data is read from the input stream.</p>
1433+
*
1434+
* @param tableName - destination table name
1435+
* @param data - data stream to insert
1436+
* @param format - format of the data in the stream
1437+
* @return {@code CompletableFuture<InsertResponse>} - a promise to insert response
1438+
*/
1439+
public CompletableFuture<InsertResponse> insert(String tableName, List<String> columnNames, InputStream data, ClickHouseFormat format) {
1440+
return insert(tableName, columnNames, data, format, new InsertSettings());
1441+
}
1442+
1443+
1444+
14311445
/**
14321446
* Sends write request to database. Input data is read from the input stream.
14331447
*
@@ -1441,6 +1455,23 @@ public CompletableFuture<InsertResponse> insert(String tableName,
14411455
InputStream data,
14421456
ClickHouseFormat format,
14431457
InsertSettings settings) {
1458+
return insert(tableName, Collections.emptyList(), data, format, settings);
1459+
}
1460+
/**
1461+
* Sends write request to database. Input data is read from the input stream.
1462+
*
1463+
* @param tableName - destination table name
1464+
* @param columnNames - list of column names to insert data into. If null or empty, all columns will be used.
1465+
* @param data - data stream to insert
1466+
* @param format - format of the data in the stream
1467+
* @param settings - insert operation settings
1468+
* @return {@code CompletableFuture<InsertResponse>} - a promise to insert response
1469+
*/
1470+
public CompletableFuture<InsertResponse> insert(String tableName,
1471+
List<String> columnNames,
1472+
InputStream data,
1473+
ClickHouseFormat format,
1474+
InsertSettings settings) {
14441475

14451476
final int writeBufferSize = settings.getInputStreamCopyBufferSize() <= 0 ?
14461477
Integer.parseInt(configuration.getOrDefault(ClientConfigProperties.CLIENT_NETWORK_BUFFER_SIZE.getKey(),
@@ -1451,7 +1482,7 @@ public CompletableFuture<InsertResponse> insert(String tableName,
14511482
throw new IllegalArgumentException("Buffer size must be greater than 0");
14521483
}
14531484

1454-
return insert(tableName, new DataStreamWriter() {
1485+
return insert(tableName, columnNames, new DataStreamWriter() {
14551486
@Override
14561487
public void onOutput(OutputStream out) throws IOException {
14571488
byte[] buffer = new byte[writeBufferSize];
@@ -1480,11 +1511,29 @@ public void onRetry() throws IOException {
14801511
* @return {@code CompletableFuture<InsertResponse>} - a promise to insert response
14811512
*/
14821513
public CompletableFuture<InsertResponse> insert(String tableName,
1514+
DataStreamWriter writer,
1515+
ClickHouseFormat format,
1516+
InsertSettings settings) {
1517+
return insert(tableName, Collections.emptyList(), writer, format, settings);
1518+
}
1519+
1520+
/**
1521+
* Does an insert request to a server. Data is pushed when a {@link DataStreamWriter#onOutput(OutputStream)} is called.
1522+
*
1523+
* @param tableName - target table name
1524+
* @param columnNames - list of column names to insert data into. If null or empty, all columns will be used.
1525+
* @param writer - {@link DataStreamWriter} implementation
1526+
* @param format - source format
1527+
* @param settings - operation settings
1528+
* @return {@code CompletableFuture<InsertResponse>} - a promise to insert response
1529+
*/
1530+
public CompletableFuture<InsertResponse> insert(String tableName,
1531+
List<String> columnNames,
14831532
DataStreamWriter writer,
14841533
ClickHouseFormat format,
14851534
InsertSettings settings) {
14861535

1487-
String operationId = (String) settings.getOperationId();
1536+
String operationId = settings.getOperationId();
14881537
ClientStatisticsHolder clientStats = null;
14891538
if (operationId != null) {
14901539
clientStats = globalClientStats.remove(operationId);
@@ -1509,8 +1558,18 @@ public CompletableFuture<InsertResponse> insert(String tableName,
15091558

15101559
settings.setOption(ClientConfigProperties.INPUT_OUTPUT_FORMAT.getKey(), format.name());
15111560
final InsertSettings finalSettings = settings;
1512-
final String sqlStmt = "INSERT INTO " + tableName + " FORMAT " + format.name();
1513-
finalSettings.serverSetting(ClickHouseHttpProto.QPARAM_QUERY_STMT, sqlStmt);
1561+
1562+
StringBuilder sqlStmt = new StringBuilder("INSERT INTO ").append(tableName);
1563+
if (columnNames != null && !columnNames.isEmpty()) {
1564+
sqlStmt.append(" (");
1565+
for (String columnName : columnNames) {
1566+
sqlStmt.append(columnName).append(", ");
1567+
}
1568+
sqlStmt.deleteCharAt(sqlStmt.length() - 2);
1569+
sqlStmt.append(")");
1570+
}
1571+
sqlStmt.append(" FORMAT ").append(format.name());
1572+
finalSettings.serverSetting(ClickHouseHttpProto.QPARAM_QUERY_STMT, sqlStmt.toString());
15141573
responseSupplier = () -> {
15151574
long startTime = System.nanoTime();
15161575
// Selecting some node

‎client-v2/src/test/java/com/clickhouse/client/datatypes/DataTypeTests.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -156,16 +156,18 @@ public void testVariantWithSimpleDataTypes() throws Exception {
156156
b.append(table).append(" ( rowId Int64, field Variant(String, ").append(dataType.name());
157157

158158
switch (dataType) {
159+
case BFloat16:
160+
// TODO: add support
161+
continue dataTypesLoop;
162+
// skipped
159163
case String:
160164
case FixedString:
161165
case Nothing:
162166
case Variant:
163167
case JSON:
164168
case Object:
165169
case Dynamic:
166-
// skipped
167-
continue dataTypesLoop;
168-
170+
// no tests or tested in other tests
169171
case Decimal:
170172
case Decimal32:
171173
case Decimal64:
@@ -179,6 +181,11 @@ public void testVariantWithSimpleDataTypes() throws Exception {
179181
case AggregateFunction:
180182
case Enum8:
181183
case Enum16:
184+
case Enum:
185+
case Nullable: // virtual type
186+
case LowCardinality: // virtual type
187+
case LineString: // same as Ring
188+
case MultiLineString: // same as MultiPolygon
182189
// tested separately
183190
continue dataTypesLoop;
184191

@@ -414,6 +421,9 @@ public void testDynamicWithPrimitives() throws Exception {
414421
int rowId = 0;
415422
for (ClickHouseDataType dataType : ClickHouseDataType.values()) {
416423
switch (dataType) {
424+
case BFloat16:
425+
// TODO: add support
426+
continue;
417427
case Array:
418428
case Map:
419429
case AggregateFunction:
@@ -428,6 +438,11 @@ public void testDynamicWithPrimitives() throws Exception {
428438
case Tuple:
429439
case Variant:
430440
case Decimal: // virtual type
441+
case Nullable: // virtual type
442+
case LowCardinality: // virtual type
443+
case Enum: // virtual type
444+
case LineString: // same as Ring
445+
case MultiLineString: // same as MultiPolygon
431446
// no tests or tested in other tests
432447
continue;
433448
default:
@@ -447,7 +462,7 @@ public void testDynamicWithPrimitives() throws Exception {
447462
}
448463
}
449464

450-
Assert.assertNotNull(value);
465+
Assert.assertNotNull(value, "Value for " + dataType.name() + " should not be null.");
451466

452467
List<DTOForDynamicPrimitivesTests> data = new ArrayList<>();
453468
data.add(new DTOForDynamicPrimitivesTests(rowId++, value));

‎client-v2/src/test/java/com/clickhouse/client/insert/InsertTests.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,31 @@ public void insertRawDataSimple(String tableName) throws Exception {
294294
assertEquals((int)response.getWrittenRows(), numberOfRecords );
295295
}
296296

297+
@Test(groups = { "integration" })
298+
public void insertRawDataFewerColumns() throws Exception {
299+
final String tableName = "raw_data_select_columns_table";
300+
final String createSQL = "CREATE TABLE " + tableName +
301+
" (Id UInt32, event_ts Timestamp, name String, p1 Int64, p2 String, p3 String, p4 Int8) ENGINE = MergeTree() ORDER BY ()";
302+
List<String> columnNames = Arrays.asList("Id", "event_ts", "name", "p1", "p2");
303+
304+
initTable(tableName, createSQL);
305+
306+
settings.setInputStreamCopyBufferSize(8198 * 2);
307+
ByteArrayOutputStream data = new ByteArrayOutputStream();
308+
PrintWriter writer = new PrintWriter(data);
309+
for (int i = 0; i < 1000; i++) {
310+
writer.printf("%d\t%s\t%s\t%d\t%s\n", i, "2021-01-01 00:00:00", "name" + i, i, "p2");
311+
}
312+
writer.flush();
313+
InsertResponse response = client.insert(tableName, columnNames, new ByteArrayInputStream(data.toByteArray()),
314+
ClickHouseFormat.TSV, settings).get(30, TimeUnit.SECONDS);
315+
OperationMetrics metrics = response.getMetrics();
316+
assertEquals((int)response.getWrittenRows(), 1000 );
317+
318+
List<GenericRecord> records = client.queryAll("SELECT * FROM " + tableName);
319+
assertEquals(records.size(), 1000);
320+
}
321+
297322
@DataProvider(name = "insertRawDataSimpleDataProvider")
298323
public static Object[][] insertRawDataSimpleDataProvider() {
299324
return new Object[][] {

0 commit comments

Comments
 (0)