Skip to content

Commit 69fe388

Browse files
chernserchernser
committed
Merge branch 'main' into v2_connection_problems_investigation
2 parents a61536d + 252aa6c commit 69fe388

File tree

20 files changed

+403
-239
lines changed

20 files changed

+403
-239
lines changed

‎clickhouse-jdbc/pom.xml‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,18 +40,15 @@
4040
<groupId>${project.parent.groupId}</groupId>
4141
<artifactId>clickhouse-http-client</artifactId>
4242
<version>${revision}</version>
43-
<optional>true</optional>
4443
</dependency>
4544
<dependency>
4645
<groupId>org.apache.commons</groupId>
4746
<artifactId>commons-compress</artifactId>
48-
<optional>true</optional>
4947
</dependency>
5048

5149
<dependency>
5250
<groupId>org.lz4</groupId>
5351
<artifactId>lz4-pure-java</artifactId>
54-
<optional>true</optional>
5552
</dependency>
5653
<dependency>
5754
<groupId>com.google.code.gson</groupId>
@@ -100,7 +97,13 @@
10097
<artifactId>zstd-jni</artifactId>
10198
<scope>provided</scope>
10299
</dependency>
100+
<dependency>
101+
<groupId>org.slf4j</groupId>
102+
<artifactId>slf4j-api</artifactId>
103+
<version>${slf4j.version}</version>
104+
</dependency>
103105

106+
<!-- Test Dependencies -->
104107
<dependency>
105108
<groupId>${project.parent.groupId}</groupId>
106109
<artifactId>clickhouse-client</artifactId>
@@ -530,6 +533,76 @@
530533
</filters>
531534
</configuration>
532535
</execution>
536+
537+
538+
<execution>
539+
<id>shade-all-v2</id>
540+
<phase>package</phase>
541+
<goals>
542+
<goal>shade</goal>
543+
</goals>
544+
<configuration>
545+
<shadedArtifactAttached>true</shadedArtifactAttached>
546+
<createDependencyReducedPom>true</createDependencyReducedPom>
547+
<createSourcesJar>true</createSourcesJar>
548+
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
549+
<shadedClassifierName>shaded-all</shadedClassifierName>
550+
<relocations>
551+
<relocation>
552+
<pattern>com.google</pattern>
553+
<shadedPattern>${shade.base}.google</shadedPattern>
554+
</relocation>
555+
<relocation>
556+
<pattern>io.opencensus</pattern>
557+
<shadedPattern>${shade.base}.opencensus</shadedPattern>
558+
</relocation>
559+
<relocation>
560+
<pattern>io.perfmark</pattern>
561+
<shadedPattern>${shade.base}.perfmark</shadedPattern>
562+
</relocation>
563+
<relocation>
564+
<pattern>net.jpountz</pattern>
565+
<shadedPattern>${shade.base}.jpountz</shadedPattern>
566+
</relocation>
567+
<relocation>
568+
<pattern>okio</pattern>
569+
<shadedPattern>${shade.base}.okio</shadedPattern>
570+
</relocation>
571+
<relocation>
572+
<pattern>org.apache</pattern>
573+
<shadedPattern>${shade.base}.apache</shadedPattern>
574+
</relocation>
575+
</relocations>
576+
<transformers>
577+
<transformer
578+
implementation="org.apache.maven.plugins.shade.resource.ApacheLicenseResourceTransformer" />
579+
<transformer
580+
implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer" />
581+
<transformer
582+
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
583+
<transformer
584+
implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
585+
<manifestEntries>
586+
<Automatic-Module-Name>${project.groupId}.jdbc</Automatic-Module-Name>
587+
<Main-Class>${mainClass}</Main-Class>
588+
<Specification-Title>${spec.title}</Specification-Title>
589+
<Specification-Version>${spec.version}</Specification-Version>
590+
</manifestEntries>
591+
</transformer>
592+
</transformers>
593+
<filters>
594+
<filter>
595+
<artifact>*:*</artifact>
596+
<excludes>
597+
<exclude>google/**</exclude>
598+
<exclude>org/checkerframework/**</exclude>
599+
<exclude>org/codehaus/**</exclude>
600+
<exclude>**/module-info.class</exclude>
601+
</excludes>
602+
</filter>
603+
</filters>
604+
</configuration>
605+
</execution>
533606
</executions>
534607
</plugin>
535608
<plugin>

‎client-v2/pom.xml‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
<properties>
2020
<apache.httpclient.version>5.3.1</apache.httpclient.version>
21+
<shade.base>${project.groupId}.shaded</shade.base>
2122
</properties>
2223

2324
<dependencies>
@@ -157,29 +158,54 @@
157158
<artifactId>maven-shade-plugin</artifactId>
158159
<executions>
159160
<execution>
160-
<id>shade</id>
161+
<id>shade-all</id>
161162
<phase>package</phase>
162163
<goals>
163164
<goal>shade</goal>
164165
</goals>
165166
<configuration>
166167
<shadedArtifactAttached>true</shadedArtifactAttached>
167-
<shadedClassifierName>shaded</shadedClassifierName>
168+
<shadedClassifierName>all</shadedClassifierName>
168169
<createDependencyReducedPom>true</createDependencyReducedPom>
169170
<createSourcesJar>true</createSourcesJar>
170171
<promoteTransitiveDependencies>true</promoteTransitiveDependencies>
171-
<artifactSet>
172-
<includes>
173-
<include>com.clickhouse:clickhouse-data</include>
174-
<include>com.clickhouse:clickhouse-client</include>
175-
<include>org.lz4:lz4-pure-java</include>
176-
</includes>
177-
</artifactSet>
178172
<relocations>
179173
<relocation>
174+
<pattern>org.slf4j</pattern>
175+
<shadedPattern>${shade.base}.org.slf4j</shadedPattern>
176+
</relocation>
177+
178+
<relocation>
179+
<pattern>org.apache</pattern>
180+
<shadedPattern>${shade.base}.org.apache</shadedPattern>
181+
</relocation>
182+
183+
<relocation>
184+
<!-- lz4 -->
180185
<pattern>net.jpountz</pattern>
181-
<shadedPattern>${shade.base}.jpountz</shadedPattern>
186+
<shadedPattern>${shade.base}.net.jpountz.lz4</shadedPattern>
187+
</relocation>
188+
189+
<relocation>
190+
<pattern>org.ow2</pattern>
191+
<shadedPattern>${shade.base}.org.ow2</shadedPattern>
192+
</relocation>
193+
194+
<relocation>
195+
<pattern>com.fasterxml</pattern>
196+
<shadedPattern>${shade.base}.com.fasterxml</shadedPattern>
197+
</relocation>
198+
199+
<relocation>
200+
<pattern>org.roaringbitmap</pattern>
201+
<shadedPattern>${shade.base}.org.roaringbitmap</shadedPattern>
182202
</relocation>
203+
204+
<relocation>
205+
<pattern>org.objectweb</pattern>
206+
<shadedPattern>${shade.base}.org.objectweb</shadedPattern>
207+
</relocation>
208+
183209
</relocations>
184210
<transformers>
185211
<transformer
@@ -199,6 +225,7 @@
199225
</filters>
200226
</configuration>
201227
</execution>
228+
202229
</executions>
203230
</plugin>
204231
<plugin>

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,13 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
281281
*/
282282
double[] getDoubleArray(String colName);
283283

284+
/**
285+
*
286+
* @param colName
287+
* @return
288+
*/
289+
boolean[] getBooleanArray(String colName);
290+
284291
/**
285292
* Reads column with name `colName` as a string.
286293
*
@@ -503,6 +510,8 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
503510
*/
504511
double[] getDoubleArray(int index);
505512

513+
boolean[] getBooleanArray(int index);
514+
506515
Object[] getTuple(int index);
507516

508517
Object[] getTuple(String colName);

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/NativeFormatReader.java‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import com.clickhouse.client.api.data_formats.internal.BinaryStreamReader;
55
import com.clickhouse.client.api.query.QuerySettings;
66
import com.clickhouse.data.ClickHouseColumn;
7+
import com.clickhouse.data.ClickHouseDataType;
78

89
import java.io.EOFException;
910
import java.io.IOException;
@@ -58,13 +59,24 @@ private boolean readBlock() throws IOException {
5859
for (int i = 0; i < nColumns; i++) {
5960
ClickHouseColumn column = ClickHouseColumn.of(BinaryStreamReader.readString(input),
6061
BinaryStreamReader.readString(input));
62+
6163
names.add(column.getColumnName());
6264
types.add(column.getDataType().name());
6365

6466
List<Object> values = new ArrayList<>(nRows);
65-
for (int j = 0; j < nRows; j++) {
66-
Object value = binaryStreamReader.readValue(column);
67-
values.add(value);
67+
if (column.isArray()) {
68+
int[] sizes = new int[nRows];
69+
for (int j = 0; j < nRows; j++) {
70+
sizes[j] = Math.toIntExact(binaryStreamReader.readLongLE());
71+
}
72+
for (int j = 0; j < nRows; j++) {
73+
values.add(binaryStreamReader.readArrayItem(column.getNestedColumns().get(0), sizes[0]));
74+
}
75+
} else {
76+
for (int j = 0; j < nRows; j++) {
77+
Object value = binaryStreamReader.readValue(column);
78+
values.add(value);
79+
}
6880
}
6981
currentBlock.add(values);
7082
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,11 @@ public double[] getDoubleArray(String colName) {
511511
return getPrimitiveArray(colName);
512512
}
513513

514+
@Override
515+
public boolean[] getBooleanArray(String colName) {
516+
return getPrimitiveArray(colName);
517+
}
518+
514519
@Override
515520
public boolean hasValue(int colIndex) {
516521
return currentRecord.containsKey(getSchema().indexToName(colIndex - 1));
@@ -646,6 +651,11 @@ public double[] getDoubleArray(int index) {
646651
return getPrimitiveArray(schema.indexToName(index));
647652
}
648653

654+
@Override
655+
public boolean[] getBooleanArray(int index) {
656+
return getPrimitiveArray(schema.indexToName(index));
657+
}
658+
649659
@Override
650660
public Object[] getTuple(int index) {
651661
return readValue(index);

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryReaderBackedRecord.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,11 @@ public double[] getDoubleArray(String colName) {
158158
return reader.getDoubleArray(colName);
159159
}
160160

161+
@Override
162+
public boolean[] getBooleanArray(String colName) {
163+
return reader.getBooleanArray(colName);
164+
}
165+
161166
@Override
162167
public String getString(int index) {
163168
return reader.getString(index);
@@ -298,6 +303,11 @@ public double[] getDoubleArray(int index) {
298303
return reader.getDoubleArray(index);
299304
}
300305

306+
@Override
307+
public boolean[] getBooleanArray(int index) {
308+
return reader.getBooleanArray(index);
309+
}
310+
301311
@Override
302312
public Object[] getTuple(int index) {
303313
return reader.getTuple(index);

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReader.java‎

Lines changed: 44 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,6 @@
2828
import java.util.TimeZone;
2929
import java.util.UUID;
3030

31-
import static com.clickhouse.data.ClickHouseDataType.toObjectType;
32-
3331
/**
3432
* This class is not thread safe and should not be shared between multiple threads.
3533
* Internally it may use a shared buffer to read data from the input stream.
@@ -521,24 +519,59 @@ public static byte[] readNBytesLE(InputStream input, byte[] buffer, int offset,
521519
* @throws IOException when IO error occurs
522520
*/
523521
public ArrayValue readArray(ClickHouseColumn column) throws IOException {
524-
Class<?> itemType = column.getArrayBaseColumn().getDataType().getWiderPrimitiveClass();
525-
if (column.getArrayBaseColumn().isNullable()) {
526-
itemType = toObjectType(itemType);
527-
}
528522
int len = readVarInt(input);
529-
ArrayValue array = new ArrayValue(column.getArrayNestedLevel() > 1 ? ArrayValue.class : itemType, len);
530-
531523
if (len == 0) {
532-
return array;
524+
return new ArrayValue(Object.class, 0);
533525
}
534526

535-
for (int i = 0; i < len; i++) {
536-
array.set(i, readValue(column.getNestedColumns().get(0)));
527+
ArrayValue array;
528+
ClickHouseColumn itemTypeColumn = column.getNestedColumns().get(0);
529+
if (column.getArrayNestedLevel() == 1) {
530+
array = readArrayItem(itemTypeColumn, len);
531+
532+
} else {
533+
array = new ArrayValue(ArrayValue.class, len);
534+
for (int i = 0; i < len; i++) {
535+
array.set(i, readArray(itemTypeColumn));
536+
}
537537
}
538538

539539
return array;
540540
}
541541

542+
public ArrayValue readArrayItem(ClickHouseColumn itemTypeColumn, int len) throws IOException {
543+
ArrayValue array;
544+
if (itemTypeColumn.isNullable()) {
545+
array = new ArrayValue(Object.class, len);
546+
for (int i = 0; i < len; i++) {
547+
array.set(i, readValue(itemTypeColumn));
548+
}
549+
} else {
550+
Object firstValue = readValue(itemTypeColumn);
551+
Class<?> itemClass = firstValue.getClass();
552+
if (firstValue instanceof Byte) {
553+
itemClass = byte.class;
554+
} else if (firstValue instanceof Character) {
555+
itemClass = char.class;
556+
} else if (firstValue instanceof Short) {
557+
itemClass = short.class;
558+
} else if (firstValue instanceof Integer) {
559+
itemClass = int.class;
560+
} else if (firstValue instanceof Long) {
561+
itemClass = long.class;
562+
} else if (firstValue instanceof Boolean) {
563+
itemClass = boolean.class;
564+
}
565+
566+
array = new ArrayValue(itemClass, len);
567+
array.set(0, firstValue);
568+
for (int i = 1; i < len; i++) {
569+
array.set(i, readValue(itemTypeColumn));
570+
}
571+
}
572+
return array;
573+
}
574+
542575
public void skipValue(ClickHouseColumn column) throws IOException {
543576
readValue(column, null);
544577
}
@@ -557,8 +590,6 @@ public static class ArrayValue {
557590

558591
try {
559592
if (itemType.isArray()) {
560-
array = Array.newInstance(ArrayValue.class, length);
561-
} else if (itemType == List.class) {
562593
array = Array.newInstance(Object[].class, length);
563594
} else {
564595
array = Array.newInstance(itemType, length);

0 commit comments

Comments
 (0)