Merge branch 'main' into clientv2_retries

Author: chernser

.github/workflows/build.yml

@@ -51,15 +51,20 @@ jobs:
         uses: actions/setup-java@v4
         with:
           distribution: "temurin"
-          java-version: 8
+          java-version: |
+            8
+            21
           cache: "maven"
       - name: Build and install libraries
         run: mvn --batch-mode --no-transfer-progress --show-version --strict-checksums --threads 2 -Dmaven.wagon.rto=30000 -DclickhouseVersion=$PREFERRED_LTS_VERSION -Dj8 install
       - name: Compile examples
         run: |
           export LIB_VER=$(grep '<revision>' pom.xml | sed -e 's|[[:space:]]*<[/]*revision>[[:space:]]*||g')
           find `pwd`/examples -type f -name pom.xml -exec sed -i -e "s|\(<clickhouse-java.version>\).*\(<\)|\1$LIB_VER\2|g" {} \;
-          for d in $(ls -d `pwd`/examples/*/); do cd $d && mvn  --batch-mode --no-transfer-progress clean compile; done
+          for d in $(ls -d `pwd`/examples/*/); do \
+            if [ -e $d/pom.xml ]; then cd $d && mvn  --batch-mode --no-transfer-progress clean compile; fi;
+            if [ -e $d/gradlew ]; then cd $d && ./gradlew clean build; fi;
+          done
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: failure()
@@ -265,6 +270,7 @@ jobs:
           java-version: |
             8
             17
+            21
           cache: "maven"
       - name: Setup Toolchain
         shell: bash

.github/workflows/nightly.yml

@@ -12,7 +12,7 @@ on:
 
 env:
   CHC_BRANCH: "main"
-  CHC_VERSION: "0.6.3"
+  CHC_VERSION: "0.6.4"
 
 jobs:
   nightly:

.github/workflows/release.yml

@@ -6,7 +6,7 @@ on:
       version:
         description: "Release version"
         required: true
-        default: "0.6.3-SNAPSHOT"
+        default: "0.6.4-SNAPSHOT"
 
 jobs:
   release:

CHANGELOG.md

@@ -1,5 +1,56 @@
 ## Latest
 
+## 0.6.4
+
+### Deprecations
+- Following components will be deprecated and archived in next release:
+  - clickhouse-cli-client
+  - clickhouse-grpc-client
+- No more builds for non-lts Java versions - no more Java 9 release builds.
+- Lowest supported Java version will be 11.
+  - Java 11 support will be ended before the end of 2023. 
+  - It is recommended to use Java 21.
+
+### Important Changes
+- [Client-V1] Fix for handling DateTime without timezone when `session_timezone` is set. Now server timezone 
+is parsed from server response when present (https://github.com/ClickHouse/clickhouse-java/issues/1464)
+
+### New Features 
+- [Client-V1/Apache HTTP] More configuration parameters for connection management. Useful for tuning performance.
+(https://github.com/ClickHouse/clickhouse-java/pull/1771)
+    - com.clickhouse.client.config.ClickHouseClientOption#CONNECTION_TTL - to configure connection time-to-live
+    - com.clickhouse.client.http.config.ClickHouseHttpOption#KEEP_ALIVE_TIMEOUT - to configure keep-alive timeout
+    - com.clickhouse.client.http.config.ClickHouseHttpOption#CONNECTION_REUSE_STRATEGY - defines how connection pool behaves.
+If `FIFO` is selected then connections are reused in the order they were created. It results in even distribution of connections.
+If `LIFO` is selected then connections are reused as soon they are returned to the pool. 
+Note: only for `APACHE_HTTP_CLIENT` connection provider. 
+    - Additionally switched to using LAX connection pool for Apache Connection Manager to improve performance 
+for concurrent requests.
+- [Client-V2] Connection pool configuration https://github.com/ClickHouse/clickhouse-java/pull/1766
+    - com.clickhouse.client.api.Client.Builder.setConnectionRequestTimeout - to configure connection request timeout.
+Important when there are no connections available in the pool to fail fast. 
+    - com.clickhouse.client.api.Client.Builder.setMaxConnections - configures how soft limit of connections per host.
+Note: Total number of connections is unlimited because in most cases there is one host.
+    - com.clickhouse.client.api.Client.Builder.setConnectionTTL - to limit connection live ignoring keep-alive from server.
+    - com.clickhouse.client.api.Client.Builder.setConnectionReuseStrategy - to configure how connections are used.
+Select FIFO to reuse connections evenly or LIFO (default) to reuse the most recently active connections.
+- [Client-V2] All operations are now executed in calling thread to avoid extra threads creation. 
+Async operations can be enabled by `com.clickhouse.client.api.Client.Builder.useAsyncRequests` (https://github.com/ClickHouse/clickhouse-java/pull/1767)
+- [Client-V2] Content and HTTP native compression is supported now Currently only LZ4 is available. (https://github.com/ClickHouse/clickhouse-java/pull/1761) 
+- [Client-V2] HTTPS support added. Required to communicate with ClickHouse Cloud Services.
+Client certificates are supported, too. (https://github.com/ClickHouse/clickhouse-java/pull/1753)
+- [Client-V2] Added support for HTTP proxy (https://github.com/ClickHouse/clickhouse-java/pull/1748)
+
+### Documentation
+- [Client-V2] Spring Demo Service as usage example (https://github.com/ClickHouse/clickhouse-java/pull/1765)
+- [Client-V2] Examples for using text based formats (https://github.com/ClickHouse/clickhouse-java/pull/1752)
+
+
+### Bug Fixes
+- [Client-V2] Data is read fully from a stream. Important for Cloud instances (https://github.com/ClickHouse/clickhouse-java/pull/1759)
+- [Client-V2] Timezone from a server response is now used to parse DateTime values (https://github.com/ClickHouse/clickhouse-java/pull/1763)
+- [Client-V1] Timezone from a server response is now used to parse DateTime values (https://github.com/ClickHouse/clickhouse-java/issues/1464)
+
 ## 0.6.3
 
 ### Important Changes

clickhouse-cli-client/src/test/java/com/clickhouse/client/cli/ClickHouseCommandLineClientTest.java

@@ -10,6 +10,7 @@
 import com.clickhouse.client.ClientIntegrationTest;
 import com.clickhouse.client.cli.config.ClickHouseCommandLineOption;
 import com.clickhouse.data.ClickHouseCompression;
+import com.clickhouse.data.ClickHouseFormat;
 import org.testcontainers.containers.GenericContainer;
 import org.testng.SkipException;
 import org.testng.annotations.BeforeClass;
@@ -199,4 +200,9 @@ public void testLoadBalancingPolicyFailover(ClickHouseLoadBalancingPolicy loadBa
     public void testFailover() {
         throw new SkipException("Skip due to failover is not supported");
     }
+
+    @Override
+    public void testServerTimezoneAppliedFromHeader(ClickHouseFormat format) throws Exception {
+        throw new SkipException("Skip due to session timezone is not supported");
+    }
 }

clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseStreamResponse.java

@@ -3,10 +3,13 @@
 import java.io.IOException;
 import java.io.Serializable;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
 
+import com.clickhouse.client.config.ClickHouseClientOption;
+import com.clickhouse.config.ClickHouseOption;
 import com.clickhouse.data.ClickHouseColumn;
 import com.clickhouse.data.ClickHouseDataProcessor;
 import com.clickhouse.data.ClickHouseDataStreamFactory;
@@ -69,6 +72,11 @@ protected ClickHouseStreamResponse(ClickHouseConfig config, ClickHouseInputStrea
         this.timeZone = timeZone;
         boolean hasError = true;
         try {
+            if (timeZone != null && config.isUseServerTimeZone() && !config.getUseTimeZone().equals(timeZone)) {
+                Map<ClickHouseOption, Serializable> configOptions = new HashMap<>(config.getAllOptions());
+                configOptions.put(ClickHouseClientOption.SERVER_TIME_ZONE, timeZone.getID());
+                config = new ClickHouseConfig(configOptions);
+            }
             this.processor = ClickHouseDataStreamFactory.getInstance().getProcessor(config, input, null, settings,
                     columns);
             hasError = false;

clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java

@@ -425,7 +425,16 @@ public enum ClickHouseClientOption implements ClickHouseOption {
     /**
      * Query ID to be attached to an operation
      */
-    QUERY_ID("query_id", "", "Query id");
+    QUERY_ID("query_id", "", "Query id"),
+
+
+    /**
+     * Connection time to live in milliseconds. 0 or negative number means no limit.
+     * Can be used to override keep-alive time suggested by a server.
+     */
+    CONNECTION_TTL("connection_ttl", 0L,
+            "Connection time to live in milliseconds. 0 or negative number means no limit."),
+    ;
 
     private final String key;
     private final Serializable defaultValue;

clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java

@@ -70,6 +70,13 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -2695,4 +2702,42 @@ public void testFailover() throws ClickHouseException {
             Assert.assertEquals(response.firstRecord().getValue(0).asInteger(), 1);
         }
     }
+
+    @Test(groups = {"integration"}, dataProvider = "testServerTimezoneAppliedFromHeaderProvider")
+    public void testServerTimezoneAppliedFromHeader(ClickHouseFormat format) throws Exception {
+        System.out.println("Testing with " + format + " format");
+        ClickHouseNode server = getServer();
+
+        ZoneId custZoneId = ZoneId.of("America/Los_Angeles");
+
+        final String sql = "SELECT now(), toDateTime(now(), 'UTC') as utc_time, serverTimezone() SETTINGS session_timezone = 'America/Los_Angeles'";
+        try (ClickHouseClient client = getClient();
+             ClickHouseResponse response = newRequest(client, server)
+                     .query(sql, UUID.randomUUID().toString())
+                     .option(ClickHouseClientOption.FORMAT, format)
+                     .executeAndWait()) {
+
+            Assert.assertEquals(response.getTimeZone().getID(), "America/Los_Angeles", "Timezone should be applied from the query settings");
+
+            ClickHouseRecord record = response.firstRecord();
+            final ZonedDateTime now = record.getValue(0).asZonedDateTime();
+            final ZonedDateTime utcTime = record.getValue(1).asZonedDateTime();
+            final String serverTimezone = record.getValue(2).asString();
+            Assert.assertNotEquals(serverTimezone, "America/Los_Angeles", "Server timezone should be applied from the query settings");
+
+
+            System.out.println("Now in America/Los_Angeles: " + now);
+            System.out.println("UTC Time: " + utcTime);
+            System.out.println("UTC Time: " + utcTime.withZoneSameInstant(custZoneId));
+            Assert.assertEquals(now, utcTime.withZoneSameInstant(custZoneId));
+        }
+    }
+
+    @DataProvider(name = "testServerTimezoneAppliedFromHeaderProvider")
+    public static ClickHouseFormat[] testServerTimezoneAppliedFromHeaderProvider() {
+        return new ClickHouseFormat[]{
+                ClickHouseFormat.TabSeparatedWithNamesAndTypes,
+                ClickHouseFormat.RowBinaryWithNamesAndTypes
+        };
+    }
 }

clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcResponse.java

@@ -29,7 +29,7 @@ static void checkError(Result result) {
 
     protected ClickHouseGrpcResponse(ClickHouseConfig config, Map<String, Serializable> settings,
             ClickHouseStreamObserver observer) throws IOException {
-        super(config, observer.getInputStream(), settings, null, observer.getSummary(), null);
+        super(config, observer.getInputStream(), settings, null, observer.getSummary(), observer.getTimeZone());
 
         this.observer = observer;
     }

clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseStreamObserver.java

@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.util.TimeZone;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
@@ -44,6 +45,8 @@ public class ClickHouseStreamObserver implements StreamObserver<Result> {
 
     private final AtomicReference<IOException> errorRef;
 
+    private TimeZone timeZone;
+
     protected ClickHouseStreamObserver(ClickHouseConfig config, ClickHouseNode server, ClickHouseOutputStream output) {
         this.server = server;
 
@@ -128,6 +131,11 @@ protected boolean updateStatus(Result result) {
             }
         }
 
+        String tz = result.getTimeZone();
+        if (!tz.isEmpty()) {
+            timeZone = TimeZone.getTimeZone(result.getTimeZone());
+        }
+
         return proceed;
     }
 
@@ -221,4 +229,8 @@ public boolean awaitCompletion(long timeout, TimeUnit unit) throws InterruptedEx
     public ClickHouseInputStream getInputStream() {
         return this.input;
     }
+
+    public TimeZone getTimeZone() {
+        return timeZone;
+    }
 }