From 47b33cbbf34a992c40972235b9e879401d8c23b3 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 8 Aug 2025 09:13:24 -0700 Subject: [PATCH 1/7] added clusterAllReplicas --- jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index bec3d83c5..0279709ee 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -309,7 +309,7 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException { stmt.execute("SYSTEM FLUSH LOGS"); final String logQuery ="SELECT http_user_agent " + - " FROM system.query_log WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'"; + " FROM clusterAllReplicas('default', system.query_log) WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'"; try (ResultSet rs = stmt.executeQuery(logQuery)) { Assert.assertTrue(rs.next()); String userAgent = rs.getString("http_user_agent"); From c8269430a6d9f9268132980069fbf4b9c31d2080 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 8 Aug 2025 09:35:50 -0700 Subject: [PATCH 2/7] fixed JDBC test for client info to match by query id --- jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 0279709ee..5bb703f34 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -306,10 +306,12 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException { final String testQuery = "SELECT '" + UUID.randomUUID() + "'"; stmt.execute(testQuery); + String queryId = ((StatementImpl)stmt).getLastQueryId(); stmt.execute("SYSTEM FLUSH LOGS"); + final String logQuery ="SELECT http_user_agent " + - " FROM clusterAllReplicas('default', system.query_log) WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'"; + " FROM clusterAllReplicas('default', system.query_log) WHERE query_id = " + stmt.enquoteLiteral(queryId); try (ResultSet rs = stmt.executeQuery(logQuery)) { Assert.assertTrue(rs.next()); String userAgent = rs.getString("http_user_agent"); From f6785a6e4d9efbb4d94f32d08190fdf2e0015a26 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 8 Aug 2025 09:45:07 -0700 Subject: [PATCH 3/7] removed clusterAllReplicas --- jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 5bb703f34..1ca181174 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -310,8 +310,7 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException { stmt.execute("SYSTEM FLUSH LOGS"); - final String logQuery ="SELECT http_user_agent " + - " FROM clusterAllReplicas('default', system.query_log) WHERE query_id = " + stmt.enquoteLiteral(queryId); + final String logQuery ="SELECT http_user_agent FROM system.query_log WHERE query_id = " + stmt.enquoteLiteral(queryId); try (ResultSet rs = stmt.executeQuery(logQuery)) { Assert.assertTrue(rs.next()); String userAgent = rs.getString("http_user_agent"); From b66594e236803a3644896b8ed8a2f7e0c2328441 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 8 Aug 2025 09:59:32 -0700 Subject: [PATCH 4/7] closing result set to finalize request --- jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index 1ca181174..a29728594 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -307,6 +307,7 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException { final String testQuery = "SELECT '" + UUID.randomUUID() + "'"; stmt.execute(testQuery); String queryId = ((StatementImpl)stmt).getLastQueryId(); + stmt.getResultSet().close(); // close result set to finalize request. stmt.execute("SYSTEM FLUSH LOGS"); From 3deeb335176d14a86fb62a03da0e8d342327e364 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Fri, 8 Aug 2025 10:20:57 -0700 Subject: [PATCH 5/7] closing result set to finalize request --- jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index a29728594..c2100f287 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -342,6 +342,7 @@ private void influenceUserAgentTest(String clientName, String urlParam) throws S final String testQuery = "SELECT '" + UUID.randomUUID() + "'"; stmt.execute(testQuery); + stmt.getResultSet().close(); // finalize request stmt.execute("SYSTEM FLUSH LOGS"); final String logQuery ="SELECT http_user_agent " + From da945b83e75c4b81cfce0f0dd0158ac606452f9d Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 11 Aug 2025 09:27:14 -0700 Subject: [PATCH 6/7] added clusterAllReplicas for reading query_log --- .../test/java/com/clickhouse/client/HttpTransportTests.java | 6 +++--- .../src/test/java/com/clickhouse/jdbc/ConnectionTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java index ae495c786..f46dfc12a 100644 --- a/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java @@ -806,7 +806,7 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA client.execute("SYSTEM FLUSH LOGS").get().close(); List logRecords = client.queryAll("SELECT http_user_agent, http_referer, " + - " forwarded_for FROM system.query_log WHERE query_id = '" + q1Id + "'"); + " forwarded_for FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + q1Id + "'"); Assert.assertFalse(logRecords.isEmpty(), "No records found in query log"); for (GenericRecord record : logRecords) { @@ -864,7 +864,7 @@ public void testClientName(String clientName, boolean setWithUpdate, String user client.execute("SYSTEM FLUSH LOGS").get().close(); List logRecords = client.queryAll("SELECT query_id, client_name, http_user_agent, http_referer " + - " FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'"); + " FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'"); Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); final String logUserAgent = logRecords.get(0).getString("http_user_agent"); Assert.assertTrue(logUserAgent.startsWith(expectedClientNameStartsWith), @@ -905,7 +905,7 @@ public void testClientNameThruRawOptions(String property, String value, boolean client.execute("SYSTEM FLUSH LOGS").get().close(); List logRecords = client.queryAll("SELECT query_id, client_name, http_user_agent, http_referer " + - " FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'"); + " FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'"); Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); final String logUserAgent = logRecords.get(0).getString("http_user_agent"); Assert.assertTrue(logUserAgent.startsWith(value), diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java index c2100f287..42374f757 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java @@ -311,7 +311,7 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException { stmt.execute("SYSTEM FLUSH LOGS"); - final String logQuery ="SELECT http_user_agent FROM system.query_log WHERE query_id = " + stmt.enquoteLiteral(queryId); + final String logQuery ="SELECT http_user_agent FROM clusterAllReplicas('default', system.query_log) WHERE query_id = " + stmt.enquoteLiteral(queryId); try (ResultSet rs = stmt.executeQuery(logQuery)) { Assert.assertTrue(rs.next()); String userAgent = rs.getString("http_user_agent"); @@ -346,7 +346,7 @@ private void influenceUserAgentTest(String clientName, String urlParam) throws S stmt.execute("SYSTEM FLUSH LOGS"); final String logQuery ="SELECT http_user_agent " + - " FROM system.query_log WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'"; + " FROM clusterAllReplicas('default', system.query_log) WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'"; try (ResultSet rs = stmt.executeQuery(logQuery)) { Assert.assertTrue(rs.next()); String userAgent = rs.getString("http_user_agent"); From 1cb243f8eab389bcc8b5e3fd367e9e64078719e2 Mon Sep 17 00:00:00 2001 From: Sergey Chernov Date: Mon, 11 Aug 2025 10:06:34 -0700 Subject: [PATCH 7/7] moved client related tests to proper place. fixed reading quer log --- .../com/clickhouse/client/ClientTests.java | 108 ++++++++++++++++++ .../clickhouse/client/query/QueryTests.java | 95 --------------- 2 files changed, 108 insertions(+), 95 deletions(-) diff --git a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java index da6b38658..fe592a160 100644 --- a/client-v2/src/test/java/com/clickhouse/client/ClientTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/ClientTests.java @@ -5,6 +5,7 @@ import com.clickhouse.client.api.ClientException; import com.clickhouse.client.api.ClientFaultCause; import com.clickhouse.client.api.ConnectionReuseStrategy; +import com.clickhouse.client.api.command.CommandResponse; import com.clickhouse.client.api.enums.Protocol; import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream; import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy; @@ -13,16 +14,25 @@ import com.clickhouse.client.api.query.QuerySettings; import com.clickhouse.client.api.query.Records; import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.query.QueryTests; +import com.clickhouse.data.ClickHouseVersion; +import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.testng.Assert; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import org.testng.util.Strings; import java.net.ConnectException; +import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -310,6 +320,104 @@ public void testWithOldDefaults() { } } + @DataProvider(name = "sessionRoles") + private static Object[][] sessionRoles() { + return new Object[][]{ + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2,☺"}}, + {new String[]{"ROL1", "ROL2"}}, + }; + } + + @Test(groups = {"integration"}, dataProvider = "sessionRoles") + public void testOperationCustomRoles(String[] roles) throws Exception { + if (isVersionMatch("(,24.3]", newClient().build())) { + return; + } + + String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$"; + final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; + try (Client client = newClient().build()) { + client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); + client.execute("CREATE ROLE " + rolesList).get().close(); + client.execute("DROP USER IF EXISTS some_user").get().close(); + client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'").get().close(); + client.execute("GRANT " + rolesList + " TO some_user").get().close(); + } + + try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { + QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles)); + List resp = userClient.queryAll("SELECT currentRoles()", settings); + Set roleSet = new HashSet<>(Arrays.asList(roles)); + Set currentRoles = new HashSet (resp.get(0).getList(1)); + Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); + } + } + + @DataProvider(name = "clientSessionRoles") + private static Object[][] clientSessionRoles() { + return new Object[][]{ + {new String[]{"ROL1", "ROL2"}}, + {new String[]{"ROL1", "ROL2,☺"}}, + }; + } + @Test(groups = {"integration"}, dataProvider = "clientSessionRoles") + public void testClientCustomRoles(String[] roles) throws Exception { + if (isVersionMatch("(,24.3]", newClient().build())) { + return; + } + + String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$"; + final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; + try (Client client = newClient().build()) { + client.execute("DROP ROLE IF EXISTS " + rolesList).get().close(); + client.execute("CREATE ROLE " + rolesList).get().close(); + client.execute("DROP USER IF EXISTS some_user").get().close(); + client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'").get().close(); + client.execute("GRANT " + rolesList + " TO some_user").get().close(); + } + + try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { + userClient.setDBRoles(Arrays.asList(roles)); + List resp = userClient.queryAll("SELECT currentRoles()"); + Set roleSet = new HashSet<>(Arrays.asList(roles)); + Set currentRoles = new HashSet (resp.get(0).getList(1)); + Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); + } + } + + + @Test(groups = {"integration"}) + public void testLogComment() throws Exception { + + String logComment = "Test log comment"; + QuerySettings settings = new QuerySettings() + .setQueryId(UUID.randomUUID().toString()) + .logComment(logComment); + + try (Client client = newClient().build()) { + + try (QueryResponse response = client.query("SELECT 1", settings).get()) { + Assert.assertNotNull(response.getQueryId()); + Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId())); + } + + client.execute("SYSTEM FLUSH LOGS").get().close(); + + List logRecords = client.queryAll("SELECT query_id, log_comment FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'"); + Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); + Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment); + } + } + + public boolean isVersionMatch(String versionExpression, Client client) { + List serverVersion = client.queryAll("SELECT version()"); + return ClickHouseVersion.of(serverVersion.get(0).getString(1)).check(versionExpression); + } + + protected Client.Builder newClient() { ClickHouseNode node = getServer(ClickHouseProtocol.HTTP); boolean isSecure = isCloud(); diff --git a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java index 46a949b38..5b08248b7 100644 --- a/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java @@ -1902,101 +1902,6 @@ public static BigDecimal cropDecimal(BigDecimal value, int scale) { return new BigDecimal(bi, scale); } - @DataProvider(name = "sessionRoles") - private static Object[][] sessionRoles() { - return new Object[][]{ - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2,☺"}}, - {new String[]{"ROL1", "ROL2"}}, - }; - } - - @Test(groups = {"integration"}, dataProvider = "sessionRoles", dataProviderClass = QueryTests.class) - public void testOperationCustomRoles(String[] roles) throws Exception { - if (isVersionMatch("(,24.3]")) { - return; - } - - String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$"; - final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; - try (CommandResponse resp = client.execute("DROP ROLE IF EXISTS " + rolesList).get()) { - } - try (CommandResponse resp = client.execute("CREATE ROLE " + rolesList).get()) { - } - try (CommandResponse resp = client.execute("DROP USER IF EXISTS some_user").get()) { - } - try (CommandResponse resp = client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'" ).get()) { - } - try (CommandResponse resp = client.execute("GRANT " + rolesList + " TO some_user").get()) { - } - - - try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { - QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles)); - List resp = userClient.queryAll("SELECT currentRoles()", settings); - Set roleSet = new HashSet<>(Arrays.asList(roles)); - Set currentRoles = new HashSet (resp.get(0).getList(1)); - Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); - } - } - - @DataProvider(name = "clientSessionRoles") - private static Object[][] clientSessionRoles() { - return new Object[][]{ - {new String[]{"ROL1", "ROL2"}}, - {new String[]{"ROL1", "ROL2,☺"}}, - }; - } - @Test(groups = {"integration"}, dataProvider = "clientSessionRoles", dataProviderClass = QueryTests.class) - public void testClientCustomRoles(String[] roles) throws Exception { - if (isVersionMatch("(,24.3]")) { - return; - } - - String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$"; - final String rolesList = "\"" + Strings.join("\",\"", roles) + "\""; - try (CommandResponse resp = client.execute("DROP ROLE IF EXISTS " + rolesList).get()) { - } - try (CommandResponse resp = client.execute("CREATE ROLE " + rolesList).get()) { - } - try (CommandResponse resp = client.execute("DROP USER IF EXISTS some_user").get()) { - } - try (CommandResponse resp = client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'" ).get()) { - } - try (CommandResponse resp = client.execute("GRANT " + rolesList + " TO some_user").get()) { - } - - try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) { - userClient.setDBRoles(Arrays.asList(roles)); - List resp = userClient.queryAll("SELECT currentRoles()"); - Set roleSet = new HashSet<>(Arrays.asList(roles)); - Set currentRoles = new HashSet (resp.get(0).getList(1)); - Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles); - } - } - - - @Test(groups = {"integration"}) - public void testLogComment() throws Exception { - - String logComment = "Test log comment"; - QuerySettings settings = new QuerySettings() - .setQueryId(UUID.randomUUID().toString()) - .logComment(logComment); - try (QueryResponse response = client.query("SELECT 1", settings).get()) { - Assert.assertNotNull(response.getQueryId()); - Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId())); - } - - try (CommandResponse resp = client.execute("SYSTEM FLUSH LOGS").get()) { - } - - List logRecords = client.queryAll("SELECT query_id, log_comment FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'"); - Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId()); - Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment); - } @Test(groups = { "integration" }, enabled = true) public void testReadingBitmap() throws Exception { final String tableName = "bitmaps_test_table";