Skip to content

[repo] Fix tests reading query_log by closing query before flushing logs. #2528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Aug 11, 2025
Merged

[repo] Fix tests reading query_log by closing query before flushing logs. #2528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
47b33cb
added clusterAllReplicas
chernser Aug 8, 2025
c826943
fixed JDBC test for client info to match by query id
chernser Aug 8, 2025
f6785a6
removed clusterAllReplicas
chernser Aug 8, 2025
b66594e
closing result set to finalize request
chernser Aug 8, 2025
3deeb33
closing result set to finalize request
chernser Aug 8, 2025
da945b8
added clusterAllReplicas for reading query_log
chernser Aug 11, 2025
1cb243f
moved client related tests to proper place. fixed reading quer log
chernser Aug 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 108 additions & 0 deletions client-v2/src/test/java/com/clickhouse/client/ClientTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.clickhouse.client.api.ClientException;
import com.clickhouse.client.api.ClientFaultCause;
import com.clickhouse.client.api.ConnectionReuseStrategy;
import com.clickhouse.client.api.command.CommandResponse;
import com.clickhouse.client.api.enums.Protocol;
import com.clickhouse.client.api.internal.ClickHouseLZ4OutputStream;
import com.clickhouse.client.api.metadata.DefaultColumnToMethodMatchingStrategy;
Expand All @@ -13,16 +14,25 @@
import com.clickhouse.client.api.query.QuerySettings;
import com.clickhouse.client.api.query.Records;
import com.clickhouse.client.config.ClickHouseClientOption;
import com.clickhouse.client.query.QueryTests;
import com.clickhouse.data.ClickHouseVersion;
import org.apache.commons.lang3.RandomStringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import org.testng.util.Strings;

import java.net.ConnectException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
Expand Down Expand Up @@ -310,6 +320,104 @@ public void testWithOldDefaults() {
}
}

@DataProvider(name = "sessionRoles")
private static Object[][] sessionRoles() {
return new Object[][]{
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2,☺"}},
{new String[]{"ROL1", "ROL2"}},
};
}

@Test(groups = {"integration"}, dataProvider = "sessionRoles")
public void testOperationCustomRoles(String[] roles) throws Exception {
if (isVersionMatch("(,24.3]", newClient().build())) {
return;
}

String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$";
final String rolesList = "\"" + Strings.join("\",\"", roles) + "\"";
try (Client client = newClient().build()) {
client.execute("DROP ROLE IF EXISTS " + rolesList).get().close();
client.execute("CREATE ROLE " + rolesList).get().close();
client.execute("DROP USER IF EXISTS some_user").get().close();
client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'").get().close();
client.execute("GRANT " + rolesList + " TO some_user").get().close();
}

try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) {
QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles));
List<GenericRecord> resp = userClient.queryAll("SELECT currentRoles()", settings);
Set<String> roleSet = new HashSet<>(Arrays.asList(roles));
Set<String> currentRoles = new HashSet<String> (resp.get(0).getList(1));
Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles);
}
}

@DataProvider(name = "clientSessionRoles")
private static Object[][] clientSessionRoles() {
return new Object[][]{
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2,☺"}},
};
}
@Test(groups = {"integration"}, dataProvider = "clientSessionRoles")
public void testClientCustomRoles(String[] roles) throws Exception {
if (isVersionMatch("(,24.3]", newClient().build())) {
return;
}

String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$";
final String rolesList = "\"" + Strings.join("\",\"", roles) + "\"";
try (Client client = newClient().build()) {
client.execute("DROP ROLE IF EXISTS " + rolesList).get().close();
client.execute("CREATE ROLE " + rolesList).get().close();
client.execute("DROP USER IF EXISTS some_user").get().close();
client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'").get().close();
client.execute("GRANT " + rolesList + " TO some_user").get().close();
}

try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) {
userClient.setDBRoles(Arrays.asList(roles));
List<GenericRecord> resp = userClient.queryAll("SELECT currentRoles()");
Set<String> roleSet = new HashSet<>(Arrays.asList(roles));
Set<String> currentRoles = new HashSet<String> (resp.get(0).getList(1));
Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles);
}
}


@Test(groups = {"integration"})
public void testLogComment() throws Exception {

String logComment = "Test log comment";
QuerySettings settings = new QuerySettings()
.setQueryId(UUID.randomUUID().toString())
.logComment(logComment);

try (Client client = newClient().build()) {

try (QueryResponse response = client.query("SELECT 1", settings).get()) {
Assert.assertNotNull(response.getQueryId());
Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId()));
}

client.execute("SYSTEM FLUSH LOGS").get().close();

List<GenericRecord> logRecords = client.queryAll("SELECT query_id, log_comment FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'");
Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId());
Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment);
}
}

public boolean isVersionMatch(String versionExpression, Client client) {
List<GenericRecord> serverVersion = client.queryAll("SELECT version()");
return ClickHouseVersion.of(serverVersion.get(0).getString(1)).check(versionExpression);
}


protected Client.Builder newClient() {
ClickHouseNode node = getServer(ClickHouseProtocol.HTTP);
boolean isSecure = isCloud();
Expand Down
6 changes: 3 additions & 3 deletions client-v2/src/test/java/com/clickhouse/client/HttpTransportTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -806,7 +806,7 @@ public void testUserAgentHasCompleteProductName(String clientName, Pattern userA
client.execute("SYSTEM FLUSH LOGS").get().close();

List<GenericRecord> logRecords = client.queryAll("SELECT http_user_agent, http_referer, " +
" forwarded_for FROM system.query_log WHERE query_id = '" + q1Id + "'");
" forwarded_for FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + q1Id + "'");
Assert.assertFalse(logRecords.isEmpty(), "No records found in query log");

for (GenericRecord record : logRecords) {
Expand Down Expand Up @@ -864,7 +864,7 @@ public void testClientName(String clientName, boolean setWithUpdate, String user
client.execute("SYSTEM FLUSH LOGS").get().close();

List<GenericRecord> logRecords = client.queryAll("SELECT query_id, client_name, http_user_agent, http_referer " +
" FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'");
" FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'");
Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId());
final String logUserAgent = logRecords.get(0).getString("http_user_agent");
Assert.assertTrue(logUserAgent.startsWith(expectedClientNameStartsWith),
Expand Down Expand Up @@ -905,7 +905,7 @@ public void testClientNameThruRawOptions(String property, String value, boolean
client.execute("SYSTEM FLUSH LOGS").get().close();

List<GenericRecord> logRecords = client.queryAll("SELECT query_id, client_name, http_user_agent, http_referer " +
" FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'");
" FROM clusterAllReplicas('default', system.query_log) WHERE query_id = '" + settings.getQueryId() + "'");
Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId());
final String logUserAgent = logRecords.get(0).getString("http_user_agent");
Assert.assertTrue(logUserAgent.startsWith(value),
Expand Down
95 changes: 0 additions & 95 deletions client-v2/src/test/java/com/clickhouse/client/query/QueryTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1902,101 +1902,6 @@ public static BigDecimal cropDecimal(BigDecimal value, int scale) {
return new BigDecimal(bi, scale);
}

@DataProvider(name = "sessionRoles")
private static Object[][] sessionRoles() {
return new Object[][]{
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2,☺"}},
{new String[]{"ROL1", "ROL2"}},
};
}

@Test(groups = {"integration"}, dataProvider = "sessionRoles", dataProviderClass = QueryTests.class)
public void testOperationCustomRoles(String[] roles) throws Exception {
if (isVersionMatch("(,24.3]")) {
return;
}

String password = "^1A" + RandomStringUtils.random(12, true, true) + "3b$";
final String rolesList = "\"" + Strings.join("\",\"", roles) + "\"";
try (CommandResponse resp = client.execute("DROP ROLE IF EXISTS " + rolesList).get()) {
}
try (CommandResponse resp = client.execute("CREATE ROLE " + rolesList).get()) {
}
try (CommandResponse resp = client.execute("DROP USER IF EXISTS some_user").get()) {
}
try (CommandResponse resp = client.execute("CREATE USER some_user IDENTIFIED BY '" + password + "'" ).get()) {
}
try (CommandResponse resp = client.execute("GRANT " + rolesList + " TO some_user").get()) {
}


try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) {
QuerySettings settings = new QuerySettings().setDBRoles(Arrays.asList(roles));
List<GenericRecord> resp = userClient.queryAll("SELECT currentRoles()", settings);
Set<String> roleSet = new HashSet<>(Arrays.asList(roles));
Set<String> currentRoles = new HashSet<String> (resp.get(0).getList(1));
Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles);
}
}

@DataProvider(name = "clientSessionRoles")
private static Object[][] clientSessionRoles() {
return new Object[][]{
{new String[]{"ROL1", "ROL2"}},
{new String[]{"ROL1", "ROL2,☺"}},
};
}
@Test(groups = {"integration"}, dataProvider = "clientSessionRoles", dataProviderClass = QueryTests.class)
public void testClientCustomRoles(String[] roles) throws Exception {
if (isVersionMatch("(,24.3]")) {
return;
}

String password = "^1A" + RandomStringUtils.random(12, true, true) + "3B$";
final String rolesList = "\"" + Strings.join("\",\"", roles) + "\"";
try (CommandResponse resp = client.execute("DROP ROLE IF EXISTS " + rolesList).get()) {
}
try (CommandResponse resp = client.execute("CREATE ROLE " + rolesList).get()) {
}
try (CommandResponse resp = client.execute("DROP USER IF EXISTS some_user").get()) {
}
try (CommandResponse resp = client.execute("CREATE USER some_user IDENTIFIED WITH sha256_password BY '" + password + "'" ).get()) {
}
try (CommandResponse resp = client.execute("GRANT " + rolesList + " TO some_user").get()) {
}

try (Client userClient = newClient().setUsername("some_user").setPassword(password).build()) {
userClient.setDBRoles(Arrays.asList(roles));
List<GenericRecord> resp = userClient.queryAll("SELECT currentRoles()");
Set<String> roleSet = new HashSet<>(Arrays.asList(roles));
Set<String> currentRoles = new HashSet<String> (resp.get(0).getList(1));
Assert.assertEquals(currentRoles, roleSet, "Roles " + roleSet + " not found in " + currentRoles);
}
}


@Test(groups = {"integration"})
public void testLogComment() throws Exception {

String logComment = "Test log comment";
QuerySettings settings = new QuerySettings()
.setQueryId(UUID.randomUUID().toString())
.logComment(logComment);
try (QueryResponse response = client.query("SELECT 1", settings).get()) {
Assert.assertNotNull(response.getQueryId());
Assert.assertTrue(response.getQueryId().startsWith(settings.getQueryId()));
}

try (CommandResponse resp = client.execute("SYSTEM FLUSH LOGS").get()) {
}

List<GenericRecord> logRecords = client.queryAll("SELECT query_id, log_comment FROM system.query_log WHERE query_id = '" + settings.getQueryId() + "'");
Assert.assertEquals(logRecords.get(0).getString("query_id"), settings.getQueryId());
Assert.assertEquals(logRecords.get(0).getString("log_comment"), logComment);
}
@Test(groups = { "integration" }, enabled = true)
public void testReadingBitmap() throws Exception {
final String tableName = "bitmaps_test_table";
Expand Down
9 changes: 6 additions & 3 deletions jdbc-v2/src/test/java/com/clickhouse/jdbc/ConnectionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,10 +306,12 @@ public void setAndGetClientInfoTest(String clientName) throws SQLException {

final String testQuery = "SELECT '" + UUID.randomUUID() + "'";
stmt.execute(testQuery);
String queryId = ((StatementImpl)stmt).getLastQueryId();
stmt.getResultSet().close(); // close result set to finalize request.
stmt.execute("SYSTEM FLUSH LOGS");

final String logQuery ="SELECT http_user_agent " +
" FROM system.query_log WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'";

final String logQuery ="SELECT http_user_agent FROM clusterAllReplicas('default', system.query_log) WHERE query_id = " + stmt.enquoteLiteral(queryId);
try (ResultSet rs = stmt.executeQuery(logQuery)) {
Assert.assertTrue(rs.next());
String userAgent = rs.getString("http_user_agent");
Expand Down Expand Up @@ -340,10 +342,11 @@ private void influenceUserAgentTest(String clientName, String urlParam) throws S

final String testQuery = "SELECT '" + UUID.randomUUID() + "'";
stmt.execute(testQuery);
stmt.getResultSet().close(); // finalize request
stmt.execute("SYSTEM FLUSH LOGS");

final String logQuery ="SELECT http_user_agent " +
" FROM system.query_log WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'";
" FROM clusterAllReplicas('default', system.query_log) WHERE query = '" + testQuery.replaceAll("'", "\\\\'") + "'";
try (ResultSet rs = stmt.executeQuery(logQuery)) {
Assert.assertTrue(rs.next());
String userAgent = rs.getString("http_user_agent");
Expand Down
Loading