Skip to content

Commit 68c7fb3

Browse files
chernserchernser
committed
implemented value function for columns. fixed returning solid result set for database metadata
1 parent 3e4e092 commit 68c7fb3

File tree

7 files changed

+170
-37
lines changed

7 files changed

+170
-37
lines changed

‎clickhouse-data/src/main/java/com/clickhouse/data/ClickHouseColumn.java‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import java.io.Serializable;
3939
import java.lang.reflect.Array;
4040
import java.math.BigInteger;
41+
import java.sql.SQLException;
4142
import java.time.OffsetDateTime;
4243
import java.util.ArrayList;
4344
import java.util.Arrays;
@@ -90,6 +91,7 @@ public final class ClickHouseColumn implements Serializable {
9091
private List<ClickHouseColumn> nested;
9192
private List<String> parameters;
9293
private ClickHouseEnum enumConstants;
94+
private ValueFunction valueFunction;
9395

9496
private int arrayLevel;
9597
private ClickHouseColumn arrayBaseColumn;
@@ -787,6 +789,18 @@ public boolean isNestedType() {
787789
return dataType.isNested();
788790
}
789791

792+
public boolean hasValueFunction() {
793+
return valueFunction != null;
794+
}
795+
796+
public void setValueFunction(ValueFunction valueFunction) {
797+
this.valueFunction = valueFunction;
798+
}
799+
800+
public ValueFunction getValueFunction() {
801+
return valueFunction;
802+
}
803+
790804
public int getArrayNestedLevel() {
791805
return arrayLevel;
792806
}
@@ -1126,4 +1140,9 @@ public String toString() {
11261140
}
11271141
return builder.append(' ').append(originalTypeName).toString();
11281142
}
1143+
1144+
public interface ValueFunction {
1145+
1146+
Object produceValue(Object[] row);
1147+
}
11291148
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/ClickHouseBinaryFormatReader.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.clickhouse.client.api.data_formats;
22

33
import com.clickhouse.client.api.metadata.TableSchema;
4+
import com.clickhouse.data.ClickHouseColumn;
45
import com.clickhouse.data.value.ClickHouseBitmap;
56
import com.clickhouse.data.value.ClickHouseGeoMultiPolygonValue;
67
import com.clickhouse.data.value.ClickHouseGeoPointValue;
@@ -545,4 +546,13 @@ public interface ClickHouseBinaryFormatReader extends AutoCloseable {
545546
TemporalAmount getTemporalAmount(int index);
546547

547548
TemporalAmount getTemporalAmount(String colName);
549+
550+
/**
551+
* ! Experimental ! Might change in the future.
552+
* Sets a value function of a column. If column has a value function then reader will pass current row
553+
* as Object[] to a function. The least is responsible for returning correct value or null.
554+
* @param index - column index starting with 1
555+
* @param function - function that will be used to calculate column value from current row.
556+
*/
557+
default void setValueFunction(int index, ClickHouseColumn.ValueFunction function) {}
548558
}

‎client-v2/src/main/java/com/clickhouse/client/api/data_formats/internal/AbstractBinaryFormatReader.java‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,11 @@ public boolean readToPOJO(Map<String, POJOFieldDeserializer> deserializers, Obje
131131
return true;
132132
}
133133

134+
@Override
135+
public void setValueFunction(int index, ClickHouseColumn.ValueFunction function) {
136+
columns[index - 1].setValueFunction(function);
137+
}
138+
134139
/**
135140
* It is still internal method and should be used with care.
136141
* Usually this method is called to read next record into internal object and affects hasNext() method.
@@ -148,7 +153,11 @@ public boolean readRecord(Map<String, Object> record) throws IOException {
148153
}
149154

150155
boolean firstColumn = true;
156+
boolean hasValueFunctionColumn = false;
151157
for (ClickHouseColumn column : columns) {
158+
if (column.hasValueFunction()) {
159+
hasValueFunctionColumn = true;
160+
}
152161
try {
153162
Object val = binaryStreamReader.readValue(column);
154163
if (val != null) {
@@ -165,6 +174,16 @@ public boolean readRecord(Map<String, Object> record) throws IOException {
165174
throw e;
166175
}
167176
}
177+
178+
if (hasValueFunctionColumn) {
179+
// This variant of readRecord is called only for POJO serialization and this logic should be avoided.
180+
Object[] row = record.values().toArray();
181+
for (ClickHouseColumn column : columns) {
182+
if (column.hasValueFunction()) {
183+
record.put(column.getColumnName(), column.getValueFunction().produceValue(row));
184+
}
185+
}
186+
}
168187
return true;
169188
}
170189

@@ -174,9 +193,14 @@ protected boolean readRecord(Object[] record) throws IOException {
174193
}
175194

176195
boolean firstColumn = true;
196+
boolean hasValueFunctionColumn = false;
177197
for (int i = 0; i < columns.length; i++) {
178198
try {
179-
Object val = binaryStreamReader.readValue(columns[i]);
199+
ClickHouseColumn column = columns[i];
200+
if (column.hasValueFunction()) {
201+
hasValueFunctionColumn = true;
202+
}
203+
Object val = binaryStreamReader.readValue(column);
180204
if (val != null) {
181205
record[i] = val;
182206
} else {
@@ -191,9 +215,19 @@ protected boolean readRecord(Object[] record) throws IOException {
191215
throw e;
192216
}
193217
}
218+
219+
if (hasValueFunctionColumn) {
220+
for (int i = 0; i < columns.length; i++) {
221+
ClickHouseColumn column = columns[i];
222+
if (column.hasValueFunction()) {
223+
record[i] = column.getValueFunction().produceValue(record);
224+
}
225+
}
226+
}
194227
return true;
195228
}
196229

230+
197231
@Override
198232
public <T> T readValue(int colIndex) {
199233
if (colIndex < 1 || colIndex > getSchema().getColumns().size()) {

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/ResultSetImpl.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.clickhouse.client.api.data_formats.ClickHouseBinaryFormatReader;
44
import com.clickhouse.client.api.metadata.TableSchema;
55
import com.clickhouse.client.api.query.QueryResponse;
6+
import com.clickhouse.data.ClickHouseColumn;
67
import com.clickhouse.data.ClickHouseDataType;
78
import com.clickhouse.jdbc.internal.ExceptionUtils;
89
import com.clickhouse.jdbc.internal.JdbcUtils;
@@ -40,7 +41,7 @@
4041

4142
public class ResultSetImpl implements ResultSet, JdbcV2Wrapper {
4243
private static final Logger log = LoggerFactory.getLogger(ResultSetImpl.class);
43-
private ResultSetMetaData metaData;
44+
private ResultSetMetaDataImpl metaData;
4445
protected ClickHouseBinaryFormatReader reader;
4546
private QueryResponse response;
4647
private boolean closed;
@@ -138,6 +139,14 @@ public void close() throws SQLException {
138139
}
139140
}
140141

142+
public void setValueFunction(int colIndex, ClickHouseColumn.ValueFunction valueFunction) {
143+
reader.setValueFunction(colIndex, valueFunction);
144+
}
145+
146+
public void hideLastNColumns(int n) {
147+
metaData.setColumnCount(metaData.getOriginalColumnCount() - n);
148+
}
149+
141150
@Override
142151
public boolean wasNull() throws SQLException {
143152
checkClosed();

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/DatabaseMetaDataImpl.java‎

Lines changed: 40 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111
import com.clickhouse.jdbc.ResultSetImpl;
1212
import com.clickhouse.jdbc.internal.ExceptionUtils;
1313
import com.clickhouse.jdbc.internal.JdbcUtils;
14-
import com.clickhouse.jdbc.internal.MetadataResultSet;
1514
import com.clickhouse.logging.Logger;
1615
import com.clickhouse.logging.LoggerFactory;
1716

@@ -22,6 +21,7 @@
2221
import java.sql.SQLException;
2322
import java.sql.SQLFeatureNotSupportedException;
2423
import java.sql.SQLType;
24+
import java.sql.Types;
2525
import java.util.Arrays;
2626

2727
public class DatabaseMetaDataImpl implements java.sql.DatabaseMetaData, JdbcV2Wrapper {
@@ -222,7 +222,7 @@ public String getStringFunctions() throws SQLException {
222222
public String getSystemFunctions() throws SQLException {
223223
// took from below URL(not from system.functions):
224224
// https://clickhouse.com/docs/en/sql-reference/functions/other-functions/
225-
return "bar,basename,blockNumber,blockSerializedSize,blockSize,buildId,byteSize,countDigits,currentDatabase,currentProfiles,currentRoles,currentUser,defaultProfiles,defaultRoles,defaultValueOfArgumentType,defaultValueOfTypeName,dumpColumnStructure,enabledProfiles,enabledRoles,errorCodeToName,filesystemAvailable,filesystemCapacity,filesystemFree,finalizeAggregation,formatReadableQuantity,formatReadableSize,formatReadableTimeDelta,FQDN,getMacro,getServerPort,getSetting,getSizeOfEnumType,greatest,hasColumnInTable,hostName,identity,ifNotFinite,ignore,indexHint,initializeAggregation,initialQueryID,isConstant,isDecimalOverflow,isFinite,isInfinite,isNaN,joinGet,least,MACNumToString,MACStringToNum,MACStringToOUI,materialize,modelEvaluate,neighbor,queryID,randomFixedString,randomPrintableASCII,randomString,randomStringUTF8,replicate,rowNumberInAllBlocks,rowNumberInBlock,runningAccumulate,runningConcurrency,runningDifference,runningDifferenceStartingWithFirstValue,shardCount ,shardNum,sleep,sleepEachRow,tcpPort,throwIf,toColumnTypeName,toTypeName,transform,uptime,version,visibleWidth";
225+
return "bar,basename,blockNumber,blockSerializedSize,blockSize,buildId,byteSize,countDigits,currentDatabase,currentProfiles,currentRoles,currentUser,defaultProfiles,defaultRoles,defaultValueOfArgumentType,defaultValueOfTypeName,dumpColumnStructure,enabledProfiles,enabledRoles,errorCodeToName,filesystemAvailable,filesystemCapacity,filesystemFree,finalizeAggregation,formatReadableQuantity,formatReadableSize,formatReadableTimeDelta,FQDN,getMacro,getServerPort,getSetting,getSizeOfEnumType,greatest,hasColumnInTable,hostName,identity,ifNotFinite,ignore,indexHint,initializeAggregation,initialQueryID,isConstant,isDecimalOverflow,isFinite,isInfinite,isNaN,joinGet,least,MACNumToString,MACStringToNum,MACStringToOUI,materialize,modelEvaluate,neighbor,queryID,randomFixedString,randomPrintableASCII,randomString,randomStringUTF8,replicate,rowNumberInAllBlocks,rowNumberInBlock,runningAccumulate,runningConcurrency,runningDifference,runningDifferenceStartingWithFirstValue,shardCount ,shardNum,sleep,sleepEachRow,tcpPort,throwIf,toColumnTypeName,toTypeName,e,uptime,version,visibleWidth";
226226
}
227227

228228
@Override
@@ -830,18 +830,19 @@ public ResultSet getTableTypes() throws SQLException {
830830
}
831831
}
832832

833-
private static final ClickHouseColumn DATA_TYPE_COL = ClickHouseColumn.of("DATA_TYPE", ClickHouseDataType.Int32.name()) ;
833+
private static final int GET_COLUMNS_TYPE_NAME_COL = 6;
834+
835+
private static final int GET_COLUMNS_DATA_TYPE_COL = 5;
834836
@Override
835837
@SuppressWarnings({"squid:S2095", "squid:S2077"})
836838
public ResultSet getColumns(String catalog, String schemaPattern, String tableNamePattern, String columnNamePattern) throws SQLException {
837-
//TODO: Best way to convert type to JDBC data type
838839
// TODO: handle useCatalogs == true and return schema catalog name
839-
String sql = "SELECT " +
840+
final String sql = "SELECT " +
840841
catalogPlaceholder + " AS TABLE_CAT, " +
841842
"database AS TABLE_SCHEM, " +
842843
"table AS TABLE_NAME, " +
843844
"name AS COLUMN_NAME, " +
844-
"system.columns.type AS DATA_TYPE, " +
845+
"toInt32(" + Types.OTHER + ") AS DATA_TYPE, " +
845846
"type AS TYPE_NAME, " +
846847
"toInt32(" + generateSqlTypeSizes("system.columns.type") + ") AS COLUMN_SIZE, " +
847848
"toInt32(0) AS BUFFER_LENGTH, " +
@@ -867,8 +868,9 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa
867868
" AND name LIKE " + SQLUtils.enquoteLiteral(columnNamePattern == null ? "%" : columnNamePattern) +
868869
" ORDER BY TABLE_SCHEM, TABLE_NAME, ORDINAL_POSITION";
869870
try {
870-
return new MetadataResultSet((ResultSetImpl) connection.createStatement().executeQuery(sql))
871-
.transform(DATA_TYPE_COL.getColumnName(), DATA_TYPE_COL, DatabaseMetaDataImpl::columnDataTypeToSqlType);
871+
ResultSetImpl rs = (ResultSetImpl) connection.createStatement().executeQuery(sql);
872+
rs.setValueFunction(GET_COLUMNS_DATA_TYPE_COL, GET_COLUMNS_DATA_TYPE_FUNC);
873+
return rs;
872874
} catch (Exception e) {
873875
throw ExceptionUtils.toSqlState(e);
874876
}
@@ -887,17 +889,22 @@ private static String generateSqlTypeSizes(String columnName) {
887889
return sql.toString();
888890
}
889891

890-
private static String columnDataTypeToSqlType(String value) {
891-
SQLType type = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(value);
892-
if (type == null) {
893-
try {
894-
type = JdbcUtils.convertToSqlType(ClickHouseColumn.of("v1", value).getDataType());
895-
} catch (Exception e) {
896-
log.error("Failed to convert column data type to SQL type: {}", value, e);
897-
type = JDBCType.OTHER; // In case of error, return SQL type 0
892+
private static final ClickHouseColumn.ValueFunction GET_COLUMNS_DATA_TYPE_FUNC = dataTypeValueFunction(GET_COLUMNS_TYPE_NAME_COL);
893+
894+
private static ClickHouseColumn.ValueFunction dataTypeValueFunction(int srcColIndex) {
895+
return row -> {
896+
String typeName = (String) row[srcColIndex - 1];
897+
SQLType type = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(typeName);
898+
if (type == null) {
899+
try {
900+
type = JdbcUtils.convertToSqlType(ClickHouseColumn.of("v1", typeName).getDataType());
901+
} catch (Exception e) {
902+
log.error("Failed to convert column data type to SQL type: {}", typeName, e);
903+
type = JDBCType.OTHER; // In case of error, return SQL type 0
904+
}
898905
}
899-
}
900-
return String.valueOf(type.getVendorTypeNumber());
906+
return type.getVendorTypeNumber();
907+
};
901908
}
902909

903910
@Override
@@ -1067,26 +1074,23 @@ public ResultSet getCrossReference(String parentCatalog, String parentSchema, St
10671074
}
10681075
}
10691076

1070-
private static final ClickHouseColumn NULLABLE_COL = ClickHouseColumn.of("NULLABLE", ClickHouseDataType.Int16.name());
1077+
private static final int TYPE_INFO_DATA_TYPE_COL = 2;
1078+
private static final int TYPE_INFO_NULLABILITY_COL = 7;
10711079
@Override
10721080
@SuppressWarnings({"squid:S2095"})
10731081
public ResultSet getTypeInfo() throws SQLException {
10741082
try {
1075-
return new MetadataResultSet((ResultSetImpl) connection.createStatement().executeQuery(DATA_TYPE_INFO_SQL))
1076-
.transform(DATA_TYPE_COL.getColumnName(), DATA_TYPE_COL, DatabaseMetaDataImpl::dataTypeToSqlTypeInt)
1077-
.transform(NULLABLE_COL.getColumnName(), NULLABLE_COL, DatabaseMetaDataImpl::dataTypeNullability);
1083+
ResultSetImpl rs = (ResultSetImpl) connection.createStatement().executeQuery(DATA_TYPE_INFO_SQL);
1084+
rs.setValueFunction(TYPE_INFO_DATA_TYPE_COL, TYPE_INFO_DATA_TYPE_VALUE_FUNC);
1085+
rs.setValueFunction(TYPE_INFO_NULLABILITY_COL, DatabaseMetaDataImpl::dataTypeNullability);
1086+
return rs;
10781087
} catch (Exception e) {
10791088
throw ExceptionUtils.toSqlState(e);
10801089
}
10811090
}
10821091

1083-
private static String dataTypeToSqlTypeInt(String type) {
1084-
SQLType sqlType = JdbcUtils.CLICKHOUSE_TYPE_NAME_TO_SQL_TYPE_MAP.get(type);
1085-
return sqlType == null ? String.valueOf(JDBCType.OTHER.getVendorTypeNumber()) :
1086-
String.valueOf(sqlType.getVendorTypeNumber());
1087-
}
1088-
1089-
private static String dataTypeNullability(String type) {
1092+
private static String dataTypeNullability(Object[] row) {
1093+
String type = (String) row[DATA_TYPE_INFO_SQL_TYPE_NAME_COL - 1];
10901094
if (type.equals(ClickHouseDataType.Nullable.name()) || type.equals(ClickHouseDataType.Dynamic.name())) {
10911095
return String.valueOf(java.sql.DatabaseMetaData.typeNullable);
10921096
}
@@ -1095,21 +1099,23 @@ private static String dataTypeNullability(String type) {
10951099

10961100
private static final String DATA_TYPE_INFO_SQL = getDataTypeInfoSql();
10971101

1102+
private static final int DATA_TYPE_INFO_SQL_TYPE_NAME_COL = 13;
1103+
10981104
private static String getDataTypeInfoSql() {
10991105
StringBuilder sql = new StringBuilder("SELECT " +
11001106
"name AS TYPE_NAME, " +
1101-
"if(empty(alias_to), name, alias_to) AS DATA_TYPE, " + // passing type name or alias if exists to map then
1107+
"0::Int32 AS DATA_TYPE, " + // passing type name or alias if exists to map then
11021108
"attrs.c2::Nullable(Int32) AS PRECISION, " +
11031109
"NULL::Nullable(String) AS LITERAL_PREFIX, " +
11041110
"NULL::Nullable(String) AS LITERAL_SUFFIX, " +
11051111
"NULL::Nullable(String) AS CREATE_PARAMS, " +
1106-
"name AS NULLABLE, " + // passing type name to map for nullable
1112+
"0::Int16 AS NULLABLE, " + // passing type name to map for nullable
11071113
"not(dt.case_insensitive)::Boolean AS CASE_SENSITIVE, " +
11081114
java.sql.DatabaseMetaData.typeSearchable + "::Int16 AS SEARCHABLE, " +
11091115
"not(attrs.c3)::Boolean AS UNSIGNED_ATTRIBUTE, " +
11101116
"false AS FIXED_PREC_SCALE, " +
11111117
"false AS AUTO_INCREMENT, " +
1112-
"name AS LOCAL_TYPE_NAME, " +
1118+
"if(empty(alias_to), name, alias_to) AS LOCAL_TYPE_NAME, " +
11131119
"attrs.c4::Nullable(Int16) AS MINIMUM_SCALE, " +
11141120
"attrs.c5::Nullable(Int16) AS MAXIMUM_SCALE, " +
11151121
"0::Nullable(Int32) AS SQL_DATA_TYPE, " +
@@ -1134,6 +1140,8 @@ private static String getDataTypeInfoSql() {
11341140
return sql.toString();
11351141
}
11361142

1143+
private static final ClickHouseColumn.ValueFunction TYPE_INFO_DATA_TYPE_VALUE_FUNC = dataTypeValueFunction(DATA_TYPE_INFO_SQL_TYPE_NAME_COL);
1144+
11371145
@Override
11381146
public ResultSet getIndexInfo(String catalog, String schema, String table, boolean unique, boolean approximate) throws SQLException {
11391147
try {

‎jdbc-v2/src/main/java/com/clickhouse/jdbc/metadata/ResultSetMetaDataImpl.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
import com.clickhouse.jdbc.JdbcV2Wrapper;
66
import com.clickhouse.jdbc.internal.ExceptionUtils;
77
import com.clickhouse.jdbc.internal.JdbcUtils;
8+
import com.google.common.collect.ImmutableList;
9+
import com.google.common.collect.UnmodifiableListIterator;
810

911
import java.sql.SQLException;
1012
import java.util.List;
@@ -22,13 +24,16 @@ public class ResultSetMetaDataImpl implements java.sql.ResultSetMetaData, JdbcV2
2224

2325
private final Map<ClickHouseDataType, Class<?>> typeClassMap;
2426

27+
private int columnCount;
28+
2529
public ResultSetMetaDataImpl(List<ClickHouseColumn> columns, String schema, String catalog, String tableName,
2630
Map<ClickHouseDataType, Class<?>> typeClassMap) {
27-
this.columns = columns;
31+
this.columns = ImmutableList.copyOf(columns);
2832
this.schema = schema;
2933
this.catalog = catalog;
3034
this.tableName = tableName;
3135
this.typeClassMap = typeClassMap;
36+
this.columnCount = columns.size();
3237
}
3338

3439
private ClickHouseColumn getColumn(int column) throws SQLException {
@@ -41,9 +46,24 @@ private ClickHouseColumn getColumn(int column) throws SQLException {
4146

4247
@Override
4348
public int getColumnCount() throws SQLException {
49+
return columnCount;
50+
}
51+
52+
public int getOriginalColumnCount() {
4453
return columns.size();
4554
}
4655

56+
/**
57+
* This method used to truncate list of column so it is possible to
58+
* "hide" columns from the end of the list.
59+
* Note: we use this to implement column replacement. it is needed when DB calculation is too hard compare to a
60+
* programmatic approach.
61+
* @param columnCount
62+
*/
63+
public void setColumnCount(int columnCount) {
64+
this.columnCount = columnCount;
65+
}
66+
4767
@Override
4868
public boolean isAutoIncrement(int column) throws SQLException {
4969
return false; // no auto-incremental types

0 commit comments

Comments
 (0)