Skip to content

Commit 0475a49

Browse files
zaleslawzaleslaw
committed
Fixing numerical types (#547)
* Update handling of SQL Types in PostgreSQL and add test cases Extended the handling of SQL Types TINYINT and SMALLINT in PostgreSql.kt to address issues reported against these data types. * Continued the enhancement of TableColumnMetadata by adding `javaClassName` as a new property. Also made changes to handle TINYINT and SMALLINT types more effectively in MySql and MariaDb classes. * Refactor type handling of TINYINT and SMALLINT in SQL classes * In this commit, TINYINT and SMALLINT types are refactored across several SQL related classes. * Remove unnecessary imports and add ignore annotations to tests
1 parent 4bb9b15 commit 0475a49

File tree

8 files changed

+296
-37
lines changed

8 files changed

+296
-37
lines changed

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MariaDb.kt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema
55
import java.sql.ResultSet
66
import org.jetbrains.kotlinx.dataframe.io.TableMetadata
77
import kotlin.reflect.KType
8+
import kotlin.reflect.full.createType
89

910
/**
1011
* Represents the MariaDb database type.
@@ -17,6 +18,10 @@ public object MariaDb : DbType("mariadb") {
1718
get() = "org.mariadb.jdbc.Driver"
1819

1920
override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? {
21+
if(tableColumnMetadata.sqlTypeName == "SMALLINT") {
22+
val kType = Short::class.createType(nullable = tableColumnMetadata.isNullable)
23+
return ColumnSchema.Value(kType)
24+
}
2025
return null
2126
}
2227

@@ -32,6 +37,8 @@ public object MariaDb : DbType("mariadb") {
3237
}
3338

3439
override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? {
40+
if(tableColumnMetadata.sqlTypeName == "SMALLINT")
41+
return Short::class.createType(nullable = tableColumnMetadata.isNullable)
3542
return null
3643
}
3744
}

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/MySql.kt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import java.sql.ResultSet
66
import java.util.Locale
77
import org.jetbrains.kotlinx.dataframe.io.TableMetadata
88
import kotlin.reflect.KType
9+
import kotlin.reflect.full.createType
910

1011
/**
1112
* Represents the MySql database type.
@@ -18,6 +19,10 @@ public object MySql : DbType("mysql") {
1819
get() = "com.mysql.jdbc.Driver"
1920

2021
override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? {
22+
if(tableColumnMetadata.sqlTypeName == "INT UNSIGNED") {
23+
val kType = Long::class.createType(nullable = tableColumnMetadata.isNullable)
24+
return ColumnSchema.Value(kType)
25+
}
2126
return null
2227
}
2328

@@ -45,6 +50,8 @@ public object MySql : DbType("mysql") {
4550
}
4651

4752
override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? {
53+
if(tableColumnMetadata.sqlTypeName == "INT UNSIGNED")
54+
return Long::class.createType(nullable = tableColumnMetadata.isNullable)
4855
return null
4956
}
5057
}

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/PostgreSql.kt

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import java.sql.ResultSet
66
import java.util.Locale
77
import org.jetbrains.kotlinx.dataframe.io.TableMetadata
88
import kotlin.reflect.KType
9-
import kotlin.reflect.typeOf
9+
import kotlin.reflect.full.createType
1010

1111
/**
1212
* Represents the PostgreSql database type.
@@ -19,6 +19,12 @@ public object PostgreSql : DbType("postgresql") {
1919
get() = "org.postgresql.Driver"
2020

2121
override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? {
22+
// TODO: could be a wrapper of convertSqlTypeToKType
23+
if (tableColumnMetadata.sqlTypeName == "money") // because of https://github.com/pgjdbc/pgjdbc/issues/425
24+
{
25+
val kType = String::class.createType(nullable = tableColumnMetadata.isNullable)
26+
return ColumnSchema.Value(kType)
27+
}
2228
return null
2329
}
2430

@@ -36,7 +42,7 @@ public object PostgreSql : DbType("postgresql") {
3642

3743
override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? {
3844
if(tableColumnMetadata.sqlTypeName == "money") // because of https://github.com/pgjdbc/pgjdbc/issues/425
39-
return typeOf<String>()
45+
return String::class.createType(nullable = tableColumnMetadata.isNullable)
4046
return null
4147
}
4248
}

‎dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,16 @@ private const val MULTIPLE_SQL_QUERY_SEPARATOR = ";"
6565
* @property [sqlTypeName] the SQL data type of the column.
6666
* @property [jdbcType] the JDBC data type of the column produced from [java.sql.Types].
6767
* @property [size] the size of the column.
68+
* @property [javaClassName] the class name in Java.
6869
* @property [isNullable] true if column could contain nulls.
6970
*/
7071
public data class TableColumnMetadata(
7172
val name: String,
7273
val sqlTypeName: String,
7374
val jdbcType: Int,
7475
val size: Int,
75-
val isNullable: Boolean = false
76+
val javaClassName: String,
77+
val isNullable: Boolean = false,
7678
)
7779

7880
/**
@@ -521,8 +523,9 @@ private fun getTableColumnsMetadata(rs: ResultSet): MutableList<TableColumnMetad
521523
val size = metaData.getColumnDisplaySize(i)
522524
val type = metaData.getColumnTypeName(i)
523525
val jdbcType = metaData.getColumnType(i)
526+
val javaClassName = metaData.getColumnClassName(i)
524527

525-
tableColumns += TableColumnMetadata(name, type, jdbcType, size, isNullable)
528+
tableColumns += TableColumnMetadata(name, type, jdbcType, size, javaClassName, isNullable)
526529
}
527530
return tableColumns
528531
}
@@ -642,8 +645,8 @@ private fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetada
642645
private fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType {
643646
val jdbcTypeToKTypeMapping = mapOf(
644647
Types.BIT to Boolean::class,
645-
Types.TINYINT to Byte::class,
646-
Types.SMALLINT to Short::class,
648+
Types.TINYINT to Int::class,
649+
Types.SMALLINT to Int::class,
647650
Types.INTEGER to Int::class,
648651
Types.BIGINT to Long::class,
649652
Types.FLOAT to Float::class,
@@ -681,6 +684,7 @@ private fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata
681684
Types.TIME_WITH_TIMEZONE to Time::class,
682685
Types.TIMESTAMP_WITH_TIMEZONE to Timestamp::class
683686
)
687+
// TODO: check mapping of JDBC types and classes correctly
684688
val kClass = jdbcTypeToKTypeMapping[tableColumnMetadata.jdbcType] ?: String::class
685689
return kClass.createType(nullable = tableColumnMetadata.isNullable)
686690
}

‎dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/h2Test.kt

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ import java.sql.Connection
1717
import java.sql.DriverManager
1818
import java.sql.ResultSet
1919
import java.sql.SQLException
20+
import org.jetbrains.kotlinx.dataframe.api.add
21+
import org.jetbrains.kotlinx.dataframe.api.select
2022
import kotlin.reflect.typeOf
2123

2224
private const val URL = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=MySQL;DATABASE_TO_UPPER=false"
@@ -52,14 +54,14 @@ interface TestTableData {
5254
val binaryVaryingCol: ByteArray?
5355
val binaryLargeObjectCol: ByteArray?
5456
val booleanCol: Boolean?
55-
val tinyIntCol: Byte?
56-
val smallIntCol: Short?
57+
val tinyIntCol: Int?
58+
val smallIntCol: Int?
5759
val integerCol: Int?
5860
val bigIntCol: Long?
59-
val numericCol: Double?
61+
val numericCol: BigDecimal?
6062
val realCol: Float?
6163
val doublePrecisionCol: Double?
62-
val decFloatCol: Double?
64+
val decFloatCol: BigDecimal?
6365
val dateCol: String?
6466
val timeCol: String?
6567
val timeWithTimeZoneCol: String?
@@ -235,9 +237,56 @@ class JdbcTest {
235237
""".trimIndent()
236238
).executeUpdate()
237239

238-
val df = DataFrame.readSqlTable(connection, "TestTable").cast<TestTableData>()
240+
val tableName = "TestTable"
241+
val df = DataFrame.readSqlTable(connection, tableName).cast<TestTableData>()
239242
df.rowsCount() shouldBe 3
240243
df.filter { it[TestTableData::integerCol]!! > 1000 }.rowsCount() shouldBe 2
244+
245+
// testing numeric columns
246+
val result = df.select("tinyIntCol")
247+
.add("tinyIntCol2") { it[TestTableData::tinyIntCol] }
248+
249+
result[0][1] shouldBe 1
250+
251+
val result1 = df.select("smallIntCol")
252+
.add("smallIntCol2") { it[TestTableData::smallIntCol] }
253+
254+
result1[0][1] shouldBe 100
255+
256+
val result2 = df.select("bigIntCol")
257+
.add("bigIntCol2") { it[TestTableData::bigIntCol] }
258+
259+
result2[0][1] shouldBe 100000
260+
261+
val result3 = df.select("numericCol")
262+
.add("numericCol2") { it[TestTableData::numericCol] }
263+
264+
BigDecimal("123.45").compareTo(result3[0][1] as BigDecimal) shouldBe 0
265+
266+
val result4 = df.select("realCol")
267+
.add("realCol2") { it[TestTableData::realCol] }
268+
269+
result4[0][1] shouldBe 1.23f
270+
271+
val result5 = df.select("doublePrecisionCol")
272+
.add("doublePrecisionCol2") { it[TestTableData::doublePrecisionCol] }
273+
274+
result5[0][1] shouldBe 3.14
275+
276+
val result6 = df.select("decFloatCol")
277+
.add("decFloatCol2") { it[TestTableData::decFloatCol] }
278+
279+
BigDecimal("2.71").compareTo(result6[0][1] as BigDecimal) shouldBe 0
280+
281+
val schema = DataFrame.getSchemaForSqlTable(connection, tableName)
282+
283+
schema.columns["tinyIntCol"]!!.type shouldBe typeOf<Int?>()
284+
schema.columns["smallIntCol"]!!.type shouldBe typeOf<Int?>()
285+
schema.columns["bigIntCol"]!!.type shouldBe typeOf<Long?>()
286+
schema.columns["numericCol"]!!.type shouldBe typeOf<BigDecimal?>()
287+
schema.columns["realCol"]!!.type shouldBe typeOf<Float?>()
288+
schema.columns["doublePrecisionCol"]!!.type shouldBe typeOf<Double?>()
289+
schema.columns["decFloatCol"]!!.type shouldBe typeOf<BigDecimal?>()
241290
}
242291

243292
@Test

‎dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mariadbTest.kt

Lines changed: 69 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import java.math.BigDecimal
1313
import java.sql.Connection
1414
import java.sql.DriverManager
1515
import java.sql.SQLException
16+
import org.jetbrains.kotlinx.dataframe.api.add
17+
import org.jetbrains.kotlinx.dataframe.api.select
1618
import org.junit.Ignore
1719
import kotlin.reflect.typeOf
1820

@@ -26,16 +28,16 @@ interface Table1MariaDb {
2628
val id: Int
2729
val bitCol: Boolean
2830
val tinyintCol: Int
29-
val smallintCol: Int
31+
val smallintCol: Short?
3032
val mediumintCol: Int
31-
val mediumintUnsignedCol: Long
33+
val mediumintUnsignedCol: Int
3234
val integerCol: Int
3335
val intCol: Int
3436
val integerUnsignedCol: Long
3537
val bigintCol: Long
3638
val floatCol: Float
3739
val doubleCol: Double
38-
val decimalCol: Double
40+
val decimalCol: BigDecimal
3941
val dateCol: String
4042
val datetimeCol: String
4143
val timestampCol: String
@@ -64,7 +66,7 @@ interface Table2MariaDb {
6466
val tinyintCol: Int?
6567
val smallintCol: Int?
6668
val mediumintCol: Int?
67-
val mediumintUnsignedCol: Long?
69+
val mediumintUnsignedCol: Int?
6870
val integerCol: Int?
6971
val intCol: Int?
7072
val integerUnsignedCol: Long?
@@ -138,7 +140,7 @@ class MariadbTest {
138140
id INT AUTO_INCREMENT PRIMARY KEY,
139141
bitCol BIT NOT NULL,
140142
tinyintCol TINYINT NOT NULL,
141-
smallintCol SMALLINT NOT NULL,
143+
smallintCol SMALLINT,
142144
mediumintCol MEDIUMINT NOT NULL,
143145
mediumintUnsignedCol MEDIUMINT UNSIGNED NOT NULL,
144146
integerCol INTEGER NOT NULL,
@@ -387,4 +389,66 @@ class MariadbTest {
387389
table2Df[0][11] shouldBe 20.0
388390
table2Df[0][26] shouldBe null
389391
}
392+
393+
@Test
394+
fun `reading numeric types`() {
395+
val df1 = DataFrame.readSqlTable(connection, "table1").cast<Table1MariaDb>()
396+
397+
val result = df1.select("tinyintCol")
398+
.add("tinyintCol2") { it[Table1MariaDb::tinyintCol] }
399+
400+
result[0][1] shouldBe 1
401+
402+
val result1 = df1.select("smallintCol")
403+
.add("smallintCol2") { it[Table1MariaDb::smallintCol] }
404+
405+
result1[0][1] shouldBe 10
406+
407+
val result2 = df1.select("mediumintCol")
408+
.add("mediumintCol2") { it[Table1MariaDb::mediumintCol] }
409+
410+
result2[0][1] shouldBe 100
411+
412+
val result3 = df1.select("mediumintUnsignedCol")
413+
.add("mediumintUnsignedCol2") { it[Table1MariaDb::mediumintUnsignedCol] }
414+
415+
result3[0][1] shouldBe 100
416+
417+
val result4 = df1.select("integerUnsignedCol")
418+
.add("integerUnsignedCol2") { it[Table1MariaDb::integerUnsignedCol] }
419+
420+
result4[0][1] shouldBe 100L
421+
422+
val result5 = df1.select("bigintCol")
423+
.add("bigintCol2") { it[Table1MariaDb::bigintCol] }
424+
425+
result5[0][1] shouldBe 100
426+
427+
val result6 = df1.select("floatCol")
428+
.add("floatCol2") { it[Table1MariaDb::floatCol] }
429+
430+
result6[0][1] shouldBe 10.0f
431+
432+
val result7 = df1.select("doubleCol")
433+
.add("doubleCol2") { it[Table1MariaDb::doubleCol] }
434+
435+
result7[0][1] shouldBe 10.0
436+
437+
val result8 = df1.select("decimalCol")
438+
.add("decimalCol2") { it[Table1MariaDb::decimalCol] }
439+
440+
result8[0][1] shouldBe BigDecimal("10")
441+
442+
val schema = DataFrame.getSchemaForSqlTable(connection, "table1")
443+
444+
schema.columns["tinyintCol"]!!.type shouldBe typeOf<Int>()
445+
schema.columns["smallintCol"]!!.type shouldBe typeOf<Short?>()
446+
schema.columns["mediumintCol"]!!.type shouldBe typeOf<Int>()
447+
schema.columns["mediumintUnsignedCol"]!!.type shouldBe typeOf<Int>()
448+
schema.columns["integerUnsignedCol"]!!.type shouldBe typeOf<Long>()
449+
schema.columns["bigintCol"]!!.type shouldBe typeOf<Long>()
450+
schema.columns["floatCol"]!!.type shouldBe typeOf<Float>()
451+
schema.columns["doubleCol"]!!.type shouldBe typeOf<Double>()
452+
schema.columns["decimalCol"]!!.type shouldBe typeOf<BigDecimal>()
453+
}
390454
}

0 commit comments

Comments
 (0)