Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.

Commit a3088d2

Browse files
Gabor Fehergatorsmile
authored andcommitted
[SPARK-20555][SQL] Fix mapping of Oracle DECIMAL types to Spark types in read path
## What changes were proposed in this pull request? This PR is to revert some code changes in the read path of apache#14377. The original fix is apache#17830 When merging this PR, please give the credit to gaborfeher ## How was this patch tested? Added a test case to OracleIntegrationSuite.scala Author: Gabor Feher <[email protected]> Author: gatorsmile <[email protected]> Closes apache#18408 from gatorsmile/OracleType. (cherry picked from commit b837bf9) Signed-off-by: gatorsmile <[email protected]>
1 parent 3394b06 commit a3088d2

File tree

2 files changed

+45
-24
lines changed

2 files changed

+45
-24
lines changed

external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.sql.jdbc
1919

2020
import java.sql.{Connection, Date, Timestamp}
2121
import java.util.Properties
22+
import java.math.BigDecimal
2223

2324
import org.apache.spark.sql.Row
2425
import org.apache.spark.sql.test.SharedSQLContext
@@ -87,8 +88,31 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
8788
|USING org.apache.spark.sql.jdbc
8889
|OPTIONS (url '$jdbcUrl', dbTable 'datetime1', oracle.jdbc.mapDateToTimestamp 'false')
8990
""".stripMargin.replaceAll("\n", " "))
91+
92+
93+
conn.prepareStatement("CREATE TABLE numerics (b DECIMAL(1), f DECIMAL(3, 2), i DECIMAL(10))").executeUpdate();
94+
conn.prepareStatement(
95+
"INSERT INTO numerics VALUES (4, 1.23, 9999999999)").executeUpdate();
96+
conn.commit();
9097
}
9198

99+
100+
test("SPARK-16625 : Importing Oracle numeric types") {
101+
val df = sqlContext.read.jdbc(jdbcUrl, "numerics", new Properties);
102+
val rows = df.collect()
103+
assert(rows.size == 1)
104+
val row = rows(0)
105+
// The main point of the below assertions is not to make sure that these Oracle types are
106+
// mapped to decimal types, but to make sure that the returned values are correct.
107+
// A value > 1 from DECIMAL(1) is correct:
108+
assert(row.getDecimal(0).compareTo(BigDecimal.valueOf(4)) == 0)
109+
// A value with fractions from DECIMAL(3, 2) is correct:
110+
assert(row.getDecimal(1).compareTo(BigDecimal.valueOf(1.23)) == 0)
111+
// A value > Int.MaxValue from DECIMAL(10) is correct:
112+
assert(row.getDecimal(2).compareTo(BigDecimal.valueOf(9999999999l)) == 0)
113+
}
114+
115+
92116
test("SPARK-12941: String datatypes to be mapped to Varchar in Oracle") {
93117
// create a sample dataframe with string type
94118
val df1 = sparkContext.parallelize(Seq(("foo"))).toDF("x")
@@ -148,27 +172,28 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
148172
val dfRead = spark.read.jdbc(jdbcUrl, tableName, props)
149173
val rows = dfRead.collect()
150174
// verify the data type is inserted
151-
val types = rows(0).toSeq.map(x => x.getClass.toString)
152-
assert(types(0).equals("class java.lang.Boolean"))
153-
assert(types(1).equals("class java.lang.Integer"))
154-
assert(types(2).equals("class java.lang.Long"))
155-
assert(types(3).equals("class java.lang.Float"))
156-
assert(types(4).equals("class java.lang.Float"))
157-
assert(types(5).equals("class java.lang.Integer"))
158-
assert(types(6).equals("class java.lang.Integer"))
159-
assert(types(7).equals("class java.lang.String"))
160-
assert(types(8).equals("class [B"))
161-
assert(types(9).equals("class java.sql.Date"))
162-
assert(types(10).equals("class java.sql.Timestamp"))
175+
val types = dfRead.schema.map(field => field.dataType)
176+
assert(types(0).equals(DecimalType(1, 0)))
177+
assert(types(1).equals(DecimalType(10, 0)))
178+
assert(types(2).equals(DecimalType(19, 0)))
179+
assert(types(3).equals(DecimalType(19, 4)))
180+
assert(types(4).equals(DecimalType(19, 4)))
181+
assert(types(5).equals(DecimalType(3, 0)))
182+
assert(types(6).equals(DecimalType(5, 0)))
183+
assert(types(7).equals(StringType))
184+
assert(types(8).equals(BinaryType))
185+
assert(types(9).equals(DateType))
186+
assert(types(10).equals(TimestampType))
187+
163188
// verify the value is the inserted correct or not
164189
val values = rows(0)
165-
assert(values.getBoolean(0).equals(booleanVal))
166-
assert(values.getInt(1).equals(integerVal))
167-
assert(values.getLong(2).equals(longVal))
168-
assert(values.getFloat(3).equals(floatVal))
169-
assert(values.getFloat(4).equals(doubleVal.toFloat))
170-
assert(values.getInt(5).equals(byteVal.toInt))
171-
assert(values.getInt(6).equals(shortVal.toInt))
190+
assert(values.getDecimal(0).compareTo(BigDecimal.valueOf(1)) == 0)
191+
assert(values.getDecimal(1).compareTo(BigDecimal.valueOf(integerVal)) == 0)
192+
assert(values.getDecimal(2).compareTo(BigDecimal.valueOf(longVal)) == 0)
193+
assert(values.getDecimal(3).compareTo(BigDecimal.valueOf(floatVal)) == 0)
194+
assert(values.getDecimal(4).compareTo(BigDecimal.valueOf(doubleVal)) == 0)
195+
assert(values.getDecimal(5).compareTo(BigDecimal.valueOf(byteVal)) == 0)
196+
assert(values.getDecimal(6).compareTo(BigDecimal.valueOf(shortVal)) == 0)
172197
assert(values.getString(7).equals(stringVal))
173198
assert(values.getAs[Array[Byte]](8).mkString.equals("678"))
174199
assert(values.getDate(9).equals(dateVal))
@@ -177,7 +202,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSQLCo
177202

178203
test("SPARK-19318: connection property keys should be case-sensitive") {
179204
def checkRow(row: Row): Unit = {
180-
assert(row.getInt(0) == 1)
205+
assert(row.getDecimal(0).equals(BigDecimal.valueOf(1)))
181206
assert(row.getDate(1).equals(Date.valueOf("1991-11-09")))
182207
assert(row.getTimestamp(2).equals(Timestamp.valueOf("1996-01-01 01:23:45")))
183208
}

sql/core/src/main/scala/org/apache/spark/sql/jdbc/OracleDialect.scala

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,6 @@ private case object OracleDialect extends JdbcDialect {
4343
// Not sure if there is a more robust way to identify the field as a float (or other
4444
// numeric types that do not specify a scale.
4545
case _ if scale == -127L => Option(DecimalType(DecimalType.MAX_PRECISION, 10))
46-
case 1 => Option(BooleanType)
47-
case 3 | 5 | 10 => Option(IntegerType)
48-
case 19 if scale == 0L => Option(LongType)
49-
case 19 if scale == 4L => Option(FloatType)
5046
case _ => None
5147
}
5248
} else {

0 commit comments

Comments
 (0)