Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.

Commit 6265119

Browse files
gatorsmilecloud-fan
authored andcommitted
[SPARK-20211][SQL][BACKPORT-2.2] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0
### What changes were proposed in this pull request? This PR is to backport apache#18244 to 2.2 --- The precision and scale of decimal values are wrong when the input is BigDecimal between -1.0 and 1.0. The BigDecimal's precision is the digit count starts from the leftmost nonzero digit based on the [JAVA's BigDecimal definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html). However, our Decimal decision follows the database decimal standard, which is the total number of digits, including both to the left and the right of the decimal point. Thus, this PR is to fix the issue by doing the conversion. Before this PR, the following queries failed: ```SQL select 1 > 0.0001 select floor(0.0001) select ceil(0.0001) ``` ### How was this patch tested? Added test cases. Author: gatorsmile <[email protected]> Closes apache#18297 from gatorsmile/backport18244.
1 parent 9bdc835 commit 6265119

File tree

4 files changed

+176
-2
lines changed

4 files changed

+176
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,15 @@ final class Decimal extends Ordered[Decimal] with Serializable {
126126
def set(decimal: BigDecimal): Decimal = {
127127
this.decimalVal = decimal
128128
this.longVal = 0L
129-
this._precision = decimal.precision
129+
if (decimal.precision <= decimal.scale) {
130+
// For Decimal, we expect the precision is equal to or large than the scale, however,
131+
// in BigDecimal, the digit count starts from the leftmost nonzero digit of the exact
132+
// result. For example, the precision of 0.01 equals to 1 based on the definition, but
133+
// the scale is 2. The expected precision should be 3.
134+
this._precision = decimal.scale + 1
135+
} else {
136+
this._precision = decimal.precision
137+
}
130138
this._scale = decimal.scale
131139
this
132140
}

sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester {
3232

3333
test("creating decimals") {
3434
checkDecimal(new Decimal(), "0", 1, 0)
35+
checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
36+
checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
37+
checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
38+
checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
39+
checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
40+
checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
41+
checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
42+
checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
43+
checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
44+
checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
3545
checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
3646
checkDecimal(Decimal(BigDecimal("10.030"), 4, 1), "10.0", 4, 1)
3747
checkDecimal(Decimal(BigDecimal("-9.95"), 4, 1), "-10.0", 4, 1)

sql/core/src/test/resources/sql-tests/inputs/arithmetic.sql

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,3 +32,27 @@ select 1 - 2;
3232
select 2 * 5;
3333
select 5 % 3;
3434
select pmod(-7, 3);
35+
36+
-- math functions
37+
select cot(1);
38+
select cot(null);
39+
select cot(0);
40+
select cot(-1);
41+
42+
-- ceil and ceiling
43+
select ceiling(0);
44+
select ceiling(1);
45+
select ceil(1234567890123456);
46+
select ceiling(1234567890123456);
47+
select ceil(0.01);
48+
select ceiling(-0.10);
49+
50+
-- floor
51+
select floor(0);
52+
select floor(1);
53+
select floor(1234567890123456);
54+
select floor(0.01);
55+
select floor(-0.10);
56+
57+
-- comparison operator
58+
select 1 > 0.00001

sql/core/src/test/resources/sql-tests/results/arithmetic.sql.out

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 28
2+
-- Number of queries: 44
33

44

55
-- !query 0
@@ -224,3 +224,135 @@ select pmod(-7, 3)
224224
struct<pmod(-7, 3):int>
225225
-- !query 27 output
226226
2
227+
228+
229+
-- !query 28
230+
select cot(1)
231+
-- !query 28 schema
232+
struct<>
233+
-- !query 28 output
234+
org.apache.spark.sql.AnalysisException
235+
Undefined function: 'cot'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
236+
237+
238+
-- !query 29
239+
select cot(null)
240+
-- !query 29 schema
241+
struct<>
242+
-- !query 29 output
243+
org.apache.spark.sql.AnalysisException
244+
Undefined function: 'cot'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
245+
246+
247+
-- !query 30
248+
select cot(0)
249+
-- !query 30 schema
250+
struct<>
251+
-- !query 30 output
252+
org.apache.spark.sql.AnalysisException
253+
Undefined function: 'cot'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
254+
255+
256+
-- !query 31
257+
select cot(-1)
258+
-- !query 31 schema
259+
struct<>
260+
-- !query 31 output
261+
org.apache.spark.sql.AnalysisException
262+
Undefined function: 'cot'. This function is neither a registered temporary function nor a permanent function registered in the database 'default'.; line 1 pos 7
263+
264+
265+
-- !query 32
266+
select ceiling(0)
267+
-- !query 32 schema
268+
struct<CEIL(CAST(0 AS DOUBLE)):bigint>
269+
-- !query 32 output
270+
0
271+
272+
273+
-- !query 33
274+
select ceiling(1)
275+
-- !query 33 schema
276+
struct<CEIL(CAST(1 AS DOUBLE)):bigint>
277+
-- !query 33 output
278+
1
279+
280+
281+
-- !query 34
282+
select ceil(1234567890123456)
283+
-- !query 34 schema
284+
struct<CEIL(1234567890123456):bigint>
285+
-- !query 34 output
286+
1234567890123456
287+
288+
289+
-- !query 35
290+
select ceiling(1234567890123456)
291+
-- !query 35 schema
292+
struct<CEIL(1234567890123456):bigint>
293+
-- !query 35 output
294+
1234567890123456
295+
296+
297+
-- !query 36
298+
select ceil(0.01)
299+
-- !query 36 schema
300+
struct<CEIL(0.01):decimal(1,0)>
301+
-- !query 36 output
302+
1
303+
304+
305+
-- !query 37
306+
select ceiling(-0.10)
307+
-- !query 37 schema
308+
struct<CEIL(-0.10):decimal(1,0)>
309+
-- !query 37 output
310+
0
311+
312+
313+
-- !query 38
314+
select floor(0)
315+
-- !query 38 schema
316+
struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
317+
-- !query 38 output
318+
0
319+
320+
321+
-- !query 39
322+
select floor(1)
323+
-- !query 39 schema
324+
struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
325+
-- !query 39 output
326+
1
327+
328+
329+
-- !query 40
330+
select floor(1234567890123456)
331+
-- !query 40 schema
332+
struct<FLOOR(1234567890123456):bigint>
333+
-- !query 40 output
334+
1234567890123456
335+
336+
337+
-- !query 41
338+
select floor(0.01)
339+
-- !query 41 schema
340+
struct<FLOOR(0.01):decimal(1,0)>
341+
-- !query 41 output
342+
0
343+
344+
345+
-- !query 42
346+
select floor(-0.10)
347+
-- !query 42 schema
348+
struct<FLOOR(-0.10):decimal(1,0)>
349+
-- !query 42 output
350+
-1
351+
352+
353+
-- !query 43
354+
select 1 > 0.00001
355+
-- !query 43 schema
356+
struct<(CAST(1 AS BIGINT) > 0):boolean>
357+
-- !query 43 output
358+
true

0 commit comments

Comments
 (0)