Skip to content

Commit df42e8c

Browse files
shah-nirmitNirmit Shah
shah-nirmit
and
Nirmit Shah
authored
Fix float to string conversion for boundary values (#3656)
In #3413 , we added new Cast functions for float to varchar/char conversion . We were using the postgres double_to_shortest_decimal_buf API to convert float to string , which would return special values like Infinity/-Infinity for very big/small values , also we were using rounding logic by multiplication and then dividing the result which could result in overflow, Now we fallback to string formatting using psprintf (postgres flavour of printf) and format specifier to fix the precision. This change also solves the CONVERT function not working correctly with very small float values . Issues Resolved BABEL-5709 Cherry-pick #3654 Co-authored-by: Nirmit Shah <nirmisha@amazon.com>
1 parent 0d8f39c commit df42e8c

18 files changed

+232
-104
lines changed

‎contrib/babelfishpg_common/src/varchar.c‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1199,10 +1199,7 @@ float82varchar(PG_FUNCTION_ARGS)
11991199
/* When No Typmod is defined Default Length is 30 */
12001200
int maxlen = (typmod == -1) ? 30 : (typmod - VARHDRSZ);
12011201
Datum res;
1202-
/* 32 length as double_to_shortest_decimal_buf always returns string with length less that 30*/
1203-
char *ascii = (char *) palloc0(32);
1204-
1205-
/* round to 6 decimal digits */
1202+
char *result;
12061203
if (unlikely(isinf(num)|| isnan(num)))
12071204
{
12081205
ereport(ERROR,
@@ -1211,15 +1208,13 @@ float82varchar(PG_FUNCTION_ARGS)
12111208
}
12121209
else
12131210
{
1214-
num = round(num * 1000000.0) / 1000000.0;
1211+
result = psprintf("%.6g",num);
12151212
}
12161213

1217-
double_to_shortest_decimal_buf(num, ascii);
1218-
12191214
/* Check if the number fits within the specified length */
12201215
if (maxlen > 0)
12211216
{
1222-
size_t str_len = strlen(ascii);
1217+
size_t str_len = strlen(result);
12231218
if (str_len > maxlen)
12241219
{
12251220
ereport(ERROR,
@@ -1229,7 +1224,7 @@ float82varchar(PG_FUNCTION_ARGS)
12291224
}
12301225

12311226
res = DirectFunctionCall3(varcharin,
1232-
CStringGetDatum(ascii),
1227+
CStringGetDatum(result),
12331228
ObjectIdGetDatum(0),
12341229
Int32GetDatum(typmod));
12351230

@@ -1245,8 +1240,7 @@ float82bpchar(PG_FUNCTION_ARGS)
12451240
/* When No Typmod is defined Default Length is 30 */
12461241
int maxlen = (typmod == -1) ? 30 : (typmod - VARHDRSZ);
12471242
Datum res;
1248-
/* 32 length as double_to_shortest_decimal_buf always returns string with length less that 30*/
1249-
char *ascii = (char *) palloc0(32);
1243+
char *result;
12501244
char *buf_padded;
12511245
int str_len;
12521246

@@ -1259,13 +1253,11 @@ float82bpchar(PG_FUNCTION_ARGS)
12591253
}
12601254
else
12611255
{
1262-
num = round(num * 1000000.0) / 1000000.0;
1256+
result = psprintf("%.6g",num);
12631257
}
12641258

1265-
double_to_shortest_decimal_buf(num, ascii);
1266-
12671259
/* Check if the number fits within the specified length */
1268-
str_len = strlen(ascii);
1260+
str_len = strlen(result);
12691261
if (str_len > maxlen)
12701262
{
12711263
ereport(ERROR,
@@ -1275,7 +1267,7 @@ float82bpchar(PG_FUNCTION_ARGS)
12751267

12761268
/* Right pad float value with the spaces */
12771269
buf_padded = (char *) palloc0(maxlen + 1);
1278-
memcpy(buf_padded, ascii, str_len);
1270+
memcpy(buf_padded, result, str_len);
12791271
memset(buf_padded + str_len, ' ', maxlen - str_len);
12801272

12811273
res = DirectFunctionCall3(bpcharin,

‎contrib/babelfishpg_tsql/sql/sys_function_helpers.sql‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10992,12 +10992,16 @@ BEGIN
1099210992
v_format := '9D99999EEEE';
1099310993
v_result := to_char(v_sign::NUMERIC * ceiling(v_floatval), v_format);
1099410994
v_result := to_char(substring(v_result, 1, 8)::NUMERIC, 'FM9D99999')::NUMERIC::TEXT || substring(v_result, 9);
10995+
ELSIF (v_floatval < 0.0001 AND v_floatval != 0) THEN
10996+
v_format := '9D99999EEEE';
10997+
v_result := to_char(v_sign::NUMERIC * v_floatval, v_format);
10998+
v_result := to_char(substring(v_result, 1, 8)::NUMERIC, 'FM9D99999')::NUMERIC::TEXT || substring(v_result, 9);
1099510999
ELSE
10996-
IF (6 - v_integral_digits < v_decimal_digits) AND (trunc(abs(v_floatval)) != 0) THEN
10997-
v_decimal_digits := 6 - v_integral_digits;
10998-
ELSIF (6 - v_integral_digits < v_decimal_digits) THEN
10999-
v_decimal_digits := 6;
11000-
END IF;
11000+
IF (6 - v_integral_digits < v_decimal_digits) AND (trunc(abs(v_floatval)) != 0) THEN
11001+
v_decimal_digits := 6 - v_integral_digits;
11002+
ELSIF (6 - v_integral_digits < v_decimal_digits) THEN
11003+
v_decimal_digits := 6;
11004+
END IF;
1100111005
v_format := (pow(10, v_integral_digits)-10)::TEXT || 'D';
1100211006
IF (v_decimal_digits > 0) THEN
1100311007
v_format := v_format || (pow(10, v_decimal_digits)-1)::TEXT;

‎contrib/babelfishpg_tsql/sql/upgrades/babelfishpg_tsql--5.1.0--5.2.0.sql‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -677,12 +677,16 @@ BEGIN
677677
v_format := '9D99999EEEE';
678678
v_result := to_char(v_sign::NUMERIC * ceiling(v_floatval), v_format);
679679
v_result := to_char(substring(v_result, 1, 8)::NUMERIC, 'FM9D99999')::NUMERIC::TEXT || substring(v_result, 9);
680+
ELSIF (v_floatval < 0.0001 AND v_floatval != 0) THEN
681+
v_format := '9D99999EEEE';
682+
v_result := to_char(v_sign::NUMERIC * v_floatval, v_format);
683+
v_result := to_char(substring(v_result, 1, 8)::NUMERIC, 'FM9D99999')::NUMERIC::TEXT || substring(v_result, 9);
680684
ELSE
681-
IF (6 - v_integral_digits < v_decimal_digits) AND (trunc(abs(v_floatval)) != 0) THEN
682-
v_decimal_digits := 6 - v_integral_digits;
683-
ELSIF (6 - v_integral_digits < v_decimal_digits) THEN
684-
v_decimal_digits := 6;
685-
END IF;
685+
IF (6 - v_integral_digits < v_decimal_digits) AND (trunc(abs(v_floatval)) != 0) THEN
686+
v_decimal_digits := 6 - v_integral_digits;
687+
ELSIF (6 - v_integral_digits < v_decimal_digits) THEN
688+
v_decimal_digits := 6;
689+
END IF;
686690
v_format := (pow(10, v_integral_digits)-10)::TEXT || 'D';
687691
IF (v_decimal_digits > 0) THEN
688692
v_format := v_format || (pow(10, v_decimal_digits)-1)::TEXT;

‎test/JDBC/expected/BABEL-1193.out‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,8 @@ SELECT * FROM t14;
221221
GO
222222
~~START~~
223223
varchar#!#varchar#!#varchar#!#varchar#!#varchar#!#varchar#!#varchar#!#varchar
224-
12#!#4553#!#123456#!#12.345#!#2344.456#!#12.34#!#456.33#!#1123.68
225-
-12#!#-1234#!#-123456#!#-12.345#!#-2344.456#!#12.34#!#-456.33#!#-1123.68
224+
12#!#4553#!#123456#!#12.345#!#2344.46#!#12.34#!#456.33#!#1123.68
225+
-12#!#-1234#!#-123456#!#-12.345#!#-2344.46#!#12.34#!#-456.33#!#-1123.68
226226
0#!#0#!#0#!#0#!#0#!#0.00#!#0.00#!#0.00
227227
~~END~~
228228

@@ -237,8 +237,8 @@ SELECT * FROM t15;
237237
GO
238238
~~START~~
239239
char#!#char#!#char#!#char#!#char#!#char#!#char#!#char
240-
12 #!#4553 #!#123456 #!#12.345 #!#2344.456 #!#12.34 #!# 456.33#!# 1123.68
241-
-12 #!#-1234 #!#-123456 #!#-12.345 #!#-2344.456 #!#12.34 #!# -456.33#!# -1123.68
240+
12 #!#4553 #!#123456 #!#12.345 #!#2344.46 #!#12.34 #!# 456.33#!# 1123.68
241+
-12 #!#-1234 #!#-123456 #!#-12.345 #!#-2344.46 #!#12.34 #!# -456.33#!# -1123.68
242242
0 #!#0 #!#0 #!#0 #!#0 #!#0.00 #!# 0.00#!# 0.00
243243
~~END~~
244244

‎test/JDBC/expected/BABEL-889-vu-verify.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ select cast(cast(cast('3.1415926' as float) as sql_variant) as varchar);
8686
go
8787
~~START~~
8888
varchar
89-
3.141593
89+
3.14159
9090
~~END~~
9191

9292
-- real

‎test/JDBC/expected/BABEL-889.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ select cast(cast(cast('3.1415926' as float) as sql_variant) as varchar);
8686
go
8787
~~START~~
8888
varchar
89-
3.141593
89+
3.14159
9090
~~END~~
9191

9292
-- real

‎test/JDBC/expected/babel_cursor.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ varchar
267267

268268
~~START~~
269269
varchar
270-
@var_a: 4444.4444
270+
@var_a: 4444.44
271271
~~END~~
272272

273273

@@ -699,7 +699,7 @@ varchar
699699

700700
~~START~~
701701
varchar
702-
@var_d: 4444.5444
702+
@var_d: 4444.54
703703
~~END~~
704704

705705
~~START~~

‎test/JDBC/expected/babel_functions_cast.out‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ select CAST(CAST(11234561231231.234 AS float) AS varchar(30));
6262
GO
6363
~~START~~
6464
varchar
65-
11234561231231.234
65+
1.12346e+13
6666
~~END~~
6767

6868
select CAST('123' AS int);
@@ -395,7 +395,7 @@ select TRY_CAST(CAST(11234561231231.234 AS float) AS varchar(30));
395395
GO
396396
~~START~~
397397
varchar
398-
11234561231231.234
398+
1.12346e+13
399399
~~END~~
400400

401401
select TRY_CAST('123' AS int);

‎test/JDBC/expected/non_default_server_collation/chinese_prc_ci_as/BABEL-889-vu-verify.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ select cast(cast(cast('3.1415926' as float) as sql_variant) as varchar);
8686
go
8787
~~START~~
8888
varchar
89-
3.141593
89+
3.14159
9090
~~END~~
9191

9292
-- real

‎test/JDBC/expected/non_default_server_collation/chinese_prc_ci_as/BABEL-889.out‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ select cast(cast(cast('3.1415926' as float) as sql_variant) as varchar);
8686
go
8787
~~START~~
8888
varchar
89-
3.141593
89+
3.14159
9090
~~END~~
9191

9292
-- real

0 commit comments

Comments
 (0)