Scale of decimal literal should always be positive in Calcite (#4401)

* Fix IT

Signed-off-by: Lantao Jin <[email protected]>

* remove unused class

Signed-off-by: Lantao Jin <[email protected]>

---------

Signed-off-by: Lantao Jin <[email protected]>

Author: LantaoJin

core/src/main/java/org/opensearch/sql/ast/expression/Literal.java

@@ -7,6 +7,7 @@
 
 import com.google.common.collect.ImmutableList;
 import java.math.BigDecimal;
+import java.text.DecimalFormat;
 import java.util.List;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -25,7 +26,21 @@ public class Literal extends UnresolvedExpression {
 
   public Literal(Object value, DataType dataType) {
     if (dataType == DataType.DECIMAL && value instanceof Double) {
-      this.value = BigDecimal.valueOf((Double) value);
+      // This branch is only used in testing for backward compatibility:
+      // We accept decimal literal by Literal(double, DataType.DECIMAL).
+      // Some double values such as 0.0001 will be converted to string "1.0E-4" and finally
+      // generate decimal 0.00010. So here we parse a decimal text to Double then convert it
+      // to BigDecimal as well.
+      // In v2, a decimal literal will be converted back to double in resolving expression
+      // via ExprDoubleValue.
+      // In v3, a decimal literal will be kept in Calcite RexNode and converted back to double
+      // in runtime.
+      DecimalFormat df = new DecimalFormat();
+      df.setMaximumFractionDigits(38);
+      df.setMinimumFractionDigits(1);
+      df.setGroupingUsed(false);
+      String plain = df.format(value);
+      this.value = new BigDecimal(plain);
     } else {
       this.value = value;
     }

core/src/main/java/org/opensearch/sql/executor/OpenSearchTypeSystem.java

@@ -17,9 +17,23 @@
 
 public class OpenSearchTypeSystem extends RelDataTypeSystemImpl {
   public static final RelDataTypeSystem INSTANCE = new OpenSearchTypeSystem();
+  // same with Spark DecimalType.MAX_PRECISION
+  public static int MAX_PRECISION = 38;
+  // same with Spark DecimalType.MAX_SCALE
+  public static int MAX_SCALE = 38;
 
   private OpenSearchTypeSystem() {}
 
+  @Override
+  public int getMaxNumericPrecision() {
+    return MAX_PRECISION;
+  }
+
+  @Override
+  public int getMaxNumericScale() {
+    return MAX_SCALE;
+  }
+
   @Override
   public RelDataType deriveAvgAggType(RelDataTypeFactory typeFactory, RelDataType argumentType) {
     if (SqlTypeName.DECIMAL == argumentType.getSqlTypeName()) {

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4391.yml

@@ -0,0 +1,37 @@
+setup:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : true
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"id": 1}'
+
+---
+teardown:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : false
+
+---
+"big decimal literal":
+  - skip:
+      features:
+        - headers
+        - allowed_warnings
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: source=test | eval floor = FLOOR(9223372036854775807.0000001) | fields floor
+
+  - match: { total: 1 }
+  - match: {"datarows": [[9.223372036854776E18]]}

opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java

@@ -98,6 +98,7 @@ public static ExprValue getExprValueFromSqlType(
           return ExprValueUtils.fromObjectValue(rs.getFloat(i));
 
         case Types.DECIMAL:
+          return ExprValueUtils.fromObjectValue(rs.getBigDecimal(i));
         case Types.NUMERIC:
         case Types.DOUBLE:
           return ExprValueUtils.fromObjectValue(rs.getDouble(i));

ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java

@@ -575,16 +575,7 @@ public UnresolvedExpression visitIntegerLiteral(IntegerLiteralContext ctx) {
 
   @Override
   public UnresolvedExpression visitDecimalLiteral(DecimalLiteralContext ctx) {
-    // For backward compatibility, we accept decimal literal by `Literal(double, DataType.DECIMAL)`
-    // The double value will be converted to decimal by BigDecimal.valueOf((Double) value),
-    // some double values such as 0.0001 will be converted to string "1.0E-4" and finally
-    // generate decimal 0.00010. So here we parse a decimal text to Double then convert it
-    // to BigDecimal as well.
-    // In v2, a decimal literal will be converted back to double in resolving expression
-    // via ExprDoubleValue.
-    // In v3, a decimal literal will be kept in Calcite RexNode and converted back to double
-    // in runtime.
-    return new Literal(BigDecimal.valueOf(Double.parseDouble(ctx.getText())), DataType.DECIMAL);
+    return new Literal(new BigDecimal(ctx.getText()), DataType.DECIMAL);
   }
 
   @Override

ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLMathFunctionTest.java

@@ -170,6 +170,22 @@ public void testFloor() {
     verifyPPLToSparkSQL(root, expectedSparkSql);
   }
 
+  @Test
+  public void testFloorBigDecimal() {
+    RelNode root =
+        getRelNode(
+            "source=EMP | head 1 | eval FLOOR1 = floor(9223372036854775807.0000001) | fields"
+                + " FLOOR1");
+    String expectedLogical =
+        "LogicalProject(FLOOR1=[FLOOR(9223372036854775807.0000001:DECIMAL(26, 7))])\n"
+            + "  LogicalSort(fetch=[1])\n"
+            + "    LogicalTableScan(table=[[scott, EMP]])\n";
+    verifyLogical(root, expectedLogical);
+    String expectedSparkSql =
+        "SELECT FLOOR(9223372036854775807.0000001) `FLOOR1`\nFROM `scott`.`EMP`\nLIMIT 1";
+    verifyPPLToSparkSQL(root, expectedSparkSql);
+  }
+
   @Test
   public void testLn() {
     RelNode root = getRelNode("source=EMP | eval LN = ln(2) | fields LN");