PLUGIN-1893 : Adding flag via field for backward compatibility of precisionless number to decimal

Author: sahusanket

oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java

@@ -112,7 +112,8 @@ protected DBConnectorPath getDBConnectorPath(String path) {
 
   @Override
   protected SchemaReader getSchemaReader(String sessionID) {
-    return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestampConn());
+    return new OracleSourceSchemaReader(sessionID, config.getTreatAsOldTimestampConn(),
+                                        config.getTreatPrecisionlessNumAsDeciConn());
   }
 
   @Override

oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnectorConfig.java

@@ -41,12 +41,14 @@ public OracleConnectorConfig(String host, int port, String user, String password
 
   public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
                                String connectionArguments, String connectionType, String database) {
-    this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null);
+    this(host, port, user, password, jdbcPluginName, connectionArguments, connectionType, database, null, null, null,
+         null);
   }
 
   public OracleConnectorConfig(String host, int port, String user, String password, String jdbcPluginName,
                                String connectionArguments, String connectionType, String database,
-                               String role, Boolean useSSL, String treatAsOldTimestampConn) {
+                               String role, Boolean useSSL, String treatAsOldTimestampConn,
+                               String treatPrecisionlessNumAsDeciConn) {
 
     this.host = host;
     this.port = port;
@@ -59,6 +61,7 @@ public OracleConnectorConfig(String host, int port, String user, String password
     this.role = role;
     this.useSSL = useSSL;
     this.treatAsOldTimestampConn = treatAsOldTimestampConn;
+    this.treatPrecisionlessNumAsDeciConn = treatPrecisionlessNumAsDeciConn;
   }
 
   @Override
@@ -90,6 +93,11 @@ public String getConnectionString() {
   @Nullable
   public String treatAsOldTimestampConn;
 
+  @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI_CONN)
+  @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.")
+  @Nullable
+  public String treatPrecisionlessNumAsDeciConn;
+
   @Override
   protected int getDefaultPort() {
     return 1521;
@@ -116,6 +124,10 @@ public Boolean getTreatAsOldTimestampConn() {
     return Boolean.parseBoolean(treatAsOldTimestampConn);
   }
 
+  public Boolean getTreatPrecisionlessNumAsDeciConn() {
+    return Boolean.parseBoolean(treatPrecisionlessNumAsDeciConn);
+  }
+
   @Override
   public Properties getConnectionArgumentsProperties() {
     Properties prop = super.getConnectionArgumentsProperties();

oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConstants.java

@@ -45,6 +45,8 @@ private OracleConstants() {
   public static final String USE_SSL = "useSSL";
   public static final String TREAT_AS_OLD_TIMESTAMP = "treatAsOldTimestamp";
   public static final String TREAT_AS_OLD_TIMESTAMP_CONN = "treatAsOldTimestampConn";
+  public static final String TREAT_PRECISIONLESSNUM_AS_DECI = "treatPrecisionlessNumAsDeci";
+  public static final String TREAT_PRECISIONLESSNUM_AS_DECI_CONN = "treatPrecisionlessNumAsDeciConn";
 
   /**
    * Constructs the Oracle connection string based on the provided connection type, host, port, and database.

oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java

@@ -74,7 +74,13 @@ protected SchemaReader getSchemaReader() {
       treatAsOldTimestamp = Boolean.parseBoolean(sourceConfig.getTreatAsOldTimestamp());
     }
 
-    return new OracleSourceSchemaReader(null, treatAsOldTimestamp);
+    boolean treatPrecisionlessNumAsDeci = oracleSourceConfig.getConnection().getTreatPrecisionlessNumAsDeciConn();
+    if (sourceConfig.getTreatPrecisionlessNumAsDeci().equalsIgnoreCase("true") ||
+      sourceConfig.getTreatPrecisionlessNumAsDeci().equalsIgnoreCase("false")) {
+      treatPrecisionlessNumAsDeci = Boolean.parseBoolean(sourceConfig.getTreatPrecisionlessNumAsDeci());
+    }
+
+    return new OracleSourceSchemaReader(null, treatAsOldTimestamp, treatPrecisionlessNumAsDeci);
   }
 
   @Override
@@ -139,14 +145,21 @@ public static class OracleSourceConfig extends AbstractDBSpecificSourceConfig {
     @Nullable
     private String treatAsOldTimestamp;
 
+    @Name(OracleConstants.TREAT_PRECISIONLESSNUM_AS_DECI)
+    @Description("A hidden field to handle precision less number as CDAP's decimal per old behavior.")
+    @Nullable
+    public String treatPrecisionlessNumAsDeci;
+
     public OracleSourceConfig(String host, int port, String user, String password, String jdbcPluginName,
                               String connectionArguments, String connectionType, String database, String role,
                               int defaultBatchValue, int defaultRowPrefetch,
                               String importQuery, Integer numSplits, int fetchSize,
                               String boundingQuery, String splitBy, Boolean useSSL, String treatAsOldTimestampConn,
-                              String treatAsOldTimestamp) {
+                              String treatAsOldTimestamp, String treatPrecisionlessNumAsDeciConn,
+                              String treatPrecisionlessNumAsDeci) {
       this.connection = new OracleConnectorConfig(host, port, user, password, jdbcPluginName, connectionArguments,
-                                                  connectionType, database, role, useSSL, treatAsOldTimestampConn);
+                                                  connectionType, database, role, useSSL, treatAsOldTimestampConn,
+                                                  treatPrecisionlessNumAsDeciConn);
       this.defaultBatchValue = defaultBatchValue;
       this.defaultRowPrefetch = defaultRowPrefetch;
       this.fetchSize = fetchSize;
@@ -155,6 +168,7 @@ public OracleSourceConfig(String host, int port, String user, String password, S
       this.boundingQuery = boundingQuery;
       this.splitBy = splitBy;
       this.treatAsOldTimestamp = treatAsOldTimestamp;
+      this.treatPrecisionlessNumAsDeci = treatPrecisionlessNumAsDeci;
     }
 
     @Override
@@ -185,6 +199,10 @@ public String getTreatAsOldTimestamp() {
       return treatAsOldTimestamp == null ? "" : treatAsOldTimestamp;
     }
 
+    public String getTreatPrecisionlessNumAsDeci() {
+      return treatPrecisionlessNumAsDeci == null ? "" : treatPrecisionlessNumAsDeci;
+    }
+
     @Override
     public void validate(FailureCollector collector) {
       ConfigUtil.validateConnection(this, useConnection, connection, collector);

oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSourceSchemaReader.java

@@ -66,13 +66,16 @@ public class OracleSourceSchemaReader extends CommonSchemaReader {
 
   private final String sessionID;
   private final Boolean isTimestampOldBehavior;
+  private final Boolean isPrecisionlessNumAsDecimal;
 
   public OracleSourceSchemaReader() {
-    this(null, false);
+    this(null, false, false);
   }
-  public OracleSourceSchemaReader(String sessionID, boolean isTimestampOldBehavior) {
+  public OracleSourceSchemaReader(String sessionID, boolean isTimestampOldBehavior,
+                                  boolean isPrecisionlessNumAsDecimal) {
     this.sessionID = sessionID;
     this.isTimestampOldBehavior = isTimestampOldBehavior;
+    this.isPrecisionlessNumAsDecimal = isPrecisionlessNumAsDecimal;
   }
 
   @Override
@@ -109,12 +112,24 @@ public Schema getSchema(ResultSetMetaData metadata, int index) throws SQLExcepti
           // For a Number type without specified precision and scale, precision will be 0 and scale will be -127
           if (precision == 0) {
             // reference : https://docs.oracle.com/cd/B28359_01/server.111/b28318/datatype.htm#CNCPT1832
-            LOG.warn(String.format("Field '%s' is a %s type without precision and scale, "
-                    + "converting into STRING type to avoid any precision loss.",
-                metadata.getColumnName(index),
-                metadata.getColumnTypeName(index),
-                metadata.getColumnName(index)));
-            return Schema.of(Schema.Type.STRING);
+            if (isPrecisionlessNumAsDecimal) {
+              precision = 38;
+              scale = 0;
+              LOG.warn(String.format("%s type with undefined precision and scale is detected, "
+                                       + "there may be a precision loss while running the pipeline. "
+                                       + "Please define an output precision and scale for field '%s' to avoid "
+                                       + "precision loss.",
+                                     metadata.getColumnTypeName(index),
+                                     metadata.getColumnName(index)));
+              return Schema.decimalOf(precision, scale);
+            } else {
+              LOG.warn(String.format("Field '%s' is a %s type without precision and scale, "
+                                       + "converting into STRING type to avoid any precision loss.",
+                                     metadata.getColumnName(index),
+                                     metadata.getColumnTypeName(index),
+                                     metadata.getColumnName(index)));
+              return Schema.of(Schema.Type.STRING);
+            }
           }
           return Schema.decimalOf(precision, scale);
         }

oracle-plugin/widgets/Oracle-batchsource.json

@@ -255,6 +255,15 @@
           "widget-attributes": {
             "default": "default-no"
           }
+        },
+        {
+          "widget-type": "hidden",
+          "label": "Treat precision less number as Decimal(old behavior)",
+          "name": "treatPrecisionlessNumAsDeci",
+          "description": "If edited, then this will take precedence over other properties and the edited value shall be true or false only",
+          "widget-attributes": {
+            "default": "default-no"
+          }
         }
       ]
     }

oracle-plugin/widgets/Oracle-connector.json

@@ -137,6 +137,14 @@
           "widget-attributes": {
             "default": "false"
           }
+        },
+        {
+          "widget-type": "hidden",
+          "label": "Treat precision less number as Decimal(old behavior)",
+          "name": "treatPrecisionlessNumAsDeciConn",
+          "widget-attributes": {
+            "default": "false"
+          }
         }
       ]
     },