Merge pull request #135 from cloudsufi/feature/PLUGIN-1942

sgarg-CS · web-flow · commit 9812f6466b8c · 2026-02-03T23:18:53.000+05:30
PLUGIN-1942: Update the plugin code to replace `enableNewDataTypes` with `legacyMapping`
diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java
@@ -311,11 +311,11 @@ private SchemaType getSchemaTypeBasedOnUseConnection(Boolean useConnection) {
    * @param accessToken Access Token to use
    * @param valueType Type of value (Actual/Display)
    * @param schemaType Enum to determine which approach to take to fetch schema.
-   * @param enableNewDataTypes Flag to enable new data types
+   * @param legacyMapping Boolean Flag to determine whether to use legacy mapping for data types.
    * @return schema for given ServiceNow table
    */
   public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType,
-                                 SchemaType schemaType, Boolean enableNewDataTypes)
+                                 SchemaType schemaType, Boolean legacyMapping)
       throws ServiceNowAPIException {
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
       this.conf.getRestApiEndpoint(), tableName, true, schemaType)
@@ -327,7 +327,7 @@ public Schema fetchTableSchema(String tableName, String accessToken, SourceValue
     List<ServiceNowColumn> columns = new ArrayList<>();
 
     if (schemaType == SchemaType.METADATA_API_BASED) {
-      return prepareSchemaWithMetadataAPI(restAPIResponse, columns, tableName, valueType, enableNewDataTypes);
+      return prepareSchemaWithMetadataAPI(restAPIResponse, columns, tableName, valueType, legacyMapping);
     } else if (schemaType == SchemaType.SCHEMA_API_BASED) {
       return prepareSchemaWithSchemaAPI(restAPIResponse, columns, tableName);
     } else {
@@ -386,7 +386,7 @@ private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List<
    * @throws RuntimeException if the response does not contain valid column information.
    */
   private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
-    String tableName, SourceValueType valueType, Boolean enableNewDataTypes) throws ServiceNowAPIException {
+    String tableName, SourceValueType valueType, Boolean legacyMapping) throws ServiceNowAPIException {
     MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody());
 
     if (metadataAPISchemaResponse.getResult() == null || metadataAPISchemaResponse.getResult().getColumns() == null ||
@@ -411,7 +411,7 @@ private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, Lis
         columns.add(new ServiceNowColumn(field.getName(), field.getInternalType()));
       }
     }
-    return SchemaBuilder.constructSchema(tableName, columns, enableNewDataTypes);
+    return SchemaBuilder.constructSchema(tableName, columns, legacyMapping);
   }
 
   /**
diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowRecordConverter.java
@@ -84,7 +84,7 @@ public class ServiceNowRecordConverter {
     ));
 
   public static void convertToValue(String fieldName, Schema fieldSchema, Map<String, String> record,
-    StructuredRecord.Builder recordBuilder, Boolean enableNewDataTypes) {
+    StructuredRecord.Builder recordBuilder, Boolean legacyMapping) {
     String fieldValue = record.get(fieldName);
     if (fieldValue == null || fieldValue.isEmpty()) {
       // Set 'null' value as it is
@@ -130,7 +130,7 @@ public static void convertToValue(String fieldName, Schema fieldSchema, Map<Stri
         recordBuilder.set(fieldName, convertToBooleanValue(fieldValue));
         return;
       case ARRAY:
-        recordBuilder.set(fieldName, enableNewDataTypes.equals(Boolean.TRUE) ? convertToList(fieldValue) : fieldValue);
+        recordBuilder.set(fieldName, legacyMapping.equals(Boolean.FALSE) ? convertToList(fieldValue) : fieldValue);
         return;
       default:
         throw new IllegalStateException(
diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowBaseSourceConfig.java
@@ -75,10 +75,11 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig {
   @Description("The number of records to fetch from ServiceNow. Default is 5000.")
   private Integer pageSize;
   
-  @Name(ServiceNowConstants.PROPERTY_ENABLE_NEW_DATA_TYPES)
+  @Name(ServiceNowConstants.PROPERTY_LEGACY_MAPPING)
   @Nullable
-  @Description("Enable support for new data types such as array (glide_list) ")
-  private Boolean enableNewDataTypes;
+  @Description("Specifies whether to use legacy mapping for data types. If true, uses legacy mapping; if false, uses" +
+    " updated mapping. Default is true.")
+  private Boolean legacyMapping;
 
   /**
    * Constructor for ServiceNowSourceConfig object.
@@ -98,15 +99,15 @@ public class ServiceNowBaseSourceConfig extends ServiceNowBaseConfig {
   public ServiceNowBaseSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint,
                                     String user, String password, String tableNameField, String valueType,
                                     @Nullable String startDate, @Nullable String endDate, Integer pageSize,
-                                    Boolean enableNewDataTypes) {
+                                    Boolean legacyMapping) {
     super(clientId, clientSecret, restApiEndpoint, user, password);
     this.referenceName = referenceName;
     this.tableNameField = tableNameField;
     this.valueType = valueType;
     this.startDate = startDate;
     this.endDate = endDate;
     this.pageSize = pageSize;
-    this.enableNewDataTypes = enableNewDataTypes;
+    this.legacyMapping = legacyMapping;
   }
 
   public String getReferenceName() {
@@ -131,8 +132,8 @@ public Integer getPageSize() {
     return pageSize == null ? ServiceNowConstants.PAGE_SIZE : pageSize;
   }
 
-  public Boolean getEnableNewDataTypes() {
-    return enableNewDataTypes == null ? Boolean.FALSE : enableNewDataTypes;
+  public Boolean getLegacyMapping() {
+    return legacyMapping == null ? Boolean.TRUE : legacyMapping;
   }
 
   /**
diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java
@@ -36,7 +36,7 @@ public class ServiceNowMultiRecordReader extends ServiceNowBaseRecordReader {
 
   private final ServiceNowMultiSourceConfig multiSourcePluginConf;
   private ServiceNowTableAPIClientImpl restApi;
-  private Boolean enableNewDataTypes;
+  private Boolean legacyMapping;
 
   ServiceNowMultiRecordReader(ServiceNowMultiSourceConfig multiSourcePluginConf) {
     super();
@@ -83,7 +83,7 @@ public StructuredRecord getCurrentValue() throws IOException {
       for (Schema.Field field : tableFields) {
         String fieldName = field.getName();
         ServiceNowRecordConverter.convertToValue(fieldName, field.getSchema(), row, recordBuilder,
-          multiSourcePluginConf.getEnableNewDataTypes());
+          multiSourcePluginConf.getLegacyMapping());
       }
     } catch (Exception e) {
       throw new IOException("Error decoding row from table " + tableName, e);
diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfig.java
@@ -55,9 +55,9 @@ public class ServiceNowMultiSourceConfig extends ServiceNowBaseSourceConfig {
   public ServiceNowMultiSourceConfig(String referenceName, String clientId, String clientSecret, String restApiEndpoint,
                                      String user, String password, String tableNameField, String valueType,
                                      @Nullable String startDate, @Nullable String endDate, Integer pageSize,
-                                     String tableNames, Boolean enableNewDataTypes) {
+                                     String tableNames, Boolean legacyMapping) {
     super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate,
-          endDate, pageSize, enableNewDataTypes);
+          endDate, pageSize, legacyMapping);
     this.tableNames = tableNames;
   }
 
diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java
@@ -38,7 +38,7 @@ public class ServiceNowRecordReader extends ServiceNowBaseRecordReader {
   private static final Logger LOG = LoggerFactory.getLogger(ServiceNowRecordReader.class);
   private final ServiceNowSourceConfig pluginConf;
   private ServiceNowTableAPIClientImpl restApi;
-  private Boolean enableNewDataTypes;
+  private Boolean legacyMapping;
 
   public ServiceNowRecordReader(ServiceNowSourceConfig pluginConf) {
     super();
@@ -96,7 +96,7 @@ public StructuredRecord getCurrentValue() throws IOException {
       for (Schema.Field field : tableFields) {
         String fieldName = field.getName();
         ServiceNowRecordConverter.convertToValue(fieldName, field.getSchema(), row, recordBuilder,
-          pluginConf.getEnableNewDataTypes());
+          pluginConf.getLegacyMapping());
       }
     } catch (Exception e) {
       LOG.error("Error decoding row from table " + tableName, e);
diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfig.java
@@ -79,9 +79,9 @@ public ServiceNowSourceConfig(String referenceName, String queryMode, @Nullable
                                 @Nullable String tableNameField, @Nullable String tableName, String clientId,
                                 String clientSecret, String restApiEndpoint, String user, String password,
                                 String valueType, @Nullable String startDate, @Nullable String endDate,
-                                Integer pageSize, Boolean enableNewDataTypes) {
+                                Integer pageSize, Boolean legacyMapping) {
     super(referenceName, clientId, clientSecret, restApiEndpoint, user, password, tableNameField, valueType, startDate,
-          endDate, pageSize, enableNewDataTypes);
+          endDate, pageSize, legacyMapping);
     this.referenceName = referenceName;
     this.queryMode = queryMode;
     this.applicationName = applicationName;
diff --git a/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java
@@ -34,27 +34,27 @@ public class SchemaBuilder {
    * @param columns   The list of ServiceNowColumn objects that will be added as Schema.Field
    * @return The instance of Schema object
    */
-  public static Schema constructSchema(String tableName, List<ServiceNowColumn> columns, boolean enableNewDataTypes) {
+  public static Schema constructSchema(String tableName, List<ServiceNowColumn> columns, boolean legacyMapping) {
     SchemaBuilder schemaBuilder = new SchemaBuilder();
-    List<Schema.Field> fields = schemaBuilder.constructSchemaFields(columns, enableNewDataTypes);
+    List<Schema.Field> fields = schemaBuilder.constructSchemaFields(columns, legacyMapping);
 
     return Schema.recordOf(tableName, fields);
   }
 
-  private List<Schema.Field> constructSchemaFields(List<ServiceNowColumn> columns, Boolean enableNewDataTypes) {
+  private List<Schema.Field> constructSchemaFields(List<ServiceNowColumn> columns, Boolean legacyMapping) {
     return columns.stream()
-      .map(o -> transformToField(o, enableNewDataTypes))
+      .map(o -> transformToField(o, legacyMapping))
       .filter(Objects::nonNull)
       .collect(Collectors.toList());
   }
 
-  private Schema.Field transformToField(ServiceNowColumn column, Boolean enableNewDataTypes) {
+  private Schema.Field transformToField(ServiceNowColumn column, Boolean legacyMapping) {
     String name = column.getFieldName();
     if (Strings.isNullOrEmpty(name)) {
       return null;
     }
 
-    Schema schema = createSchema(column, enableNewDataTypes);
+    Schema schema = createSchema(column, legacyMapping);
     if (schema == null) {
       return null;
     }
@@ -64,7 +64,7 @@ private Schema.Field transformToField(ServiceNowColumn column, Boolean enableNew
       : Schema.Field.of(name, Schema.nullableOf(schema));
   }
 
-  private Schema createSchema(ServiceNowColumn column, Boolean enableNewDataTypes) {
+  private Schema createSchema(ServiceNowColumn column, Boolean legacyMapping) {
     switch (column.getTypeName().toLowerCase()) {
       case "decimal":
         return Schema.of(Schema.Type.DOUBLE);
@@ -79,7 +79,7 @@ private Schema createSchema(ServiceNowColumn column, Boolean enableNewDataTypes)
       case "glide_time":
         return Schema.of(Schema.LogicalType.TIME_MICROS);
       case "glide_list":
-        return enableNewDataTypes.equals(Boolean.TRUE) ? Schema.arrayOf(Schema.of(Schema.Type.STRING)) :
+        return legacyMapping.equals(Boolean.FALSE) ? Schema.arrayOf(Schema.of(Schema.Type.STRING)) :
           Schema.of(Schema.Type.STRING);
       case "reference":
       case "currency":
diff --git a/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java b/src/main/java/io/cdap/plugin/servicenow/util/ServiceNowConstants.java
@@ -54,7 +54,7 @@ public interface ServiceNowConstants {
   /**
     * Configuration property name used to enable new data types.
    */
-  String PROPERTY_ENABLE_NEW_DATA_TYPES = "enableNewDataTypes";
+  String PROPERTY_LEGACY_MAPPING = "legacyMapping";
 
   /**
    * Configuration property name used to specify table names.
diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java
@@ -253,7 +253,7 @@ public void testFetchTableSchema_WithGlide_ListType_ParseAsArray() throws Except
     RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null);
     Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any());
     Schema schema = implSpy.fetchTableSchema("u_custom_13", "dummy-access-token",
-      SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.METADATA_API_BASED, true);
+      SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.METADATA_API_BASED, false);
     Assert.assertNotNull(schema);
     Assert.assertEquals("record", schema.getDisplayName());
     Assert.assertEquals(2, schema.getFields().size());
@@ -291,7 +291,7 @@ public void testFetchTableSchema_WithGlide_ListType_ParseAsString() throws Excep
     RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null);
     Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any());
     Schema schema = implSpy.fetchTableSchema("u_custom_13", "dummy-access-token",
-     SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.METADATA_API_BASED, false);
+     SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.METADATA_API_BASED, true);
     Assert.assertNotNull(schema);
     Assert.assertEquals("record", schema.getDisplayName());
     Assert.assertEquals(2, schema.getFields().size());
diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java
@@ -170,7 +170,7 @@ public void testFetchDataOnInvalidTable()
             .setStartDate("2021-01-01")
             .setEndDate("2022-02-18")
             .setPageSize(10)
-            .setEnableNewDataTypes(false)
+            .setLegacyMapping(true)
             .setTableNameField("tablename")
             .buildMultiSource();
 
diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java
@@ -77,7 +77,7 @@ public void initializeTests() {
       .setStartDate("2021-12-30")
       .setEndDate("2021-12-31")
       .setPageSize(10)
-      .setEnableNewDataTypes(false)
+      .setLegacyMapping(true)
       .setTableNameField("tablename")
       .build();
 
@@ -241,7 +241,7 @@ public void testConvertToValue_WithArrayFieldType_ParseAsString() {
     Map<String, String> inputMap = new HashMap<>();
     inputMap.put("ArrayField", "\"service_sys_id_1_1\",\"service_sys_id_2_1\"");
     StructuredRecord.Builder recordBuilder = StructuredRecord.builder(recordSchema);
-    ServiceNowRecordConverter.convertToValue("ArrayField", fieldSchema, inputMap, recordBuilder, false);
+    ServiceNowRecordConverter.convertToValue("ArrayField", fieldSchema, inputMap, recordBuilder, true);
     StructuredRecord record = recordBuilder.build();
     Assert.assertEquals(inputMap.get("ArrayField"), record.get("ArrayField"));
   }
@@ -256,7 +256,7 @@ public void testConvertToValue_WithArrayFieldType_ParseAsArray() {
     Map<String, String> inputMap = new HashMap<>();
     inputMap.put("ArrayField", "\"service_sys_id_1_1\",\"service_sys_id_2_1\"");
     StructuredRecord.Builder recordBuilder = StructuredRecord.builder(recordSchema);
-    ServiceNowRecordConverter.convertToValue("ArrayField", fieldSchema, inputMap, recordBuilder, true);
+    ServiceNowRecordConverter.convertToValue("ArrayField", fieldSchema, inputMap, recordBuilder, false);
     StructuredRecord record = recordBuilder.build();
     Assert.assertEquals(Arrays.asList(inputMap.get("ArrayField").split(",")), record.get("ArrayField"));
   }
diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigHelper.java
@@ -59,7 +59,7 @@ The name of the ServiceNow table(s) from which data to be fetched.
     private String startDate = "";
     private String endDate = "";
     private Integer pageSize = 5000;
-    private Boolean enableNewDataTypes = false;
+    private Boolean legacyMapping = true;
 
     public ConfigBuilder setReferenceName(String referenceName) {
       this.referenceName = referenceName;
@@ -136,20 +136,20 @@ public ConfigBuilder setPageSize(Integer pageSize) {
       return this;
     }
 
-    public ConfigBuilder setEnableNewDataTypes(Boolean enableNewDataTypes) {
-      this.enableNewDataTypes = enableNewDataTypes;
+    public ConfigBuilder setLegacyMapping(Boolean legacyMapping) {
+      this.legacyMapping = legacyMapping;
       return this;
     }
 
     public ServiceNowSourceConfig build() {
       return new ServiceNowSourceConfig(referenceName, queryMode, applicationName, tableNameField, tableName,
         clientId, clientSecret, restApiEndpoint, user, password, valueType, startDate, endDate, pageSize,
-          enableNewDataTypes);
+          legacyMapping);
     }
 
     public ServiceNowMultiSourceConfig buildMultiSource() {
       return new ServiceNowMultiSourceConfig(referenceName, clientId, clientSecret, restApiEndpoint, user, password,
-        tableNameField, valueType, startDate, endDate, pageSize, tableNames, enableNewDataTypes);
+        tableNameField, valueType, startDate, endDate, pageSize, tableNames, legacyMapping);
     }
 
 
diff --git a/widgets/ServiceNow-batchsource.json b/widgets/ServiceNow-batchsource.json
@@ -189,8 +189,8 @@
         },
         {
           "widget-type": "hidden",
-          "label": "Enable New Data Types",
-          "name": "enableNewDataTypes",
+          "label": "Legacy Mapping",
+          "name": "legacyMapping",
           "widget-attributes": {
             "on": {
               "value": "true",
@@ -200,7 +200,7 @@
               "value": "false",
               "label": "NO"
             },
-            "default": "false"
+            "default": "true"
           }
         }
       ]
diff --git a/widgets/ServiceNowMultiSource-batchsource.json b/widgets/ServiceNowMultiSource-batchsource.json
@@ -151,8 +151,8 @@
         },
         {
           "widget-type": "hidden",
-          "label": "Enable New Data Types",
-          "name": "enableNewDataTypes",
+          "label": "Legacy Mapping",
+          "name": "legacyMapping",
           "widget-attributes": {
             "on": {
               "value": "true",
@@ -162,7 +162,7 @@
               "value": "false",
               "label": "NO"
             },
-            "default": "false"
+            "default": "true"
           }
         }
       ]
