Changes done for schema backward compatibility.

Author: prince-cs

src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java

@@ -61,7 +61,7 @@ public static void getRecordFromServiceNowTable(String query, String tableName)
         System.getenv("SERVICE_NOW_PASSWORD"),
         "", "", "", null);
 
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query);
   }
 

src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java

@@ -65,7 +65,7 @@ public static void initializeServiceNowSourceConfig() {
   @Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM")
   public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Product Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'name':'" + uniqueId + "','price':'2500'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -76,7 +76,7 @@ public static void createRecordInProductCatalogItemTable() throws IOException, S
   public static void createRecordInReceivingSlipLineTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Receiving Slip Line table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'number':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -88,7 +88,7 @@ public static void createRecordInReceivingSlipLineTable()
   public static void updateRecordInAgentAssistRecommendationTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'active':'false','name':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -99,7 +99,7 @@ public static void updateRecordInAgentAssistRecommendationTable()
   public static void updateRecordInAgentVendorCatalogItem()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Vendor Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'out_of_stock':'false','product_id':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -109,7 +109,7 @@ public static void updateRecordInAgentVendorCatalogItem()
   @Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING")
   public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Service Offering table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," +
         " 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";

src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java

@@ -28,13 +28,10 @@
 import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig;
 import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
 import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
+import io.cdap.plugin.servicenow.util.SchemaType;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.SourceValueType;
-import org.apache.http.HttpStatus;
-import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
-import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
 
-import java.io.IOException;
 import javax.annotation.Nullable;
 
 /**
@@ -58,6 +55,11 @@ public ServiceNowBaseConfig(String clientId, String clientSecret, String restApi
     this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password);
   }
 
+  @Nullable
+  public Boolean getUseConnection() {
+    return useConnection;
+  }
+
   @Nullable
   public ServiceNowConnectorConfig getConnection() {
     return connection;
@@ -83,7 +85,7 @@ public void validateCredentials(FailureCollector collector) {
   @VisibleForTesting
   public void validateServiceNowConnection(FailureCollector collector) {
     try {
-      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection);
+      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection);
       restApi.getAccessToken();
     } catch (Exception e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",
@@ -123,13 +125,13 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
                             String tableField) {
     // Call API to fetch first record from the table
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      connection.getRestApiEndpoint(), tableName, false)
+      connection.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED)
       .setExcludeReferenceLink(true)
       .setDisplayValue(valueType)
       .setLimit(1);
 
     RestAPIResponse apiResponse = null;
-    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection);
+    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection, useConnection);
     try {
       String accessToken = serviceNowTableAPIClient.getAccessToken();
       requestBuilder.setAuthHeader(accessToken);

src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java

@@ -34,9 +34,12 @@
 import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
 import io.cdap.plugin.servicenow.sink.model.APIResponse;
 import io.cdap.plugin.servicenow.sink.model.CreateRecordAPIResponse;
+import io.cdap.plugin.servicenow.sink.model.MetadataAPISchemaResponse;
+import io.cdap.plugin.servicenow.sink.model.SchemaField;
 import io.cdap.plugin.servicenow.sink.model.SchemaResponse;
 import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField;
 import io.cdap.plugin.servicenow.util.SchemaBuilder;
+import io.cdap.plugin.servicenow.util.SchemaType;
 import io.cdap.plugin.servicenow.util.ServiceNowColumn;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.SourceValueType;
@@ -72,10 +75,12 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient {
   private static final String GLIDE_DATE_TIME_DATATYPE = "glide_date_time";
   private static final Gson GSON = new Gson();
   private final ServiceNowConnectorConfig conf;
+  public final Boolean useConnection;
   public static JsonArray serviceNowJsonResultArray;
 
-  public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf) {
+  public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf, Boolean useConnection) {
     this.conf = conf;
+    this.useConnection = useConnection;
   }
 
   public String getAccessToken() throws ServiceNowAPIException {
@@ -129,7 +134,7 @@ public List<Map<String, String>> fetchTableRecords(
       int limit)
       throws ServiceNowAPIException {
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, false)
+      this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
       .setExcludeReferenceLink(true)
       .setDisplayValue(valueType)
       .setLimit(limit);
@@ -178,7 +183,6 @@ private int getRecordCountFromHeader(RestAPIResponse apiResponse) {
 
   public List<Map<String, String>> parseResponseToResultListOfMap(String responseBody) {
 
-
     JsonObject jo = GSON.fromJson(responseBody, JsonObject.class);
     JsonArray ja = jo.getAsJsonArray(ServiceNowConstants.RESULT);
 
@@ -257,7 +261,7 @@ public List<Map<String, String>> fetchTableRecordsRetryableMode(String tableName
   public Schema fetchTableSchema(String tableName, FailureCollector collector) {
     Schema schema = null;
     try {
-      schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE);
+      schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE, useConnection);
     } catch (Exception e) {
       LOG.error("Failed to fetch schema on table {}", tableName, e);
       collector.addFailure(String.format("Connection failed. Unable to fetch schema for table: %s. Cause: %s",
@@ -267,20 +271,37 @@ public Schema fetchTableSchema(String tableName, FailureCollector collector) {
   }
 
   @VisibleForTesting
-  public SchemaResponse parseSchemaResponse(String responseBody) {
+  public MetadataAPISchemaResponse parseSchemaResponse(String responseBody) {
+    return GSON.fromJson(responseBody, MetadataAPISchemaResponse.class);
+  }
+
+  public SchemaResponse parseSchemaResponseWithoutMetadata(String responseBody) {
     return GSON.fromJson(responseBody, SchemaResponse.class);
   }
 
   /**
    * Fetches the table schema from ServiceNow
    *
    * @param tableName ServiceNow table name for which schema is getting fetched
+   * @param valueType Whether to fetch schema for actual value or display value
+   * @param useConnection This flag is added to backward compatibility with 1.1 release,
+   *                      in case of null for useConnection, schema will be string based.
    * @return schema for given ServiceNow table
    * @throws ServiceNowAPIException
    */
-  public Schema fetchTableSchema(String tableName, SourceValueType valueType)
+  public Schema fetchTableSchema(String tableName, SourceValueType valueType, Boolean useConnection)
       throws ServiceNowAPIException {
-      return fetchTableSchema(tableName, getAccessToken(), valueType);
+    SchemaType schemaType = getSchemaTypeBasedOnUseConnection(useConnection);
+    return fetchTableSchema(tableName, getAccessToken(), valueType, schemaType);
+  }
+
+  private SchemaType getSchemaTypeBasedOnUseConnection(Boolean useConnection) {
+    // use connection was added in release/1.2, so it will be null for users who are upgrading from release/1.1
+    // This is added to support backward compatibility for 1.1 users.
+    if (useConnection == null) {
+      return SchemaType.STRING_BASED;
+    }
+    return SchemaType.SCHEMA_API_BASED;
   }
 
   /**
@@ -289,25 +310,88 @@ public Schema fetchTableSchema(String tableName, SourceValueType valueType)
    * @param tableName ServiceNow table name for which schema is getting fetched
    * @param accessToken Access Token to use
    * @param valueType Type of value (Actual/Display)
+   * @param schemaType Enum to determine which approach to take to fetch schema.
    * @return schema for given ServiceNow table
    */
-  public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType)
+  public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType,
+                                 SchemaType schemaType)
       throws ServiceNowAPIException {
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, true)
+      this.conf.getRestApiEndpoint(), tableName, true, schemaType)
       .setExcludeReferenceLink(true);
 
     RestAPIResponse restAPIResponse;
     requestBuilder.setAuthHeader(accessToken);
     restAPIResponse = executeGetWithRetries(requestBuilder.build());
-    SchemaResponse schemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody());
     List<ServiceNowColumn> columns = new ArrayList<>();
 
-    if (schemaResponse.getResult() == null && schemaResponse.getResult().getColumns().isEmpty()) {
+    if (schemaType == SchemaType.METADATA_API_BASED) {
+      return prepareSchemaWithMetadataAPI(restAPIResponse, columns, tableName, valueType);
+    } else if (schemaType == SchemaType.SCHEMA_API_BASED) {
+      return prepareSchemaWithSchemaAPI(restAPIResponse, columns, tableName);
+    } else {
+      return prepareStringBasedSchema(restAPIResponse, columns, tableName);
+    }
+  }
+
+  /**
+   * Processes a schema response obtained from the ServiceNow Table API (without using Metadata API)
+   * and constructs a {@link Schema} object based on the parsed column definitions.
+   *
+   * <p>This method parses the raw JSON response body into a list of schema fields,
+   * extracts the internal column types, and appends them to the provided column list.
+   * The final schema is constructed using the {@link SchemaBuilder} utility.</p>
+   *
+   * @param restAPIResponse The raw API response received from the ServiceNow Table API.
+   * @param columns         A list to which parsed {@link ServiceNowColumn} objects will be added.
+   * @param tableName       The name of the table for which the schema is being constructed.
+   *
+   * @return A {@link Schema} object representing the table structure as interpreted from the Schema API.
+   *
+   * @throws RuntimeException if the schema response is null or contains no result.
+   */
+  private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
+                                            String tableName) {
+    SchemaResponse schemaResponse =
+      parseSchemaResponseWithoutMetadata(restAPIResponse.getResponseBody());
+
+    if (schemaResponse.getResult() == null && schemaResponse.getResult().isEmpty()) {
       throw new RuntimeException("Error - Schema Response does not contain any result");
     }
 
-    for (ServiceNowSchemaField field : schemaResponse.getResult().getColumns().values()) {
+    for (SchemaField field : schemaResponse.getResult()) {
+      columns.add(new ServiceNowColumn(field.getName(), field.getInternalType()));
+    }
+    return SchemaBuilder.constructSchema(tableName, columns);
+  }
+
+  /**
+   * Parses a ServiceNow schema response obtained via the Metadata API and constructs a
+   * {@link Schema} object using the extracted field information and value type preferences.
+   *
+   * <p>This method reads the JSON response, extracts the column metadata including field names
+   * and data types, and adds each as a {@link ServiceNowColumn} to the provided list. The choice
+   * between internal values and display values is based on the {@code valueType} parameter.</p>
+   *
+   * @param restAPIResponse The response returned from the ServiceNow Metadata API.
+   * @param columns         A list to which parsed {@link ServiceNowColumn} definitions will be added.
+   * @param tableName       The name of the ServiceNow table for which the schema is being generated.
+   * @param valueType       The value type preference (e.g., {@code SHOW_DISPLAY_VALUE} or {@code USE_INTERNAL_VALUE}).
+   *                        Determines whether to use display types or internal types in the resulting schema.
+   *
+   * @return A {@link Schema} object representing the table structure as interpreted from the Metadata API.
+   *
+   * @throws RuntimeException if the response does not contain valid column information.
+   */
+  private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
+                                              String tableName, SourceValueType valueType) {
+    MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody());
+
+    if (metadataAPISchemaResponse.getResult() == null && metadataAPISchemaResponse.getResult().getColumns().isEmpty()) {
+      throw new RuntimeException("Error - Schema Response does not contain any result");
+    }
+
+    for (ServiceNowSchemaField field : metadataAPISchemaResponse.getResult().getColumns().values()) {
       if (valueType.equals(SourceValueType.SHOW_DISPLAY_VALUE) &&
         !Objects.equals(field.getType(), field.getInternalType())) {
         columns.add(new ServiceNowColumn(field.getName(), field.getType()));
@@ -343,7 +427,7 @@ public int getTableRecordCount(String tableName)
    */
   public int getTableRecordCount(String tableName, String accessToken) throws ServiceNowAPIException {
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, false)
+      this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
       .setExcludeReferenceLink(true)
       .setDisplayValue(SourceValueType.SHOW_DISPLAY_VALUE)
       .setLimit(1);
@@ -363,7 +447,7 @@ public int getTableRecordCount(String tableName, String accessToken) throws Serv
    */
   public String createRecord(String tableName, HttpEntity entity) throws IOException, ServiceNowAPIException {
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, false);
+      this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection));
     String systemID;
     RestAPIResponse apiResponse = null;
     try {
@@ -398,7 +482,7 @@ public Map<String, String> getRecordFromServiceNowTable(String tableName, String
       throws ServiceNowAPIException {
 
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      this.conf.getRestApiEndpoint(), tableName, false)
+      this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
       .setQuery(query);
 
     RestAPIResponse restAPIResponse;
@@ -409,4 +493,29 @@ public Map<String, String> getRecordFromServiceNowTable(String tableName, String
     APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class);
     return apiResponse.getResult().get(0);
   }
+
+  /**
+   * Processes the response obtained from the ServiceNow Table API
+   * and constructs a {@link Schema} object based on the first record.
+   *
+   * @param restAPIResponse The raw API response received from the ServiceNow Table API.
+   * @param columns         A list to which parsed {@link ServiceNowColumn} objects will be added.
+   * @param tableName       The name of the table for which the schema is being constructed.
+   *
+   * @return A {@link Schema} object representing the table structure as interpreted from the Schema API.
+   *
+   * @throws RuntimeException if the schema response is null or contains no result.
+   */
+  private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
+                                          String tableName) {
+    List<Map<String, String>> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody());
+    if (result != null && !result.isEmpty()) {
+      Map<String, String> firstRecord = result.get(0);
+      for (String key : firstRecord.keySet()) {
+        columns.add(new ServiceNowColumn(key, "string"));
+      }
+    }
+    return SchemaBuilder.constructSchema(tableName, columns);
+  }
+
 }