Merge pull request data-integrations#110 from data-integrations/backward-compatability-schema-fix

 [PLUGIN-1902] Changes done for schema backward compatibility along with metadata API addition for DTS

Author: vikasrathee-cs

src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java

@@ -62,7 +62,7 @@ public static void getRecordFromServiceNowTable(String query, String tableName)
         System.getenv("SERVICE_NOW_PASSWORD"),
         "", "", "", null);
 
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query);
   }
 

src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java

@@ -65,7 +65,7 @@ public static void initializeServiceNowSourceConfig() {
   @Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM")
   public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Product Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'name':'" + uniqueId + "','price':'2500'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -76,7 +76,7 @@ public static void createRecordInProductCatalogItemTable() throws IOException, S
   public static void createRecordInReceivingSlipLineTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Receiving Slip Line table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'number':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -88,7 +88,7 @@ public static void createRecordInReceivingSlipLineTable()
   public static void createRecordInDateTimeTable()
     throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Date time table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), false);
     String recordDetails = "{'u_date': '2025-05-28 15:07'," +
       "  'u_datetime1': '28.05.2025 03:07:56 PM'," +
       "  'u_datetime2': '28.05.2025 03.07.56 PM'," +
@@ -110,7 +110,7 @@ public static void createRecordInDateTimeTable()
 
   @After(order = 2, value = "@SN_DATE_TIME_TABLE")
   public static void deleteRecord() throws ServiceNowAPIException, IOException {
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), false);
     tableAPIClient.deleteRecordFromServiceNowTable(TablesInTableMode.DATE_TIME_TABLE.value, TestSetupHooks.systemId);
     BeforeActions.scenario.write("Record in Date time table: " + systemId + " deleted successfully");
   }
@@ -119,7 +119,7 @@ public static void deleteRecord() throws ServiceNowAPIException, IOException {
   public static void updateRecordInAgentAssistRecommendationTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'active':'false','name':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -130,7 +130,7 @@ public static void updateRecordInAgentAssistRecommendationTable()
   public static void updateRecordInAgentVendorCatalogItem()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Vendor Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'out_of_stock':'false','product_id':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -140,7 +140,7 @@ public static void updateRecordInAgentVendorCatalogItem()
   @Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING")
   public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Service Offering table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," +
         " 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";

src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java

@@ -28,13 +28,10 @@
 import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig;
 import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
 import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
+import io.cdap.plugin.servicenow.util.SchemaType;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.SourceValueType;
-import org.apache.http.HttpStatus;
-import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
-import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
 
-import java.io.IOException;
 import javax.annotation.Nullable;
 
 /**
@@ -58,6 +55,11 @@ public ServiceNowBaseConfig(String clientId, String clientSecret, String restApi
     this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password);
   }
 
+  @Nullable
+  public Boolean getUseConnection() {
+    return useConnection;
+  }
+
   @Nullable
   public ServiceNowConnectorConfig getConnection() {
     return connection;
@@ -83,7 +85,7 @@ public void validateCredentials(FailureCollector collector) {
   @VisibleForTesting
   public void validateServiceNowConnection(FailureCollector collector) {
     try {
-      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection);
+      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection);
       restApi.getAccessToken();
     } catch (Exception e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",
@@ -123,13 +125,13 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
                             String tableField) {
     // Call API to fetch first record from the table
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      connection.getRestApiEndpoint(), tableName, false)
+      connection.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED)
       .setExcludeReferenceLink(true)
       .setDisplayValue(valueType)
       .setLimit(1);
 
     RestAPIResponse apiResponse = null;
-    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection);
+    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection, useConnection);
     try {
       String accessToken = serviceNowTableAPIClient.getAccessToken();
       requestBuilder.setAuthHeader(accessToken);