Merge pull request #120 from cloudsufi/bump-up-and-cherrypick

[cherrypick] [PLUGIN-1902] Bump up to 1.2.7-SNAPSHOT and Changes done for schema backward compatibility along with metadata API addition for DTS

Author: vikasrathee-cs

pom.xml

@@ -20,7 +20,7 @@
 
   <groupId>io.cdap.plugin</groupId>
   <artifactId>servicenow-plugins</artifactId>
-  <version>1.2.6</version>
+  <version>1.2.7-SNAPSHOT</version>
   <name>ServiceNow Plugins</name>
   <packaging>jar</packaging>
   <description>Plugins for ServiceNow</description>

src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java

@@ -61,7 +61,7 @@ public static void getRecordFromServiceNowTable(String query, String tableName)
         System.getenv("SERVICE_NOW_PASSWORD"),
         "", "", "", null);
 
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query);
   }
 

src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java

@@ -65,7 +65,7 @@ public static void initializeServiceNowSourceConfig() {
   @Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM")
   public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Product Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'name':'" + uniqueId + "','price':'2500'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -76,18 +76,50 @@ public static void createRecordInProductCatalogItemTable() throws IOException, S
   public static void createRecordInReceivingSlipLineTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Receiving Slip Line table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'number':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
     systemId = tableAPIClient.createRecord(TablesInTableMode.RECEIVING_SLIP_LINE.value, entity);
+    BeforeActions.scenario.write("New Record in Receiving Slip Line table: " + systemId + " created successfully");
+  }
+
+  @Before(order = 2, value = "@SN_DATE_TIME_TABLE")
+  public static void createRecordInDateTimeTable()
+    throws IOException, ServiceNowAPIException {
+    BeforeActions.scenario.write("Create new record in Date time table");
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), false);
+    String recordDetails = "{'u_date': '2025-05-28 15:07'," +
+      "  'u_datetime1': '28.05.2025 03:07:56 PM'," +
+      "  'u_datetime2': '28.05.2025 03.07.56 PM'," +
+      "  'u_datetime3': '2025-05-28 15:07:56'," +
+      "  'u_datetime4': '28.05.2025 15:07:56'," +
+      "  'u_datetime5': '28.05.2025 15.07.56'," +
+      "  'u_datetime6': '28-05-2025 15:07:56'," +
+      "  'u_datetime7': '28-05-2025 15.07.56'," +
+      "  'u_datetime8': '05/28/2025 15:07:56'," +
+      "  'u_datetime9': '28/05/2025 15:07:56'," +
+      "  'u_datetime10': '05-28-2025 15:07:56'," +
+      "  'u_datetime11': '28-05-25 15.07.56'," +
+      "  'u_datetime13': '05-28-2025 15:07'," +
+      "  'u_datetime': '28-05-2025 15.07'}";
+    StringEntity entity = new StringEntity(recordDetails);
+    systemId = tableAPIClient.createRecordInDisplayMode(TablesInTableMode.DATE_TIME_TABLE.value, entity);
+    BeforeActions.scenario.write("New Record in Date time table: " + systemId + " created successfully");
+  }
+
+  @After(order = 2, value = "@SN_DATE_TIME_TABLE")
+  public static void deleteRecord() throws ServiceNowAPIException, IOException {
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), false);
+    tableAPIClient.deleteRecordFromServiceNowTable(TablesInTableMode.DATE_TIME_TABLE.value, TestSetupHooks.systemId);
+    BeforeActions.scenario.write("Record in Date time table: " + systemId + " deleted successfully");
   }
 
   @Before(order = 2, value = "@SN_UPDATE_AGENT_ASSIST_RECOMMENDATION")
   public static void updateRecordInAgentAssistRecommendationTable()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'active':'false','name':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -98,7 +130,7 @@ public static void updateRecordInAgentAssistRecommendationTable()
   public static void updateRecordInAgentVendorCatalogItem()
       throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Vendor Catalog Item table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'out_of_stock':'false','product_id':'" + uniqueId + "'}";
     StringEntity entity = new StringEntity(recordDetails);
@@ -108,7 +140,7 @@ public static void updateRecordInAgentVendorCatalogItem()
   @Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING")
   public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException {
     BeforeActions.scenario.write("Create new record in Service Offering table");
-    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
+    ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
     String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10);
     String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," +
         " 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";
@@ -234,6 +266,7 @@ public static void updateTempSourceBQTableForServiceOffering() throws IOExceptio
   public static void setTempTargetBQTable() {
     bqTargetTable = "TestSN_table" + RandomStringUtils.randomAlphanumeric(10);
     BeforeActions.scenario.write("BigQuery Target table name: " + bqTargetTable);
+    PluginPropertyUtils.addPluginProp("bqTargetTable", bqTargetTable);
   }
 
   @Before(order = 1, value = "@CONNECTION")
@@ -243,7 +276,7 @@ public static void setNewConnectionName() {
     BeforeActions.scenario.write("New Connection name: " + connectionName);
   }
 
-  @After(order = 1, value = "@BQ_SINK_CLEANUP")
+  @After(order = 1, value = "@BQ_SINK_CLEANUP or @BQ_SINK")
   public static void deleteTempTargetBQTable() throws IOException, InterruptedException {
     try {
       BigQueryClient.dropBqQuery(bqTargetTable);
@@ -257,4 +290,14 @@ public static void deleteTempTargetBQTable() throws IOException, InterruptedExce
       }
     }
   }
+
+  @After(order = 1, value = "@BQ_SOURCE_TEST_RECEIVING_SLIP_LINE or @BQ_SOURCE_UPDATE_SERVICE_OFFERING or " +
+    "@BQ_SOURCE_UPDATE_VENDOR_CATALOG_ITEM or @BQ_SOURCE_UPDATE_AGENT_ASSIST_RECOMMENDATION or " +
+    "@BQ_SOURCE_UPDATE_RECEIVING_SLIP_LINE or @BQ_SOURCE_SERVICE_OFFERING or @BQ_SOURCE_VENDOR_CATALOG_ITEM or " +
+    "@BQ_SOURCE_AGENT_ASSIST_RECOMMENDATION")
+  public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
+    BigQueryClient.dropBqQuery(bqSourceTable);
+    PluginPropertyUtils.removePluginProp("bqSourceTable");
+    BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
+  }
 }

src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java

@@ -28,13 +28,10 @@
 import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig;
 import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
 import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
+import io.cdap.plugin.servicenow.util.SchemaType;
 import io.cdap.plugin.servicenow.util.ServiceNowConstants;
 import io.cdap.plugin.servicenow.util.SourceValueType;
-import org.apache.http.HttpStatus;
-import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
-import org.apache.oltu.oauth2.common.exception.OAuthSystemException;
 
-import java.io.IOException;
 import javax.annotation.Nullable;
 
 /**
@@ -58,6 +55,11 @@ public ServiceNowBaseConfig(String clientId, String clientSecret, String restApi
     this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password);
   }
 
+  @Nullable
+  public Boolean getUseConnection() {
+    return useConnection;
+  }
+
   @Nullable
   public ServiceNowConnectorConfig getConnection() {
     return connection;
@@ -83,7 +85,7 @@ public void validateCredentials(FailureCollector collector) {
   @VisibleForTesting
   public void validateServiceNowConnection(FailureCollector collector) {
     try {
-      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection);
+      ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection);
       restApi.getAccessToken();
     } catch (Exception e) {
       collector.addFailure("Unable to connect to ServiceNow Instance.",
@@ -123,13 +125,13 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
                             String tableField) {
     // Call API to fetch first record from the table
     ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
-      connection.getRestApiEndpoint(), tableName, false)
+      connection.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED)
       .setExcludeReferenceLink(true)
       .setDisplayValue(valueType)
       .setLimit(1);
 
     RestAPIResponse apiResponse = null;
-    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection);
+    ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection, useConnection);
     try {
       String accessToken = serviceNowTableAPIClient.getAccessToken();
       requestBuilder.setAuthHeader(accessToken);