Skip to content

[PLUGIN-1902] Changes done for schema backward compatibility along with metadata API addition for DTS #100

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
prince-cs wants to merge 3 commits into from
Closed

[PLUGIN-1902] Changes done for schema backward compatibility along with metadata API addition for DTS #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f137461
Changes done for schema backward compatibility.
prince-cs Jun 26, 2025
88ac58d
Changes done for schema backward compatibility.
vikasrathee-cs Jul 15, 2025
b24f5ed
Changes done for schema backward compatibility.
vikasrathee-cs Jul 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public static void getRecordFromServiceNowTable(String query, String tableName)
System.getenv("SERVICE_NOW_PASSWORD"),
"", "", "", null);

ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query);
}

Expand Down
10 changes: 5 additions & 5 deletions src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ public static void initializeServiceNowSourceConfig() {
@Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM")
public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Product Catalog Item table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'name':'" + uniqueId + "','price':'2500'}";
StringEntity entity = new StringEntity(recordDetails);
Expand All @@ -76,7 +76,7 @@ public static void createRecordInProductCatalogItemTable() throws IOException, S
public static void createRecordInReceivingSlipLineTable()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Receiving Slip Line table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'number':'" + uniqueId + "'}";
StringEntity entity = new StringEntity(recordDetails);
Expand All @@ -88,7 +88,7 @@ public static void createRecordInReceivingSlipLineTable()
public static void updateRecordInAgentAssistRecommendationTable()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'active':'false','name':'" + uniqueId + "'}";
StringEntity entity = new StringEntity(recordDetails);
Expand All @@ -99,7 +99,7 @@ public static void updateRecordInAgentAssistRecommendationTable()
public static void updateRecordInAgentVendorCatalogItem()
throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Vendor Catalog Item table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'out_of_stock':'false','product_id':'" + uniqueId + "'}";
StringEntity entity = new StringEntity(recordDetails);
Expand All @@ -109,7 +109,7 @@ public static void updateRecordInAgentVendorCatalogItem()
@Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING")
public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException {
BeforeActions.scenario.write("Create new record in Service Offering table");
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection());
ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true);
String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10);
String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," +
" 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}";
Expand Down
16 changes: 9 additions & 7 deletions src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,10 @@
import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig;
import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig;
import io.cdap.plugin.servicenow.util.SchemaType;
import io.cdap.plugin.servicenow.util.ServiceNowConstants;
import io.cdap.plugin.servicenow.util.SourceValueType;
import org.apache.http.HttpStatus;
import org.apache.oltu.oauth2.common.exception.OAuthProblemException;
import org.apache.oltu.oauth2.common.exception.OAuthSystemException;

import java.io.IOException;
import javax.annotation.Nullable;

/**
Expand All @@ -58,6 +55,11 @@ public ServiceNowBaseConfig(String clientId, String clientSecret, String restApi
this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password);
}

@Nullable
public Boolean getUseConnection() {
return useConnection;
}

@Nullable
public ServiceNowConnectorConfig getConnection() {
return connection;
Expand All @@ -83,7 +85,7 @@ public void validateCredentials(FailureCollector collector) {
@VisibleForTesting
public void validateServiceNowConnection(FailureCollector collector) {
try {
ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection);
ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection);
restApi.getAccessToken();
} catch (Exception e) {
collector.addFailure("Unable to connect to ServiceNow Instance.",
Expand Down Expand Up @@ -123,13 +125,13 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo
String tableField) {
// Call API to fetch first record from the table
ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
connection.getRestApiEndpoint(), tableName, false)
connection.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED)
.setExcludeReferenceLink(true)
.setDisplayValue(valueType)
.setLimit(1);

RestAPIResponse apiResponse = null;
ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection);
ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection, useConnection);
try {
String accessToken = serviceNowTableAPIClient.getAccessToken();
requestBuilder.setAuthHeader(accessToken);
Expand Down
140 changes: 124 additions & 16 deletions src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,12 @@
import io.cdap.plugin.servicenow.restapi.RestAPIResponse;
import io.cdap.plugin.servicenow.sink.model.APIResponse;
import io.cdap.plugin.servicenow.sink.model.CreateRecordAPIResponse;
import io.cdap.plugin.servicenow.sink.model.MetadataAPISchemaResponse;
import io.cdap.plugin.servicenow.sink.model.SchemaField;
import io.cdap.plugin.servicenow.sink.model.SchemaResponse;
import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField;
import io.cdap.plugin.servicenow.util.SchemaBuilder;
import io.cdap.plugin.servicenow.util.SchemaType;
import io.cdap.plugin.servicenow.util.ServiceNowColumn;
import io.cdap.plugin.servicenow.util.ServiceNowConstants;
import io.cdap.plugin.servicenow.util.SourceValueType;
Expand Down Expand Up @@ -70,10 +73,11 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient {
private static final String OAUTH_URL_TEMPLATE = "%s/oauth_token.do";
private static final Gson GSON = new Gson();
private final ServiceNowConnectorConfig conf;
public static JsonArray serviceNowJsonResultArray;
public final Boolean useConnection;

public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf) {
public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf, Boolean useConnection) {
this.conf = conf;
this.useConnection = useConnection;
}

public String getAccessToken() throws ServiceNowAPIException {
Expand Down Expand Up @@ -127,7 +131,7 @@ public List<Map<String, String>> fetchTableRecords(
int limit)
throws ServiceNowAPIException {
ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
this.conf.getRestApiEndpoint(), tableName, false)
this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
.setExcludeReferenceLink(true)
.setDisplayValue(valueType)
.setLimit(limit);
Expand Down Expand Up @@ -176,7 +180,6 @@ private int getRecordCountFromHeader(RestAPIResponse apiResponse) {

public List<Map<String, String>> parseResponseToResultListOfMap(String responseBody) {


JsonObject jo = GSON.fromJson(responseBody, JsonObject.class);
JsonArray ja = jo.getAsJsonArray(ServiceNowConstants.RESULT);

Expand Down Expand Up @@ -255,7 +258,7 @@ public List<Map<String, String>> fetchTableRecordsRetryableMode(String tableName
public Schema fetchTableSchema(String tableName, FailureCollector collector) {
Schema schema = null;
try {
schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE);
schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE, useConnection);
} catch (Exception e) {
LOG.error("Failed to fetch schema on table {}", tableName, e);
collector.addFailure(String.format("Connection failed. Unable to fetch schema for table: %s. Cause: %s",
Expand All @@ -265,20 +268,37 @@ public Schema fetchTableSchema(String tableName, FailureCollector collector) {
}

@VisibleForTesting
public SchemaResponse parseSchemaResponse(String responseBody) {
public MetadataAPISchemaResponse parseSchemaResponse(String responseBody) {
return GSON.fromJson(responseBody, MetadataAPISchemaResponse.class);
}

public SchemaResponse parseSchemaResponseWithoutMetadata(String responseBody) {
return GSON.fromJson(responseBody, SchemaResponse.class);
}

/**
* Fetches the table schema from ServiceNow
*
* @param tableName ServiceNow table name for which schema is getting fetched
* @param valueType Whether to fetch schema for actual value or display value
* @param useConnection This flag is added to backward compatibility with 1.1 release,
* in case of null for useConnection, schema will be string based.
* @return schema for given ServiceNow table
* @throws ServiceNowAPIException
*/
public Schema fetchTableSchema(String tableName, SourceValueType valueType)
public Schema fetchTableSchema(String tableName, SourceValueType valueType, Boolean useConnection)
throws ServiceNowAPIException {
return fetchTableSchema(tableName, getAccessToken(), valueType);
SchemaType schemaType = getSchemaTypeBasedOnUseConnection(useConnection);
return fetchTableSchema(tableName, getAccessToken(), valueType, schemaType);
}

private SchemaType getSchemaTypeBasedOnUseConnection(Boolean useConnection) {
// use connection was added in release/1.2, so it will be null for users who are upgrading from release/1.1
// This is added to support backward compatibility for 1.1 users.
if (useConnection == null) {
return SchemaType.STRING_BASED;
}
return SchemaType.SCHEMA_API_BASED;
}

/**
Expand All @@ -287,25 +307,88 @@ public Schema fetchTableSchema(String tableName, SourceValueType valueType)
* @param tableName ServiceNow table name for which schema is getting fetched
* @param accessToken Access Token to use
* @param valueType Type of value (Actual/Display)
* @param schemaType Enum to determine which approach to take to fetch schema.
* @return schema for given ServiceNow table
*/
public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType)
public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType,
SchemaType schemaType)
throws ServiceNowAPIException {
ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
this.conf.getRestApiEndpoint(), tableName, true)
this.conf.getRestApiEndpoint(), tableName, true, schemaType)
.setExcludeReferenceLink(true);

RestAPIResponse restAPIResponse;
requestBuilder.setAuthHeader(accessToken);
restAPIResponse = executeGetWithRetries(requestBuilder.build());
SchemaResponse schemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody());
List<ServiceNowColumn> columns = new ArrayList<>();

if (schemaResponse.getResult() == null && schemaResponse.getResult().getColumns().isEmpty()) {
if (schemaType == SchemaType.METADATA_API_BASED) {
return prepareSchemaWithMetadataAPI(restAPIResponse, columns, tableName, valueType);
} else if (schemaType == SchemaType.SCHEMA_API_BASED) {
return prepareSchemaWithSchemaAPI(restAPIResponse, columns, tableName);
} else {
return prepareStringBasedSchema(restAPIResponse, columns, tableName);
}
}

/**
* Processes a schema response obtained from the ServiceNow Table API (without using Metadata API)
* and constructs a {@link Schema} object based on the parsed column definitions.
*
* <p>This method parses the raw JSON response body into a list of schema fields,
* extracts the internal column types, and appends them to the provided column list.
* The final schema is constructed using the {@link SchemaBuilder} utility.</p>
*
* @param restAPIResponse The raw API response received from the ServiceNow Table API.
* @param columns A list to which parsed {@link ServiceNowColumn} objects will be added.
* @param tableName The name of the table for which the schema is being constructed.
*
* @return A {@link Schema} object representing the table structure as interpreted from the Schema API.
*
* @throws RuntimeException if the schema response is null or contains no result.
*/
private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
String tableName) {
SchemaResponse schemaResponse =
parseSchemaResponseWithoutMetadata(restAPIResponse.getResponseBody());

if (schemaResponse.getResult() == null && schemaResponse.getResult().isEmpty()) {
throw new RuntimeException("Error - Schema Response does not contain any result");
}

for (ServiceNowSchemaField field : schemaResponse.getResult().getColumns().values()) {
for (SchemaField field : schemaResponse.getResult()) {
columns.add(new ServiceNowColumn(field.getName(), field.getInternalType()));
}
return SchemaBuilder.constructSchema(tableName, columns);
}

/**
* Parses a ServiceNow schema response obtained via the Metadata API and constructs a
* {@link Schema} object using the extracted field information and value type preferences.
*
* <p>This method reads the JSON response, extracts the column metadata including field names
* and data types, and adds each as a {@link ServiceNowColumn} to the provided list. The choice
* between internal values and display values is based on the {@code valueType} parameter.</p>
*
* @param restAPIResponse The response returned from the ServiceNow Metadata API.
* @param columns A list to which parsed {@link ServiceNowColumn} definitions will be added.
* @param tableName The name of the ServiceNow table for which the schema is being generated.
* @param valueType The value type preference (e.g., {@code SHOW_DISPLAY_VALUE} or {@code USE_INTERNAL_VALUE}).
* Determines whether to use display types or internal types in the resulting schema.
*
* @return A {@link Schema} object representing the table structure as interpreted from the Metadata API.
*
* @throws RuntimeException if the response does not contain valid column information.
*/
private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
String tableName, SourceValueType valueType) {
MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody());

if (metadataAPISchemaResponse.getResult() == null && metadataAPISchemaResponse.getResult().getColumns().isEmpty()) {
throw new RuntimeException("Error - Schema Response does not contain any result");
}

for (ServiceNowSchemaField field : metadataAPISchemaResponse.getResult().getColumns().values()) {
if (valueType.equals(SourceValueType.SHOW_DISPLAY_VALUE) &&
!Objects.equals(field.getType(), field.getInternalType())) {
columns.add(new ServiceNowColumn(field.getName(), field.getType()));
Expand Down Expand Up @@ -338,7 +421,7 @@ public int getTableRecordCount(String tableName)
*/
public int getTableRecordCount(String tableName, String accessToken) throws ServiceNowAPIException {
ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
this.conf.getRestApiEndpoint(), tableName, false)
this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
.setExcludeReferenceLink(true)
.setDisplayValue(SourceValueType.SHOW_DISPLAY_VALUE)
.setLimit(1);
Expand All @@ -358,7 +441,7 @@ public int getTableRecordCount(String tableName, String accessToken) throws Serv
*/
public String createRecord(String tableName, HttpEntity entity) throws IOException, ServiceNowAPIException {
ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
this.conf.getRestApiEndpoint(), tableName, false);
this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection));
String systemID;
RestAPIResponse apiResponse = null;
try {
Expand Down Expand Up @@ -393,7 +476,7 @@ public Map<String, String> getRecordFromServiceNowTable(String tableName, String
throws ServiceNowAPIException {

ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder(
this.conf.getRestApiEndpoint(), tableName, false)
this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection))
.setQuery(query);

RestAPIResponse restAPIResponse;
Expand All @@ -404,4 +487,29 @@ public Map<String, String> getRecordFromServiceNowTable(String tableName, String
APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class);
return apiResponse.getResult().get(0);
}

/**
* Processes the response obtained from the ServiceNow Table API
* and constructs a {@link Schema} object based on the first record.
*
* @param restAPIResponse The raw API response received from the ServiceNow Table API.
* @param columns A list to which parsed {@link ServiceNowColumn} objects will be added.
* @param tableName The name of the table for which the schema is being constructed.
*
* @return A {@link Schema} object representing the table structure as interpreted from the Schema API.
*
* @throws RuntimeException if the schema response is null or contains no result.
*/
private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List<ServiceNowColumn> columns,
String tableName) {
List<Map<String, String>> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody());
if (result != null && !result.isEmpty()) {
Map<String, String> firstRecord = result.get(0);
for (String key : firstRecord.keySet()) {
columns.add(new ServiceNowColumn(key, "string"));
}
}
return SchemaBuilder.constructSchema(tableName, columns);
}

}
Loading
Loading