diff --git a/src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java b/src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java index 8b43da6f..f7c359fb 100644 --- a/src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java +++ b/src/e2e-test/java/io/cdap/plugin/servicenowsink/actions/ServiceNowSinkPropertiesPageActions.java @@ -61,7 +61,7 @@ public static void getRecordFromServiceNowTable(String query, String tableName) System.getenv("SERVICE_NOW_PASSWORD"), "", "", "", null); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); responseFromServiceNowTable = tableAPIClient.getRecordFromServiceNowTable(tableName, query); } diff --git a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java index f284ace8..d06ec583 100644 --- a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java @@ -65,7 +65,7 @@ public static void initializeServiceNowSourceConfig() { @Before(order = 2, value = "@SN_PRODUCT_CATALOG_ITEM") public static void createRecordInProductCatalogItemTable() throws IOException, ServiceNowAPIException { BeforeActions.scenario.write("Create new record in Product Catalog Item table"); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); String uniqueId = "TestProductCatalogItem" + RandomStringUtils.randomAlphanumeric(10); String recordDetails = "{'name':'" + uniqueId + "','price':'2500'}"; StringEntity entity = new StringEntity(recordDetails); @@ -76,7 +76,7 @@ public static void createRecordInProductCatalogItemTable() throws IOException, S public static void createRecordInReceivingSlipLineTable() throws IOException, ServiceNowAPIException { BeforeActions.scenario.write("Create new record in Receiving Slip Line table"); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); String uniqueId = "TestReceivingSlipLine" + RandomStringUtils.randomAlphanumeric(10); String recordDetails = "{'number':'" + uniqueId + "'}"; StringEntity entity = new StringEntity(recordDetails); @@ -88,7 +88,7 @@ public static void createRecordInReceivingSlipLineTable() public static void updateRecordInAgentAssistRecommendationTable() throws IOException, ServiceNowAPIException { BeforeActions.scenario.write("Create new record in Agent Assist Recommendation table"); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); String uniqueId = "TestAgentAssist" + RandomStringUtils.randomAlphanumeric(10); String recordDetails = "{'active':'false','name':'" + uniqueId + "'}"; StringEntity entity = new StringEntity(recordDetails); @@ -99,7 +99,7 @@ public static void updateRecordInAgentAssistRecommendationTable() public static void updateRecordInAgentVendorCatalogItem() throws IOException, ServiceNowAPIException { BeforeActions.scenario.write("Create new record in Vendor Catalog Item table"); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); String uniqueId = "TestVendorCatalog" + RandomStringUtils.randomAlphanumeric(10); String recordDetails = "{'out_of_stock':'false','product_id':'" + uniqueId + "'}"; StringEntity entity = new StringEntity(recordDetails); @@ -109,7 +109,7 @@ public static void updateRecordInAgentVendorCatalogItem() @Before(order = 2, value = "@SN_UPDATE_SERVICE_OFFERING") public static void updateRecordInServiceOffering() throws IOException, ServiceNowAPIException { BeforeActions.scenario.write("Create new record in Service Offering table"); - ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection()); + ServiceNowTableAPIClientImpl tableAPIClient = new ServiceNowTableAPIClientImpl(config.getConnection(), true); String uniqueId = "TestServiceOffering" + RandomStringUtils.randomAlphanumeric(10); String recordDetails = "{'purchase_date':'2022-05-28','end_date':'2022-06-05 15:00:00'," + " 'start_date':'2022-05-25 15:00:00','number':'" + uniqueId + "'}"; diff --git a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java index 945a9e39..9bdcbdd7 100644 --- a/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/ServiceNowBaseConfig.java @@ -28,13 +28,10 @@ import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.source.ServiceNowSourceConfig; +import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceValueType; -import org.apache.http.HttpStatus; -import org.apache.oltu.oauth2.common.exception.OAuthProblemException; -import org.apache.oltu.oauth2.common.exception.OAuthSystemException; -import java.io.IOException; import javax.annotation.Nullable; /** @@ -58,6 +55,11 @@ public ServiceNowBaseConfig(String clientId, String clientSecret, String restApi this.connection = new ServiceNowConnectorConfig(clientId, clientSecret, restApiEndpoint, user, password); } + @Nullable + public Boolean getUseConnection() { + return useConnection; + } + @Nullable public ServiceNowConnectorConfig getConnection() { return connection; @@ -83,7 +85,7 @@ public void validateCredentials(FailureCollector collector) { @VisibleForTesting public void validateServiceNowConnection(FailureCollector collector) { try { - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection); + ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(connection, useConnection); restApi.getAccessToken(); } catch (Exception e) { collector.addFailure("Unable to connect to ServiceNow Instance.", @@ -123,13 +125,13 @@ public void validateTable(String tableName, SourceValueType valueType, FailureCo String tableField) { // Call API to fetch first record from the table ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - connection.getRestApiEndpoint(), tableName, false) + connection.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED) .setExcludeReferenceLink(true) .setDisplayValue(valueType) .setLimit(1); RestAPIResponse apiResponse = null; - ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection); + ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(connection, useConnection); try { String accessToken = serviceNowTableAPIClient.getAccessToken(); requestBuilder.setAuthHeader(accessToken); diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 3c0afaa1..e291965b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -34,9 +34,12 @@ import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.sink.model.APIResponse; import io.cdap.plugin.servicenow.sink.model.CreateRecordAPIResponse; +import io.cdap.plugin.servicenow.sink.model.MetadataAPISchemaResponse; +import io.cdap.plugin.servicenow.sink.model.SchemaField; import io.cdap.plugin.servicenow.sink.model.SchemaResponse; import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; import io.cdap.plugin.servicenow.util.SchemaBuilder; +import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceValueType; @@ -70,10 +73,11 @@ public class ServiceNowTableAPIClientImpl extends RestAPIClient { private static final String OAUTH_URL_TEMPLATE = "%s/oauth_token.do"; private static final Gson GSON = new Gson(); private final ServiceNowConnectorConfig conf; - public static JsonArray serviceNowJsonResultArray; + public final Boolean useConnection; - public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf) { + public ServiceNowTableAPIClientImpl(ServiceNowConnectorConfig conf, Boolean useConnection) { this.conf = conf; + this.useConnection = useConnection; } public String getAccessToken() throws ServiceNowAPIException { @@ -127,7 +131,7 @@ public List> fetchTableRecords( int limit) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false) + this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection)) .setExcludeReferenceLink(true) .setDisplayValue(valueType) .setLimit(limit); @@ -176,7 +180,6 @@ private int getRecordCountFromHeader(RestAPIResponse apiResponse) { public List> parseResponseToResultListOfMap(String responseBody) { - JsonObject jo = GSON.fromJson(responseBody, JsonObject.class); JsonArray ja = jo.getAsJsonArray(ServiceNowConstants.RESULT); @@ -255,7 +258,7 @@ public List> fetchTableRecordsRetryableMode(String tableName public Schema fetchTableSchema(String tableName, FailureCollector collector) { Schema schema = null; try { - schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE); + schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE, useConnection); } catch (Exception e) { LOG.error("Failed to fetch schema on table {}", tableName, e); collector.addFailure(String.format("Connection failed. Unable to fetch schema for table: %s. Cause: %s", @@ -265,7 +268,11 @@ public Schema fetchTableSchema(String tableName, FailureCollector collector) { } @VisibleForTesting - public SchemaResponse parseSchemaResponse(String responseBody) { + public MetadataAPISchemaResponse parseSchemaResponse(String responseBody) { + return GSON.fromJson(responseBody, MetadataAPISchemaResponse.class); + } + + public SchemaResponse parseSchemaResponseWithoutMetadata(String responseBody) { return GSON.fromJson(responseBody, SchemaResponse.class); } @@ -273,12 +280,25 @@ public SchemaResponse parseSchemaResponse(String responseBody) { * Fetches the table schema from ServiceNow * * @param tableName ServiceNow table name for which schema is getting fetched + * @param valueType Whether to fetch schema for actual value or display value + * @param useConnection This flag is added to backward compatibility with 1.1 release, + * in case of null for useConnection, schema will be string based. * @return schema for given ServiceNow table * @throws ServiceNowAPIException */ - public Schema fetchTableSchema(String tableName, SourceValueType valueType) + public Schema fetchTableSchema(String tableName, SourceValueType valueType, Boolean useConnection) throws ServiceNowAPIException { - return fetchTableSchema(tableName, getAccessToken(), valueType); + SchemaType schemaType = getSchemaTypeBasedOnUseConnection(useConnection); + return fetchTableSchema(tableName, getAccessToken(), valueType, schemaType); + } + + private SchemaType getSchemaTypeBasedOnUseConnection(Boolean useConnection) { + // use connection was added in release/1.2, so it will be null for users who are upgrading from release/1.1 + // This is added to support backward compatibility for 1.1 users. + if (useConnection == null) { + return SchemaType.STRING_BASED; + } + return SchemaType.SCHEMA_API_BASED; } /** @@ -287,25 +307,88 @@ public Schema fetchTableSchema(String tableName, SourceValueType valueType) * @param tableName ServiceNow table name for which schema is getting fetched * @param accessToken Access Token to use * @param valueType Type of value (Actual/Display) + * @param schemaType Enum to determine which approach to take to fetch schema. * @return schema for given ServiceNow table */ - public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType) + public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType, + SchemaType schemaType) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, true) + this.conf.getRestApiEndpoint(), tableName, true, schemaType) .setExcludeReferenceLink(true); RestAPIResponse restAPIResponse; requestBuilder.setAuthHeader(accessToken); restAPIResponse = executeGetWithRetries(requestBuilder.build()); - SchemaResponse schemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody()); List columns = new ArrayList<>(); - if (schemaResponse.getResult() == null && schemaResponse.getResult().getColumns().isEmpty()) { + if (schemaType == SchemaType.METADATA_API_BASED) { + return prepareSchemaWithMetadataAPI(restAPIResponse, columns, tableName, valueType); + } else if (schemaType == SchemaType.SCHEMA_API_BASED) { + return prepareSchemaWithSchemaAPI(restAPIResponse, columns, tableName); + } else { + return prepareStringBasedSchema(restAPIResponse, columns, tableName); + } + } + + /** + * Processes a schema response obtained from the ServiceNow Table API (without using Metadata API) + * and constructs a {@link Schema} object based on the parsed column definitions. + * + *

This method parses the raw JSON response body into a list of schema fields, + * extracts the internal column types, and appends them to the provided column list. + * The final schema is constructed using the {@link SchemaBuilder} utility.

+ * + * @param restAPIResponse The raw API response received from the ServiceNow Table API. + * @param columns A list to which parsed {@link ServiceNowColumn} objects will be added. + * @param tableName The name of the table for which the schema is being constructed. + * + * @return A {@link Schema} object representing the table structure as interpreted from the Schema API. + * + * @throws RuntimeException if the schema response is null or contains no result. + */ + private Schema prepareSchemaWithSchemaAPI(RestAPIResponse restAPIResponse, List columns, + String tableName) { + SchemaResponse schemaResponse = + parseSchemaResponseWithoutMetadata(restAPIResponse.getResponseBody()); + + if (schemaResponse.getResult() == null && schemaResponse.getResult().isEmpty()) { throw new RuntimeException("Error - Schema Response does not contain any result"); } - for (ServiceNowSchemaField field : schemaResponse.getResult().getColumns().values()) { + for (SchemaField field : schemaResponse.getResult()) { + columns.add(new ServiceNowColumn(field.getName(), field.getInternalType())); + } + return SchemaBuilder.constructSchema(tableName, columns); + } + + /** + * Parses a ServiceNow schema response obtained via the Metadata API and constructs a + * {@link Schema} object using the extracted field information and value type preferences. + * + *

This method reads the JSON response, extracts the column metadata including field names + * and data types, and adds each as a {@link ServiceNowColumn} to the provided list. The choice + * between internal values and display values is based on the {@code valueType} parameter.

+ * + * @param restAPIResponse The response returned from the ServiceNow Metadata API. + * @param columns A list to which parsed {@link ServiceNowColumn} definitions will be added. + * @param tableName The name of the ServiceNow table for which the schema is being generated. + * @param valueType The value type preference (e.g., {@code SHOW_DISPLAY_VALUE} or {@code USE_INTERNAL_VALUE}). + * Determines whether to use display types or internal types in the resulting schema. + * + * @return A {@link Schema} object representing the table structure as interpreted from the Metadata API. + * + * @throws RuntimeException if the response does not contain valid column information. + */ + private Schema prepareSchemaWithMetadataAPI(RestAPIResponse restAPIResponse, List columns, + String tableName, SourceValueType valueType) { + MetadataAPISchemaResponse metadataAPISchemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody()); + + if (metadataAPISchemaResponse.getResult() == null && metadataAPISchemaResponse.getResult().getColumns().isEmpty()) { + throw new RuntimeException("Error - Schema Response does not contain any result"); + } + + for (ServiceNowSchemaField field : metadataAPISchemaResponse.getResult().getColumns().values()) { if (valueType.equals(SourceValueType.SHOW_DISPLAY_VALUE) && !Objects.equals(field.getType(), field.getInternalType())) { columns.add(new ServiceNowColumn(field.getName(), field.getType())); @@ -338,7 +421,7 @@ public int getTableRecordCount(String tableName) */ public int getTableRecordCount(String tableName, String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false) + this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection)) .setExcludeReferenceLink(true) .setDisplayValue(SourceValueType.SHOW_DISPLAY_VALUE) .setLimit(1); @@ -358,7 +441,7 @@ public int getTableRecordCount(String tableName, String accessToken) throws Serv */ public String createRecord(String tableName, HttpEntity entity) throws IOException, ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false); + this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection)); String systemID; RestAPIResponse apiResponse = null; try { @@ -393,7 +476,7 @@ public Map getRecordFromServiceNowTable(String tableName, String throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - this.conf.getRestApiEndpoint(), tableName, false) + this.conf.getRestApiEndpoint(), tableName, false, getSchemaTypeBasedOnUseConnection(useConnection)) .setQuery(query); RestAPIResponse restAPIResponse; @@ -404,4 +487,29 @@ public Map getRecordFromServiceNowTable(String tableName, String APIResponse apiResponse = GSON.fromJson(restAPIResponse.getResponseBody(), APIResponse.class); return apiResponse.getResult().get(0); } + + /** + * Processes the response obtained from the ServiceNow Table API + * and constructs a {@link Schema} object based on the first record. + * + * @param restAPIResponse The raw API response received from the ServiceNow Table API. + * @param columns A list to which parsed {@link ServiceNowColumn} objects will be added. + * @param tableName The name of the table for which the schema is being constructed. + * + * @return A {@link Schema} object representing the table structure as interpreted from the Schema API. + * + * @throws RuntimeException if the schema response is null or contains no result. + */ + private Schema prepareStringBasedSchema(RestAPIResponse restAPIResponse, List columns, + String tableName) { + List> result = parseResponseToResultListOfMap(restAPIResponse.getResponseBody()); + if (result != null && !result.isEmpty()) { + Map firstRecord = result.get(0); + for (String key : firstRecord.keySet()) { + columns.add(new ServiceNowColumn(key, "string")); + } + } + return SchemaBuilder.constructSchema(tableName, columns); + } + } diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java index aa654cf9..31aeaa3a 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java @@ -18,6 +18,7 @@ import com.google.common.base.Joiner; import io.cdap.plugin.servicenow.restapi.RestAPIRequest; +import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.SourceValueType; import java.net.URLEncoder; @@ -48,9 +49,17 @@ public class ServiceNowTableAPIRequestBuilder extends RestAPIRequest.Builder { */ private static final String METADATA_API_URL_TEMPLATE = "%s/api/now/ui/meta/%s"; - public ServiceNowTableAPIRequestBuilder(String instanceBaseUrl, String tableName, boolean isSchemaRequired) { + public ServiceNowTableAPIRequestBuilder(String instanceBaseUrl, String tableName, boolean isSchemaRequired, + SchemaType schemaType) { if (isSchemaRequired) { - this.setUrl(String.format(METADATA_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + if (SchemaType.METADATA_API_BASED == schemaType) { + this.setUrl(String.format(METADATA_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + } else if (SchemaType.SCHEMA_API_BASED == schemaType) { + this.setUrl(String.format(SCHEMA_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + } else { + this.setUrl(String.format(TABLE_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + this.setLimit(1); + } } else { this.setUrl(String.format(TABLE_API_URL_TEMPLATE, instanceBaseUrl, tableName)); } diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index fbf5dda4..81aa364c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -44,6 +44,7 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIRequestBuilder; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.source.ServiceNowInputFormat; +import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import io.cdap.plugin.servicenow.util.SourceQueryMode; @@ -89,7 +90,7 @@ public void test(ConnectorContext connectorContext) throws ValidationException { @Override public BrowseDetail browse(ConnectorContext connectorContext, BrowseRequest browseRequest) throws IOException { - ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config, true); try { String accessToken = serviceNowTableAPIClient.getAccessToken(); return browse(connectorContext, accessToken); @@ -127,11 +128,11 @@ public BrowseDetail browse(ConnectorContext connectorContext, */ private TableList listTables(String accessToken) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - config.getRestApiEndpoint(), OBJECT_TABLE_LIST, false); + config.getRestApiEndpoint(), OBJECT_TABLE_LIST, false, SchemaType.SCHEMA_API_BASED); requestBuilder.setAuthHeader(accessToken); requestBuilder.setAcceptHeader(MediaType.APPLICATION_JSON); requestBuilder.setContentTypeHeader(MediaType.APPLICATION_JSON); - ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse apiResponse = serviceNowTableAPIClient.executeGetWithRetries(requestBuilder.build()); return GSON.fromJson(apiResponse.getResponseBody(), TableList.class); @@ -173,11 +174,11 @@ public List sample(ConnectorContext connectorContext, SampleRe private List getTableData(String tableName, int limit) throws OAuthProblemException, OAuthSystemException, ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( - config.getRestApiEndpoint(), tableName, false) + config.getRestApiEndpoint(), tableName, false, SchemaType.SCHEMA_API_BASED) .setExcludeReferenceLink(true) .setDisplayValue(SourceValueType.SHOW_DISPLAY_VALUE) .setLimit(limit); - ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl serviceNowTableAPIClient = new ServiceNowTableAPIClientImpl(config, true); String accessToken = serviceNowTableAPIClient.getAccessToken(); requestBuilder.setAuthHeader(accessToken); requestBuilder.setResponseHeaders(ServiceNowConstants.HEADER_NAME_TOTAL_COUNT); @@ -212,7 +213,8 @@ private Schema getSchema(String tableName) { config, tableName, null, - SourceValueType.SHOW_DISPLAY_VALUE); + SourceValueType.SHOW_DISPLAY_VALUE, + true); Schema schema = tableInfo.stream().findFirst().isPresent() ? tableInfo.stream().findFirst().get().getSchema() : null; return schema; diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java index 61114276..cb7f25e5 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorConfig.java @@ -124,7 +124,7 @@ public void validateCredentialsFields(FailureCollector collector) { public void validateConnection(FailureCollector collector) { try { - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(this); + ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(this, true); restApi.getAccessToken(); } catch (Exception e) { collector.addFailure("Unable to connect to ServiceNow Instance.", diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java index 8438055c..f0f02b0b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfig.java @@ -147,7 +147,8 @@ void validateSchema(Schema schema, FailureCollector collector) { || containsMacro(ServiceNowConstants.PROPERTY_OPERATION) || containsMacro(PROPERTY_EXTERNAL_ID_FIELD)) { return; } - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(this.getConnection()); + ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(this.getConnection(), + this.getUseConnection()); Schema tableSchema = restApi.fetchTableSchema(tableName, collector); if (tableSchema == null) { throw collector.getOrThrowException(); diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResponse.java new file mode 100644 index 00000000..c9b671f3 --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResponse.java @@ -0,0 +1,34 @@ +/* + * Copyright © 2022 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +/** + * Model class for Schema Response from Column Metadata API + */ +public class MetadataAPISchemaResponse { + + private final MetadataAPISchemaResult result; + + public MetadataAPISchemaResponse(MetadataAPISchemaResult result) { + this.result = result; + } + + public MetadataAPISchemaResult getResult() { + return result; + } + +} diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResult.java similarity index 93% rename from src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java rename to src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResult.java index 80b11440..9275a1af 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/MetadataAPISchemaResult.java @@ -50,10 +50,10 @@ * In this example, the map will contain keys like {@code "state"} and {@code "active"}, * each pointing to a {@code ServiceNowSchemaField} instance with metadata about that field. */ -public class ServiceNowSchemaResult { +public class MetadataAPISchemaResult { private final Map columns; - public ServiceNowSchemaResult(Map columns) { + public MetadataAPISchemaResult(Map columns) { this.columns = columns; } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaField.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaField.java new file mode 100644 index 00000000..0677ecba --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaField.java @@ -0,0 +1,50 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +/** + * Model class for Schema Field from Schema API + */ +public class SchemaField { + private final String label; + private final String exampleValue; + private final String internalType; + private final String name; + + public SchemaField(String label, String exampleValue, String internalType, String name) { + this.label = label; + this.exampleValue = exampleValue; + this.internalType = internalType; + this.name = name; + } + + public String getLabel() { + return label; + } + + public String getExampleValue() { + return exampleValue; + } + + public String getInternalType() { + return internalType; + } + + public String getName() { + return name; + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java index ec85f245..c355633e 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java @@ -1,5 +1,5 @@ /* - * Copyright © 2022 Cask Data, Inc. + * Copyright © 2025 Cask Data, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,19 +16,21 @@ package io.cdap.plugin.servicenow.sink.model; +import java.util.List; + /** * Model class for Schema Response from Schema API */ public class SchemaResponse { - private final ServiceNowSchemaResult result; + private final List result; - public SchemaResponse(ServiceNowSchemaResult result) { + public SchemaResponse(List result) { this.result = result; } - public ServiceNowSchemaResult getResult() { + public List getResult() { return result; } - + } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java index 80209e06..7dee308b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java @@ -19,7 +19,7 @@ import com.google.gson.annotations.SerializedName; /** - * Model class for Schema Field from Schema API + * Model class for Schema Field from Column Metadata API */ public class ServiceNowSchemaField { private final String label; diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java index af346c4c..6acdb5d2 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/service/ServiceNowSinkAPIRequestImpl.java @@ -71,7 +71,7 @@ public class ServiceNowSinkAPIRequestImpl { public ServiceNowSinkAPIRequestImpl(ServiceNowSinkConfig conf) { this.config = conf; - restApi = new ServiceNowTableAPIClientImpl(config.getConnection()); + restApi = new ServiceNowTableAPIClientImpl(config.getConnection(), config.getUseConnection()); } public RestRequest getRestRequest(JsonObject jsonObject) { diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java index 190f2272..bcf5f932 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java @@ -63,7 +63,8 @@ public static List setInput(Configuration jobConfig, Source // Depending on conf value fetch the list of fields for each table and create schema object // return the schema object for each table as ServiceNowTableInfo List tableInfos = fetchTableInfo(mode, conf.getConnection(), conf.getTableName(), - conf.getApplicationName(), conf.getValueType()); + conf.getApplicationName(), conf.getValueType(), + conf.getUseConnection()); jobConf.setTableInfos(tableInfos); return tableInfos; @@ -72,10 +73,11 @@ public static List setInput(Configuration jobConfig, Source public static List fetchTableInfo(SourceQueryMode mode, ServiceNowConnectorConfig conf, @Nullable String tableName, @Nullable SourceApplication application, - @Nullable SourceValueType valueType) { + @Nullable SourceValueType valueType, + @Nullable Boolean useConnection) { // When mode = Table, fetch details from the table name provided in plugin config if (mode == SourceQueryMode.TABLE) { - ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf, valueType); + ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf, valueType, useConnection); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -85,7 +87,7 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser List tableNames = application.getTableNames(); for (String table : tableNames) { - ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, valueType); + ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, valueType, useConnection); if (tableInfo == null) { continue; } @@ -97,14 +99,15 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf, - SourceValueType valueType) { + SourceValueType valueType, + @Nullable Boolean useConnection) { // Call API to fetch first record from the table - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf); + ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf, useConnection); Schema schema = null; int recordCount = 0; try { - schema = restApi.fetchTableSchema(tableName, valueType); + schema = restApi.fetchTableSchema(tableName, valueType, useConnection); recordCount = restApi.getTableRecordCount(tableName); } catch (ServiceNowAPIException e) { throw new RuntimeException(String.format("Error in fetching table metadata due to reason: %s", e.getMessage()), diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java index 96140ba1..485466eb 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java @@ -67,20 +67,22 @@ public static Set setInput(Configuration jobConfig, // Depending on conf value fetch the list of fields for each table and create schema object // return the schema object for each table as ServiceNowTableInfo - Set tableInfos = fetchTablesInfo(conf.getConnection(), conf.getTableNames()); + Set tableInfos = fetchTablesInfo(conf.getConnection(), conf.getTableNames(), + conf.getUseConnection()); jobConf.setTableInfos(tableInfos.stream().collect(Collectors.toList())); return tableInfos; } - static Set fetchTablesInfo(ServiceNowConnectorConfig conf, String tableNames) { + static Set fetchTablesInfo(ServiceNowConnectorConfig conf, String tableNames, + Boolean useConnection) { Set tablesInfos = new LinkedHashSet<>(); Set tableNameSet = getList(tableNames); for (String table : tableNameSet) { - ServiceNowTableInfo tableInfo = getTableMetaData(table, conf); + ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, useConnection); if (tableInfo == null) { continue; } @@ -90,9 +92,10 @@ static Set fetchTablesInfo(ServiceNowConnectorConfig conf, return tablesInfos; } - private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf) { + private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf, + Boolean useConnection) { // Call API to fetch first record from the table - ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf); + ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf, useConnection); Schema schema; int recordCount; @@ -100,7 +103,7 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow // Use actual value type as connector config does not have an option to select value type // This is used for ServiceNowMultiSource and provides structure of the table and being dependent on // connector config, makes it a read only function - schema = restApi.fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE); + schema = restApi.fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE, true); recordCount = restApi.getTableRecordCount(tableName); } catch (ServiceNowAPIException e) { throw new RuntimeException(e); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index 4f8b56dd..b46eae3b 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -22,13 +22,8 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowRecordConverter; -import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.oltu.oauth2.common.exception.OAuthProblemException; -import org.apache.oltu.oauth2.common.exception.OAuthSystemException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; @@ -51,7 +46,8 @@ public class ServiceNowMultiRecordReader extends ServiceNowBaseRecordReader { public void initialize(InputSplit split, TaskAttemptContext context) { this.split = (ServiceNowInputSplit) split; this.pos = 0; - restApi = new ServiceNowTableAPIClientImpl(multiSourcePluginConf.getConnection()); + restApi = new ServiceNowTableAPIClientImpl(multiSourcePluginConf.getConnection(), + multiSourcePluginConf.getUseConnection()); tableName = ((ServiceNowInputSplit) split).getTableName(); tableNameField = multiSourcePluginConf.getTableNameField(); fetchSchema(restApi); @@ -108,7 +104,8 @@ void fetchData() throws ServiceNowAPIException { private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { // Fetch the schema try { - Schema tempSchema = restApi.fetchTableSchema(tableName, multiSourcePluginConf.getValueType()); + Schema tempSchema = restApi.fetchTableSchema(tableName, multiSourcePluginConf.getValueType(), + multiSourcePluginConf.getUseConnection()); tableFields = tempSchema.getFields(); List schemaFields = new ArrayList<>(tableFields); schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java index 511d9cc5..41d58c7c 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReader.java @@ -116,7 +116,7 @@ private void fetchData() throws ServiceNowAPIException { protected void initialize(InputSplit split) { this.split = (ServiceNowInputSplit) split; this.pos = 0; - restApi = new ServiceNowTableAPIClientImpl(pluginConf.getConnection()); + restApi = new ServiceNowTableAPIClientImpl(pluginConf.getConnection(), pluginConf.getUseConnection()); tableName = ((ServiceNowInputSplit) split).getTableName(); tableNameField = pluginConf.getTableNameField(); } diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java index 4844e662..7ed6ee55 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java @@ -83,7 +83,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { conf.getConnection(), conf.getTableName(), conf.getApplicationName(), - conf.getValueType()); + conf.getValueType(), + conf.getUseConnection()); stageConfigurer.setOutputSchema(tableInfo.stream().findFirst().get().getSchema()); } else if (conf.getQueryMode() == SourceQueryMode.REPORTING) { stageConfigurer.setOutputSchema(null); diff --git a/src/main/java/io/cdap/plugin/servicenow/util/SchemaType.java b/src/main/java/io/cdap/plugin/servicenow/util/SchemaType.java new file mode 100644 index 00000000..3903a7d0 --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/util/SchemaType.java @@ -0,0 +1,29 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.util; + +/** + * This class has all the Schema Types allowed in the plugin. Added for backward compatibility + */ +public enum SchemaType { + // All the fields will be of String type as it was in release/1.1. + STRING_BASED, + // All the fields will be based on Schema API provided by service now, will be used in newer versions for CDAP. + SCHEMA_API_BASED, + // All the fields will be based on Metadata API provided by service now, will be used only for DTS. + METADATA_API_BASED +} diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java index 8382c0a4..05d5e0d5 100644 --- a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -3,6 +3,7 @@ import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; +import io.cdap.plugin.servicenow.util.SchemaType; import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.http.HttpResponse; @@ -26,7 +27,7 @@ public class ServiceNowTableAPIClientImplTest { @Test public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws ServiceNowAPIException { ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); List> mockResults = new ArrayList<>(); mockResults.add(new HashMap() {{ @@ -61,7 +62,7 @@ public void testFetchTableRecordsRetryableMode_RetriesAndSucceeds() throws Servi public void testFetchTableRecordsRetryableMode_nonRetryable() throws ServiceNowAPIException { ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); HttpResponse mockResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); @@ -85,29 +86,28 @@ public void testFetchTableRecordsRetryableMode_nonRetryable() @Test public void testFetchTableSchema_ActualValueType() throws Exception { ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); String jsonResponse = "{\n" + - " \"result\": {\n" + - " \"columns\": {\n" + - " \"active\": {\n" + - " \"label\": \"Active\",\n" + - " \"name\": \"active\",\n" + - " \"type\": \"string\",\n" + - " \"internal_type\": \"boolean\"\n" + - " },\n" + - " \"user_name\": {\n" + - " \"label\": \"Username\",\n" + - " \"name\": \"user_name\",\n" + - " \"type\": \"string\",\n" + - " \"internal_type\": \"string\"\n" + - " }\n" + + " \"result\": [\n" + + " {\n" + + " \"internalType\": \"boolean\",\n" + + " \"label\": \"Active\",\n" + + " \"exampleValue\": \"\",\n" + + " \"name\": \"active\"\n" + + " },\n" + + " {\n" + + " \"internalType\": \"string\",\n" + + " \"label\": \"Username\",\n" + + " \"exampleValue\": \"\",\n" + + " \"name\": \"user_name\"\n" + " }\n" + - " }\n" + + " ]\n" + "}"; RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); - Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_ACTUAL_VALUE); + Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", + SourceValueType.SHOW_ACTUAL_VALUE, SchemaType.SCHEMA_API_BASED); Assert.assertNotNull(schema); Assert.assertEquals("record", schema.getDisplayName()); Assert.assertEquals(2, schema.getFields().size()); @@ -119,34 +119,34 @@ public void testFetchTableSchema_ActualValueType() throws Exception { @Test public void testFetchTableSchema_DisplayValueType() throws Exception { + ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig, true); ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); String jsonResponse = "{\n" + - " \"result\": {\n" + - " \"columns\": {\n" + - " \"active\": {\n" + - " \"label\": \"Active\",\n" + - " \"name\": \"active\",\n" + - " \"type\": \"string\",\n" + - " \"internal_type\": \"boolean\"\n" + - " },\n" + - " \"user_name\": {\n" + - " \"label\": \"Username\",\n" + - " \"name\": \"user_name\",\n" + - " \"type\": \"string\",\n" + - " \"internal_type\": \"string\"\n" + - " }\n" + + " \"result\": [\n" + + " {\n" + + " \"internalType\": \"boolean\",\n" + + " \"label\": \"Active\",\n" + + " \"exampleValue\": \"\",\n" + + " \"name\": \"active\"\n" + + " },\n" + + " {\n" + + " \"internalType\": \"string\",\n" + + " \"label\": \"Username\",\n" + + " \"exampleValue\": \"\",\n" + + " \"name\": \"user_name\"\n" + " }\n" + - " }\n" + + " ]\n" + "}"; RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); - Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE); + Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", + SourceValueType.SHOW_DISPLAY_VALUE, SchemaType.SCHEMA_API_BASED); Assert.assertNotNull(schema); Assert.assertEquals("record", schema.getDisplayName()); Assert.assertEquals(2, schema.getFields().size()); - Assert.assertEquals(Schema.Type.STRING, + Assert.assertEquals(Schema.Type.BOOLEAN, schema.getField("active").getSchema().getUnionSchemas().get(0).getType()); Assert.assertEquals(Schema.Type.STRING, schema.getField("user_name").getSchema().getUnionSchemas().get(0).getType()); diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index 2ecc805c..2240e8b5 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -97,8 +97,7 @@ public void testTest() throws Exception { ConnectorContext context = new MockConnectorContext(new MockConnectorConfigurer()); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); ServiceNowConnector serviceNowConnector = new ServiceNowConnector(serviceNowSourceConfig.getConnection()); serviceNowConnector.test(context); Assert.assertEquals(0, collector.getValidationFailures().size()); @@ -116,8 +115,7 @@ public void testTestWithInvalidToken() throws Exception { public void testGenerateSpec() throws Exception { ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map map = new HashMap<>(); List> result = new ArrayList<>(); map.put("key", "value"); @@ -137,8 +135,7 @@ public void testGenerateSpec() throws Exception { Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); - PowerMockito.whenNew(OAuthClient.class). - withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); Mockito.when(oAuthClient.accessToken(Mockito.any(), Mockito.anyString(), Mockito.any(Class.class))). thenReturn(accessTokenResponse); @@ -163,7 +160,7 @@ public void testGenerateSpec() throws Exception { SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Mockito.when(ServiceNowInputFormat.fetchTableInfo(mode, serviceNowSourceConfig.getConnection(), serviceNowSourceConfig.getTableName(), - null, valueType)).thenReturn(list); + null, valueType, true)).thenReturn(list); ConnectorSpec connectorSpec = serviceNowConnector.generateSpec(new MockConnectorContext (new MockConnectorConfigurer()), diff --git a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java index da15c5b4..b28b24ad 100644 --- a/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/restapi/RestAPIClientTest.java @@ -48,7 +48,7 @@ public void testExecuteGet_throwRetryableException() throws IOException { RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); Assert.assertTrue(actualResponse.getException().isErrorRetryable()); @@ -72,7 +72,7 @@ public void testExecuteGet_throwNonRetryableException() throws IOException { RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); RestAPIResponse actualResponse = client.executeGet(request); Assert.assertNotNull(actualResponse.getException()); Assert.assertFalse(actualResponse.getException().isErrorRetryable()); @@ -99,7 +99,7 @@ public void testExecuteGet_StatusOk() throws IOException { RestAPIRequest request = builder.build(); ServiceNowConnectorConfig config = Mockito.mock(ServiceNowConnectorConfig.class); - ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config); + ServiceNowTableAPIClientImpl client = new ServiceNowTableAPIClientImpl(config, true); client.executeGet(request); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java index 1a0ce79f..259e4582 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowRecordWriterTest.java @@ -92,8 +92,7 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { JsonObject jsonObject = new JsonObject(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map map = new HashMap<>(); List> result = new ArrayList<>(); map.put("key", "value"); @@ -104,8 +103,7 @@ public void testWriteWithUnSuccessfulApiResponse() throws Exception { Mockito.when(restApi.executePost(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); - PowerMockito.whenNew(OAuthClient.class). - withArguments(Mockito.any(URLConnectionClient.class)).thenReturn(oAuthClient); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); OAuthJSONAccessTokenResponse accessTokenResponse = Mockito.mock(OAuthJSONAccessTokenResponse.class); Mockito.when(oAuthClient.accessToken(Mockito.any(), Mockito.anyString(), Mockito.any(Class.class))). thenReturn(accessTokenResponse); @@ -136,8 +134,7 @@ public void testWriteWithSuccessFulApiResponse() throws Exception { JsonObject jsonObject = new JsonObject(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); @@ -185,8 +182,7 @@ public void testWriteWithUnservicedRequests() throws Exception { JsonObject jsonObject = new JsonObject(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); ServiceNowSinkAPIRequestImpl serviceNowSinkAPIRequest = Mockito.mock(ServiceNowSinkAPIRequestImpl.class); PowerMockito.whenNew(ServiceNowSinkAPIRequestImpl.class).withParameterTypes(ServiceNowSinkConfig.class) .withArguments(Mockito.any(ServiceNowSinkConfig.class)).thenReturn(serviceNowSinkAPIRequest); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index a0b5b826..f58702b5 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -27,9 +27,9 @@ import io.cdap.plugin.servicenow.restapi.RestAPIClient; import io.cdap.plugin.servicenow.restapi.RestAPIRequest; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; -import io.cdap.plugin.servicenow.sink.model.SchemaResponse; +import io.cdap.plugin.servicenow.sink.model.MetadataAPISchemaResponse; +import io.cdap.plugin.servicenow.sink.model.MetadataAPISchemaResult; import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; -import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaResult; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -58,7 +58,7 @@ @RunWith(PowerMockRunner.class) @PrepareForTest({RestAPIClient.class, HttpClientBuilder.class, RestAPIResponse.class, - ServiceNowTableAPIClientImpl.class}) + ServiceNowTableAPIClientImpl.class, ServiceNowSinkConfig.class}) public class ServiceNowSinkConfigTest { @Rule @@ -276,8 +276,7 @@ public void testValidateSchema() throws Exception { .build(), collector)); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withArguments(Mockito.any(ServiceNowSinkConfig.class)) - .thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Schema schema = Schema.recordOf("record", Schema.Field.of("id", Schema.of(Schema.Type.LONG)), Schema.Field.of("price", Schema.of(Schema.Type.DOUBLE))); @@ -302,8 +301,8 @@ public void testValidateSchema() throws Exception { "sys_class_name", "sys_class_name"); Map columns = new HashMap<>(); columns.put("sys_class_name", schemaField); - ServiceNowSchemaResult schemaResult = new ServiceNowSchemaResult(columns); - SchemaResponse schemaResponse = new SchemaResponse(schemaResult); + MetadataAPISchemaResult schemaResult = new MetadataAPISchemaResult(columns); + MetadataAPISchemaResponse metadataAPISchemaResponse = new MetadataAPISchemaResponse(schemaResult); HttpResponse mockResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); @@ -329,7 +328,7 @@ public void testValidateSchema() throws Exception { Mockito.when(restApi.executeGetWithRetries(Mockito.any(RestAPIRequest.class))).thenReturn(restAPIResponse); Mockito.when(restApi.fetchTableSchema(Mockito.anyString(), Mockito.any(FailureCollector.class))).thenReturn(schema); Mockito.when(restApi.parseSchemaResponse(restAPIResponse.getResponseBody())) - .thenReturn(schemaResponse); + .thenReturn(metadataAPISchemaResponse); try { config.validateSchema(schema, collector); collector.getOrThrowException(); @@ -349,8 +348,7 @@ public void testValidateSchemaWithOperation() throws Exception { .build(), collector)); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + Schema schema = Schema.recordOf("record", Schema.Field.of("sys_class_name", Schema.of(Schema.Type.STRING))); List> result = new ArrayList<>(); @@ -359,16 +357,11 @@ public void testValidateSchemaWithOperation() throws Exception { result.add(map); Map headers = new HashMap<>(); String responseBody = "{\n" + - " \"result\": {\n" + - " \"columns\": {\n" + - " \"sys_class_name\": {\n" + - " \"label\": \"Class\",\n" + - " \"internal_type\": \"sys_class_name\",\n" + - " \"name\": \"sys_class_name\",\n" + - " \"type\": \"sys_class_name\"\n" + - " }\n" + + " \"result\": [\n" + + " {\n" + + " \"sys_class_name\": \"class\"\n" + " }\n" + - " }\n" + + " ]\n" + "}"; RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java index 1a3f2942..3f748d61 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkTest.java @@ -86,8 +86,7 @@ public void testConfigurePipeline() throws Exception { MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -110,8 +109,7 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getFailureCollector()).thenReturn(mockFailureCollector); Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value"); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index bbfe8fae..2af4e573 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -70,8 +70,7 @@ public void initializeTests() { public void testFetchTableInfo() throws Exception { SourceQueryMode mode = SourceQueryMode.TABLE; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value"); @@ -163,15 +162,14 @@ public void testFetchTableInfo() throws Exception { SourceApplication application = SourceApplication.PROCUREMENT; SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertEquals(1, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType).size()); + application, valueType, true).size()); } @Test public void testFetchTableInfoReportingMode() throws Exception { SourceQueryMode mode = SourceQueryMode.REPORTING; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value"); @@ -263,15 +261,14 @@ public void testFetchTableInfoReportingMode() throws Exception { SourceApplication application = SourceApplication.PROCUREMENT; SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertEquals(4, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType).size()); + application, valueType, true).size()); } @Test public void testFetchTableInfoWithEmptyTableName() throws Exception { SourceQueryMode mode = SourceQueryMode.TABLE; ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value"); @@ -305,6 +302,6 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { SourceApplication application = SourceApplication.PROCUREMENT; SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertTrue(ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application, valueType).isEmpty()); + application, valueType, true).isEmpty()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java index 1a9d4724..2d430302 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormatTest.java @@ -70,16 +70,16 @@ public void testFetchTablesInfo() { serviceNowTableInfos.add(serviceNowTableInfo); PowerMockito.mockStatic(ServiceNowMultiInputFormat.class); SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; - PowerMockito.when(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "table")). + PowerMockito.when(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "table", true)). thenReturn(serviceNowTableInfos); Assert.assertEquals(1, ServiceNowMultiInputFormat - .fetchTablesInfo(connectorConfig, "table") + .fetchTablesInfo(connectorConfig, "table", true) .size()); } @Test public void testFetchTablesInfoWithEmptyTableNames() { SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; - Assert.assertTrue(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "").isEmpty()); + Assert.assertTrue(ServiceNowMultiInputFormat.fetchTablesInfo(connectorConfig, "", true).isEmpty()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 0169d88b..ad56697a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -141,7 +141,7 @@ public void testFetchData() throws ServiceNowAPIException, IOException { response.setResult(results); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); try { - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType())) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType(), true)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); } catch (RuntimeException @@ -196,7 +196,7 @@ public void testFetchDataOnInvalidTable() ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); try { - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType())) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType(), true)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); } catch (RuntimeException e) { diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java index 9022f3a4..c0a0302a 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceConfigTest.java @@ -20,9 +20,7 @@ import io.cdap.cdap.etl.api.validation.ValidationFailure; import io.cdap.cdap.etl.mock.validation.MockFailureCollector; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; -import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.restapi.RestAPIResponse; -import org.apache.http.HttpStatus; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -108,8 +106,7 @@ public void testValidate() throws Exception { .buildMultiSource(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); @@ -204,8 +201,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { .buildMultiSource(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -237,8 +233,7 @@ public void testValidateReferenceName() throws Exception { .buildMultiSource(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); @@ -339,8 +334,7 @@ public void testValidateWhenTableFieldNameIsEmpty() throws Exception { .buildMultiSource(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); Map map = new HashMap<>(); List> result = new ArrayList<>(); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java index a87af2f0..033f05ae 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiSourceTest.java @@ -88,8 +88,7 @@ public void testConfigurePipeline() throws Exception { MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map map = new HashMap<>(); List> result = new ArrayList<>(); map.put("key", "value"); @@ -171,8 +170,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -212,8 +210,7 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getFailureCollector()).thenReturn(mockFailureCollector); Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value"); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index 55f5e608..1a2e0729 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -277,13 +277,12 @@ public void testFetchData() throws Exception { response.setColumns(columns); response.setResult(results); response.setTotalRecordCount(1); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. getEndDate(), split.getOffset(), serviceNowSourceConfig.getPageSize())).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName, valueType)) + Mockito.when(restApi.fetchTableSchema(tableName, valueType, true)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); @@ -331,13 +330,12 @@ public void testFetchDataReportingMode() throws Exception { response.setColumns(columns); response.setResult(results); response.setTotalRecordCount(1); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecordsRetryableMode(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), serviceNowSourceConfig.getPageSize())).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType(), true)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); @@ -364,15 +362,14 @@ public void testFetchDataOnInvalidTable() throws Exception { ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); List> results = new ArrayList<>(); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Mockito.when(restApi.fetchTableRecords(tableName, serviceNowSourceConfig.getValueType(), serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), serviceNowSourceConfig.getPageSize())).thenReturn(results); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); - Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType(), true)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertFalse(serviceNowRecordReader.nextKeyValue()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java index 7a11cce9..d9bc48c6 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceConfigTest.java @@ -618,8 +618,7 @@ public void testValidateWhenTableIsEmpty() throws Exception { .build(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -648,8 +647,7 @@ public void testValidateWhenTableNameIsInvalid() throws Exception { .build(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); String errorMessage = "Http call returned 400 response code"; HttpResponse mockResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 7bbae49e..29871ad1 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -100,8 +100,7 @@ public void testConfigurePipeline() throws Exception { MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token1"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); Map map = new HashMap<>(); List> result = new ArrayList<>(); map.put("key", "value"); @@ -171,7 +170,7 @@ public void testConfigurePipeline() throws Exception { "}"; PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), - Mockito.any(), Mockito.any())).thenReturn(tableInfo); + Mockito.any(), Mockito.any(), Mockito.any())).thenReturn(tableInfo); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result); @@ -204,8 +203,7 @@ public void testConfigurePipelineWithEmptyTable() throws Exception { MockPipelineConfigurer mockPipelineConfigurer = new MockPipelineConfigurer(null, plugins); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); Mockito.when(restApi.getAccessToken()).thenReturn("token"); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map headers = new HashMap<>(); String responseBody = "{\n" + @@ -231,8 +229,7 @@ public void testPrepareRun() throws Exception { Mockito.when(context.getFailureCollector()).thenReturn(mockFailureCollector); Mockito.when(context.getArguments()).thenReturn(mockArguments); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); - PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withParameterTypes(ServiceNowConnectorConfig.class) - .withArguments(Mockito.any(ServiceNowConnectorConfig.class)).thenReturn(restApi); + PowerMockito.whenNew(ServiceNowTableAPIClientImpl.class).withAnyArguments().thenReturn(restApi); List> result = new ArrayList<>(); Map map = new HashMap<>(); map.put("key", "value");