diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java index 2971be6f..3c0afaa1 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImpl.java @@ -48,15 +48,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.io.UnsupportedEncodingException; import java.lang.reflect.Type; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; import javax.annotation.Nullable; /** @@ -247,6 +246,7 @@ public List> fetchTableRecordsRetryableMode(String tableName } /** + * Fetch schema for actual value type * @param tableName ServiceNow table name for which schema is getting fetched * @param collector FailureCollector * @return schema for given ServiceNow table @@ -255,7 +255,7 @@ public List> fetchTableRecordsRetryableMode(String tableName public Schema fetchTableSchema(String tableName, FailureCollector collector) { Schema schema = null; try { - schema = fetchTableSchema(tableName); + schema = fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE); } catch (Exception e) { LOG.error("Failed to fetch schema on table {}", tableName, e); collector.addFailure(String.format("Connection failed. Unable to fetch schema for table: %s. Cause: %s", @@ -276,9 +276,9 @@ public SchemaResponse parseSchemaResponse(String responseBody) { * @return schema for given ServiceNow table * @throws ServiceNowAPIException */ - public Schema fetchTableSchema(String tableName) + public Schema fetchTableSchema(String tableName, SourceValueType valueType) throws ServiceNowAPIException { - return fetchTableSchema(tableName, getAccessToken()); + return fetchTableSchema(tableName, getAccessToken(), valueType); } /** @@ -286,25 +286,32 @@ public Schema fetchTableSchema(String tableName) * * @param tableName ServiceNow table name for which schema is getting fetched * @param accessToken Access Token to use + * @param valueType Type of value (Actual/Display) * @return schema for given ServiceNow table */ - public Schema fetchTableSchema(String tableName, String accessToken) + public Schema fetchTableSchema(String tableName, String accessToken, SourceValueType valueType) throws ServiceNowAPIException { ServiceNowTableAPIRequestBuilder requestBuilder = new ServiceNowTableAPIRequestBuilder( this.conf.getRestApiEndpoint(), tableName, true) .setExcludeReferenceLink(true); - RestAPIResponse apiResponse; + RestAPIResponse restAPIResponse; requestBuilder.setAuthHeader(accessToken); - apiResponse = executeGetWithRetries(requestBuilder.build()); - SchemaResponse response = parseSchemaResponse(apiResponse.getResponseBody()); + restAPIResponse = executeGetWithRetries(requestBuilder.build()); + SchemaResponse schemaResponse = parseSchemaResponse(restAPIResponse.getResponseBody()); List columns = new ArrayList<>(); - if (response.getResult() == null && response.getResult().isEmpty()) { + if (schemaResponse.getResult() == null && schemaResponse.getResult().getColumns().isEmpty()) { throw new RuntimeException("Error - Schema Response does not contain any result"); } - for (ServiceNowSchemaField field : response.getResult()) { - columns.add(new ServiceNowColumn(field.getName(), field.getInternalType())); + + for (ServiceNowSchemaField field : schemaResponse.getResult().getColumns().values()) { + if (valueType.equals(SourceValueType.SHOW_DISPLAY_VALUE) && + !Objects.equals(field.getType(), field.getInternalType())) { + columns.add(new ServiceNowColumn(field.getName(), field.getType())); + } else { + columns.add(new ServiceNowColumn(field.getName(), field.getInternalType())); + } } return SchemaBuilder.constructSchema(tableName, columns); } diff --git a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java index d1f905b2..aa654cf9 100644 --- a/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java +++ b/src/main/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIRequestBuilder.java @@ -43,9 +43,14 @@ public class ServiceNowTableAPIRequestBuilder extends RestAPIRequest.Builder { */ private static final String SCHEMA_API_URL_TEMPLATE = "%s/api/now/doc/table/schema/%s"; + /** + * ServiceNow API URL to fetch column metadata + */ + private static final String METADATA_API_URL_TEMPLATE = "%s/api/now/ui/meta/%s"; + public ServiceNowTableAPIRequestBuilder(String instanceBaseUrl, String tableName, boolean isSchemaRequired) { if (isSchemaRequired) { - this.setUrl(String.format(SCHEMA_API_URL_TEMPLATE, instanceBaseUrl, tableName)); + this.setUrl(String.format(METADATA_API_URL_TEMPLATE, instanceBaseUrl, tableName)); } else { this.setUrl(String.format(TABLE_API_URL_TEMPLATE, instanceBaseUrl, tableName)); } diff --git a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java index b2331eed..fbf5dda4 100644 --- a/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java +++ b/src/main/java/io/cdap/plugin/servicenow/connector/ServiceNowConnector.java @@ -16,7 +16,6 @@ package io.cdap.plugin.servicenow.connector; import com.google.gson.Gson; -import com.google.gson.JsonObject; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Name; import io.cdap.cdap.api.annotation.Plugin; @@ -206,8 +205,14 @@ private List getTableData(String tableName, int limit) @Nullable private Schema getSchema(String tableName) { SourceQueryMode mode = SourceQueryMode.TABLE; - List tableInfo = ServiceNowInputFormat.fetchTableInfo(mode, config, tableName, - null); + // Use display type schema as connector shows a limited number of values + // and display value type provides easy to read values + List tableInfo = ServiceNowInputFormat.fetchTableInfo( + mode, + config, + tableName, + null, + SourceValueType.SHOW_DISPLAY_VALUE); Schema schema = tableInfo.stream().findFirst().isPresent() ? tableInfo.stream().findFirst().get().getSchema() : null; return schema; diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java index ea5191ef..ec85f245 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/SchemaResponse.java @@ -16,20 +16,18 @@ package io.cdap.plugin.servicenow.sink.model; -import java.util.List; - /** * Model class for Schema Response from Schema API */ public class SchemaResponse { - private final List result; + private final ServiceNowSchemaResult result; - public SchemaResponse(List result) { + public SchemaResponse(ServiceNowSchemaResult result) { this.result = result; } - public List getResult() { + public ServiceNowSchemaResult getResult() { return result; } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java index e3c44c38..80209e06 100644 --- a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaField.java @@ -16,30 +16,29 @@ package io.cdap.plugin.servicenow.sink.model; +import com.google.gson.annotations.SerializedName; + /** * Model class for Schema Field from Schema API */ public class ServiceNowSchemaField { private final String label; - private final String exampleValue; + @SerializedName("internal_type") private final String internalType; private final String name; + private final String type; - public ServiceNowSchemaField(String label, String exampleValue, String internalType, String name) { + public ServiceNowSchemaField(String label, String internalType, String name, String type) { this.label = label; - this.exampleValue = exampleValue; this.internalType = internalType; this.name = name; + this.type = type; } public String getLabel() { return label; } - public String getExampleValue() { - return exampleValue; - } - public String getInternalType() { return internalType; } @@ -47,4 +46,8 @@ public String getInternalType() { public String getName() { return name; } + + public String getType() { + return type; + } } diff --git a/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java new file mode 100644 index 00000000..80b11440 --- /dev/null +++ b/src/main/java/io/cdap/plugin/servicenow/sink/model/ServiceNowSchemaResult.java @@ -0,0 +1,63 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.servicenow.sink.model; + +import java.util.Map; + +/** + * Model class for Schema Result returned by the ServiceNow Column Schema API. + * + *

The {@code columns} map contains metadata for each column in the ServiceNow table. + * The key of the map is the column's internal name (as used in the table schema), + * and the value is a {@link ServiceNowSchemaField} object containing the details for that column. + * + *

Example JSON from ServiceNow: + * 

+ * {
+ *   "result": {
+ *     "columns": {
+ *       "state": {
+ *         "label": "State",
+ *         "type": "string",
+ *         "internal_type": "integer",
+ *         "name": "state"
+ *       },
+ *       "active": {
+ *         "label": "Active",
+ *         "type": "boolean",
+ *         "internal_type": "boolean",
+ *         "name": "active"
+ *       }
+ *     }
+ *   }
+ * }
+ *
+ * + * In this example, the map will contain keys like {@code "state"} and {@code "active"}, + * each pointing to a {@code ServiceNowSchemaField} instance with metadata about that field. + */ +public class ServiceNowSchemaResult { + private final Map columns; + + public ServiceNowSchemaResult(Map columns) { + this.columns = columns; + } + + public Map getColumns() { + return columns; + } +} diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java index a77ad335..190f2272 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormat.java @@ -21,10 +21,10 @@ import io.cdap.plugin.servicenow.apiclient.ServiceNowAPIException; import io.cdap.plugin.servicenow.apiclient.ServiceNowTableAPIClientImpl; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; -import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; +import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -32,8 +32,6 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.RecordReader; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.oltu.oauth2.common.exception.OAuthProblemException; -import org.apache.oltu.oauth2.common.exception.OAuthSystemException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,7 +63,7 @@ public static List setInput(Configuration jobConfig, Source // Depending on conf value fetch the list of fields for each table and create schema object // return the schema object for each table as ServiceNowTableInfo List tableInfos = fetchTableInfo(mode, conf.getConnection(), conf.getTableName(), - conf.getApplicationName()); + conf.getApplicationName(), conf.getValueType()); jobConf.setTableInfos(tableInfos); return tableInfos; @@ -73,10 +71,11 @@ public static List setInput(Configuration jobConfig, Source public static List fetchTableInfo(SourceQueryMode mode, ServiceNowConnectorConfig conf, @Nullable String tableName, - @Nullable SourceApplication application) { + @Nullable SourceApplication application, + @Nullable SourceValueType valueType) { // When mode = Table, fetch details from the table name provided in plugin config if (mode == SourceQueryMode.TABLE) { - ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf); + ServiceNowTableInfo tableInfo = getTableMetaData(tableName, conf, valueType); return (tableInfo == null) ? Collections.emptyList() : Collections.singletonList(tableInfo); } @@ -86,7 +85,7 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser List tableNames = application.getTableNames(); for (String table : tableNames) { - ServiceNowTableInfo tableInfo = getTableMetaData(table, conf); + ServiceNowTableInfo tableInfo = getTableMetaData(table, conf, valueType); if (tableInfo == null) { continue; } @@ -96,14 +95,16 @@ public static List fetchTableInfo(SourceQueryMode mode, Ser return tableInfos; } - private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNowConnectorConfig conf) { + private static ServiceNowTableInfo getTableMetaData(String tableName, + ServiceNowConnectorConfig conf, + SourceValueType valueType) { // Call API to fetch first record from the table ServiceNowTableAPIClientImpl restApi = new ServiceNowTableAPIClientImpl(conf); Schema schema = null; int recordCount = 0; try { - schema = restApi.fetchTableSchema(tableName); + schema = restApi.fetchTableSchema(tableName, valueType); recordCount = restApi.getTableRecordCount(tableName); } catch (ServiceNowAPIException e) { throw new RuntimeException(String.format("Error in fetching table metadata due to reason: %s", e.getMessage()), diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java index 0d492c6a..96140ba1 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiInputFormat.java @@ -24,6 +24,7 @@ import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.ServiceNowTableInfo; +import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; @@ -96,7 +97,10 @@ private static ServiceNowTableInfo getTableMetaData(String tableName, ServiceNow Schema schema; int recordCount; try { - schema = restApi.fetchTableSchema(tableName); + // Use actual value type as connector config does not have an option to select value type + // This is used for ServiceNowMultiSource and provides structure of the table and being dependent on + // connector config, makes it a read only function + schema = restApi.fetchTableSchema(tableName, SourceValueType.SHOW_ACTUAL_VALUE); recordCount = restApi.getTableRecordCount(tableName); } catch (ServiceNowAPIException e) { throw new RuntimeException(e); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java index 09f87e69..4f8b56dd 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReader.java @@ -108,7 +108,7 @@ void fetchData() throws ServiceNowAPIException { private void fetchSchema(ServiceNowTableAPIClientImpl restApi) { // Fetch the schema try { - Schema tempSchema = restApi.fetchTableSchema(tableName); + Schema tempSchema = restApi.fetchTableSchema(tableName, multiSourcePluginConf.getValueType()); tableFields = tempSchema.getFields(); List schemaFields = new ArrayList<>(tableFields); schemaFields.add(Schema.Field.of(tableNameField, Schema.of(Schema.Type.STRING))); diff --git a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java index ec9babda..4844e662 100644 --- a/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java +++ b/src/main/java/io/cdap/plugin/servicenow/source/ServiceNowSource.java @@ -82,7 +82,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { List tableInfo = ServiceNowInputFormat.fetchTableInfo(conf.getQueryMode(collector), conf.getConnection(), conf.getTableName(), - conf.getApplicationName()); + conf.getApplicationName(), + conf.getValueType()); stageConfigurer.setOutputSchema(tableInfo.stream().findFirst().get().getSchema()); } else if (conf.getQueryMode() == SourceQueryMode.REPORTING) { stageConfigurer.setOutputSchema(null); diff --git a/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java b/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java index 2b3701da..9ef719da 100644 --- a/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java +++ b/src/main/java/io/cdap/plugin/servicenow/util/SchemaBuilder.java @@ -86,6 +86,7 @@ private Schema createSchema(ServiceNowColumn column) { case "guid": case "translated_html": case "journal": + case "choice": case "string": default: return Schema.of(Schema.Type.STRING); diff --git a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java index 99640a5b..8382c0a4 100644 --- a/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/apiclient/ServiceNowTableAPIClientImplTest.java @@ -1,6 +1,8 @@ package io.cdap.plugin.servicenow.apiclient; +import io.cdap.cdap.api.data.schema.Schema; import io.cdap.plugin.servicenow.connector.ServiceNowConnectorConfig; +import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.http.HttpResponse; @@ -12,8 +14,8 @@ import org.junit.rules.ExpectedException; import org.mockito.Mockito; -import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,4 +81,74 @@ public void testFetchTableRecordsRetryableMode_nonRetryable() implSpy.fetchTableRecordsRetryableMode( "test", SourceValueType.SHOW_DISPLAY_VALUE, "", "", 0, 0); } + + @Test + public void testFetchTableSchema_ActualValueType() throws Exception { + ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); + String jsonResponse = "{\n" + + " \"result\": {\n" + + " \"columns\": {\n" + + " \"active\": {\n" + + " \"label\": \"Active\",\n" + + " \"name\": \"active\",\n" + + " \"type\": \"string\",\n" + + " \"internal_type\": \"boolean\"\n" + + " },\n" + + " \"user_name\": {\n" + + " \"label\": \"Username\",\n" + + " \"name\": \"user_name\",\n" + + " \"type\": \"string\",\n" + + " \"internal_type\": \"string\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); + Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_ACTUAL_VALUE); + Assert.assertNotNull(schema); + Assert.assertEquals("record", schema.getDisplayName()); + Assert.assertEquals(2, schema.getFields().size()); + Assert.assertEquals(Schema.Type.BOOLEAN, + schema.getField("active").getSchema().getUnionSchemas().get(0).getType()); + Assert.assertEquals(Schema.Type.STRING, + schema.getField("user_name").getSchema().getUnionSchemas().get(0).getType()); + } + + @Test + public void testFetchTableSchema_DisplayValueType() throws Exception { + ServiceNowConnectorConfig mockConfig = Mockito.mock(ServiceNowConnectorConfig.class); + ServiceNowTableAPIClientImpl impl = new ServiceNowTableAPIClientImpl(mockConfig); + ServiceNowTableAPIClientImpl implSpy = Mockito.spy(impl); + String jsonResponse = "{\n" + + " \"result\": {\n" + + " \"columns\": {\n" + + " \"active\": {\n" + + " \"label\": \"Active\",\n" + + " \"name\": \"active\",\n" + + " \"type\": \"string\",\n" + + " \"internal_type\": \"boolean\"\n" + + " },\n" + + " \"user_name\": {\n" + + " \"label\": \"Username\",\n" + + " \"name\": \"user_name\",\n" + + " \"type\": \"string\",\n" + + " \"internal_type\": \"string\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"; + RestAPIResponse mockResponse = new RestAPIResponse(Collections.emptyMap(), jsonResponse, null); + Mockito.doReturn(mockResponse).when(implSpy).executeGetWithRetries(Mockito.any()); + Schema schema = implSpy.fetchTableSchema("sys_user", "dummy-access-token", SourceValueType.SHOW_DISPLAY_VALUE); + Assert.assertNotNull(schema); + Assert.assertEquals("record", schema.getDisplayName()); + Assert.assertEquals(2, schema.getFields().size()); + Assert.assertEquals(Schema.Type.STRING, + schema.getField("active").getSchema().getUnionSchemas().get(0).getType()); + Assert.assertEquals(Schema.Type.STRING, + schema.getField("user_name").getSchema().getUnionSchemas().get(0).getType()); + } } diff --git a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java index 1d2c3733..2ecc805c 100644 --- a/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/connector/ServiceNowConnectorTest.java @@ -163,7 +163,7 @@ public void testGenerateSpec() throws Exception { SourceValueType valueType = SourceValueType.SHOW_DISPLAY_VALUE; Mockito.when(ServiceNowInputFormat.fetchTableInfo(mode, serviceNowSourceConfig.getConnection(), serviceNowSourceConfig.getTableName(), - null)).thenReturn(list); + null, valueType)).thenReturn(list); ConnectorSpec connectorSpec = serviceNowConnector.generateSpec(new MockConnectorContext (new MockConnectorConfigurer()), diff --git a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java index 42841db4..a0b5b826 100644 --- a/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/sink/ServiceNowSinkConfigTest.java @@ -29,6 +29,7 @@ import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.sink.model.SchemaResponse; import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaField; +import io.cdap.plugin.servicenow.sink.model.ServiceNowSchemaResult; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; @@ -299,9 +300,10 @@ public void testValidateSchema() throws Exception { "}"; ServiceNowSchemaField schemaField = new ServiceNowSchemaField("Class", "sys_class_name", "sys_class_name", "sys_class_name"); - List schemaFields = new ArrayList<>(); - schemaFields.add(schemaField); - SchemaResponse schemaResponse = new SchemaResponse(schemaFields); + Map columns = new HashMap<>(); + columns.put("sys_class_name", schemaField); + ServiceNowSchemaResult schemaResult = new ServiceNowSchemaResult(columns); + SchemaResponse schemaResponse = new SchemaResponse(schemaResult); HttpResponse mockResponse = Mockito.mock(HttpResponse.class); Mockito.when(mockResponse.getStatusLine()).thenReturn(Mockito.mock(StatusLine.class)); Mockito.when(mockResponse.getStatusLine().getStatusCode()).thenReturn(httpStatus); @@ -357,14 +359,16 @@ public void testValidateSchemaWithOperation() throws Exception { result.add(map); Map headers = new HashMap<>(); String responseBody = "{\n" + - " \"result\": [\n" + - " {\n" + - " \"label\": \"Class\",\n" + - " \"internalType\": \"sys_class_name\",\n" + - " \"exampleValue\": \"\",\n" + - " \"name\": \"sys_class_name\"\n" + - " }\n" + - " ]\n" + + " \"result\": {\n" + + " \"columns\": {\n" + + " \"sys_class_name\": {\n" + + " \"label\": \"Class\",\n" + + " \"internal_type\": \"sys_class_name\",\n" + + " \"name\": \"sys_class_name\",\n" + + " \"type\": \"sys_class_name\"\n" + + " }\n" + + " }\n" + + " }\n" + "}"; RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); OAuthClient oAuthClient = Mockito.mock(OAuthClient.class); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java index f6e2feab..bbfe8fae 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowInputFormatTest.java @@ -22,6 +22,7 @@ import io.cdap.plugin.servicenow.restapi.RestAPIResponse; import io.cdap.plugin.servicenow.util.SourceApplication; import io.cdap.plugin.servicenow.util.SourceQueryMode; +import io.cdap.plugin.servicenow.util.SourceValueType; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.impl.client.CloseableHttpClient; @@ -160,8 +161,9 @@ public void testFetchTableInfo() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; + SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertEquals(1, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application).size()); + application, valueType).size()); } @Test @@ -259,8 +261,9 @@ public void testFetchTableInfoReportingMode() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; + SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertEquals(4, ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application).size()); + application, valueType).size()); } @Test @@ -300,7 +303,8 @@ public void testFetchTableInfoWithEmptyTableName() throws Exception { PowerMockito.when(RestAPIResponse.parse(ArgumentMatchers.any(), ArgumentMatchers.anyString())). thenReturn(response); SourceApplication application = SourceApplication.PROCUREMENT; + SourceValueType valueType = SourceValueType.SHOW_ACTUAL_VALUE; Assert.assertTrue(ServiceNowInputFormat.fetchTableInfo(mode, connectorConfig, "table", - application).isEmpty()); + application, valueType).isEmpty()); } } diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java index 975d367f..0169d88b 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowMultiRecordReaderTest.java @@ -141,7 +141,7 @@ public void testFetchData() throws ServiceNowAPIException, IOException { response.setResult(results); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); try { - Mockito.when(restApi.fetchTableSchema(tableName)) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); } catch (RuntimeException @@ -196,7 +196,7 @@ public void testFetchDataOnInvalidTable() ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); try { - Mockito.when(restApi.fetchTableSchema(tableName)) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowMultiSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowMultiRecordReader.initialize(split, null); } catch (RuntimeException e) { diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java index f518fc28..55f5e608 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowRecordReaderTest.java @@ -28,6 +28,7 @@ import io.cdap.plugin.servicenow.util.ServiceNowColumn; import io.cdap.plugin.servicenow.util.ServiceNowConstants; import io.cdap.plugin.servicenow.util.SourceQueryMode; +import io.cdap.plugin.servicenow.util.SourceValueType; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -252,6 +253,7 @@ public void testConvertToBooleanValueForInvalidFieldValue() { @Test public void testFetchData() throws Exception { String tableName = serviceNowSourceConfig.getTableName(); + SourceValueType valueType = serviceNowSourceConfig.getValueType(); ServiceNowTableAPIClientImpl restApi = Mockito.mock(ServiceNowTableAPIClientImpl.class); ServiceNowInputSplit split = new ServiceNowInputSplit(tableName, 1); ServiceNowRecordReader serviceNowRecordReader = new ServiceNowRecordReader(serviceNowSourceConfig); @@ -281,7 +283,7 @@ public void testFetchData() throws Exception { serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig. getEndDate(), split.getOffset(), serviceNowSourceConfig.getPageSize())).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName)) + Mockito.when(restApi.fetchTableSchema(tableName, valueType)) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); @@ -335,7 +337,7 @@ public void testFetchDataReportingMode() throws Exception { serviceNowSourceConfig.getStartDate(), serviceNowSourceConfig.getEndDate(), split.getOffset(), serviceNowSourceConfig.getPageSize())).thenReturn(results); - Mockito.when(restApi.fetchTableSchema(tableName)) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertTrue(serviceNowRecordReader.nextKeyValue()); @@ -370,7 +372,7 @@ public void testFetchDataOnInvalidTable() throws Exception { serviceNowSourceConfig.getPageSize())).thenReturn(results); ServiceNowTableDataResponse response = new ServiceNowTableDataResponse(); response.setResult(results); - Mockito.when(restApi.fetchTableSchema(tableName)) + Mockito.when(restApi.fetchTableSchema(tableName, serviceNowSourceConfig.getValueType())) .thenReturn(Schema.recordOf(Schema.Field.of("calendar_integration", Schema.of(Schema.Type.STRING)))); serviceNowRecordReader.initialize(split); Assert.assertFalse(serviceNowRecordReader.nextKeyValue()); diff --git a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java index 0d7a8dda..7bbae49e 100644 --- a/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java +++ b/src/test/java/io/cdap/plugin/servicenow/source/ServiceNowSourceTest.java @@ -171,7 +171,7 @@ public void testConfigurePipeline() throws Exception { "}"; PowerMockito.mockStatic(ServiceNowInputFormat.class); Mockito.when(ServiceNowInputFormat.fetchTableInfo(Mockito.any(), Mockito.any(), Mockito.anyString(), - Mockito.any())).thenReturn(tableInfo); + Mockito.any(), Mockito.any())).thenReturn(tableInfo); RestAPIResponse restAPIResponse = new RestAPIResponse(headers, responseBody, null); Mockito.when(restApi.executeGetWithRetries(Mockito.any())).thenReturn(restAPIResponse); Mockito.when(restApi.parseResponseToResultListOfMap(restAPIResponse.getResponseBody())).thenReturn(result);