diff --git a/pom.xml b/pom.xml index c22dd0c4..2befa87a 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ Salesforce plugins io.cdap.plugin salesforce-plugins - 1.6.8 + 1.6.9 jar Salesforce Plugins https://github.com/data-integrations/salesforce diff --git a/src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java b/src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java index 06da81a0..68166d58 100644 --- a/src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java +++ b/src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java @@ -127,11 +127,11 @@ public static void verifyNoErrorsFoundSuccessMessage() { } private static AuthenticatorCredentials setAuthenticationCredentialsOfAdminUser() { - return new AuthenticatorCredentials(PluginPropertyUtils.pluginProp("admin.username"), + return new AuthenticatorCredentials(null, PluginPropertyUtils.pluginProp("admin.username"), PluginPropertyUtils.pluginProp("admin.password"), PluginPropertyUtils.pluginProp("admin.consumer.key"), PluginPropertyUtils.pluginProp("admin.consumer.secret"), - PluginPropertyUtils.pluginProp("login.url"), null, null, null + PluginPropertyUtils.pluginProp("login.url"), null, null, null, null ); } diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 810280ca..73ad4b8e 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -60,51 +60,170 @@ simple.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string ChildToParent.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string"},\ {"key":"Account_Name","value":"string"}] -lead.schema=[{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\ - {"key":"MasterRecordId","value":"string"},{"key":"LastName","value":"string"},\ - {"key":"FirstName","value":"string"},{"key":"Salutation","value":"string"},\ - {"key":"MiddleName","value":"string"},{"key":"Suffix","value":"string"},{"key":"Name","value":"string"},\ - {"key":"Title","value":"string"},{"key":"Company","value":"string"},\ - {"key":"Street","value":"string"},\ - {"key":"City","value":"string"},{"key":"State","value":"string"},\ - {"key":"PostalCode","value":"string"},{"key":"Country","value":"string"},\ - {"key":"Latitude","value":"double"},{"key":"Longitude","value":"double"},\ - {"key":"GeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\ - {"key":"MobilePhone","value":"string"},{"key":"Email","value":"string"},\ - {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\ - {"key":"LeadSource","value":"string"},{"key":"Status","value":"string"},\ - {"key":"Industry","value":"string"},{"key":"Rating","value":"string"},\ - {"key":"NumberOfEmployees","value":"int"},{"key":"OwnerId","value":"string"},\ - {"key":"IsConverted","value":"boolean"},{"key":"ConvertedDate","value":"date"},\ - {"key":"ConvertedAccountId","value":"string"},\ - {"key":"ConvertedContactId","value":"string"},{"key":"ConvertedOpportunityId","value":"string"},\ - {"key":"IsUnreadByOwner","value":"boolean"},{"key":"CreatedDate","value":"timestamp"},\ - {"key":"CreatedById","value":"string"},{"key":"LastModifiedDate","value":"timestamp"},\ - {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\ - {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\ - {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\ - {"key":"JigsawContactId","value":"string"},{"key":"EmailBouncedReason","value":"string"},\ - {"key":"EmailBouncedDate","value":"timestamp"}] +lead.schema=[{"key":"Id","value":"string"},\ +{"key":"IsDeleted","value":"boolean"},\ +{"key":"MasterRecordId","value":"string"},\ +{"key":"LastName","value":"string"},\ +{"key":"FirstName","value":"string"},\ +{"key":"Salutation","value":"string"},\ +{"key":"Name","value":"string"},\ +{"key":"Title","value":"string"},\ +{"key":"Company","value":"string"},\ +{"key":"Street","value":"string"},\ +{"key":"City","value":"string"},\ +{"key":"State","value":"string"},\ +{"key":"PostalCode","value":"string"},\ +{"key":"Country","value":"string"},\ +{"key":"Latitude","value":"double"},\ +{"key":"Longitude","value":"double"},\ +{"key":"GeocodeAccuracy","value":"string"},\ +{"key":"Phone","value":"string"},\ +{"key":"MobilePhone","value":"string"},\ +{"key":"Fax","value":"string"},\ +{"key":"Email","value":"string"},\ +{"key":"Website","value":"string"},\ +{"key":"PhotoUrl","value":"string"},\ +{"key":"Description","value":"string"},\ +{"key":"LeadSource","value":"string"},\ +{"key":"Status","value":"string"},\ +{"key":"Industry","value":"string"},\ +{"key":"Rating","value":"string"},\ +{"key":"AnnualRevenue","value":"double"},\ +{"key":"NumberOfEmployees","value":"int"},\ +{"key":"OwnerId","value":"string"},\ +{"key":"IsConverted","value":"boolean"},\ +{"key":"ConvertedDate","value":"date"},\ +{"key":"ConvertedAccountId","value":"string"},\ +{"key":"ConvertedContactId","value":"string"},\ +{"key":"ConvertedOpportunityId","value":"string"},\ +{"key":"IsUnreadByOwner","value":"boolean"},\ +{"key":"CreatedDate","value":"timestamp"},\ +{"key":"CreatedById","value":"string"},\ +{"key":"LastModifiedDate","value":"timestamp"},\ +{"key":"LastModifiedById","value":"string"},\ +{"key":"SystemModstamp","value":"timestamp"},\ +{"key":"LastActivityDate","value":"date"},\ +{"key":"LastViewedDate","value":"timestamp"},\ +{"key":"LastReferencedDate","value":"timestamp"},\ +{"key":"Jigsaw","value":"string"},\ +{"key":"JigsawContactId","value":"string"},\ +{"key":"EmailBouncedReason","value":"string"},\ +{"key":"EmailBouncedDate","value":"timestamp"},\ +{"key":"IndividualId","value":"string"},\ +{"key":"Follow_Up__c","value":"boolean"},\ +{"key":"npe01__Preferred_Email__c","value":"string"},\ +{"key":"npe01__Preferred_Phone__c","value":"string"},\ +{"key":"npsp__Batch__c","value":"string"},\ +{"key":"npsp__CompanyCity__c","value":"string"},\ +{"key":"npsp__CompanyCountry__c","value":"string"},\ +{"key":"npsp__CompanyPostalCode__c","value":"string"},\ +{"key":"npsp__CompanyState__c","value":"string"},\ +{"key":"npsp__CompanyStreet__c","value":"string"}] -account.schema= [{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\ - {"key":"MasterRecordId","value":"string"},{"key":"Name","value":"string"},\ - {"key":"Type","value":"string"},\ - {"key":"ParentId","value":"string"},{"key":"BillingStreet","value":"string"},{"key":"BillingCity","value":"string"},\ - {"key":"BillingState","value":"string"},{"key":"BillingPostalCode","value":"string"},\ - {"key":"BillingCountry","value":"string"},\ - {"key":"BillingLatitude","value":"double"},{"key":"BillingLongitude","value":"double"},\ - {"key":"BillingGeocodeAccuracy","value":"string"},{"key":"ShippingStreet","value":"string"},\ - {"key":"ShippingCity","value":"string"},{"key":"ShippingState","value":"string"},\ - {"key":"ShippingPostalCode","value":"string"},{"key":"ShippingCountry","value":"string"},\ - {"key":"ShippingLatitude","value":"double"},{"key":"ShippingLongitude","value":"double"},\ - {"key":"ShippingGeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\ - {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\ - {"key":"Industry","value":"string"},\ - {"key":"NumberOfEmployees","value":"int"},{"key":"Description","value":"string"},{"key":"OwnerId","value":"string"},\ - {"key":"CreatedDate","value":"timestamp"},{"key":"CreatedById","value":"string"},\ - {"key":"LastModifiedDate","value":"timestamp"},\ - {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\ - {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\ - {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\ - {"key":"JigsawCompanyId","value":"string"},{"key":"AccountSource","value":"string"},\ - {"key":"SicDesc","value":"string"}] +account.schema= [{"key":"Id","value":"string"},\ +{"key":"IsDeleted","value":"boolean"},\ +{"key":"MasterRecordId","value":"string"},\ +{"key":"Name","value":"string"},\ +{"key":"Type","value":"string"},\ +{"key":"RecordTypeId","value":"string"},\ +{"key":"ParentId","value":"string"},\ +{"key":"BillingStreet","value":"string"},\ +{"key":"BillingCity","value":"string"},\ +{"key":"BillingState","value":"string"},\ +{"key":"BillingPostalCode","value":"string"},\ +{"key":"BillingCountry","value":"string"},\ +{"key":"BillingLatitude","value":"double"},\ +{"key":"BillingLongitude","value":"double"},\ +{"key":"BillingGeocodeAccuracy","value":"string"},\ +{"key":"ShippingStreet","value":"string"},\ +{"key":"ShippingCity","value":"string"},\ +{"key":"ShippingState","value":"string"},\ +{"key":"ShippingPostalCode","value":"string"},\ +{"key":"ShippingCountry","value":"string"},\ +{"key":"ShippingLatitude","value":"double"},\ +{"key":"ShippingLongitude","value":"double"},\ +{"key":"ShippingGeocodeAccuracy","value":"string"},\ +{"key":"Phone","value":"string"},\ +{"key":"Fax","value":"string"},\ +{"key":"AccountNumber","value":"string"},\ +{"key":"Website","value":"string"},\ +{"key":"PhotoUrl","value":"string"},\ +{"key":"Sic","value":"string"},\ +{"key":"Industry","value":"string"},\ +{"key":"AnnualRevenue","value":"double"},\ +{"key":"NumberOfEmployees","value":"int"},\ +{"key":"Ownership","value":"string"},\ +{"key":"TickerSymbol","value":"string"},\ +{"key":"Description","value":"string"},\ +{"key":"Rating","value":"string"},\ +{"key":"Site","value":"string"},\ +{"key":"OwnerId","value":"string"},\ +{"key":"CreatedDate","value":"timestamp"},\ +{"key":"CreatedById","value":"string"},\ +{"key":"LastModifiedDate","value":"timestamp"},\ +{"key":"LastModifiedById","value":"string"},\ +{"key":"SystemModstamp","value":"timestamp"},\ +{"key":"LastActivityDate","value":"date"},\ +{"key":"LastViewedDate","value":"timestamp"},\ +{"key":"LastReferencedDate","value":"timestamp"},\ +{"key":"Jigsaw","value":"string"},\ +{"key":"JigsawCompanyId","value":"string"},\ +{"key":"AccountSource","value":"string"},\ +{"key":"SicDesc","value":"string"},\ +{"key":"Copy_Billing_Address_to_Shipping_Address__c","value":"boolean"},\ +{"key":"Test_External_Id__c","value":"string"},\ +{"key":"custom_field__c","value":"string"},\ +{"key":"npe01__One2OneContact__c","value":"string"},\ +{"key":"npe01__SYSTEMIsIndividual__c","value":"boolean"},\ +{"key":"npe01__SYSTEM_AccountType__c","value":"string"},\ +{"key":"npo02__AverageAmount__c","value":"double"},\ +{"key":"npo02__Best_Gift_Year_Total__c","value":"double"},\ +{"key":"npo02__Best_Gift_Year__c","value":"string"},\ +{"key":"npo02__FirstCloseDate__c","value":"date"},\ +{"key":"npo02__Formal_Greeting__c","value":"string"},\ +{"key":"npo02__HouseholdPhone__c","value":"string"},\ +{"key":"npo02__Informal_Greeting__c","value":"string"},\ +{"key":"npo02__LargestAmount__c","value":"double"},\ +{"key":"npo02__LastCloseDate__c","value":"date"},\ +{"key":"npo02__LastMembershipAmount__c","value":"double"},\ +{"key":"npo02__LastMembershipDate__c","value":"date"},\ +{"key":"npo02__LastMembershipLevel__c","value":"string"},\ +{"key":"npo02__LastMembershipOrigin__c","value":"string"},\ +{"key":"npo02__LastOppAmount__c","value":"double"},\ +{"key":"npo02__MembershipEndDate__c","value":"date"},\ +{"key":"npo02__MembershipJoinDate__c","value":"date"},\ +{"key":"npo02__NumberOfClosedOpps__c","value":"double"},\ +{"key":"npo02__NumberOfMembershipOpps__c","value":"double"},\ +{"key":"npo02__OppAmount2YearsAgo__c","value":"double"},\ +{"key":"npo02__OppAmountLastNDays__c","value":"double"},\ +{"key":"npo02__OppAmountLastYear__c","value":"double"},\ +{"key":"npo02__OppAmountThisYear__c","value":"double"},\ +{"key":"npo02__OppsClosed2YearsAgo__c","value":"double"},\ +{"key":"npo02__OppsClosedLastNDays__c","value":"double"},\ +{"key":"npo02__OppsClosedLastYear__c","value":"double"},\ +{"key":"npo02__OppsClosedThisYear__c","value":"double"},\ +{"key":"npo02__SYSTEM_CUSTOM_NAMING__c","value":"string"},\ +{"key":"npo02__SmallestAmount__c","value":"double"},\ +{"key":"npo02__TotalMembershipOppAmount__c","value":"double"},\ +{"key":"npo02__TotalOppAmount__c","value":"double"},\ +{"key":"npsp__Membership_Span__c","value":"double"},\ +{"key":"npsp__Membership_Status__c","value":"string"},\ +{"key":"npsp__Sustainer__c","value":"string"},\ +{"key":"npsp__All_Members_Deceased__c","value":"boolean"},\ +{"key":"npsp__Batch__c","value":"string"},\ +{"key":"npsp__CustomizableRollups_UseSkewMode__c","value":"boolean"},\ +{"key":"npsp__Funding_Focus__c","value":"string"},\ +{"key":"npsp__Grantmaker__c","value":"boolean"},\ +{"key":"npsp__Matching_Gift_Administrator_Name__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Amount_Max__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Amount_Min__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Annual_Employee_Max__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Comments__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Company__c","value":"boolean"},\ +{"key":"npsp__Matching_Gift_Email__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Info_Updated__c","value":"date"},\ +{"key":"npsp__Matching_Gift_Percent__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Phone__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Request_Deadline__c","value":"string"},\ +{"key":"npsp__Number_of_Household_Members__c","value":"double"},\ +{"key":"npsp__Undeliverable_Address__c","value":"boolean"}] diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java index 1fa74af1..b48c0f23 100644 --- a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java +++ b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java @@ -22,6 +22,7 @@ import io.cdap.plugin.salesforce.SalesforceSchemaUtil; import io.cdap.plugin.salesforce.SalesforceTransformUtil; +import java.math.BigDecimal; import java.util.List; import java.util.Map; import java.util.Objects; @@ -78,7 +79,11 @@ private Object convertValue(String fieldName, Object value, Schema fieldSchema) case BOOLEAN: return Boolean.parseBoolean(castValue(value, fieldName, String.class)); case INT: - return Integer.parseInt(castValue(value, fieldName, String.class)); + // 'Number' field in 'Site' sObject is defined as INT in schema, + // however, the actual data is received as a Decimal (e.g., 0.0) + // Wrapping as BigDecimal to safely extract the int value and avoid cast exceptions. + String data = castValue(value, fieldName, String.class); + return new BigDecimal(data).intValue(); case LONG: return Long.parseLong(castValue(value, fieldName, String.class)); case FLOAT: diff --git a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java index 24fc0ab5..e9fbc6fc 100644 --- a/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java +++ b/src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.StreamSupport; @@ -53,6 +54,9 @@ public class SoapRecordToMapTransformer { Iterable subValues = () -> child.getChildren(SUB_QUERY_FIELDS_PARENT); List> subQueryValues = StreamSupport.stream(subValues.spliterator(), false) + // Since null XmlObject instances were encountered in the customer transfer runs, a + // validation check is now applied to exclude them during processing. + .filter(Objects::nonNull) .map(subValue -> transformRowToMap(subValue, childSObjectDescriptor)) .collect(Collectors.toList()); result.put(childSObjectDescriptor.getName(), subQueryValues); diff --git a/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java b/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java index cb3f8c40..3491c140 100644 --- a/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java +++ b/src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java @@ -562,4 +562,46 @@ public void testCaseSenstiveMultipleResults() throws Exception { assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords); } + @Test + public void testCaseIntTypeWithDecimalDataInResults() throws Exception { + String csvString1 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" + + "\"0061i000003XNcBAAW\",\"false\",\"1500.0\",\"1234.56\"\n"; + String csvString2 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" + + "\"0061i000003XNcCAAW\",\"false\",\"112500.0\",\"1234.56\"\n" + + "\"0061i000003XNcDAAW\",\"false\",\"220000.0\",\"1234.56\"\n"; + + Schema schema = Schema.recordOf("output", + Schema.Field.of("id", Schema.of(Schema.Type.STRING)), + Schema.Field.of("isDeleted", Schema.of(Schema.Type.BOOLEAN)), + Schema.Field.of("ExpectedRevenue", Schema.of(Schema.Type.INT)), + Schema.Field.of("TotalRevenue", Schema.of(Schema.Type.DOUBLE)) + ); + + List> expectedRecords = new ImmutableList.Builder>() + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcBAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 1500) + .put("TotalRevenue", 1234.56) + .build() + ) + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcCAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 112500) + .put("TotalRevenue", 1234.56) + .build() + ) + .add(new ImmutableMap.Builder() + .put("id", "0061i000003XNcDAAW") + .put("isDeleted", false) + .put("ExpectedRevenue", 220000) + .put("TotalRevenue", 1234.56) + .build() + ) + .build(); + + assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords); + } + }