Fixes addressing two issues identified during DTS transfer

pom.xml

@@ -22,7 +22,7 @@
   <name>Salesforce plugins</name>
   <groupId>io.cdap.plugin</groupId>
   <artifactId>salesforce-plugins</artifactId>
-  <version>1.6.8</version>
+  <version>1.6.9</version>
   <packaging>jar</packaging>
   <description>Salesforce Plugins</description>
   <url>https://github.com/data-integrations/salesforce</url>

src/e2e-test/java/io/cdap/plugin/salesforcebatchsource/actions/SalesforcePropertiesPageActions.java

@@ -127,11 +127,11 @@ public static void verifyNoErrorsFoundSuccessMessage() {
   }
 
   private static AuthenticatorCredentials setAuthenticationCredentialsOfAdminUser() {
-    return new AuthenticatorCredentials(PluginPropertyUtils.pluginProp("admin.username"),
+    return new AuthenticatorCredentials(null, PluginPropertyUtils.pluginProp("admin.username"),
       PluginPropertyUtils.pluginProp("admin.password"),
       PluginPropertyUtils.pluginProp("admin.consumer.key"),
       PluginPropertyUtils.pluginProp("admin.consumer.secret"),
-      PluginPropertyUtils.pluginProp("login.url"), null, null, null
+      PluginPropertyUtils.pluginProp("login.url"), null, null, null, null
     );
   }
 

src/e2e-test/resources/pluginParameters.properties

@@ -60,51 +60,170 @@ simple.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string
 ChildToParent.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string"},\
   {"key":"Account_Name","value":"string"}]
 
-lead.schema=[{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\
-  {"key":"MasterRecordId","value":"string"},{"key":"LastName","value":"string"},\
-  {"key":"FirstName","value":"string"},{"key":"Salutation","value":"string"},\
-  {"key":"MiddleName","value":"string"},{"key":"Suffix","value":"string"},{"key":"Name","value":"string"},\
-  {"key":"Title","value":"string"},{"key":"Company","value":"string"},\
-  {"key":"Street","value":"string"},\
-  {"key":"City","value":"string"},{"key":"State","value":"string"},\
-  {"key":"PostalCode","value":"string"},{"key":"Country","value":"string"},\
-  {"key":"Latitude","value":"double"},{"key":"Longitude","value":"double"},\
-  {"key":"GeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\
-  {"key":"MobilePhone","value":"string"},{"key":"Email","value":"string"},\
-  {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\
-  {"key":"LeadSource","value":"string"},{"key":"Status","value":"string"},\
-  {"key":"Industry","value":"string"},{"key":"Rating","value":"string"},\
-  {"key":"NumberOfEmployees","value":"int"},{"key":"OwnerId","value":"string"},\
-  {"key":"IsConverted","value":"boolean"},{"key":"ConvertedDate","value":"date"},\
-  {"key":"ConvertedAccountId","value":"string"},\
-  {"key":"ConvertedContactId","value":"string"},{"key":"ConvertedOpportunityId","value":"string"},\
-  {"key":"IsUnreadByOwner","value":"boolean"},{"key":"CreatedDate","value":"timestamp"},\
-  {"key":"CreatedById","value":"string"},{"key":"LastModifiedDate","value":"timestamp"},\
-  {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\
-  {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\
-  {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\
-  {"key":"JigsawContactId","value":"string"},{"key":"EmailBouncedReason","value":"string"},\
-  {"key":"EmailBouncedDate","value":"timestamp"}]
+lead.schema=[{"key":"Id","value":"string"},\
+{"key":"IsDeleted","value":"boolean"},\
+{"key":"MasterRecordId","value":"string"},\
+{"key":"LastName","value":"string"},\
+{"key":"FirstName","value":"string"},\
+{"key":"Salutation","value":"string"},\
+{"key":"Name","value":"string"},\
+{"key":"Title","value":"string"},\
+{"key":"Company","value":"string"},\
+{"key":"Street","value":"string"},\
+{"key":"City","value":"string"},\
+{"key":"State","value":"string"},\
+{"key":"PostalCode","value":"string"},\
+{"key":"Country","value":"string"},\
+{"key":"Latitude","value":"double"},\
+{"key":"Longitude","value":"double"},\
+{"key":"GeocodeAccuracy","value":"string"},\
+{"key":"Phone","value":"string"},\
+{"key":"MobilePhone","value":"string"},\
+{"key":"Fax","value":"string"},\
+{"key":"Email","value":"string"},\
+{"key":"Website","value":"string"},\
+{"key":"PhotoUrl","value":"string"},\
+{"key":"Description","value":"string"},\
+{"key":"LeadSource","value":"string"},\
+{"key":"Status","value":"string"},\
+{"key":"Industry","value":"string"},\
+{"key":"Rating","value":"string"},\
+{"key":"AnnualRevenue","value":"double"},\
+{"key":"NumberOfEmployees","value":"int"},\
+{"key":"OwnerId","value":"string"},\
+{"key":"IsConverted","value":"boolean"},\
+{"key":"ConvertedDate","value":"date"},\
+{"key":"ConvertedAccountId","value":"string"},\
+{"key":"ConvertedContactId","value":"string"},\
+{"key":"ConvertedOpportunityId","value":"string"},\
+{"key":"IsUnreadByOwner","value":"boolean"},\
+{"key":"CreatedDate","value":"timestamp"},\
+{"key":"CreatedById","value":"string"},\
+{"key":"LastModifiedDate","value":"timestamp"},\
+{"key":"LastModifiedById","value":"string"},\
+{"key":"SystemModstamp","value":"timestamp"},\
+{"key":"LastActivityDate","value":"date"},\
+{"key":"LastViewedDate","value":"timestamp"},\
+{"key":"LastReferencedDate","value":"timestamp"},\
+{"key":"Jigsaw","value":"string"},\
+{"key":"JigsawContactId","value":"string"},\
+{"key":"EmailBouncedReason","value":"string"},\
+{"key":"EmailBouncedDate","value":"timestamp"},\
+{"key":"IndividualId","value":"string"},\
+{"key":"Follow_Up__c","value":"boolean"},\
+{"key":"npe01__Preferred_Email__c","value":"string"},\
+{"key":"npe01__Preferred_Phone__c","value":"string"},\
+{"key":"npsp__Batch__c","value":"string"},\
+{"key":"npsp__CompanyCity__c","value":"string"},\
+{"key":"npsp__CompanyCountry__c","value":"string"},\
+{"key":"npsp__CompanyPostalCode__c","value":"string"},\
+{"key":"npsp__CompanyState__c","value":"string"},\
+{"key":"npsp__CompanyStreet__c","value":"string"}]
 
-account.schema= [{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\
-  {"key":"MasterRecordId","value":"string"},{"key":"Name","value":"string"},\
-  {"key":"Type","value":"string"},\
-  {"key":"ParentId","value":"string"},{"key":"BillingStreet","value":"string"},{"key":"BillingCity","value":"string"},\
-  {"key":"BillingState","value":"string"},{"key":"BillingPostalCode","value":"string"},\
-  {"key":"BillingCountry","value":"string"},\
-  {"key":"BillingLatitude","value":"double"},{"key":"BillingLongitude","value":"double"},\
-  {"key":"BillingGeocodeAccuracy","value":"string"},{"key":"ShippingStreet","value":"string"},\
-  {"key":"ShippingCity","value":"string"},{"key":"ShippingState","value":"string"},\
-  {"key":"ShippingPostalCode","value":"string"},{"key":"ShippingCountry","value":"string"},\
-  {"key":"ShippingLatitude","value":"double"},{"key":"ShippingLongitude","value":"double"},\
-  {"key":"ShippingGeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\
-  {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\
-  {"key":"Industry","value":"string"},\
-  {"key":"NumberOfEmployees","value":"int"},{"key":"Description","value":"string"},{"key":"OwnerId","value":"string"},\
-  {"key":"CreatedDate","value":"timestamp"},{"key":"CreatedById","value":"string"},\
-  {"key":"LastModifiedDate","value":"timestamp"},\
-  {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\
-  {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\
-  {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\
-  {"key":"JigsawCompanyId","value":"string"},{"key":"AccountSource","value":"string"},\
-  {"key":"SicDesc","value":"string"}]
+account.schema= [{"key":"Id","value":"string"},\
+{"key":"IsDeleted","value":"boolean"},\
+{"key":"MasterRecordId","value":"string"},\
+{"key":"Name","value":"string"},\
+{"key":"Type","value":"string"},\
+{"key":"RecordTypeId","value":"string"},\
+{"key":"ParentId","value":"string"},\
+{"key":"BillingStreet","value":"string"},\
+{"key":"BillingCity","value":"string"},\
+{"key":"BillingState","value":"string"},\
+{"key":"BillingPostalCode","value":"string"},\
+{"key":"BillingCountry","value":"string"},\
+{"key":"BillingLatitude","value":"double"},\
+{"key":"BillingLongitude","value":"double"},\
+{"key":"BillingGeocodeAccuracy","value":"string"},\
+{"key":"ShippingStreet","value":"string"},\
+{"key":"ShippingCity","value":"string"},\
+{"key":"ShippingState","value":"string"},\
+{"key":"ShippingPostalCode","value":"string"},\
+{"key":"ShippingCountry","value":"string"},\
+{"key":"ShippingLatitude","value":"double"},\
+{"key":"ShippingLongitude","value":"double"},\
+{"key":"ShippingGeocodeAccuracy","value":"string"},\
+{"key":"Phone","value":"string"},\
+{"key":"Fax","value":"string"},\
+{"key":"AccountNumber","value":"string"},\
+{"key":"Website","value":"string"},\
+{"key":"PhotoUrl","value":"string"},\
+{"key":"Sic","value":"string"},\
+{"key":"Industry","value":"string"},\
+{"key":"AnnualRevenue","value":"double"},\
+{"key":"NumberOfEmployees","value":"int"},\
+{"key":"Ownership","value":"string"},\
+{"key":"TickerSymbol","value":"string"},\
+{"key":"Description","value":"string"},\
+{"key":"Rating","value":"string"},\
+{"key":"Site","value":"string"},\
+{"key":"OwnerId","value":"string"},\
+{"key":"CreatedDate","value":"timestamp"},\
+{"key":"CreatedById","value":"string"},\
+{"key":"LastModifiedDate","value":"timestamp"},\
+{"key":"LastModifiedById","value":"string"},\
+{"key":"SystemModstamp","value":"timestamp"},\
+{"key":"LastActivityDate","value":"date"},\
+{"key":"LastViewedDate","value":"timestamp"},\
+{"key":"LastReferencedDate","value":"timestamp"},\
+{"key":"Jigsaw","value":"string"},\
+{"key":"JigsawCompanyId","value":"string"},\
+{"key":"AccountSource","value":"string"},\
+{"key":"SicDesc","value":"string"},\
+{"key":"Copy_Billing_Address_to_Shipping_Address__c","value":"boolean"},\
+{"key":"Test_External_Id__c","value":"string"},\
+{"key":"custom_field__c","value":"string"},\
+{"key":"npe01__One2OneContact__c","value":"string"},\
+{"key":"npe01__SYSTEMIsIndividual__c","value":"boolean"},\
+{"key":"npe01__SYSTEM_AccountType__c","value":"string"},\
+{"key":"npo02__AverageAmount__c","value":"double"},\
+{"key":"npo02__Best_Gift_Year_Total__c","value":"double"},\
+{"key":"npo02__Best_Gift_Year__c","value":"string"},\
+{"key":"npo02__FirstCloseDate__c","value":"date"},\
+{"key":"npo02__Formal_Greeting__c","value":"string"},\
+{"key":"npo02__HouseholdPhone__c","value":"string"},\
+{"key":"npo02__Informal_Greeting__c","value":"string"},\
+{"key":"npo02__LargestAmount__c","value":"double"},\
+{"key":"npo02__LastCloseDate__c","value":"date"},\
+{"key":"npo02__LastMembershipAmount__c","value":"double"},\
+{"key":"npo02__LastMembershipDate__c","value":"date"},\
+{"key":"npo02__LastMembershipLevel__c","value":"string"},\
+{"key":"npo02__LastMembershipOrigin__c","value":"string"},\
+{"key":"npo02__LastOppAmount__c","value":"double"},\
+{"key":"npo02__MembershipEndDate__c","value":"date"},\
+{"key":"npo02__MembershipJoinDate__c","value":"date"},\
+{"key":"npo02__NumberOfClosedOpps__c","value":"double"},\
+{"key":"npo02__NumberOfMembershipOpps__c","value":"double"},\
+{"key":"npo02__OppAmount2YearsAgo__c","value":"double"},\
+{"key":"npo02__OppAmountLastNDays__c","value":"double"},\
+{"key":"npo02__OppAmountLastYear__c","value":"double"},\
+{"key":"npo02__OppAmountThisYear__c","value":"double"},\
+{"key":"npo02__OppsClosed2YearsAgo__c","value":"double"},\
+{"key":"npo02__OppsClosedLastNDays__c","value":"double"},\
+{"key":"npo02__OppsClosedLastYear__c","value":"double"},\
+{"key":"npo02__OppsClosedThisYear__c","value":"double"},\
+{"key":"npo02__SYSTEM_CUSTOM_NAMING__c","value":"string"},\
+{"key":"npo02__SmallestAmount__c","value":"double"},\
+{"key":"npo02__TotalMembershipOppAmount__c","value":"double"},\
+{"key":"npo02__TotalOppAmount__c","value":"double"},\
+{"key":"npsp__Membership_Span__c","value":"double"},\
+{"key":"npsp__Membership_Status__c","value":"string"},\
+{"key":"npsp__Sustainer__c","value":"string"},\
+{"key":"npsp__All_Members_Deceased__c","value":"boolean"},\
+{"key":"npsp__Batch__c","value":"string"},\
+{"key":"npsp__CustomizableRollups_UseSkewMode__c","value":"boolean"},\
+{"key":"npsp__Funding_Focus__c","value":"string"},\
+{"key":"npsp__Grantmaker__c","value":"boolean"},\
+{"key":"npsp__Matching_Gift_Administrator_Name__c","value":"string"},\
+{"key":"npsp__Matching_Gift_Amount_Max__c","value":"double"},\
+{"key":"npsp__Matching_Gift_Amount_Min__c","value":"double"},\
+{"key":"npsp__Matching_Gift_Annual_Employee_Max__c","value":"double"},\
+{"key":"npsp__Matching_Gift_Comments__c","value":"string"},\
+{"key":"npsp__Matching_Gift_Company__c","value":"boolean"},\
+{"key":"npsp__Matching_Gift_Email__c","value":"string"},\
+{"key":"npsp__Matching_Gift_Info_Updated__c","value":"date"},\
+{"key":"npsp__Matching_Gift_Percent__c","value":"double"},\
+{"key":"npsp__Matching_Gift_Phone__c","value":"string"},\
+{"key":"npsp__Matching_Gift_Request_Deadline__c","value":"string"},\
+{"key":"npsp__Number_of_Household_Members__c","value":"double"},\
+{"key":"npsp__Undeliverable_Address__c","value":"boolean"}]

src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/MapToRecordTransformer.java

@@ -22,6 +22,7 @@
 import io.cdap.plugin.salesforce.SalesforceSchemaUtil;
 import io.cdap.plugin.salesforce.SalesforceTransformUtil;
 
+import java.math.BigDecimal;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -78,7 +79,11 @@ private Object convertValue(String fieldName, Object value, Schema fieldSchema)
       case BOOLEAN:
         return Boolean.parseBoolean(castValue(value, fieldName, String.class));
       case INT:
-        return Integer.parseInt(castValue(value, fieldName, String.class));
+        // 'Number' field in 'Site' sObject is defined as INT in schema,
+        // however, the actual data is received as a Decimal (e.g., 0.0)
+        // Wrapping as BigDecimal to safely extract the int value and avoid cast exceptions.
+        String data = castValue(value, fieldName, String.class);
+        return new BigDecimal(data).intValue();
       case LONG:
         return Long.parseLong(castValue(value, fieldName, String.class));
       case FLOAT:

src/main/java/io/cdap/plugin/salesforce/plugin/source/batch/SoapRecordToMapTransformer.java

@@ -23,6 +23,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
 
@@ -53,6 +54,9 @@ public class SoapRecordToMapTransformer {
       Iterable<XmlObject> subValues = () -> child.getChildren(SUB_QUERY_FIELDS_PARENT);
 
       List<Map<String, String>> subQueryValues = StreamSupport.stream(subValues.spliterator(), false)
+          // Since null XmlObject instances were encountered in the customer transfer runs, a
+          // validation check is now applied to exclude them during processing.
+          .filter(Objects::nonNull)
         .map(subValue -> transformRowToMap(subValue, childSObjectDescriptor))
         .collect(Collectors.toList());
       result.put(childSObjectDescriptor.getName(), subQueryValues);

src/test/java/io/cdap/plugin/salesforce/plugin/source/batch/SalesforceBulkRecordReaderTest.java

@@ -562,4 +562,46 @@ public void testCaseSenstiveMultipleResults() throws Exception {
     assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords);
   }
 
+  @Test
+  public void testCaseIntTypeWithDecimalDataInResults() throws Exception {
+    String csvString1 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" +
+      "\"0061i000003XNcBAAW\",\"false\",\"1500.0\",\"1234.56\"\n";
+    String csvString2 = "\"Id\",\"IsDeleted\",\"ExpectedRevenue\",\"TotalRevenue\"\n" +
+      "\"0061i000003XNcCAAW\",\"false\",\"112500.0\",\"1234.56\"\n" +
+      "\"0061i000003XNcDAAW\",\"false\",\"220000.0\",\"1234.56\"\n";
+
+    Schema schema = Schema.recordOf("output",
+                                    Schema.Field.of("id", Schema.of(Schema.Type.STRING)),
+                                    Schema.Field.of("isDeleted", Schema.of(Schema.Type.BOOLEAN)),
+                                    Schema.Field.of("ExpectedRevenue", Schema.of(Schema.Type.INT)),
+                                    Schema.Field.of("TotalRevenue", Schema.of(Schema.Type.DOUBLE))
+    );
+
+    List<Map<String, Object>> expectedRecords = new ImmutableList.Builder<Map<String, Object>>()
+      .add(new ImmutableMap.Builder<String, Object>()
+             .put("id", "0061i000003XNcBAAW")
+             .put("isDeleted", false)
+             .put("ExpectedRevenue", 1500)
+             .put("TotalRevenue", 1234.56)
+             .build()
+      )
+      .add(new ImmutableMap.Builder<String, Object>()
+             .put("id", "0061i000003XNcCAAW")
+             .put("isDeleted", false)
+             .put("ExpectedRevenue", 112500)
+             .put("TotalRevenue", 1234.56)
+             .build()
+      )
+      .add(new ImmutableMap.Builder<String, Object>()
+             .put("id", "0061i000003XNcDAAW")
+             .put("isDeleted", false)
+             .put("ExpectedRevenue", 220000)
+             .put("TotalRevenue", 1234.56)
+             .build()
+      )
+      .build();
+
+    assertRecordReaderOutputRecords(new String[] {csvString1, csvString2}, schema, expectedRecords);
+  }
+
 }