diff --git a/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTime.feature b/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTime.feature index 6d11e04f..dfe25bb8 100644 --- a/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTime.feature +++ b/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTime.feature @@ -15,11 +15,12 @@ @SalesforceSalesCloud @SFMultiObjectsBatchSource @Smoke +# Skipping these tests due to permission issues for creating custom fields in sObjects. Will Run it once it get resolves. @Regression Feature: Salesforce Multi Objects Batch Source - Run time Scenarios - @MULTIBATCH-TS-SF-RNTM-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 + @MULTIBATCH-TS-SF-RNTM-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890 Scenario: Verify user should be able to preview, deploy and run pipeline for valid White List When Open Datafusion Project to configure pipeline And Select data pipeline type as: "Batch" @@ -58,7 +59,7 @@ Feature: Salesforce Multi Objects Batch Source - Run time Scenarios Then Validate the values of records transferred to target Big Query table is equal to the values from multi object source table - @MULTIBATCH-TS-SF-RNTM-02 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 + @MULTIBATCH-TS-SF-RNTM-02 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890 Scenario: Verify user should be able to preview, deploy and run pipeline for valid Black List When Open Datafusion Project to configure pipeline And Select data pipeline type as: "Batch" @@ -99,7 +100,7 @@ Feature: Salesforce Multi Objects Batch Source - Run time Scenarios Then Validate the values of records transferred to target Big Query table is equal to the values from multi object source table - @MULTIBATCH-TS-SF-RNTM-03 @CONNECTION @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 + @MULTIBATCH-TS-SF-RNTM-03 @CONNECTION @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890 Scenario: Verify user should be able to deploy and run the pipeline using connection manager functionality When Open Datafusion Project to configure pipeline And Select plugin: "Salesforce Multi Objects" from the plugins list as: "Source" diff --git a/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTimeWithMacros.feature b/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTimeWithMacros.feature index 9a14592e..f2698f20 100644 --- a/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTimeWithMacros.feature +++ b/src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTimeWithMacros.feature @@ -15,11 +15,12 @@ @SalesforceSalesCloud @SFMultiObjectsBatchSource @Smoke +# Skipping these tests due to permission issues for creating custom fields in sObjects. Will Run it once it get resolves. @Regression Feature: Salesforce Multi Objects Batch Source - Run time Scenarios with Macro - @MULTIBATCH-TS-SF-RNTM-MACRO-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 + @MULTIBATCH-TS-SF-RNTM-MACRO-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890 Scenario: Verify user should be able to preview, deploy a pipeline when plugin is configured with macros for WhiteList When Open Datafusion Project to configure pipeline And Select data pipeline type as: "Batch" diff --git a/src/e2e-test/features/salesforcestreamingsource/DesignTime.feature b/src/e2e-test/features/salesforcestreamingsource/DesignTime.feature index d74bec5b..870bbad2 100644 --- a/src/e2e-test/features/salesforcestreamingsource/DesignTime.feature +++ b/src/e2e-test/features/salesforcestreamingsource/DesignTime.feature @@ -27,6 +27,7 @@ Feature: Salesforce Streaming Source - Design time scenarios And Navigate to the properties page of plugin: "Salesforce" And fill Authentication properties for Salesforce Admin user And Enter input plugin property: "pushTopicName" with value: "topic.name" + And Enter input plugin property: "sObjectName" with value: "sobject.Automation_custom_c" And Select dropdown plugin property: "pushTopicNotifyCreate" with option value: "Enabled" And Select dropdown plugin property: "pushTopicNotifyUpdate" with option value: "Enabled" And Select dropdown plugin property: "pushTopicNotifyDelete" with option value: "Enabled" diff --git a/src/e2e-test/java/io/cdap/plugin/BQValidation.java b/src/e2e-test/java/io/cdap/plugin/BQValidation.java index 444047a1..dd105e5e 100644 --- a/src/e2e-test/java/io/cdap/plugin/BQValidation.java +++ b/src/e2e-test/java/io/cdap/plugin/BQValidation.java @@ -51,7 +51,7 @@ public class BQValidation { public static boolean validateSalesforceAndBQRecordValues(String objectName, String bqTable) throws IOException, InterruptedException { - String uniqueRecordId = SalesforceClient.queryObjectId(objectName); + List uniqueRecordIds = SalesforceClient.queryObjectId(objectName); List bigQueryResponse = new ArrayList<>(); List bigQueryRows = new ArrayList<>(); @@ -60,8 +60,11 @@ public static boolean validateSalesforceAndBQRecordValues(String objectName, Str JsonObject jsonData = gson.fromJson(String.valueOf(rows), JsonObject.class); bigQueryResponse.add(jsonData); } - List sObjectResponse; - sObjectResponse = SalesforceClient.queryObject(uniqueRecordId, objectName); + List sObjectResponse = new ArrayList<>(); + for (String recordId : uniqueRecordIds) { + JsonObject record = SalesforceClient.queryObject(recordId, objectName); + sObjectResponse.add(record); + } return compareSalesforceAndJsonData(sObjectResponse, bigQueryResponse, bqTable); } @@ -82,9 +85,12 @@ public static boolean validateSalesforceMultiObjectToBQRecordValues() throws IOE JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class); bigQueryResponse.add(jsonData); } - String uniqueRecordId = SalesforceClient.queryObjectId(currentObject); - List sObjectResponse; - sObjectResponse = SalesforceClient.queryObject(uniqueRecordId, currentObject); + List uniqueRecordIds = SalesforceClient.queryObjectId(currentObject); + List sObjectResponse = new ArrayList<>(); + for (String recordId : uniqueRecordIds) { + JsonObject record = SalesforceClient.queryObject(recordId, currentObject); + sObjectResponse.add(record); + } boolean isValid = compareSalesforceAndJsonData( sObjectResponse, bigQueryResponse, currentTargetTable); @@ -141,14 +147,9 @@ public static boolean compareSalesforceAndJsonData(List salesforceDa Assert.fail("bigQueryData is null"); return result; } - int jsonObjectIdx = 0; - if (salesforceData.size() > 0) { - salesforceData.get(jsonObjectIdx).entrySet().size(); - } - // Get the column count of the first JsonObject in bigQueryData - int columnCountSource = 0; - if (bigQueryData.size() > 0) { - columnCountSource = bigQueryData.get(jsonObjectIdx).entrySet().size(); + if (salesforceData.isEmpty() || bigQueryData.isEmpty()) { + Assert.fail("One or both datasets are empty"); + return result; } BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService(); @@ -158,9 +159,11 @@ public static boolean compareSalesforceAndJsonData(List salesforceDa TableId tableRef = TableId.of(projectId, dataset, tableName); // Get the table schema Schema schema = bigQuery.getTable(tableRef).getDefinition().getSchema(); - // Iterate over the fields - int currentColumnCount = 1; - while (currentColumnCount <= columnCountSource) { + + for (int rowIndex = 0; rowIndex < salesforceData.size(); rowIndex++) { + JsonObject salesforceRow = salesforceData.get(rowIndex); + JsonObject bigQueryRow = bigQueryData.get(rowIndex); + for (Field field : schema.getFields()) { String columnName = field.getName(); String columnType = field.getType().toString(); @@ -168,29 +171,27 @@ public static boolean compareSalesforceAndJsonData(List salesforceDa switch (columnType) { case "BOOLEAN": - boolean sourceAsBoolean = salesforceData.get(jsonObjectIdx).get(columnName).getAsBoolean(); - boolean targetAsBoolean = bigQueryData.get(jsonObjectIdx).get(columnName).getAsBoolean(); + boolean sourceAsBoolean = salesforceRow.get(columnName).getAsBoolean(); + boolean targetAsBoolean = bigQueryRow.get(columnName).getAsBoolean(); Assert.assertEquals("Different values found for column : %s", sourceAsBoolean, targetAsBoolean); break; case "FLOAT": - double sourceVal = salesforceData.get(jsonObjectIdx).get(columnName).getAsDouble(); - double targetVal = bigQueryData.get(jsonObjectIdx).get(columnName).getAsDouble(); + double sourceVal = salesforceRow.get(columnName).getAsDouble(); + double targetVal = bigQueryRow.get(columnName).getAsDouble(); Assert.assertEquals(String.format("Different values found for column: %s", columnName), 0, Double.compare(sourceVal, targetVal)); break; case "TIMESTAMP": OffsetDateTime sourceTimestamp = OffsetDateTime.parse( - salesforceData.get(jsonObjectIdx) - .get(columnName) + salesforceRow.get(columnName) .getAsString(), DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ") ); OffsetDateTime targetTimestamp = OffsetDateTime.parse( - bigQueryData.get(jsonObjectIdx) - .get(columnName) + bigQueryRow.get(columnName) .getAsString() ); Assert.assertEquals("Different values found for column : %s", sourceTimestamp, targetTimestamp); @@ -200,23 +201,21 @@ public static boolean compareSalesforceAndJsonData(List salesforceDa DateTimeFormatter formatterSource = DateTimeFormatter.ofPattern("HH:mm:ss.SSSX"); DateTimeFormatter formatterTarget = DateTimeFormatter.ofPattern("HH:mm:ss"); LocalTime sourceTime = LocalTime.parse( - salesforceData.get(jsonObjectIdx) - .get(columnName) + salesforceRow.get(columnName) .getAsString(), formatterSource ); LocalTime targetTime = LocalTime.parse( - bigQueryData.get(jsonObjectIdx) - .get(columnName) + bigQueryRow.get(columnName) .getAsString(), formatterTarget ); Assert.assertEquals("Different values found for column : %s", sourceTime, targetTime); break; case "DATE": - JsonElement jsonElementSource = salesforceData.get(jsonObjectIdx).get(columnName); + JsonElement jsonElementSource = salesforceRow.get(columnName); Date sourceDate = (jsonElementSource != null && !jsonElementSource.isJsonNull()) ? Date.valueOf( jsonElementSource.getAsString()) : null; - JsonElement jsonElementTarget = bigQueryData.get(jsonObjectIdx).get(columnName); + JsonElement jsonElementTarget = bigQueryRow.get(columnName); Date targetDate = (jsonElementTarget != null && !jsonElementTarget.isJsonNull()) ? Date.valueOf( jsonElementTarget.getAsString()) : null; Assert.assertEquals("Different values found for column : %s", sourceDate, targetDate); @@ -231,21 +230,19 @@ public static boolean compareSalesforceAndJsonData(List salesforceDa if (columnName.equals("Col_GeoLocation__c")) { break; } else { - JsonElement sourceElement = salesforceData.get(jsonObjectIdx).get(columnName); + JsonElement sourceElement = salesforceRow.get(columnName); String sourceString = (sourceElement != null && !sourceElement.isJsonNull()) ? sourceElement.getAsString() : null; - JsonElement targetElement = bigQueryData.get(jsonObjectIdx).get(columnName); + JsonElement targetElement = bigQueryRow.get(columnName); String targetString = (targetElement != null && !targetElement.isJsonNull()) ? targetElement.getAsString() : null; Assert.assertEquals(String.format("Different values found for column : %s", columnName), String.valueOf(sourceString), String.valueOf(targetString)); - break; + } + break; } } - currentColumnCount++; } - jsonObjectIdx++; - } Assert.assertFalse("Number of rows in Source table is greater than the number of rows in Target table", salesforceData.size() > bigQueryData.size()); return true; diff --git a/src/e2e-test/java/io/cdap/plugin/salesforcestreamingsource/stepsdesign/DesignTimeSteps.java b/src/e2e-test/java/io/cdap/plugin/salesforcestreamingsource/stepsdesign/DesignTimeSteps.java index cba23494..948fef70 100644 --- a/src/e2e-test/java/io/cdap/plugin/salesforcestreamingsource/stepsdesign/DesignTimeSteps.java +++ b/src/e2e-test/java/io/cdap/plugin/salesforcestreamingsource/stepsdesign/DesignTimeSteps.java @@ -35,6 +35,7 @@ import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.util.List; import java.util.concurrent.TimeUnit; /** * Design-time steps of Salesforce Streaming plugins. @@ -70,8 +71,10 @@ public void fillUniqueTopicNameInRuntimeArguments(String runtimeArgumentKey) { @Then("Update existing salesforce records") public void updateExistingSalesforceRecords() { - String uniqueRecordId = SalesforceClient.queryObjectId(customObject); - SalesforceClient.updateObject(uniqueRecordId, customObject); + List uniqueRecordIds = SalesforceClient.queryObjectId(customObject); + for (String recordId : uniqueRecordIds) { + SalesforceClient.updateObject(recordId, customObject); + } } diff --git a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java index 69064110..8031c648 100644 --- a/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java +++ b/src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java @@ -37,6 +37,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.text.ParseException; +import java.util.List; import java.util.NoSuchElementException; import java.util.UUID; @@ -65,16 +66,20 @@ public static void createTestObject2() throws UnsupportedEncodingException, Pars @After(order = 2, value = "@DELETE_TEST_DATA") public static void deleteTestObject() { - String uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM__C.value); - SalesforceClient.deleteId(uniqueRecordId, SObjects.AUTOMATION_CUSTOM__C.value); - BeforeActions.scenario.write("Record - " + uniqueRecordId + " deleted successfully"); + List uniqueRecordIds = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM__C.value); + for (String recordId : uniqueRecordIds) { + SalesforceClient.deleteId(recordId, SObjects.AUTOMATION_CUSTOM__C.value); + BeforeActions.scenario.write("Record - " + recordId + " deleted successfully"); + } } @After(order = 2, value = "@DELETE_TEST_DATA2") public static void deleteTestObject2() { - String uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM2__C.value); - SalesforceClient.deleteId(uniqueRecordId, SObjects.AUTOMATION_CUSTOM2__C.value); - BeforeActions.scenario.write("Record - " + uniqueRecordId + " deleted successfully"); + List uniqueRecordIds = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM2__C.value); + for (String recordId : uniqueRecordIds) { + SalesforceClient.deleteId(recordId, SObjects.AUTOMATION_CUSTOM__C.value); + BeforeActions.scenario.write("Record - " + recordId + " deleted successfully"); + } } diff --git a/src/e2e-test/java/io/cdap/plugin/tests/runner/TestRunner.java b/src/e2e-test/java/io/cdap/plugin/tests/runner/TestRunner.java index 0caa19c1..3e8c8275 100644 --- a/src/e2e-test/java/io/cdap/plugin/tests/runner/TestRunner.java +++ b/src/e2e-test/java/io/cdap/plugin/tests/runner/TestRunner.java @@ -32,7 +32,7 @@ "io.cdap.plugin.salesforcemultiobjectsbatchsource.stepsdesign", "io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.tests.hooks", "stepsdesign"}, - tags = {"@Regression"}, + tags = {"@Regression and not @Plugin-1890"}, monochrome = true, plugin = {"pretty", "html:target/cucumber-html-report", "json:target/cucumber-reports/cucumber.json", "junit:target/cucumber-reports/cucumber.xml"} diff --git a/src/e2e-test/java/io/cdap/plugin/utils/SalesforceClient.java b/src/e2e-test/java/io/cdap/plugin/utils/SalesforceClient.java index 466b714b..6b27385d 100644 --- a/src/e2e-test/java/io/cdap/plugin/utils/SalesforceClient.java +++ b/src/e2e-test/java/io/cdap/plugin/utils/SalesforceClient.java @@ -149,7 +149,7 @@ public static String createObject(JSONObject objectJson, String objectName) thro return uniqueRecordId; } - public static List queryObject(String id, String objectName) { + public static JsonObject queryObject(String id, String objectName) { getAccessToken(); HttpClient httpClient = HttpClientBuilder.create().build(); String baseUri = loginInstanceUrl + REST_ENDPOINT + API_VERSION; @@ -167,12 +167,12 @@ public static List queryObject(String id, String objectName) { String responseString = EntityUtils.toString(response.getEntity()); Gson gson = new Gson(); JsonObject objectResponseInJson = gson.fromJson(responseString, JsonObject.class); - sfobjectResponse.add(objectResponseInJson); + return objectResponseInJson; } } catch (IOException ioException) { logger.info("Error in establishing connection to Salesforce: " + ioException); } - return sfobjectResponse; + return null; } public static void deletePushTopic(String pushTopicName) { @@ -233,10 +233,11 @@ public static void deleteId(String id, String objectName) { } } - public static String queryObjectId(String objectName) { + public static List queryObjectId(String objectName) { getAccessToken(); HttpClient httpClient = HttpClientBuilder.create().build(); String baseUri = loginInstanceUrl + REST_ENDPOINT + API_VERSION; + List idList = new ArrayList<>(); try { String query = "SELECT Id FROM " + objectName; @@ -259,14 +260,15 @@ public static String queryObjectId(String objectName) { JsonArray records = queryResponse.getAsJsonArray("records"); for (JsonElement record : records) { JsonObject recordObject = record.getAsJsonObject(); - uniqueRecordId = recordObject.get("Id").getAsString(); - logger.info("Queried Object id from response: " + uniqueRecordId); + String id = recordObject.get("Id").getAsString(); + idList.add(id); + logger.info("Queried Object id from response: " + id); } } } catch (IOException ioException) { logger.info("Error in establishing connection to Salesforce: " + ioException); } - return uniqueRecordId; + return idList; } public static void updateObject(String id, String objectName) { diff --git a/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt b/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt index 17d217bd..91243d51 100644 --- a/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt +++ b/src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt @@ -1,3 +1,2 @@ create table `DATASET.TABLE_NAME` (Name STRING, Col_Timestamp__c TIMESTAMP, Col_Date__c DATE, Col_Currency__c FLOAT64, - Col_Email__c STRING, Col_Number__c FLOAT64, Col_GeoLocation__Latitude__s FLOAT64, -Col_GeoLocation__Longitude__s FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING) + Col_Email__c STRING, Col_Number__c FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING) diff --git a/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt b/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt index 6dd90956..333f531c 100644 --- a/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt +++ b/src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt @@ -1,5 +1,5 @@ insert into `DATASET.TABLE_NAME` (Name, Col_Timestamp__c, Col_Date__c, Col_Currency__c, Col_Email__c, Col_Number__c, -Col_GeoLocation__Latitude__s, Col_GeoLocation__Longitude__s, Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values -('adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',898365444,37.794116,-122.3432, +Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values +('adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',898365444, '984746334','abc/123','20:26:34','find'); diff --git a/src/e2e-test/resources/pluginParameters.properties b/src/e2e-test/resources/pluginParameters.properties index 625132d1..b2252425 100644 --- a/src/e2e-test/resources/pluginParameters.properties +++ b/src/e2e-test/resources/pluginParameters.properties @@ -18,7 +18,7 @@ invalid.admin.consumer.secret=lmnop891011 #SOQL Query simple.query=SELECT Id, Name, Phone FROM Account test.query=SELECT Id,Name,Col_Timestamp__c,Col_Date__c,Col_Currency__c,Col_Email__c,Col_Number__c,\ - Col_GeoLocation__Latitude__s,Col_GeoLocation__Longitude__s,Col__c,Col_Url__c,Col_Time__c,Col_Text__c FROM Automation_custom__c + Col__c,Col_Url__c,Col_Time__c,Col_Text__c FROM Automation_custom__c where.query=SELECT name FROM Opportunity WHERE StageName='Needs Analysis' groupby.query=SELECT CampaignId, AVG(Amount) FROM Opportunity GROUP BY CampaignId childtoparent.query=SELECT Id, Name, Account.Name FROM Contact WHERE Account.Industry = 'Chemicals' @@ -75,54 +75,174 @@ simple.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string ChildToParent.query.schema=[{"key":"Id","value":"string"},{"key":"Name","value":"string"},\ {"key":"Account_Name","value":"string"}] -lead.schema=[{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\ - {"key":"MasterRecordId","value":"string"},{"key":"LastName","value":"string"},\ - {"key":"FirstName","value":"string"},{"key":"Salutation","value":"string"},\ - {"key":"MiddleName","value":"string"},{"key":"Suffix","value":"string"},{"key":"Name","value":"string"},\ - {"key":"Title","value":"string"},{"key":"Company","value":"string"},\ - {"key":"Street","value":"string"},\ - {"key":"City","value":"string"},{"key":"State","value":"string"},\ - {"key":"PostalCode","value":"string"},{"key":"Country","value":"string"},\ - {"key":"Latitude","value":"double"},{"key":"Longitude","value":"double"},\ - {"key":"GeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\ - {"key":"MobilePhone","value":"string"},{"key":"Email","value":"string"},\ - {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\ - {"key":"LeadSource","value":"string"},{"key":"Status","value":"string"},\ - {"key":"Industry","value":"string"},{"key":"Rating","value":"string"},\ - {"key":"NumberOfEmployees","value":"int"},{"key":"OwnerId","value":"string"},\ - {"key":"IsConverted","value":"boolean"},{"key":"ConvertedDate","value":"date"},\ - {"key":"ConvertedAccountId","value":"string"},\ - {"key":"ConvertedContactId","value":"string"},{"key":"ConvertedOpportunityId","value":"string"},\ - {"key":"IsUnreadByOwner","value":"boolean"},{"key":"CreatedDate","value":"timestamp"},\ - {"key":"CreatedById","value":"string"},{"key":"LastModifiedDate","value":"timestamp"},\ - {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\ - {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\ - {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\ - {"key":"JigsawContactId","value":"string"},{"key":"EmailBouncedReason","value":"string"},\ - {"key":"EmailBouncedDate","value":"timestamp"}] +lead.schema=[{"key":"Id","value":"string"},\ +{"key":"IsDeleted","value":"boolean"},\ +{"key":"MasterRecordId","value":"string"},\ +{"key":"LastName","value":"string"},\ +{"key":"FirstName","value":"string"},\ +{"key":"Salutation","value":"string"},\ +{"key":"Name","value":"string"},\ +{"key":"Title","value":"string"},\ +{"key":"Company","value":"string"},\ +{"key":"Street","value":"string"},\ +{"key":"City","value":"string"},\ +{"key":"State","value":"string"},\ +{"key":"PostalCode","value":"string"},\ +{"key":"Country","value":"string"},\ +{"key":"Latitude","value":"double"},\ +{"key":"Longitude","value":"double"},\ +{"key":"GeocodeAccuracy","value":"string"},\ +{"key":"Phone","value":"string"},\ +{"key":"MobilePhone","value":"string"},\ +{"key":"Fax","value":"string"},\ +{"key":"Email","value":"string"},\ +{"key":"Website","value":"string"},\ +{"key":"PhotoUrl","value":"string"},\ +{"key":"Description","value":"string"},\ +{"key":"LeadSource","value":"string"},\ +{"key":"Status","value":"string"},\ +{"key":"Industry","value":"string"},\ +{"key":"Rating","value":"string"},\ +{"key":"AnnualRevenue","value":"double"},\ +{"key":"NumberOfEmployees","value":"int"},\ +{"key":"OwnerId","value":"string"},\ +{"key":"IsConverted","value":"boolean"},\ +{"key":"ConvertedDate","value":"date"},\ +{"key":"ConvertedAccountId","value":"string"},\ +{"key":"ConvertedContactId","value":"string"},\ +{"key":"ConvertedOpportunityId","value":"string"},\ +{"key":"IsUnreadByOwner","value":"boolean"},\ +{"key":"CreatedDate","value":"timestamp"},\ +{"key":"CreatedById","value":"string"},\ +{"key":"LastModifiedDate","value":"timestamp"},\ +{"key":"LastModifiedById","value":"string"},\ +{"key":"SystemModstamp","value":"timestamp"},\ +{"key":"LastActivityDate","value":"date"},\ +{"key":"LastViewedDate","value":"timestamp"},\ +{"key":"LastReferencedDate","value":"timestamp"},\ +{"key":"Jigsaw","value":"string"},\ +{"key":"JigsawContactId","value":"string"},\ +{"key":"EmailBouncedReason","value":"string"},\ +{"key":"EmailBouncedDate","value":"timestamp"},\ +{"key":"IndividualId","value":"string"},\ +{"key":"Follow_Up__c","value":"boolean"},\ +{"key":"npe01__Preferred_Email__c","value":"string"},\ +{"key":"npe01__Preferred_Phone__c","value":"string"},\ +{"key":"npsp__Batch__c","value":"string"},\ +{"key":"npsp__CompanyCity__c","value":"string"},\ +{"key":"npsp__CompanyCountry__c","value":"string"},\ +{"key":"npsp__CompanyPostalCode__c","value":"string"},\ +{"key":"npsp__CompanyState__c","value":"string"},\ +{"key":"npsp__CompanyStreet__c","value":"string"}] -account.schema= [{"key":"Id","value":"string"},{"key":"IsDeleted","value":"boolean"},\ - {"key":"MasterRecordId","value":"string"},{"key":"Name","value":"string"},\ - {"key":"Type","value":"string"},\ - {"key":"ParentId","value":"string"},{"key":"BillingStreet","value":"string"},{"key":"BillingCity","value":"string"},\ - {"key":"BillingState","value":"string"},{"key":"BillingPostalCode","value":"string"},\ - {"key":"BillingCountry","value":"string"},\ - {"key":"BillingLatitude","value":"double"},{"key":"BillingLongitude","value":"double"},\ - {"key":"BillingGeocodeAccuracy","value":"string"},{"key":"ShippingStreet","value":"string"},\ - {"key":"ShippingCity","value":"string"},{"key":"ShippingState","value":"string"},\ - {"key":"ShippingPostalCode","value":"string"},{"key":"ShippingCountry","value":"string"},\ - {"key":"ShippingLatitude","value":"double"},{"key":"ShippingLongitude","value":"double"},\ - {"key":"ShippingGeocodeAccuracy","value":"string"},{"key":"Phone","value":"string"},\ - {"key":"Website","value":"string"},{"key":"PhotoUrl","value":"string"},\ - {"key":"Industry","value":"string"},\ - {"key":"NumberOfEmployees","value":"int"},{"key":"Description","value":"string"},{"key":"OwnerId","value":"string"},\ - {"key":"CreatedDate","value":"timestamp"},{"key":"CreatedById","value":"string"},\ - {"key":"LastModifiedDate","value":"timestamp"},\ - {"key":"LastModifiedById","value":"string"},{"key":"SystemModstamp","value":"timestamp"},\ - {"key":"LastActivityDate","value":"date"},{"key":"LastViewedDate","value":"timestamp"},\ - {"key":"LastReferencedDate","value":"timestamp"},{"key":"Jigsaw","value":"string"},\ - {"key":"JigsawCompanyId","value":"string"},{"key":"AccountSource","value":"string"},\ - {"key":"SicDesc","value":"string"}] + +account.schema= [{"key":"Id","value":"string"},\ +{"key":"IsDeleted","value":"boolean"},\ +{"key":"MasterRecordId","value":"string"},\ +{"key":"Name","value":"string"},\ +{"key":"Type","value":"string"},\ +{"key":"RecordTypeId","value":"string"},\ +{"key":"ParentId","value":"string"},\ +{"key":"BillingStreet","value":"string"},\ +{"key":"BillingCity","value":"string"},\ +{"key":"BillingState","value":"string"},\ +{"key":"BillingPostalCode","value":"string"},\ +{"key":"BillingCountry","value":"string"},\ +{"key":"BillingLatitude","value":"double"},\ +{"key":"BillingLongitude","value":"double"},\ +{"key":"BillingGeocodeAccuracy","value":"string"},\ +{"key":"ShippingStreet","value":"string"},\ +{"key":"ShippingCity","value":"string"},\ +{"key":"ShippingState","value":"string"},\ +{"key":"ShippingPostalCode","value":"string"},\ +{"key":"ShippingCountry","value":"string"},\ +{"key":"ShippingLatitude","value":"double"},\ +{"key":"ShippingLongitude","value":"double"},\ +{"key":"ShippingGeocodeAccuracy","value":"string"},\ +{"key":"Phone","value":"string"},\ +{"key":"Fax","value":"string"},\ +{"key":"AccountNumber","value":"string"},\ +{"key":"Website","value":"string"},\ +{"key":"PhotoUrl","value":"string"},\ +{"key":"Sic","value":"string"},\ +{"key":"Industry","value":"string"},\ +{"key":"AnnualRevenue","value":"double"},\ +{"key":"NumberOfEmployees","value":"int"},\ +{"key":"Ownership","value":"string"},\ +{"key":"TickerSymbol","value":"string"},\ +{"key":"Description","value":"string"},\ +{"key":"Rating","value":"string"},\ +{"key":"Site","value":"string"},\ +{"key":"OwnerId","value":"string"},\ +{"key":"CreatedDate","value":"timestamp"},\ +{"key":"CreatedById","value":"string"},\ +{"key":"LastModifiedDate","value":"timestamp"},\ +{"key":"LastModifiedById","value":"string"},\ +{"key":"SystemModstamp","value":"timestamp"},\ +{"key":"LastActivityDate","value":"date"},\ +{"key":"LastViewedDate","value":"timestamp"},\ +{"key":"LastReferencedDate","value":"timestamp"},\ +{"key":"Jigsaw","value":"string"},\ +{"key":"JigsawCompanyId","value":"string"},\ +{"key":"AccountSource","value":"string"},\ +{"key":"SicDesc","value":"string"},\ +{"key":"Copy_Billing_Address_to_Shipping_Address__c","value":"boolean"},\ +{"key":"Test_External_Id__c","value":"string"},\ +{"key":"custom_field__c","value":"string"},\ +{"key":"npe01__One2OneContact__c","value":"string"},\ +{"key":"npe01__SYSTEMIsIndividual__c","value":"boolean"},\ +{"key":"npe01__SYSTEM_AccountType__c","value":"string"},\ +{"key":"npo02__AverageAmount__c","value":"double"},\ +{"key":"npo02__Best_Gift_Year_Total__c","value":"double"},\ +{"key":"npo02__Best_Gift_Year__c","value":"string"},\ +{"key":"npo02__FirstCloseDate__c","value":"date"},\ +{"key":"npo02__Formal_Greeting__c","value":"string"},\ +{"key":"npo02__HouseholdPhone__c","value":"string"},\ +{"key":"npo02__Informal_Greeting__c","value":"string"},\ +{"key":"npo02__LargestAmount__c","value":"double"},\ +{"key":"npo02__LastCloseDate__c","value":"date"},\ +{"key":"npo02__LastMembershipAmount__c","value":"double"},\ +{"key":"npo02__LastMembershipDate__c","value":"date"},\ +{"key":"npo02__LastMembershipLevel__c","value":"string"},\ +{"key":"npo02__LastMembershipOrigin__c","value":"string"},\ +{"key":"npo02__LastOppAmount__c","value":"double"},\ +{"key":"npo02__MembershipEndDate__c","value":"date"},\ +{"key":"npo02__MembershipJoinDate__c","value":"date"},\ +{"key":"npo02__NumberOfClosedOpps__c","value":"double"},\ +{"key":"npo02__NumberOfMembershipOpps__c","value":"double"},\ +{"key":"npo02__OppAmount2YearsAgo__c","value":"double"},\ +{"key":"npo02__OppAmountLastNDays__c","value":"double"},\ +{"key":"npo02__OppAmountLastYear__c","value":"double"},\ +{"key":"npo02__OppAmountThisYear__c","value":"double"},\ +{"key":"npo02__OppsClosed2YearsAgo__c","value":"double"},\ +{"key":"npo02__OppsClosedLastNDays__c","value":"double"},\ +{"key":"npo02__OppsClosedLastYear__c","value":"double"},\ +{"key":"npo02__OppsClosedThisYear__c","value":"double"},\ +{"key":"npo02__SYSTEM_CUSTOM_NAMING__c","value":"string"},\ +{"key":"npo02__SmallestAmount__c","value":"double"},\ +{"key":"npo02__TotalMembershipOppAmount__c","value":"double"},\ +{"key":"npo02__TotalOppAmount__c","value":"double"},\ +{"key":"npsp__Membership_Span__c","value":"double"},\ +{"key":"npsp__Membership_Status__c","value":"string"},\ +{"key":"npsp__Sustainer__c","value":"string"},\ +{"key":"npsp__All_Members_Deceased__c","value":"boolean"},\ +{"key":"npsp__Batch__c","value":"string"},\ +{"key":"npsp__CustomizableRollups_UseSkewMode__c","value":"boolean"},\ +{"key":"npsp__Funding_Focus__c","value":"string"},\ +{"key":"npsp__Grantmaker__c","value":"boolean"},\ +{"key":"npsp__Matching_Gift_Administrator_Name__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Amount_Max__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Amount_Min__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Annual_Employee_Max__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Comments__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Company__c","value":"boolean"},\ +{"key":"npsp__Matching_Gift_Email__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Info_Updated__c","value":"date"},\ +{"key":"npsp__Matching_Gift_Percent__c","value":"double"},\ +{"key":"npsp__Matching_Gift_Phone__c","value":"string"},\ +{"key":"npsp__Matching_Gift_Request_Deadline__c","value":"string"},\ +{"key":"npsp__Number_of_Household_Members__c","value":"double"},\ +{"key":"npsp__Undeliverable_Address__c","value":"boolean"}] CreateBQTableQueryFile=BigQuery/BigQueryCreateTableQuery.txt InsertBQDataQueryFile=BigQuery/BigQueryInsertDataQuery.txt @@ -132,7 +252,7 @@ bigQueryDatatypesColumnsList=(Id,Name,Col_Timestamp__c,Col_Date__c,Col_Currency_ testData={"Name":"hello","Col_Timestamp__c":"2023-06-14T07:04:56.000+0000","Col_Date__c":"2023-06-14",\ "Col_Currency__c":123.456,"Col_Email__c":"hello443@gmail.com","Col_Number__c":1008.0,\ - "Col_GeoLocation__Latitude__s":37.794016,"Col_GeoLocation__Longitude__s":-122.395016,"Col__c":"984746334",\ + "Col__c":"984746334",\ "Col_Url__c":"abc/123","Col_Time__c":"05:00:00.000Z","Col_Text__c":"shsss"} testObjectData={"Name":"dummy","Col_Timestamp__c":"2023-06-14T07:04:56.000+0000","Col_Date__c":"2023-06-14",\