Skip to content

Commit bdc4404

Browse files
AnkitCLIAnkitCLI
committed
SF e2e test fixes
1 parent 6edaa7a commit bdc4404

File tree

11 files changed

+237
-108
lines changed

11 files changed

+237
-108
lines changed

‎src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTime.feature‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@
1515
@SalesforceSalesCloud
1616
@SFMultiObjectsBatchSource
1717
@Smoke
18+
# Skipping these tests due to permission issues for creating custom fields in sObjects. Will Run it once it get resolves.
1819
@Regression
1920

2021
Feature: Salesforce Multi Objects Batch Source - Run time Scenarios
2122

22-
@MULTIBATCH-TS-SF-RNTM-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2
23+
@MULTIBATCH-TS-SF-RNTM-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890
2324
Scenario: Verify user should be able to preview, deploy and run pipeline for valid White List
2425
When Open Datafusion Project to configure pipeline
2526
And Select data pipeline type as: "Batch"
@@ -58,7 +59,7 @@ Feature: Salesforce Multi Objects Batch Source - Run time Scenarios
5859
Then Validate the values of records transferred to target Big Query table is equal to the values from multi object source table
5960

6061

61-
@MULTIBATCH-TS-SF-RNTM-02 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2
62+
@MULTIBATCH-TS-SF-RNTM-02 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890
6263
Scenario: Verify user should be able to preview, deploy and run pipeline for valid Black List
6364
When Open Datafusion Project to configure pipeline
6465
And Select data pipeline type as: "Batch"
@@ -99,7 +100,7 @@ Feature: Salesforce Multi Objects Batch Source - Run time Scenarios
99100
Then Validate the values of records transferred to target Big Query table is equal to the values from multi object source table
100101

101102

102-
@MULTIBATCH-TS-SF-RNTM-03 @CONNECTION @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2
103+
@MULTIBATCH-TS-SF-RNTM-03 @CONNECTION @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890
103104
Scenario: Verify user should be able to deploy and run the pipeline using connection manager functionality
104105
When Open Datafusion Project to configure pipeline
105106
And Select plugin: "Salesforce Multi Objects" from the plugins list as: "Source"

‎src/e2e-test/features/salesforcemultiobjectsbatchsource/RunTimeWithMacros.feature‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,12 @@
1515
@SalesforceSalesCloud
1616
@SFMultiObjectsBatchSource
1717
@Smoke
18+
# Skipping these tests due to permission issues for creating custom fields in sObjects. Will Run it once it get resolves.
1819
@Regression
1920

2021
Feature: Salesforce Multi Objects Batch Source - Run time Scenarios with Macro
2122

22-
@MULTIBATCH-TS-SF-RNTM-MACRO-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2
23+
@MULTIBATCH-TS-SF-RNTM-MACRO-01 @BQ_SINK_MULTI_TEST @CREATE_TEST_DATA @CREATE_TEST_DATA2 @DELETE_TEST_DATA @DELETE_TEST_DATA2 @Plugin-1890
2324
Scenario: Verify user should be able to preview, deploy a pipeline when plugin is configured with macros for WhiteList
2425
When Open Datafusion Project to configure pipeline
2526
And Select data pipeline type as: "Batch"

‎src/e2e-test/features/salesforcestreamingsource/DesignTime.feature‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ Feature: Salesforce Streaming Source - Design time scenarios
2727
And Navigate to the properties page of plugin: "Salesforce"
2828
And fill Authentication properties for Salesforce Admin user
2929
And Enter input plugin property: "pushTopicName" with value: "topic.name"
30+
And Enter input plugin property: "sObjectName" with value: "sobject.Automation_custom_c"
3031
And Select dropdown plugin property: "pushTopicNotifyCreate" with option value: "Enabled"
3132
And Select dropdown plugin property: "pushTopicNotifyUpdate" with option value: "Enabled"
3233
And Select dropdown plugin property: "pushTopicNotifyDelete" with option value: "Enabled"

‎src/e2e-test/java/io/cdap/plugin/BQValidation.java‎

Lines changed: 34 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ public class BQValidation {
5151

5252
public static boolean validateSalesforceAndBQRecordValues(String objectName, String bqTable) throws
5353
IOException, InterruptedException {
54-
String uniqueRecordId = SalesforceClient.queryObjectId(objectName);
54+
List<String> uniqueRecordIds = SalesforceClient.queryObjectId(objectName);
5555

5656
List<JsonObject> bigQueryResponse = new ArrayList<>();
5757
List<Object> bigQueryRows = new ArrayList<>();
@@ -60,8 +60,11 @@ public static boolean validateSalesforceAndBQRecordValues(String objectName, Str
6060
JsonObject jsonData = gson.fromJson(String.valueOf(rows), JsonObject.class);
6161
bigQueryResponse.add(jsonData);
6262
}
63-
List<JsonObject> sObjectResponse;
64-
sObjectResponse = SalesforceClient.queryObject(uniqueRecordId, objectName);
63+
List<JsonObject> sObjectResponse = new ArrayList<>();
64+
for (String recordId : uniqueRecordIds) {
65+
JsonObject record = SalesforceClient.queryObject(recordId, objectName);
66+
sObjectResponse.add(record);
67+
}
6568
return compareSalesforceAndJsonData(sObjectResponse, bigQueryResponse, bqTable);
6669
}
6770

@@ -82,9 +85,12 @@ public static boolean validateSalesforceMultiObjectToBQRecordValues() throws IOE
8285
JsonObject jsonData = gson.fromJson(String.valueOf(row), JsonObject.class);
8386
bigQueryResponse.add(jsonData);
8487
}
85-
String uniqueRecordId = SalesforceClient.queryObjectId(currentObject);
86-
List<JsonObject> sObjectResponse;
87-
sObjectResponse = SalesforceClient.queryObject(uniqueRecordId, currentObject);
88+
List<String> uniqueRecordIds = SalesforceClient.queryObjectId(currentObject);
89+
List<JsonObject> sObjectResponse = new ArrayList<>();
90+
for (String recordId : uniqueRecordIds) {
91+
JsonObject record = SalesforceClient.queryObject(recordId, currentObject);
92+
sObjectResponse.add(record);
93+
}
8894
boolean isValid = compareSalesforceAndJsonData(
8995
sObjectResponse, bigQueryResponse, currentTargetTable);
9096

@@ -141,14 +147,9 @@ public static boolean compareSalesforceAndJsonData(List<JsonObject> salesforceDa
141147
Assert.fail("bigQueryData is null");
142148
return result;
143149
}
144-
int jsonObjectIdx = 0;
145-
if (salesforceData.size() > 0) {
146-
salesforceData.get(jsonObjectIdx).entrySet().size();
147-
}
148-
// Get the column count of the first JsonObject in bigQueryData
149-
int columnCountSource = 0;
150-
if (bigQueryData.size() > 0) {
151-
columnCountSource = bigQueryData.get(jsonObjectIdx).entrySet().size();
150+
if (salesforceData.isEmpty() || bigQueryData.isEmpty()) {
151+
Assert.fail("One or both datasets are empty");
152+
return result;
152153
}
153154

154155
BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
@@ -158,39 +159,39 @@ public static boolean compareSalesforceAndJsonData(List<JsonObject> salesforceDa
158159
TableId tableRef = TableId.of(projectId, dataset, tableName);
159160
// Get the table schema
160161
Schema schema = bigQuery.getTable(tableRef).getDefinition().getSchema();
161-
// Iterate over the fields
162-
int currentColumnCount = 1;
163-
while (currentColumnCount <= columnCountSource) {
162+
163+
for (int rowIndex = 0; rowIndex < salesforceData.size(); rowIndex++) {
164+
JsonObject salesforceRow = salesforceData.get(rowIndex);
165+
JsonObject bigQueryRow = bigQueryData.get(rowIndex);
166+
164167
for (Field field : schema.getFields()) {
165168
String columnName = field.getName();
166169
String columnType = field.getType().toString();
167170

168171
switch (columnType) {
169172

170173
case "BOOLEAN":
171-
boolean sourceAsBoolean = salesforceData.get(jsonObjectIdx).get(columnName).getAsBoolean();
172-
boolean targetAsBoolean = bigQueryData.get(jsonObjectIdx).get(columnName).getAsBoolean();
174+
boolean sourceAsBoolean = salesforceRow.get(columnName).getAsBoolean();
175+
boolean targetAsBoolean = bigQueryRow.get(columnName).getAsBoolean();
173176
Assert.assertEquals("Different values found for column : %s", sourceAsBoolean, targetAsBoolean);
174177
break;
175178

176179
case "FLOAT":
177-
double sourceVal = salesforceData.get(jsonObjectIdx).get(columnName).getAsDouble();
178-
double targetVal = bigQueryData.get(jsonObjectIdx).get(columnName).getAsDouble();
180+
double sourceVal = salesforceRow.get(columnName).getAsDouble();
181+
double targetVal = bigQueryRow.get(columnName).getAsDouble();
179182
Assert.assertEquals(String.format("Different values found for column: %s", columnName), 0,
180183
Double.compare(sourceVal, targetVal));
181184
break;
182185

183186
case "TIMESTAMP":
184187
OffsetDateTime sourceTimestamp = OffsetDateTime.parse(
185-
salesforceData.get(jsonObjectIdx)
186-
.get(columnName)
188+
salesforceRow.get(columnName)
187189
.getAsString(),
188190
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
189191
);
190192

191193
OffsetDateTime targetTimestamp = OffsetDateTime.parse(
192-
bigQueryData.get(jsonObjectIdx)
193-
.get(columnName)
194+
bigQueryRow.get(columnName)
194195
.getAsString()
195196
);
196197
Assert.assertEquals("Different values found for column : %s", sourceTimestamp, targetTimestamp);
@@ -200,23 +201,21 @@ public static boolean compareSalesforceAndJsonData(List<JsonObject> salesforceDa
200201
DateTimeFormatter formatterSource = DateTimeFormatter.ofPattern("HH:mm:ss.SSSX");
201202
DateTimeFormatter formatterTarget = DateTimeFormatter.ofPattern("HH:mm:ss");
202203
LocalTime sourceTime = LocalTime.parse(
203-
salesforceData.get(jsonObjectIdx)
204-
.get(columnName)
204+
salesforceRow.get(columnName)
205205
.getAsString(), formatterSource
206206
);
207207
LocalTime targetTime = LocalTime.parse(
208-
bigQueryData.get(jsonObjectIdx)
209-
.get(columnName)
208+
bigQueryRow.get(columnName)
210209
.getAsString(), formatterTarget
211210
);
212211
Assert.assertEquals("Different values found for column : %s", sourceTime, targetTime);
213212
break;
214213

215214
case "DATE":
216-
JsonElement jsonElementSource = salesforceData.get(jsonObjectIdx).get(columnName);
215+
JsonElement jsonElementSource = salesforceRow.get(columnName);
217216
Date sourceDate = (jsonElementSource != null && !jsonElementSource.isJsonNull()) ? Date.valueOf(
218217
jsonElementSource.getAsString()) : null;
219-
JsonElement jsonElementTarget = bigQueryData.get(jsonObjectIdx).get(columnName);
218+
JsonElement jsonElementTarget = bigQueryRow.get(columnName);
220219
Date targetDate = (jsonElementTarget != null && !jsonElementTarget.isJsonNull()) ? Date.valueOf(
221220
jsonElementTarget.getAsString()) : null;
222221
Assert.assertEquals("Different values found for column : %s", sourceDate, targetDate);
@@ -231,21 +230,19 @@ public static boolean compareSalesforceAndJsonData(List<JsonObject> salesforceDa
231230
if (columnName.equals("Col_GeoLocation__c")) {
232231
break;
233232
} else {
234-
JsonElement sourceElement = salesforceData.get(jsonObjectIdx).get(columnName);
233+
JsonElement sourceElement = salesforceRow.get(columnName);
235234
String sourceString = (sourceElement != null && !sourceElement.isJsonNull())
236235
? sourceElement.getAsString() : null;
237-
JsonElement targetElement = bigQueryData.get(jsonObjectIdx).get(columnName);
236+
JsonElement targetElement = bigQueryRow.get(columnName);
238237
String targetString = (targetElement != null && !targetElement.isJsonNull())
239238
? targetElement.getAsString() : null;
240239
Assert.assertEquals(String.format("Different values found for column : %s", columnName),
241240
String.valueOf(sourceString), String.valueOf(targetString));
242-
break;
241+
}
242+
break;
243243
}
244244
}
245-
currentColumnCount++;
246245
}
247-
jsonObjectIdx++;
248-
}
249246
Assert.assertFalse("Number of rows in Source table is greater than the number of rows in Target table",
250247
salesforceData.size() > bigQueryData.size());
251248
return true;

‎src/e2e-test/java/io/cdap/plugin/salesforcestreamingsource/stepsdesign/DesignTimeSteps.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
import java.io.IOException;
3737
import java.io.UnsupportedEncodingException;
38+
import java.util.List;
3839
import java.util.concurrent.TimeUnit;
3940
/**
4041
* Design-time steps of Salesforce Streaming plugins.
@@ -70,8 +71,10 @@ public void fillUniqueTopicNameInRuntimeArguments(String runtimeArgumentKey) {
7071

7172
@Then("Update existing salesforce records")
7273
public void updateExistingSalesforceRecords() {
73-
String uniqueRecordId = SalesforceClient.queryObjectId(customObject);
74-
SalesforceClient.updateObject(uniqueRecordId, customObject);
74+
List<String> uniqueRecordId = SalesforceClient.queryObjectId(customObject);
75+
for (String record : uniqueRecordId) {
76+
SalesforceClient.updateObject(record, customObject);
77+
}
7578
}
7679

7780

‎src/e2e-test/java/io/cdap/plugin/tests/hooks/TestSetupHooks.java‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
import java.nio.file.Files;
3838
import java.nio.file.Paths;
3939
import java.text.ParseException;
40+
import java.util.List;
4041
import java.util.NoSuchElementException;
4142
import java.util.UUID;
4243

@@ -65,15 +66,19 @@ public static void createTestObject2() throws UnsupportedEncodingException, Pars
6566

6667
@After(order = 2, value = "@DELETE_TEST_DATA")
6768
public static void deleteTestObject() {
68-
String uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM__C.value);
69-
SalesforceClient.deleteId(uniqueRecordId, SObjects.AUTOMATION_CUSTOM__C.value);
69+
List<String> uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM__C.value);
70+
for (String record : uniqueRecordId) {
71+
SalesforceClient.deleteId(record, SObjects.AUTOMATION_CUSTOM__C.value);
72+
}
7073
BeforeActions.scenario.write("Record - " + uniqueRecordId + " deleted successfully");
7174
}
7275

7376
@After(order = 2, value = "@DELETE_TEST_DATA2")
7477
public static void deleteTestObject2() {
75-
String uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM2__C.value);
76-
SalesforceClient.deleteId(uniqueRecordId, SObjects.AUTOMATION_CUSTOM2__C.value);
78+
List<String> uniqueRecordId = SalesforceClient.queryObjectId(SObjects.AUTOMATION_CUSTOM2__C.value);
79+
for (String record : uniqueRecordId) {
80+
SalesforceClient.deleteId(record, SObjects.AUTOMATION_CUSTOM__C.value);
81+
}
7782
BeforeActions.scenario.write("Record - " + uniqueRecordId + " deleted successfully");
7883
}
7984

‎src/e2e-test/java/io/cdap/plugin/tests/runner/TestRunner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
"io.cdap.plugin.salesforcemultiobjectsbatchsource.stepsdesign",
3333
"io.cdap.plugin.bigquery.stepsdesign",
3434
"io.cdap.plugin.tests.hooks", "stepsdesign"},
35-
tags = {"@Regression"},
35+
tags = {"@Regression and not @Plugin-1890"},
3636
monochrome = true,
3737
plugin = {"pretty", "html:target/cucumber-html-report", "json:target/cucumber-reports/cucumber.json",
3838
"junit:target/cucumber-reports/cucumber.xml"}

‎src/e2e-test/java/io/cdap/plugin/utils/SalesforceClient.java‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@ public static String createObject(JSONObject objectJson, String objectName) thro
149149
return uniqueRecordId;
150150
}
151151

152-
public static List<JsonObject> queryObject(String id, String objectName) {
152+
public static JsonObject queryObject(String id, String objectName) {
153153
getAccessToken();
154154
HttpClient httpClient = HttpClientBuilder.create().build();
155155
String baseUri = loginInstanceUrl + REST_ENDPOINT + API_VERSION;
@@ -167,12 +167,12 @@ public static List<JsonObject> queryObject(String id, String objectName) {
167167
String responseString = EntityUtils.toString(response.getEntity());
168168
Gson gson = new Gson();
169169
JsonObject objectResponseInJson = gson.fromJson(responseString, JsonObject.class);
170-
sfobjectResponse.add(objectResponseInJson);
170+
return objectResponseInJson;
171171
}
172172
} catch (IOException ioException) {
173173
logger.info("Error in establishing connection to Salesforce: " + ioException);
174174
}
175-
return sfobjectResponse;
175+
return null;
176176
}
177177

178178
public static void deletePushTopic(String pushTopicName) {
@@ -233,10 +233,11 @@ public static void deleteId(String id, String objectName) {
233233
}
234234
}
235235

236-
public static String queryObjectId(String objectName) {
236+
public static List<String> queryObjectId(String objectName) {
237237
getAccessToken();
238238
HttpClient httpClient = HttpClientBuilder.create().build();
239239
String baseUri = loginInstanceUrl + REST_ENDPOINT + API_VERSION;
240+
List<String> idList = new ArrayList<>();
240241

241242
try {
242243
String query = "SELECT Id FROM " + objectName;
@@ -259,14 +260,15 @@ public static String queryObjectId(String objectName) {
259260
JsonArray records = queryResponse.getAsJsonArray("records");
260261
for (JsonElement record : records) {
261262
JsonObject recordObject = record.getAsJsonObject();
262-
uniqueRecordId = recordObject.get("Id").getAsString();
263-
logger.info("Queried Object id from response: " + uniqueRecordId);
263+
String id = recordObject.get("Id").getAsString();
264+
idList.add(id);
265+
logger.info("Queried Object id from response: " + idList);
264266
}
265267
}
266268
} catch (IOException ioException) {
267269
logger.info("Error in establishing connection to Salesforce: " + ioException);
268270
}
269-
return uniqueRecordId;
271+
return idList;
270272
}
271273

272274
public static void updateObject(String id, String objectName) {

‎src/e2e-test/resources/BigQuery/BigQueryCreateTableQuery.txt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,2 @@
11
create table `DATASET.TABLE_NAME` (Name STRING, Col_Timestamp__c TIMESTAMP, Col_Date__c DATE, Col_Currency__c FLOAT64,
2-
Col_Email__c STRING, Col_Number__c FLOAT64, Col_GeoLocation__Latitude__s FLOAT64,
3-
Col_GeoLocation__Longitude__s FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING)
2+
Col_Email__c STRING, Col_Number__c FLOAT64, Col__c STRING, Col_Url__c STRING, Col_Time__c TIME, Col_Text__c STRING)

‎src/e2e-test/resources/BigQuery/BigQueryInsertDataQuery.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
insert into `DATASET.TABLE_NAME` (Name, Col_Timestamp__c, Col_Date__c, Col_Currency__c, Col_Email__c, Col_Number__c,
2-
Col_GeoLocation__Latitude__s, Col_GeoLocation__Longitude__s, Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values
3-
('adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',898365444,37.794116,-122.3432,
2+
Col__c, Col_Url__c, Col_Time__c, Col_Text__c) values
3+
('adam','2019-03-10 04:50:01 UTC','2021-01-28',61.823765812,'skfdsfds@gmail.com',898365444,
44
'984746334','abc/123','20:26:34','find');
55

0 commit comments

Comments
 (0)