Skip to content

Commit a2a7d51

Browse files
psainicspsainics
authored
Merge branch 'develop' into fem/big-query
2 parents 1ea8bba + b49a189 commit a2a7d51

File tree

5 files changed

+78
-45
lines changed

5 files changed

+78
-45
lines changed

‎src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
@BigQueryMultiTable_Sink
1616
Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
1717

18-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_DELETE_TABLES_TEST
18+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_TWO_SINK_BQMT_TEST
1919
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
2020
Given Open Datafusion Project to configure pipeline
2121
When Expand Plugin group in the LHS plugins list: "Source"
@@ -89,7 +89,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
8989
Then Verify the pipeline status is "Succeeded"
9090
Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
9191

92-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
92+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
9393
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
9494
Given Open Datafusion Project to configure pipeline
9595
When Expand Plugin group in the LHS plugins list: "Source"
@@ -131,7 +131,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
131131
Then Verify the pipeline status is "Succeeded"
132132
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
133133

134-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
134+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
135135
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
136136
Given Open Datafusion Project to configure pipeline
137137
When Expand Plugin group in the LHS plugins list: "Source"
@@ -174,7 +174,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
174174
Then Verify the pipeline status is "Succeeded"
175175
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
176176

177-
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
177+
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
178178
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
179179
Given Open Datafusion Project to configure pipeline
180180
When Expand Plugin group in the LHS plugins list: "Source"

‎src/e2e-test/features/gcs/source/GCSToGCSAdditonalTests.feature‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,9 @@ Feature: GCS source - Verification of GCS to GCS Additional Tests successful
266266
Then Deploy the pipeline
267267
Then Run the Pipeline in Runtime
268268
Then Wait till pipeline is in running state
269+
Then Open and capture logs
269270
And Verify the pipeline status is "Failed"
271+
Then Close the pipeline logs
270272
Then Open Pipeline logs and verify Log entries having below listed Level and Message:
271273
| Level | Message |
272274
| ERROR | errorMessageMultipleFileWithFirstRowAsHeaderEnabled |
@@ -300,7 +302,9 @@ Feature: GCS source - Verification of GCS to GCS Additional Tests successful
300302
Then Deploy the pipeline
301303
Then Run the Pipeline in Runtime
302304
Then Wait till pipeline is in running state
305+
Then Open and capture logs
303306
And Verify the pipeline status is "Failed"
307+
Then Close the pipeline logs
304308
Then Open Pipeline logs and verify Log entries having below listed Level and Message:
305309
| Level | Message |
306310
| ERROR | errorMessageMultipleFileWithoutClearDefaultSchema |

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package io.cdap.plugin.bigquery.stepsdesign;
1818

1919
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
2021
import io.cucumber.java.en.Then;
2122
import org.junit.Assert;
2223

@@ -32,11 +33,10 @@ public class BigQueryMultiTable {
3233
@Then("Validate data transferred from BigQuery To BigQueryMultiTable is equal")
3334
public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
3435
throws IOException, InterruptedException {
35-
List<String> sourceTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqSourceTable"),
36-
PluginPropertyUtils.pluginProp("bqSourceTable2"));
36+
List<String> sourceTables = Arrays.asList(TestSetupHooks.bqSourceTable, TestSetupHooks.bqSourceTable2);
3737

38-
List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqTargetTable"),
39-
PluginPropertyUtils.pluginProp("bqTargetTable2"));
38+
List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqmtTargetTable"),
39+
PluginPropertyUtils.pluginProp("bqmtTargetTable2"));
4040
boolean recordsMatched = BigQueryMultiTableValidation.validateBQToBigQueryMultiTable(sourceTables, targetTables);
4141
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
4242
"of the records in the source table", recordsMatched);
@@ -46,8 +46,8 @@ public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
4646
public void validateDataTransferredFromBigQueryToBigQueryMultiTableInOneTableIsEqual()
4747
throws IOException, InterruptedException {
4848
boolean recordsMatched = BigQueryMultiTableValidation.
49-
validateBQToBigQueryMultiTable(Collections.singletonList(PluginPropertyUtils.pluginProp("bqSourceTable")),
50-
Collections.singletonList(PluginPropertyUtils.pluginProp("bqTargetTable")));
49+
validateBQToBigQueryMultiTable(Collections.singletonList(TestSetupHooks.bqSourceTable),
50+
Collections.singletonList(TestSetupHooks.bqmtTargetTable));
5151
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
5252
"of the records in the source table", recordsMatched);
5353
}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 59 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ public class TestSetupHooks {
6565
public static String gcsSourceBucketName = StringUtils.EMPTY;
6666
public static String gcsTargetBucketName = StringUtils.EMPTY;
6767
public static String bqTargetTable = StringUtils.EMPTY;
68+
public static String bqmtTargetTable = StringUtils.EMPTY;
69+
public static String bqmtTargetTable2 = StringUtils.EMPTY;
6870
public static String bqSourceTable = StringUtils.EMPTY;
6971
public static String bqSourceTable2 = StringUtils.EMPTY;
7072
public static String bqTargetTable2 = StringUtils.EMPTY;
@@ -298,7 +300,8 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce
298300
"@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_EXISTING_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or " +
299301
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST or @BQ_UPSERT_SOURCE_TEST or " +
300302
"@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_INSERT_INT_SOURCE_TEST or " +
301-
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST")
303+
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST or " +
304+
"@BQ_SINGLE_SOURCE_BQMT_TEST")
302305
public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
303306
BigQueryClient.dropBqQuery(bqSourceTable);
304307
PluginPropertyUtils.removePluginProp("bqSourceTable");
@@ -1554,20 +1557,22 @@ public static void emptyExistingBigTableInstanceAndTableName() {
15541557
bigtableExistingTargetTable = StringUtils.EMPTY;
15551558
}
15561559

1557-
@Before(order = 1, value = "@BQ_EXISTING_TARGET_TEST")
1560+
@Before(order = 2, value = "@BQ_EXISTING_TARGET_TEST")
15581561
public static void createSinkTables() throws IOException, InterruptedException {
1559-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1560-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1561-
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqTargetTable + "` " +
1562+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1563+
bqmtTargetTable2 = PluginPropertyUtils.pluginProp("bqmtTargetTable2");
1564+
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqmtTargetTable + "` " +
15621565
"(ID INT64, tablename STRING," +
15631566
"Price FLOAT64, Customer_Exists BOOL ) ");
15641567

1565-
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqTargetTable2 + "` " +
1568+
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqmtTargetTable2 + "` " +
15661569
"(ID INT64, tablename STRING," +
15671570
"Price FLOAT64, Customer_Exists BOOL ) ");
15681571

1569-
PluginPropertyUtils.addPluginProp("bqTargetTable", bqTargetTable);
1570-
PluginPropertyUtils.addPluginProp("bqTargetTable2", bqTargetTable2);
1572+
PluginPropertyUtils.addPluginProp("bqmtTargetTable", bqmtTargetTable);
1573+
PluginPropertyUtils.addPluginProp("bqmtTargetTable2", bqmtTargetTable2);
1574+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " created successfully");
1575+
BeforeActions.scenario.write("BQ Target table2 - " + bqmtTargetTable2 + " created successfully");
15711576
}
15721577
@Before(order = 1, value = "@BQ_SOURCE_UPDATE_TEST")
15731578
public static void createSourceTables() throws IOException, InterruptedException {
@@ -1602,48 +1607,70 @@ public static void createSourceTables() throws IOException, InterruptedException
16021607

16031608
PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable);
16041609
PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
1610+
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " created successfully");
1611+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " created successfully");
16051612
}
16061613

1607-
@After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1608-
public static void deleteAllBqTables() throws IOException, InterruptedException {
1609-
BigQueryClient.dropBqQuery(bqSourceTable);
1614+
@After(order = 2, value = "@BQ_SINK_BQMT_TEST")
1615+
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
1616+
try {
1617+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1618+
BigQueryClient.dropBqQuery(bqmtTargetTable);
1619+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " deleted successfully");
1620+
} catch (BigQueryException e) {
1621+
if (e.getMessage().contains("Not found: Table")) {
1622+
BeforeActions.scenario.write("BQ Target Table " + bqmtTargetTable + " does not exist");
1623+
} else {
1624+
Assert.fail(e.getMessage());
1625+
}
1626+
}
1627+
}
1628+
1629+
@After(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
1630+
public static void deleteTempSource2BQTable() throws IOException, InterruptedException {
16101631
BigQueryClient.dropBqQuery(bqSourceTable2);
1611-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1612-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1613-
BigQueryClient.dropBqQuery(bqTargetTable);
1614-
BigQueryClient.dropBqQuery(bqTargetTable2);
1615-
PluginPropertyUtils.removePluginProp("bqSourceTable");
1632+
bqSourceTable2 = PluginPropertyUtils.pluginProp("bqSourceTable2");
16161633
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1617-
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
16181634
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1619-
BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1620-
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
16211635
}
16221636

1623-
@After(order = 1, value = "@BQ_SINK_BQMT_TEST")
1624-
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
1637+
@After(order = 2, value = "@BQ_TWO_SOURCE_BQMT_TEST or @BQ_SOURCE_UPDATE_TEST")
1638+
public static void deleteTwoSourceTables() throws IOException, InterruptedException {
16251639
try {
1626-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1627-
BigQueryClient.dropBqQuery(bqTargetTable);
16281640
BigQueryClient.dropBqQuery(bqSourceTable);
1629-
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1641+
BigQueryClient.dropBqQuery(bqSourceTable2);
16301642
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
1631-
bqTargetTable = StringUtils.EMPTY;
1643+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " deleted successfully");
1644+
PluginPropertyUtils.removePluginProp("bqSourceTable");
1645+
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1646+
16321647
} catch (BigQueryException e) {
16331648
if (e.getMessage().contains("Not found: Table")) {
1634-
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
16351649
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
1650+
BeforeActions.scenario.write("BQ Source Table2 " + bqSourceTable2 + " does not exist");
16361651
} else {
16371652
Assert.fail(e.getMessage());
16381653
}
16391654
}
16401655
}
16411656

1642-
@After(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
1643-
public static void deleteTempSource2BQTable() throws IOException, InterruptedException {
1644-
BigQueryClient.dropBqQuery(bqSourceTable2);
1645-
bqSourceTable2 = PluginPropertyUtils.pluginProp("bqSourceTable2");
1646-
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1647-
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1657+
@After(order = 2, value = "@BQ_TWO_SINK_BQMT_TEST")
1658+
public static void deleteTwoTargetTables() throws IOException, InterruptedException {
1659+
try {
1660+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1661+
bqmtTargetTable2 = PluginPropertyUtils.pluginProp("bqmtTargetTable2");
1662+
BigQueryClient.dropBqQuery(bqmtTargetTable);
1663+
BigQueryClient.dropBqQuery(bqmtTargetTable2);
1664+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " deleted successfully");
1665+
BeforeActions.scenario.write("BQ Target table2 - " + bqmtTargetTable2 + " deleted successfully");
1666+
1667+
} catch (BigQueryException e) {
1668+
if (e.getMessage().contains("Not found: Table")) {
1669+
BeforeActions.scenario.write("BQ Target Table " + bqmtTargetTable + " does not exist");
1670+
BeforeActions.scenario.write("BQ Target Table2 " + bqmtTargetTable2 + " does not exist");
1671+
} else {
1672+
Assert.fail(e.getMessage());
1673+
}
1674+
}
16481675
}
16491676
}

‎src/e2e-test/resources/pluginParameters.properties‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
projectId=cdf-athena
22
datasetprojectId=cdf-athena
3-
dataset=testing_bqmt
3+
dataset=testbq_bqmt
44
wrongSourcePath=gs://00000000-e2e-0014a44f-81be-4501-8360-0ddca192492
55
serviceAccountType=filePath
66
serviceAccount=auto-detect
@@ -350,8 +350,10 @@ dsExpectedFile=testdata/BQValidationExpectedFiles/datastoreexpectedfile
350350
## BQMT-PLUGIN-PROPERTIES-START
351351
bqmtCreateTableQueryFile=testdata/BigQuery/BqmtCreateTableQuery.txt
352352
bqmtInsertDataQueryFile=testdata/BigQuery/BqmtInsertDataQuery.txt
353-
bqTargetTable=tabA
354-
bqTargetTable2=tabB
353+
bqTargetTable=dummy
354+
bqTargetTable2=dummy
355+
bqmtTargetTable=tabA
356+
bqmtTargetTable2=tabB
355357
## BQMT-PLUGIN-PROPERTIES-END
356358

357359
##CLOUDBIGTABLE-PLUGIN-PROPERTIES-START

0 commit comments

Comments
 (0)