Skip to content

Commit e54c286

Browse files
psainicsAnkitCLI
psainics
authored and
AnkitCLI
committed
bqmt fixes
1 parent 601f62b commit e54c286

File tree

6 files changed

+92
-47
lines changed

6 files changed

+92
-47
lines changed

‎src/e2e-test/features/bigquery/multitablesink/BigQueryMultiTableAdditional.feature‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
@BigQueryMultiTable_Sink
1616
Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable successful data transfer
1717

18-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_DELETE_TABLES_TEST
18+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_TWO_SINK_BQMT_TEST
1919
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two new tables
2020
Given Open Datafusion Project to configure pipeline
2121
When Expand Plugin group in the LHS plugins list: "Source"
@@ -89,7 +89,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
8989
Then Verify the pipeline status is "Succeeded"
9090
Then Validate data transferred from BigQuery To BigQueryMultiTable in one table is equal
9191

92-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
92+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
9393
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables
9494
Given Open Datafusion Project to configure pipeline
9595
When Expand Plugin group in the LHS plugins list: "Source"
@@ -131,7 +131,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
131131
Then Verify the pipeline status is "Succeeded"
132132
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
133133

134-
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
134+
@BQ_TWO_SOURCE_BQMT_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
135135
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables using truncate
136136
Given Open Datafusion Project to configure pipeline
137137
When Expand Plugin group in the LHS plugins list: "Source"
@@ -174,7 +174,7 @@ Feature: BigQueryMultiTable sink -Verification of BigQuery to BigQueryMultiTable
174174
Then Verify the pipeline status is "Succeeded"
175175
Then Validate data transferred from BigQuery To BigQueryMultiTable is equal
176176

177-
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_DELETE_TABLES_TEST
177+
@BQ_SOURCE_UPDATE_TEST @BQ_EXISTING_TARGET_TEST @BQ_TWO_SINK_BQMT_TEST
178178
Scenario: Verify data successfully transferred from BigQuery To BigQueryMultiTable in two existing tables after updating schema
179179
Given Open Datafusion Project to configure pipeline
180180
When Expand Plugin group in the LHS plugins list: "Source"

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/sinkrunner/TestRunner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
features = {"src/e2e-test/features"},
2828
glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
2929
"stepsdesign", "io.cdap.plugin.common.stepsdesign"},
30-
tags = {"@BigQuery_Sink and not @CDAP-20830"},
30+
tags = {"@BigQuery_Sinkz and not @CDAP-20830"},
3131
//TODO: Enable test once issue is fixed https://cdap.atlassian.net/browse/CDAP-20830
3232
monochrome = true,
3333
plugin = {"pretty", "html:target/cucumber-html-report/bigquery-sink",

‎src/e2e-test/java/io/cdap/plugin/bigquery/runners/sourcerunner/TestRunner.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
features = {"src/e2e-test/features"},
2828
glue = {"io.cdap.plugin.bigquery.stepsdesign", "io.cdap.plugin.gcs.stepsdesign",
2929
"stepsdesign", "io.cdap.plugin.common.stepsdesign", "io.cdap.plugin.pubsub.stepsdesign"},
30-
tags = {"@BigQuery_Source"},
30+
tags = {"@BigQuery_Sourcez"},
3131
monochrome = true,
3232
plugin = {"pretty", "html:target/cucumber-html-report/bigquery-source",
3333
"json:target/cucumber-reports/cucumber-bigquery-source.json",

‎src/e2e-test/java/io/cdap/plugin/bigquery/stepsdesign/BigQueryMultiTable.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package io.cdap.plugin.bigquery.stepsdesign;
1818

1919
import io.cdap.e2e.utils.PluginPropertyUtils;
20+
import io.cdap.plugin.common.stepsdesign.TestSetupHooks;
2021
import io.cucumber.java.en.Then;
2122
import org.junit.Assert;
2223

@@ -35,8 +36,7 @@ public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
3536
List<String> sourceTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqSourceTable"),
3637
PluginPropertyUtils.pluginProp("bqSourceTable2"));
3738

38-
List<String> targetTables = Arrays.asList(PluginPropertyUtils.pluginProp("bqTargetTable"),
39-
PluginPropertyUtils.pluginProp("bqTargetTable2"));
39+
List<String> targetTables = Arrays.asList(TestSetupHooks.bqmtTargetTable, TestSetupHooks.bqmtTargetTable2);
4040
boolean recordsMatched = BigQueryMultiTableValidation.validateBQToBigQueryMultiTable(sourceTables, targetTables);
4141
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
4242
"of the records in the source table", recordsMatched);
@@ -46,8 +46,8 @@ public void validateDataTransferredFromBigQueryToBigQueryMultiTableIsEqual()
4646
public void validateDataTransferredFromBigQueryToBigQueryMultiTableInOneTableIsEqual()
4747
throws IOException, InterruptedException {
4848
boolean recordsMatched = BigQueryMultiTableValidation.
49-
validateBQToBigQueryMultiTable(Collections.singletonList(PluginPropertyUtils.pluginProp("bqSourceTable")),
50-
Collections.singletonList(PluginPropertyUtils.pluginProp("bqTargetTable")));
49+
validateBQToBigQueryMultiTable(Collections.singletonList(TestSetupHooks.bqSourceTable),
50+
Collections.singletonList(TestSetupHooks.bqmtTargetTable));
5151
Assert.assertTrue("Value of records transferred to the BQ sink should be equal to the value " +
5252
"of the records in the source table", recordsMatched);
5353
}

‎src/e2e-test/java/io/cdap/plugin/common/stepsdesign/TestSetupHooks.java‎

Lines changed: 77 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ public class TestSetupHooks {
6565
public static String gcsSourceBucketName = StringUtils.EMPTY;
6666
public static String gcsTargetBucketName = StringUtils.EMPTY;
6767
public static String bqTargetTable = StringUtils.EMPTY;
68+
public static String bqmtTargetTable = StringUtils.EMPTY;
69+
public static String bqmtTargetTable2 = StringUtils.EMPTY;
6870
public static String bqSourceTable = StringUtils.EMPTY;
6971
public static String bqSourceTable2 = StringUtils.EMPTY;
7072
public static String bqTargetTable2 = StringUtils.EMPTY;
@@ -294,16 +296,16 @@ public static void createTempSourceBQTable() throws IOException, InterruptedExce
294296
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " created successfully");
295297
}
296298

297-
@After(order = 1, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
299+
@After(order = 2, value = "@BQ_SOURCE_TEST or @BQ_PARTITIONED_SOURCE_TEST or @BQ_SOURCE_DATATYPE_TEST or " +
298300
"@BQ_INSERT_SOURCE_TEST or @BQ_UPDATE_SINK_TEST or @BQ_EXISTING_SOURCE_TEST or @BQ_EXISTING_SINK_TEST or " +
299301
"@BQ_EXISTING_SOURCE_DATATYPE_TEST or @BQ_EXISTING_SINK_DATATYPE_TEST or @BQ_UPSERT_SOURCE_TEST or " +
300302
"@BQ_NULL_MODE_SOURCE_TEST or @BQ_UPDATE_SOURCE_DEDUPE_TEST or @BQ_INSERT_INT_SOURCE_TEST or " +
301-
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST")
303+
"@BQ_TIME_SOURCE_TEST or @BQ_UPSERT_DEDUPE_SOURCE_TEST or @BQ_PRIMARY_RECORD_SOURCE_TEST or " +
304+
"@BQ_SINGLE_SOURCE_BQMT_TEST")
302305
public static void deleteTempSourceBQTable() throws IOException, InterruptedException {
303306
BigQueryClient.dropBqQuery(bqSourceTable);
304307
PluginPropertyUtils.removePluginProp("bqSourceTable");
305308
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
306-
bqSourceTable = StringUtils.EMPTY;
307309
}
308310

309311
/**
@@ -1554,20 +1556,22 @@ public static void emptyExistingBigTableInstanceAndTableName() {
15541556
bigtableExistingTargetTable = StringUtils.EMPTY;
15551557
}
15561558

1557-
@Before(order = 1, value = "@BQ_EXISTING_TARGET_TEST")
1559+
@Before(order = 2, value = "@BQ_EXISTING_TARGET_TEST")
15581560
public static void createSinkTables() throws IOException, InterruptedException {
1559-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1560-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1561-
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqTargetTable + "` " +
1561+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1562+
bqmtTargetTable2 = PluginPropertyUtils.pluginProp("bqmtTargetTable2");
1563+
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqmtTargetTable + "` " +
15621564
"(ID INT64, tablename STRING," +
15631565
"Price FLOAT64, Customer_Exists BOOL ) ");
15641566

1565-
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqTargetTable2 + "` " +
1567+
io.cdap.e2e.utils.BigQueryClient.getSoleQueryResult("create table `" + datasetName + "." + bqmtTargetTable + "` " +
15661568
"(ID INT64, tablename STRING," +
15671569
"Price FLOAT64, Customer_Exists BOOL ) ");
15681570

1569-
PluginPropertyUtils.addPluginProp("bqTargetTable", bqTargetTable);
1570-
PluginPropertyUtils.addPluginProp("bqTargetTable2", bqTargetTable2);
1571+
PluginPropertyUtils.addPluginProp("bqmtTargetTable", bqmtTargetTable);
1572+
PluginPropertyUtils.addPluginProp("bqmtTargetTable2", bqmtTargetTable2);
1573+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " created successfully");
1574+
BeforeActions.scenario.write("BQ Target table2 - " + bqmtTargetTable2 + " created successfully");
15711575
}
15721576
@Before(order = 1, value = "@BQ_SOURCE_UPDATE_TEST")
15731577
public static void createSourceTables() throws IOException, InterruptedException {
@@ -1602,48 +1606,87 @@ public static void createSourceTables() throws IOException, InterruptedException
16021606

16031607
PluginPropertyUtils.addPluginProp("bqSourceTable", bqSourceTable);
16041608
PluginPropertyUtils.addPluginProp("bqSourceTable2", bqSourceTable2);
1609+
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " created successfully");
1610+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " created successfully");
1611+
}
1612+
1613+
// @After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1614+
// public static void deleteAllBqTables() throws IOException, InterruptedException {
1615+
// BigQueryClient.dropBqQuery(bqSourceTable);
1616+
// BigQueryClient.dropBqQuery(bqSourceTable2);
1617+
// bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1618+
// bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1619+
// BigQueryClient.dropBqQuery(bqTargetTable);
1620+
// BigQueryClient.dropBqQuery(bqTargetTable2);
1621+
// PluginPropertyUtils.removePluginProp("bqSourceTable");
1622+
// PluginPropertyUtils.removePluginProp("bqSourceTable2");
1623+
// BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
1624+
// BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1625+
// BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1626+
// BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
1627+
// }
1628+
1629+
@After(order = 2, value = "@BQ_SINK_BQMT_TEST")
1630+
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
1631+
try {
1632+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1633+
BigQueryClient.dropBqQuery(bqmtTargetTable);
1634+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " deleted successfully");
1635+
} catch (BigQueryException e) {
1636+
if (e.getMessage().contains("Not found: Table")) {
1637+
BeforeActions.scenario.write("BQ Target Table " + bqmtTargetTable + " does not exist");
1638+
} else {
1639+
Assert.fail(e.getMessage());
1640+
}
1641+
}
16051642
}
16061643

1607-
@After(order = 1, value = "@BQ_DELETE_TABLES_TEST")
1608-
public static void deleteAllBqTables() throws IOException, InterruptedException {
1609-
BigQueryClient.dropBqQuery(bqSourceTable);
1644+
@After(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
1645+
public static void deleteTempSource2BQTable() throws IOException, InterruptedException {
16101646
BigQueryClient.dropBqQuery(bqSourceTable2);
1611-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1612-
bqTargetTable2 = PluginPropertyUtils.pluginProp("bqTargetTable2");
1613-
BigQueryClient.dropBqQuery(bqTargetTable);
1614-
BigQueryClient.dropBqQuery(bqTargetTable2);
1615-
PluginPropertyUtils.removePluginProp("bqSourceTable");
1647+
bqSourceTable2 = PluginPropertyUtils.pluginProp("bqSourceTable2");
16161648
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1617-
BeforeActions.scenario.write("BQ source Table " + bqSourceTable + " deleted successfully");
16181649
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1619-
BeforeActions.scenario.write("BQ target Table " + bqTargetTable + " deleted successfully");
1620-
BeforeActions.scenario.write("BQ target Table2 " + bqTargetTable2 + " deleted successfully");
16211650
}
16221651

1623-
@After(order = 1, value = "@BQ_SINK_BQMT_TEST")
1624-
public static void deleteTargetBqmtTable() throws IOException, InterruptedException {
1652+
@After(order = 2, value = "@BQ_TWO_SOURCE_BQMT_TEST or @BQ_SOURCE_UPDATE_TEST")
1653+
public static void deleteTwoSourceTables() throws IOException, InterruptedException {
16251654
try {
1626-
bqTargetTable = PluginPropertyUtils.pluginProp("bqTargetTable");
1627-
BigQueryClient.dropBqQuery(bqTargetTable);
16281655
BigQueryClient.dropBqQuery(bqSourceTable);
1629-
BeforeActions.scenario.write("BQ Target table - " + bqTargetTable + " deleted successfully");
1656+
BigQueryClient.dropBqQuery(bqSourceTable2);
16301657
BeforeActions.scenario.write("BQ Source table - " + bqSourceTable + " deleted successfully");
1631-
bqTargetTable = StringUtils.EMPTY;
1658+
BeforeActions.scenario.write("BQ Source table2 - " + bqSourceTable2 + " deleted successfully");
1659+
PluginPropertyUtils.removePluginProp("bqSourceTable");
1660+
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1661+
16321662
} catch (BigQueryException e) {
16331663
if (e.getMessage().contains("Not found: Table")) {
1634-
BeforeActions.scenario.write("BQ Target Table " + bqTargetTable + " does not exist");
16351664
BeforeActions.scenario.write("BQ Source Table " + bqSourceTable + " does not exist");
1665+
BeforeActions.scenario.write("BQ Source Table2 " + bqSourceTable2 + " does not exist");
16361666
} else {
16371667
Assert.fail(e.getMessage());
16381668
}
16391669
}
16401670
}
16411671

1642-
@After(order = 1, value = "@BQ_SECONDARY_RECORD_SOURCE_TEST")
1643-
public static void deleteTempSource2BQTable() throws IOException, InterruptedException {
1644-
BigQueryClient.dropBqQuery(bqSourceTable2);
1645-
bqSourceTable2 = PluginPropertyUtils.pluginProp("bqSourceTable2");
1646-
PluginPropertyUtils.removePluginProp("bqSourceTable2");
1647-
BeforeActions.scenario.write("BQ source Table2 " + bqSourceTable2 + " deleted successfully");
1672+
@After(order = 2, value = "@BQ_TWO_SINK_BQMT_TEST")
1673+
public static void deleteTwoTargetTables() throws IOException, InterruptedException {
1674+
try {
1675+
bqmtTargetTable = PluginPropertyUtils.pluginProp("bqmtTargetTable");
1676+
bqmtTargetTable2 = PluginPropertyUtils.pluginProp("bqmtTargetTable2");
1677+
BigQueryClient.dropBqQuery(bqmtTargetTable);
1678+
BigQueryClient.dropBqQuery(bqmtTargetTable2);
1679+
BeforeActions.scenario.write("BQ Target table - " + bqmtTargetTable + " deleted successfully");
1680+
BeforeActions.scenario.write("BQ Target table2 - " + bqmtTargetTable2 + " deleted successfully");
1681+
1682+
} catch (BigQueryException e) {
1683+
if (e.getMessage().contains("Not found: Table")) {
1684+
BeforeActions.scenario.write("BQ Target Table " + bqmtTargetTable + " does not exist");
1685+
BeforeActions.scenario.write("BQ Target Table2 " + bqmtTargetTable2 + " does not exist");
1686+
} else {
1687+
Assert.fail(e.getMessage());
1688+
}
1689+
}
16481690
}
1691+
16491692
}

‎src/e2e-test/resources/pluginParameters.properties‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
projectId=cdf-athena
22
datasetprojectId=cdf-athena
3-
dataset=testing_bqmt
3+
dataset=testbq_bqmt
44
wrongSourcePath=gs://00000000-e2e-0014a44f-81be-4501-8360-0ddca192492
55
serviceAccountType=filePath
66
serviceAccount=auto-detect
@@ -350,8 +350,10 @@ dsExpectedFile=testdata/BQValidationExpectedFiles/datastoreexpectedfile
350350
## BQMT-PLUGIN-PROPERTIES-START
351351
bqmtCreateTableQueryFile=testdata/BigQuery/BqmtCreateTableQuery.txt
352352
bqmtInsertDataQueryFile=testdata/BigQuery/BqmtInsertDataQuery.txt
353-
bqTargetTable=tabA
354-
bqTargetTable2=tabB
353+
bqTargetTable=dummy
354+
bqTargetTable2=dummy
355+
bqmtTargetTable=tabA
356+
bqmtTargetTable2=tabB
355357
## BQMT-PLUGIN-PROPERTIES-END
356358

357359
##CLOUDBIGTABLE-PLUGIN-PROPERTIES-START

0 commit comments

Comments
 (0)