Fixing integration test cases (#197)

Author: apoorvmittal10

kcbq-connector/src/integration-test/java/com/wepay/kafka/connect/bigquery/it/utils/TableClearer.java

@@ -21,6 +21,7 @@
 import com.google.cloud.bigquery.BigQuery;
 
 import com.wepay.kafka.connect.bigquery.BigQueryHelper;
+import com.wepay.kafka.connect.bigquery.utils.FieldNameSanitizer;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -44,10 +45,14 @@ public static void main(String[] args) {
     }
     BigQuery bigQuery = new BigQueryHelper().setKeySource(keySource).connect(args[1], args[0]);
     for (int i = tablesStart; i < args.length; i++) {
-      if (bigQuery.delete(args[2], args[i])) {
-        logger.info("Table {} in dataset {} deleted successfully", args[i], args[2]);
+      // May be consider using sanitizeTopics property value in future to decide table name
+      // sanitization but as currently we always run test cases with sanitizeTopics value as true
+      // hence sanitize table name prior delete. This is required else it makes test cases flaky.
+      String table = FieldNameSanitizer.sanitizeName(args[i]);
+      if (bigQuery.delete(args[2], table)) {
+        logger.info("Table {} in dataset {} deleted successfully", table, args[2]);
       } else {
-        logger.info("Table {} in dataset {} does not exist", args[i], args[2]);
+        logger.info("Table {} in dataset {} does not exist", table, args[2]);
       }
     }
   }

kcbq-connector/test/integrationtest.sh

@@ -215,15 +215,15 @@ warn 'Deleting existing BigQuery test tables and existing GCS bucket'
 test_tables=
 test_topics=
 for file in "$BASE_DIR"/resources/test_schemas/*; do
-        test_tables+="${test_tables:+ }$(basename "kcbq_test_${file/-/_}")"
-        test_topics+="${test_topics:+,}$(basename "kcbq_test_$file")"
+        test_tables+="${test_tables:+ }kcbq_test_$(basename "${file/-/_}")"
+        test_topics+="${test_topics:+,}kcbq_test_$(basename "$file")"
 done
 
 "$GRADLEW" -p "$BASE_DIR/.." \
     -Pkcbq_test_keyfile="$KCBQ_TEST_KEYFILE" \
     -Pkcbq_test_project="$KCBQ_TEST_PROJECT" \
     -Pkcbq_test_dataset="$KCBQ_TEST_DATASET" \
-    -Pkcbq_test_tables="test_tables" \
+    -Pkcbq_test_tables="$test_tables" \
     -Pkcbq_test_bucket="$KCBQ_TEST_BUCKET" \
     -Pkcbq_test_keysource="$KCBQ_TEST_KEYSOURCE" \
     integrationTestPrep