Merge remote-tracking branch 'origin/issue/CDM-13' into issue/97

Author: mieslep

.github/workflows/cdm-integrationtest.yml

@@ -0,0 +1,23 @@
+name: Build and test jar with integration tests
+run-name: ${{ github.actor }} is testing out GitHub Actions 🚀
+on: [push]
+jobs:
+  Explore-GitHub-Actions:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up JDK 8
+        uses: actions/setup-java@v3
+        with:
+          java-version: '8'
+          distribution: 'temurin'
+          cache: maven
+      - name: Build with Maven
+        run: mvn -B package --file pom.xml
+
+      - name: Test SIT with cdm
+        working-directory: SIT
+        run: |
+          echo "Testing SIT with cdm"
+          chmod o+rx ./environment.sh
+          make -f Makefile

SIT/Makefile

@@ -1,6 +1,8 @@
 
-all: setup test_smoke env_teardown
+all: set_permissions setup test_smoke env_teardown
 
+set_permissions:
+	chmod -R 777 ./*.sh
 setup: build env_setup
 
 reset: build env_reset
@@ -13,7 +15,7 @@ test_smoke: reset test_smoke_cmd
 test_smoke_cmd:
 	./test.sh -p smoke
 
-env_setup: 
+env_setup:
 	./environment.sh -m setup -j ../target/cassandra-data-migrator*.jar
 env_reset:
 	./environment.sh -m reset -j ../target/cassandra-data-migrator*.jar

SIT/cdm.sh

@@ -16,7 +16,6 @@
     <connector.version>3.2.0</connector.version>
     <cassandra.version>3.11.13</cassandra.version>
     <junit.version>5.9.1</junit.version>
-    <mockito.version>2.28.2</mockito.version>
   </properties>
 
   <distributionManagement>

SIT/common.sh

@@ -67,6 +67,7 @@ public void getDataAndInsert(BigInteger min, BigInteger max) {
                             printCounts(false);
                         }
 
+                        // exclusion filter below
                         if (cqlHelper.hasFilterColumn()) {
                             String col = (String) cqlHelper.getData(cqlHelper.getFilterColType(), cqlHelper.getFilterColIndex(), originRow);
                             if (col.trim().equalsIgnoreCase(cqlHelper.getFilterColValue())) {

SIT/environment.sh

@@ -40,6 +40,19 @@ public MigrateDataType(String dataType) {
         }
         this.typeClass = getType(this.type);
 
+        if (this.type >= minType && this.type <= maxType) {
+            isValid = true;
+            for (Object o : subTypes) {
+                if (null == o || Object.class == o) {
+                    isValid = false;
+                }
+            }
+        }
+        else {
+            isValid = false;
+        }
+        this.typeClass = getType(this.type);
+
         if (this.type >= minType && this.type <= maxType) {
             isValid = true;
             for (Object o : subTypes) {
@@ -138,64 +151,6 @@ public boolean isValid() {
         return isValid;
     }
 
-    public boolean parse(String s) {
-        try {
-            switch (this.type) {
-                case 0:
-                    return true;
-                case 1:
-                    Integer.parseInt(s);
-                    return true;
-                case 2:
-                    Long.parseLong(s);
-                    return true;
-                case 3:
-                    Double.parseDouble(s);
-                    return true;
-                case 4:
-                    Instant.parse(s);
-                    return true;
-                case 9:
-                    UUID.fromString(s);
-                    return true;
-                case 10:
-                    Boolean.parseBoolean(s);
-                    return true;
-                case 12:
-                    Float.parseFloat(s);
-                    return true;
-                case 13:
-                    Byte.parseByte(s);
-                    return true;
-                case 14:
-                    BigDecimal.valueOf(Double.parseDouble(s));
-                    return true;
-                case 15:
-                    LocalDate.parse(s);
-                    return true;
-                case 17:
-                    BigInteger.valueOf(Long.parseLong(s));
-                    return true;
-                case 18:
-                    LocalTime.parse(s);
-                    return true;
-                case 19:
-                    Short.parseShort(s);
-                    return true;
-                case 5: // Map
-                case 6: // List
-                case 7: // ByteBuffer
-                case 8: // Set
-                case 11: // TupleValue
-                case 16: // UDT
-                default:
-                    throw new IllegalArgumentException("MigrateDataType " + this.type + "(" + this.typeClass.getName() + ") cannot currently parse this type");
-            }
-        } catch (NumberFormatException e) {
-            return false;
-        }
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

SIT/test.sh

@@ -14,25 +14,19 @@
 public class Util {
 
     public static String getSparkProp(SparkConf sc, String prop) {
-        if (!sc.contains(prop)) {
-            return sc.get(prop.replace("origin", "source").replace("target", "destination"));
+        String retVal = PropertyHelper.getInstance(sc).getAsString(prop);
+        if (StringUtils.isEmpty(retVal) && (prop.contains("origin") || prop.contains("target"))) {
+            retVal = PropertyHelper.getInstance(sc).getAsString(prop.replace("origin", "source").replace("target", "destination"));
         }
         if (!KnownProperties.isKnown(prop)) {
             throw new IllegalArgumentException("Unknown property: " + prop + "; this is a bug in the code: the property is not configured in KnownProperties.java");
         }
-        return PropertyHelper.getInstance(sc).getAsString(prop);
+        return retVal;
     }
 
     public static String getSparkPropOr(SparkConf sc, String prop, String defaultVal) {
-        if (!sc.contains(prop)) {
-            return sc.get(prop.replace("origin", "source").replace("target", "destination"), defaultVal);
-        }
         String retVal = getSparkProp(sc,prop);
-        if (null == retVal) {
-            PropertyHelper.getInstance(sc).setProperty(prop, defaultVal);
-            return getSparkProp(sc,prop);
-        }
-        return retVal;
+        return StringUtils.isEmpty(retVal) ? defaultVal : retVal;
     }
 
     public static String getSparkPropOrEmpty(SparkConf sc, String prop) {