Merge pull request #107 from datastax/issue/CDM-13

CDM-13 : properties now retrieved via helpers rather than SparkConf

Author: mieslep

pom.xml

@@ -15,7 +15,7 @@
     <scalatest.version>3.2.12</scalatest.version>
     <connector.version>3.2.0</connector.version>
     <cassandra.version>3.11.13</cassandra.version>
-    <junit.version>4.13.2</junit.version>
+    <junit.version>5.9.1</junit.version>
   </properties>
 
   <distributionManagement>
@@ -44,6 +44,10 @@
           <groupId>log4j</groupId>
           <artifactId>log4j</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.apache.logging.log4j</groupId>
+          <artifactId>log4j-slf4j-impl</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>
@@ -101,8 +105,8 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
       <version>${junit.version}</version>
       <scope>test</scope>
     </dependency>
@@ -178,9 +182,6 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
         <version>2.22.2</version>
-        <configuration>
-          <skipTests>true</skipTests>
-        </configuration>
       </plugin>
       <!-- enable scalatest -->
       <plugin>

src/main/java/datastax/astra/migrate/MigrateDataType.java

@@ -13,23 +13,52 @@
 
 public class MigrateDataType {
     Class typeClass = Object.class;
+    String dataTypeString = "";
+    int type = -1;
     List<Class> subTypes = new ArrayList<Class>();
+    private boolean isValid = false;
+    private static int minType = 0;
+    private static int maxType = 19;
 
     public MigrateDataType(String dataType) {
+        dataTypeString = dataType;
         if (dataType.contains("%")) {
             int count = 1;
             for (String type : dataType.split("%")) {
+                int typeAsInt = typeAsInt(type);
                 if (count == 1) {
-                    typeClass = getType(Integer.parseInt(type));
+                    this.type = typeAsInt;
                 } else {
-                    subTypes.add(getType(Integer.parseInt(type)));
+                    subTypes.add(getType(typeAsInt));
                 }
                 count++;
             }
         } else {
-            int type = Integer.parseInt(dataType);
-            typeClass = getType(type);
+            this.type = typeAsInt(dataType);
         }
+        this.typeClass = getType(this.type);
+
+        if (this.type >= minType && this.type <= maxType) {
+            isValid = true;
+            for (Object o : subTypes) {
+                if (null == o || Object.class == o) {
+                    isValid = false;
+                }
+            }
+        }
+        else {
+            isValid = false;
+        }
+    }
+
+    private int typeAsInt(String dataType) {
+        int rtn = -1;
+        try {
+            rtn = Integer.parseInt(dataType);
+        } catch (NumberFormatException e) {
+            rtn = -1;
+        }
+        return rtn;
     }
 
     public boolean diff(Object source, Object astra) {
@@ -91,4 +120,25 @@ private Class getType(int type) {
         return Object.class;
     }
 
+    public Class getType() {
+        return this.typeClass;
+    }
+
+    public boolean isValid() {
+        return isValid;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        MigrateDataType that = (MigrateDataType) o;
+        return type == that.type &&
+                Objects.equals(subTypes, that.subTypes);
+    }
+
+    @Override
+    public String toString() {
+        return dataTypeString;
+    }
 }

src/main/java/datastax/astra/migrate/Util.java

@@ -1,6 +1,8 @@
 package datastax.astra.migrate;
 
 import com.datastax.oss.driver.api.core.ConsistencyLevel;
+import datastax.astra.migrate.properties.KnownProperties;
+import datastax.astra.migrate.properties.PropertyHelper;
 import org.apache.commons.lang.StringUtils;
 import org.apache.spark.SparkConf;
 
@@ -12,21 +14,19 @@
 public class Util {
 
     public static String getSparkProp(SparkConf sc, String prop) {
-        try {
-            return sc.get(prop);
-        } catch (NoSuchElementException nse) {
-            String newProp = prop.replace("origin", "source").replace("target", "destination");
-            return sc.get(newProp);
+        String retVal = PropertyHelper.getInstance(sc).getAsString(prop);
+        if (StringUtils.isEmpty(retVal) && (prop.contains("origin") || prop.contains("target"))) {
+            retVal = PropertyHelper.getInstance(sc).getAsString(prop.replace("origin", "source").replace("target", "destination"));
+        }
+        if (!KnownProperties.isKnown(prop)) {
+            throw new IllegalArgumentException("Unknown property: " + prop + "; this is a bug in the code: the property is not configured in KnownProperties.java");
         }
+        return retVal;
     }
 
     public static String getSparkPropOr(SparkConf sc, String prop, String defaultVal) {
-        try {
-            return sc.get(prop);
-        } catch (NoSuchElementException nse) {
-            String newProp = prop.replace("origin", "source").replace("target", "destination");
-            return sc.get(newProp, defaultVal);
-        }
+        String retVal = getSparkProp(sc,prop);
+        return StringUtils.isEmpty(retVal) ? defaultVal : retVal;
     }
 
     public static String getSparkPropOrEmpty(SparkConf sc, String prop) {