Refactor testing modules + testing spark3 bump

Author: Calvin Wang

pydeequ/__init__.py

@@ -19,7 +19,8 @@
 from pydeequ.analyzers import AnalysisRunner
 from pydeequ.checks import Check, CheckLevel
 
-deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"
+# deequ_maven_coord = "com.amazon.deequ:deequ:1.1.0_spark-2.4-scala-2.11"
+deequ_maven_coord = "com.amazon.deequ:deequ:1.1.0_spark-3.0-scala-2.12"
 f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all"
 
 class PyDeequSession:

tests/test_analyzers.py

@@ -1,14 +1,12 @@
 import unittest
 from pyspark.sql import SparkSession, Row, DataFrame
 from pydeequ.analyzers import *
-from pydeequ import PyDeequSession
+from pydeequ import *
 
 class TestAnalyzers(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"  # TODO: get Maven Coord from Configs
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all"  # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                       .builder
                       .master('local[*]')

tests/test_anomaly_detection.py

@@ -4,12 +4,11 @@
 from pydeequ.anomaly_detection import *
 from pydeequ.repository import *
 from pydeequ.analyzers import *
+from pydeequ import *
 
 class TestAnomalies(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all" # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                  .builder
                  .master('local[*]')

tests/test_checks.py

@@ -2,14 +2,11 @@
 from pyspark.sql import SparkSession, Row, DataFrame
 from pydeequ.verification import *
 from pydeequ.checks import *
-import py4j
+from pydeequ import *
 
 class TestChecks(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        # TODO share spark context between test cases?
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3" # TODO get Maven Coord from Configs
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all" # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                  .builder
                  .master('local[*]')

tests/test_pandas_utils.py

@@ -6,16 +6,14 @@
 from pydeequ.profiles import ColumnProfilerRunBuilder, ColumnProfilerRunner
 from pydeequ.verification import *
 from pydeequ.checks import *
-from pydeequ import PyDeequSession
+from pydeequ import *
 from pandas import DataFrame as pandasDF
 import numpy as np
 
 class TestPandasUtils(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"  # TODO: get Maven Coord from Configs
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all"  # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                       .builder
                       .master('local[*]')

tests/test_profiles.py

@@ -2,13 +2,11 @@
 from pydeequ.profiles import ColumnProfilerRunBuilder, ColumnProfilerRunner
 from pydeequ.analyzers import KLLParameters
 from pyspark.sql import SparkSession, Row
-
+from pydeequ import *
 
 class TestProfiles(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all" # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                  .builder
                  .master('local[*]')

tests/test_repository.py

@@ -4,13 +4,12 @@
 from pydeequ.repository import *
 from pydeequ.verification import *
 from pydeequ.checks import *
+from pydeequ import *
 
 
 class TestRepository(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all"  # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                       .builder
                       .master('local[*]')

tests/test_scala_utils.py

@@ -1,14 +1,12 @@
 import unittest
 from pydeequ.scala_utils import ScalaFunction1, ScalaFunction2
 from pyspark.sql import SparkSession
-
+from pydeequ import *
 
 class TestScalaUtils(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        # TODO share spark context between test cases?
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3-rc2" # TODO get Maven Coord from Configs
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all" # This package is excluded because it causes an error in the SparkSession fig
+        # deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3-rc2" # TODO This ran rc2?
         cls.spark = (SparkSession
                  .builder
                  .master('local[*]')

tests/test_suggestions.py

@@ -2,13 +2,11 @@
 from pyspark.sql import SparkSession, Row, DataFrame
 from pydeequ.suggestions import *
 import json
-
+from pydeequ import *
 
 class TestSuggestions(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
-        deequ_maven_coord = "com.amazon.deequ:deequ:1.0.3"
-        f2j_maven_coord = "net.sourceforge.f2j:arpack_combined_all"  # This package is excluded because it causes an error in the SparkSession fig
         cls.spark = (SparkSession
                       .builder
                       .master('local[*]')