Modify 'PatternMatch' function in analyzers.py and 'hasPattern' function in checks.py (#66)

* Add code to func: hasPattern, cuz there is no code in ver1.0.0

* modify PatternMatch func, cuz previous regex will not match anything

* add test cases for analyzer.PatternMatch & checker.hasPattern

---------

Co-authored-by: Jason <[email protected]>
Co-authored-by: Ethan_C_Lin <[email protected]>

Author: 3 people

pydeequ/analyzers.py

@@ -664,7 +664,7 @@ class PatternMatch(_AnalyzerObject):
 
     def __init__(self, column, pattern_regex: str, *pattern_groupNames, where: str = None):
         self.column = column
-        self.pattern_regex = (pattern_regex,)
+        self.pattern_regex = pattern_regex
         if pattern_groupNames:
             raise NotImplementedError("pattern_groupNames have not been implemented yet.")
         self.pattern_groupNames = None
@@ -679,7 +679,7 @@ def _analyzer_jvm(self):
         """
         return self._deequAnalyzers.PatternMatch(
             self.column,
-            self._jvm.scala.util.matching.Regex(str(self.pattern_regex), None),
+            self._jvm.scala.util.matching.Regex(self.pattern_regex, None),
             # TODO: revisit bc scala constructor does some weird implicit type casting from python str -> java list
             #  if we don't cast it to str()
             self._jvm.scala.Option.apply(self.where),

pydeequ/checks.py

@@ -564,6 +564,13 @@ def hasPattern(self, column, pattern, assertion=None, name=None, hint=None):
         :param str hint: A hint that states why a constraint could have failed.
         :return: hasPattern self: A Check object that runs the condition on the column.
         """
+        assertion_func = ScalaFunction1(self._spark_session.sparkContext._gateway, assertion) if assertion \
+            else getattr(self._Check, "hasPattern$default$2")()
+        name = self._jvm.scala.Option.apply(name)
+        hint = self._jvm.scala.Option.apply(hint)
+        pattern_regex = self._jvm.scala.util.matching.Regex(pattern, None)
+        self._Check = self._Check.hasPattern(column, pattern_regex, assertion_func, name, hint)
+        return self
 
     def containsCreditCardNumber(self, column, assertion=None, hint=None):
         """

tests/test_analyzers.py

@@ -493,8 +493,18 @@ def test_MutualInformation(self):
     def test_fail_MutualInformation(self):
         self.assertEqual(self.MutualInformation(["b", "d"]), [])
 
-    # TODO: Revisit when PatternMatch class is sorted out
     def test_PatternMatch(self):
+        result = (
+            self.AnalysisRunner.onData(self.df).addAnalyzer(PatternMatch(column="a", pattern_regex="ba(r|z)")).run()
+        )
+        result_df = AnalyzerContext.successMetricsAsDataFrame(self.spark, result)
+        result_json = AnalyzerContext.successMetricsAsJson(self.spark, result)
+        df_from_json = self.spark.read.json(self.sc.parallelize([result_json]))
+        self.assertEqual(df_from_json.select("value").collect(), result_df.select("value").collect())
+        self.assertEqual(result_df.select("value").collect(), [Row(value=0.6666666666666666)])
+
+    @pytest.mark.xfail(reason="@unittest.expectedFailure")
+    def test_fail_PatternMatch(self):
         result = (
             self.AnalysisRunner.onData(self.df).addAnalyzer(PatternMatch(column="a", pattern_regex="ba(r|z)")).run()
         )

tests/test_checks.py

@@ -395,6 +395,18 @@ def satisfies(self, columnCondition, constraintName, assertion=None, hint=None):
         df = VerificationResult.checkResultsAsDataFrame(self.spark, result)
         return df.select("constraint_status").collect()
 
+    def hasPattern(self, column, pattern, assertion=None, name=None, hint=None):
+        check = Check(self.spark, CheckLevel.Warning, "test hasPattern")
+        result = (
+            VerificationSuite(self.spark)
+            .onData(self.df)
+            .addCheck(check.hasPattern(column, pattern, assertion, name, hint))
+            .run()
+        )
+
+        df = VerificationResult.checkResultsAsDataFrame(self.spark, result)
+        return df.select("constraint_status").collect()
+
     def isLessThanOrEqualTo(self, columnA, columnB, assertion=None, hint=None):
         check = Check(self.spark, CheckLevel.Warning, "test isLessThanOrEqualTo")
         result = (
@@ -1120,6 +1132,14 @@ def test_fail_satisfies(self):
         )
         self.assertEqual(self.satisfies('a = "zoo"', "find a", lambda x: x == 1), [Row(constraint_status="Success")])
 
+    def test_hasPattern(self):
+        self.assertEqual(self.hasPattern("ssn", "\d{3}\-\d{2}\-\d{4}", lambda x: x == 2 / 3), [Row(constraint_status="Success")])
+
+    @pytest.mark.xfail(reason="@unittest.expectedFailure")
+    def test_fail_hasPattern(self):
+        self.assertEqual(self.hasPattern("ssn", r"\d{3}\-\d{2}\-\d{4}", lambda x: x == 2 / 3), [Row(constraint_status="Failure")])
+        self.assertEqual(self.hasPattern("ssn", r"\d{3}\d{2}\d{4}", lambda x: x == 2 / 3), [Row(constraint_status="Failure")])
+        
     def test_isNonNegative(self):
         self.assertEqual(self.isNonNegative("b"), [Row(constraint_status="Success")])
         self.assertEqual(