Skip to content

Commit c869fe5

Browse files
authored
Upgrade Deequ to 2.0.8 (#241)
- This version introduces default parameters for certain classes in Deequ, which breaks PyDeequ. - As part of this change, we pass in the default value for these parameters at the time of constructing these classes.
1 parent 48ed442 commit c869fe5

File tree

2 files changed

+21
-6
lines changed

2 files changed

+21
-6
lines changed

pydeequ/configs.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@
55

66

77
SPARK_TO_DEEQU_COORD_MAPPING = {
8-
"3.5": "com.amazon.deequ:deequ:2.0.7-spark-3.5",
9-
"3.3": "com.amazon.deequ:deequ:2.0.7-spark-3.3",
10-
"3.2": "com.amazon.deequ:deequ:2.0.7-spark-3.2",
11-
"3.1": "com.amazon.deequ:deequ:2.0.7-spark-3.1"
8+
"3.5": "com.amazon.deequ:deequ:2.0.8-spark-3.5",
9+
"3.3": "com.amazon.deequ:deequ:2.0.8-spark-3.3",
10+
"3.2": "com.amazon.deequ:deequ:2.0.8-spark-3.2",
11+
"3.1": "com.amazon.deequ:deequ:2.0.8-spark-3.1"
1212
}
1313

1414

pydeequ/suggestions.py

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -229,8 +229,11 @@ def rule_jvm(self):
229229
default_category_sorter = scala_get_default_argument(
230230
self._deequSuggestions.rules.FractionalCategoricalRangeRule, 2
231231
)
232+
default_interval_strategy = scala_get_default_argument(
233+
self._deequSuggestions.rules.FractionalCategoricalRangeRule, 3
234+
)
232235
return self._deequSuggestions.rules.FractionalCategoricalRangeRule(
233-
self.targetDataCoverageFraction, default_category_sorter
236+
self.targetDataCoverageFraction, default_category_sorter, default_interval_strategy
234237
)
235238

236239

@@ -252,7 +255,19 @@ class RetainCompletenessRule(_RulesObject):
252255

253256
@property
254257
def rule_jvm(self):
255-
return self._deequSuggestions.rules.RetainCompletenessRule()
258+
default_min_completeness = scala_get_default_argument(
259+
self._deequSuggestions.rules.RetainCompletenessRule, 1
260+
)
261+
default_max_completeness = scala_get_default_argument(
262+
self._deequSuggestions.rules.RetainCompletenessRule, 2
263+
)
264+
default_interval_strategy = scala_get_default_argument(
265+
self._deequSuggestions.rules.RetainCompletenessRule, 3
266+
)
267+
268+
return self._deequSuggestions.rules.RetainCompletenessRule(
269+
default_min_completeness, default_max_completeness, default_interval_strategy
270+
)
256271

257272

258273
class RetainTypeRule(_RulesObject):

0 commit comments

Comments
 (0)