make release-tag: Merge branch 'main' into stable

Author: amontanez24

HISTORY.md

@@ -1,5 +1,21 @@
 # History
 
+## v1.18.2 - 2025-10-15
+
+### Bugs Fixed
+
+* Error when fitting using nullable integer data with categorical transformer - Issue [#1036](https://github.com/sdv-dev/RDT/issues/1036) by @R-Palazzo
+
+### Maintenance
+
+* Remove excluding Faker v37.11.0 when they have released the fix for python 3.9 - Issue [#1043](https://github.com/sdv-dev/RDT/issues/1043) by @R-Palazzo
+
+## v1.18.1 - 2025-09-11
+
+### Bugs Fixed
+
+* fill_nan_with_none produces pandas FutureWarning - Issue [#1027](https://github.com/sdv-dev/RDT/issues/1027) by @R-Palazzo
+
 ## v1.18.0 - 2025-08-14
 
 ### New Features

latest_requirements.txt

@@ -1,6 +1,6 @@
-Faker==37.5.3
+Faker==37.8.0
 copulas==0.12.3
-numpy==2.3.2
-pandas==2.3.1
-scikit-learn==1.7.1
-scipy==1.16.1
+numpy==2.3.3
+pandas==2.3.3
+scikit-learn==1.7.2
+scipy==1.16.2

pyproject.toml

@@ -39,7 +39,7 @@ dependencies = [
     "scikit-learn>=1.1.3;python_version>='3.11' and python_version<'3.12'",
     "scikit-learn>=1.3.1;python_version>='3.12' and python_version<'3.13'",
     "scikit-learn>=1.5.2;python_version>='3.13'",
-    'Faker>=17',
+    'Faker>=17, !=37.11.0',
     'python-dateutil>=2.9',
 ]
 
@@ -144,7 +144,7 @@ collect_ignore = ['pyproject.toml']
 exclude_lines = ['NotImplementedError()']
 
 [tool.bumpversion]
-current_version = "1.18.0"
+current_version = "1.18.2.dev1"
 parse = '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?'
 serialize = [
     '{major}.{minor}.{patch}.{release}{candidate}',

rdt/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = '[email protected]'
-__version__ = '1.18.0'
+__version__ = '1.18.2.dev1'
 
 
 import sys

rdt/transformers/categorical.py

@@ -59,12 +59,12 @@ def _order_categories(self, unique_data):
         nans = pd.isna(unique_data)
         if self.order_by == 'alphabetical':
             # pylint: disable=invalid-unary-operand-type
-            if any(map(lambda item: not isinstance(item, str), unique_data[~nans])):  # noqa: C417
+            if any(not isinstance(item, str) for item in unique_data[~nans]):
                 raise TransformerInputError(
                     "The data must be of type string if order_by is 'alphabetical'."
                 )
         elif self.order_by == 'numerical_value':
-            if not np.issubdtype(unique_data.dtype.type, np.number):
+            if any(not np.issubdtype(type(item), np.number) for item in unique_data[~nans]):
                 raise TransformerInputError(
                     "The data must be numerical if order_by is 'numerical_value'."
                 )

rdt/transformers/utils.py

@@ -175,6 +175,17 @@ def strings_from_regex(regex, max_repeat=16):
     return _from_generators(generators, max_repeat), np.prod(sizes, dtype=np.complex128).real
 
 
+def _fill_nan_with_none_series(data):
+    dtype = data.dtype
+    if isinstance(dtype, pd.CategoricalDtype):
+        sentinel = object()
+        data = data.cat.add_categories([sentinel])
+        data = data.fillna(sentinel).replace({sentinel: None})
+        return pd.Series(pd.Categorical(data, categories=dtype.categories), index=data.index)
+
+    return data.astype('object').where(~data.isna(), None)
+
+
 def fill_nan_with_none(data):
     """Replace all nan values with None.
 
@@ -185,7 +196,10 @@ def fill_nan_with_none(data):
         data:
             Original data with nan values replaced by None.
     """
-    return data.infer_objects().fillna(np.nan).replace([np.nan], [None])
+    if isinstance(data, pd.DataFrame):
+        return data.apply(_fill_nan_with_none_series)
+
+    return _fill_nan_with_none_series(data)
 
 
 def flatten_column_list(column_list):

static_code_analysis.txt

@@ -1,10 +1,10 @@
-Run started:2025-08-14 20:37:52.987737
+Run started:2025-10-15 16:09:17.276277
 
 Test results:
 	No issues identified.
 
 Code scanned:
-	Total lines of code: 6323
+	Total lines of code: 6333
 	Total lines skipped (#nosec): 0
 	Total potential issues skipped due to specifically being disabled (e.g., #nosec BXXX): 0
 

tests/unit/transformers/test_categorical.py

@@ -198,6 +198,24 @@ def test__fit(self):
         assert transformer.frequencies == expected_frequencies
         assert transformer.intervals == expected_intervals
 
+    def test_fit_with_nullable_integer_dtype(self):
+        """Test that the ``fit`` method works with nullable integer columns."""
+        # Setup
+        data = pd.DataFrame({'example': [1, 2, 3, None]}, dtype='Int64')
+        transformer = UniformEncoder()
+
+        # Run
+        transformer.fit(data=data, column='example')
+
+        # Assert
+        expected_frequencies = {
+            1: 0.25,
+            2: 0.25,
+            3: 0.25,
+            None: 0.25,
+        }
+        assert transformer.frequencies == expected_frequencies
+
     def test__set_fitted_parameters(self):
         """Test the ``_set_fitted_parameters`` method."""
         # Setup

tests/unit/transformers/test_utils.py

@@ -16,6 +16,7 @@
     _any,
     _cast_to_type,
     _extract_timezone_from_a_string,
+    _fill_nan_with_none_series,
     _get_utc_offset,
     _handle_enforce_uniqueness_and_cardinality_rule,
     _max_repeat,
@@ -167,6 +168,45 @@ def test_fill_nan_with_none_no_warning():
     pd.testing.assert_series_equal(result, expected)
 
 
+def test__fill_nan_with_none_series():
+    """Test the ``_fill_nan_with_none_series`` method."""
+    # Setup
+    series = pd.Series([1.0, 2.0, 3.0, np.nan], dtype='object')
+    categorical_serie = pd.Series(['a', 'b', 'c', 'd', np.nan], dtype='category')
+
+    # Run
+    result = _fill_nan_with_none_series(series)
+    result_categorical = _fill_nan_with_none_series(categorical_serie)
+
+    # Assert
+    expected_result = pd.Series([1.0, 2.0, 3.0, None], dtype='object')
+    pd.testing.assert_series_equal(result, expected_result)
+    expected_result_categorical = pd.Series(
+        pd.Categorical(['a', 'b', 'c', 'd', None], categories=['a', 'b', 'c', 'd'])
+    )
+    pd.testing.assert_series_equal(result_categorical, expected_result_categorical)
+
+
+def test_fill_nan_with_none_series():
+    """Test the `fill_nan_with_none_series` function."""
+    # Setup
+    series = pd.Series([1.0, 2.0, 3.0, np.nan], dtype='object')
+    data = pd.DataFrame({'col1': series})
+    data_2 = pd.DataFrame({'col1': series, 'col2': ['a', 'b', 'c', np.nan]})
+
+    # Run
+    result_series = _fill_nan_with_none_series(series)
+    result_data = fill_nan_with_none(data)
+    result_data_2 = fill_nan_with_none(data_2)
+
+    # Assert
+    expected_result = pd.Series([1.0, 2.0, 3.0, None], dtype='object')
+    expected_result_data_2 = pd.DataFrame({'col1': expected_result, 'col2': ['a', 'b', 'c', None]})
+    pd.testing.assert_series_equal(result_series, expected_result)
+    pd.testing.assert_frame_equal(result_data, pd.DataFrame({'col1': expected_result}))
+    pd.testing.assert_frame_equal(result_data_2, expected_result_data_2)
+
+
 def test_check_nan_in_transform():
     """Test ``check_nan_in_transform`` method.