make release-tag: Merge branch 'master' into stable

amontanez24 · amontanez24 · commit cd7f5d81ff37 · 2023-07-12T15:34:31.000-05:00
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
diff --git a/.github/workflows/minimum.yml b/.github/workflows/minimum.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v1
-    - name: Set up Python 3.8
+    - name: Set up Python 3.9
       uses: actions/setup-python@v2
       with:
-        python-version: 3.8
+        python-version: 3.9
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
diff --git a/.github/workflows/readme.yml b/.github/workflows/readme.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest, macos-latest]   # skip windows bc rundoc fails
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v1
diff --git a/HISTORY.md b/HISTORY.md
@@ -1,5 +1,29 @@
 # History
 
+## 1.6.0 -2023-07-12
+
+This release adds the ability to generate missing values to the `AnonymizedFaker`. Users can now provide the `missing_value_generation` parameter during initialization. They can set it to `None` to not generate any missing values, or `'random'` to generate random missing values in the same proportion as the fitted data.
+
+Additionally, this release improves the `NullTransformer` by allowing nulls to be replaced on the forward transform even if `missing_value_generation` is set to None. It also fixes a bug that was causing the `UnixTimestampEncoder` to return a different dtype than the input on `reverse_transform`. This was particularly problematic when datetime columns are represented as ints.
+
+### New Features
+
+* AnonymizedFaker should be able to model and generate missing values - Issue [#660](https://github.com/sdv-dev/RDT/issues/660) by @R-Palazzo
+
+### Bugs
+
+* The datetime transformers don't give me back the same dtype sometimes - Issue [#657](https://github.com/sdv-dev/RDT/issues/657) by @frances-h
+* RDT NullTransformer doesn't replace nulls if missing_value_generation is None - Issue [#658](https://github.com/sdv-dev/RDT/issues/658) by @amontanez24
+
+### Maintenance
+
+* Remove python 3.7 builds - Issue [#663](https://github.com/sdv-dev/RDT/issues/663) by @amontanez24
+* Drop support for Python 3.7 - Issue [#666](https://github.com/sdv-dev/RDT/issues/666) by @amontanez24
+
+### Internal
+
+* Add add-on modules to sys.modules - Issue [#653](https://github.com/sdv-dev/RDT/issues/653) by @amontanez24
+
 ## 1.5.0 - 2023-06-01
 
 This release adds a new parameter called `missing_value_generation` to the initialization of certain transformers to specify how missing values should be created. The parameter can be used in the `FloatFormatter`, `BinaryEncoder`, `UnixTimestampEncoder`, `OptimizedTimestampEncoder`, `GaussianNormalizer` and `ClusterBasedNormalizer`. Additionally, it fixes a bug that was causing every column that had nulls to generate them in the same place.
diff --git a/INSTALL.md b/INSTALL.md
@@ -3,7 +3,7 @@
 ## Requirements
 
 **RDT** has been developed and tested on
-[Python 3.7, 3.8, 3.9, 3.10 and 3.11](https://www.python.org/downloads/)
+[Python 3.8, 3.9, 3.10 and 3.11](https://www.python.org/downloads/)
 
 Also, although it is not strictly required, the usage of a [virtualenv](
 https://virtualenv.pypa.io/en/latest/) is highly recommended in order to avoid
diff --git a/rdt/__init__.py b/rdt/__init__.py
@@ -5,12 +5,13 @@
 
 __author__ = 'DataCebo, Inc.'
 __email__ = 'info@sdv.dev'
-__version__ = '1.5.0'
+__version__ = '1.6.0.dev1'
 
 
 import sys
 import warnings
 from operator import attrgetter
+from types import ModuleType
 
 import numpy as np
 import pandas as pd
@@ -140,7 +141,13 @@ def _get_addon_target(addon_path_name):
 
 
 def _find_addons():
-    """Find and load all RDT add-ons."""
+    """Find and load all RDT add-ons.
+
+    If the add-on is a module, we add it both to the target module and to
+    ``system.modules`` so that they can be imported from the top of a file as follows:
+
+    from top_module.addon_module import x
+    """
     group = 'rdt_modules'
     for entry_point in iter_entry_points(group=group):
         try:
@@ -157,6 +164,11 @@ def _find_addons():
             warnings.warn(msg)
             continue
 
+        if isinstance(addon, ModuleType):
+            addon_module_name = f'{addon_target.__name__}.{addon_name}'
+            if addon_module_name not in sys.modules:
+                sys.modules[addon_module_name] = addon
+
         setattr(addon_target, addon_name, addon)
 
 
diff --git a/rdt/transformers/addons/addons_setup.py b/rdt/transformers/addons/addons_setup.py
@@ -43,7 +43,6 @@ def _build_setup(addon_json):
             'License :: Free for non-commercial use',
             'Natural Language :: English',
             'Programming Language :: Python :: 3',
-            'Programming Language :: Python :: 3.7',
             'Programming Language :: Python :: 3.8',
             'Programming Language :: Python :: 3.9',
             'Programming Language :: Python :: 3.10',
@@ -59,7 +58,7 @@ def _build_setup(addon_json):
         long_description_content_type='text/markdown',
         name=addon_name,
         packages=find_namespace_packages(include=[addon_module]),
-        python_requires='>=3.7,<3.12',
+        python_requires='>=3.8,<3.12',
         url='https://github.com/sdv-dev/RDT',
         version=RDT_VERSION,
         zip_safe=False,
diff --git a/rdt/transformers/datetime.py b/rdt/transformers/datetime.py
@@ -1,7 +1,7 @@
 """Transformer for datetime data."""
 import numpy as np
 import pandas as pd
-from pandas.api.types import is_datetime64_dtype
+from pandas.api.types import is_datetime64_dtype, is_numeric_dtype
 from pandas.core.tools.datetimes import _guess_datetime_format_for_array
 
 from rdt.transformers.base import BaseTransformer
@@ -55,7 +55,26 @@ def __init__(self, missing_value_replacement='mean', model_missing_values=None,
         self._dtype = None
 
     def _convert_to_datetime(self, data):
-        if data.dtype == 'object':
+        """Convert datetime column into datetime dtype.
+
+        Convert the datetime column to datetime dtype using the ``datetime_format``.
+        All non-numeric columns will automatically be cast to datetimes. Numeric columns
+        with a ``datetime_format`` will be treated as strings and cast to datetime. Numeric
+        columns without a ``datetime_format`` will be treated as already converted datetimes.
+
+        Args:
+            data (pandas.Series):
+                The datetime column.
+
+        Raises:
+            - ``TypeError`` if data cannot be converted to datetime.
+            - ``ValueError`` if data does not match the specified datetime format
+
+        Returns:
+            pandas.Series:
+                The datetime column converted to the datetime dtype.
+        """
+        if self.datetime_format or not is_numeric_dtype(data):
             try:
                 pandas_datetime_format = None
                 if self.datetime_format:
@@ -138,13 +157,16 @@ def _reverse_transform(self, data):
         data = self._reverse_transform_helper(data)
         datetime_data = pd.to_datetime(data)
         if self.datetime_format:
-            if self._dtype == 'object':
-                datetime_data = datetime_data.dt.strftime(self.datetime_format)
-            elif is_datetime64_dtype(self._dtype) and '.%f' not in self.datetime_format:
+            if is_datetime64_dtype(self._dtype) and '.%f' not in self.datetime_format:
                 datetime_data = pd.to_datetime(
                     datetime_data.dt.strftime(self.datetime_format),
                     format=self.datetime_format,
                 )
+            else:
+                datetime_data = datetime_data.dt.strftime(self.datetime_format).astype(self._dtype)
+        elif is_numeric_dtype(self._dtype):
+            datetime_data = pd.to_numeric(datetime_data.astype('object'), errors='coerce')
+            datetime_data = datetime_data.astype(self._dtype)
 
         return datetime_data
 
diff --git a/rdt/transformers/null.py b/rdt/transformers/null.py
@@ -98,11 +98,11 @@ def fit(self, data):
             data (pandas.Series):
                 Data to transform.
         """
+        self._missing_value_replacement = self._get_missing_value_replacement(data)
         if self._missing_value_generation is not None:
             null_values = data.isna().to_numpy()
             self.nulls = null_values.any()
 
-            self._missing_value_replacement = self._get_missing_value_replacement(data)
             if not self.nulls and self.models_missing_values():
                 self._missing_value_generation = None
                 guidance_message = (
@@ -126,9 +126,6 @@ def transform(self, data):
         Returns:
             numpy.ndarray
         """
-        if self._missing_value_generation is None:
-            return data.to_numpy()
-
         isna = data.isna()
         if isna.any() and self._missing_value_replacement is not None:
             data = data.fillna(self._missing_value_replacement)
diff --git a/rdt/transformers/pii/anonymizer.py b/rdt/transformers/pii/anonymizer.py
@@ -36,6 +36,14 @@ class AnonymizedFaker(BaseTransformer):
             Whether or not to ensure that the new anonymized data is all unique. If it isn't
             possible to create the requested number of rows, then an error will be raised.
             Defaults to ``False``.
+        missing_value_generation (str or None):
+            The way missing values are being handled. There are two strategies:
+
+                * ``random``: Randomly generates missing values based on the percentage of
+                  missing values.
+                * ``None``: Don't learn anything during fit. Then during reverse transform,
+                  don't create any missing values.
+
     """
 
     IS_GENERATOR = True
@@ -85,7 +93,7 @@ def _check_locales(self):
             )
 
     def __init__(self, provider_name=None, function_name=None, function_kwargs=None,
-                 locales=None, enforce_uniqueness=False):
+                 locales=None, enforce_uniqueness=False, missing_value_generation='random'):
         super().__init__()
         self.data_length = None
         self.enforce_uniqueness = enforce_uniqueness
@@ -107,6 +115,15 @@ def __init__(self, provider_name=None, function_name=None, function_kwargs=None,
         if self.locales:
             self._check_locales()
 
+        if missing_value_generation not in ['random', None]:
+            raise TransformerInputError(
+                f"Missing value generation '{missing_value_generation}' is not supported "
+                "for AnonymizedFaker. Please use either 'random' or None."
+            )
+
+        self.missing_value_generation = missing_value_generation
+        self._nan_frequency = 0.0
+
     def reset_randomization(self):
         """Create a new ``Faker`` instance."""
         super().reset_randomization()
@@ -138,6 +155,8 @@ def _fit(self, data):
         """
         self._set_faker_seed(data)
         self.data_length = len(data)
+        if self.missing_value_generation == 'random':
+            self._nan_frequency = data.isna().sum() / len(data)
 
     def _transform(self, _data):
         """Drop the input column by returning ``None``."""
@@ -170,6 +189,11 @@ def _reverse_transform(self, data):
                 f"('{self.get_input_column()}')."
             ) from exception
 
+        if self.missing_value_generation == 'random':
+            num_nans = int(self._nan_frequency * sample_size)
+            nan_indices = np.random.choice(sample_size, num_nans, replace=False)
+            reverse_transformed[nan_indices] = np.nan
+
         return reverse_transformed
 
     def __repr__(self):
diff --git a/setup.cfg b/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.0
+current_version = 1.6.0.dev1
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?
diff --git a/setup.py b/setup.py
@@ -111,7 +111,6 @@
         'License :: Free for non-commercial use',
         'Natural Language :: English',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
@@ -135,11 +134,11 @@
         include=['rdt', 'rdt.*'],
         exclude=['rdt.transformers.addons.*']
     ),
-    python_requires='>=3.7,<3.12',
+    python_requires='>=3.8,<3.12',
     setup_requires=setup_requires,
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/sdv-dev/RDT',
-    version='1.5.0',
+    version='1.6.0.dev1',
     zip_safe=False,
 )
diff --git a/tests/integration/transformers/pii/test_anonymizer.py b/tests/integration/transformers/pii/test_anonymizer.py
@@ -69,6 +69,26 @@ def test_anonymizedfaker_with_nans():
         'id': [1, 2, 3, 4, 5],
     })
 
+    pd.testing.assert_frame_equal(transformed, expected_transformed)
+    assert len(reverse_transform['username']) == 5
+    assert reverse_transform['username'].isna().sum() == 1
+
+
+def test_anonymizedfaker_with_nans_missing_value_generation_none():
+    """End to end test settings missing_value_generation=None."""
+    data = pd.DataFrame({
+        'id': [1, 2, 3, 4, 5],
+        'username': ['a', np.nan, 'c', 'd', 'e']
+    })
+
+    instance = AnonymizedFaker(missing_value_generation=None)
+    transformed = instance.fit_transform(data, 'username')
+    reverse_transform = instance.reverse_transform(transformed)
+
+    expected_transformed = pd.DataFrame({
+        'id': [1, 2, 3, 4, 5],
+    })
+
     pd.testing.assert_frame_equal(transformed, expected_transformed)
     assert len(reverse_transform['username']) == 5
     assert reverse_transform['username'].isna().sum() == 0
@@ -102,7 +122,7 @@ def test_anonymizedfaker_custom_provider_with_nans():
 
     pd.testing.assert_frame_equal(transformed, expected_transformed)
     assert len(reverse_transform['cc']) == 5
-    assert reverse_transform['cc'].isna().sum() == 0
+    assert reverse_transform['cc'].isna().sum() == 1
 
 
 def test_anonymizedfaker_enforce_uniqueness():
diff --git a/tests/integration/transformers/test_boolean.py b/tests/integration/transformers/test_boolean.py
@@ -67,9 +67,9 @@ def test_boolean_missing_value_replacement_mode(self):
         pd.testing.assert_frame_equal(reverse, data)
 
     def test_boolean_missing_value_generation_none(self):
-        """Test the BinaryEncoder when `missing_value_generation` is None.
+        """Test the BinaryEncoder when ``missing_value_generation`` is None.
 
-        In this test we should get `nans` on the transformed data.
+        In this test, the nans should be replacd by the mode on the transformed data.
         """
         # Setup
         data = pd.DataFrame([True, True, None, False], columns=['bool'])
@@ -85,8 +85,7 @@ def test_boolean_missing_value_generation_none(self):
         reverse = transformer.reverse_transform(transformed)
 
         # Assert
-        expected_transformed = pd.DataFrame({
-            'bool': [1., 1., None, 0.],
-        })
+        expected_transformed = pd.DataFrame({'bool': [1., 1., 1., 0.]})
+        expected_reversed = pd.DataFrame({'bool': [True, True, True, False]})
         pd.testing.assert_frame_equal(transformed, expected_transformed)
-        pd.testing.assert_frame_equal(reverse, data)
+        pd.testing.assert_frame_equal(reverse, expected_reversed, check_dtype=False)
diff --git a/tests/integration/transformers/test_datetime.py b/tests/integration/transformers/test_datetime.py
diff --git a/tests/integration/transformers/test_numerical.py b/tests/integration/transformers/test_numerical.py
diff --git a/tests/unit/test___init__.py b/tests/unit/test___init__.py
diff --git a/tests/unit/transformers/pii/test_anonymizer.py b/tests/unit/transformers/pii/test_anonymizer.py
diff --git a/tests/unit/transformers/test_datetime.py b/tests/unit/transformers/test_datetime.py
diff --git a/tests/unit/transformers/test_null.py b/tests/unit/transformers/test_null.py
diff --git a/tox.ini b/tox.ini
