Merge branch 'main' into feature-branch-download-demo

Author: fealho

latest_requirements.txt

@@ -4,7 +4,7 @@ ctgan==0.11.0
 deepecho==0.7.0
 graphviz==0.21
 numpy==2.3.3
-pandas==2.3.2
+pandas==2.3.3
 platformdirs==4.4.0
 rdt==1.18.1
 sdmetrics==0.23.0

pyproject.toml

@@ -38,7 +38,7 @@ dependencies = [
     'copulas>=0.12.1',
     'ctgan>=0.11.0',
     'deepecho>=0.7.0',
-    'rdt>=1.17.0',
+    'rdt>=1.18.2',
     'sdmetrics>=0.21.0',
     'platformdirs>=4.0',
     'pyyaml>=6.0.1',

sdv/sequential/par.py

@@ -11,9 +11,10 @@
 import tqdm
 from rdt.transformers import FloatFormatter
 
-from sdv._utils import MODELABLE_SDTYPES, _cast_to_iterable, _groupby_list
+from sdv._utils import MODELABLE_SDTYPES, _cast_to_iterable, _groupby_list, _is_datetime_type
 from sdv.cag import ProgrammableConstraint
 from sdv.cag._utils import _validate_constraints_single_table
+from sdv.constraints.utils import cast_to_datetime64
 from sdv.errors import SamplingError, SynthesizerInputError
 from sdv.metadata.errors import InvalidMetadataError
 from sdv.metadata.metadata import Metadata
@@ -37,6 +38,11 @@
 LOGGER = logging.getLogger(__name__)
 
 
+def _diff_and_bfill(series):
+    """Compute the diff of a pandas Series and backfill the first NaN."""
+    return series.diff().bfill()
+
+
 class PARSynthesizer(LossValuesMixin, MissingModuleMixin, BaseSynthesizer):
     """Synthesizer for sequential data.
 
@@ -310,20 +316,25 @@ def _transform_sequence_index(self, data):
         sequence_index_context = sequence_index_context.rename(
             columns={self._sequence_index: f'{self._sequence_index}.context'}
         )
+
+        if _is_datetime_type(sequence_index[self._sequence_index]):
+            sequence_index[self._sequence_index] = cast_to_datetime64(
+                sequence_index[self._sequence_index]
+            ).astype(np.int64)
+
         if all(sequence_index[self._sequence_key].nunique() == 1):
-            sequence_index_sequence = sequence_index[[self._sequence_index]].diff().bfill()
+            diff_series = sequence_index[self._sequence_index].diff().bfill()
         else:
-            sequence_index_sequence = (
-                sequence_index.groupby(self._sequence_key)
-                .apply(lambda x: x[self._sequence_index].diff().bfill())
-                .droplevel(1)
-                .reset_index()
-            )
+            diff_series = sequence_index.groupby(self._sequence_key, group_keys=False)[
+                self._sequence_index
+            ].transform(_diff_and_bfill)
 
+        sequence_index_sequence = diff_series.to_frame(name=self._sequence_index)
         if all(sequence_index_sequence[self._sequence_index].isna()):
             fill_value = 0
         else:
             fill_value = min(sequence_index_sequence[self._sequence_index].dropna())
+
         sequence_index_sequence = sequence_index_sequence.fillna(fill_value)
 
         data[self._sequence_index] = sequence_index_sequence[self._sequence_index].to_numpy()
@@ -573,7 +584,7 @@ def _sample_from_par(self, context, sequence_length=None):
                     pd.DataFrame({self._sequence_index: diffs})
                 )[self._sequence_index].to_numpy()
                 start_index = context_columns.index(f'{self._sequence_index}.context')
-                start = context_values[start_index]
+                start = context_values.iloc[start_index]
                 sequence[sequence_index_idx] = np.cumsum(diffs) - diffs[0] + start
 
             # Reformat as a DataFrame

sdv/single_table/_dayz_utils.py

@@ -42,8 +42,8 @@ def detect_column_parameters(data, metadata, table_name):
         if sdtype == 'numerical':
             column_parameters[column_name] = {
                 'num_decimal_digits': learn_rounding_digits(data[column_name]),
-                'min_value': data[column_name].min().item(),
-                'max_value': data[column_name].max().item(),
+                'min_value': data[column_name].min(),
+                'max_value': data[column_name].max(),
             }
         elif sdtype == 'datetime':
             datetime_format = column_metadata.get('datetime_format', None)
@@ -63,13 +63,13 @@ def detect_column_parameters(data, metadata, table_name):
                 'start_timestamp': start_timestamp,
                 'end_timestamp': end_timestamp,
             }
-        elif sdtype in ['categorical', 'boolean']:
+        elif sdtype == 'categorical':
             column_parameters[column_name] = {
                 'category_values': data[column_name].dropna().unique().tolist()
             }
 
-        column_parameters[column_name]['missing_values_proportion'] = (
-            data[column_name].isna().mean().item()
+        column_parameters[column_name]['missing_values_proportion'] = float(
+            data[column_name].isna().mean()
         )
 
     return {'columns': column_parameters}

sdv/single_table/dayz.py

@@ -132,7 +132,7 @@ def _validate_categorical_parameters(column_parameters, column_table_msg):
         raise SynthesizerProcessingError(msg)
 
 
-def _validate_missing_value_parameters(column_parameters, column_table_msg):
+def _validate_missing_value_parameters(column_parameters, column_table_msg, is_key_column):
     missing_values_proportion = column_parameters['missing_values_proportion']
     if not _is_numerical(missing_values_proportion) or (
         missing_values_proportion < 0.0 or missing_values_proportion > 1.0
@@ -142,9 +142,15 @@ def _validate_missing_value_parameters(column_parameters, column_table_msg):
             'must be a float between 0.0 and 1.0.'
         )
         raise SynthesizerProcessingError(msg)
+    elif is_key_column and missing_values_proportion != 0:
+        msg = (
+            f"Invalid 'missing_values_proportion' parameter for {column_table_msg}. Primary "
+            "and alternate keys must have 'missing_values_proportion' parameter set to zero."
+        )
+        raise SynthesizerProcessingError(msg)
 
 
-def _validate_column_parameters(table, column, column_metadata, column_parameters):
+def _validate_column_parameters(table, column, column_metadata, column_parameters, is_key_column):
     column_table_msg = f"column '{column}' in table '{table}'"
     sdtype = column_metadata['sdtype']
     sdtype_parameters = SDTYPE_TO_PARAMETERS.get(sdtype, COLUMN_PARAMETER_KEYS)
@@ -165,7 +171,7 @@ def _validate_column_parameters(table, column, column_metadata, column_parameter
         _validate_categorical_parameters(column_parameters, column_table_msg)
 
     if 'missing_values_proportion' in column_parameters:
-        _validate_missing_value_parameters(column_parameters, column_table_msg)
+        _validate_missing_value_parameters(column_parameters, column_table_msg, is_key_column)
 
 
 def _validate_table_parameters(table, table_metadata, table_parameters):
@@ -186,9 +192,11 @@ def _validate_table_parameters(table, table_metadata, table_parameters):
         )
         raise SynthesizerProcessingError(msg)
 
+    key_columns = table_metadata._get_primary_and_alternate_keys()
     for column, column_parameters in table_parameters.get('columns', {}).items():
+        is_key_column = column in key_columns
         _validate_column_parameters(
-            table, column, table_metadata.columns[column], column_parameters
+            table, column, table_metadata.columns[column], column_parameters, is_key_column
         )
 
 

tasks.py

@@ -13,7 +13,9 @@
 from packaging.version import Version
 
 COMPARISONS = {'>=': operator.ge, '>': operator.gt, '<': operator.lt, '<=': operator.le}
-
+EXTERNAL_DEPENDENCY_CAPS = {
+    'torch': '2.9.0'
+}
 
 if not hasattr(inspect, 'getargspec'):
     inspect.getargspec = inspect.getfullargspec
@@ -86,6 +88,8 @@ def install_minimum(c):
     if minimum_versions:
         install_deps = ' '.join(minimum_versions)
         c.run(f'python -m pip install {install_deps}')
+        for dep, cap in EXTERNAL_DEPENDENCY_CAPS.items():
+            c.run(f'python -m pip install "{dep}<{cap}"')
 
 
 @task

tests/integration/multi_table/test_dayz.py

@@ -24,7 +24,6 @@ def test_create_parameters_end_to_end(self):
                         'guest_email': {'missing_values_proportion': 0.0},
                         'hotel_id': {'missing_values_proportion': 0.0},
                         'has_rewards': {
-                            'category_values': [False, True],
                             'missing_values_proportion': 0.0,
                         },
                         'room_type': {

tests/integration/single_table/test_dayz.py

@@ -23,7 +23,6 @@ def test_create_parameters_end_to_end(self):
                     'columns': {
                         'guest_email': {'missing_values_proportion': 0.0},
                         'has_rewards': {
-                            'category_values': [False, True],
                             'missing_values_proportion': 0.0,
                         },
                         'room_type': {

tests/unit/sequential/test_par.py

@@ -14,11 +14,24 @@
 from sdv.metadata.metadata import Metadata
 from sdv.metadata.single_table import SingleTableMetadata
 from sdv.sampling import Condition
-from sdv.sequential.par import PARSynthesizer
+from sdv.sequential.par import PARSynthesizer, _diff_and_bfill
 from sdv.single_table.base import BaseSynthesizer
 from sdv.single_table.copulas import GaussianCopulaSynthesizer
 
 
+def test__diff_and_bfill():
+    """Test the ``_diff_and_bfill`` method."""
+    # Setup
+    data = pd.Series([10, 15, 20, 30])
+
+    # Run
+    result = _diff_and_bfill(data)
+
+    # Assert
+    expected = pd.Series([5.0, 5.0, 5.0, 10.0])
+    pd.testing.assert_series_equal(result, expected)
+
+
 class TestPARSynthesizer:
     def get_metadata(self, add_sequence_key=True, add_sequence_index=False):
         metadata = Metadata()
@@ -283,6 +296,7 @@ def test_validate_context_columns_unique_per_sequence_key(self):
         with pytest.raises(InvalidDataError, match=err_msg):
             instance.validate(data)
 
+    @pytest.mark.filterwarnings('error::FutureWarning')
     def test__transform_sequence(self):
         # Setup
         metadata = self.get_metadata(add_sequence_index=True)
@@ -310,6 +324,7 @@ def test__transform_sequence(self):
         assert list(par.extended_columns.keys()) == ['time']
         assert par.extended_columns['time'].enforce_min_max_values is True
 
+    @pytest.mark.filterwarnings('error::FutureWarning')
     def test__transform_sequence_index_single_instances(self):
         # Setup
         metadata = self.get_metadata(add_sequence_index=True)
@@ -332,6 +347,7 @@ def test__transform_sequence_index_single_instances(self):
         assert list(par.extended_columns.keys()) == ['time']
         assert par.extended_columns['time'].enforce_min_max_values is True
 
+    @pytest.mark.filterwarnings('error::FutureWarning')
     def test__transform_sequence_index_non_unique_sequence_key(self):
         # Setup
         metadata = self.get_metadata(add_sequence_index=True)
@@ -833,6 +849,7 @@ def test__sample_from_par_with_sequence_key(self, tqdm_mock):
         })
         pd.testing.assert_frame_equal(sampled, expected_output)
 
+    @pytest.mark.filterwarnings('error::FutureWarning')
     @patch('sdv.sequential.par.tqdm')
     def test__sample_from_par_with_sequence_index(self, tqdm_mock):
         """Test that the method handles the sequence index properly.
@@ -1245,6 +1262,9 @@ def test_sample_with_all_null_column_categorical(self):
         assert result['all_null_cat_col'].isna().all()
         assert len(result) > 0
 
+    @pytest.mark.filterwarnings(
+        'error:Series.__getitem__ treating keys as positions is deprecated:FutureWarning'
+    )
     def test_sample_with_multiple_all_null_columns(self):
         """Test that sampling works correctly with multiple all-null columns."""
         # Setup
@@ -1257,15 +1277,21 @@ def test_sample_with_multiple_all_null_columns(self):
             'all_null_col2': [np.nan] * 9,
         })
 
-        metadata = Metadata()
-        metadata.add_table('table')
-        metadata.add_column('time', 'table', sdtype='datetime')
-        metadata.add_column('gender', 'table', sdtype='categorical')
-        metadata.add_column('name', 'table', sdtype='id')
-        metadata.add_column('measurement', 'table', sdtype='numerical')
-        metadata.add_column('all_null_col1', 'table', sdtype='numerical')
-        metadata.add_column('all_null_col2', 'table', sdtype='categorical')
-        metadata.set_sequence_key('name', 'table')
+        metadata = Metadata().load_from_dict({
+            'tables': {
+                'table': {
+                    'columns': {
+                        'time': {'sdtype': 'datetime'},
+                        'gender': {'sdtype': 'categorical'},
+                        'name': {'sdtype': 'id'},
+                        'measurement': {'sdtype': 'numerical'},
+                        'all_null_col1': {'sdtype': 'numerical'},
+                        'all_null_col2': {'sdtype': 'categorical'},
+                    },
+                    'sequence_key': 'name',
+                }
+            }
+        })
 
         # Run
         synthesizer = PARSynthesizer(metadata=metadata, epochs=1)

tests/unit/single_table/test__dayz_utils.py

@@ -27,20 +27,26 @@ def test_detect_column_parameter():
     """Test the `detect_column_parameters` method."""
     # Setup
     data = pd.DataFrame({
+        'pk': [0, 1, 2, 3],
         'num_col': [1.0, 2.5, 3.0, None],
         'cat_col': ['A', 'B', 'A', None],
         'date_col': ['2020-01-01', '2020-01-02', None, None],
         'date_col_2': ['2020 Jan 01', '2020 Jan 02', '2020 Jan 03', None],
+        'alt_key': ['id0', 'id1', 'id2', 'id3'],
     })
     metadata = Metadata.load_from_dict({
         'tables': {
             'table_name': {
                 'columns': {
+                    'pk': {'sdtype': 'id'},
                     'num_col': {'sdtype': 'numerical'},
                     'cat_col': {'sdtype': 'categorical'},
                     'date_col': {'sdtype': 'datetime', 'datetime_format': '%Y-%m-%d'},
                     'date_col_2': {'sdtype': 'datetime'},
-                }
+                    'alt_key': {'sdtype': 'ssn'},
+                },
+                'primary_key': 'pk',
+                'alternate_keys': ['alt_key'],
             }
         }
     })
@@ -50,6 +56,7 @@ def test_detect_column_parameter():
     # Assert
     assert result == {
         'columns': {
+            'pk': {'missing_values_proportion': 0.0},
             'num_col': {
                 'num_decimal_digits': 1,
                 'min_value': 1.0,
@@ -70,6 +77,7 @@ def test_detect_column_parameter():
                 'end_timestamp': '2020-01-03 00:00:00',
                 'missing_values_proportion': 0.25,
             },
+            'alt_key': {'missing_values_proportion': 0.0},
         }
     }