make release-tag: Merge branch 'master' into stable

Author: csala

HISTORY.md

@@ -1,5 +1,15 @@
 # History
 
+## v0.3.1 - 2021-01-27
+
+### Improvements
+
+* Check discrete_columns valid before fitting - [Issue #35](https://github.com/sdv-dev/CTGAN/issues/35) by @fealho
+
+## Bugs fixed
+
+* ValueError: max() arg is an empty sequence - [Issue #115](https://github.com/sdv-dev/CTGAN/issues/115) by @fealho
+
 ## v0.3.0 - 2020-12-18
 
 In this release we add a new TVAE model which was presented in the original CTGAN paper.

conda/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = 'ctgan' %}
-{% set version = '0.3.0' %}
+{% set version = '0.3.1.dev3' %}
 
 package:
   name: "{{ name|lower }}"

ctgan/__init__.py

@@ -4,7 +4,7 @@
 
 __author__ = 'MIT Data To AI Lab'
 __email__ = 'dailabmit@gmail.com'
-__version__ = '0.3.0'
+__version__ = '0.3.1.dev3'
 
 from ctgan.demo import load_demo
 from ctgan.synthesizers.ctgan import CTGANSynthesizer

ctgan/__main__.py

@@ -66,11 +66,11 @@ def main():
     if args.load:
         model = CTGANSynthesizer.load(args.load)
     else:
-        generator_dims = [int(x) for x in args.generator_dims.split(',')]
-        discriminator_dims = [int(x) for x in args.discriminator_dims.split(',')]
+        generator_dim = [int(x) for x in args.generator_dim.split(',')]
+        discriminator_dim = [int(x) for x in args.discriminator_dim.split(',')]
         model = CTGANSynthesizer(
-            embedding_dim=args.embedding_dim, generator_dims=generator_dims,
-            discriminator_dims=discriminator_dims, generator_lr=args.generator_lr,
+            embedding_dim=args.embedding_dim, generator_dim=generator_dim,
+            discriminator_dim=discriminator_dim, generator_lr=args.generator_lr,
             generator_decay=args.generator_decay, discriminator_lr=args.discriminator_lr,
             discriminator_decay=args.discriminator_decay, batch_size=args.batch_size,
             epochs=args.epochs)

ctgan/data_sampler.py

@@ -41,7 +41,7 @@ def is_discrete_column(column_info):
         # Prepare an interval matrix for efficiently sample conditional vector
         max_category = max(
             [column_info[0].dim for column_info in output_info
-             if is_discrete_column(column_info)])
+             if is_discrete_column(column_info)], default=0)
 
         self._discrete_column_cond_st = np.zeros(n_discrete_columns, dtype='int32')
         self._discrete_column_n_category = np.zeros(
@@ -133,7 +133,7 @@ def sample_data(self, n, col, opt):
             n rows of matrix data.
         """
         if col is None:
-            idx = np.random.randint(len(self._data), n)
+            idx = np.random.randint(len(self._data), size=n)
             return self._data[idx]
 
         idx = []

ctgan/synthesizers/ctgan.py

@@ -1,6 +1,7 @@
 import warnings
 
 import numpy as np
+import pandas as pd
 import torch
 from packaging import version
 from torch import optim
@@ -13,13 +14,13 @@
 
 class Discriminator(Module):
 
-    def __init__(self, input_dim, dis_dims, pack=10):
+    def __init__(self, input_dim, discriminator_dim, pack=10):
         super(Discriminator, self).__init__()
         dim = input_dim * pack
         self.pack = pack
         self.packdim = dim
         seq = []
-        for item in list(dis_dims):
+        for item in list(discriminator_dim):
             seq += [Linear(dim, item), LeakyReLU(0.2), Dropout(0.5)]
             dim = item
 
@@ -222,6 +223,31 @@ def _cond_loss(self, data, c, m):
 
         return (loss * m).sum() / data.size()[0]
 
+    def _validate_discrete_columns(self, train_data, discrete_columns):
+        """Check whether ``discrete_columns`` exists in ``train_data``.
+
+        Args:
+            train_data (numpy.ndarray or pandas.DataFrame):
+                Training Data. It must be a 2-dimensional numpy array or a pandas.DataFrame.
+            discrete_columns (list-like):
+                List of discrete columns to be used to generate the Conditional
+                Vector. If ``train_data`` is a Numpy array, this list should
+                contain the integer indices of the columns. Otherwise, if it is
+                a ``pandas.DataFrame``, this list should contain the column names.
+        """
+        if isinstance(train_data, pd.DataFrame):
+            invalid_columns = set(discrete_columns) - set(train_data.columns)
+        elif isinstance(train_data, np.ndarray):
+            invalid_columns = []
+            for column in discrete_columns:
+                if column < 0 or column >= train_data.shape[1]:
+                    invalid_columns.append(column)
+        else:
+            raise TypeError('``train_data`` should be either pd.DataFrame or np.array.')
+
+        if invalid_columns:
+            raise ValueError('Invalid columns found: {}'.format(invalid_columns))
+
     def fit(self, train_data, discrete_columns=tuple(), epochs=None):
         """Fit the CTGAN Synthesizer models to the training data.
 
@@ -234,6 +260,8 @@ def fit(self, train_data, discrete_columns=tuple(), epochs=None):
                 contain the integer indices of the columns. Otherwise, if it is
                 a ``pandas.DataFrame``, this list should contain the column names.
         """
+        self._validate_discrete_columns(train_data, discrete_columns)
+
         if epochs is None:
             epochs = self._epochs
         else:

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.0
+current_version = 0.3.1.dev3
 commit = True
 tag = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<candidate>\d+))?

setup.py

@@ -15,9 +15,9 @@
     'torch<2,>=1.0',
     'torchvision<1,>=0.4.2',
     'scikit-learn<0.24,>=0.21',
-    'rdt>=0.2.7,<0.3',
     'numpy<2,>=1.17.4',
     'pandas<1.1.5,>=0.24',
+    'rdt>=0.2.7,<0.4',
     'packaging',
 ]
 
@@ -99,6 +99,6 @@
     test_suite='tests',
     tests_require=tests_require,
     url='https://github.com/sdv-dev/CTGAN',
-    version='0.3.0',
+    version='0.3.1.dev3',
     zip_safe=False,
 )

tests/integration/test_ctgan.py

@@ -9,12 +9,30 @@
 model are not checked.
 """
 
+import tempfile as tf
+
 import numpy as np
 import pandas as pd
+import pytest
 
 from ctgan.synthesizers.ctgan import CTGANSynthesizer
 
 
+def test_ctgan_no_categoricals():
+    data = pd.DataFrame({
+        'continuous': np.random.random(1000)
+    })
+
+    ctgan = CTGANSynthesizer(epochs=1)
+    ctgan.fit(data, [])
+
+    sampled = ctgan.sample(100)
+
+    assert sampled.shape == (100, 1)
+    assert isinstance(sampled, pd.DataFrame)
+    assert set(sampled.columns) == {'continuous'}
+
+
 def test_ctgan_dataframe():
     data = pd.DataFrame({
         'continuous': np.random.random(100),
@@ -120,10 +138,33 @@ def test_save_load():
 
     ctgan = CTGANSynthesizer(epochs=1)
     ctgan.fit(data, discrete_columns)
-    ctgan.save("test_ctgan.pkl")
 
-    ctgan = CTGANSynthesizer.load("test_ctgan.pkl")
+    with tf.TemporaryDirectory() as temporary_directory:
+        ctgan.save(temporary_directory + "test_tvae.pkl")
+        ctgan = CTGANSynthesizer.load(temporary_directory + "test_tvae.pkl")
 
     sampled = ctgan.sample(1000)
     assert set(sampled.columns) == {'continuous', 'discrete'}
     assert set(sampled['discrete'].unique()) == {'a', 'b', 'c'}
+
+
+def test_wrong_discrete_columns_dataframe():
+    data = pd.DataFrame({
+        'discrete': ['a', 'b']
+    })
+    discrete_columns = ['b', 'c']
+
+    ctgan = CTGANSynthesizer(epochs=1)
+    with pytest.raises(ValueError):
+        ctgan.fit(data, discrete_columns)
+
+
+def test_wrong_discrete_columns_numpy():
+    data = pd.DataFrame({
+        'discrete': ['a', 'b']
+    })
+    discrete_columns = [0, 1]
+
+    ctgan = CTGANSynthesizer(epochs=1)
+    with pytest.raises(ValueError):
+        ctgan.fit(data.to_numpy(), discrete_columns)

tests/integration/test_tvae.py

@@ -9,6 +9,8 @@
 model are not checked.
 """
 
+import tempfile as tf
+
 import numpy as np
 import pandas as pd
 
@@ -70,11 +72,12 @@ def test_save_load():
     })
     discrete_columns = ['discrete']
 
-    tvae = TVAESynthesizer(epochs=1)
+    tvae = TVAESynthesizer(epochs=10)
     tvae.fit(data, discrete_columns)
-    tvae.save("test_tvae.pkl")
 
-    tvae = TVAESynthesizer.load("test_tvae.pkl")
+    with tf.TemporaryDirectory() as temporary_directory:
+        tvae.save(temporary_directory + "test_tvae.pkl")
+        tvae = TVAESynthesizer.load(temporary_directory + "test_tvae.pkl")
 
     sampled = tvae.sample(1000)
     assert set(sampled.columns) == {'continuous', 'discrete'}