Surface more detailed error info when detecting metadata from dataframes (#2358)

Author: R-Palazzo

sdv/metadata/multi_table.py

@@ -544,7 +544,7 @@ def detect_table_from_dataframe(self, table_name, data):
         """
         self._validate_table_not_detected(table_name)
         table = SingleTableMetadata()
-        table._detect_columns(data)
+        table._detect_columns(data, table_name)
         self.tables[table_name] = table
         self._log_detected_table(table)
 

sdv/metadata/single_table.py

@@ -595,35 +595,53 @@ def _detect_primary_key(self, data):
 
         return None
 
-    def _detect_columns(self, data):
+    def _detect_columns(self, data, table_name=None):
         """Detect the columns' sdtypes from the data.
 
         Args:
             data (pandas.DataFrame):
                 The data to be analyzed.
+            table_name (str):
+                The name of the table to be analyzed. Defaults to ``None``.
         """
         old_columns = data.columns
         data.columns = data.columns.astype(str)
         for field in data:
-            column_data = data[field]
-            clean_data = column_data.dropna()
-            dtype = clean_data.infer_objects().dtype.kind
-
-            sdtype = self._detect_pii_column(field)
-            if sdtype is None:
-                if dtype in self._DTYPES_TO_SDTYPES:
-                    sdtype = self._DTYPES_TO_SDTYPES[dtype]
-                elif dtype in ['i', 'f', 'u']:
-                    sdtype = self._determine_sdtype_for_numbers(column_data)
-
-                elif dtype == 'O':
-                    sdtype = self._determine_sdtype_for_objects(column_data)
+            try:
+                column_data = data[field]
+                clean_data = column_data.dropna()
+                dtype = clean_data.infer_objects().dtype.kind
 
+                sdtype = self._detect_pii_column(field)
                 if sdtype is None:
-                    raise InvalidMetadataError(
-                        f"Unsupported data type for column '{field}' (kind: {dtype})."
-                        "The valid data types are: 'object', 'int', 'float', 'datetime', 'bool'."
-                    )
+                    if dtype in self._DTYPES_TO_SDTYPES:
+                        sdtype = self._DTYPES_TO_SDTYPES[dtype]
+                    elif dtype in ['i', 'f', 'u']:
+                        sdtype = self._determine_sdtype_for_numbers(column_data)
+
+                    elif dtype == 'O':
+                        sdtype = self._determine_sdtype_for_objects(column_data)
+
+                    if sdtype is None:
+                        table_str = f"table '{table_name}' " if table_name else ''
+                        error_message = (
+                            f"Unsupported data type for {table_str}column '{field}' (kind: {dtype}"
+                            "). The valid data types are: 'object', 'int', 'float', 'datetime',"
+                            " 'bool'."
+                        )
+                        raise InvalidMetadataError(error_message)
+
+            except Exception as e:
+                error_type = type(e).__name__
+                if error_type == 'InvalidMetadataError':
+                    raise e
+
+                table_str = f"table '{table_name}' " if table_name else ''
+                error_message = (
+                    f"Unable to detect metadata for {table_str}column '{field}' due to an invalid "
+                    f'data format.\n {error_type}: {e}'
+                )
+                raise InvalidMetadataError(error_message) from e
 
             column_dict = {'sdtype': sdtype}
             sdtype_in_reference = sdtype in self._REFERENCE_TO_SDTYPE.values()

tests/integration/metadata/test_metadata.py

@@ -2,9 +2,11 @@
 import re
 from copy import deepcopy
 
+import pandas as pd
 import pytest
 
 from sdv.datasets.demo import download_demo
+from sdv.metadata.errors import InvalidMetadataError
 from sdv.metadata.metadata import Metadata
 from sdv.metadata.multi_table import MultiTableMetadata
 from sdv.metadata.single_table import SingleTableMetadata
@@ -520,3 +522,34 @@ def test_anonymize():
 
     assert anonymized.tables['table1'].to_dict() == table1_metadata.anonymize().to_dict()
     assert anonymized.tables['table2'].to_dict() == table2_metadata.anonymize().to_dict()
+
+
+def test_detect_from_dataframes_invalid_format():
+    """Test the ``detect_from_dataframes`` method with an invalid data format."""
+    # Setup
+    dict_data = [
+        {
+            'key1': i,
+            'key2': f'string_{i}',
+            'key3': 1.5,
+        }
+        for i in range(100)
+    ]
+    data = {
+        'table_1': pd.DataFrame({
+            'dict_column': dict_data,
+            'numerical': [1.2] * 100,
+        }),
+        'table_2': pd.DataFrame({
+            'numerical': [1.5] * 10,
+            'categorical': ['A'] * 10,
+        }),
+    }
+    expected_error = re.escape(
+        "Unable to detect metadata for table 'table_1' column 'dict_column' due to an "
+        "invalid data format.\n TypeError: unhashable type: 'dict'"
+    )
+
+    # Run and Assert
+    with pytest.raises(InvalidMetadataError, match=expected_error):
+        Metadata.detect_from_dataframes(data)

tests/unit/metadata/test_multi_table.py

@@ -2499,7 +2499,7 @@ def test_detect_table_from_dataframe(self, single_table_mock, log_mock):
         metadata.detect_table_from_dataframe('table', data)
 
         # Assert
-        single_table_mock.return_value._detect_columns.assert_called_once_with(data)
+        single_table_mock.return_value._detect_columns.assert_called_once_with(data, 'table')
         assert metadata.tables == {'table': single_table_mock.return_value}
 
         expected_log_calls = call(

tests/unit/metadata/test_single_table.py

@@ -1277,9 +1277,8 @@ def test__detect_columns_with_error(self, mock__get_datetime_format):
 
         expected_error_message = re.escape(
             "Unsupported data type for column 'complex_dtype' (kind: c)."
-            "The valid data types are: 'object', 'int', 'float', 'datetime', 'bool'."
+            " The valid data types are: 'object', 'int', 'float', 'datetime', 'bool'."
         )
-
         with pytest.raises(InvalidMetadataError, match=expected_error_message):
             instance._detect_columns(non_supported_data)
 
@@ -1296,6 +1295,31 @@ def test__detect_columns_with_error(self, mock__get_datetime_format):
         instance._determine_sdtype_for_objects.assert_called_once()
         mock__get_datetime_format.assert_called_once()
 
+    def test__detect_columns_invalid_data_format(self):
+        """Test the ``_detect_columns`` method with an invalid data format."""
+        # Setup
+        instance = SingleTableMetadata()
+        dict_data = [
+            {
+                'key1': i,
+                'key2': f'string_{i}',
+                'key3': np.random.random(),  # random float
+            }
+            for i in range(100)
+        ]
+        data = pd.DataFrame({
+            'dict_column': dict_data,
+            'numerical': [1.2] * 100,
+        })
+        expected_error_message = re.escape(
+            "Unable to detect metadata for column 'dict_column' due to an invalid data format."
+            "\n TypeError: unhashable type: 'dict'"
+        )
+
+        # Run and Assert
+        with pytest.raises(InvalidMetadataError, match=expected_error_message):
+            instance._detect_columns(data)
+
     def test__detect_primary_key_missing_sdtypes(self):
         """The method should raise an error if not all sdtypes were detected."""
         # Setup