Merge branch 'main' into feat/logging-new

Author: pushpam345

app/etl/transform.py

@@ -2,68 +2,57 @@
 import logging as lg
 # TODO (Find & Fix)
 
-logger = lg.getLogger(__name__)
-logger.setLevel(lg.DEBUG)
-
-
-def transform(df: pd.DataFrame) -> pd.DataFrame:
-    """
-    Transform data by cleaning and standardizing it.
-
-    Args:
-        df: Input DataFrame
-
-    Returns:
-        Transformed DataFrame
-    """
-    if df.empty:
-        # TODO (Find & Fix): Should raise a ValueError if DataFrame is empty
-        pass
-
-        # Create a copy to avoid modifying original
-    df_transformed = df.copy()
-
-    # TODO (Find & Fix): Use logging instead of print
-    logger.info(f"🔄 Starting transformation of {len(df_transformed)} rows")
-
-    # Handle duplicates
-    initial_rows = len(df_transformed)
-    # Removing duplicates
-    df_transformed = df_transformed.drop_duplicates()
-
-    duplicates_removed = initial_rows - len(df_transformed)
+def _remove_duplicates(df: pd.DataFrame) -> pd.DataFrame:
+    """Handles duplicate removal."""
+    initial_rows = len(df)
+    df = df.drop_duplicates()
+    duplicates_removed = initial_rows - len(df)
     if duplicates_removed > 0:
-        # Number of duplicates removed
-        logger.info(f"✅ Removed {duplicates_removed} duplicate rows.")
+        print(f"✅ Removed {duplicates_removed} duplicate rows.")
+    return df
 
-    # Handle null values in numeric columns
-    numeric_columns = df_transformed.select_dtypes(include=['number']).columns
+def _handle_nulls(df: pd.DataFrame) -> pd.DataFrame:
+    """Handles null values (currently a placeholder)."""
+    numeric_columns = df.select_dtypes(include=['number']).columns
     for col in numeric_columns:
         # TODO (Find & Fix): Nulls in numeric columns are not handled
         pass
-
-    # Handle null values in text columns
-    text_columns = df_transformed.select_dtypes(include=['object']).columns
+        text_columns = df.select_dtypes(include=['object']).columns
     for col in text_columns:
         # TODO (Find & Fix): Nulls in text columns are not handled
         pass
+    return df
 
-    # Standardize date columns (look for common date column names)
-    date_columns = [col for col in df_transformed.columns
-                    if any(keyword in col.lower() for keyword in ['date', 'time', 'created', 'updated'])]
-
+def _standardize_dates(df: pd.DataFrame) -> pd.DataFrame:
+    """Handles date standardization."""
+    date_columns = [col for col in df.columns 
+                      if any(keyword in col.lower() for keyword in ['date', 'time', 'created', 'updated'])]
     for col in date_columns:
         try:
-            df_transformed[col] = pd.to_datetime(
-                df_transformed[col], errors='coerce', infer_datetime_format=True)
+            df[col] = pd.to_datetime(df[col], errors='coerce', format='mixed')
             # Standardize all dates to 'YYYY-MM-DD HH:MM:SS'
-            df_transformed[col] = df_transformed[col].dt.strftime(
-                '%Y-%m-%d %H:%M:%S')
-
-            logger.info(
-                f"✅ Standardized date column '{col}' (e.g., {df_transformed[col].iloc[0]})")
+            df[col] = df[col].dt.strftime('%Y-%m-%d %H:%M:%S')
+            
+            print(f"✅ Standardized date column '{col}' (e.g., {df[col].iloc[0]})")
         except Exception as e:
-            logger.error(f"⚠️ Could not standardize column '{col}': {e}")
+            print(f"⚠️ Could not standardize column '{col}': {e}")
+    return df
 
+def _validate_types(df: pd.DataFrame) -> pd.DataFrame:
     # TODO (Find & Fix): Text columns are not cleaned (strip, lowercase)
-    return df_transformed
+    pass
+    return df
+def transform(df: pd.DataFrame) -> pd.DataFrame:
+    if df.empty:
+        # TODO (Find &Fix): Should raise a ValueError if DataFrame is empty
+        pass
+    
+    df_transformed = df.copy()
+    
+    print(f"🔄 Starting transformation of {len(df_transformed)} rows")  # TODO (Find & Fix): Use logging instead of print
+    df_transformed = _remove_duplicates(df_transformed)
+    df_transformed = _handle_nulls(df_transformed)
+    df_transformed = _standardize_dates(df_transformed)
+    df_transformed = _validate_types(df_transformed)
+    
+    return df_transformed

tests/__pycache__/__init__.cpython-310.pyc

@@ -0,0 +1,52 @@
+import pytest
+import numpy as np
+import pandas as pd
+import numpy as np
+from pandas.testing import assert_frame_equal
+from app.etl.transform import (
+    _remove_duplicates,
+    _handle_nulls,
+    _standardize_dates,
+    _validate_types
+)
+
+def test_remove_duplicates():
+    data = {'id': [1, 1, 2], 'name': ['a', 'a', 'b']}
+    input_df = pd.DataFrame(data)
+    expected_data = {'id': [1, 2], 'name': ['a', 'b']}
+    expected_df = pd.DataFrame(expected_data).reset_index(drop=True)
+    result_df = _remove_duplicates(input_df).reset_index(drop=True)
+    assert len(result_df) == 2
+    assert_frame_equal(result_df, expected_df)
+
+def test_handle_nulls_does_nothing():
+    data = {'numeric': [1, np.nan], 'text': ['a', np.nan]}
+    input_df = pd.DataFrame(data)
+    expected_df = input_df.copy()
+    result_df = _handle_nulls(input_df)
+    assert_frame_equal(result_df, expected_df)
+
+def test_standardize_dates():
+        data = {'report_date': ['2023-01-01 12:00:00', '02/25/2022', 'bad-date']}
+        df = pd.DataFrame(data)
+
+        result_df = _standardize_dates(df)
+
+        expected = [
+            '2023-01-01 12:00:00',
+            '2022-02-25 00:00:00',
+            np.nan
+        ]
+        assert result_df['report_date'].tolist() == expected
+
+def test_validate_types_does_nothing():
+    """
+    Tests that _validate_types currently does nothing
+    (as per the 'pass' in the logic).
+    """
+    data = {'name': [' Alice ', ' Bob ']}
+    input_df = pd.DataFrame(data)
+    expected_df = input_df.copy()
+    
+    result_df = _validate_types(input_df)
+    assert_frame_equal(result_df, expected_df)