fix: metric alignment with inmemory backend (#2116)

Co-authored-by: Shahules786 <[email protected]>

Author: jjmachan

experimental/pyproject.toml

@@ -42,6 +42,7 @@ tracing = ["langfuse", "mlflow"]
 [project.entry-points."ragas.backends"]
 "local/csv" = "ragas_experimental.backends.local_csv:LocalCSVBackend"
 "local/jsonl" = "ragas_experimental.backends.local_jsonl:LocalJSONLBackend"
+"inmemory" = "ragas_experimental.backends.inmemory:InMemoryBackend"
 
 [tool.setuptools.packages.find]
 include = ["ragas_experimental*", "ragas_examples*"]

experimental/ragas_experimental/backends/inmemory.py

@@ -0,0 +1,130 @@
+"""In-memory backend for temporary dataset and experiment storage."""
+
+import typing as t
+from copy import deepcopy
+
+from pydantic import BaseModel
+
+from .base import BaseBackend
+
+
+class InMemoryBackend(BaseBackend):
+    """Backend that stores datasets and experiments in memory.
+
+    This backend is designed for temporary storage of datasets and experiments
+    that don't need persistence. It's particularly useful for:
+    - train/test splits that are temporary
+    - intermediate datasets during processing
+    - testing and development
+
+    Features:
+    - No configuration required
+    - Preserves all data types exactly (unlike CSV backend)
+    - Separate storage for datasets and experiments
+    - Instance isolation (multiple instances don't share data)
+    - Thread-safe for basic operations
+
+    Usage:
+        backend = InMemoryBackend()
+        backend.save_dataset("my_dataset", data)
+        loaded_data = backend.load_dataset("my_dataset")
+    """
+
+    def __init__(self):
+        """Initialize the backend with empty storage."""
+        self._datasets: t.Dict[str, t.List[t.Dict[str, t.Any]]] = {}
+        self._experiments: t.Dict[str, t.List[t.Dict[str, t.Any]]] = {}
+
+    def load_dataset(self, name: str) -> t.List[t.Dict[str, t.Any]]:
+        """Load dataset by name.
+
+        Args:
+            name: Dataset identifier
+
+        Returns:
+            List of dictionaries representing dataset rows. Empty list for empty datasets.
+
+        Raises:
+            FileNotFoundError: If dataset doesn't exist
+        """
+        if name not in self._datasets:
+            raise FileNotFoundError(f"Dataset '{name}' not found")
+
+        # Return a deep copy to prevent accidental modification
+        return deepcopy(self._datasets[name])
+
+    def load_experiment(self, name: str) -> t.List[t.Dict[str, t.Any]]:
+        """Load experiment by name.
+
+        Args:
+            name: Experiment identifier
+
+        Returns:
+            List of dictionaries representing experiment results. Empty list for empty experiments.
+
+        Raises:
+            FileNotFoundError: If experiment doesn't exist
+        """
+        if name not in self._experiments:
+            raise FileNotFoundError(f"Experiment '{name}' not found")
+
+        # Return a deep copy to prevent accidental modification
+        return deepcopy(self._experiments[name])
+
+    def save_dataset(
+        self,
+        name: str,
+        data: t.List[t.Dict[str, t.Any]],
+        data_model: t.Optional[t.Type[BaseModel]] = None,
+    ) -> None:
+        """Save dataset with given name.
+
+        Args:
+            name: Dataset identifier for storage
+            data: List of dictionaries to save
+            data_model: Optional Pydantic model for validation context (ignored)
+
+        Notes:
+            - Overwrites existing dataset with same name
+            - Handles empty data list gracefully
+            - data_model is ignored (for compatibility with BaseBackend interface)
+        """
+        # Store a deep copy to prevent accidental modification of original data
+        self._datasets[name] = deepcopy(data)
+
+    def save_experiment(
+        self,
+        name: str,
+        data: t.List[t.Dict[str, t.Any]],
+        data_model: t.Optional[t.Type[BaseModel]] = None,
+    ) -> None:
+        """Save experiment with given name.
+
+        Args:
+            name: Experiment identifier for storage
+            data: List of dictionaries to save
+            data_model: Optional Pydantic model for validation context (ignored)
+
+        Notes:
+            - Overwrites existing experiment with same name
+            - Handles empty data list gracefully
+            - data_model is ignored (for compatibility with BaseBackend interface)
+        """
+        # Store a deep copy to prevent accidental modification of original data
+        self._experiments[name] = deepcopy(data)
+
+    def list_datasets(self) -> t.List[str]:
+        """List all available dataset names.
+
+        Returns:
+            Sorted list of dataset names
+        """
+        return sorted(self._datasets.keys())
+
+    def list_experiments(self) -> t.List[str]:
+        """List all available experiment names.
+
+        Returns:
+            Sorted list of experiment names
+        """
+        return sorted(self._experiments.keys())

experimental/ragas_experimental/dataset.py

@@ -9,7 +9,11 @@
 
 from pydantic import BaseModel
 
+if t.TYPE_CHECKING:
+    from pandas import DataFrame as PandasDataFrame
+
 from .backends import BaseBackend, get_registry
+from .backends.inmemory import InMemoryBackend
 
 # For backwards compatibility, use typing_extensions for older Python versions
 try:
@@ -190,6 +194,59 @@ def load(
             # Unvalidated mode - keep as dicts but wrapped in Dataset API
             return cls(name, backend, None, dict_data)
 
+    @classmethod
+    def from_pandas(
+        cls: t.Type[Self],
+        dataframe: "PandasDataFrame",
+        name: str,
+        backend: t.Union[BaseBackend, str],
+        data_model: t.Optional[t.Type[T]] = None,
+        **kwargs,
+    ) -> Self:
+        """Create a DataTable from a pandas DataFrame.
+
+        Args:
+            dataframe: The pandas DataFrame to convert
+            name: Name of the dataset
+            backend: Either a BaseBackend instance or backend name string (e.g., "local/csv")
+            data_model: Optional Pydantic model for validation
+            **kwargs: Additional arguments passed to backend constructor (when using string backend)
+
+        Returns:
+            DataTable instance with data from the DataFrame
+
+        Examples:
+            # Using string backend name
+            dataset = Dataset.load_from_pandas(df, "my_data", "local/csv", root_dir="./data")
+
+            # Using backend instance
+            backend = LocalCSVBackend(root_dir="./data")
+            dataset = Dataset.load_from_pandas(df, "my_data", backend)
+        """
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ImportError(
+                "pandas is not installed. Please install it to use this function."
+            )
+
+        if not isinstance(dataframe, pd.DataFrame):
+            raise TypeError(f"Expected pandas DataFrame, got {type(dataframe)}")
+
+        # Convert DataFrame to list of dictionaries
+        dict_data = dataframe.to_dict(orient="records")
+
+        # Resolve backend if string
+        backend = cls._resolve_backend(backend, **kwargs)
+
+        if data_model:
+            # Validated mode - convert dicts to Pydantic models
+            validated_data = [data_model(**d) for d in dict_data]
+            return cls(name, backend, data_model, validated_data)
+        else:
+            # Unvalidated mode - keep as dicts but wrapped in DataTable API
+            return cls(name, backend, None, dict_data)
+
     def save(self) -> None:
         """Save dataset - converts to dicts if needed"""
         dict_data: t.List[t.Dict[str, t.Any]] = []
@@ -252,6 +309,27 @@ def validate_with(self, data_model: t.Type[T]) -> Self:
             data=validated_data,
         )
 
+    def to_pandas(self) -> "PandasDataFrame":
+        """Convert the dataset to a pandas DataFrame."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ImportError(
+                "pandas is not installed. Please install it to use this function."
+            )
+
+        # Convert data to list of dictionaries
+        dict_data: t.List[t.Dict[str, t.Any]] = []
+        for item in self._data:
+            if isinstance(item, BaseModel):
+                dict_data.append(item.model_dump())
+            elif isinstance(item, dict):
+                dict_data.append(item)
+            else:
+                raise TypeError(f"Unexpected type in dataset: {type(item)}")
+
+        return pd.DataFrame(dict_data)
+
     def append(self, item: t.Union[t.Dict, BaseModel]) -> None:
         """Add item to dataset with validation if model exists"""
         if self.data_model is not None:
@@ -290,21 +368,17 @@ def __str__(self):
 
         return f"{self.DATATABLE_TYPE}(name={self.name}, {data_model_str} len={len(self._data)})"
 
-    __repr__ = __str__
-
-
-class Dataset(DataTable[T]):
-    """Dataset class for managing dataset entries.
-
-    Inherits all functionality from DataTable. This class represents
-    datasets specifically (as opposed to experiments).
-    """
+    def get_row_value(self, row, key: str):
+        """Helper method to get value from row (dict or BaseModel)"""
 
-    DATATABLE_TYPE = "Dataset"
+        if isinstance(row, dict):
+            return row.get(key)
+        else:
+            return getattr(row, key, None)
 
     def train_test_split(
         self, test_size: float = 0.2, random_state: t.Optional[int] = None
-    ) -> t.Tuple["Dataset[T]", "Dataset[T]"]:
+    ) -> t.Tuple["DataTable[T]", "DataTable[T]"]:
         """Split the dataset into training and testing sets.
 
         Args:
@@ -327,6 +401,9 @@ def train_test_split(
         split_index = int(len(self._data) * (1 - test_size))
 
         # Create new dataset instances with proper initialization
+        # Use inmemory backend for split datasets (temporary datasets)
+        inmemory_backend = InMemoryBackend()
+
         # Handle type-safe constructor calls based on data_model presence
         if self.data_model is not None:
             # Validated dataset case - data should be List[T]
@@ -335,14 +412,14 @@ def train_test_split(
 
             train_dataset = type(self)(
                 name=f"{self.name}_train",
-                backend=self.backend,
+                backend=inmemory_backend,
                 data_model=self.data_model,
                 data=train_data,
             )
 
             test_dataset = type(self)(
                 name=f"{self.name}_test",
-                backend=self.backend,
+                backend=inmemory_backend,
                 data_model=self.data_model,
                 data=test_data,
             )
@@ -353,16 +430,32 @@ def train_test_split(
 
             train_dataset = type(self)(
                 name=f"{self.name}_train",
-                backend=self.backend,
+                backend=inmemory_backend,
                 data_model=None,
                 data=train_data,
             )
 
             test_dataset = type(self)(
                 name=f"{self.name}_test",
-                backend=self.backend,
+                backend=inmemory_backend,
                 data_model=None,
                 data=test_data,
             )
 
+        # save to inmemory backend
+        train_dataset.save()
+        test_dataset.save()
+
         return train_dataset, test_dataset
+
+    __repr__ = __str__
+
+
+class Dataset(DataTable[T]):
+    """Dataset class for managing dataset entries.
+
+    Inherits all functionality from DataTable. This class represents
+    datasets specifically (as opposed to experiments).
+    """
+
+    DATATABLE_TYPE = "Dataset"