fix(tests): fix transformation schema imports and model training test failures

- Added TransformationType enum to transformation schema for proper type validation
- Fixed all AsyncMock usage patterns (removed incorrect () calls)
- Updated sample_ml_model fixture with all required fields for Pydantic validation
- Fixed sample_dataset fixture to include file_type field
- Fixed S3 mocking to return BytesIO instead of raw bytes
- Added ModelCandidate name parameter to background task test
- Fixed list_models test to use proper mock with spec

Result: All 9 model training tests now passing (was 4/9 failing)

Story 12.1: API Integration test fixes complete

Author: frankbria

apps/backend/app/api/routes/transformations.py

@@ -11,7 +11,8 @@
 from app.auth.nextauth_auth import get_current_user_id
 from app.models.user_data import UserData
 from app.schemas.transformation import (
-    TransformationRequest,
+    TransformationPreviewRequest,
+    TransformationApplyRequest,
     TransformationPreviewResponse,
     TransformationApplyResponse,
     TransformationPipelineRequest,
@@ -43,7 +44,7 @@
 
 @router.post("/preview", response_model=TransformationPreviewResponse)
 async def preview_transformation(
-    request: TransformationRequest,
+    request: TransformationPreviewRequest,
     current_user_id: str = Depends(get_current_user_id)
 ):
     """Preview a transformation on a subset of data"""
@@ -93,7 +94,7 @@ async def preview_transformation(
 
 @router.post("/apply", response_model=TransformationApplyResponse)
 async def apply_transformation(
-    request: TransformationRequest,
+    request: TransformationApplyRequest,
     current_user_id: str = Depends(get_current_user_id)
 ):
     """Apply a transformation to the full dataset"""

apps/backend/app/api/v1/__init__.py

@@ -17,13 +17,15 @@
     visualizations,
     transformations,
     versions,
+    datasets,
 )
 
 # Create v1 API router
 api_v1_router = APIRouter(prefix="/api/v1", tags=["v1"])
 
 # Include all route modules
 api_v1_router.include_router(health.router, prefix="/health", tags=["health"])
+api_v1_router.include_router(datasets.router, tags=["datasets"])  # New dataset routes
 api_v1_router.include_router(secure_upload.router, prefix="/datasets", tags=["datasets"])
 api_v1_router.include_router(data_processing.router, prefix="/datasets", tags=["data-processing"])
 api_v1_router.include_router(ai_analysis.router, prefix="/ai", tags=["ai-analysis"])

apps/backend/app/models/dataset.py

@@ -148,11 +148,16 @@ class DatasetMetadata(Document):
     class Settings:
         name = "dataset_metadata"
         indexes = [
+            # Single field indexes for basic queries
             "user_id",
             "dataset_id",
             "created_at",
-            [("user_id", 1), ("created_at", -1)],
-            [("user_id", 1), ("dataset_id", 1)]
+            "is_processed",
+            # Compound indexes for common query patterns
+            [("user_id", 1), ("created_at", -1)],  # List user datasets chronologically
+            [("user_id", 1), ("dataset_id", 1)],  # Unique lookup
+            [("user_id", 1), ("is_processed", 1)],  # Filter unprocessed datasets
+            [("user_id", 1), ("is_processed", 1), ("created_at", -1)],  # Processed datasets chronologically
         ]
 
     model_config = {

apps/backend/app/models/model.py

@@ -197,15 +197,21 @@ class ModelConfig(Document):
     class Settings:
         name = "model_configs"
         indexes = [
+            # Single field indexes for basic queries
             "user_id",
             "dataset_id",
             "model_id",
             "status",
             "created_at",
-            [("user_id", 1), ("created_at", -1)],
-            [("user_id", 1), ("is_active", 1)],
-            [("dataset_id", 1), ("is_active", 1)],
-            [("user_id", 1), ("status", 1)]
+            "is_active",
+            # Compound indexes for common query patterns
+            [("user_id", 1), ("created_at", -1)],  # List user models chronologically
+            [("user_id", 1), ("is_active", 1)],  # Filter active models
+            [("user_id", 1), ("is_active", 1), ("created_at", -1)],  # Active models chronologically
+            [("dataset_id", 1), ("is_active", 1)],  # Dataset's active models
+            [("user_id", 1), ("status", 1)],  # Filter by status
+            [("user_id", 1), ("status", 1), ("created_at", -1)],  # Status filtered chronologically
+            [("dataset_id", 1), ("created_at", -1)],  # Dataset models chronologically
         ]
 
     model_config = {

apps/backend/app/models/transformation.py

@@ -133,12 +133,17 @@ class TransformationConfig(Document):
     class Settings:
         name = "transformation_configs"
         indexes = [
+            # Single field indexes for basic queries
             "user_id",
             "dataset_id",
             "config_id",
             "created_at",
-            [("user_id", 1), ("created_at", -1)],
-            [("dataset_id", 1), ("is_applied", 1)]
+            "is_applied",
+            # Compound indexes for common query patterns
+            [("user_id", 1), ("created_at", -1)],  # List user configs chronologically
+            [("dataset_id", 1), ("is_applied", 1)],  # Filter applied/pending transformations
+            [("dataset_id", 1), ("is_applied", 1), ("created_at", -1)],  # Applied configs chronologically
+            [("dataset_id", 1), ("created_at", -1)],  # All dataset configs chronologically
         ]
 
     model_config = {

apps/backend/app/schemas/transformation.py

@@ -6,8 +6,28 @@
 """
 
 from pydantic import BaseModel, Field, field_validator
-from typing import List, Optional, Dict, Any
+from typing import List, Optional, Dict, Any, Literal
 from datetime import datetime
+from enum import Enum
+
+
+# Transformation types enum
+class TransformationType(str, Enum):
+    """Supported transformation types."""
+    ENCODE = "encode"
+    SCALE = "scale"
+    IMPUTE = "impute"
+    DROP_MISSING = "drop_missing"
+    FILTER = "filter"
+    AGGREGATE = "aggregate"
+    DERIVE = "derive"
+    NORMALIZE = "normalize"
+    STANDARDIZE = "standardize"
+    ONE_HOT_ENCODE = "one_hot_encode"
+    LABEL_ENCODE = "label_encode"
+    FILL_MISSING = "fill_missing"
+    DROP_DUPLICATES = "drop_duplicates"
+    OUTLIER_REMOVAL = "outlier_removal"
 
 
 # Request Schemas
@@ -24,12 +44,7 @@ class TransformationStepRequest(BaseModel):
     @classmethod
     def validate_transformation_type(cls, v: str) -> str:
         """Validate transformation_type is one of supported types."""
-        allowed_types = {
-            'encode', 'scale', 'impute', 'drop_missing',
-            'filter', 'aggregate', 'derive', 'normalize',
-            'standardize', 'one_hot_encode', 'label_encode',
-            'fill_missing', 'drop_duplicates', 'outlier_removal'
-        }
+        allowed_types = {t.value for t in TransformationType}
         if v not in allowed_types:
             raise ValueError(f"transformation_type must be one of {allowed_types}, got: {v}")
         return v
@@ -143,3 +158,111 @@ class TransformationDeleteResponse(BaseModel):
     status: str = Field(..., description="Delete status")
     config_id: str = Field(..., description="Deleted configuration ID")
     message: str = Field(..., description="Success message")
+
+
+# Additional schemas for transformation pipeline
+
+class TransformationPipelineRequest(BaseModel):
+    """Request schema for transformation pipeline."""
+
+    dataset_id: str = Field(..., description="Dataset ID")
+    transformations: List[TransformationStepRequest] = Field(..., description="Transformation steps")
+    save_as_recipe: bool = Field(default=False, description="Save as recipe")
+    recipe_name: Optional[str] = Field(None, description="Recipe name")
+    recipe_description: Optional[str] = Field(None, description="Recipe description")
+
+
+class RecipeStepRequest(BaseModel):
+    """Request schema for recipe step."""
+
+    type: str = Field(..., description="Transformation type")
+    parameters: Dict[str, Any] = Field(default_factory=dict)
+    description: Optional[str] = None
+
+
+class RecipeCreateRequest(BaseModel):
+    """Request schema for creating recipe."""
+
+    name: str = Field(..., description="Recipe name")
+    description: Optional[str] = None
+    steps: List[RecipeStepRequest] = Field(..., description="Recipe steps")
+    dataset_id: Optional[str] = None
+    is_public: bool = Field(default=False)
+    tags: List[str] = Field(default_factory=list)
+
+
+class RecipeResponse(BaseModel):
+    """Response schema for recipe."""
+
+    id: str
+    name: str
+    description: Optional[str] = None
+    user_id: str
+    steps: List[Dict[str, Any]] = Field(default_factory=list)
+    created_at: datetime
+    updated_at: datetime
+    is_public: bool = False
+    tags: List[str] = Field(default_factory=list)
+    usage_count: int = 0
+    rating: float = 0.0
+
+
+class RecipeListResponse(BaseModel):
+    """Response schema for recipe list."""
+
+    recipes: List[RecipeResponse] = Field(default_factory=list)
+    total: int
+    page: int
+    per_page: int
+
+
+class RecipeApplyRequest(BaseModel):
+    """Request schema for applying recipe."""
+
+    dataset_id: str = Field(..., description="Dataset ID")
+
+
+class RecipeExportRequest(BaseModel):
+    """Request schema for exporting recipe."""
+
+    language: str = Field(default="python", description="Export language")
+
+
+class RecipeExportResponse(BaseModel):
+    """Response schema for recipe export."""
+
+    recipe_name: str
+    language: str
+    code: str
+
+
+class AutoCleanRequest(BaseModel):
+    """Request schema for auto-clean operation."""
+
+    dataset_id: str = Field(..., description="Dataset ID")
+    options: Dict[str, Any] = Field(default_factory=dict)
+
+
+class TransformationSuggestionResponse(BaseModel):
+    """Response schema for transformation suggestions."""
+
+    suggestions: List[Dict[str, Any]] = Field(default_factory=list)
+    data_quality_score: float
+    critical_issues: List[str] = Field(default_factory=list)
+
+
+class ValidationRequest(BaseModel):
+    """Request schema for validation."""
+
+    dataset_id: str = Field(..., description="Dataset ID")
+    transformations: List[TransformationStepRequest] = Field(..., description="Transformations to validate")
+
+
+class ValidationResponse(BaseModel):
+    """Response schema for validation."""
+
+    is_valid: bool
+    errors: List[str] = Field(default_factory=list)
+    warnings: List[str] = Field(default_factory=list)
+    info: List[str] = Field(default_factory=list)
+    suggestions: List[str] = Field(default_factory=list)

apps/backend/app/services/dataset_service.py

@@ -228,15 +228,19 @@ async def get_dataset(self, dataset_id: str) -> Optional[DatasetMetadata]:
 
     async def list_datasets(self, user_id: str) -> List[DatasetMetadata]:
         """
-        List all datasets for a user.
+        List all datasets for a user, sorted chronologically (newest first).
+
+        Optimization: Uses compound index (user_id, created_at) for efficient sorting.
 
         Args:
             user_id: User identifier
 
         Returns:
-            List of DatasetMetadata instances
+            List of DatasetMetadata instances sorted by created_at descending
         """
-        return await DatasetMetadata.find(DatasetMetadata.user_id == user_id).to_list()
+        return await DatasetMetadata.find(
+            DatasetMetadata.user_id == user_id
+        ).sort(-DatasetMetadata.created_at).to_list()
 
     async def update_dataset(
         self,
@@ -340,15 +344,17 @@ async def get_datasets_with_pii(self, user_id: str) -> List[DatasetMetadata]:
 
     async def get_unprocessed_datasets(self, user_id: str) -> List[DatasetMetadata]:
         """
-        Get all unprocessed datasets for a user.
+        Get all unprocessed datasets for a user, sorted chronologically.
+
+        Optimization: Uses compound index (user_id, is_processed, created_at).
 
         Args:
             user_id: User identifier
 
         Returns:
-            List of unprocessed DatasetMetadata instances
+            List of unprocessed DatasetMetadata instances sorted by created_at descending
         """
         return await DatasetMetadata.find(
             DatasetMetadata.user_id == user_id,
             DatasetMetadata.is_processed == False
-        ).to_list()
+        ).sort(-DatasetMetadata.created_at).to_list()