feat: cache scraped offers + auto-create providers with defaults

- Add Redis cache between preview and refresh to avoid double scraping
- Preview caches offers (TTL 5min), refresh uses cache if available
- Fix progression continuity: 80% → 82% → 90% → 100% with cache
- Auto-create missing providers with default scraper URLs from classes
- Fix provider query param (was sent in body, now in query string)
- Use datetime.fromisoformat() instead of dateutil (no extra dep)
- Faster polling (500ms) during sync for responsive progress
- Close preview modal immediately when applying changes

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <[email protected]>

Author: Clément VALENTIN

apps/api/Dockerfile

@@ -1,26 +1,43 @@
-FROM python:3.12-slim
+# =============================================================================
+# Stage 1: Dependencies (cached separately from code)
+# =============================================================================
+FROM python:3.12-slim AS deps
 
 WORKDIR /app
 
-# Install system dependencies
+# Install system dependencies (rarely changes - cached)
 RUN apt-get update && apt-get install -y \
     gcc \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy ONLY dependency files first (better cache)
+COPY pyproject.toml README.md ./
+
+# Install dependencies (cached if pyproject.toml unchanged)
+# uv crashes under QEMU emulation, so we use pip for multi-arch builds
+RUN pip install --no-cache-dir .
+
+# =============================================================================
+# Stage 2: Production image
+# =============================================================================
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install runtime dependencies only (no gcc needed)
+RUN apt-get update && apt-get install -y \
     tzdata \
     && rm -rf /var/lib/apt/lists/*
 
 # Set timezone
 ENV TZ=Europe/Paris
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-# Copy dependency files
-COPY pyproject.toml .
-COPY README.md .
-
-# Install dependencies using pip (compatible with cross-platform builds)
-# uv crashes under QEMU emulation, so we use pip for multi-arch builds
-RUN pip install --no-cache-dir .
+# Copy installed packages from deps stage
+COPY --from=deps /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=deps /usr/local/bin /usr/local/bin
 
-# Copy application code
+# Copy application code (changes frequently - last layer)
 COPY . .
 
 # Expose port

apps/api/src/routers/admin.py

@@ -6,13 +6,15 @@
 import logging
 import json
 import asyncio
-from typing import Optional
+from typing import Optional, List
 from ..models import User, PDL, EnergyProvider, EnergyOffer
 from ..models.database import get_db
 from ..middleware import require_admin, require_permission, get_current_user
 from ..schemas import APIResponse
-from ..services import rate_limiter, cache_service
+from ..services import rate_limiter
 from ..services.price_update_service import PriceUpdateService
+from ..config import settings
+import redis.asyncio as redis
 
 logger = logging.getLogger(__name__)
 
@@ -27,6 +29,55 @@
     "progress": 0
 }
 
+# Cache pour les offres scrapées (évite de re-scraper entre preview et refresh)
+# TTL de 5 minutes - les offres scrapées sont réutilisées si le refresh est fait rapidement
+SCRAPED_OFFERS_CACHE_TTL = 300  # 5 minutes
+
+
+async def _get_redis_client():
+    """Get Redis client"""
+    return await redis.from_url(settings.REDIS_URL, encoding="utf-8", decode_responses=True)
+
+
+async def _cache_scraped_offers(provider: str, offers: List[dict]) -> None:
+    """Cache les offres scrapées pour éviter un double scraping"""
+    try:
+        client = await _get_redis_client()
+        cache_key = f"scraped_offers:{provider}"
+        await client.setex(cache_key, SCRAPED_OFFERS_CACHE_TTL, json.dumps(offers))
+        await client.close()
+        logger.info(f"Cached {len(offers)} scraped offers for {provider} (TTL: {SCRAPED_OFFERS_CACHE_TTL}s)")
+    except Exception as e:
+        logger.error(f"Failed to cache offers for {provider}: {e}")
+
+
+async def _get_cached_offers(provider: str) -> List[dict] | None:
+    """Récupère les offres scrapées du cache si disponibles"""
+    try:
+        client = await _get_redis_client()
+        cache_key = f"scraped_offers:{provider}"
+        cached = await client.get(cache_key)
+        await client.close()
+        if cached:
+            offers = json.loads(cached)
+            logger.info(f"Found {len(offers)} cached offers for {provider}")
+            return offers
+    except Exception as e:
+        logger.error(f"Failed to get cached offers for {provider}: {e}")
+    return None
+
+
+async def _clear_cached_offers(provider: str) -> None:
+    """Supprime les offres scrapées du cache après utilisation"""
+    try:
+        client = await _get_redis_client()
+        cache_key = f"scraped_offers:{provider}"
+        await client.delete(cache_key)
+        await client.close()
+        logger.info(f"Cleared cached offers for {provider}")
+    except Exception as e:
+        logger.error(f"Failed to clear cached offers for {provider}: {e}")
+
 
 def _update_scraper_progress(step: str, progress: int):
     """Update scraper progress status"""
@@ -952,6 +1003,10 @@ async def preview_offers_update(
                         }
                     )
 
+                # Cache scraped offers for later refresh (avoids re-scraping)
+                if preview_result.get("scraped_offers"):
+                    await _cache_scraped_offers(provider, preview_result["scraped_offers"])
+
                 return APIResponse(
                     success=True,
                     data={
@@ -1099,9 +1154,25 @@ async def refresh_offers(
                         }
                     )
 
-                _update_scraper_progress(f"Téléchargement des tarifs {provider}", 20)
-                result = await service.update_provider(provider)
-                _update_scraper_progress("Mise à jour de la base de données", 80)
+                # Check for cached offers from preview (avoids re-scraping)
+                cached_offers = await _get_cached_offers(provider)
+
+                if cached_offers:
+                    # Continue from where preview left off (80%)
+                    # Preview: 0% → 20% (download) → 80% (analysis done)
+                    # Refresh with cache: 80% → 90% (DB update) → 100% (done)
+                    _update_scraper_progress(f"Utilisation des données en cache pour {provider}", 82)
+                else:
+                    _update_scraper_progress(f"Téléchargement des tarifs {provider}", 20)
+
+                result = await service.update_provider(provider, cached_offers=cached_offers)
+
+                # Progress depends on whether we used cache
+                _update_scraper_progress("Mise à jour de la base de données", 90 if cached_offers else 80)
+
+                # Clear cache after use
+                if cached_offers:
+                    await _clear_cached_offers(provider)
 
                 if not result.get("success"):
                     return APIResponse(
@@ -1116,8 +1187,9 @@ async def refresh_offers(
                 return APIResponse(
                     success=True,
                     data={
-                        "message": f"Successfully updated {provider}",
-                        "result": result
+                        "message": f"Successfully updated {provider}" + (" (from cache)" if cached_offers else ""),
+                        "result": result,
+                        "used_cache": cached_offers is not None
                     }
                 )
             else:
@@ -1350,31 +1422,40 @@ async def list_available_scrapers(
 
 @router.get("/providers", response_model=APIResponse)
 async def list_providers(
-    include_missing_scrapers: bool = Query(False, description="Include providers with scrapers that don't exist in DB yet"),
     current_user: User = Depends(require_permission('offers')),
     db: AsyncSession = Depends(get_db)
 ) -> APIResponse:
     """
     List all energy providers
 
-    Args:
-        include_missing_scrapers: If True, also returns providers with scrapers not yet in DB
+    Automatically creates missing providers from scrapers with default values.
 
     Returns:
         APIResponse with list of providers
     """
     try:
+        # First, ensure all scrapers have corresponding providers in DB
+        service = PriceUpdateService(db)
+        existing_result = await db.execute(select(EnergyProvider.name))
+        existing_names = {row[0] for row in existing_result.fetchall()}
+
+        # Create missing providers with defaults
+        for scraper_name in PriceUpdateService.SCRAPERS.keys():
+            if scraper_name not in existing_names:
+                logger.info(f"Auto-creating missing provider: {scraper_name}")
+                await service._get_or_create_provider(scraper_name)
+
+        await db.commit()
+
+        # Now fetch all providers
         result = await db.execute(
             select(EnergyProvider).order_by(EnergyProvider.name)
         )
         providers = result.scalars().all()
 
         providers_data = []
-        existing_names = set()
 
         for provider in providers:
-            existing_names.add(provider.name)
-
             # Count active offers
             offers_result = await db.execute(
                 select(func.count()).select_from(EnergyOffer).where(
@@ -1389,6 +1470,11 @@ async def list_providers(
             # Check if this provider has a scraper
             has_scraper = provider.name in PriceUpdateService.SCRAPERS
 
+            # Get default URLs if provider has none
+            scraper_urls = provider.scraper_urls
+            if not scraper_urls and has_scraper:
+                scraper_urls = PriceUpdateService.get_default_scraper_urls(provider.name)
+
             providers_data.append({
                 "id": provider.id,
                 "name": provider.name,
@@ -1397,30 +1483,11 @@ async def list_providers(
                 "is_active": provider.is_active,
                 "active_offers_count": offers_count,
                 "has_scraper": has_scraper,
-                "scraper_urls": provider.scraper_urls,
+                "scraper_urls": scraper_urls,
                 "created_at": provider.created_at.isoformat(),
                 "updated_at": provider.updated_at.isoformat(),
             })
 
-        # Add providers with scrapers that don't exist in DB yet
-        if include_missing_scrapers:
-            for scraper_name in PriceUpdateService.SCRAPERS.keys():
-                if scraper_name not in existing_names:
-                    # Generate a placeholder ID and default values
-                    providers_data.append({
-                        "id": f"scraper-{scraper_name.lower().replace(' ', '-')}",
-                        "name": scraper_name,
-                        "logo_url": None,
-                        "website": None,
-                        "is_active": False,
-                        "active_offers_count": 0,
-                        "has_scraper": True,
-                        "scraper_urls": None,
-                        "created_at": None,
-                        "updated_at": None,
-                        "not_in_database": True,  # Flag to indicate this is a placeholder
-                    })
-
         # Sort by name
         providers_data.sort(key=lambda x: x["name"])
 

apps/api/src/services/cache.py

@@ -71,6 +71,31 @@ async def delete(self, key: str) -> bool:
         except Exception:
             return False
 
+    async def get_raw(self, key: str) -> Optional[str]:
+        """Get cached value without decryption (for non-sensitive data)"""
+        if not self.redis_client:
+            return None
+
+        try:
+            data = await self.redis_client.get(key)
+            if data:
+                return data.decode() if isinstance(data, bytes) else data
+            return None
+        except Exception:
+            return None
+
+    async def set_raw(self, key: str, value: str, ttl: Optional[int] = None) -> bool:
+        """Cache value without encryption (for non-sensitive data)"""
+        if not self.redis_client:
+            return False
+
+        try:
+            cache_ttl = ttl if ttl is not None else self.ttl
+            await self.redis_client.setex(key, cache_ttl, value)
+            return True
+        except Exception:
+            return False
+
     async def delete_pattern(self, pattern: str) -> int:
         """Delete all keys matching pattern"""
         if not self.redis_client:

apps/api/src/services/price_scrapers/base.py

@@ -103,9 +103,14 @@ def __init__(
         self.valid_from = valid_from
         self.valid_to = valid_to
 
-    def to_dict(self) -> Dict[str, Any]:
-        """Convert to dictionary for database insertion"""
-        return {
+    def to_dict(self, for_json: bool = False) -> Dict[str, Any]:
+        """
+        Convert to dictionary for database insertion or JSON serialization.
+
+        Args:
+            for_json: If True, convert datetime to ISO strings for JSON serialization
+        """
+        result = {
             "name": self.name,
             "offer_type": self.offer_type,
             "description": self.description,
@@ -131,12 +136,22 @@ def to_dict(self) -> Dict[str, Any]:
             "peak_day_price": self.peak_day_price,
             "hc_schedules": self.hc_schedules,
             "power_kva": self.power_kva,
-            "valid_from": self.valid_from,
-            "valid_to": self.valid_to,
-            "price_updated_at": datetime.now(UTC),
             "is_active": True,
         }
 
+        if for_json:
+            # Convert datetime to ISO string for JSON serialization (cache)
+            result["valid_from"] = self.valid_from.isoformat() if self.valid_from else None
+            result["valid_to"] = self.valid_to.isoformat() if self.valid_to else None
+            result["price_updated_at"] = datetime.now(UTC).isoformat()
+        else:
+            # Keep native datetime for database insertion
+            result["valid_from"] = self.valid_from
+            result["valid_to"] = self.valid_to
+            result["price_updated_at"] = datetime.now(UTC)
+
+        return result
+
 
 class BasePriceScraper(ABC):
     """Abstract base class for price scrapers"""