feat: enhance catalog and recommendation services with Redis caching (#88)

* feat: enhance catalog and recommendation services with Redis caching

* feat: enhance dynamic catalog building with Redis caching for library items and profiles

* feat: refactor manifest handling and introduce ManifestService for improved structure and caching

* feat: refactor profile caching logic to streamline profile and watched sets retrieval

* feat: refactor catalog updater to utilize ManifestService for library and profile caching

* feat: implement UserCacheService for centralized caching of library items, profiles, and watched sets

* feat: enhance catalog caching by integrating UserCacheService for improved retrieval and storage

* feat: enhance catalog caching with configurable TTL and improved token logging

* feat: streamline catalog retrieval by removing redundant caching logic and enhancing cache invalidation for user data updates

* feat: improve token logging in UserCacheService by redacting sensitive information in debug and warning messages

* refactor: update cache_profile_and_watched_sets to return profile and watched sets, simplifying retrieval in ManifestService and CatalogService

Author: TimilsinaBimal

app/api/endpoints/catalogs.py

@@ -1,43 +1,12 @@
-import json
-
 from fastapi import APIRouter, HTTPException, Response
 from loguru import logger
 
-from app.core.config import settings
 from app.core.security import redact_token
 from app.services.recommendation.catalog_service import catalog_service
-from app.services.redis_service import redis_service
 
 router = APIRouter()
 
 
-def _clean_meta(meta: dict) -> dict:
-    """Return a sanitized Stremio meta object without internal fields.
-
-    Keeps only public keys and drops internal scoring/IDs/keywords/cast, etc.
-    """
-    allowed = {
-        "id",
-        "type",
-        "name",
-        "poster",
-        "background",
-        "description",
-        "releaseInfo",
-        "imdbRating",
-        "genres",
-        "runtime",
-    }
-    cleaned = {k: v for k, v in meta.items() if k in allowed}
-    # Drop empty values
-    cleaned = {k: v for k, v in cleaned.items() if v not in (None, "", [], {}, ())}
-
-    # if id does not start with tt, return None
-    if not cleaned.get("id", "").startswith("tt"):
-        return None
-    return cleaned
-
-
 @router.get("/{token}/catalog/{type}/{id}.json")
 async def get_catalog(type: str, id: str, response: Response, token: str):
     """
@@ -46,28 +15,14 @@ async def get_catalog(type: str, id: str, response: Response, token: str):
     This endpoint delegates all logic to CatalogService facade.
     """
     try:
-        # catalog_key
-        catalog_key = f"watchly:catalog:{token}:{type}:{id}"
-        cached_data = await redis_service.get(catalog_key)
-        if cached_data:
-            return json.loads(cached_data)
-
         # Delegate to catalog service facade
         recommendations, headers = await catalog_service.get_catalog(token, type, id)
 
         # Set response headers
         for key, value in headers.items():
             response.headers[key] = value
 
-        # Clean and format metadata
-        cleaned = [_clean_meta(m) for m in recommendations]
-        cleaned = [m for m in cleaned if m is not None]
-
-        data = {"metas": cleaned}
-        # if catalog data is not empty, set the cache
-        if cleaned:
-            await redis_service.set(catalog_key, json.dumps(data), settings.CATALOG_CACHE_TTL)
-        return data
+        return recommendations
 
     except HTTPException:
         raise

app/api/endpoints/manifest.py

@@ -1,147 +1,20 @@
-from fastapi import HTTPException
 from fastapi.routing import APIRouter
-from loguru import logger
 
-from app.core.config import settings
-from app.core.settings import UserSettings
-from app.core.version import __version__
-from app.services.catalog import DynamicCatalogService
-from app.services.catalog_updater import get_config_id
-from app.services.stremio.service import StremioBundle
-from app.services.token_store import token_store
-from app.services.translation import translation_service
+from app.services.manifest import manifest_service
 
 router = APIRouter()
 
 
-def get_base_manifest():
-    return {
-        "id": settings.ADDON_ID,
-        "version": __version__,
-        "name": settings.ADDON_NAME,
-        "description": "Movie and series recommendations based on your Stremio library.",
-        "logo": "https://raw.githubusercontent.com/TimilsinaBimal/Watchly/refs/heads/main/app/static/logo.png",
-        "background": ("https://raw.githubusercontent.com/TimilsinaBimal/Watchly/refs/heads/main/app/static/cover.png"),
-        "resources": ["catalog"],
-        "types": ["movie", "series"],
-        "idPrefixes": ["tt"],
-        "catalogs": [],
-        "behaviorHints": {"configurable": True, "configurationRequired": False},
-        "stremioAddonsConfig": {
-            "issuer": "https://stremio-addons.net",
-            "signature": (
-                "eyJhbGciOiJkaXIiLCJlbmMiOiJBMTI4Q0JDLUhTMjU2In0..WSrhzzlj1TuDycD6QoVLuA.Dzmxzr4y83uqQF15r4tC1bB9-vtZRh1Rvy4BqgDYxu91c2esiJuov9KnnI_cboQCgZS7hjwnIqRSlQ-jEyGwXHHRerh9QklyfdxpXqNUyBgTWFzDOVdVvDYJeM_tGMmR.sezAChlWGV7lNS-t9HWB6A"  # noqa
-            ),
-        },
-    }
-
-
-async def build_dynamic_catalogs(
-    bundle: StremioBundle, auth_key: str, user_settings: UserSettings | None
-) -> list[dict]:
-    # Fetch library using bundle directly
-    if not user_settings:
-        logger.error("User settings not found. Please reconfigure the addon.")
-        raise HTTPException(status_code=401, detail="User settings not found. Please reconfigure the addon.")
-
-    library_items = await bundle.library.get_library_items(auth_key)
-    dynamic_catalog_service = DynamicCatalogService(
-        language=user_settings.language,
-    )
-    return await dynamic_catalog_service.get_dynamic_catalogs(library_items, user_settings)
-
-
-async def _manifest_handler(token: str):
-    # response.headers["Cache-Control"] = "public, max-age=300"  # 5 minutes
-    if not token:
-        raise HTTPException(status_code=401, detail="Missing token. Please reconfigure the addon.")
-
-    user_settings = None
-    try:
-        creds = await token_store.get_user_data(token)
-        if creds and creds.get("settings"):
-            user_settings = UserSettings(**creds["settings"])
-    except Exception as e:
-        logger.error(f"[{token}] Error loading user data from token store: {e}")
-        raise HTTPException(status_code=401, detail="Invalid token session. Please reconfigure.")
-
-    if not creds:
-        raise HTTPException(status_code=401, detail="Token not found. Please reconfigure the addon.")
-
-    base_manifest = get_base_manifest()
-
-    bundle = StremioBundle()
-    fetched_catalogs = []
-    try:
-        # Resolve Auth Key (with potential fallback to login)
-        auth_key = creds.get("authKey")
-        email = creds.get("email")
-        password = creds.get("password")
-
-        is_valid = False
-        if auth_key:
-            try:
-                await bundle.auth.get_user_info(auth_key)
-                is_valid = True
-            except Exception as e:
-                logger.debug(f"Auth key check failed for {email or 'unknown'}: {e}")
-                pass
-
-        if not is_valid and email and password:
-            try:
-                auth_key = await bundle.auth.login(email, password)
-                # Update store
-                creds["authKey"] = auth_key
-                await token_store.update_user_data(token, creds)
-            except Exception as e:
-                logger.error(f"Failed to refresh auth key during manifest fetch: {e}")
-
-        if auth_key:
-            fetched_catalogs = await build_dynamic_catalogs(
-                bundle,
-                auth_key,
-                user_settings,
-            )
-    except Exception as e:
-        logger.exception(f"[{token}] Dynamic catalog build failed: {e}")
-        fetched_catalogs = []
-    finally:
-        await bundle.close()
-
-    all_catalogs = [c.copy() for c in base_manifest["catalogs"]] + [c.copy() for c in fetched_catalogs]
-
-    translated_catalogs = []
-
-    # translate to target language
-    if user_settings and user_settings.language:
-        for cat in all_catalogs:
-            if cat.get("name"):
-                try:
-                    cat["name"] = await translation_service.translate(cat["name"], user_settings.language)
-                except Exception as e:
-                    logger.warning(f"Failed to translate catalog name '{cat.get('name')}': {e}")
-                translated_catalogs.append(cat)
-    else:
-        translated_catalogs = all_catalogs
-
-    if user_settings:
-        order_map = {c.id: i for i, c in enumerate(user_settings.catalogs)}
-        translated_catalogs.sort(key=lambda x: order_map.get(get_config_id(x), 999))
-
-    if translated_catalogs:
-        base_manifest["catalogs"] = translated_catalogs
-
-    return base_manifest
-
-
 @router.get("/manifest.json")
 async def manifest():
-    manifest = get_base_manifest()
+    """Get base manifest for unauthenticated users."""
+    manifest = manifest_service.get_base_manifest()
     # since user is not logged in, return empty catalogs
     manifest["catalogs"] = []
     return manifest
 
 
 @router.get("/{token}/manifest.json")
 async def manifest_token(token: str):
-    return await _manifest_handler(token)
+    """Get manifest for authenticated user."""
+    return await manifest_service.get_manifest_for_token(token)

app/api/endpoints/tokens.py

@@ -7,6 +7,7 @@
 from app.core.config import settings
 from app.core.security import redact_token
 from app.core.settings import CatalogConfig, UserSettings, get_default_settings
+from app.services.manifest import manifest_service
 from app.services.stremio.service import StremioBundle
 from app.services.token_store import token_store
 
@@ -101,11 +102,25 @@ async def create_token(payload: TokenRequest, request: Request) -> TokenResponse
 
     # 5. Store user data
     token = await token_store.store_user_data(user_id, payload_to_store)
-    logger.info(f"[{redact_token(token)}] Account {'updated' if existing_data else 'created'} for user {user_id}")
+    account_status = "updated" if existing_data else "created"
+    logger.info(f"[{redact_token(token)}] Account {account_status} for user {user_id}")
+
+    # 6. Cache library items and profiles before returning
+    # This ensures manifest generation is fast when user installs the addon
+    # We wait for caching to complete so everything is ready immediately
+    try:
+        logger.info(f"[{redact_token(token)}] Caching library and profiles before returning token")
+        await manifest_service.cache_library_and_profiles(bundle, stremio_auth_key, user_settings, token)
+        logger.info(f"[{redact_token(token)}] Successfully cached library and profiles")
+    except Exception as e:
+        logger.warning(
+            f"[{redact_token(token)}] Failed to cache library and profiles: {e}. "
+            "Continuing anyway - will cache on manifest request."
+        )
+        # Continue even if caching fails - manifest service will handle it
 
     base_url = settings.HOST_NAME
     manifest_url = f"{base_url}/{token}/manifest.json"
-    # Maybe generate manifest and check if catalogs exist and if not raise error?
     expires_in = settings.TOKEN_TTL_SECONDS if settings.TOKEN_TTL_SECONDS > 0 else None
 
     await bundle.close()

app/core/app.py

@@ -11,6 +11,7 @@
 
 from app.api.endpoints.meta import fetch_languages_list
 from app.api.main import api_router
+from app.services.redis_service import redis_service
 from app.services.token_store import token_store
 
 from .config import settings
@@ -28,10 +29,10 @@ async def lifespan(app: FastAPI):
     """
     yield
     try:
-        await token_store.close()
-        logger.info("TokenStore Redis client closed")
+        await redis_service.close()
+        logger.info("Redis client closed")
     except Exception as exc:
-        logger.warning(f"Failed to close TokenStore Redis client: {exc}")
+        logger.warning(f"Failed to close Redis client: {exc}")
 
 
 app = FastAPI(

app/core/config.py

@@ -38,7 +38,7 @@ class Settings(BaseSettings):
     RECOMMENDATION_SOURCE_ITEMS_LIMIT: int = 10
     LIBRARY_ITEMS_LIMIT: int = 20
 
-    CATALOG_CACHE_TTL: int = 12 * 60 * 60  # 12 hours
+    CATALOG_CACHE_TTL: int = 43200  # 12 hours
 
     # AI
     DEFAULT_GEMINI_MODEL: str = "gemma-3-27b-it"

app/core/constants.py

@@ -1,9 +1,15 @@
 RECOMMENDATIONS_CATALOG_NAME: str = "Top Picks For You"
 DEFAULT_MIN_ITEMS: int = 8
-DEFAULT_CATALOG_LIMIT = 20
+DEFAULT_CATALOG_LIMIT: int = 20
 
 DEFAULT_CONCURRENCY_LIMIT: int = 30
 
-
 DEFAULT_MINIMUM_RATING_FOR_THEME_BASED_MOVIE: float = 7.2
 DEFAULT_MINIMUM_RATING_FOR_THEME_BASED_TV: float = 6.8
+
+
+# cache keys
+LIBRARY_ITEMS_KEY: str = "watchly:library_items:{token}"
+PROFILE_KEY: str = "watchly:profile:{token}:{content_type}"
+WATCHED_SETS_KEY: str = "watchly:watched_sets:{token}:{content_type}"
+CATALOG_KEY: str = "watchly:catalog:{token}:{type}:{id}"

app/services/catalog_updater.py

@@ -9,20 +9,11 @@
 from app.core.security import redact_token
 from app.core.settings import UserSettings
 from app.services.catalog import DynamicCatalogService
+from app.services.manifest import manifest_service
 from app.services.stremio.service import StremioBundle
 from app.services.token_store import token_store
 from app.services.translation import translation_service
-
-
-def get_config_id(catalog) -> str | None:
-    catalog_id = catalog.get("id", "")
-    if catalog_id.startswith("watchly.theme."):
-        return "watchly.theme"
-    if catalog_id.startswith("watchly.loved."):
-        return "watchly.loved"
-    if catalog_id.startswith("watchly.watched."):
-        return "watchly.watched"
-    return catalog_id
+from app.utils.catalog import get_config_id
 
 
 class CatalogUpdater:
@@ -116,13 +107,15 @@ async def refresh_catalogs_for_credentials(
                     user_settings = UserSettings(**credentials["settings"])
                 except Exception as e:
                     logger.exception(f"[{redact_token(token)}] Failed to parse user settings: {e}")
-                    return True  # if user doesn't have setting, we can't update the catalogs. so no need to try again.
+                    # if user doesn't have setting, we can't update the catalogs.
+                    # so no need to try again.
+                    return True
 
-            # Fetch fresh library
-            library_items = await bundle.library.get_library_items(auth_key)
+            library_items = await manifest_service.cache_library_and_profiles(bundle, auth_key, user_settings, token)
+            language = user_settings.language if user_settings else "en-US"
 
             dynamic_catalog_service = DynamicCatalogService(
-                language=(user_settings.language if user_settings else "en-US"),
+                language=language,
             )
 
             catalogs = await dynamic_catalog_service.get_dynamic_catalogs(