DISCO-3202 - Enrich the curated-recommendations endpoint with icon-urls

Author: gruberb

merino/curated_recommendations/protocol.py

@@ -85,6 +85,7 @@ class CuratedRecommendation(CorpusItem):
     __typename: TypeName = TypeName.RECOMMENDATION
     tileId: Annotated[int, Field(strict=True, ge=MIN_TILE_ID, le=MAX_TILE_ID)]
     receivedRank: int
+    icon: str | None = None
 
     @model_validator(mode="before")
     def set_tileId(cls, values):

merino/providers/manifest/__init__.py

@@ -3,13 +3,13 @@
 from timeit import default_timer as timer
 import logging
 from merino.utils import metrics
-from merino.providers.manifest.provider import Provider
+from merino.providers.manifest.provider import ManifestProvider
 from merino.providers.manifest.backends.manifest import ManifestBackend
 from merino.configs import settings
 
 logger = logging.getLogger(__name__)
 
-provider: Provider | None = None
+provider: ManifestProvider | None = None
 
 
 async def init_provider() -> None:
@@ -20,7 +20,7 @@ async def init_provider() -> None:
     global provider
     start = timer()
 
-    provider = Provider(
+    provider = ManifestProvider(
         backend=ManifestBackend(),
         resync_interval_sec=settings.manifest.resync_interval_sec,
         cron_interval_sec=settings.manifest.cron_interval_sec,
@@ -37,7 +37,7 @@ async def init_provider() -> None:
     )
 
 
-def get_provider() -> Provider:
+def get_provider() -> ManifestProvider:
     """Return manifest provider"""
     if provider is None:
         raise ValueError("Manifest provider has not been initialized.")

merino/providers/manifest/provider.py

@@ -11,12 +11,13 @@
     ManifestBackend,
     ManifestBackendError,
     ManifestData,
+    Domain,
 )
 
 logger = logging.getLogger(__name__)
 
 
-class Provider:
+class ManifestProvider:
     """Provide access to in-memory manifest data fetched from GCS."""
 
     manifest_data: ManifestData | None
@@ -86,3 +87,25 @@ def _should_fetch(self) -> bool:
     def get_manifest_data(self) -> ManifestData | None:
         """Return manifest data"""
         return self.manifest_data
+
+    def get_domain_info(self, domain: str) -> Domain | None:
+        """Get domain information from manifest data.
+
+        Args:
+            domain: Domain name (e.g., 'google.com')
+
+        Returns:
+            Domain object if found, None otherwise
+        """
+        if not self.manifest_data or not self.manifest_data.domains:
+            return None
+
+        # Clean up domain
+        domain = domain.lower().strip()
+
+        # Find matching domain
+        for domain_info in self.manifest_data.domains:
+            if domain_info.domain == domain:
+                return domain_info
+
+        return None

merino/services/__init__.py

@@ -0,0 +1 @@
+"""Services package for Merino application."""

merino/services/icon_enrichment/protocol.py

@@ -0,0 +1,16 @@
+"""Protocol for icon enrichment service."""
+
+from typing import Protocol
+from merino.curated_recommendations.protocol import CuratedRecommendationsResponse
+
+
+class IconEnricher(Protocol):
+    """Protocol for the icon enricher."""
+
+    def enrich_response(self, response: CuratedRecommendationsResponse) -> None:
+        """Enrich response with icons."""
+        ...
+
+    def get_icon_url(self, url: str) -> str | None:
+        """Get icon URL for a given URL."""
+        ...

merino/services/icon_enrichment/service.py

@@ -0,0 +1,70 @@
+"""A Service to retrieve domain data from the manifest provider."""
+
+from urllib.parse import urlparse
+import logging
+
+from pydantic import HttpUrl
+
+from merino.providers.manifest.provider import ManifestProvider
+from merino.curated_recommendations.protocol import (
+    CuratedRecommendationsResponse,
+    CuratedRecommendation,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class IconEnrichmentService:
+    """Service to enrich recommendations with icons from manifest provider."""
+
+    def __init__(self, manifest_provider: ManifestProvider):
+        self.manifest_provider = manifest_provider
+
+    def get_icon_url(self, url: HttpUrl) -> str | None:
+        """Get icon URL for a domain.
+
+        Args:
+            url: Full URL to extract domain from
+
+        Returns:
+            Icon URL if found, None otherwise
+        """
+        try:
+            # Extract domain from URL and normalize it
+            url_str = str(url)
+            domain = urlparse(url_str).netloc.replace("www.", "")
+            if domain_info := self.manifest_provider.get_domain_info(domain):
+                return domain_info.icon
+        except Exception as e:
+            logger.warning(f"Error getting icon for URL {url}: {e}")
+        return None
+
+    def enrich_response(self, response: CuratedRecommendationsResponse) -> None:
+        """Enrich response with icons from manifest provider.
+
+        Args:
+            response: Response to enrich with icons
+        """
+
+        def add_icon(rec: CuratedRecommendation) -> None:
+            if rec.url:
+                if icon_url := self.get_icon_url(rec.url):
+                    rec.icon = icon_url
+
+        # Process main recommendations
+        for rec in response.data:
+            add_icon(rec)
+
+        # Process feeds if they exist
+        if response.feeds:
+            # Need to know recommendations
+            if response.feeds.need_to_know:
+                for rec in response.feeds.need_to_know.recommendations:
+                    add_icon(rec)
+
+            # Process any section that has recommendations
+            for section_name in response.feeds.model_fields_set:
+                if section := getattr(response.feeds, section_name):
+                    if hasattr(section, "recommendations"):
+                        for rec in section.recommendations:
+                            add_icon(rec)

merino/web/api_v1.py

@@ -28,6 +28,7 @@
 from merino.providers.suggest.base import BaseProvider, SuggestionRequest
 
 from merino.providers.manifest.backends.protocol import ManifestData
+from merino.services.icon_enrichment.service import IconEnrichmentService
 from merino.utils import task_runner
 
 from merino.utils.api.cache_control import get_ttl_for_cache_control_header_for_suggestions
@@ -39,7 +40,7 @@
 )
 
 from merino.web.models_v1 import ProviderResponse, SuggestResponse
-from merino.providers.manifest.provider import Provider as ManifestProvider
+from merino.providers.manifest.provider import ManifestProvider
 
 logger = logging.getLogger(__name__)
 router = APIRouter()
@@ -293,6 +294,7 @@ async def providers(
 async def curated_content(
     curated_recommendations_request: CuratedRecommendationsRequest,
     provider: CuratedRecommendationsProvider = Depends(get_corpus_api_provider),
+    manifest_provider: ManifestProvider = Depends(get_manifest_provider),
 ) -> CuratedRecommendationsResponse:
     """Query Merino for curated recommendations.
 
@@ -320,7 +322,14 @@ async def curated_content(
 
     [curated-topics-doc]: https://mozilla-hub.atlassian.net/wiki/x/LQDaMg
     """
-    return await provider.fetch(curated_recommendations_request)
+    # Get base response
+    response = await provider.fetch(curated_recommendations_request)
+
+    # Enrich with icons
+    icon_service = IconEnrichmentService(manifest_provider)
+    icon_service.enrich_response(response)
+
+    return response
 
 
 @router.get(

tests/integration/api/conftest.py

@@ -6,11 +6,16 @@
 
 from logging import LogRecord
 from typing import Iterator, Generator
+from unittest.mock import AsyncMock
 
 import pytest
 import orjson
 from starlette.testclient import TestClient
 from aiodogstatsd import Client as AioDogstatsdClient
+
+from merino.providers.manifest import ManifestProvider
+from merino.providers.manifest.backends.manifest import ManifestBackend
+from merino.providers.manifest.backends.protocol import GetManifestResultCode, ManifestData
 from merino.utils.gcs.gcs_uploader import GcsUploader
 from contextlib import nullcontext
 from merino.curated_recommendations.fakespot_backend.protocol import (
@@ -134,3 +139,24 @@ def fakespot_feed() -> FakespotFeed:
         footerCopy=FAKESPOT_FOOTER_COPY,
         cta=FakespotCTA(ctaCopy=FAKESPOT_CTA_COPY, url=FAKESPOT_CTA_URL),
     )
+
+
+@pytest.fixture
+def mock_manifest_backend(mock_manifest_data):
+    """Mock ManifestBackend that returns our test data."""
+    backend = ManifestBackend()
+    backend.fetch = AsyncMock(
+        return_value=(GetManifestResultCode.SUCCESS, ManifestData(**mock_manifest_data))
+    )
+    return backend
+
+
+@pytest.fixture
+def manifest_provider(mock_manifest_backend):
+    """Override the manifest provider fixture with our mocked data."""
+    provider = ManifestProvider(
+        backend=mock_manifest_backend,
+        resync_interval_sec=86400,
+        cron_interval_sec=3600,
+    )
+    return provider

tests/integration/api/v1/curated_recommendations/test_curated_recommendations.py

@@ -46,6 +46,10 @@
 )
 from merino.curated_recommendations.protocol import CuratedRecommendation
 from merino.main import app
+from merino.providers.manifest import ManifestProvider
+from merino.providers.manifest.backends.manifest import ManifestBackend
+from merino.providers.manifest import get_provider as get_manifest_provider
+from merino.providers.manifest.backends.protocol import Domain
 from tests.integration.api.conftest import fakespot_feed
 
 
@@ -121,6 +125,28 @@ def extended_expiration_corpus_backend(
     )
 
 
+@pytest.fixture
+def manifest_provider():
+    """Mock manifest provider with test data"""
+    backend = ManifestBackend()
+    provider = ManifestProvider(
+        backend=backend,
+        resync_interval_sec=86400,
+        cron_interval_sec=3600,
+    )
+    return provider
+
+
+@pytest.fixture(autouse=True)
+def setup_manifest_provider(manifest_provider):
+    """Set up the manifest provider dependency"""
+    app.dependency_overrides[get_manifest_provider] = lambda: manifest_provider
+    yield
+    # Clean up after test
+    if get_manifest_provider in app.dependency_overrides:
+        del app.dependency_overrides[get_manifest_provider]
+
+
 @pytest.fixture(name="corpus_provider")
 def provider(
     corpus_backend: CorpusApiBackend,
@@ -1529,3 +1555,67 @@ def mock_post_by_days_ago(*args, **kwargs):
             else:
                 # Check that only today's items are returned if in control or not in the experiment.
                 assert set(days_ago_counter.keys()) == {0}
+
+
+@pytest.mark.asyncio
+async def test_curated_recommendations_enriched_with_icons(
+    manifest_provider,
+    corpus_http_client,
+    fixture_request_data,
+):
+    """Test the enrichment of a curated recommendation with an added icon-url."""
+    manifest_provider.manifest_data.domains = [
+        Domain(
+            rank=2,
+            title="Microsoft – AI, Cloud, Productivity, Computing, Gaming & Apps",
+            url="https://www.microsoft.com",
+            domain="microsoft.com",
+            icon="https://merino-images.services.mozilla.com/favicons/microsoft-icon.png",
+            categories=["Business", "Information Technology"],
+            serp_categories=[0],
+        )
+    ]
+
+    mocked_response = {
+        "data": {
+            "scheduledSurface": {
+                "items": [
+                    {
+                        "id": "scheduledSurfaceItemId-ABC",
+                        "corpusItem": {
+                            "id": "corpusItemId-XYZ",
+                            "url": "https://www.microsoft.com/some-article?utm_source=firefox-newtab-en-us",
+                            "title": "Some MS Article",
+                            "excerpt": "All about Microsoft something",
+                            "topic": "tech",
+                            "publisher": "ExamplePublisher",
+                            "isTimeSensitive": False,
+                            "imageUrl": "https://somewhere.com/test.jpg",
+                        },
+                    }
+                ]
+            }
+        }
+    }
+    corpus_http_client.post.return_value = Response(
+        status_code=200,
+        json=mocked_response,
+        request=fixture_request_data,
+    )
+
+    async with AsyncClient(app=app, base_url="http://test") as ac:
+        response = await ac.post(
+            "/api/v1/curated-recommendations",
+            json={"locale": "en-US"},
+        )
+        assert response.status_code == 200
+
+    data = response.json()
+    items = data["data"]
+    assert len(items) == 1
+
+    item = items[0]
+    assert item["url"] == "https://www.microsoft.com/some-article?utm_source=firefox-newtab-en-us"
+
+    assert "icon" in item
+    assert item["icon"] == "https://merino-images.services.mozilla.com/favicons/microsoft-icon.png"