Switch web feed analysis to Haiku and add web feed usage metrics to Grafana

samuelclay · claude · samuelclay · commit 1d30bbf95ddb · 2026-03-05T11:02:35.000-08:00
Web feed LLM analysis was using Opus ($9.50/day) for XPath extraction — a
structured task that Haiku handles well (~$0.50/day). Also adds full web feed
usage tracking: Redis counters for analyses, subscriptions, hint refines,
variant selections, and success/failure rates, exposed via Prometheus endpoint
with 13 new Grafana dashboard panels.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/apps/monitor/urls.py b/apps/monitor/urls.py
@@ -55,4 +55,5 @@
     url(r"^user-searches?$", UserSearches.as_view(), name="user_searches"),
     url(r"^trending-feeds?$", TrendingFeeds.as_view(), name="trending_feeds"),
     url(r"^trending-subscriptions?$", TrendingSubscriptions.as_view(), name="trending_subscriptions"),
+    url(r"^webfeed-usage?$", WebFeedUsage.as_view(), name="webfeed_usage"),
 ]
diff --git a/apps/monitor/views/__init__.py b/apps/monitor/views/__init__.py
@@ -23,3 +23,4 @@
 from apps.monitor.views.newsblur_updates import Updates
 from apps.monitor.views.newsblur_user_searches import UserSearches
 from apps.monitor.views.newsblur_users import Users
+from apps.monitor.views.newsblur_webfeed_usage import WebFeedUsage
diff --git a/apps/monitor/views/newsblur_webfeed_usage.py b/apps/monitor/views/newsblur_webfeed_usage.py
@@ -0,0 +1,85 @@
+import time
+
+from django.shortcuts import render
+from django.views import View
+
+from apps.statistics.rtrending_webfeeds import RTrendingWebFeed
+from apps.webfeed.models import MWebFeedConfig
+
+
+class WebFeedUsage(View):
+    def get(self, request):
+        """
+        Prometheus metrics endpoint for web feed usage.
+
+        Tracks analyses, subscriptions, hint refinements, variant selections,
+        success/failure rates, and overall funnel conversion.
+        """
+        start_time = time.time()
+
+        formatted_data = {}
+        chart_name = "webfeed_usage"
+        chart_type = "gauge"
+
+        stats = RTrendingWebFeed.get_stats_for_prometheus()
+
+        # Simple counters
+        for key in [
+            "analyses_today",
+            "analyses_with_hint_today",
+            "reanalyses_today",
+            "subscriptions_today",
+            "unique_urls_analyzed_today",
+            "unique_urls_subscribed_today",
+            "unique_users_analyzing_today",
+            "unique_users_subscribing_today",
+            "analysis_success_today",
+            "analysis_fail_today",
+            "conversion_rate_pct",
+        ]:
+            formatted_data[key] = f'{chart_name}{{metric="{key}"}} {stats[key]}'
+
+        # Variant choices
+        for variant_idx in range(5):
+            count = stats["variant_choices"].get(str(variant_idx), 0)
+            formatted_data[f"variant_{variant_idx}"] = (
+                f'{chart_name}{{metric="variant_choice",variant="{variant_idx}"}} {count}'
+            )
+
+        # Daily totals for 7-day chart
+        daily_totals = RTrendingWebFeed.get_daily_totals(days=7)
+        for date_str, analyses, subscriptions, unique_users in daily_totals:
+            formatted_data[f"daily_analyses_{date_str}"] = (
+                f'{chart_name}{{metric="daily_analyses",date="{date_str}"}} {analyses}'
+            )
+            formatted_data[f"daily_subs_{date_str}"] = (
+                f'{chart_name}{{metric="daily_subscriptions",date="{date_str}"}} {subscriptions}'
+            )
+            formatted_data[f"daily_users_{date_str}"] = (
+                f'{chart_name}{{metric="daily_unique_users",date="{date_str}"}} {unique_users}'
+            )
+
+        # MongoDB counts for total active web feeds
+        try:
+            total_active = MWebFeedConfig.objects.count()
+            needs_reanalysis = MWebFeedConfig.objects.filter(needs_reanalysis=True).count()
+        except Exception:
+            total_active = 0
+            needs_reanalysis = 0
+
+        formatted_data["total_active_webfeeds"] = (
+            f'{chart_name}{{metric="total_active_webfeeds"}} {total_active}'
+        )
+        formatted_data["webfeeds_needing_reanalysis"] = (
+            f'{chart_name}{{metric="webfeeds_needing_reanalysis"}} {needs_reanalysis}'
+        )
+
+        elapsed_ms = (time.time() - start_time) * 1000
+        formatted_data["scrape_duration"] = f'{chart_name}{{metric="scrape_duration_ms"}} {elapsed_ms:.1f}'
+
+        context = {
+            "data": formatted_data,
+            "chart_name": chart_name,
+            "chart_type": chart_type,
+        }
+        return render(request, "monitor/prometheus_data.html", context, content_type="text/plain")
diff --git a/apps/statistics/rtrending_webfeeds.py b/apps/statistics/rtrending_webfeeds.py
@@ -0,0 +1,174 @@
+import datetime
+import hashlib
+
+import redis
+from django.conf import settings
+
+
+class RTrendingWebFeed:
+    """
+    Tracks web feed usage events: analyses, subscriptions, hint refinements,
+    variant selections, and re-analyses.
+
+    Redis Key Structure:
+    - wfAna:{date}       -> sorted set {url_hash: count} for analyses
+    - wfSub:{date}       -> sorted set {url_hash: count} for subscriptions
+    - wfAnaUsers:{date}  -> set of user IDs who analyzed
+    - wfSubUsers:{date}  -> set of user IDs who subscribed
+    - wfHints:{date}     -> integer counter for hint/refine analyses
+    - wfReanalyze:{date} -> integer counter for re-analyses
+    - wfVariant:{date}   -> sorted set {variant_index: count}
+    - wfAnaSuccess:{date} -> integer counter for successful analyses
+    - wfAnaFail:{date}    -> integer counter for failed analyses
+
+    All keys expire after 35 days.
+    """
+
+    TTL_DAYS = 35
+
+    @classmethod
+    def _redis(cls):
+        return redis.Redis(connection_pool=settings.REDIS_STATISTICS_POOL)
+
+    @classmethod
+    def _today(cls):
+        return datetime.date.today().strftime("%Y-%m-%d")
+
+    @classmethod
+    def _ttl(cls):
+        return cls.TTL_DAYS * 24 * 60 * 60
+
+    @classmethod
+    def _url_hash(cls, url):
+        return hashlib.sha256(url.encode("utf-8")).hexdigest()[:16]
+
+    @classmethod
+    def record_analysis(cls, user_id, url, has_hint=False):
+        r = cls._redis()
+        today = cls._today()
+        ttl = cls._ttl()
+        url_hash = cls._url_hash(url)
+
+        pipe = r.pipeline()
+        pipe.zincrby(f"wfAna:{today}", 1, url_hash)
+        pipe.expire(f"wfAna:{today}", ttl)
+        pipe.sadd(f"wfAnaUsers:{today}", str(user_id))
+        pipe.expire(f"wfAnaUsers:{today}", ttl)
+        if has_hint:
+            pipe.incr(f"wfHints:{today}")
+            pipe.expire(f"wfHints:{today}", ttl)
+        pipe.execute()
+
+    @classmethod
+    def record_reanalysis(cls, user_id):
+        r = cls._redis()
+        today = cls._today()
+        ttl = cls._ttl()
+
+        pipe = r.pipeline()
+        pipe.incr(f"wfReanalyze:{today}")
+        pipe.expire(f"wfReanalyze:{today}", ttl)
+        pipe.sadd(f"wfAnaUsers:{today}", str(user_id))
+        pipe.expire(f"wfAnaUsers:{today}", ttl)
+        pipe.execute()
+
+    @classmethod
+    def record_analysis_result(cls, success=True):
+        r = cls._redis()
+        today = cls._today()
+        ttl = cls._ttl()
+
+        key = f"wfAnaSuccess:{today}" if success else f"wfAnaFail:{today}"
+        pipe = r.pipeline()
+        pipe.incr(key)
+        pipe.expire(key, ttl)
+        pipe.execute()
+
+    @classmethod
+    def record_subscription(cls, user_id, url, variant_index):
+        r = cls._redis()
+        today = cls._today()
+        ttl = cls._ttl()
+        url_hash = cls._url_hash(url)
+
+        pipe = r.pipeline()
+        pipe.zincrby(f"wfSub:{today}", 1, url_hash)
+        pipe.expire(f"wfSub:{today}", ttl)
+        pipe.sadd(f"wfSubUsers:{today}", str(user_id))
+        pipe.expire(f"wfSubUsers:{today}", ttl)
+        pipe.zincrby(f"wfVariant:{today}", 1, str(variant_index))
+        pipe.expire(f"wfVariant:{today}", ttl)
+        pipe.execute()
+
+    @classmethod
+    def get_daily_totals(cls, days=7):
+        """Get daily totals for analyses, subscriptions, and unique users."""
+        r = cls._redis()
+        results = []
+
+        for i in range(days):
+            day = (datetime.date.today() - datetime.timedelta(days=i)).strftime("%Y-%m-%d")
+
+            pipe = r.pipeline()
+            pipe.zrange(f"wfAna:{day}", 0, -1, withscores=True)
+            pipe.zrange(f"wfSub:{day}", 0, -1, withscores=True)
+            pipe.scard(f"wfAnaUsers:{day}")
+            vals = pipe.execute()
+
+            analyses = sum(int(s) for _, s in vals[0])
+            subscriptions = sum(int(s) for _, s in vals[1])
+            unique_users = vals[2]
+
+            results.append((day, analyses, subscriptions, unique_users))
+
+        return results
+
+    @classmethod
+    def get_stats_for_prometheus(cls):
+        r = cls._redis()
+        today = cls._today()
+
+        pipe = r.pipeline()
+        pipe.zrange(f"wfAna:{today}", 0, -1, withscores=True)        # 0: analyses
+        pipe.zrange(f"wfSub:{today}", 0, -1, withscores=True)        # 1: subscriptions
+        pipe.scard(f"wfAnaUsers:{today}")                              # 2: unique analyzing users
+        pipe.scard(f"wfSubUsers:{today}")                              # 3: unique subscribing users
+        pipe.get(f"wfHints:{today}")                                   # 4: hints
+        pipe.get(f"wfReanalyze:{today}")                               # 5: re-analyses
+        pipe.zrange(f"wfVariant:{today}", 0, -1, withscores=True)     # 6: variant choices
+        pipe.get(f"wfAnaSuccess:{today}")                              # 7: successes
+        pipe.get(f"wfAnaFail:{today}")                                 # 8: failures
+        vals = pipe.execute()
+
+        analyses_total = sum(int(s) for _, s in vals[0])
+        unique_urls_analyzed = len(vals[0])
+        subscriptions_total = sum(int(s) for _, s in vals[1])
+        unique_urls_subscribed = len(vals[1])
+        unique_users_analyzing = vals[2]
+        unique_users_subscribing = vals[3]
+        hints = int(vals[4] or 0)
+        reanalyses = int(vals[5] or 0)
+        variant_choices = {
+            (v.decode() if isinstance(v, bytes) else v): int(s) for v, s in vals[6]
+        }
+        successes = int(vals[7] or 0)
+        failures = int(vals[8] or 0)
+
+        conversion_pct = 0
+        if analyses_total > 0:
+            conversion_pct = round(subscriptions_total / analyses_total * 100, 1)
+
+        return {
+            "analyses_today": analyses_total,
+            "analyses_with_hint_today": hints,
+            "reanalyses_today": reanalyses,
+            "subscriptions_today": subscriptions_total,
+            "unique_urls_analyzed_today": unique_urls_analyzed,
+            "unique_urls_subscribed_today": unique_urls_subscribed,
+            "unique_users_analyzing_today": unique_users_analyzing,
+            "unique_users_subscribing_today": unique_users_subscribing,
+            "analysis_success_today": successes,
+            "analysis_fail_today": failures,
+            "variant_choices": variant_choices,
+            "conversion_rate_pct": conversion_pct,
+        }
diff --git a/apps/webfeed/tasks.py b/apps/webfeed/tasks.py
@@ -307,11 +307,14 @@ def publish(event_type, extra=None):
         logging.user(user, f"~BB~FWWeb Feed: Fetching page ~SB{url}~SN")
 
         # Step 1: Fetch page HTML
+        from apps.statistics.rtrending_webfeeds import RTrendingWebFeed
+
         page_html = fetch_page_html(url)
         if not page_html:
             error_msg = "Could not fetch the page. The site may be blocking requests."
             publish_event("error", {"error": error_msg})
             logging.user(user, f"~BB~FWWeb Feed: ~FR~SBFetch failed~SN~FW for ~SB{url}~SN")
+            RTrendingWebFeed.record_analysis_result(success=False)
             return {"code": -1, "message": error_msg}
 
         html_hash = hashlib.sha256(page_html[:10000].encode("utf-8", errors="replace")).hexdigest()[:16]
@@ -335,10 +338,10 @@ def publish(event_type, extra=None):
         publish_event("progress", {"message": "Finding story patterns..."})
 
         # Step 2: Call Claude for XPath analysis
-        from apps.ask_ai.providers import LLM_EXCEPTIONS, get_provider
+        from apps.ask_ai.providers import LLM_EXCEPTIONS, get_briefing_provider
 
         messages = get_analysis_messages(url, cleaned_html, story_hint=story_hint)
-        provider, model_id, _ = get_provider("opus")
+        provider, model_id = get_briefing_provider("haiku")
 
         if not provider.is_configured():
             error_msg = "Anthropic API key not configured"
@@ -379,11 +382,13 @@ def publish(event_type, extra=None):
             error_msg = "Failed to parse AI response. Please try again."
             publish_event("error", {"error": error_msg})
             logging.user(user, f"~BB~FWWeb Feed: ~FR~SBJSON parse failed~SN~FW: {response_text[:200]}")
+            RTrendingWebFeed.record_analysis_result(success=False)
             return {"code": -1, "message": error_msg}
 
         if not isinstance(variants, list) or len(variants) == 0:
             error_msg = "No story patterns found on this page."
             publish_event("error", {"error": error_msg})
+            RTrendingWebFeed.record_analysis_result(success=False)
             return {"code": -1, "message": error_msg}
 
         # Extract page title
@@ -447,6 +452,8 @@ def publish(event_type, extra=None):
         )
         publish_event("complete")
 
+        RTrendingWebFeed.record_analysis_result(success=True)
+
         return {
             "code": 1,
             "message": "Analysis complete",
@@ -460,6 +467,12 @@ def publish(event_type, extra=None):
             publish_event("error", {"error": error_msg})
         if user:
             logging.user(user, f"~BB~FWWeb Feed: ~FR~SBLLM error~SN~FW - {e}")
+        try:
+            from apps.statistics.rtrending_webfeeds import RTrendingWebFeed
+
+            RTrendingWebFeed.record_analysis_result(success=False)
+        except Exception:
+            pass
         return {"code": -1, "message": error_msg}
 
     except Exception as e:
@@ -468,4 +481,10 @@ def publish(event_type, extra=None):
             publish_event("error", {"error": error_msg})
         if user:
             logging.user(user, f"~BB~FWWeb Feed: ~FR~SBUnexpected error~SN~FW - {e}")
+        try:
+            from apps.statistics.rtrending_webfeeds import RTrendingWebFeed
+
+            RTrendingWebFeed.record_analysis_result(success=False)
+        except Exception:
+            pass
         return {"code": -1, "message": error_msg}
diff --git a/apps/webfeed/views.py b/apps/webfeed/views.py
@@ -9,6 +9,8 @@
 from utils.user_functions import ajax_login_required
 from utils.view_functions import required_params
 
+from apps.statistics.rtrending_webfeeds import RTrendingWebFeed
+
 from .models import MWebFeedConfig
 from .tasks import AnalyzeWebFeedPage
 
@@ -40,6 +42,8 @@ def analyze(request):
         f"~BB~FWWeb Feed: Analyzing ~SB{url}~SN" + (f" (hint: {story_hint})" if story_hint else ""),
     )
 
+    RTrendingWebFeed.record_analysis(request.user.pk, url, has_hint=bool(story_hint))
+
     AnalyzeWebFeedPage.apply_async(
         kwargs={
             "user_id": request.user.pk,
@@ -157,6 +161,8 @@ def subscribe(request):
     except Exception as e:
         logging.user(request.user, f"~BB~FWWeb Feed: ~FR~SBFavicon import failed~SN~FW - {e}")
 
+    RTrendingWebFeed.record_subscription(request.user.pk, url, variant_index)
+
     logging.user(request.user, f"~BB~FWWeb Feed: Subscribed to ~SB{url}~SN (feed {feed.pk})")
 
     # Trigger background fetch for archive subscribers
@@ -203,6 +209,8 @@ def reanalyze(request):
 
     url = feed.feed_address[len("webfeed:") :]
 
+    RTrendingWebFeed.record_reanalysis(request.user.pk)
+
     logging.user(request.user, f"~BB~FWWeb Feed: Re-analyzing ~SB{url}~SN (feed {feed_id})")
 
     AnalyzeWebFeedPage.apply_async(
diff --git a/docker/grafana/dashboards/newsblur_dashboard.json b/docker/grafana/dashboards/newsblur_dashboard.json
diff --git a/docker/prometheus/prometheus.consul.yml.j2 b/docker/prometheus/prometheus.consul.yml.j2
diff --git a/docker/prometheus/prometheus.docker.yml b/docker/prometheus/prometheus.docker.yml

Original file line number	Diff line number	Diff line change
`@@ -55,4 +55,5 @@`
`55`	`55`	`url(r"^user-searches?$", UserSearches.as_view(), name="user_searches"),`
`56`	`56`	`url(r"^trending-feeds?$", TrendingFeeds.as_view(), name="trending_feeds"),`
`57`	`57`	`url(r"^trending-subscriptions?$", TrendingSubscriptions.as_view(), name="trending_subscriptions"),`
	`58`	`+ url(r"^webfeed-usage?$", WebFeedUsage.as_view(), name="webfeed_usage"),`
`58`	`59`	`]`