use an algorithm approach for calculating how often to query

Author: Thomas Stromberg

poll/poll.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"log/slog"
+	"math"
 	"sync"
 	"time"
 
@@ -120,8 +121,8 @@ func (m *Monitor) CheckAll(ctx context.Context) error {
 
 		// Use any subscriber's thread info to check intervals (they should all be the same)
 		thread := info.thread
-		interval, reason := calculateInterval(thread.LastPostTime, thread.LastPolledAt)
-		timeSinceLastPoll := time.Now().Sub(thread.LastPolledAt)
+		interval, reason := CalculateInterval(thread.LastPostTime, thread.LastPolledAt)
+		timeSinceLastPoll := time.Since(thread.LastPolledAt)
 		needsCheck := timeSinceLastPoll >= interval
 
 		// Format times for logging, handling zero values
@@ -472,48 +473,58 @@ func (m *Monitor) checkThreadForSubscribers(ctx context.Context, info *threadChe
 	return hasUpdates, savedEmails, nil
 }
 
-// calculateInterval determines how often to poll a thread based on activity.
-// Returns the interval duration and a human-readable reason explaining the decision.
+// CalculateInterval determines how often to poll a thread based on activity.
+// Uses exponential backoff: the longer since the last post, the less frequently we check.
+// Formula: interval = min(minInterval * 2^(hours_since_post / scaleFactor), maxInterval)
+//
+// This provides smooth scaling:
+//   - 0h since post → 5 minutes
+//   - 3h since post → 10 minutes
+//   - 6h since post → 20 minutes
+//   - 12h since post → 80 minutes
+//   - 24h+ since post → 4 hours (capped)
+//
 // NEVER returns 0s - always returns a minimum interval to prevent polling loops.
-func calculateInterval(lastPostTime, lastPolledAt time.Time) (time.Duration, string) {
+func CalculateInterval(lastPostTime, lastPolledAt time.Time) (time.Duration, string) {
 	const minInterval = 5 * time.Minute // Minimum safe interval
+	const maxInterval = 4 * time.Hour   // Maximum interval for inactive threads
+	const scaleFactor = 3.0             // Hours before interval doubles (smaller = more aggressive backoff)
 
-	// If never polled before, use minimum interval
+	// CRITICAL: These should NEVER be zero after subscription creation.
+	// If they are, it indicates a serious bug in subscription or polling logic.
 	if lastPolledAt.IsZero() {
-		return minInterval, "never polled before (using minimum interval)"
+		return maxInterval, "CRITICAL ERROR: LastPolledAt is zero (this should never happen - bug in subscription creation)"
 	}
 
-	// If no post time recorded, use maximum interval (something is wrong)
 	if lastPostTime.IsZero() {
-		return 6 * time.Hour, "ERROR: no post time recorded (using maximum interval to avoid polling loop)"
+		return maxInterval, "CRITICAL ERROR: LastPostTime is zero (this should never happen - timestamp validation failed)"
 	}
 
 	// Calculate time since last post
-	timeSinceLastPost := time.Since(lastPostTime)
+	hoursSincePost := time.Since(lastPostTime).Hours()
 
-	var interval time.Duration
+	// Exponential backoff: interval doubles every scaleFactor hours
+	// Example with scaleFactor=3: 0h→5m, 3h→10m, 6h→20m, 9h→40m, 12h→80m
+	multiplier := math.Pow(2.0, hoursSincePost/scaleFactor)
+	interval := time.Duration(float64(minInterval) * multiplier)
+
+	// Clamp to min/max bounds
+	if interval > maxInterval {
+		interval = maxInterval
+	}
+	if interval < minInterval {
+		interval = minInterval
+	}
+
+	// Format reason with readable time units
 	var reason string
 	switch {
-	case timeSinceLastPost < 30*time.Minute:
-		interval = 5 * time.Minute
-		reason = "very active thread (post < 30m ago)"
-	case timeSinceLastPost < 2*time.Hour:
-		interval = 10 * time.Minute
-		reason = "active thread (post < 2h ago)"
-	case timeSinceLastPost < 6*time.Hour:
-		interval = 20 * time.Minute
-		reason = "moderately active thread (post < 6h ago)"
-	case timeSinceLastPost < 24*time.Hour:
-		interval = 1 * time.Hour
-		reason = "daily active thread (post < 24h ago)"
+	case hoursSincePost < 1:
+		reason = fmt.Sprintf("%.0f minutes since last post", hoursSincePost*60)
+	case hoursSincePost < 48:
+		reason = fmt.Sprintf("%.1f hours since last post", hoursSincePost)
 	default:
-		interval = 6 * time.Hour
-		reason = "inactive thread (post > 24h ago)"
-	}
-
-	// Safety check: never return 0s interval
-	if interval == 0 {
-		return minInterval, "ERROR: interval calculation resulted in 0s (using minimum interval)"
+		reason = fmt.Sprintf("%.0f days since last post", hoursSincePost/24)
 	}
 
 	return interval, reason

server/subscribe.go

@@ -1,12 +1,14 @@
 package server
 
 import (
+	"context"
 	"fmt"
 	"net/http"
 	"strings"
 	"time"
 
 	"advrider-notifier/pkg/notifier"
+	"advrider-notifier/poll"
 )
 
 func (s *Server) handleSubscribe(w http.ResponseWriter, r *http.Request) {
@@ -146,6 +148,8 @@ func (s *Server) handleSubscribe(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	now := time.Now().UTC()
+
 	s.logger.Info("Creating subscription with latest post ID",
 		"email", email,
 		"thread_id", threadID,
@@ -154,14 +158,15 @@ func (s *Server) handleSubscribe(w http.ResponseWriter, r *http.Request) {
 		"last_post_time", lastPostTime.Format(time.RFC3339))
 
 	// Add thread to subscription
+	// Set LastPolledAt to now since we just fetched the thread during subscription
 	sub.Threads[threadID] = &notifier.Thread{
 		ThreadURL:    baseThreadURL,
 		ThreadID:     threadID,
 		ThreadTitle:  threadTitle,
 		LastPostID:   post.ID,
 		LastPostTime: lastPostTime,
-		LastPolledAt: time.Time{}, // Will be set on first poll
-		CreatedAt:    time.Now().UTC(),
+		LastPolledAt: now, // Set to now since we just verified the thread
+		CreatedAt:    now,
 	}
 
 	if err := s.store.Save(r.Context(), sub); err != nil {
@@ -170,22 +175,60 @@ func (s *Server) handleSubscribe(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	s.logger.Info("Subscription created", "email", email, "thread_id", threadID, "ip", ip)
+
+	// Trigger immediate poll to notify all existing subscribers about any new posts
+	// This runs asynchronously so we don't block the HTTP response
+	// Use background context since we don't want this tied to the HTTP request lifecycle
+	go func() {
+		pollCtx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+		defer cancel()
+
+		s.logger.Info("Triggering poll after new subscription", "thread_url", baseThreadURL, "email", email)
+		if err := s.poller.CheckAll(pollCtx); err != nil {
+			s.logger.Warn("Post-subscription poll failed", "error", err)
+		}
+	}()
+
 	// Send welcome email
 	userAgent := r.Header.Get("User-Agent")
 	if err := s.emailer.SendWelcome(r.Context(), sub, sub.Threads[threadID], ip, userAgent); err != nil {
 		// Log error but don't fail the subscription
 		s.logger.Warn("Failed to send welcome email", "email", email, "error", err)
 	}
 
-	s.logger.Info("Subscription created", "email", email, "thread_id", threadID, "ip", ip)
+	// Calculate next crawl time based on thread activity
+	interval, reason := poll.CalculateInterval(lastPostTime, now)
+	nextCrawlTime := now.Add(interval)
+
+	// Format time delta for display
+	var crawlTimeStr string
+	switch {
+	case interval < time.Hour:
+		crawlTimeStr = fmt.Sprintf("%d minutes", int(interval.Minutes()))
+	case interval < 2*time.Hour:
+		crawlTimeStr = "1 hour"
+	default:
+		crawlTimeStr = fmt.Sprintf("%d hours", int(interval.Hours()))
+	}
+
+	s.logger.Info("Subscription completed",
+		"email", email,
+		"thread_id", threadID,
+		"next_crawl_in", interval.String(),
+		"crawl_reason", reason)
 
 	// Set cookie to remember email address
 	setEmailCookie(w, email)
 
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	w.Header().Set("X-Content-Type-Options", "nosniff")
 	w.WriteHeader(http.StatusOK)
-	if err := templates.ExecuteTemplate(w, "subscribed.tmpl", map[string]string{"Email": email}); err != nil {
+	if err := templates.ExecuteTemplate(w, "subscribed.tmpl", map[string]any{
+		"Email":       email,
+		"CrawlTime":   crawlTimeStr,
+		"NextCrawlAt": nextCrawlTime.Format("3:04 PM MST"),
+	}); err != nil {
 		s.logger.Error("Failed to render template", "template", "subscribed.tmpl", "error", err)
 		http.Error(w, "Internal server error", http.StatusInternalServerError)
 	}

server/tmpl/subscribed.tmpl

@@ -19,6 +19,7 @@
 		<div class="icon">✓</div>
 		<h1>Subscription Created!</h1>
 		<p>You'll receive an email at <strong>{{.Email}}</strong> whenever new posts appear on this thread.</p>
+		<p style="font-size: 15px; color: #666; margin-top: 16px;">Next check scheduled in approximately <strong>{{.CrawlTime}}</strong> ({{.NextCrawlAt}})</p>
 		<p style="font-size: 15px; color: #999;">Each email will include a secure link to manage your subscriptions.</p>
 		<a href="/" class="button">Subscribe to Another Thread</a>
 	</div>