Merge pull request #57 from priyanshujain/feat-websearch-phase3

feat(websearch): phase 3 — ranking, health tracking, httpclient, bing

Author: priyanshujain

config/config.go

@@ -143,6 +143,7 @@ type WebSearchConfig struct {
 	Proxy    string        `yaml:"proxy,omitempty"`
 	Timeout  string        `yaml:"timeout,omitempty"`
 	CacheTTL string        `yaml:"cache_ttl,omitempty"`
+	Backends []string      `yaml:"backends,omitempty"`
 }
 
 type ContactsConfig struct {

docs/websearch.md

@@ -0,0 +1,200 @@
+# WebSearch — Strategy & Design Decisions
+
+A Go metasearch data source for OpenBotKit. No API keys. No external services. Just HTTP requests + HTML parsing running locally.
+
+Inspired by [ddgs](https://github.com/deedy5/ddgs) (Python), rebuilt in Go as a first-class openbotkit source.
+
+---
+
+## Problem
+
+Every open-source coding agent delegates web search to their model provider's API. There is no standalone, self-contained, local web search capability that agents can use. We need a Go equivalent integrated into openbotkit — search, news, and page fetching accessible via CLI, consumed by agents as skills.
+
+## Design Principles
+
+1. **Follows openbotkit patterns** — Source interface, CLI commands via `obk websearch`, skills as SKILL.md/REFERENCE.md, SQLite for caching/history.
+2. **Zero dependencies on external services** — No API keys, no Docker, no servers. Just HTTP scraping.
+3. **Composable** — Search, news, and fetch are separate skills. The agent orchestrates.
+4. **Resilient** — Multiple backends with concurrent dispatch, automatic fallback, per-host rate limiting, and health tracking.
+
+---
+
+## Architecture
+
+```
+Skills Layer
+  web-search, web-fetch, web-news (SKILL.md + REFERENCE.md each)
+        │
+CLI Layer (obk websearch ...)
+  search, news, fetch, backends, history, cache clear
+        │
+source/websearch/ (Go package)
+  ┌─────────────────────────────────────────────┐
+  │  Orchestrator                                │
+  │  - Backend selection (auto / explicit)       │
+  │  - Concurrent dispatch (errgroup)            │
+  │  - Result ranking & deduplication            │
+  │  - Health tracking (exponential cooldown)    │
+  ├─────────────────────────────────────────────┤
+  │  Engines (each: HTTP request + HTML parse)   │
+  │  DDG, Brave, Mojeek, Wikipedia, Yahoo,       │
+  │  Yandex, Google, Bing                        │
+  ├─────────────────────────────────────────────┤
+  │  httpclient/                                 │
+  │  - Wraps internal/browser (utls transport)   │
+  │  - UA rotation (4 browser profiles)          │
+  │  - Per-host token bucket rate limiting       │
+  ├─────────────────────────────────────────────┤
+  │  SQLite (cache + history)                    │
+  │  search_cache, fetch_cache, search_history   │
+  └─────────────────────────────────────────────┘
+```
+
+---
+
+## Key Decisions
+
+### 1. Flat package over engine/ subdirectory
+
+**Decision:** Engines live directly in `source/websearch/` (e.g., `duckduckgo.go`, `brave.go`), not in a `source/websearch/engine/` subdirectory.
+
+**Why:** The engines are small (50-120 lines each) and tightly coupled to the orchestrator. A subdirectory adds import ceremony for no benefit. Each engine file is self-contained — struct, constructor, `Search()`/`News()` method, and any helper functions.
+
+### 2. No parse/ package
+
+**Decision:** HTML parsing helpers stay private and co-located with each engine.
+
+**Why:** Each engine has its own HTML structure and parsing quirks. Extracting shared "parse helpers" would create false abstractions — the parsing code between DuckDuckGo and Brave has nothing in common. Functions are small, private, and belong next to their callers.
+
+### 3. HTTPDoer interface over concrete *http.Client
+
+**Decision:** Engines accept `HTTPDoer` interface (`Do(*http.Request) (*http.Response, error)`) instead of `*http.Client`.
+
+**Why:** This lets engines work with both `*http.Client` (in tests, using httptest) and `*httpclient.Client` (in production, with UA rotation + rate limiting). Engines stay testable without mocking infrastructure.
+
+### 4. Multi-backend fallback over exponential backoff
+
+**Decision:** No `cenkalti/backoff` dependency. Retry strategy is multi-backend fallback, not per-request retries.
+
+**Why:** This is a CLI tool, not a long-running service. When a user runs a search, they want results in seconds, not after a backoff sequence. If DuckDuckGo fails, we immediately try Brave, Mojeek, etc. The health tracker prevents repeatedly hitting a broken backend (exponential cooldown: 30s → 5min). This gives better UX than retrying the same failing backend with increasing delays.
+
+### 5. Concurrent dispatch with errgroup
+
+**Decision:** All backends in the auto set run concurrently via `errgroup`, not sequentially.
+
+**Why:** With 4 default backends, sequential dispatch means total latency = sum of all backend latencies. Concurrent dispatch means latency = max(backend latencies). Results are collected with a mutex, then sorted by engine priority before ranking to maintain deterministic output.
+
+### 6. Separate fetch client with SSRF protection
+
+**Decision:** `fetchClient()` (for `obk websearch fetch`) uses a raw `*http.Client` with SSRF guards. `httpClient()` (for search engines) uses `*httpclient.Client` without SSRF guards.
+
+**Why:** Search engines hit known external hosts (duckduckgo.com, bing.com, etc.) — SSRF isn't a concern. But `fetch` takes arbitrary user-provided URLs, so it must resolve DNS and block private IPs (loopback, RFC1918, link-local) to prevent SSRF. It also pins to the first resolved IP to prevent DNS rebinding (TOCTOU).
+
+### 7. UA rotation per client instance, not per request
+
+**Decision:** A random browser profile (Chrome/Firefox/Safari/Edge) is selected when `httpclient.Client` is created and reused for all requests from that client.
+
+**Why:** A real browser sends the same UA across a session. Rotating per-request looks suspicious to anti-bot systems. The client is created once per `WebSearch` instance and reused across searches.
+
+### 8. Auto backend set: DDG + Brave + Mojeek + Wikipedia
+
+**Decision:** The default `auto` set uses DuckDuckGo, Brave, Mojeek, and Wikipedia. Yahoo, Yandex, Google, and Bing are opt-in only via `--backend <name>`.
+
+**Why:**
+- **DuckDuckGo** — Most reliable scraping target. No-JS HTML endpoint.
+- **Brave** — Good quality, less aggressive anti-bot than Google.
+- **Mojeek** — Very permissive, independent index (not Bing/Google derivative).
+- **Wikipedia** — JSON API (no scraping needed), always high-quality for factual queries.
+- **Google** — Most aggressive anti-bot, CAPTCHAs likely. Opt-in only.
+- **Bing** — Disabled in ddgs too. Opt-in only.
+- **Yahoo/Yandex** — Redirect URL unwrapping adds fragility. Opt-in only.
+
+Users can override via config (`websearch.backends` list) or `--backend` flag.
+
+### 9. Ranking: frequency + token scoring + Wikipedia priority
+
+**Decision:** Results are ranked by: multi-backend appearance bonus, query token scoring (title weight 2x, snippet weight 1x), Wikipedia +10 bonus. Stable sort preserves original order for equal scores.
+
+**Why:** Simple, predictable, no ML. A result appearing from 3 backends is likely more relevant than one from 1 backend. Title matches matter more than snippet matches. Wikipedia is almost always the best single result for factual queries — the +10 bonus ensures it surfaces first without suppressing other results.
+
+### 10. Cache key includes page number
+
+**Decision:** Cache key = `sha256(query|category|backend|region|timeLimit|page)`.
+
+**Why:** Without page in the key, searching "golang" page 1 then "golang" page 2 returns page 1's cached results. Learned this from a bug caught in review.
+
+### 11. Best-effort caching and history, not transactional
+
+**Decision:** `putSearchCache`, `putFetchCache`, and `putSearchHistory` log warnings on failure but don't propagate errors.
+
+**Why:** Cache and history are convenience features. A failed cache write shouldn't make a successful search return an error. The slog.Warn ensures failures are visible for debugging.
+
+### 12. Bing URL unwrapping via base64
+
+**Decision:** Bing wraps result URLs in `/ck/a?u=<encoded>` redirects. We decode inline: strip first 2 chars of the `u` param, base64url decode the rest.
+
+**Why:** Following Bing's real redirect URL isn't reliable (may require cookies/sessions). The base64 encoding scheme was reverse-engineered from ddgs and is stable.
+
+---
+
+## What We Chose Not to Build
+
+| Feature | Reason |
+|---------|--------|
+| Exponential backoff (`cenkalti/backoff`) | Multi-backend fallback is the retry strategy for a CLI tool |
+| `SearchError` structured error type | Plain `fmt.Errorf` is sufficient; errors flow through cobra's `RunE` |
+| `--proxy` CLI flag / `WEBSEARCH_PROXY` env var | Proxy works via config.yaml; CLI flag adds complexity for a rarely-used feature |
+| Session recovery (401/403 retry) | Multi-backend fallback handles this implicitly |
+| Rate limiter eviction | Not needed for CLI (process is short-lived); TODO left for library use |
+| Daemon cache warming | Deferred — low value until usage patterns are established |
+
+---
+
+## Backend Reference
+
+| Backend | Method | Priority | Auto | Notes |
+|---------|--------|----------|------|-------|
+| DuckDuckGo | POST `html.duckduckgo.com/html/` | 1 | Yes | Most reliable. No-JS endpoint. Max 499 char query. |
+| Brave | GET `search.brave.com/search` | 1 | Yes | Good quality, moderate anti-bot. |
+| Mojeek | GET `www.mojeek.com/search` | 1 | Yes | Very permissive. Independent index. |
+| Wikipedia | GET `en.wikipedia.org/w/api.php` | 2 | Yes | JSON API. +10 ranking bonus. Filters disambiguation. |
+| Yahoo | GET `search.yahoo.com/search` | 1 | No | Requires redirect URL unwrapping. |
+| Yandex | GET `yandex.com/search/site/` | 1 | No | Uses random searchid. |
+| Google | GET `www.google.com/search` | 0 | No | Most aggressive anti-bot. Lowest priority. |
+| Bing | GET `www.bing.com/search` | 0 | No | base64 URL unwrapping. Ad filtering. |
+
+News backends: DuckDuckGo (VQD token + JSON API), Yahoo (HTML scraping).
+
+---
+
+## Security
+
+- **SSRF protection on fetch**: DNS resolution + private IP blocking + IP pinning (anti-DNS-rebinding)
+- **No SSRF on search**: Engines only hit known external hosts
+- **SQL injection**: All queries use parameterized placeholders (`db.Rebind("... ?")`)
+- **Query length limit**: 2000 chars max to prevent abuse
+- **Response body limit**: 10MB hard cap on fetch, body size limits on news/VQD responses
+
+---
+
+## Dependencies
+
+| Package | Purpose |
+|---------|---------|
+| `github.com/PuerkitoBio/goquery` | HTML parsing with CSS selectors |
+| `golang.org/x/time/rate` | Per-host token bucket rate limiting |
+| `golang.org/x/sync/errgroup` | Concurrent backend dispatch |
+| `github.com/JohannesKaufmann/html-to-markdown` | HTML → Markdown for fetch |
+| `github.com/refraction-networking/utls` | TLS fingerprint impersonation (via internal/browser) |
+
+---
+
+## Prior Art
+
+| Project | Language | Gap |
+|---------|----------|-----|
+| [ddgs](https://github.com/deedy5/ddgs) | Python | Python-only. Uses `primp` (Rust) for TLS. |
+| [SearXNG](https://github.com/searxng/searxng) | Python | Requires Docker. Server-based. |
+| [Djarvur/ddg-search](https://github.com/Djarvur/ddg-search) | Go | DDG only, no multi-backend. |
+
+This fills the gap: a Go data source with multi-backend search, skills integration, and agent-first CLI design.

go.mod

@@ -22,6 +22,7 @@ require (
 	go.mau.fi/whatsmeow v0.0.0-20260227112304-c9652e4448a2
 	golang.org/x/net v0.50.0
 	golang.org/x/oauth2 v0.35.0
+	golang.org/x/sync v0.19.0
 	golang.org/x/time v0.14.0
 	google.golang.org/api v0.269.0
 	google.golang.org/protobuf v1.36.11
@@ -143,7 +144,6 @@ require (
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/exp v0.0.0-20260212183809-81e46e3db34a // indirect
-	golang.org/x/sync v0.19.0 // indirect
 	golang.org/x/sys v0.41.0 // indirect
 	golang.org/x/text v0.34.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect

internal/cli/websearch/backends.go

@@ -26,6 +26,7 @@ var backendsCmd = &cobra.Command{
 			{Name: "yandex", Priority: 1, News: false},
 			{Name: "google", Priority: 0, News: false},
 			{Name: "wikipedia", Priority: 2, News: false},
+			{Name: "bing", Priority: 0, News: false},
 		}
 		return json.NewEncoder(os.Stdout).Encode(backends)
 	},

internal/cli/websearch/cache.go

@@ -0,0 +1,50 @@
+package websearch
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/priyanshujain/openbotkit/config"
+	wssrc "github.com/priyanshujain/openbotkit/source/websearch"
+	"github.com/priyanshujain/openbotkit/store"
+	"github.com/spf13/cobra"
+)
+
+var cacheCmd = &cobra.Command{
+	Use:   "cache",
+	Short: "Manage websearch cache",
+}
+
+var cacheClearCmd = &cobra.Command{
+	Use:   "clear",
+	Short: "Clear all cached search results and fetched pages",
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfg, err := config.Load()
+		if err != nil {
+			return fmt.Errorf("load config: %w", err)
+		}
+
+		db, err := store.Open(store.SQLiteConfig(cfg.WebSearchDataDSN()))
+		if err != nil {
+			return fmt.Errorf("open db: %w", err)
+		}
+		defer db.Close()
+
+		if err := wssrc.Migrate(db); err != nil {
+			return fmt.Errorf("migrate: %w", err)
+		}
+
+		ws := wssrc.New(wssrc.Config{WebSearch: cfg.WebSearch}, wssrc.WithDB(db))
+		if err := ws.ClearCaches(); err != nil {
+			return err
+		}
+
+		return json.NewEncoder(os.Stdout).Encode(map[string]string{"status": "cleared"})
+	},
+}
+
+func init() {
+	cacheCmd.AddCommand(cacheClearCmd)
+}

internal/cli/websearch/history.go

@@ -0,0 +1,69 @@
+package websearch
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/priyanshujain/openbotkit/config"
+	wssrc "github.com/priyanshujain/openbotkit/source/websearch"
+	"github.com/priyanshujain/openbotkit/store"
+	"github.com/spf13/cobra"
+)
+
+type historyEntry struct {
+	Query       string `json:"query"`
+	Category    string `json:"category"`
+	ResultCount int    `json:"result_count"`
+	Backends    string `json:"backends"`
+	SearchMs    int64  `json:"search_ms"`
+	CreatedAt   string `json:"created_at"`
+}
+
+var historyCmd = &cobra.Command{
+	Use:   "history",
+	Short: "Show recent search history",
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfg, err := config.Load()
+		if err != nil {
+			return fmt.Errorf("load config: %w", err)
+		}
+
+		db, err := store.Open(store.SQLiteConfig(cfg.WebSearchDataDSN()))
+		if err != nil {
+			return fmt.Errorf("open db: %w", err)
+		}
+		defer db.Close()
+
+		if err := wssrc.Migrate(db); err != nil {
+			return fmt.Errorf("migrate: %w", err)
+		}
+
+		limit, _ := cmd.Flags().GetInt("limit")
+
+		rows, err := db.Query("SELECT query, category, result_count, backends, search_ms, created_at FROM search_history ORDER BY created_at DESC LIMIT ?", limit)
+		if err != nil {
+			return fmt.Errorf("query history: %w", err)
+		}
+		defer rows.Close()
+
+		var entries []historyEntry
+		for rows.Next() {
+			var e historyEntry
+			if err := rows.Scan(&e.Query, &e.Category, &e.ResultCount, &e.Backends, &e.SearchMs, &e.CreatedAt); err != nil {
+				return fmt.Errorf("scan row: %w", err)
+			}
+			entries = append(entries, e)
+		}
+		if entries == nil {
+			entries = []historyEntry{}
+		}
+
+		return json.NewEncoder(os.Stdout).Encode(entries)
+	},
+}
+
+func init() {
+	historyCmd.Flags().Int("limit", 20, "Maximum number of entries to show")
+}

internal/cli/websearch/news.go

@@ -25,6 +25,7 @@ var newsCmd = &cobra.Command{
 		backend, _ := cmd.Flags().GetString("backend")
 		timeLimit, _ := cmd.Flags().GetString("time-limit")
 		region, _ := cmd.Flags().GetString("region")
+		page, _ := cmd.Flags().GetInt("page")
 		noCache, _ := cmd.Flags().GetBool("no-cache")
 
 		var opts []wssrc.Option
@@ -42,6 +43,7 @@ var newsCmd = &cobra.Command{
 			Backend:    backend,
 			TimeLimit:  timeLimit,
 			Region:     region,
+			Page:       page,
 			NoCache:    noCache,
 		})
 		if err != nil {
@@ -57,5 +59,6 @@ func init() {
 	newsCmd.Flags().StringP("backend", "b", "auto", "News backend (auto, duckduckgo, yahoo)")
 	newsCmd.Flags().StringP("time-limit", "t", "", "Time limit (d=day, w=week, m=month)")
 	newsCmd.Flags().StringP("region", "r", "us-en", "Region for news results")
+	newsCmd.Flags().IntP("page", "p", 1, "Page number for pagination")
 	newsCmd.Flags().Bool("no-cache", false, "Bypass result cache")
 }