DB query optimizations (#5)

* Use FTS and precompute aggregate package stats

Avoids expensive aggregate calculations and FTS is much faster for text
search.

* Fix NULL crash on empty DB and search regression for collapsed slugs

Wrap active_plugins/active_themes SUM in COALESCE to prevent NOT NULL
constraint failures when no active packages exist. Add LIKE fallback
for slug searches so compact queries like "woocommerceblocks" still
match hyphenated names like "woo-commerce-blocks".

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Fix migration numbering and keep package stats fresh

---------

Co-authored-by: Ben Word <ben@benword.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 3 people

cmd/wpcomposer/cmd/dev.go

@@ -128,6 +128,9 @@ func runDev(cmd *cobra.Command, args []string) error {
 	_ = packages.FinishSyncRun(ctx, application.DB, syncRun.RowID, "completed", map[string]any{
 		"updated": succeeded.Load(), "failed": failed.Load(),
 	})
+	if err := packages.RefreshSiteStats(ctx, application.DB); err != nil {
+		return fmt.Errorf("refreshing package stats: %w", err)
+	}
 	application.Logger.Info("dev: metadata fetched", "updated", succeeded.Load(), "failed", failed.Load())
 
 	// 5. Build

cmd/wpcomposer/cmd/discover.go

@@ -102,6 +102,9 @@ func discoverFromConfig(ctx context.Context, pkgType string, limit, concurrency
 	}
 
 	application.Logger.Info("discovery complete", "succeeded", succeeded.Load(), "failed", failed.Load())
+	if err := packages.RefreshSiteStats(ctx, application.DB); err != nil {
+		return fmt.Errorf("refreshing package stats: %w", err)
+	}
 	if failed.Load() > 0 {
 		return fmt.Errorf("discovery completed with %d failures", failed.Load())
 	}
@@ -183,6 +186,10 @@ func discoverFromSVN(ctx context.Context, pkgType string, limit int) error {
 		}
 	}
 
+	if err := packages.RefreshSiteStats(ctx, application.DB); err != nil {
+		return fmt.Errorf("refreshing package stats: %w", err)
+	}
+
 	if totalFailed > 0 {
 		return fmt.Errorf("SVN discovery completed with %d failures", totalFailed)
 	}

cmd/wpcomposer/cmd/update.go

@@ -196,6 +196,10 @@ func runUpdate(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("finishing sync run: %w", err)
 	}
 
+	if err := packages.RefreshSiteStats(ctx, application.DB); err != nil {
+		return fmt.Errorf("refreshing package stats: %w", err)
+	}
+
 	application.Logger.Info("update complete",
 		"updated", succeeded.Load(),
 		"failed", failed.Load(),

internal/db/migrate_test.go

@@ -0,0 +1,39 @@
+package db
+
+import (
+	"database/sql"
+	"testing"
+
+	wpcomposergo "github.com/roots/wp-composer"
+)
+
+func TestMigrateCreatesPackageStatsAndFTS(t *testing.T) {
+	dbPath := t.TempDir() + "/test.db"
+	database, err := Open(dbPath)
+	if err != nil {
+		t.Fatalf("opening db: %v", err)
+	}
+	t.Cleanup(func() { _ = database.Close() })
+
+	if err := Migrate(database, wpcomposergo.Migrations); err != nil {
+		t.Fatalf("running migrations: %v", err)
+	}
+
+	assertObjectExists(t, database, "table", "package_stats")
+	assertObjectExists(t, database, "table", "packages_fts")
+	assertObjectExists(t, database, "trigger", "packages_fts_insert")
+	assertObjectExists(t, database, "trigger", "packages_fts_update")
+	assertObjectExists(t, database, "trigger", "packages_fts_delete")
+}
+
+func assertObjectExists(t *testing.T, database *sql.DB, objType, name string) {
+	t.Helper()
+	var count int
+	err := database.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type = ? AND name = ?`, objType, name).Scan(&count)
+	if err != nil {
+		t.Fatalf("querying sqlite_master for %s %s: %v", objType, name, err)
+	}
+	if count != 1 {
+		t.Fatalf("expected %s %s to exist", objType, name)
+	}
+}

internal/http/handlers.go

@@ -537,20 +537,40 @@ type indexStats struct {
 
 func queryIndexStats(ctx context.Context, db *sql.DB) indexStats {
 	var s indexStats
-	_ = db.QueryRowContext(ctx, "SELECT COALESCE(SUM(wp_composer_installs_total), 0) FROM packages WHERE type = 'plugin'").Scan(&s.PluginInstalls)
-	_ = db.QueryRowContext(ctx, "SELECT COALESCE(SUM(wp_composer_installs_total), 0) FROM packages WHERE type = 'theme'").Scan(&s.ThemeInstalls)
+	_ = db.QueryRowContext(ctx, "SELECT plugin_installs, theme_installs FROM package_stats WHERE id = 1").Scan(&s.PluginInstalls, &s.ThemeInstalls)
 	return s
 }
 
+// collapseSlug strips hyphens, underscores, and spaces to produce a
+// compact form suitable for LIKE-matching against similarly collapsed names.
+func collapseSlug(s string) string {
+	s = strings.ToLower(s)
+	s = strings.ReplaceAll(s, "-", "")
+	s = strings.ReplaceAll(s, "_", "")
+	s = strings.ReplaceAll(s, " ", "")
+	return s
+}
+
+// ftsQuery converts a user search string into an FTS5 query.
+// Each token becomes a prefix search, joined with AND.
+// e.g. "woo commerce" -> "woo* AND commerce*"
+func ftsQuery(s string) string {
+	words := strings.Fields(s)
+	for i, w := range words {
+		// Escape double quotes to prevent FTS5 syntax injection
+		w = strings.ReplaceAll(w, `"`, `""`)
+		words[i] = `"` + w + `"` + "*"
+	}
+	return strings.Join(words, " AND ")
+}
+
 func queryPackages(ctx context.Context, db *sql.DB, f publicFilters, page, limit int) ([]packageRow, int, error) {
 	where := "is_active = 1"
 	args := []any{}
 
-	if f.Search != "" {
-		normalized := "%" + strings.NewReplacer("-", "", " ", "").Replace(f.Search) + "%"
-		s := "%" + f.Search + "%"
-		where += " AND (REPLACE(REPLACE(name, '-', ''), ' ', '') LIKE ? OR display_name LIKE ? OR description LIKE ?)"
-		args = append(args, normalized, s, s)
+	if q := ftsQuery(f.Search); q != "" {
+		where += " AND (id IN (SELECT rowid FROM packages_fts WHERE packages_fts MATCH ?) OR REPLACE(name, '-', '') LIKE ?)"
+		args = append(args, q, "%"+collapseSlug(f.Search)+"%")
 	}
 	if f.Type != "" {
 		where += " AND type = ?"
@@ -568,9 +588,13 @@ func queryPackages(ctx context.Context, db *sql.DB, f publicFilters, page, limit
 	}
 
 	var total int
-	countQ := "SELECT COUNT(*) FROM packages WHERE " + where
-	if err := db.QueryRowContext(ctx, countQ, args...).Scan(&total); err != nil {
-		return nil, 0, err
+	if f.Search == "" && f.Type == "" {
+		_ = db.QueryRowContext(ctx, "SELECT active_plugins + active_themes FROM package_stats WHERE id = 1").Scan(&total)
+	} else {
+		countQ := "SELECT COUNT(*) FROM packages WHERE " + where
+		if err := db.QueryRowContext(ctx, countQ, args...).Scan(&total); err != nil {
+			return nil, 0, err
+		}
 	}
 
 	offset := (page - 1) * limit
@@ -693,11 +717,9 @@ func queryDashboardStats(ctx context.Context, db *sql.DB) map[string]any {
 		CurrentBuild  string
 	}
 
-	_ = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM packages WHERE is_active = 1").Scan(&s.TotalPackages)
-	_ = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM packages WHERE is_active = 1 AND type = 'plugin'").Scan(&s.ActivePlugins)
-	_ = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM packages WHERE is_active = 1 AND type = 'theme'").Scan(&s.ActiveThemes)
-	_ = db.QueryRowContext(ctx, "SELECT COALESCE(SUM(wp_composer_installs_total), 0) FROM packages").Scan(&s.TotalInstalls)
-	_ = db.QueryRowContext(ctx, "SELECT COALESCE(SUM(wp_composer_installs_30d), 0) FROM packages").Scan(&s.Installs30d)
+	_ = db.QueryRowContext(ctx, `SELECT active_plugins, active_themes, active_plugins + active_themes,
+		plugin_installs + theme_installs, installs_30d FROM package_stats WHERE id = 1`).Scan(
+		&s.ActivePlugins, &s.ActiveThemes, &s.TotalPackages, &s.TotalInstalls, &s.Installs30d)
 
 	stats["Stats"] = s
 	return stats
@@ -707,11 +729,9 @@ func queryAdminPackages(ctx context.Context, db *sql.DB, f adminFilters, page, l
 	where := "1=1"
 	args := []any{}
 
-	if f.Search != "" {
-		normalized := "%" + strings.NewReplacer("-", "", " ", "").Replace(f.Search) + "%"
-		s := "%" + f.Search + "%"
-		where += " AND (REPLACE(REPLACE(name, '-', ''), ' ', '') LIKE ? OR display_name LIKE ? OR description LIKE ?)"
-		args = append(args, normalized, s, s)
+	if q := ftsQuery(f.Search); q != "" {
+		where += " AND (id IN (SELECT rowid FROM packages_fts WHERE packages_fts MATCH ?) OR REPLACE(name, '-', '') LIKE ?)"
+		args = append(args, q, "%"+collapseSlug(f.Search)+"%")
 	}
 	if f.Type != "" {
 		where += " AND type = ?"

internal/packages/package.go

@@ -337,6 +337,25 @@ func DeactivatePackage(ctx context.Context, db *sql.DB, id int64) error {
 	return nil
 }
 
+// RefreshSiteStats recomputes the package_stats row from the packages table.
+func RefreshSiteStats(ctx context.Context, db *sql.DB) error {
+	_, err := db.ExecContext(ctx, `
+		INSERT OR REPLACE INTO package_stats (id, active_plugins, active_themes, plugin_installs, theme_installs, installs_30d, updated_at)
+		SELECT 1,
+			COALESCE(SUM(CASE WHEN type = 'plugin' THEN 1 ELSE 0 END), 0),
+			COALESCE(SUM(CASE WHEN type = 'theme' THEN 1 ELSE 0 END), 0),
+			COALESCE(SUM(CASE WHEN type = 'plugin' THEN wp_composer_installs_total ELSE 0 END), 0),
+			COALESCE(SUM(CASE WHEN type = 'theme' THEN wp_composer_installs_total ELSE 0 END), 0),
+			COALESCE(SUM(wp_composer_installs_30d), 0),
+			datetime('now')
+		FROM packages
+		WHERE is_active = 1`)
+	if err != nil {
+		return fmt.Errorf("refreshing package stats: %w", err)
+	}
+	return nil
+}
+
 func boolToInt(b bool) int {
 	if b {
 		return 1

internal/packages/package_test.go

@@ -46,14 +46,23 @@ func setupTestDB(t *testing.T) *sql.DB {
 			updated_at TEXT NOT NULL,
 			UNIQUE(type, name)
 		);
-		CREATE TABLE sync_runs (
-			id INTEGER PRIMARY KEY,
-			started_at TEXT NOT NULL,
-			finished_at TEXT,
-			status TEXT NOT NULL,
-			meta_json TEXT NOT NULL DEFAULT '{}'
-		);
-	`)
+			CREATE TABLE sync_runs (
+				id INTEGER PRIMARY KEY,
+				started_at TEXT NOT NULL,
+				finished_at TEXT,
+				status TEXT NOT NULL,
+				meta_json TEXT NOT NULL DEFAULT '{}'
+			);
+			CREATE TABLE package_stats (
+				id INTEGER PRIMARY KEY CHECK (id = 1),
+				active_plugins INTEGER NOT NULL DEFAULT 0,
+				active_themes INTEGER NOT NULL DEFAULT 0,
+				plugin_installs INTEGER NOT NULL DEFAULT 0,
+				theme_installs INTEGER NOT NULL DEFAULT 0,
+				installs_30d INTEGER NOT NULL DEFAULT 0,
+				updated_at TEXT NOT NULL DEFAULT ''
+			);
+		`)
 	if err != nil {
 		t.Fatalf("creating tables: %v", err)
 	}
@@ -172,6 +181,89 @@ func TestDeactivatePackage(t *testing.T) {
 	}
 }
 
+func TestRefreshSiteStats(t *testing.T) {
+	database := setupTestDB(t)
+	ctx := context.Background()
+
+	cur := "1.0.0"
+	p1 := &Package{
+		Type:                    "plugin",
+		Name:                    "plugin-one",
+		VersionsJSON:            "{}",
+		CurrentVersion:          &cur,
+		IsActive:                true,
+		WpComposerInstallsTotal: 100,
+		WpComposerInstalls30d:   25,
+	}
+	p2 := &Package{
+		Type:                    "plugin",
+		Name:                    "plugin-two",
+		VersionsJSON:            "{}",
+		CurrentVersion:          &cur,
+		IsActive:                false,
+		WpComposerInstallsTotal: 999,
+		WpComposerInstalls30d:   999,
+	}
+	t1 := &Package{
+		Type:                    "theme",
+		Name:                    "theme-one",
+		VersionsJSON:            "{}",
+		CurrentVersion:          &cur,
+		IsActive:                true,
+		WpComposerInstallsTotal: 50,
+		WpComposerInstalls30d:   5,
+	}
+
+	if err := UpsertPackage(ctx, database, p1); err != nil {
+		t.Fatalf("upserting plugin-one: %v", err)
+	}
+	if err := UpsertPackage(ctx, database, p2); err != nil {
+		t.Fatalf("upserting plugin-two: %v", err)
+	}
+	if err := UpsertPackage(ctx, database, t1); err != nil {
+		t.Fatalf("upserting theme-one: %v", err)
+	}
+
+	// Install counters are maintained by telemetry aggregation, not package upserts.
+	if _, err := database.Exec(`UPDATE packages SET wp_composer_installs_total = 100, wp_composer_installs_30d = 25 WHERE name = 'plugin-one'`); err != nil {
+		t.Fatalf("updating plugin-one counters: %v", err)
+	}
+	if _, err := database.Exec(`UPDATE packages SET wp_composer_installs_total = 999, wp_composer_installs_30d = 999 WHERE name = 'plugin-two'`); err != nil {
+		t.Fatalf("updating plugin-two counters: %v", err)
+	}
+	if _, err := database.Exec(`UPDATE packages SET wp_composer_installs_total = 50, wp_composer_installs_30d = 5 WHERE name = 'theme-one'`); err != nil {
+		t.Fatalf("updating theme-one counters: %v", err)
+	}
+
+	if err := RefreshSiteStats(ctx, database); err != nil {
+		t.Fatalf("refreshing site stats: %v", err)
+	}
+
+	var activePlugins, activeThemes, pluginInstalls, themeInstalls, installs30d int
+	err := database.QueryRow(`SELECT active_plugins, active_themes, plugin_installs, theme_installs, installs_30d FROM package_stats WHERE id = 1`).Scan(
+		&activePlugins, &activeThemes, &pluginInstalls, &themeInstalls, &installs30d,
+	)
+	if err != nil {
+		t.Fatalf("querying package_stats: %v", err)
+	}
+
+	if activePlugins != 1 {
+		t.Errorf("active_plugins = %d, want 1", activePlugins)
+	}
+	if activeThemes != 1 {
+		t.Errorf("active_themes = %d, want 1", activeThemes)
+	}
+	if pluginInstalls != 100 {
+		t.Errorf("plugin_installs = %d, want 100", pluginInstalls)
+	}
+	if themeInstalls != 50 {
+		t.Errorf("theme_installs = %d, want 50", themeInstalls)
+	}
+	if installs30d != 30 {
+		t.Errorf("installs_30d = %d, want 30", installs30d)
+	}
+}
+
 func TestGetPackagesNeedingUpdate(t *testing.T) {
 	database := setupTestDB(t)
 	ctx := context.Background()

internal/telemetry/aggregate.go

@@ -5,6 +5,8 @@ import (
 	"database/sql"
 	"fmt"
 	"time"
+
+	"github.com/roots/wp-composer/internal/packages"
 )
 
 // AggregateInstalls recomputes wp_composer_installs_total, wp_composer_installs_30d,
@@ -68,6 +70,10 @@ func AggregateInstalls(ctx context.Context, db *sql.DB) (AggregateResult, error)
 	}
 	resetCount, _ := resetResult.RowsAffected()
 
+	if err := packages.RefreshSiteStats(ctx, db); err != nil {
+		return AggregateResult{}, err
+	}
+
 	return AggregateResult{
 		PackagesUpdated: totalUpdated,
 		PackagesReset:   resetCount,

internal/telemetry/telemetry_test.go

@@ -36,6 +36,16 @@ func setupTestDB(t *testing.T) *sql.DB {
 			created_at TEXT NOT NULL, updated_at TEXT NOT NULL,
 			UNIQUE(type, name)
 		);
+		CREATE TABLE package_stats (
+			id INTEGER PRIMARY KEY CHECK (id = 1),
+			active_plugins INTEGER NOT NULL DEFAULT 0,
+			active_themes INTEGER NOT NULL DEFAULT 0,
+			plugin_installs INTEGER NOT NULL DEFAULT 0,
+			theme_installs INTEGER NOT NULL DEFAULT 0,
+			installs_30d INTEGER NOT NULL DEFAULT 0,
+			updated_at TEXT NOT NULL DEFAULT ''
+		);
+		INSERT INTO package_stats (id) VALUES (1);
 		CREATE TABLE install_events (
 			id INTEGER PRIMARY KEY,
 			package_id INTEGER NOT NULL REFERENCES packages(id) ON DELETE CASCADE,

migrations/010_create_package_stats.sql

@@ -0,0 +1,24 @@
+-- +goose Up
+CREATE TABLE package_stats (
+    id INTEGER PRIMARY KEY CHECK (id = 1),
+    active_plugins INTEGER NOT NULL DEFAULT 0,
+    active_themes INTEGER NOT NULL DEFAULT 0,
+    plugin_installs INTEGER NOT NULL DEFAULT 0,
+    theme_installs INTEGER NOT NULL DEFAULT 0,
+    installs_30d INTEGER NOT NULL DEFAULT 0,
+    updated_at TEXT NOT NULL
+);
+
+INSERT INTO package_stats (id, active_plugins, active_themes, plugin_installs, theme_installs, installs_30d, updated_at)
+    SELECT 1,
+        COALESCE(SUM(CASE WHEN type = 'plugin' THEN 1 ELSE 0 END), 0),
+        COALESCE(SUM(CASE WHEN type = 'theme' THEN 1 ELSE 0 END), 0),
+        COALESCE(SUM(CASE WHEN type = 'plugin' THEN wp_composer_installs_total ELSE 0 END), 0),
+        COALESCE(SUM(CASE WHEN type = 'theme' THEN wp_composer_installs_total ELSE 0 END), 0),
+        COALESCE(SUM(wp_composer_installs_30d), 0),
+        datetime('now')
+    FROM packages
+    WHERE is_active = 1;
+
+-- +goose Down
+DROP TABLE package_stats;