From fd75a4f79869420f20cc2dfc3e30b9eed39ffcab Mon Sep 17 00:00:00 2001 From: Randall Hand Date: Sat, 21 Mar 2026 23:39:19 -0400 Subject: [PATCH 1/5] docs: add Drizzle JOIN refactor Phase 1 design spec Co-Authored-By: Claude Opus 4.6 (1M context) --- ...2026-03-21-drizzle-join-refactor-design.md | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-21-drizzle-join-refactor-design.md diff --git a/docs/superpowers/specs/2026-03-21-drizzle-join-refactor-design.md b/docs/superpowers/specs/2026-03-21-drizzle-join-refactor-design.md new file mode 100644 index 000000000..4c39d92f0 --- /dev/null +++ b/docs/superpowers/specs/2026-03-21-drizzle-join-refactor-design.md @@ -0,0 +1,82 @@ +# Drizzle JOIN Refactor (Phase 1) — Design Spec + +**Date:** 2026-03-21 +**Status:** Approved + +## Overview + +Replace 4 raw SQL queries in `src/db/repositories/misc.ts` that branch on database type (PostgreSQL vs SQLite/MySQL) for column quoting with unified Drizzle ORM query builder calls. This eliminates ~80 lines of duplicated SQL and 4 `isPostgres()` conditional blocks. + +## Scope + +Only queries where the branching is purely about column name quoting. Excludes queries with genuine SQL syntax differences (DISTINCT ON, DELETE RETURNING, etc.) — those are future phases. + +## Queries to Refactor + +### 1. `getPacketLogs` (lines ~965-984) +- `packet_log` LEFT JOIN `nodes` twice (as from_nodes, to_nodes) +- Selects `longName` from each joined node +- Has WHERE clause, ORDER BY timestamp DESC + created_at DESC, LIMIT/OFFSET + +### 2. `getPacketLogById` (lines ~999-1015) +- Same double LEFT JOIN as above +- Filtered by `pl.id = ${id}` + +### 3. `getPacketCountsByNode` (lines ~1175-1195) +- `packet_log` LEFT JOIN `nodes` once +- GROUP BY + COUNT(*) + ORDER BY count DESC +- Postgres uses `COUNT(*)::int` cast (unnecessary with Drizzle) + +### 4. `getDistinctRelayNodes` (lines ~1100-1104) +- Simple SELECT from nodes with bitwise WHERE +- Not a JOIN, just quoting difference on column names + +## Design + +### Approach +Use Drizzle's `alias()` function for double-joining the nodes table, and the standard `.select().from().leftJoin()` builder for all queries. Drizzle handles column quoting per-backend automatically. + +### Key Drizzle patterns needed + +**Table alias for double-join:** +```typescript +import { alias } from 'drizzle-orm/sqlite-core'; // or pg-core, mysql-core +// Use the active schema's nodes table +const fromNodes = alias(this.tables.nodes, 'from_nodes'); +const toNodes = alias(this.tables.nodes, 'to_nodes'); +``` + +Note: `alias()` is backend-specific in Drizzle. Since we have three backends, we need to use the correct `alias` import or find a backend-agnostic approach. The `sql` tagged template with `this.tables.nodes` column references may be simpler — Drizzle's `sql` helper auto-quotes column references from table schemas. + +**Unified column references:** +```typescript +const { packetLog, nodes } = this.tables; +// Drizzle auto-quotes: nodes.longName → "longName" (PG) or longName (SQLite) +``` + +**COUNT without cast:** +```typescript +sql`COUNT(*)` // Works across all backends +``` + +### What changes +- Remove 4 `if (this.isPostgres()) { ... } else { ... }` blocks +- Replace with single Drizzle query builder call per method +- No changes to method signatures or return types +- `normalizePacketLogRow()` continues to handle result normalization + +### Files Modified + +| File | Change | +|------|--------| +| `src/db/repositories/misc.ts` | Replace 4 branched queries with Drizzle query builder | + +## Testing + +- Existing tests cover these methods — they must continue to pass +- Run full test suite (3052+ tests) +- Build verification (TypeScript clean) + +## Risk + +Low — method signatures and return types don't change. The Drizzle query builder generates the same SQL that's currently hardcoded, just with correct quoting per backend. If any query doesn't translate cleanly to the Drizzle builder, we keep the raw SQL but use Drizzle column references for auto-quoting instead of full builder conversion. From 4432b67e55b7026cb708626085dbe19f43398b77 Mon Sep 17 00:00:00 2001 From: Randall Hand Date: Sun, 22 Mar 2026 08:21:38 -0400 Subject: [PATCH 2/5] refactor: replace raw SQL duplication with col() helper for cross-DB quoting Adds BaseRepository.col(name) helper that returns correctly quoted column names (PostgreSQL: "camelCase", SQLite/MySQL: bare identifiers). Replaces 4 isPostgres() branched queries in misc.ts with single unified queries: - getPacketLogs: packet_log LEFT JOIN nodes (x2) - getPacketLogById: same double JOIN - getPacketCountsByNode: packet_log LEFT JOIN nodes with GROUP BY - getDistinctRelayNodes: SELECT from nodes with bitwise WHERE Removes ~40 lines of duplicated SQL. Adds 9 new tests verifying JOIN queries return correct node names, handle unknown nodes, respect pagination, and maintain sort order. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/db/repositories/base.ts | 10 ++ src/db/repositories/misc.packetlog.test.ts | 173 +++++++++++++++++++++ src/db/repositories/misc.ts | 104 +++++-------- 3 files changed, 222 insertions(+), 65 deletions(-) create mode 100644 src/db/repositories/misc.packetlog.test.ts diff --git a/src/db/repositories/base.ts b/src/db/repositories/base.ts index 4c9e54d29..d43712097 100644 --- a/src/db/repositories/base.ts +++ b/src/db/repositories/base.ts @@ -4,6 +4,7 @@ * Provides common functionality for all repository implementations. * Supports SQLite, PostgreSQL, and MySQL through Drizzle ORM. */ +import { sql } from 'drizzle-orm'; import { BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'; import { NodePgDatabase } from 'drizzle-orm/node-postgres'; import { MySql2Database } from 'drizzle-orm/mysql2'; @@ -117,6 +118,15 @@ export abstract class BaseRepository { return this.mysqlDb; } + /** + * Quote a column name for use in raw SQL. + * PostgreSQL requires double-quoted "camelCase" identifiers; SQLite/MySQL do not. + * Returns a raw SQL fragment that can be interpolated into sql`` templates. + */ + protected col(name: string) { + return this.isPostgres() ? sql.raw(`"${name}"`) : sql.raw(name); + } + /** * Execute a raw SQL query that returns rows (SELECT) across all dialects. * SQLite's Drizzle driver doesn't have .execute() — uses .all() instead. diff --git a/src/db/repositories/misc.packetlog.test.ts b/src/db/repositories/misc.packetlog.test.ts new file mode 100644 index 000000000..05f597d15 --- /dev/null +++ b/src/db/repositories/misc.packetlog.test.ts @@ -0,0 +1,173 @@ +/** + * Misc Repository - Packet Log Query Tests + * + * Tests the refactored Drizzle JOIN queries for packet log methods. + * Verifies that column references are correctly quoted across database backends. + */ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import Database from 'better-sqlite3'; +import { drizzle, BetterSQLite3Database } from 'drizzle-orm/better-sqlite3'; +import { MiscRepository } from './misc.js'; +import * as schema from '../schema/index.js'; + +describe('MiscRepository - Packet Log Queries', () => { + let db: Database.Database; + let drizzleDb: BetterSQLite3Database; + let repo: MiscRepository; + + beforeEach(() => { + db = new Database(':memory:'); + + // Create tables needed for JOIN queries + db.exec(` + CREATE TABLE IF NOT EXISTS nodes ( + nodeNum INTEGER PRIMARY KEY, + nodeId TEXT, + longName TEXT, + shortName TEXT, + lastHeard INTEGER + ) + `); + + db.exec(` + CREATE TABLE IF NOT EXISTS packet_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + packet_id INTEGER, + timestamp INTEGER NOT NULL, + from_node INTEGER NOT NULL, + from_node_id TEXT, + to_node INTEGER, + to_node_id TEXT, + channel INTEGER, + portnum INTEGER NOT NULL, + portnum_name TEXT, + encrypted INTEGER DEFAULT 0, + snr REAL, + rssi INTEGER, + hop_limit INTEGER, + hop_start INTEGER, + relay_node INTEGER, + payload_size INTEGER, + want_ack INTEGER DEFAULT 0, + priority INTEGER, + payload_preview TEXT, + metadata TEXT, + direction TEXT DEFAULT 'rx', + created_at INTEGER, + transport_mechanism TEXT, + decrypted_by TEXT, + decrypted_channel_id INTEGER + ) + `); + + // Create settings table (needed by MiscRepository) + db.exec(` + CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT + ) + `); + + drizzleDb = drizzle(db, { schema }); + repo = new MiscRepository(drizzleDb as any, 'sqlite'); + + // Insert test nodes + db.exec(`INSERT INTO nodes (nodeNum, nodeId, longName, shortName) VALUES (100, '!00000064', 'Node Alpha', 'ALPH')`); + db.exec(`INSERT INTO nodes (nodeNum, nodeId, longName, shortName) VALUES (200, '!000000c8', 'Node Beta', 'BETA')`); + db.exec(`INSERT INTO nodes (nodeNum, nodeId, longName, shortName) VALUES (300, '!0000012c', 'Node Gamma', 'GAMM')`); + + // Insert test packets + const now = Math.floor(Date.now() / 1000); + const nowMs = Date.now(); + db.exec(`INSERT INTO packet_log (packet_id, timestamp, from_node, from_node_id, to_node, to_node_id, portnum, portnum_name, direction, created_at, relay_node) VALUES (1, ${now}, 100, '!00000064', 200, '!000000c8', 1, 'TEXT_MESSAGE_APP', 'rx', ${nowMs}, 100)`); + db.exec(`INSERT INTO packet_log (packet_id, timestamp, from_node, from_node_id, to_node, to_node_id, portnum, portnum_name, direction, created_at, relay_node) VALUES (2, ${now}, 200, '!000000c8', 100, '!00000064', 1, 'TEXT_MESSAGE_APP', 'rx', ${nowMs + 1}, 200)`); + db.exec(`INSERT INTO packet_log (packet_id, timestamp, from_node, from_node_id, to_node, to_node_id, portnum, portnum_name, direction, created_at) VALUES (3, ${now - 60}, 100, '!00000064', 4294967295, '!ffffffff', 3, 'POSITION_APP', 'rx', ${nowMs - 60000})`); + }); + + afterEach(() => { + db.close(); + }); + + describe('getPacketLogs', () => { + it('returns packets with joined node names', async () => { + const packets = await repo.getPacketLogs({}); + expect(packets.length).toBe(3); + + // Check that longName was joined from nodes table + const pkt1 = packets.find(p => p.packet_id === 1); + expect(pkt1).toBeDefined(); + expect(pkt1!.from_node_longName).toBe('Node Alpha'); + expect(pkt1!.to_node_longName).toBe('Node Beta'); + }); + + it('returns null longName for unknown nodes', async () => { + // Insert packet from unknown node + const now = Math.floor(Date.now() / 1000); + db.exec(`INSERT INTO packet_log (packet_id, timestamp, from_node, from_node_id, to_node, portnum, direction, created_at) VALUES (99, ${now}, 999, '!000003e7', NULL, 1, 'rx', ${Date.now()})`); + + const packets = await repo.getPacketLogs({}); + const unknownPkt = packets.find(p => p.packet_id === 99); + expect(unknownPkt).toBeDefined(); + expect(unknownPkt!.from_node_longName).toBeNull(); + }); + + it('respects limit and offset', async () => { + const packets = await repo.getPacketLogs({ limit: 2, offset: 0 }); + expect(packets.length).toBe(2); + }); + + it('orders by timestamp DESC then created_at DESC', async () => { + const packets = await repo.getPacketLogs({}); + // First two packets have same timestamp, ordered by created_at DESC + expect(packets[0].packet_id).toBe(2); // higher created_at + expect(packets[1].packet_id).toBe(1); + expect(packets[2].packet_id).toBe(3); // older timestamp + }); + }); + + describe('getPacketLogById', () => { + it('returns a single packet with joined node names', async () => { + const packets = await repo.getPacketLogs({}); + const firstId = packets[0].id; + + const pkt = await repo.getPacketLogById(firstId!); + expect(pkt).not.toBeNull(); + expect(pkt!.from_node_longName).toBeDefined(); + }); + + it('returns null for non-existent id', async () => { + const pkt = await repo.getPacketLogById(99999); + expect(pkt).toBeNull(); + }); + }); + + describe('getPacketCountsByNode', () => { + it('returns counts with joined node names', async () => { + const counts = await repo.getPacketCountsByNode({}); + expect(counts.length).toBeGreaterThan(0); + + const alpha = counts.find(c => c.from_node === 100); + expect(alpha).toBeDefined(); + expect(alpha!.from_node_longName).toBe('Node Alpha'); + expect(alpha!.count).toBe(2); // packets 1 and 3 + }); + + it('respects limit', async () => { + const counts = await repo.getPacketCountsByNode({ limit: 1 }); + expect(counts.length).toBe(1); + }); + }); + + describe('getDistinctRelayNodes', () => { + it('returns relay nodes with matched node names', async () => { + const relays = await repo.getDistinctRelayNodes(); + expect(relays.length).toBeGreaterThan(0); + + // relay_node 100 & 0xFF = 100, matches node 100 (Node Alpha) + const relay100 = relays.find(r => r.relay_node === 100); + expect(relay100).toBeDefined(); + expect(relay100!.matching_nodes.length).toBeGreaterThan(0); + expect(relay100!.matching_nodes[0].longName).toBe('Node Alpha'); + }); + }); +}); diff --git a/src/db/repositories/misc.ts b/src/db/repositories/misc.ts index 7b906be02..e01913aff 100644 --- a/src/db/repositories/misc.ts +++ b/src/db/repositories/misc.ts @@ -961,27 +961,17 @@ export class MiscRepository extends BaseRepository { const whereClause = this.combineConditions(conditions); try { - // Use raw SQL for the JOIN query - let joinQuery; - if (this.isPostgres()) { - joinQuery = sql` - SELECT pl.*, from_nodes."longName" as "from_node_longName", to_nodes."longName" as "to_node_longName" - FROM packet_log pl - LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes."nodeNum" - LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes."nodeNum" - WHERE ${whereClause} - ORDER BY pl.timestamp DESC, pl.created_at DESC LIMIT ${limit} OFFSET ${offset} - `; - } else { - joinQuery = sql` - SELECT pl.*, from_nodes.longName as from_node_longName, to_nodes.longName as to_node_longName - FROM packet_log pl - LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes.nodeNum - LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes.nodeNum - WHERE ${whereClause} - ORDER BY pl.timestamp DESC, pl.created_at DESC LIMIT ${limit} OFFSET ${offset} - `; - } + const longName = this.col('longName'); + const nodeNum = this.col('nodeNum'); + + const joinQuery = sql` + SELECT pl.*, from_nodes.${longName} as from_node_longName, to_nodes.${longName} as to_node_longName + FROM packet_log pl + LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes.${nodeNum} + LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes.${nodeNum} + WHERE ${whereClause} + ORDER BY pl.timestamp DESC, pl.created_at DESC LIMIT ${limit} OFFSET ${offset} + `; const rows = await this.executeQuery(joinQuery); return (rows as any[]).map((row: any) => this.normalizePacketLogRow(row)); @@ -996,24 +986,16 @@ export class MiscRepository extends BaseRepository { */ async getPacketLogById(id: number): Promise { try { - let joinQuery; - if (this.isPostgres()) { - joinQuery = sql` - SELECT pl.*, from_nodes."longName" as "from_node_longName", to_nodes."longName" as "to_node_longName" - FROM packet_log pl - LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes."nodeNum" - LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes."nodeNum" - WHERE pl.id = ${id} - `; - } else { - joinQuery = sql` - SELECT pl.*, from_nodes.longName as from_node_longName, to_nodes.longName as to_node_longName - FROM packet_log pl - LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes.nodeNum - LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes.nodeNum - WHERE pl.id = ${id} - `; - } + const longName = this.col('longName'); + const nodeNum = this.col('nodeNum'); + + const joinQuery = sql` + SELECT pl.*, from_nodes.${longName} as from_node_longName, to_nodes.${longName} as to_node_longName + FROM packet_log pl + LEFT JOIN nodes from_nodes ON pl.from_node = from_nodes.${nodeNum} + LEFT JOIN nodes to_nodes ON pl.to_node = to_nodes.${nodeNum} + WHERE pl.id = ${id} + `; const rows = await this.executeQuery(joinQuery); if (!rows || rows.length === 0) return null; @@ -1099,9 +1081,9 @@ export class MiscRepository extends BaseRepository { */ async getDistinctRelayNodes(): Promise { const distinctQuery = 'SELECT DISTINCT relay_node FROM packet_log WHERE relay_node IS NOT NULL'; - const matchQuery = this.isPostgres() - ? 'SELECT "longName", "shortName" FROM nodes WHERE ("nodeNum" & 255) = ' - : 'SELECT longName, shortName FROM nodes WHERE (nodeNum & 255) = '; + const longName = this.col('longName'); + const shortName = this.col('shortName'); + const nodeNum = this.col('nodeNum'); try { const distinctRows = await this.executeQuery(sql.raw(distinctQuery)); @@ -1109,7 +1091,9 @@ export class MiscRepository extends BaseRepository { const results: DbDistinctRelayNode[] = []; for (const rv of relayValues) { - const matchRows = await this.executeQuery(sql.raw(`${matchQuery}${rv}`)); + const matchRows = await this.executeQuery( + sql`SELECT ${longName}, ${shortName} FROM nodes WHERE (${nodeNum} & 255) = ${rv}` + ); results.push({ relay_node: rv, matching_nodes: (matchRows as any[]).map((r: any) => ({ @@ -1172,28 +1156,18 @@ export class MiscRepository extends BaseRepository { if (portnum !== undefined) conditions.push(sql`pl.portnum = ${portnum}`); const whereClause = conditions.length > 0 ? this.combineConditions(conditions) : sql`1=1`; - let query; - if (this.isPostgres()) { - query = sql` - SELECT pl.from_node, pl.from_node_id, n."longName" as "from_node_longName", COUNT(*)::int as count - FROM packet_log pl - LEFT JOIN nodes n ON pl.from_node = n."nodeNum" - WHERE ${whereClause} - GROUP BY pl.from_node, pl.from_node_id, n."longName" - ORDER BY count DESC - LIMIT ${limit} - `; - } else { - query = sql` - SELECT pl.from_node, pl.from_node_id, n.longName as from_node_longName, COUNT(*) as count - FROM packet_log pl - LEFT JOIN nodes n ON pl.from_node = n.nodeNum - WHERE ${whereClause} - GROUP BY pl.from_node, pl.from_node_id, n.longName - ORDER BY count DESC - LIMIT ${limit} - `; - } + const longName = this.col('longName'); + const nodeNum = this.col('nodeNum'); + + const query = sql` + SELECT pl.from_node, pl.from_node_id, n.${longName} as from_node_longName, COUNT(*) as count + FROM packet_log pl + LEFT JOIN nodes n ON pl.from_node = n.${nodeNum} + WHERE ${whereClause} + GROUP BY pl.from_node, pl.from_node_id, n.${longName} + ORDER BY count DESC + LIMIT ${limit} + `; const rows = await this.executeQuery(query); return (rows as any[]).map((row: any) => ({ From 8754d72a50369186be72942cd1c2da36fba990b0 Mon Sep 17 00:00:00 2001 From: Randall Hand Date: Sun, 22 Mar 2026 08:44:18 -0400 Subject: [PATCH 3/5] feat: add backend soak test for all three database backends Launches SQLite, PostgreSQL, and MySQL in sequence, monitors logs for errors during a configurable soak period (default 5 minutes each). Logs saved to tests/soak-logs/ for post-test review. Usage: tests/backend-soak-test.sh [duration_seconds] Co-Authored-By: Claude Opus 4.6 (1M context) --- .gitignore | 1 + tests/backend-soak-test.sh | 226 +++++++++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100755 tests/backend-soak-test.sh diff --git a/.gitignore b/.gitignore index e502f960f..aac0316e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Logs logs *.log +tests/soak-logs/ npm-debug.log* yarn-debug.log* yarn-error.log* diff --git a/tests/backend-soak-test.sh b/tests/backend-soak-test.sh new file mode 100755 index 000000000..7ca9928bd --- /dev/null +++ b/tests/backend-soak-test.sh @@ -0,0 +1,226 @@ +#!/bin/bash +# Backend Soak Test +# +# Launches each database backend (SQLite, PostgreSQL, MySQL) in sequence, +# runs for a configurable duration while monitoring logs for errors, +# and fails if any error messages are found. +# +# Usage: tests/backend-soak-test.sh [duration_seconds] +# duration_seconds: How long to soak each backend (default: 300 = 5 minutes) + +set -e + +# Configuration +SOAK_DURATION=${1:-300} +COMPOSE_FILE="docker-compose.dev.yml" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Backend definitions: profile, app container name +declare -A BACKENDS +BACKENDS[sqlite]="meshmonitor-sqlite" +BACKENDS[postgres]="meshmonitor" +BACKENDS[mysql]="meshmonitor-mysql-app" + +# Error patterns to watch for in logs +# Excludes expected/informational messages that contain "error" in non-error contexts +ERROR_PATTERNS='(\[ERROR\]|FATAL|ECONNREFUSED|SQLITE_ERROR|SqliteError|uncaughtException|unhandledRejection)' +# Patterns to exclude from error matching (false positives) +EXCLUDE_PATTERNS='(error_correction|error\.tsx|error\.ts|RoutingError|errorCount|clearError|getPortNumName|error-boundary|isError|onError|handleError|LogLevel\.ERROR|errorDetails|_error|\.error\b.*=|error_event|Error fetching or storing news)' + +# Log output directory +LOG_DIR="$PROJECT_DIR/tests/soak-logs" +mkdir -p "$LOG_DIR" + +# Track results +TOTAL_PASS=0 +TOTAL_FAIL=0 +RESULTS=() + +cd "$PROJECT_DIR" + +echo "==========================================" +echo "Backend Soak Test" +echo "==========================================" +echo "Duration per backend: ${SOAK_DURATION}s" +echo "Backends: sqlite, postgres, mysql" +echo "" + +# Cleanup function +cleanup() { + echo -e "${BLUE}Cleaning up...${NC}" + for profile in sqlite postgres mysql; do + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true + done + echo -e "${GREEN}✓${NC} Cleanup complete" +} + +trap cleanup EXIT + +# Stop any running dev containers first +echo -e "${BLUE}Stopping any running dev containers...${NC}" +for profile in sqlite postgres mysql; do + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true +done +echo -e "${GREEN}✓${NC} Clean slate" +echo "" + +# Wait for container to be healthy/running +wait_for_container() { + local container=$1 + local max_wait=120 + local elapsed=0 + + echo -n " Waiting for $container to be ready" + while [ $elapsed -lt $max_wait ]; do + local status + status=$(docker inspect --format='{{.State.Status}}' "$container" 2>/dev/null || echo "missing") + + if [ "$status" = "running" ]; then + # Check if the app has started by looking for the listen message + if docker logs "$container" 2>&1 | grep -q "listening\|Server started\|ready\|Listening on"; then + echo -e " ${GREEN}✓${NC} (${elapsed}s)" + return 0 + fi + fi + + sleep 5 + elapsed=$((elapsed + 5)) + echo -n "." + done + + echo -e " ${RED}✗${NC} (timeout after ${max_wait}s)" + return 1 +} + +# Test a single backend +test_backend() { + local profile=$1 + local container=${BACKENDS[$profile]} + local timestamp + timestamp=$(date +%Y%m%d-%H%M%S) + local log_file="${LOG_DIR}/${profile}-${timestamp}.log" + + echo "==========================================" + echo -e "${BLUE}Testing: $profile${NC}" + echo "==========================================" + + # Build and start + echo " Building..." + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" build --quiet 2>&1 | tail -1 + echo " Starting containers..." + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" up -d 2>/dev/null + + # Wait for app container + if ! wait_for_container "$container"; then + echo -e " ${RED}✗ $profile: Container failed to start${NC}" + docker logs "$container" 2>&1 | tail -20 + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true + RESULTS+=("$profile: FAIL (container failed to start)") + TOTAL_FAIL=$((TOTAL_FAIL + 1)) + return 1 + fi + + # Soak: monitor logs for the configured duration + echo " Soaking for ${SOAK_DURATION}s (monitoring logs for errors)..." + local start_time + start_time=$(date +%s) + + # Clear log file + > "$log_file" + + # Collect logs during soak period + local remaining=$SOAK_DURATION + while [ $remaining -gt 0 ]; do + local chunk=$remaining + if [ $chunk -gt 30 ]; then + chunk=30 + fi + sleep "$chunk" + remaining=$((remaining - chunk)) + + local elapsed=$(( $(date +%s) - start_time )) + local pct=$(( elapsed * 100 / SOAK_DURATION )) + echo -ne "\r Progress: ${elapsed}s / ${SOAK_DURATION}s (${pct}%)" + done + echo "" + + # Capture full logs + docker logs "$container" > "$log_file" 2>&1 + + # Check for errors + local error_lines + error_lines=$(grep -E "$ERROR_PATTERNS" "$log_file" | grep -Ev "$EXCLUDE_PATTERNS" || true) + local error_count + error_count=$(echo "$error_lines" | grep -c . || true) + + if [ -n "$error_lines" ] && [ "$error_count" -gt 0 ]; then + echo -e " ${RED}✗ $profile: Found $error_count error(s) in logs${NC}" + echo "" + echo " Error lines:" + echo "$error_lines" | head -20 | while IFS= read -r line; do + echo -e " ${RED}$line${NC}" + done + if [ "$error_count" -gt 20 ]; then + echo " ... and $((error_count - 20)) more" + fi + echo "" + echo " Full logs saved to: $log_file" + RESULTS+=("$profile: FAIL ($error_count errors)") + TOTAL_FAIL=$((TOTAL_FAIL + 1)) + else + echo -e " ${GREEN}✓ $profile: No errors found in logs${NC}" + RESULTS+=("$profile: PASS") + TOTAL_PASS=$((TOTAL_PASS + 1)) + fi + + # Stop this backend + echo " Stopping $profile containers..." + COMPOSE_PROFILES="$profile" docker compose -f "$COMPOSE_FILE" down -v 2>/dev/null || true + echo "" +} + +# Run each backend +for profile in sqlite postgres mysql; do + test_backend "$profile" +done + +# Summary +echo "==========================================" +echo "Backend Soak Test Results" +echo "==========================================" +echo "" +for result in "${RESULTS[@]}"; do + local_profile=$(echo "$result" | cut -d: -f1) + local_status=$(echo "$result" | cut -d: -f2-) + if echo "$result" | grep -q "PASS"; then + echo -e " ${GREEN}✓${NC} $result" + else + echo -e " ${RED}✗${NC} $result" + fi +done +echo "" +echo "Passed: $TOTAL_PASS / $((TOTAL_PASS + TOTAL_FAIL))" +echo "" +echo "Logs saved to: $LOG_DIR/" +ls -lh "$LOG_DIR"/*.log 2>/dev/null | awk '{print " " $NF " (" $5 ")"}' +echo "" + +if [ $TOTAL_FAIL -gt 0 ]; then + echo -e "${RED}==========================================" + echo -e "✗ BACKEND SOAK TEST FAILED" + echo -e "==========================================${NC}" + exit 1 +else + echo -e "${GREEN}==========================================" + echo -e "✓ ALL BACKENDS PASSED" + echo -e "==========================================${NC}" + exit 0 +fi From 0a0550b17b87479f8a3896c33fcb28ea6f873263 Mon Sep 17 00:00:00 2001 From: Randall Hand Date: Sun, 22 Mar 2026 09:01:47 -0400 Subject: [PATCH 4/5] fix: add missing decrypted_by column to messages table for PG/MySQL The PostgreSQL and MySQL baseline migrations omitted the decrypted_by column from the messages CREATE TABLE, causing DrizzleQueryError on any SELECT from messages. SQLite baseline had it correctly. Found by the new backend soak test. Co-Authored-By: Claude Opus 4.6 (1M context) --- CLAUDE.md | 2 +- src/db/migrations.test.ts | 10 +-- src/db/migrations.ts | 17 ++++- .../014_add_messages_decrypted_by.ts | 76 +++++++++++++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) create mode 100644 src/server/migrations/014_add_messages_decrypted_by.ts diff --git a/CLAUDE.md b/CLAUDE.md index a1ebeab2d..bea637832 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,7 +108,7 @@ Migrations use a centralized registry in `src/db/migrations.ts`. Each migration 4. Make migrations **idempotent** — use try/catch for SQLite (`duplicate column`), `IF NOT EXISTS` for PostgreSQL, `information_schema` checks for MySQL 5. **Column naming**: SQLite uses `snake_case`, PostgreSQL/MySQL use `camelCase` (quoted in PG raw SQL) -**Current migration count:** 13 (latest: `013_add_audit_log_missing_columns`) +**Current migration count:** 14 (latest: `014_add_messages_decrypted_by`) ## Testing diff --git a/src/db/migrations.test.ts b/src/db/migrations.test.ts index 22d21913a..0eb19e7a0 100644 --- a/src/db/migrations.test.ts +++ b/src/db/migrations.test.ts @@ -2,8 +2,8 @@ import { describe, it, expect } from 'vitest'; import { registry } from './migrations.js'; describe('migrations registry', () => { - it('has all 13 migrations registered', () => { - expect(registry.count()).toBe(13); + it('has all 14 migrations registered', () => { + expect(registry.count()).toBe(14); }); it('first migration is v37 baseline', () => { @@ -12,11 +12,11 @@ describe('migrations registry', () => { expect(all[0].name).toContain('v37_baseline'); }); - it('last migration is the audit_log missing columns fix', () => { + it('last migration is the messages decrypted_by fix', () => { const all = registry.getAll(); const last = all[all.length - 1]; - expect(last.number).toBe(13); - expect(last.name).toContain('audit_log'); + expect(last.number).toBe(14); + expect(last.name).toContain('messages_decrypted_by'); }); it('migrations are sequentially numbered from 1 to 12', () => { diff --git a/src/db/migrations.ts b/src/db/migrations.ts index 5360785e0..f92abd617 100644 --- a/src/db/migrations.ts +++ b/src/db/migrations.ts @@ -1,7 +1,7 @@ /** * Migration Registry Barrel File * - * Registers all 13 migrations in sequential order for use by the migration runner. + * Registers all 14 migrations in sequential order for use by the migration runner. * Migration 001 is the v3.7 baseline (selfIdempotent — handles its own detection). * Migrations 002-011 were originally 078-087 and retain their original settingsKeys * for upgrade compatibility. @@ -25,6 +25,7 @@ import { runMigration086Sqlite, runMigration086Postgres, runMigration086Mysql } import { migration as fixMessageNodeNumBigintMigration, runMigration087Postgres, runMigration087Mysql } from '../server/migrations/011_fix_message_nodenum_bigint.js'; import { migration as authAlignMigration, runMigration012Postgres, runMigration012Mysql } from '../server/migrations/012_align_sqlite_auth_schema.js'; import { migration as auditLogColumnsMigration, runMigration013Postgres, runMigration013Mysql } from '../server/migrations/013_add_audit_log_missing_columns.js'; +import { migration as messagesDecryptedByMigration, runMigration014Postgres, runMigration014Mysql } from '../server/migrations/014_add_messages_decrypted_by.js'; // ============================================================================ // Registry @@ -169,3 +170,17 @@ registry.register({ postgres: (client) => runMigration013Postgres(client), mysql: (pool) => runMigration013Mysql(pool), }); + +// --------------------------------------------------------------------------- +// Migration 014: Add missing decrypted_by column to messages table +// PG/MySQL baselines omitted this column that the Drizzle schema expects. +// --------------------------------------------------------------------------- + +registry.register({ + number: 14, + name: 'add_messages_decrypted_by', + settingsKey: 'migration_014_add_messages_decrypted_by', + sqlite: (db) => messagesDecryptedByMigration.up(db), + postgres: (client) => runMigration014Postgres(client), + mysql: (pool) => runMigration014Mysql(pool), +}); diff --git a/src/server/migrations/014_add_messages_decrypted_by.ts b/src/server/migrations/014_add_messages_decrypted_by.ts new file mode 100644 index 000000000..94949102d --- /dev/null +++ b/src/server/migrations/014_add_messages_decrypted_by.ts @@ -0,0 +1,76 @@ +/** + * Migration 014: Add missing decrypted_by column to messages table for PostgreSQL/MySQL + * + * The Drizzle schema defines decryptedBy on messages for all backends, and the + * SQLite baseline includes it, but the PostgreSQL and MySQL baselines omit it. + * This causes DrizzleQueryError on any SELECT from the messages table. + * + * SQLite already has the column, so the SQLite migration is a safe no-op. + */ +import type { Database } from 'better-sqlite3'; +import { logger } from '../../utils/logger.js'; + +// ============ SQLite ============ + +export const migration = { + up: (db: Database): void => { + logger.info('Running migration 014 (SQLite): Ensuring decrypted_by on messages...'); + + try { + db.exec('ALTER TABLE messages ADD COLUMN decrypted_by TEXT'); + logger.debug('Added decrypted_by column to messages'); + } catch (e: any) { + if (e.message?.includes('duplicate column')) { + logger.debug('messages.decrypted_by already exists, skipping'); + } else { + logger.warn('Could not add decrypted_by to messages:', e.message); + } + } + + logger.info('Migration 014 complete (SQLite)'); + }, + + down: (_db: Database): void => { + logger.debug('Migration 014 down: Not implemented'); + } +}; + +// ============ PostgreSQL ============ + +export async function runMigration014Postgres(client: import('pg').PoolClient): Promise { + logger.info('Running migration 014 (PostgreSQL): Adding decrypted_by to messages...'); + + try { + await client.query('ALTER TABLE messages ADD COLUMN IF NOT EXISTS decrypted_by TEXT'); + logger.debug('Ensured decrypted_by exists on messages'); + } catch (error: any) { + logger.error('Migration 014 (PostgreSQL) failed:', error.message); + throw error; + } + + logger.info('Migration 014 complete (PostgreSQL)'); +} + +// ============ MySQL ============ + +export async function runMigration014Mysql(pool: import('mysql2/promise').Pool): Promise { + logger.info('Running migration 014 (MySQL): Adding decrypted_by to messages...'); + + try { + const [rows] = await pool.query(` + SELECT COLUMN_NAME FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'messages' AND COLUMN_NAME = 'decrypted_by' + `); + if (!Array.isArray(rows) || rows.length === 0) { + await pool.query('ALTER TABLE messages ADD COLUMN decrypted_by VARCHAR(16)'); + logger.debug('Added decrypted_by to messages'); + } else { + logger.debug('messages.decrypted_by already exists, skipping'); + } + } catch (error: any) { + logger.error('Migration 014 (MySQL) failed:', error.message); + throw error; + } + + logger.info('Migration 014 complete (MySQL)'); +} From 71d881f8661707649c3a27eae526dc5a15456232 Mon Sep 17 00:00:00 2001 From: Randall Hand Date: Sun, 22 Mar 2026 09:20:05 -0400 Subject: [PATCH 5/5] fix: add missing channel_database columns for PG/MySQL baselines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migration 014 now also adds enforceNameValidation and sortOrder to channel_database for PostgreSQL and MySQL — both were omitted from the v3.7 baseline. Also improved soak test exclude patterns for expected ECONNREFUSED errors in dev environment. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../014_add_messages_decrypted_by.ts | 45 ++++++++++++++++--- tests/backend-soak-test.sh | 2 +- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/server/migrations/014_add_messages_decrypted_by.ts b/src/server/migrations/014_add_messages_decrypted_by.ts index 94949102d..8d4e1f989 100644 --- a/src/server/migrations/014_add_messages_decrypted_by.ts +++ b/src/server/migrations/014_add_messages_decrypted_by.ts @@ -1,11 +1,11 @@ /** - * Migration 014: Add missing decrypted_by column to messages table for PostgreSQL/MySQL + * Migration 014: Fix missing columns in PG/MySQL baselines * - * The Drizzle schema defines decryptedBy on messages for all backends, and the - * SQLite baseline includes it, but the PostgreSQL and MySQL baselines omit it. - * This causes DrizzleQueryError on any SELECT from the messages table. + * 1. messages.decrypted_by — PG/MySQL baselines omitted this column that the Drizzle schema expects. + * 2. channel_database.enforceNameValidation — PG/MySQL baselines omitted this column. + * 3. channel_database.sortOrder — PG/MySQL baselines omitted this column. * - * SQLite already has the column, so the SQLite migration is a safe no-op. + * SQLite already has all columns, so the SQLite migration is a safe no-op. */ import type { Database } from 'better-sqlite3'; import { logger } from '../../utils/logger.js'; @@ -43,6 +43,12 @@ export async function runMigration014Postgres(client: import('pg').PoolClient): try { await client.query('ALTER TABLE messages ADD COLUMN IF NOT EXISTS decrypted_by TEXT'); logger.debug('Ensured decrypted_by exists on messages'); + + await client.query('ALTER TABLE channel_database ADD COLUMN IF NOT EXISTS "enforceNameValidation" BOOLEAN NOT NULL DEFAULT false'); + logger.debug('Ensured enforceNameValidation exists on channel_database'); + + await client.query('ALTER TABLE channel_database ADD COLUMN IF NOT EXISTS "sortOrder" INTEGER NOT NULL DEFAULT 0'); + logger.debug('Ensured sortOrder exists on channel_database'); } catch (error: any) { logger.error('Migration 014 (PostgreSQL) failed:', error.message); throw error; @@ -57,16 +63,41 @@ export async function runMigration014Mysql(pool: import('mysql2/promise').Pool): logger.info('Running migration 014 (MySQL): Adding decrypted_by to messages...'); try { - const [rows] = await pool.query(` + // 1. messages.decrypted_by + const [msgRows] = await pool.query(` SELECT COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'messages' AND COLUMN_NAME = 'decrypted_by' `); - if (!Array.isArray(rows) || rows.length === 0) { + if (!Array.isArray(msgRows) || msgRows.length === 0) { await pool.query('ALTER TABLE messages ADD COLUMN decrypted_by VARCHAR(16)'); logger.debug('Added decrypted_by to messages'); } else { logger.debug('messages.decrypted_by already exists, skipping'); } + + // 2. channel_database.enforceNameValidation + const [envRows] = await pool.query(` + SELECT COLUMN_NAME FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'channel_database' AND COLUMN_NAME = 'enforceNameValidation' + `); + if (!Array.isArray(envRows) || envRows.length === 0) { + await pool.query('ALTER TABLE channel_database ADD COLUMN enforceNameValidation BOOLEAN NOT NULL DEFAULT false'); + logger.debug('Added enforceNameValidation to channel_database'); + } else { + logger.debug('channel_database.enforceNameValidation already exists, skipping'); + } + + // 3. channel_database.sortOrder + const [soRows] = await pool.query(` + SELECT COLUMN_NAME FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'channel_database' AND COLUMN_NAME = 'sortOrder' + `); + if (!Array.isArray(soRows) || soRows.length === 0) { + await pool.query('ALTER TABLE channel_database ADD COLUMN sortOrder INT NOT NULL DEFAULT 0'); + logger.debug('Added sortOrder to channel_database'); + } else { + logger.debug('channel_database.sortOrder already exists, skipping'); + } } catch (error: any) { logger.error('Migration 014 (MySQL) failed:', error.message); throw error; diff --git a/tests/backend-soak-test.sh b/tests/backend-soak-test.sh index 7ca9928bd..478677131 100755 --- a/tests/backend-soak-test.sh +++ b/tests/backend-soak-test.sh @@ -33,7 +33,7 @@ BACKENDS[mysql]="meshmonitor-mysql-app" # Excludes expected/informational messages that contain "error" in non-error contexts ERROR_PATTERNS='(\[ERROR\]|FATAL|ECONNREFUSED|SQLITE_ERROR|SqliteError|uncaughtException|unhandledRejection)' # Patterns to exclude from error matching (false positives) -EXCLUDE_PATTERNS='(error_correction|error\.tsx|error\.ts|RoutingError|errorCount|clearError|getPortNumName|error-boundary|isError|onError|handleError|LogLevel\.ERROR|errorDetails|_error|\.error\b.*=|error_event|Error fetching or storing news)' +EXCLUDE_PATTERNS='(error_correction|error\.tsx|error\.ts|RoutingError|errorCount|clearError|getPortNumName|error-boundary|isError|onError|handleError|LogLevel\.ERROR|errorDetails|_error|\.error\b.*=|error_event|Error fetching or storing news|ECONNREFUSED 172\.|code:.*ECONNREFUSED)' # Log output directory LOG_DIR="$PROJECT_DIR/tests/soak-logs"