|
| 1 | +"""SQLite-based append-only conversation log. |
| 2 | +
|
| 3 | +Stores all conversation turns for batch processing during sleep cycles. |
| 4 | +Uses WAL mode for concurrent read/write safety. |
| 5 | +""" |
| 6 | + |
| 7 | +from __future__ import annotations |
| 8 | + |
| 9 | +import logging |
| 10 | +import sqlite3 |
| 11 | +from datetime import datetime, timedelta |
| 12 | +from pathlib import Path |
| 13 | + |
| 14 | +logger = logging.getLogger(__name__) |
| 15 | + |
| 16 | +_SCHEMA = """ |
| 17 | +CREATE TABLE IF NOT EXISTS conversation_turns ( |
| 18 | + id INTEGER PRIMARY KEY AUTOINCREMENT, |
| 19 | + conversation_id TEXT NOT NULL, |
| 20 | + turn_index INTEGER NOT NULL, |
| 21 | + role TEXT NOT NULL, |
| 22 | + content TEXT NOT NULL, |
| 23 | + timestamp TEXT NOT NULL, |
| 24 | + processed INTEGER NOT NULL DEFAULT 0 |
| 25 | +); |
| 26 | +
|
| 27 | +CREATE INDEX IF NOT EXISTS idx_conv_id ON conversation_turns(conversation_id); |
| 28 | +CREATE INDEX IF NOT EXISTS idx_processed ON conversation_turns(processed); |
| 29 | +CREATE INDEX IF NOT EXISTS idx_timestamp ON conversation_turns(timestamp); |
| 30 | +""" |
| 31 | + |
| 32 | + |
| 33 | +class ConversationLog: |
| 34 | + """Append-only conversation log backed by SQLite. |
| 35 | +
|
| 36 | + All conversation turns are recorded for later batch processing |
| 37 | + by the sleep cycle memory extraction pipeline. |
| 38 | + """ |
| 39 | + |
| 40 | + def __init__(self, db_path: str | Path) -> None: |
| 41 | + self._db_path = str(db_path) |
| 42 | + self._conn = sqlite3.connect(self._db_path, check_same_thread=False) |
| 43 | + self._conn.execute("PRAGMA journal_mode=WAL") |
| 44 | + self._conn.executescript(_SCHEMA) |
| 45 | + self._conn.commit() |
| 46 | + |
| 47 | + def append_turn( |
| 48 | + self, |
| 49 | + conversation_id: str, |
| 50 | + turn_index: int, |
| 51 | + role: str, |
| 52 | + content: str, |
| 53 | + ) -> int: |
| 54 | + """Append a conversation turn. Returns the row id.""" |
| 55 | + now = datetime.now().isoformat() |
| 56 | + cursor = self._conn.execute( |
| 57 | + "INSERT INTO conversation_turns (conversation_id, turn_index, role, content, timestamp) " |
| 58 | + "VALUES (?, ?, ?, ?, ?)", |
| 59 | + (conversation_id, turn_index, role, content, now), |
| 60 | + ) |
| 61 | + self._conn.commit() |
| 62 | + return cursor.lastrowid or 0 |
| 63 | + |
| 64 | + def get_unprocessed_turns(self, limit: int = 500) -> list[dict]: |
| 65 | + """Get unprocessed turns ordered by conversation and turn index. |
| 66 | +
|
| 67 | + Returns list of dicts with keys: id, conversation_id, turn_index, role, content, timestamp. |
| 68 | + """ |
| 69 | + cursor = self._conn.execute( |
| 70 | + "SELECT id, conversation_id, turn_index, role, content, timestamp " |
| 71 | + "FROM conversation_turns " |
| 72 | + "WHERE processed = 0 " |
| 73 | + "ORDER BY conversation_id, turn_index " |
| 74 | + "LIMIT ?", |
| 75 | + (limit,), |
| 76 | + ) |
| 77 | + return [ |
| 78 | + { |
| 79 | + "id": row[0], |
| 80 | + "conversation_id": row[1], |
| 81 | + "turn_index": row[2], |
| 82 | + "role": row[3], |
| 83 | + "content": row[4], |
| 84 | + "timestamp": row[5], |
| 85 | + } |
| 86 | + for row in cursor.fetchall() |
| 87 | + ] |
| 88 | + |
| 89 | + def get_conversation(self, conversation_id: str) -> list[dict]: |
| 90 | + """Get all turns for a specific conversation, ordered by turn index.""" |
| 91 | + cursor = self._conn.execute( |
| 92 | + "SELECT id, conversation_id, turn_index, role, content, timestamp, processed " |
| 93 | + "FROM conversation_turns " |
| 94 | + "WHERE conversation_id = ? " |
| 95 | + "ORDER BY turn_index", |
| 96 | + (conversation_id,), |
| 97 | + ) |
| 98 | + return [ |
| 99 | + { |
| 100 | + "id": row[0], |
| 101 | + "conversation_id": row[1], |
| 102 | + "turn_index": row[2], |
| 103 | + "role": row[3], |
| 104 | + "content": row[4], |
| 105 | + "timestamp": row[5], |
| 106 | + "processed": bool(row[6]), |
| 107 | + } |
| 108 | + for row in cursor.fetchall() |
| 109 | + ] |
| 110 | + |
| 111 | + def mark_processed(self, turn_ids: list[int]) -> int: |
| 112 | + """Mark turns as processed. Returns number of rows updated.""" |
| 113 | + if not turn_ids: |
| 114 | + return 0 |
| 115 | + placeholders = ",".join("?" for _ in turn_ids) |
| 116 | + cursor = self._conn.execute( |
| 117 | + f"UPDATE conversation_turns SET processed = 1 WHERE id IN ({placeholders})", |
| 118 | + turn_ids, |
| 119 | + ) |
| 120 | + self._conn.commit() |
| 121 | + return cursor.rowcount |
| 122 | + |
| 123 | + def cleanup_old(self, days: int = 30) -> int: |
| 124 | + """Delete processed turns older than `days`. Returns number of rows deleted.""" |
| 125 | + cutoff = (datetime.now() - timedelta(days=days)).isoformat() |
| 126 | + cursor = self._conn.execute( |
| 127 | + "DELETE FROM conversation_turns WHERE processed = 1 AND timestamp < ?", |
| 128 | + (cutoff,), |
| 129 | + ) |
| 130 | + self._conn.commit() |
| 131 | + deleted = cursor.rowcount |
| 132 | + if deleted > 0: |
| 133 | + logger.info("Cleaned up %d old conversation turns (older than %d days)", deleted, days) |
| 134 | + return deleted |
| 135 | + |
| 136 | + def count(self, processed: bool | None = None) -> int: |
| 137 | + """Count turns, optionally filtered by processed status.""" |
| 138 | + if processed is None: |
| 139 | + cursor = self._conn.execute("SELECT COUNT(*) FROM conversation_turns") |
| 140 | + else: |
| 141 | + cursor = self._conn.execute( |
| 142 | + "SELECT COUNT(*) FROM conversation_turns WHERE processed = ?", |
| 143 | + (1 if processed else 0,), |
| 144 | + ) |
| 145 | + return cursor.fetchone()[0] |
| 146 | + |
| 147 | + def close(self) -> None: |
| 148 | + """Close the database connection.""" |
| 149 | + self._conn.close() |
0 commit comments