Merge pull request #4 from lazypower/fix/issue-2-extraction-correctness

Fix silent extraction lock-out (closes #2)

Author: lazypower

internal/cli/extract.go

@@ -0,0 +1,179 @@
+package cli
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/lazypower/continuity/internal/hooks"
+	"github.com/spf13/cobra"
+)
+
+var (
+	extractForce        bool
+	extractTranscript   string
+	extractBackfillEmpty bool
+)
+
+var extractCmd = &cobra.Command{
+	Use:   "extract [session-id]",
+	Short: "Re-run extraction for a session",
+	Long: `Trigger memory extraction for a completed session.
+
+Typical uses:
+  continuity extract <session-id>              — re-extract if not already done
+  continuity extract <session-id> --force      — re-extract even if marked done
+  continuity extract --backfill-empty          — unmark every session that was
+                                                 flagged as extracted but has
+                                                 no memories attributed to it
+
+When a session-id is given, continuity auto-discovers the transcript at
+~/.claude/projects/*/<session-id>.jsonl. Pass --transcript to override.
+
+Requires a running server (continuity serve).`,
+	Args: cobra.MaximumNArgs(1),
+	RunE: runExtract,
+}
+
+func init() {
+	extractCmd.Flags().BoolVar(&extractForce, "force", false, "Bypass the idempotency guard (re-extract already-extracted sessions)")
+	extractCmd.Flags().StringVar(&extractTranscript, "transcript", "", "Path to transcript JSONL (overrides auto-discovery)")
+	extractCmd.Flags().BoolVar(&extractBackfillEmpty, "backfill-empty", false, "Unmark every session marked extracted with zero attributed memories")
+}
+
+func runExtract(cmd *cobra.Command, args []string) error {
+	client := hooks.NewClient()
+	if !client.Healthy() {
+		return fmt.Errorf("continuity server is not running — start it with: continuity serve")
+	}
+
+	if extractBackfillEmpty {
+		if len(args) > 0 || extractForce || extractTranscript != "" {
+			return fmt.Errorf("--backfill-empty cannot be combined with a session-id, --force, or --transcript")
+		}
+		return runBackfillEmpty(client)
+	}
+
+	if len(args) != 1 {
+		return fmt.Errorf("session-id is required (or use --backfill-empty)")
+	}
+	sessionID := strings.TrimSpace(args[0])
+	if sessionID == "" {
+		return fmt.Errorf("session-id is required")
+	}
+
+	transcriptPath := extractTranscript
+	if transcriptPath == "" {
+		found, err := findTranscript(sessionID)
+		if err != nil {
+			return err
+		}
+		transcriptPath = found
+	}
+	if _, err := os.Stat(transcriptPath); err != nil {
+		return fmt.Errorf("transcript not readable: %w", err)
+	}
+
+	body, _ := json.Marshal(map[string]any{
+		"transcript_path": transcriptPath,
+		"force":           extractForce,
+	})
+
+	data, err := client.Post("/api/sessions/"+sessionID+"/extract", body)
+	if err != nil {
+		if len(data) > 0 {
+			var resp struct {
+				Error string `json:"error"`
+			}
+			if jsonErr := json.Unmarshal(data, &resp); jsonErr == nil && resp.Error != "" {
+				return fmt.Errorf("%s", resp.Error)
+			}
+		}
+		return fmt.Errorf("extract: %w", err)
+	}
+
+	fmt.Printf("extraction queued for %s (transcript: %s, force: %v)\n", sessionID, transcriptPath, extractForce)
+	fmt.Println("check serve.log for progress — extraction runs asynchronously")
+	return nil
+}
+
+func runBackfillEmpty(client *hooks.Client) error {
+	data, err := client.Post("/api/sessions/unmark-empty-extractions", nil)
+	if err != nil {
+		if len(data) > 0 {
+			var resp struct {
+				Error string `json:"error"`
+			}
+			if jsonErr := json.Unmarshal(data, &resp); jsonErr == nil && resp.Error != "" {
+				return fmt.Errorf("%s", resp.Error)
+			}
+		}
+		return fmt.Errorf("backfill: %w", err)
+	}
+
+	var resp struct {
+		Status   string `json:"status"`
+		Unmarked int64  `json:"unmarked"`
+		Error    string `json:"error"`
+	}
+	if err := json.Unmarshal(data, &resp); err != nil {
+		return fmt.Errorf("parse response: %w", err)
+	}
+	if resp.Error != "" {
+		return fmt.Errorf("%s", resp.Error)
+	}
+
+	fmt.Printf("unmarked %d session(s) that were extracted with no attributed memories\n", resp.Unmarked)
+	if resp.Unmarked > 0 {
+		fmt.Println("they will be re-extracted on their next Stop/SessionEnd hook,")
+		fmt.Println("or force one now with: continuity extract <session-id> --force")
+	}
+	return nil
+}
+
+// findTranscript searches ~/.claude/projects/*/<session-id>.jsonl for a
+// Claude Code transcript matching the given session id. The sessionID is
+// validated first — path separators or ".." would let a glob pattern escape
+// ~/.claude/projects, which is surprising for "auto-discovery". Callers who
+// genuinely need to point at a transcript outside that tree should pass
+// --transcript explicitly.
+func findTranscript(sessionID string) (string, error) {
+	if err := validateSessionIDForGlob(sessionID); err != nil {
+		return "", fmt.Errorf("%w — pass --transcript to point at a specific file", err)
+	}
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", fmt.Errorf("resolve home dir: %w", err)
+	}
+	pattern := filepath.Join(home, ".claude", "projects", "*", sessionID+".jsonl")
+	matches, err := filepath.Glob(pattern)
+	if err != nil {
+		return "", fmt.Errorf("glob transcripts: %w", err)
+	}
+	if len(matches) == 0 {
+		return "", fmt.Errorf("no transcript found for session %s (looked in %s)", sessionID, pattern)
+	}
+	if len(matches) > 1 {
+		return "", fmt.Errorf("multiple transcripts found for %s — pass --transcript to disambiguate:\n  %s", sessionID, strings.Join(matches, "\n  "))
+	}
+	return matches[0], nil
+}
+
+// validateSessionIDForGlob rejects session IDs that would let the
+// auto-discovery glob escape ~/.claude/projects. Real Claude Code session
+// IDs are UUIDs, but continuity imports from other sources so we don't
+// require that — we just refuse anything that would traverse the filesystem.
+func validateSessionIDForGlob(sessionID string) error {
+	if sessionID == "" {
+		return fmt.Errorf("session-id is empty")
+	}
+	if strings.ContainsAny(sessionID, `/\`) {
+		return fmt.Errorf("session-id %q contains a path separator", sessionID)
+	}
+	if sessionID == "." || sessionID == ".." || strings.Contains(sessionID, "..") {
+		return fmt.Errorf("session-id %q contains path traversal", sessionID)
+	}
+	return nil
+}

internal/cli/extract_test.go

@@ -0,0 +1,178 @@
+package cli
+
+import (
+	"encoding/json"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/lazypower/continuity/internal/engine"
+	"github.com/lazypower/continuity/internal/server"
+	"github.com/lazypower/continuity/internal/store"
+)
+
+// extractTestServer stands up an in-memory server and points the CLI client
+// at it via CONTINUITY_URL. Mirrors showTestServer.
+func extractTestServer(t *testing.T) *store.DB {
+	t.Helper()
+	db, err := store.OpenMemory()
+	if err != nil {
+		t.Fatalf("OpenMemory: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+
+	eng := engine.New(db, nil)
+	srv := server.New(db, eng, "test-version")
+	ts := httptest.NewServer(srv)
+	t.Cleanup(ts.Close)
+
+	prev := os.Getenv("CONTINUITY_URL")
+	os.Setenv("CONTINUITY_URL", ts.URL)
+	t.Cleanup(func() { os.Setenv("CONTINUITY_URL", prev) })
+
+	return db
+}
+
+func resetExtractFlags() {
+	extractForce = false
+	extractTranscript = ""
+	extractBackfillEmpty = false
+}
+
+func writeDummyTranscript(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	path := filepath.Join(dir, "transcript.jsonl")
+	entries := []map[string]any{
+		{"type": "user", "message": map[string]any{"role": "user", "content": "Hello please help me out with this task"}},
+		{"type": "assistant", "message": map[string]any{"role": "assistant", "content": "Sure, I can help."}},
+		{"type": "user", "message": map[string]any{"role": "user", "content": "Another message here"}},
+	}
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatalf("create transcript: %v", err)
+	}
+	defer f.Close()
+	for _, e := range entries {
+		data, _ := json.Marshal(e)
+		if _, err := f.Write(data); err != nil {
+			t.Fatalf("write transcript: %v", err)
+		}
+		if _, err := f.Write([]byte("\n")); err != nil {
+			t.Fatalf("write transcript: %v", err)
+		}
+	}
+	return path
+}
+
+func TestExtractCLIWithExplicitTranscript(t *testing.T) {
+	db := extractTestServer(t)
+	db.InitSession("cli-sess", "proj")
+	path := writeDummyTranscript(t)
+
+	resetExtractFlags()
+	extractTranscript = path
+
+	out, err := captureStdout(t, func() error {
+		return runExtract(extractCmd, []string{"cli-sess"})
+	})
+	if err != nil {
+		t.Fatalf("runExtract: %v", err)
+	}
+	if !strings.Contains(out, "extraction queued") {
+		t.Errorf("expected queued message, got: %s", out)
+	}
+}
+
+func TestExtractCLIBackfillEmpty(t *testing.T) {
+	db := extractTestServer(t)
+	db.InitSession("damaged", "proj")
+	db.MarkExtracted("damaged")
+
+	resetExtractFlags()
+	extractBackfillEmpty = true
+
+	out, err := captureStdout(t, func() error {
+		return runExtract(extractCmd, nil)
+	})
+	if err != nil {
+		t.Fatalf("runExtract --backfill-empty: %v", err)
+	}
+	if !strings.Contains(out, "unmarked 1 session") {
+		t.Errorf("expected unmark count, got: %s", out)
+	}
+
+	// Verify the damaged session was actually unmarked.
+	s, _ := db.GetSession("damaged")
+	if s.ExtractedAt != nil {
+		t.Error("damaged session should have been unmarked via CLI path")
+	}
+}
+
+func TestExtractCLIBackfillExclusiveFlags(t *testing.T) {
+	extractTestServer(t)
+
+	resetExtractFlags()
+	extractBackfillEmpty = true
+	extractForce = true
+
+	err := runExtract(extractCmd, nil)
+	if err == nil {
+		t.Fatal("expected error when --backfill-empty combined with --force")
+	}
+	if !strings.Contains(err.Error(), "cannot be combined") {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+func TestExtractCLIRequiresSessionID(t *testing.T) {
+	extractTestServer(t)
+	resetExtractFlags()
+
+	err := runExtract(extractCmd, nil)
+	if err == nil {
+		t.Fatal("expected error when no session-id and no --backfill-empty")
+	}
+}
+
+func TestExtractCLITranscriptMissing(t *testing.T) {
+	extractTestServer(t)
+	resetExtractFlags()
+	extractTranscript = "/no/such/path.jsonl"
+
+	err := runExtract(extractCmd, []string{"abc"})
+	if err == nil {
+		t.Fatal("expected error for missing transcript")
+	}
+	if !strings.Contains(err.Error(), "not readable") {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+func TestValidateSessionIDForGlob(t *testing.T) {
+	cases := []struct {
+		name      string
+		sessionID string
+		wantErr   bool
+	}{
+		{"empty", "", true},
+		{"uuid", "0f5d812c-69e1-40d0-9eef-5436f5721a80", false},
+		{"plain slug", "my-session", false},
+		{"forward slash", "foo/bar", true},
+		{"backslash", `foo\bar`, true},
+		{"parent traversal", "..", true},
+		{"embedded traversal", "foo..bar", true},
+		{"dot", ".", true},
+		{"absolute escape", "/etc/passwd", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateSessionIDForGlob(tc.sessionID)
+			if (err != nil) != tc.wantErr {
+				t.Errorf("validateSessionIDForGlob(%q) err=%v wantErr=%v", tc.sessionID, err, tc.wantErr)
+			}
+		})
+	}
+}

internal/cli/root.go

@@ -29,4 +29,5 @@ func init() {
 	rootCmd.AddCommand(timelineCmd)
 	rootCmd.AddCommand(installServiceCmd)
 	rootCmd.AddCommand(uninstallServiceCmd)
+	rootCmd.AddCommand(extractCmd)
 }