perf(remote): Fast SFTP walk and background indexing

- Add SFTPFs.Walk using ReadDir (1 round trip per dir vs N Stat calls)
- Add FastWalker interface, Index.Build() uses it when available
- Single file: create one-entry index without walking parent dir
- Drop CacheOnReadFs (caused overhead on directory operations)
- Build fulltext + link graph in background after TUI starts
- Add Index.AddFile for manual entry injection

Author: Benjamin-Connelly

CLAUDE.md

@@ -2,7 +2,7 @@
 
 Dual-mode markdown navigator: TUI (Bubble Tea) and web (stdlib net/http). Inter-document link navigation with history, backlinks, and broken link detection. Syntax highlighting (50+ languages), git-aware, zero-config.
 
-**Version:** v0.3.1
+**Version:** v0.3.2
 
 ## Tech Stack
 

cmd/lookit/main.go

@@ -10,7 +10,6 @@ import (
 	"time"
 
 	tea "github.com/charmbracelet/bubbletea"
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
 	"golang.org/x/term"
@@ -26,7 +25,7 @@ import (
 	"github.com/Benjamin-Connelly/lookit/internal/web"
 )
 
-var version = "v0.3.1"
+var version = "v0.3.2"
 
 var cfg *config.Config
 
@@ -664,36 +663,40 @@ func runRemote(target *remote.Target) error {
 
 	fmt.Fprintf(os.Stderr, "Connected to %s. Indexing...\n", resolved.Display())
 
-	// Build afero filesystem: SFTP reads cached in memory with 30s TTL
+	// Use SFTP filesystem directly (no CacheOnReadFs — its directory
+	// handling adds excessive round trips over SSH)
 	sftpFs := remote.NewSFTPFs(conn.SFTP())
-	cachedFs := afero.NewCacheOnReadFs(sftpFs, afero.NewMemMapFs(), 30*time.Second)
 
-	// Resolve root: if target is a file, use parent dir and pre-select
+	// Resolve root: if target is a file, build a single-entry index
+	// instead of walking the parent directory (which could be huge)
 	root := resolved.Path
 	var initialFile string
-	if info, err := cachedFs.Stat(root); err == nil && !info.IsDir() {
-		initialFile = filepath.Base(root)
-		root = filepath.Dir(root)
-	}
+	var idx *index.Index
 
-	// Build index directly from remote filesystem
-	idx := index.NewWithFs(root, cachedFs)
-	if err := idx.Build(); err != nil {
-		return fmt.Errorf("building index: %w", err)
+	info, err := sftpFs.Stat(root)
+	if err != nil {
+		return fmt.Errorf("stat remote path: %w", err)
 	}
 
-	// Build fulltext search index (in-memory for remote)
-	if err := idx.BuildFulltext(""); err != nil {
-		fmt.Fprintf(os.Stderr, "warning: fulltext index unavailable: %v\n", err)
+	if !info.IsDir() {
+		// Single file: create index with just this entry
+		initialFile = filepath.Base(root)
+		root = filepath.Dir(root)
+		idx = index.NewWithFs(root, sftpFs)
+		idx.AddFile(resolved.Path, initialFile, info.Size(), info.ModTime())
+	} else {
+		// Directory: walk via SFTP
+		idx = index.NewWithFs(root, sftpFs)
+		if err := idx.Build(); err != nil {
+			return fmt.Errorf("building index: %w", err)
+		}
 	}
-	defer idx.CloseFulltext()
 
 	links := index.NewLinkGraph()
-	links.BuildFromIndex(idx)
 
 	fmt.Fprintf(os.Stderr, "Ready. Starting TUI...\n")
 
-	// Create TUI with remote info
+	// Create TUI with remote info (fulltext + link graph build in background)
 	model := tui.New(cfg, idx, links)
 	if initialFile != "" {
 		model.SelectFile(initialFile)
@@ -703,11 +706,16 @@ func runRemote(target *remote.Target) error {
 		State:   conn.State().String(),
 	})
 
-	// Background poller: rebuild index periodically to detect remote changes
+	// Background: build fulltext + link graph, then poll for changes
 	done := make(chan struct{})
 	defer close(done)
+	defer idx.CloseFulltext()
 	lastRefresh := time.Now()
 	go func() {
+		// Build link graph and fulltext in background (reads files over SFTP)
+		links.BuildFromIndex(idx)
+		_ = idx.BuildFulltext("")
+
 		ticker := time.NewTicker(1 * time.Second)
 		defer ticker.Stop()
 		for {
@@ -735,7 +743,7 @@ func runRemote(target *remote.Target) error {
 	}()
 
 	p := tea.NewProgram(model, tea.WithAltScreen())
-	_, err := p.Run()
+	_, err = p.Run()
 	return err
 }
 

internal/index/index.go

@@ -105,6 +105,13 @@ var hiddenDirs = map[string]bool{
 	".git": true, ".hg": true, ".svn": true, ".bzr": true,
 }
 
+// FastWalker is an optional interface that filesystems can implement
+// for optimized directory traversal (e.g., SFTP uses ReadDir instead
+// of per-file Stat calls).
+type FastWalker interface {
+	Walk(root string, fn func(path string, info os.FileInfo, err error) error) error
+}
+
 // Build walks the root directory and populates the index.
 func (idx *Index) Build() error {
 	idx.mu.Lock()
@@ -116,7 +123,7 @@ func (idx *Index) Build() error {
 
 	gitignore := loadGitignore(idx.fs, idx.root)
 
-	err := afero.Walk(idx.fs, idx.root, func(path string, info os.FileInfo, err error) error {
+	walkFn := func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return nil
 		}
@@ -175,7 +182,16 @@ func (idx *Index) Build() error {
 		}
 
 		return nil
-	})
+	}
+
+	// Use fast walker if available (e.g., SFTP ReadDir is one round
+	// trip per directory vs per-file Stat calls in afero.Walk)
+	var err error
+	if fw, ok := idx.fs.(FastWalker); ok {
+		err = fw.Walk(idx.root, walkFn)
+	} else {
+		err = afero.Walk(idx.fs, idx.root, walkFn)
+	}
 	if err != nil {
 		return err
 	}
@@ -241,6 +257,25 @@ func (idx *Index) Root() string {
 	return idx.root
 }
 
+// AddFile adds a single file entry to the index without walking.
+// The path must be absolute; relPath is relative to the index root.
+func (idx *Index) AddFile(absPath, relPath string, size int64, modTime time.Time) {
+	idx.mu.Lock()
+	defer idx.mu.Unlock()
+
+	entry := FileEntry{
+		Path:       absPath,
+		RelPath:    relPath,
+		Size:       size,
+		ModTime:    modTime,
+		IsMarkdown: isMarkdown(filepath.Base(relPath)),
+	}
+	idx.entries = append(idx.entries, entry)
+	idx.byPath[relPath] = &idx.entries[len(idx.entries)-1]
+	idx.stats.FileCount++
+	idx.stats.TotalSize += size
+}
+
 // Stats returns aggregate statistics about the index.
 func (idx *Index) Stats() Stats {
 	idx.mu.RLock()

internal/remote/sftpfs.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"io"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/sftp"
@@ -141,5 +143,51 @@ func (d *sftpDir) Seek(offset int64, whence int) (int64, error) { return 0, erro
 func (d *sftpDir) Write(p []byte) (int, error)                  { return 0, errReadOnly }
 func (d *sftpDir) WriteAt(p []byte, off int64) (int, error)     { return 0, errReadOnly }
 func (d *sftpDir) WriteString(s string) (ret int, err error)    { return 0, errReadOnly }
-func (d *sftpDir) Sync() error                                  { return nil }
-func (d *sftpDir) Truncate(size int64) error                    { return errReadOnly }
+func (d *sftpDir) Sync() error                               { return nil }
+func (d *sftpDir) Truncate(size int64) error                  { return errReadOnly }
+
+// WalkFunc is the callback for Walk.
+type WalkFunc func(path string, info os.FileInfo, err error) error
+
+// Walk traverses the directory tree rooted at root using SFTP ReadDir
+// (one round trip per directory) instead of individual Stat calls.
+func (s *SFTPFs) Walk(root string, fn WalkFunc) error {
+	info, err := s.client.Stat(root)
+	if err != nil {
+		return fn(root, nil, err)
+	}
+	return s.walk(root, info, fn)
+}
+
+func (s *SFTPFs) walk(path string, info os.FileInfo, fn WalkFunc) error {
+	if !info.IsDir() {
+		return fn(path, info, nil)
+	}
+
+	if err := fn(path, info, nil); err != nil {
+		if err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	entries, err := s.client.ReadDir(path)
+	if err != nil {
+		return fn(path, info, err)
+	}
+
+	for _, entry := range entries {
+		name := entry.Name()
+		if strings.HasPrefix(name, ".") {
+			continue
+		}
+		child := path + "/" + name
+		if err := s.walk(child, entry, fn); err != nil {
+			if err == filepath.SkipDir {
+				continue
+			}
+			return err
+		}
+	}
+	return nil
+}