Merge pull request #40 from priyanshujain/feat-progressive-skill-loading

Progressive skill loading: slim SKILL.md + REFERENCE.md

Author: priyanshujain

agent/tools/load_skills.go

@@ -40,8 +40,13 @@ func (l *LoadSkillsTool) Execute(_ context.Context, input json.RawMessage) (stri
 
 	var parts []string
 	for _, name := range in.Names {
-		skillPath := filepath.Join(skills.SkillsDir(), name, "SKILL.md")
-		content, err := os.ReadFile(skillPath)
+		// Prefer REFERENCE.md (full instructions), fall back to SKILL.md.
+		refPath := filepath.Join(skills.SkillsDir(), name, "REFERENCE.md")
+		content, err := os.ReadFile(refPath)
+		if err != nil {
+			skillPath := filepath.Join(skills.SkillsDir(), name, "SKILL.md")
+			content, err = os.ReadFile(skillPath)
+		}
 		if err != nil {
 			parts = append(parts, fmt.Sprintf("--- %s ---\nError: skill not found", name))
 			continue

agent/tools/skills_test.go

@@ -15,7 +15,7 @@ func TestLoadSkills(t *testing.T) {
 	tmp := t.TempDir()
 	t.Setenv("OBK_CONFIG_DIR", tmp)
 
-	// Create test skill.
+	// Create test skill with only SKILL.md (fallback path).
 	skillDir := filepath.Join(tmp, "skills", "test-skill")
 	if err := os.MkdirAll(skillDir, 0700); err != nil {
 		t.Fatal(err)
@@ -38,6 +38,35 @@ func TestLoadSkills(t *testing.T) {
 	}
 }
 
+func TestLoadSkillsPrefersReferenceMD(t *testing.T) {
+	tmp := t.TempDir()
+	t.Setenv("OBK_CONFIG_DIR", tmp)
+
+	skillDir := filepath.Join(tmp, "skills", "test-skill")
+	if err := os.MkdirAll(skillDir, 0700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(skillDir, "SKILL.md"), []byte("---\nname: test-skill\n---\nSlim summary"), 0600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(skillDir, "REFERENCE.md"), []byte("Full reference content with schema and examples"), 0600); err != nil {
+		t.Fatal(err)
+	}
+
+	tool := &LoadSkillsTool{}
+	input, _ := json.Marshal(map[string]any{"names": []string{"test-skill"}})
+	result, err := tool.Execute(context.Background(), input)
+	if err != nil {
+		t.Fatalf("Execute: %v", err)
+	}
+	if !strings.Contains(result, "Full reference content") {
+		t.Errorf("should prefer REFERENCE.md content, got: %q", result)
+	}
+	if strings.Contains(result, "Slim summary") {
+		t.Errorf("should not contain SKILL.md content when REFERENCE.md exists, got: %q", result)
+	}
+}
+
 func TestLoadSkillsMissing(t *testing.T) {
 	tmp := t.TempDir()
 	t.Setenv("OBK_CONFIG_DIR", tmp)

assistant/CLAUDE.md

@@ -4,7 +4,7 @@ Skills are loaded from ~/.obk/skills/ — run `obk setup` to configure.
 
 ## How to access data
 
-Use the skills provided to query data via `sqlite3`, send messages via `obk`, or interact with Google Workspace via `gws`. Each skill contains the exact schema, query patterns, and command usage.
+Use the skills provided to query data via `sqlite3`, send messages via `obk`, or interact with Google Workspace via `gws`. Each skill's SKILL.md contains a summary. Before using a skill, read its REFERENCE.md file for the full schema, query patterns, and command usage.
 
 ## Messaging someone
 

internal/skills/install.go

@@ -296,14 +296,20 @@ func resolveGWSSkills(cfg *config.Config) (map[string]SkillEntry, []string, erro
 		}
 		desired[name] = se
 
-		// Copy to skills dir.
+		// Split into slim SKILL.md + full REFERENCE.md, then copy.
+		slim, ref := splitSkillFile(content)
 		destDir := filepath.Join(SkillsDir(), name)
 		if err := os.MkdirAll(destDir, 0700); err != nil {
 			return nil, nil, fmt.Errorf("create skill dir %s: %w", name, err)
 		}
-		if err := os.WriteFile(filepath.Join(destDir, "SKILL.md"), content, 0600); err != nil {
+		if err := os.WriteFile(filepath.Join(destDir, "SKILL.md"), slim, 0600); err != nil {
 			return nil, nil, fmt.Errorf("write skill %s: %w", name, err)
 		}
+		if len(ref) > 0 {
+			if err := os.WriteFile(filepath.Join(destDir, "REFERENCE.md"), ref, 0600); err != nil {
+				return nil, nil, fmt.Errorf("write reference %s: %w", name, err)
+			}
+		}
 	}
 
 	return desired, skipped, nil
@@ -378,3 +384,45 @@ func gwsServiceFromSkillName(name string) string {
 	parts := strings.SplitN(name, "-", 2)
 	return parts[0]
 }
+
+// splitSkillFile splits a full SKILL.md into a slim summary (frontmatter +
+// first paragraph + pointer) and a REFERENCE.md (remaining body).
+// If there is no body after frontmatter, ref is nil.
+func splitSkillFile(content []byte) (slim []byte, ref []byte) {
+	text := string(content)
+
+	// Parse frontmatter (between --- markers).
+	if !strings.HasPrefix(text, "---\n") {
+		return content, nil
+	}
+	end := strings.Index(text[4:], "\n---\n")
+	if end < 0 {
+		return content, nil
+	}
+	frontmatter := text[:4+end+5] // includes both --- lines and trailing newline
+	body := strings.TrimSpace(text[4+end+5:])
+	if body == "" {
+		return content, nil
+	}
+
+	// Extract first paragraph (up to first blank line) as the summary.
+	var summary, rest string
+	if idx := strings.Index(body, "\n\n"); idx >= 0 {
+		summary = body[:idx]
+		rest = strings.TrimSpace(body[idx+2:])
+	} else {
+		summary = body
+	}
+
+	// Build slim SKILL.md: frontmatter + summary + pointer.
+	slimStr := frontmatter + "\n" + summary + "\n\nRead the REFERENCE.md in this skill's directory for full instructions.\n"
+	slim = []byte(slimStr)
+
+	if rest == "" {
+		// Only one paragraph — no reference needed, keep full body in SKILL.md.
+		return content, nil
+	}
+
+	ref = []byte(rest + "\n")
+	return slim, ref
+}

internal/skills/install_test.go

@@ -218,6 +218,15 @@ func TestInstallBuiltinSkillsNoAuth(t *testing.T) {
 		t.Error("history-read SKILL.md is empty")
 	}
 
+	// Verify REFERENCE.md was written alongside SKILL.md.
+	refContent, err := os.ReadFile(filepath.Join(tmp, "skills", "history-read", "REFERENCE.md"))
+	if err != nil {
+		t.Fatalf("read history-read REFERENCE.md: %v", err)
+	}
+	if len(refContent) == 0 {
+		t.Error("history-read REFERENCE.md is empty")
+	}
+
 	// Verify schema.sql was written alongside SKILL.md.
 	schemaContent, err := os.ReadFile(filepath.Join(tmp, "skills", "history-read", "schema.sql"))
 	if err != nil {
@@ -227,14 +236,21 @@ func TestInstallBuiltinSkillsNoAuth(t *testing.T) {
 		t.Error("history-read schema.sql is empty")
 	}
 
-	// Verify memory-save SKILL.md was written.
+	// Verify memory-save SKILL.md and REFERENCE.md were written.
 	memorySaveContent, err := os.ReadFile(filepath.Join(tmp, "skills", "memory-save", "SKILL.md"))
 	if err != nil {
 		t.Fatalf("read memory-save SKILL.md: %v", err)
 	}
 	if len(memorySaveContent) == 0 {
 		t.Error("memory-save SKILL.md is empty")
 	}
+	memorySaveRef, err := os.ReadFile(filepath.Join(tmp, "skills", "memory-save", "REFERENCE.md"))
+	if err != nil {
+		t.Fatalf("read memory-save REFERENCE.md: %v", err)
+	}
+	if len(memorySaveRef) == 0 {
+		t.Error("memory-save REFERENCE.md is empty")
+	}
 
 	// Verify manifest was written.
 	m, err := LoadManifest()
@@ -423,6 +439,56 @@ func TestInstallRemovesRevokedSkills(t *testing.T) {
 	}
 }
 
+func TestSplitSkillFile(t *testing.T) {
+	tests := []struct {
+		name     string
+		input    string
+		wantSlim string
+		wantRef  string
+	}{
+		{
+			name:     "no frontmatter",
+			input:    "Just some content",
+			wantSlim: "Just some content",
+			wantRef:  "",
+		},
+		{
+			name:     "frontmatter only",
+			input:    "---\nname: test\n---\n",
+			wantSlim: "---\nname: test\n---\n",
+			wantRef:  "",
+		},
+		{
+			name:     "single paragraph body",
+			input:    "---\nname: test\n---\n\nJust a summary line.",
+			wantSlim: "---\nname: test\n---\n\nJust a summary line.",
+			wantRef:  "",
+		},
+		{
+			name:  "multi paragraph body",
+			input: "---\nname: test\ndescription: Test skill\n---\n\nFirst paragraph summary.\n\n## Commands\n\n```bash\nsome command\n```\n\n## Examples\n\nMore content here.",
+			wantSlim: "---\nname: test\ndescription: Test skill\n---\n\nFirst paragraph summary.\n\n" +
+				"Read the REFERENCE.md in this skill's directory for full instructions.\n",
+			wantRef: "## Commands\n\n```bash\nsome command\n```\n\n## Examples\n\nMore content here.\n",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			slim, ref := splitSkillFile([]byte(tt.input))
+			if string(slim) != tt.wantSlim {
+				t.Errorf("slim:\ngot:  %q\nwant: %q", string(slim), tt.wantSlim)
+			}
+			if string(ref) != tt.wantRef {
+				if tt.wantRef == "" && ref == nil {
+					return // nil and "" are equivalent for empty ref
+				}
+				t.Errorf("ref:\ngot:  %q\nwant: %q", string(ref), tt.wantRef)
+			}
+		})
+	}
+}
+
 func TestInstallIdempotent(t *testing.T) {
 	t.Setenv("OBK_CONFIG_DIR", t.TempDir())
 

skills/applenotes-read/REFERENCE.md

@@ -0,0 +1,31 @@
+## Schema
+
+Full database schema: see schema.sql in this skill directory.
+
+## Query patterns
+
+```bash
+# Recent notes
+obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes ORDER BY modified_at DESC LIMIT 20;"
+
+# Search by title
+obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes WHERE LOWER(title) LIKE '%keyword%' ORDER BY modified_at DESC LIMIT 20;"
+
+# Full text search across title and body
+obk db applenotes "SELECT modified_at, folder, title, substr(body, 1, 200) FROM applenotes_notes WHERE LOWER(title) LIKE '%term%' OR LOWER(body) LIKE '%term%' ORDER BY modified_at DESC LIMIT 10;"
+
+# Read full note
+obk db applenotes "SELECT title, folder, account, created_at, modified_at, body FROM applenotes_notes WHERE id = <id>;"
+
+# Notes in a specific folder
+obk db applenotes "SELECT modified_at, title FROM applenotes_notes WHERE LOWER(folder) = 'notes' ORDER BY modified_at DESC LIMIT 20;"
+
+# List all folders
+obk db applenotes "SELECT name, account, (SELECT COUNT(*) FROM applenotes_notes WHERE folder_id = f.apple_id) as note_count FROM applenotes_folders f ORDER BY name;"
+
+# Notes by account
+obk db applenotes "SELECT account, COUNT(*) FROM applenotes_notes GROUP BY account;"
+
+# Recently modified notes (last 7 days)
+obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes WHERE modified_at >= datetime('now', '-7 days') ORDER BY modified_at DESC;"
+```

skills/applenotes-read/SKILL.md

@@ -4,34 +4,6 @@ description: Search Apple Notes, find notes by title or content, browse notes by
 allowed-tools: Bash(obk *)
 ---
 
-## Schema
+Query synced Apple Notes from `~/.obk/applenotes/data.db`.
 
-Full database schema: see schema.sql in this skill directory.
-
-## Query patterns
-
-```bash
-# Recent notes
-obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes ORDER BY modified_at DESC LIMIT 20;"
-
-# Search by title
-obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes WHERE LOWER(title) LIKE '%keyword%' ORDER BY modified_at DESC LIMIT 20;"
-
-# Full text search across title and body
-obk db applenotes "SELECT modified_at, folder, title, substr(body, 1, 200) FROM applenotes_notes WHERE LOWER(title) LIKE '%term%' OR LOWER(body) LIKE '%term%' ORDER BY modified_at DESC LIMIT 10;"
-
-# Read full note
-obk db applenotes "SELECT title, folder, account, created_at, modified_at, body FROM applenotes_notes WHERE id = <id>;"
-
-# Notes in a specific folder
-obk db applenotes "SELECT modified_at, title FROM applenotes_notes WHERE LOWER(folder) = 'notes' ORDER BY modified_at DESC LIMIT 20;"
-
-# List all folders
-obk db applenotes "SELECT name, account, (SELECT COUNT(*) FROM applenotes_notes WHERE folder_id = f.apple_id) as note_count FROM applenotes_folders f ORDER BY name;"
-
-# Notes by account
-obk db applenotes "SELECT account, COUNT(*) FROM applenotes_notes GROUP BY account;"
-
-# Recently modified notes (last 7 days)
-obk db applenotes "SELECT modified_at, folder, title FROM applenotes_notes WHERE modified_at >= datetime('now', '-7 days') ORDER BY modified_at DESC;"
-```
+Read the REFERENCE.md in this skill's directory for the full schema and query patterns.

skills/email-read/REFERENCE.md

@@ -0,0 +1,28 @@
+## Schema
+
+Full database schema: see schema.sql in this skill directory.
+
+## Query patterns
+
+```bash
+# Recent emails
+obk db gmail "SELECT date, from_addr, subject FROM gmail_emails ORDER BY date DESC LIMIT 20;"
+
+# Search by subject
+obk db gmail "SELECT date, from_addr, subject FROM gmail_emails WHERE LOWER(subject) LIKE '%keyword%' ORDER BY date DESC LIMIT 20;"
+
+# Search by sender
+obk db gmail "SELECT date, from_addr, subject FROM gmail_emails WHERE LOWER(from_addr) LIKE '%name%' ORDER BY date DESC LIMIT 20;"
+
+# Full text search across subject and body
+obk db gmail "SELECT date, from_addr, subject, substr(body, 1, 200) FROM gmail_emails WHERE LOWER(subject) LIKE '%term%' OR LOWER(body) LIKE '%term%' ORDER BY date DESC LIMIT 10;"
+
+# Read full email
+obk db gmail "SELECT from_addr, to_addr, subject, date, body FROM gmail_emails WHERE id = <id>;"
+
+# Emails with attachments
+obk db gmail "SELECT e.date, e.from_addr, e.subject, a.filename, a.mime_type FROM gmail_emails e JOIN gmail_attachments a ON a.email_id = e.id ORDER BY e.date DESC LIMIT 20;"
+
+# Count by account
+obk db gmail "SELECT account, COUNT(*) FROM gmail_emails GROUP BY account;"
+```

skills/email-read/SKILL.md

@@ -4,31 +4,6 @@ description: Search emails, check inbox, find messages, look up correspondence,
 allowed-tools: Bash(obk *)
 ---
 
-## Schema
+Query synced Gmail emails from `~/.obk/gmail/data.db`.
 
-Full database schema: see schema.sql in this skill directory.
-
-## Query patterns
-
-```bash
-# Recent emails
-obk db gmail "SELECT date, from_addr, subject FROM gmail_emails ORDER BY date DESC LIMIT 20;"
-
-# Search by subject
-obk db gmail "SELECT date, from_addr, subject FROM gmail_emails WHERE LOWER(subject) LIKE '%keyword%' ORDER BY date DESC LIMIT 20;"
-
-# Search by sender
-obk db gmail "SELECT date, from_addr, subject FROM gmail_emails WHERE LOWER(from_addr) LIKE '%name%' ORDER BY date DESC LIMIT 20;"
-
-# Full text search across subject and body
-obk db gmail "SELECT date, from_addr, subject, substr(body, 1, 200) FROM gmail_emails WHERE LOWER(subject) LIKE '%term%' OR LOWER(body) LIKE '%term%' ORDER BY date DESC LIMIT 10;"
-
-# Read full email
-obk db gmail "SELECT from_addr, to_addr, subject, date, body FROM gmail_emails WHERE id = <id>;"
-
-# Emails with attachments
-obk db gmail "SELECT e.date, e.from_addr, e.subject, a.filename, a.mime_type FROM gmail_emails e JOIN gmail_attachments a ON a.email_id = e.id ORDER BY e.date DESC LIMIT 20;"
-
-# Count by account
-obk db gmail "SELECT account, COUNT(*) FROM gmail_emails GROUP BY account;"
-```
+Read the REFERENCE.md in this skill's directory for the full schema and query patterns.