Skip to content

Commit c14d059

Browse files
authored
Merge pull request #16 from dmoliveira/feat/selftest-automation
Add deterministic self-test automation
2 parents 6f35047 + 63a7a96 commit c14d059

File tree

5 files changed

+154
-1
lines changed

5 files changed

+154
-1
lines changed

.github/workflows/ci.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ jobs:
2222
python -m py_compile scripts/*.py
2323
python -m json.tool opencode.json > /dev/null
2424
25+
- name: Run deterministic self-tests
26+
run: |
27+
make selftest
28+
2529
- name: Installer smoke test
2630
run: |
2731
TMP_HOME="$(mktemp -d)"

CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,14 @@
22

33
All notable changes to this project are documented in this file.
44

5+
## Unreleased
6+
7+
### Adds
8+
- Added deterministic `scripts/selftest.py` to verify MCP/plugin command behavior with isolated config and environment.
9+
10+
### Changes
11+
- Added `make selftest` and wired it into CI for regression coverage.
12+
513
## v0.1.1 - 2026-02-12
614

715
### Adds

Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
.DEFAULT_GOAL := help
22

3-
.PHONY: help validate doctor doctor-json install-test release
3+
.PHONY: help validate selftest doctor doctor-json install-test release
44

55
help: ## Show available targets
66
@awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*##/ {printf "%-14s %s\n", $$1, $$2}' $(MAKEFILE_LIST)
@@ -9,6 +9,9 @@ validate: ## Validate scripts and JSON config
99
python3 -m py_compile scripts/*.py
1010
python3 -m json.tool opencode.json >/dev/null
1111

12+
selftest: ## Run deterministic command self-tests
13+
python3 scripts/selftest.py
14+
1215
doctor: ## Run plugin diagnostics (human-readable)
1316
python3 scripts/plugin_command.py doctor
1417

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,7 @@ For WakaTime, configure `~/.wakatime.cfg` with your `api_key` before enabling `w
170170
```bash
171171
make help
172172
make validate
173+
make selftest
173174
make doctor
174175
make doctor-json
175176
make install-test

scripts/selftest.py

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
#!/usr/bin/env python3
2+
3+
import json
4+
import os
5+
import shutil
6+
import subprocess
7+
import sys
8+
import tempfile
9+
from pathlib import Path
10+
11+
12+
REPO_ROOT = Path(__file__).resolve().parents[1]
13+
PLUGIN_SCRIPT = REPO_ROOT / "scripts" / "plugin_command.py"
14+
MCP_SCRIPT = REPO_ROOT / "scripts" / "mcp_command.py"
15+
BASE_CONFIG = REPO_ROOT / "opencode.json"
16+
17+
18+
def run_script(
19+
script: Path, cfg: Path, home: Path, *args: str
20+
) -> subprocess.CompletedProcess[str]:
21+
env = os.environ.copy()
22+
env["OPENCODE_CONFIG_PATH"] = str(cfg)
23+
env["HOME"] = str(home)
24+
env.pop("SUPERMEMORY_API_KEY", None)
25+
env.pop("MORPH_API_KEY", None)
26+
return subprocess.run(
27+
[sys.executable, str(script), *args],
28+
capture_output=True,
29+
text=True,
30+
env=env,
31+
check=False,
32+
)
33+
34+
35+
def expect(condition: bool, message: str) -> None:
36+
if not condition:
37+
raise AssertionError(message)
38+
39+
40+
def parse_json_output(text: str) -> dict:
41+
return json.loads(text)
42+
43+
44+
def main() -> int:
45+
with tempfile.TemporaryDirectory() as tmpdir:
46+
tmp = Path(tmpdir)
47+
home = tmp / "home"
48+
home.mkdir(parents=True, exist_ok=True)
49+
cfg = tmp / "opencode.json"
50+
shutil.copy2(BASE_CONFIG, cfg)
51+
52+
# Plugin profile lean should pass doctor.
53+
result = run_script(PLUGIN_SCRIPT, cfg, home, "profile", "lean")
54+
expect(result.returncode == 0, f"plugin profile lean failed: {result.stderr}")
55+
56+
result = run_script(PLUGIN_SCRIPT, cfg, home, "doctor", "--json")
57+
expect(
58+
result.returncode == 0,
59+
f"plugin doctor --json (lean) failed: {result.stderr}",
60+
)
61+
report = parse_json_output(result.stdout)
62+
expect(
63+
report.get("result") == "PASS", "plugin doctor should pass for lean profile"
64+
)
65+
66+
# Plugin stable should fail doctor without keys in isolated HOME.
67+
result = run_script(PLUGIN_SCRIPT, cfg, home, "profile", "stable")
68+
expect(result.returncode == 0, f"plugin profile stable failed: {result.stderr}")
69+
70+
result = run_script(PLUGIN_SCRIPT, cfg, home, "doctor", "--json")
71+
expect(
72+
result.returncode == 1,
73+
"plugin doctor stable should fail when keys are absent",
74+
)
75+
report = parse_json_output(result.stdout)
76+
problems = "\n".join(report.get("problems", []))
77+
expect("supermemory enabled" in problems, "expected supermemory key problem")
78+
expect("wakatime enabled" in problems, "expected wakatime key problem")
79+
80+
result = run_script(PLUGIN_SCRIPT, cfg, home, "setup-keys")
81+
expect(result.returncode == 0, f"plugin setup-keys failed: {result.stderr}")
82+
expect(
83+
"[supermemory]" in result.stdout, "setup-keys missing supermemory section"
84+
)
85+
expect("[wakatime]" in result.stdout, "setup-keys missing wakatime section")
86+
87+
# MCP minimal should pass with disabled warning.
88+
result = run_script(MCP_SCRIPT, cfg, home, "profile", "minimal")
89+
expect(result.returncode == 0, f"mcp profile minimal failed: {result.stderr}")
90+
91+
result = run_script(MCP_SCRIPT, cfg, home, "doctor", "--json")
92+
expect(
93+
result.returncode == 0,
94+
f"mcp doctor --json (minimal) failed: {result.stderr}",
95+
)
96+
report = parse_json_output(result.stdout)
97+
expect(
98+
report.get("result") == "PASS", "mcp doctor should pass for minimal profile"
99+
)
100+
warnings = "\n".join(report.get("warnings", []))
101+
expect(
102+
"all MCP servers are disabled" in warnings, "expected disabled MCP warning"
103+
)
104+
105+
# MCP research should enable both servers and pass.
106+
result = run_script(MCP_SCRIPT, cfg, home, "profile", "research")
107+
expect(result.returncode == 0, f"mcp profile research failed: {result.stderr}")
108+
109+
result = run_script(MCP_SCRIPT, cfg, home, "doctor", "--json")
110+
expect(
111+
result.returncode == 0,
112+
f"mcp doctor --json (research) failed: {result.stderr}",
113+
)
114+
report = parse_json_output(result.stdout)
115+
expect(
116+
report.get("result") == "PASS",
117+
"mcp doctor should pass for research profile",
118+
)
119+
expect(
120+
report.get("servers", {}).get("context7", {}).get("status") == "enabled",
121+
"context7 should be enabled in research profile",
122+
)
123+
expect(
124+
report.get("servers", {}).get("gh_grep", {}).get("status") == "enabled",
125+
"gh_grep should be enabled in research profile",
126+
)
127+
128+
print("selftest: PASS")
129+
return 0
130+
131+
132+
if __name__ == "__main__":
133+
try:
134+
raise SystemExit(main())
135+
except AssertionError as exc:
136+
print(f"selftest: FAIL - {exc}")
137+
raise SystemExit(1)

0 commit comments

Comments
 (0)