stage 5 done

icmoore · icmoore · commit 59b78a92cc2e · 2025-09-16T16:55:46.000-07:00
diff --git a/.DS_Store b/.DS_Store
diff --git a/.artifacts/defi_smoke_results.jsonl b/.artifacts/defi_smoke_results.jsonl
@@ -0,0 +1,3 @@
+{"input": {"prompt": "swap 2 ETH to USDC", "policy": {"mapper": {"confidence_threshold": 0.7}}, "expect_ok": true, "expect_label": "swap_assets"}, "output": {"ok": true, "label": "swap_assets", "score": 0.72, "reason": "shim:accept:stage-4", "artifacts": {"mapper": {"score": 0.72, "reason": "heuristic:swap", "aux": {"reason": "heuristic:swap"}}, "verify": {"ok": true, "reason": "shim:accept:stage-4"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": true, "label": "swap_assets", "expect_ok": true, "ok_match": true, "expect_label": "swap_assets", "label_match": true}}
+{"input": {"prompt": "deposit 10 ETH into aave", "context": {"oracle": {"age_sec": 5, "max_age_sec": 30}}, "policy": {"ltv_max": 0.75, "mapper": {"confidence_threshold": 0.7}}, "expect_ok": true, "expect_label": "deposit_asset"}, "output": {"ok": true, "label": "deposit_asset", "score": 0.71, "reason": "shim:accept:stage-4", "artifacts": {"mapper": {"score": 0.71, "reason": "heuristic:deposit", "aux": {"reason": "heuristic:deposit"}}, "verify": {"ok": true, "reason": "shim:accept:stage-4"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": true, "label": "deposit_asset", "expect_ok": true, "ok_match": true, "expect_label": "deposit_asset", "label_match": true}}
+{"input": {"prompt": "withdraw 5 ETH", "context": {"oracle": {"age_sec": 5, "max_age_sec": 30}, "ltv": 0.8}, "policy": {"ltv_max": 0.75, "mapper": {"confidence_threshold": 0.7}}, "expect_ok": false, "expect_label": "withdraw_asset"}, "output": {"ok": false, "label": "withdraw_asset", "score": 0.73, "reason": "ltv", "artifacts": {"mapper": {"score": 0.73, "reason": "heuristic:withdraw", "aux": {"reason": "heuristic:withdraw"}}, "verify": {"ok": false, "reason": "ltv"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": false, "label": "withdraw_asset", "expect_ok": false, "ok_match": true, "expect_label": "withdraw_asset", "label_match": true}}
diff --git a/.artifacts/defi_smoke_summary.json b/.artifacts/defi_smoke_summary.json
@@ -0,0 +1 @@
+{"total": 3, "ok": 2, "ok_acc": 0.6666666666666666, "label_acc": 1.0, "expect_ok_acc": 1.0, "exact_acc": 1.0, "sec": 8.687548875808716, "confusion": [{"pred": "deposit_asset", "expect": "deposit_asset", "count": 1}, {"pred": "swap_assets", "expect": "swap_assets", "count": 1}, {"pred": "withdraw_asset", "expect": "withdraw_asset", "count": 1}], "denominators": {"label": 3, "expect_ok": 3, "exact": 3}}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,33 +1,20 @@
-name: CI
-on:
-  push: { branches: ["**"] }
-  pull_request:
-
+name: ci
+on: [push, pull_request]
 jobs:
-  build-test:
+  build-test-bench:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ "3.10", "3.11" ]
     steps:
       - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install tools
+      - uses: actions/setup-python@v5
+        with: { python-version: "3.11" }
+      - run: pip install -e .  # keep deps light; add extras if needed
+      - name: Unit tests
         run: |
-          python -m pip install --upgrade pip
-          pip install ruff pytest
-
-      - name: Lint (ruff)
-        run: ruff . --output-format=github || (echo "::warning::Ruff issues found"; exit 1)
-
-      - name: Milestones freeze check
-        run: .github/workflows/check_freeze.sh
-
-      - name: Tests (unit placeholder)
-        run: pytest -q
-
+          PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -q tests/unit
+      - name: Bench (local-only rails)
+        run: |
+          MICROLM_DISABLE_RAILS=1 micro-lm-bench defi \
+            --file benches/defi_smoke.jsonl \
+            --out .artifacts/defi_smoke_results.jsonl \
+            --summary-out .artifacts/defi_smoke_summary.json \
+            --gate-metric exact_acc --gate-min 0.66
diff --git a/.ipynb_checkpoints/pyproject-checkpoint.toml b/.ipynb_checkpoints/pyproject-checkpoint.toml
@@ -14,4 +14,16 @@ where = ["src"]   # auto-discovers micro_lm, micro_lm.core, etc.
 package-dir = {"" = "src"}
 
 [project.scripts]
-micro-lm = "micro_lm.cli:main"
+micro-lm = "micro_lm.cli.entry:main"
+micro-lm-bench = "micro_lm.bench_cli:main"
+
+[project.optional-dependencies]
+sbert = [
+  "sentence-transformers>=2.7,<3",
+  "scikit-learn>=1.3,<2",
+  "joblib>=1.3,<2",
+]
+dev = [
+  "pytest",
+  "ruff",
+]
diff --git a/benches/.ipynb_checkpoints/defi_smoke-checkpoint.jsonl b/benches/.ipynb_checkpoints/defi_smoke-checkpoint.jsonl
@@ -0,0 +1,3 @@
+{"prompt":"swap 2 ETH to USDC","policy":{"mapper":{"confidence_threshold":0.7}}}
+{"prompt":"deposit 10 ETH into aave","context":{"oracle":{"age_sec":5,"max_age_sec":30}},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}}}
+{"prompt":"withdraw 5 ETH","context":{"oracle":{"age_sec":5,"max_age_sec":30},"ltv":0.80},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}}}
diff --git a/benches/defi_smoke.jsonl b/benches/defi_smoke.jsonl
@@ -0,0 +1,4 @@
+{"prompt":"swap 2 ETH to USDC","policy":{"mapper":{"confidence_threshold":0.7}},"expect_ok":true,"expect_label":"swap_assets"}
+{"prompt":"deposit 10 ETH into aave","context":{"oracle":{"age_sec":5,"max_age_sec":30}},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}},"expect_ok":true,"expect_label":"deposit_asset"}
+{"prompt":"withdraw 5 ETH","context":{"oracle":{"age_sec":5,"max_age_sec":30},"ltv":0.80},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}},"expect_ok":false,"expect_label":"withdraw_asset"}
+
diff --git a/pyproject.toml b/pyproject.toml
@@ -14,4 +14,16 @@ where = ["src"]   # auto-discovers micro_lm, micro_lm.core, etc.
 package-dir = {"" = "src"}
 
 [project.scripts]
-micro-lm = "micro_lm.cli:main"
+micro-lm = "micro_lm.cli.entry:main"
+micro-lm-bench = "micro_lm.bench_cli:main"
+
+[project.optional-dependencies]
+sbert = [
+  "sentence-transformers>=2.7,<3",
+  "scikit-learn>=1.3,<2",
+  "joblib>=1.3,<2",
+]
+dev = [
+  "pytest",
+  "ruff",
+]
diff --git a/src/micro_lm.egg-info/PKG-INFO b/src/micro_lm.egg-info/PKG-INFO
@@ -3,4 +3,11 @@ Name: micro-lm
 Version: 0.0.0a0
 Requires-Python: >=3.10
 License-File: LICENSE
+Provides-Extra: sbert
+Requires-Dist: sentence-transformers<3,>=2.7; extra == "sbert"
+Requires-Dist: scikit-learn<2,>=1.3; extra == "sbert"
+Requires-Dist: joblib<2,>=1.3; extra == "sbert"
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
 Dynamic: license-file
diff --git a/src/micro_lm.egg-info/SOURCES.txt b/src/micro_lm.egg-info/SOURCES.txt
@@ -2,10 +2,13 @@ LICENSE
 README.md
 pyproject.toml
 src/micro_lm/__init__.py
+src/micro_lm/bench_cli.py
+src/micro_lm/config.py
 src/micro_lm.egg-info/PKG-INFO
 src/micro_lm.egg-info/SOURCES.txt
 src/micro_lm.egg-info/dependency_links.txt
 src/micro_lm.egg-info/entry_points.txt
+src/micro_lm.egg-info/requires.txt
 src/micro_lm.egg-info/top_level.txt
 src/micro_lm/cli/__init__.py
 src/micro_lm/cli/__main__.py
diff --git a/src/micro_lm.egg-info/entry_points.txt b/src/micro_lm.egg-info/entry_points.txt
@@ -1,2 +1,3 @@
 [console_scripts]
-micro-lm = micro_lm.cli:main
+micro-lm = micro_lm.cli.entry:main
+micro-lm-bench = micro_lm.bench_cli:main
diff --git a/src/micro_lm.egg-info/requires.txt b/src/micro_lm.egg-info/requires.txt
@@ -0,0 +1,9 @@
+
+[dev]
+pytest
+ruff
+
+[sbert]
+sentence-transformers<3,>=2.7
+scikit-learn<2,>=1.3
+joblib<2,>=1.3
diff --git a/src/micro_lm/.ipynb_checkpoints/bench_cli-checkpoint.py b/src/micro_lm/.ipynb_checkpoints/bench_cli-checkpoint.py
@@ -0,0 +1,162 @@
+import argparse, json, sys, time, os
+from typing import List, Dict, Any, Tuple
+from collections import Counter
+from micro_lm.core.runner import run_micro
+
+
+def _read_lines(path: str) -> List[Dict[str, Any]]:
+    rows = []
+    with open(path, "r") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            rows.append(json.loads(line))
+    return rows
+
+
+def _safe_bool(x: Any, default=None):
+    if x is None:
+        return default
+    if isinstance(x, bool):
+        return x
+    if isinstance(x, (int, float)):
+        return bool(x)
+    if isinstance(x, str):
+        t = x.strip().lower()
+        if t in {"1", "true", "yes", "y"}:
+            return True
+        if t in {"0", "false", "no", "n"}:
+            return False
+    return default
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser("micro-lm-bench")
+    ap.add_argument("domain", choices=["defi", "arc"])
+    ap.add_argument("--file", required=True, help="JSONL with rows: {prompt, context?, policy?, backend?, rails?, T?, expect_ok?, expect_label?}")
+    ap.add_argument("--out", default=".artifacts/bench_results.jsonl")
+    ap.add_argument("--summary-out", default=None, help="Optional path to write the summary JSON")
+    ap.add_argument("--default-backend", default="sbert")
+    ap.add_argument("--default-rails", default="stage11")
+    ap.add_argument("--default-T", type=int, default=180)
+    ap.add_argument("--gate-metric", default=None, choices=["ok_acc","label_acc","expect_ok_acc","exact_acc"], help="Optional metric to gate on")
+    ap.add_argument("--gate-min", type=float, default=None, help="Minimum value for gate metric, e.g. 0.66")
+    args = ap.parse_args(argv)
+
+    rows = _read_lines(args.file)
+
+    # tallies
+    total = 0
+    ok_cnt = 0
+    label_hits = 0
+    label_den = 0
+    expect_ok_hits = 0
+    expect_ok_den = 0
+    exact_hits = 0
+    exact_den = 0
+    conf = Counter()  # (pred_label, expect_label) -> count
+
+    t0 = time.time()
+    os.makedirs(os.path.dirname(args.out), exist_ok=True)
+    with open(args.out, "w") as w:
+        for r in rows:
+            total += 1
+            out = run_micro(
+                args.domain,
+                r["prompt"],
+                context=r.get("context", {}),
+                policy=r.get("policy", {}),
+                rails=r.get("rails", args.default_rails),
+                T=int(r.get("T", args.default_T)),
+                backend=r.get("backend", args.default_backend),
+            )
+
+            # base ok
+            ok = bool(out.get("ok"))
+            ok_cnt += 1 if ok else 0
+
+            # optional expectations
+            exp_ok = _safe_bool(r.get("expect_ok"), default=None)
+            exp_label = r.get("expect_label")
+
+            # label accuracy (only if expect_label provided)
+            pred_label = out.get("label")
+            if exp_label is not None:
+                label_den += 1
+                if pred_label == exp_label:
+                    label_hits += 1
+                conf[(str(pred_label), str(exp_label))] += 1
+
+            # expected-ok accuracy (only if expect_ok provided)
+            if exp_ok is not None:
+                expect_ok_den += 1
+                if ok is exp_ok:
+                    expect_ok_hits += 1
+
+            # exact match (need both)
+            if (exp_ok is not None) and (exp_label is not None):
+                exact_den += 1
+                if (ok is exp_ok) and (pred_label == exp_label):
+                    exact_hits += 1
+
+            # Write per-row result with evaluation fields (where applicable)
+            eval_fields = {
+                "ok": ok,
+                "label": pred_label,
+            }
+            if exp_ok is not None:
+                eval_fields["expect_ok"] = exp_ok
+                eval_fields["ok_match"] = (ok is exp_ok)
+            if exp_label is not None:
+                eval_fields["expect_label"] = exp_label
+                eval_fields["label_match"] = (pred_label == exp_label)
+
+            w.write(json.dumps({"input": r, "output": out, "eval": eval_fields}) + "\n")
+
+    dt = time.time() - t0
+
+    # compute metrics
+    ok_acc = ok_cnt / total if total else 0.0
+    label_acc = label_hits / label_den if label_den else None
+    expect_ok_acc = expect_ok_hits / expect_ok_den if expect_ok_den else None
+    exact_acc = exact_hits / exact_den if exact_den else None
+
+    # tidy confusion
+    confusion = [{"pred": p, "expect": e, "count": c} for (p, e), c in sorted(conf.items(), key=lambda kv: (-kv[1], kv[0][0], kv[0][1]))]
+
+    summary = {
+        "total": total,
+        "ok": ok_cnt,
+        "ok_acc": ok_acc,
+        "label_acc": label_acc,
+        "expect_ok_acc": expect_ok_acc,
+        "exact_acc": exact_acc,
+        "sec": dt,
+        "confusion": confusion,
+        "denominators": {"label": label_den, "expect_ok": expect_ok_den, "exact": exact_den},
+    }
+
+    # print summary and optionally write it
+    print(json.dumps(summary, indent=2))
+    if args.summary_out:
+        os.makedirs(os.path.dirname(args.summary_out), exist_ok=True)
+        with open(args.summary_out, "w") as f:
+            json.dump(summary, f)
+
+    # optional gate
+    if args.gate_metric and args.gate_min is not None:
+        val = summary.get(args.gate_metric)
+        # if metric is None (no denominator), treat as fail
+        ok_gate = (val is not None) and (float(val) >= float(args.gate_min))
+        if not ok_gate:
+            print(json.dumps({"gate_metric": args.gate_metric, "value": val, "min": args.gate_min, "result": "FAIL"}))
+            sys.exit(1)
+        else:
+            print(json.dumps({"gate_metric": args.gate_metric, "value": val, "min": args.gate_min, "result": "PASS"}))
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/src/micro_lm/.ipynb_checkpoints/config-checkpoint.py b/src/micro_lm/.ipynb_checkpoints/config-checkpoint.py
@@ -0,0 +1,21 @@
+import json, os
+from typing import Any
+
+def _load_json_or_yaml(path: str) -> Any:
+    if not os.path.exists(path):
+        return {}
+    if path.endswith((".yml", ".yaml")):
+        import yaml  # type: ignore
+        with open(path, "r") as f:
+            return yaml.safe_load(f) or {}
+    with open(path, "r") as f:
+        return json.load(f)
+
+def discover_context(default="{}"):
+    # prefer explicit env, then file, then CLI default
+    fn = os.getenv("MICROLM_CONTEXT", "configs/context.yml")
+    return _load_json_or_yaml(fn) or json.loads(default)
+
+def discover_policy(default="{}"):
+    fn = os.getenv("MICROLM_POLICY", "configs/policy.yml")
+    return _load_json_or_yaml(fn) or json.loads(default)
diff --git a/src/micro_lm/.ipynb_checkpoints/pyproject-checkpoint.toml b/src/micro_lm/.ipynb_checkpoints/pyproject-checkpoint.toml
@@ -0,0 +1,17 @@
+[build-system]
+requires = ["setuptools>=68", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "micro-lm"
+version = "0.0.0a0"
+requires-python = ">=3.10"
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[project.scripts]
+micro-lm = "micro_lm.cli.entry:main"
diff --git a/src/micro_lm/__pycache__/bench_cli.cpython-310.pyc b/src/micro_lm/__pycache__/bench_cli.cpython-310.pyc
diff --git a/src/micro_lm/bench_cli.py b/src/micro_lm/bench_cli.py
diff --git a/src/micro_lm/config.py b/src/micro_lm/config.py
diff --git a/tests/.DS_Store b/tests/.DS_Store

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	+{"input": {"prompt": "swap 2 ETH to USDC", "policy": {"mapper": {"confidence_threshold": 0.7}}, "expect_ok": true, "expect_label": "swap_assets"}, "output": {"ok": true, "label": "swap_assets", "score": 0.72, "reason": "shim:accept:stage-4", "artifacts": {"mapper": {"score": 0.72, "reason": "heuristic:swap", "aux": {"reason": "heuristic:swap"}}, "verify": {"ok": true, "reason": "shim:accept:stage-4"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": true, "label": "swap_assets", "expect_ok": true, "ok_match": true, "expect_label": "swap_assets", "label_match": true}}
	`2`	+{"input": {"prompt": "deposit 10 ETH into aave", "context": {"oracle": {"age_sec": 5, "max_age_sec": 30}}, "policy": {"ltv_max": 0.75, "mapper": {"confidence_threshold": 0.7}}, "expect_ok": true, "expect_label": "deposit_asset"}, "output": {"ok": true, "label": "deposit_asset", "score": 0.71, "reason": "shim:accept:stage-4", "artifacts": {"mapper": {"score": 0.71, "reason": "heuristic:deposit", "aux": {"reason": "heuristic:deposit"}}, "verify": {"ok": true, "reason": "shim:accept:stage-4"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": true, "label": "deposit_asset", "expect_ok": true, "ok_match": true, "expect_label": "deposit_asset", "label_match": true}}
	`3`	+{"input": {"prompt": "withdraw 5 ETH", "context": {"oracle": {"age_sec": 5, "max_age_sec": 30}, "ltv": 0.8}, "policy": {"ltv_max": 0.75, "mapper": {"confidence_threshold": 0.7}}, "expect_ok": false, "expect_label": "withdraw_asset"}, "output": {"ok": false, "label": "withdraw_asset", "score": 0.73, "reason": "ltv", "artifacts": {"mapper": {"score": 0.73, "reason": "heuristic:withdraw", "aux": {"reason": "heuristic:withdraw"}}, "verify": {"ok": false, "reason": "ltv"}, "schema": {"v": 1, "keys": ["mapper", "verify"]}}}, "eval": {"ok": false, "label": "withdraw_asset", "expect_ok": false, "ok_match": true, "expect_label": "withdraw_asset", "label_match": true}}
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"total": 3, "ok": 2, "ok_acc": 0.6666666666666666, "label_acc": 1.0, "expect_ok_acc": 1.0, "exact_acc": 1.0, "sec": 8.687548875808716, "confusion": [{"pred": "deposit_asset", "expect": "deposit_asset", "count": 1}, {"pred": "swap_assets", "expect": "swap_assets", "count": 1}, {"pred": "withdraw_asset", "expect": "withdraw_asset", "count": 1}], "denominators": {"label": 3, "expect_ok": 3, "exact": 3}}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{"prompt":"swap 2 ETH to USDC","policy":{"mapper":{"confidence_threshold":0.7}}}`
	`2`	`+{"prompt":"deposit 10 ETH into aave","context":{"oracle":{"age_sec":5,"max_age_sec":30}},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}}}`
	`3`	`+{"prompt":"withdraw 5 ETH","context":{"oracle":{"age_sec":5,"max_age_sec":30},"ltv":0.80},"policy":{"ltv_max":0.75,"mapper":{"confidence_threshold":0.7}}}`