Skip to content

Commit 2e145ed

Browse files
tom-sapletta-comtom-sapletta-com
committed
refactoring
1 parent 66fdf3d commit 2e145ed

File tree

11 files changed

+6262
-14382
lines changed

11 files changed

+6262
-14382
lines changed

‎Makefile‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
.PHONY: help install install-dev install-full clean build test lint format typecheck publish publish-test docs docker
2+
.PHONY: benchmark benchmark-format benchmark-function benchmark-project benchmark-token benchmark-compare benchmark-toon examples
23

34
POETRY := $(shell command -v poetry 2>/dev/null)
45
ifeq ($(POETRY),)
@@ -298,6 +299,149 @@ run-compare: ## Compare sizes of all formats
298299
status: ## Show library status
299300
code2logic --status
300301

302+
# ============================================================================
303+
# Benchmarks
304+
# ============================================================================
305+
306+
BENCH_SAMPLES := tests/samples
307+
BENCH_OUTPUT := examples/output
308+
BENCH_LIMIT := 20
309+
BENCH_FORMATS := yaml toon logicml json
310+
311+
benchmark: benchmark-format benchmark-function benchmark-token benchmark-project benchmark-toon benchmark-compare ## Run all benchmarks (no LLM)
312+
@echo ""
313+
@echo "$(GREEN)All benchmarks completed!$(NC)"
314+
@echo "Results in $(BENCH_OUTPUT)/"
315+
@ls -lhS $(BENCH_OUTPUT)/*.json 2>/dev/null
316+
317+
benchmark-format: ## Benchmark format reproduction (yaml/toon/logicml/json)
318+
@echo "$(BLUE)━━━ Format Benchmark ━━━$(NC)"
319+
$(PYTHON) examples/15_unified_benchmark.py \
320+
--no-llm --type format \
321+
--folder $(BENCH_SAMPLES)/ \
322+
--formats $(BENCH_FORMATS) \
323+
--limit $(BENCH_LIMIT) --verbose \
324+
--output $(BENCH_OUTPUT)/benchmark_format.json
325+
326+
benchmark-function: ## Benchmark function-level reproduction
327+
@echo "$(BLUE)━━━ Function Benchmark ━━━$(NC)"
328+
$(PYTHON) examples/15_unified_benchmark.py \
329+
--no-llm --type function \
330+
--file $(BENCH_SAMPLES)/sample_functions.py \
331+
--limit 10 --verbose \
332+
--output $(BENCH_OUTPUT)/benchmark_function.json
333+
334+
benchmark-token: ## Benchmark token efficiency across formats
335+
@echo "$(BLUE)━━━ Token Efficiency Benchmark ━━━$(NC)"
336+
$(PYTHON) examples/11_token_benchmark.py \
337+
--no-llm \
338+
--folder $(BENCH_SAMPLES)/ \
339+
--formats $(BENCH_FORMATS) \
340+
--limit $(BENCH_LIMIT) --verbose \
341+
--output $(BENCH_OUTPUT)/benchmark_token.json
342+
343+
benchmark-project: ## Benchmark project-level reproduction
344+
@echo "$(BLUE)━━━ Project Benchmark ━━━$(NC)"
345+
$(PYTHON) examples/15_unified_benchmark.py \
346+
--no-llm --type project \
347+
--folder $(BENCH_SAMPLES)/ \
348+
--formats $(BENCH_FORMATS) \
349+
--limit $(BENCH_LIMIT) --verbose \
350+
--output $(BENCH_OUTPUT)/benchmark_project.json
351+
352+
benchmark-toon: ## Generate TOON + function-logic for self-analysis
353+
@echo "$(BLUE)━━━ TOON Self-Analysis ━━━$(NC)"
354+
@mkdir -p $(BENCH_OUTPUT)
355+
$(PYTHON) -m code2logic ./ -f toon --compact --name project -o ./
356+
$(PYTHON) -m code2logic ./ -f toon --compact --no-repeat-module --function-logic --with-schema --name project -o ./
357+
$(PYTHON) -m code2logic ./ -f yaml --compact --name project -o $(BENCH_OUTPUT)/
358+
$(PYTHON) -m code2logic ./ -f json --name project -o $(BENCH_OUTPUT)/
359+
$(PYTHON) -m code2logic ./ -f markdown --name project -o $(BENCH_OUTPUT)/
360+
$(PYTHON) -m code2logic ./ -f compact --name project -o $(BENCH_OUTPUT)/
361+
$(PYTHON) -m code2logic ./ -f csv -d standard --name project -o $(BENCH_OUTPUT)/
362+
@echo ""
363+
@echo "$(BLUE)Format size comparison (self-analysis):$(NC)"
364+
@printf " %-25s %10s %10s\n" "Format" "Size" "~Tokens"
365+
@printf " %-25s %10s %10s\n" "-------------------------" "----------" "----------"
366+
@for f in project.toon project.functions.toon $(BENCH_OUTPUT)/project.yaml $(BENCH_OUTPUT)/project.json $(BENCH_OUTPUT)/project.md $(BENCH_OUTPUT)/project.txt $(BENCH_OUTPUT)/project.csv; do \
367+
if [ -f "$$f" ]; then \
368+
sz=$$(wc -c < "$$f"); \
369+
tok=$$((sz / 4)); \
370+
printf " %-25s %8s B %8s\n" "$$(basename $$f)" "$$sz" "$$tok"; \
371+
fi; \
372+
done
373+
@echo ""
374+
@echo "$(GREEN)TOON files:$(NC)"
375+
@ls -lh project.toon project.functions.toon project.toon-schema.json project.functions-schema.json 2>/dev/null
376+
377+
benchmark-compare: ## Show summary comparison of all benchmark results
378+
@echo ""
379+
@echo "$(BLUE)━━━ Benchmark Summary ━━━$(NC)"
380+
@$(PYTHON) examples/benchmark_summary.py $(BENCH_OUTPUT)
381+
382+
# ============================================================================
383+
# Examples (step by step)
384+
# ============================================================================
385+
386+
examples: ## Run all examples step by step (no LLM required)
387+
@mkdir -p $(BENCH_OUTPUT)
388+
@echo "$(BLUE)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$(NC)"
389+
@echo "$(BLUE) Running all code2logic examples (--no-llm where needed)$(NC)"
390+
@echo "$(BLUE)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$(NC)"
391+
@echo ""
392+
@echo "$(YELLOW)[1/16] Quick Start — basic analysis$(NC)"
393+
$(PYTHON) examples/01_quick_start.py
394+
@echo ""
395+
@echo "$(YELLOW)[2/16] Refactoring — suggest improvements$(NC)"
396+
$(PYTHON) examples/02_refactoring.py
397+
@echo ""
398+
@echo "$(YELLOW)[3/16] Reproduction — code ↔ spec round-trip$(NC)"
399+
$(PYTHON) examples/03_reproduction.py --no-llm
400+
@echo ""
401+
@echo "$(YELLOW)[4/16] Project Analysis — multi-file$(NC)"
402+
$(PYTHON) examples/04_project.py --no-llm
403+
@echo ""
404+
@echo "$(YELLOW)[5/16] LLM Integration — provider detection$(NC)"
405+
$(PYTHON) examples/05_llm_integration.py --no-llm
406+
@echo ""
407+
@echo "$(YELLOW)[6/16] Metrics — reproduction quality$(NC)"
408+
$(PYTHON) examples/06_metrics.py --no-llm
409+
@echo ""
410+
@echo "$(YELLOW)[7/16] Format Benchmark — compare formats$(NC)"
411+
$(PYTHON) examples/08_format_benchmark.py --no-llm
412+
@echo ""
413+
@echo "$(YELLOW)[8/16] Async Benchmark — async code analysis$(NC)"
414+
$(PYTHON) examples/09_async_benchmark.py --no-llm
415+
@echo ""
416+
@echo "$(YELLOW)[9/16] Function Reproduction — per-function$(NC)"
417+
$(PYTHON) examples/10_function_reproduction.py --no-llm
418+
@echo ""
419+
@echo "$(YELLOW)[10/16] Token Benchmark — token efficiency$(NC)"
420+
$(PYTHON) examples/11_token_benchmark.py --no-llm
421+
@echo ""
422+
@echo "$(YELLOW)[11/16] Comprehensive Analysis — full pipeline$(NC)"
423+
$(PYTHON) examples/12_comprehensive_analysis.py --no-llm --limit 3
424+
@echo ""
425+
@echo "$(YELLOW)[12/16] Project Benchmark — project-level scores$(NC)"
426+
$(PYTHON) examples/13_project_benchmark.py --no-llm
427+
@echo ""
428+
@echo "$(YELLOW)[13/16] Repeatability Test — determinism check$(NC)"
429+
$(PYTHON) examples/14_repeatability_test.py --no-llm --file tests/samples/sample_functions.py --runs 2
430+
@echo ""
431+
@echo "$(YELLOW)[14/16] Unified Benchmark — all-in-one$(NC)"
432+
$(PYTHON) examples/15_unified_benchmark.py --no-llm --verbose
433+
@echo ""
434+
@echo "$(YELLOW)[15/16] Terminal Demo — rich output$(NC)"
435+
$(PYTHON) examples/16_terminal_demo.py
436+
@echo ""
437+
@echo "$(YELLOW)[16/16] Duplicate Detection$(NC)"
438+
$(PYTHON) examples/duplicate_detection.py
439+
@echo ""
440+
@echo "$(GREEN)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$(NC)"
441+
@echo "$(GREEN) All 16 examples completed successfully!$(NC)"
442+
@echo "$(GREEN)━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$(NC)"
443+
@echo " Output: $(BENCH_OUTPUT)/"
444+
301445
# ============================================================================
302446
# LLM Integration
303447
# ============================================================================

‎README.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,9 @@ pip install code2logic[nlp] # Enhanced intents
4949
## 📖 Quick Start
5050
```bash
5151
code2logic ./ -f yaml --compact --function-logic --with-schema -o project.yaml
52-
code2logic ./ -f toon --ultra-compact --function-logic --with-schema -o project.toon
52+
code2logic ./ -f toon --function-logic --with-schema --name project -o ./
53+
# Optional: include function intent/purpose column in project.functions.toon
54+
code2logic ./ -f toon --function-logic --does --name project -o ./
5355
```
5456

5557
### Command Line

‎code2logic/cli.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -668,6 +668,11 @@ def _maybe_print_pretty_help() -> bool:
668668
action='store_true',
669669
help='Reduce repeated directory prefixes in function-logic TOON section function_details by using ./file for consecutive entries in the same folder.'
670670
)
671+
parser.add_argument(
672+
'--does',
673+
action='store_true',
674+
help='Include the does/intent column in function-logic TOON output. Without this flag, the does column is omitted to save tokens.'
675+
)
671676
parser.add_argument(
672677
'--no-install',
673678
action='store_true',
@@ -1038,6 +1043,7 @@ def _maybe_print_pretty_help() -> bool:
10381043
detail=args.detail,
10391044
no_repeat_name=args.no_repeat_module,
10401045
no_repeat_details=args.no_repeat_details,
1046+
include_does=args.does,
10411047
)
10421048
else:
10431049
logic_out = logic_gen.generate(project, detail=args.detail)

‎code2logic/function_logic.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def generate_toon(
6565
detail: str = 'full',
6666
no_repeat_name: bool = False,
6767
no_repeat_details: bool = False,
68+
include_does: bool = False,
6869
) -> str:
6970
if detail == 'detailed':
7071
detail = 'full'
@@ -110,7 +111,7 @@ def generate_toon(
110111
lines.append(f" {toon._quote(details_key)}:")
111112

112113
header = f"line{dm}name{dm}sig"
113-
if detail in ('standard', 'full'):
114+
if include_does and detail in ('standard', 'full'):
114115
header += f"{dm}does"
115116
if detail == 'full':
116117
header += f"{dm}decorators{dm}calls{dm}raises"
@@ -135,7 +136,7 @@ def generate_toon(
135136
toon._quote(sig),
136137
]
137138

138-
if detail in ('standard', 'full'):
139+
if include_does and detail in ('standard', 'full'):
139140
does = self._build_does(func)
140141
row.append(toon._quote(does))
141142

‎code2logic/generators.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -209,7 +209,8 @@ def _gen_module(self, lines: List[str], m: ModuleInfo,
209209
lines.append(f"imports: [{imps}]")
210210

211211
if m.constants:
212-
lines.append(f"constants: [{', '.join(m.constants[:5])}]")
212+
cnames = [c.name if hasattr(c, 'name') else str(c) for c in m.constants[:5]]
213+
lines.append(f"constants: [{', '.join(cnames)}]")
213214
lines.append("```")
214215
lines.append("")
215216

‎docs/03-cli-reference.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ If `CODE2LOGIC_DEFAULT_PROVIDER=auto`, Code2Logic tries providers in priority or
4646
| `--ultra-compact` | | Use ultra-compact TOON format (71% smaller) |
4747
| `--no-repeat-module` | | Reduce repeated directory prefixes in TOON `modules[...]` tables by using `./file` for consecutive entries in the same folder |
4848
| `--no-repeat-details` | | Reduce repeated directory prefixes in function-logic TOON `function_details` section by using `./file` for consecutive entries in the same folder |
49+
| `--does` | | Include the `does` (intent/purpose) column in function-logic TOON output. Omitted by default to save tokens |
4950
| `--with-schema` | | Generate JSON schema alongside output |
5051
| `--verbose` | `-v` | Verbose output with timing |
5152
| `--debug` | | Debug output (very verbose) |
@@ -130,6 +131,12 @@ code2logic /path/to/project -f toon --ultra-compact --with-schema
130131

131132
# Generate function-logic as TOON + compress function_details module keys
132133
code2logic /path/to/project -f toon --function-logic --name project -o ./ --no-repeat-details
134+
135+
# Generate function-logic TOON with intent descriptions (does column)
136+
code2logic /path/to/project -f toon --function-logic --does --name project -o ./
137+
138+
# Generate function-logic TOON + schema (project.functions.toon + project.functions-schema.json)
139+
code2logic /path/to/project -f toon --function-logic --with-schema --name project -o ./
133140
```
134141

135142
Token-oriented object notation - most efficient format for LLM consumption.

‎docs/07-toon.md‎

Lines changed: 65 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -168,22 +168,81 @@ modules[4]{path,lang,lines,kb}:
168168
**Example (function-logic TOON, function_details):**
169169

170170
```bash
171-
code2logic /path/to/project -f toon --function-logic --name project -o ./ --no-repeat-details
171+
code2logic /path/to/project -f toon --function-logic --with-schema --name project -o ./ --no-repeat-details
172172
```
173173

174174
Output fragment:
175175

176176
```toon
177+
# myproject function-logic | 3 modules
178+
# Convention: name with . = method, ~name = async, cc:N shown only when >1
179+
project: myproject
180+
generated: "2026-02-25T09:00:00"
181+
modules[3]{path,lang,items}:
182+
firmware/main.py,py,4
183+
./test_main.py,py,15
184+
db/config.py,py,5
185+
177186
function_details:
178187
firmware/main.py:
179-
functions[2]{line,name,kind,sig,async,cc,does}:
180-
77,index_page,function,(),true,2,Serve the firmware UI
181-
85,health_check,function,(),true,1,Health check endpoint
188+
functions[4]{line,name,sig}:
189+
77,~index_page cc:2,()
190+
85,~health_check,()
191+
90,~status,()
192+
96,"~websocket_endpoint cc:5","(websocket:WebSocket)"
182193
./test_main.py:
183-
functions[1]{line,name,kind,sig,async,cc,does}:
184-
14,TestFirmwareSimulator.test_health_check,method,(),false,1,Test basic health endpoint
194+
functions[15]{line,name,sig}:
195+
14,TestFirmwareSimulator.test_health_check,()
196+
20,TestFirmwareSimulator.test_scenarios_fetch,()
197+
```
198+
199+
If you also want the intent/purpose column, add `--does`:
200+
201+
```bash
202+
code2logic /path/to/project -f toon --function-logic --does --name project -o ./ --no-repeat-details
185203
```
186204

205+
### Format Conventions (function-logic TOON)
206+
207+
| Convention | Meaning | Example |
208+
|---|---|---|
209+
| Name with `.` | Method | `Config.get_api_key` |
210+
| Name without `.` | Top-level function | `main` |
211+
| `~` prefix | Async | `~index_page` |
212+
| `cc:N` suffix | Cyclomatic complexity > 1 | `~index_page cc:2` |
213+
| `./file` | Same directory as previous entry | `./test_main.py` |
214+
215+
Only modules with at least one function/method are listed. Empty modules (`__init__.py`, `models.py` with 0 items) are omitted.
216+
217+
### The `--does` flag
218+
219+
By default, the `does` (intent/purpose) column is **omitted** from function-logic TOON to save tokens. Use `--does` to include it:
220+
221+
```bash
222+
# Without --does (default, compact):
223+
# functions[2]{line,name,sig}:
224+
# 77,~index_page cc:2,()
225+
226+
# With --does (adds intent column):
227+
# functions[2]{line,name,sig,does}:
228+
# 77,~index_page cc:2,(),Serve the firmware UI
229+
230+
code2logic /path/to/project -f toon --function-logic --does --name project -o ./
231+
```
232+
233+
Use `--does` when you need the LLM to understand **what each function does**, not just its signature. Omit it when you only need structure/navigation.
234+
235+
### Schema Generation
236+
237+
When `--with-schema` is used with `--function-logic` and TOON format, a JSON schema is written alongside:
238+
239+
```bash
240+
code2logic /path/to/project -f toon --function-logic --with-schema --name project -o ./
241+
# Produces: project.functions.toon + project.functions-schema.json
242+
```
243+
244+
If using `--stdout`, the function-logic schema is printed under the `=== FUNCTION_LOGIC_SCHEMA ===` section marker.
245+
187246
Notes:
188247

189248
- `--no-repeat-module` affects TOON `modules[...]` tables (main TOON output and function-logic TOON modules table).

‎examples/benchmark_summary.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/env python3
2+
"""Print benchmark summary from JSON result files."""
3+
import json
4+
import os
5+
import sys
6+
7+
8+
def main():
9+
out = sys.argv[1] if len(sys.argv) > 1 else "examples/output"
10+
11+
files = {
12+
"Format": os.path.join(out, "benchmark_format.json"),
13+
"Token": os.path.join(out, "benchmark_token.json"),
14+
"Project": os.path.join(out, "benchmark_project.json"),
15+
"Function": os.path.join(out, "benchmark_function.json"),
16+
}
17+
18+
print()
19+
print(
20+
f"{'Benchmark':<12} {'Files':>6} {'Avg Score':>10} "
21+
f"{'Syntax OK':>10} {'Runs OK':>10} {'Best Format':>14} {'Time':>8}"
22+
)
23+
print("-" * 75)
24+
25+
for name, path in files.items():
26+
if not os.path.exists(path):
27+
continue
28+
d = json.load(open(path))
29+
total = d.get("total_files", d.get("total_functions", "-"))
30+
print(
31+
f"{name:<12} {total:>6} "
32+
f"{d.get('avg_score', 0):>9.1f}% "
33+
f"{d.get('syntax_ok_rate', 0):>9.0f}% "
34+
f"{d.get('runs_ok_rate', 0):>9.0f}% "
35+
f"{d.get('best_format', '-'):>14} "
36+
f"{d.get('total_time', 0):>7.1f}s"
37+
)
38+
39+
print()
40+
41+
fmt_path = os.path.join(out, "benchmark_format.json")
42+
if os.path.exists(fmt_path):
43+
d = json.load(open(fmt_path))
44+
scores = d.get("format_scores", {})
45+
if scores:
46+
print("Format scores:")
47+
for fmt, sc in sorted(scores.items(), key=lambda x: -x[1]):
48+
bar = "\u2588" * int(sc / 2)
49+
print(f" {fmt:<10} {sc:>6.1f}% {bar}")
50+
print()
51+
52+
53+
if __name__ == "__main__":
54+
main()

0 commit comments

Comments
 (0)