Skip to content

Commit bff89d3

Browse files
tom-sapletta-comtom-sapletta-com
committed
feat(docs): CLI interface improvements
changes: - file: runner.py area: core modified: [_test_python_runs] - file: function_logic.py area: core modified: [generate_toon_schema, FunctionLogicGenerator] - file: metrics.py area: quality modified: [ReproductionMetrics, _compute_structural_metrics, count_elements] - file: behavioral_benchmark.py area: core added: [CaseResult, builder_missing, main, _values_equal, _load_module_from_path, builder_invalid, +7 more] - file: benchmark_report.py area: core added: [_calc_file_summary, _load_json, main, Artifact, _token_estimate_bytes, _calc_function_summary, +3 more] - file: benchmark_summary.py area: core modified: [main] stats: lines: "+4665/-61 (net +4604)" files: 15 complexity: "Large structural change (normalized)"
1 parent 2e145ed commit bff89d3

23 files changed

+4699
-68
lines changed

‎CHANGELOG.md‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,30 @@
1+
## [1.0.38] - 2026-02-25
2+
3+
### Summary
4+
5+
feat(docs): CLI interface improvements
6+
7+
### Docs
8+
9+
- docs: update README
10+
- docs: update 03-cli-reference.md
11+
- docs: update 07-toon.md
12+
13+
### Other
14+
15+
- build: update Makefile
16+
- update TICKET
17+
- update code2logic/benchmarks/runner.py
18+
- update code2logic/function_logic.py
19+
- update code2logic/metrics.py
20+
- update examples/behavioral_benchmark.py
21+
- update examples/benchmark_report.py
22+
- update examples/benchmark_summary.py
23+
- update function-schema.json
24+
- update function.toon
25+
- ... and 2 more
26+
27+
128
## [1.0.37] - 2026-02-25
229

330
### Summary

‎Makefile‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -313,9 +313,18 @@ benchmark: benchmark-format benchmark-function benchmark-token benchmark-project
313313
@echo "$(GREEN)All benchmarks completed!$(NC)"
314314
@echo "Results in $(BENCH_OUTPUT)/"
315315
@ls -lhS $(BENCH_OUTPUT)/*.json 2>/dev/null
316+
@echo ""
317+
@echo "$(BLUE)Generating Markdown report...$(NC)"
318+
@BENCH_OUTPUT=$(BENCH_OUTPUT) $(PYTHON) examples/benchmark_report.py >/dev/null
319+
@echo "Report: $(BENCH_OUTPUT)/BENCHMARK_REPORT.md"
316320

317321
benchmark-format: ## Benchmark format reproduction (yaml/toon/logicml/json)
318322
@echo "$(BLUE)━━━ Format Benchmark ━━━$(NC)"
323+
@mkdir -p $(BENCH_OUTPUT)
324+
@echo "# Auto-generated by make benchmark" > $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
325+
@echo "set -euo pipefail" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
326+
@echo "" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
327+
@printf '%s\n' "$(PYTHON) examples/15_unified_benchmark.py --no-llm --type format --folder $(BENCH_SAMPLES)/ --formats $(BENCH_FORMATS) --limit $(BENCH_LIMIT) --verbose --output $(BENCH_OUTPUT)/benchmark_format.json" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
319328
$(PYTHON) examples/15_unified_benchmark.py \
320329
--no-llm --type format \
321330
--folder $(BENCH_SAMPLES)/ \
@@ -325,14 +334,20 @@ benchmark-format: ## Benchmark format reproduction (yaml/toon/logicml/json)
325334

326335
benchmark-function: ## Benchmark function-level reproduction
327336
@echo "$(BLUE)━━━ Function Benchmark ━━━$(NC)"
337+
@printf '%s\n' "$(PYTHON) examples/15_unified_benchmark.py --no-llm --type function --file $(BENCH_SAMPLES)/sample_functions.py --limit 10 --verbose --output $(BENCH_OUTPUT)/benchmark_function.json" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
328338
$(PYTHON) examples/15_unified_benchmark.py \
329339
--no-llm --type function \
330340
--file $(BENCH_SAMPLES)/sample_functions.py \
331341
--limit 10 --verbose \
332342
--output $(BENCH_OUTPUT)/benchmark_function.json
343+
@echo "$(BLUE)━━━ Behavioral Benchmark (runtime equivalence) ━━━$(NC)"
344+
@printf '%s\n' "$(PYTHON) examples/behavioral_benchmark.py" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
345+
@BENCH_OUTPUT=$(BENCH_OUTPUT) BENCH_FUNCTION_JSON=$(BENCH_OUTPUT)/benchmark_function.json BENCH_FUNCTION_SOURCE=$(BENCH_SAMPLES)/sample_functions.py \
346+
$(PYTHON) examples/behavioral_benchmark.py >/dev/null
333347

334348
benchmark-token: ## Benchmark token efficiency across formats
335349
@echo "$(BLUE)━━━ Token Efficiency Benchmark ━━━$(NC)"
350+
@printf '%s\n' "$(PYTHON) examples/11_token_benchmark.py --no-llm --folder $(BENCH_SAMPLES)/ --formats $(BENCH_FORMATS) --limit $(BENCH_LIMIT) --verbose --output $(BENCH_OUTPUT)/benchmark_token.json" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
336351
$(PYTHON) examples/11_token_benchmark.py \
337352
--no-llm \
338353
--folder $(BENCH_SAMPLES)/ \
@@ -342,6 +357,7 @@ benchmark-token: ## Benchmark token efficiency across formats
342357

343358
benchmark-project: ## Benchmark project-level reproduction
344359
@echo "$(BLUE)━━━ Project Benchmark ━━━$(NC)"
360+
@printf '%s\n' "$(PYTHON) examples/15_unified_benchmark.py --no-llm --type project --folder $(BENCH_SAMPLES)/ --formats $(BENCH_FORMATS) --limit $(BENCH_LIMIT) --verbose --output $(BENCH_OUTPUT)/benchmark_project.json" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
345361
$(PYTHON) examples/15_unified_benchmark.py \
346362
--no-llm --type project \
347363
--folder $(BENCH_SAMPLES)/ \
@@ -352,18 +368,29 @@ benchmark-project: ## Benchmark project-level reproduction
352368
benchmark-toon: ## Generate TOON + function-logic for self-analysis
353369
@echo "$(BLUE)━━━ TOON Self-Analysis ━━━$(NC)"
354370
@mkdir -p $(BENCH_OUTPUT)
371+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f toon --compact --name project -o ./" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
355372
$(PYTHON) -m code2logic ./ -f toon --compact --name project -o ./
356-
$(PYTHON) -m code2logic ./ -f toon --compact --no-repeat-module --function-logic --with-schema --name project -o ./
373+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f toon --compact --no-repeat-module --function-logic function.toon --with-schema --name project -o ./" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
374+
$(PYTHON) -m code2logic ./ -f toon --compact --no-repeat-module --function-logic function.toon --with-schema --name project -o ./
375+
@cp -f project.toon $(BENCH_OUTPUT)/project.toon 2>/dev/null || true
376+
@cp -f project.toon-schema.json $(BENCH_OUTPUT)/project.toon-schema.json 2>/dev/null || true
377+
@cp -f function.toon $(BENCH_OUTPUT)/function.toon 2>/dev/null || true
378+
@cp -f function-schema.json $(BENCH_OUTPUT)/function-schema.json 2>/dev/null || true
379+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f yaml --compact --name project -o $(BENCH_OUTPUT)/" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
357380
$(PYTHON) -m code2logic ./ -f yaml --compact --name project -o $(BENCH_OUTPUT)/
381+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f json --name project -o $(BENCH_OUTPUT)/" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
358382
$(PYTHON) -m code2logic ./ -f json --name project -o $(BENCH_OUTPUT)/
383+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f markdown --name project -o $(BENCH_OUTPUT)/" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
359384
$(PYTHON) -m code2logic ./ -f markdown --name project -o $(BENCH_OUTPUT)/
385+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f compact --name project -o $(BENCH_OUTPUT)/" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
360386
$(PYTHON) -m code2logic ./ -f compact --name project -o $(BENCH_OUTPUT)/
387+
@printf '%s\n' "$(PYTHON) -m code2logic ./ -f csv -d standard --name project -o $(BENCH_OUTPUT)/" >> $(BENCH_OUTPUT)/BENCHMARK_COMMANDS.sh
361388
$(PYTHON) -m code2logic ./ -f csv -d standard --name project -o $(BENCH_OUTPUT)/
362389
@echo ""
363390
@echo "$(BLUE)Format size comparison (self-analysis):$(NC)"
364391
@printf " %-25s %10s %10s\n" "Format" "Size" "~Tokens"
365392
@printf " %-25s %10s %10s\n" "-------------------------" "----------" "----------"
366-
@for f in project.toon project.functions.toon $(BENCH_OUTPUT)/project.yaml $(BENCH_OUTPUT)/project.json $(BENCH_OUTPUT)/project.md $(BENCH_OUTPUT)/project.txt $(BENCH_OUTPUT)/project.csv; do \
393+
@for f in $(BENCH_OUTPUT)/project.toon $(BENCH_OUTPUT)/function.toon $(BENCH_OUTPUT)/project.yaml $(BENCH_OUTPUT)/project.json $(BENCH_OUTPUT)/project.md $(BENCH_OUTPUT)/project.txt $(BENCH_OUTPUT)/project.csv; do \
367394
if [ -f "$$f" ]; then \
368395
sz=$$(wc -c < "$$f"); \
369396
tok=$$((sz / 4)); \
@@ -372,7 +399,7 @@ benchmark-toon: ## Generate TOON + function-logic for self-analysis
372399
done
373400
@echo ""
374401
@echo "$(GREEN)TOON files:$(NC)"
375-
@ls -lh project.toon project.functions.toon project.toon-schema.json project.functions-schema.json 2>/dev/null
402+
@ls -lh $(BENCH_OUTPUT)/project.toon $(BENCH_OUTPUT)/function.toon $(BENCH_OUTPUT)/project.toon-schema.json $(BENCH_OUTPUT)/function-schema.json 2>/dev/null
376403

377404
benchmark-compare: ## Show summary comparison of all benchmark results
378405
@echo ""

‎README.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,9 @@ pip install code2logic[nlp] # Enhanced intents
4949
## 📖 Quick Start
5050
```bash
5151
code2logic ./ -f yaml --compact --function-logic --with-schema -o project.yaml
52-
code2logic ./ -f toon --function-logic --with-schema --name project -o ./
53-
# Optional: include function intent/purpose column in project.functions.toon
54-
code2logic ./ -f toon --function-logic --does --name project -o ./
52+
code2logic ./ -f toon --function-logic function.toon --with-schema --name project -o ./
53+
# Optional: include function intent/purpose column in function.toon
54+
code2logic ./ -f toon --function-logic function.toon --does --name project -o ./
5555
```
5656

5757
### Command Line

‎TICKET‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# TICKET
2+
#
3+
# Configuration for ticket prefixing in commit messages.
4+
#
5+
# Supported keys:
6+
# - prefix: ticket prefix to prepend to commit titles (e.g. ABC-123)
7+
# - format: formatting template. Available variables:
8+
# {ticket} - value of prefix
9+
# {title} - generated conventional commit title
10+
#
11+
# Examples:
12+
# prefix=ABC-123
13+
# format=[{ticket}] {title}
14+
15+
prefix=
16+
format=[{ticket}] {title}

‎VERSION‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.0.37
1+
1.0.38

‎code2logic/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
>>> print(output)
1919
"""
2020

21-
__version__ = "1.0.37"
21+
__version__ = "1.0.38"
2222
__author__ = "Softreck"
2323
__email__ = "info@softreck.dev"
2424
__license__ = "MIT"

‎code2logic/benchmarks/runner.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,10 @@ def _test_python_runs(code: str, timeout: int = 5) -> bool:
5454
capture_output=True, timeout=timeout
5555
)
5656
return result.returncode == 0
57+
except subprocess.TimeoutExpired:
58+
return False
5759
except Exception:
58-
return True # Timeout might mean waiting for input
60+
return False
5961

6062

6163
def _basic_syntax_ok(code: str, language: str) -> bool:

‎code2logic/function_logic.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def generate_toon_schema(self) -> str:
160160
"$schema": "https://json-schema.org/draft/2020-12/schema",
161161
"title": "Code2Logic Function-Logic TOON Schema",
162162
"description": (
163-
"Schema for project.functions.toon — compact function/method index. "
163+
"Schema for function.toon — compact function/method index. "
164164
"Conventions: name containing '.' = method (Class.method), "
165165
"~prefix = async, 'cc:N' suffix = cyclomatic complexity (only when >1)."
166166
),

‎code2logic/metrics.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -310,10 +310,12 @@ def _compute_structural_metrics(self, original: str, generated: str) -> Structur
310310
def count_elements(code: str) -> Dict[str, int]:
311311
return {
312312
'classes': len(re.findall(r'^class\s+\w+', code, re.MULTILINE)),
313-
'functions': len(re.findall(r'^def\s+\w+', code, re.MULTILINE)),
314-
'methods': len(re.findall(r'^\s+def\s+\w+', code, re.MULTILINE)),
313+
'functions': len(re.findall(r'^(?:async\s+)?def\s+\w+', code, re.MULTILINE)),
314+
'methods': len(re.findall(r'^\s+(?:async\s+)?def\s+\w+', code, re.MULTILINE)),
315315
'imports': len(re.findall(r'^(?:from|import)\s+', code, re.MULTILINE)),
316-
'attributes': len(re.findall(r'^\s+\w+\s*:\s*\w+', code, re.MULTILINE)),
316+
# Capture both annotated attributes and simple assignments.
317+
# This is still heuristic, but avoids undercounting common code.
318+
'attributes': len(re.findall(r'^\s+\w+\s*(?::\s*[^=\n]+)?\s*=', code, re.MULTILINE)),
317319
}
318320

319321
orig = count_elements(original)

‎docs/03-cli-reference.md‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ code2logic /path/to/project -f hybrid -o analysis.hybrid.yaml
120120
# Standard TOON (token-efficient)
121121
code2logic /path/to/project -f toon -o analysis.toon
122122

123+
# Function-logic TOON (outputs function.toon + optional schema)
124+
code2logic /path/to/project -f toon --compact --function-logic function.toon --name project -o ./
125+
126+
# Same as above, but also compress repeated module paths and generate JSON Schema
127+
code2logic /path/to/project -f toon --compact --no-repeat-module --function-logic function.toon --with-schema --name project -o ./
128+
123129
# Ultra-compact TOON (71% smaller, single-letter keys)
124130
code2logic /path/to/project -f toon --ultra-compact -o analysis-ultra.toon
125131

@@ -135,12 +141,37 @@ code2logic /path/to/project -f toon --function-logic --name project -o ./ --no-r
135141
# Generate function-logic TOON with intent descriptions (does column)
136142
code2logic /path/to/project -f toon --function-logic --does --name project -o ./
137143

138-
# Generate function-logic TOON + schema (project.functions.toon + project.functions-schema.json)
139-
code2logic /path/to/project -f toon --function-logic --with-schema --name project -o ./
144+
# Generate function-logic TOON + schema (function.toon + function-schema.json)
145+
code2logic /path/to/project -f toon --function-logic function.toon --with-schema --name project -o ./
140146
```
141147

142148
Token-oriented object notation - most efficient format for LLM consumption.
143149

150+
## Benchmarks (Makefile)
151+
152+
This repository includes reproducibility benchmarks that compare formats by how well they can be used to regenerate runnable code (heuristic scoring).
153+
154+
```bash
155+
make benchmark
156+
```
157+
158+
Generated artifacts (written to `examples/output/`):
159+
160+
- `BENCHMARK_REPORT.md`
161+
- Links all artifacts and summarizes the run
162+
- `BENCHMARK_COMMANDS.sh`
163+
- Exact commands used to generate each artifact
164+
- `benchmark_format.json`, `benchmark_project.json`, `benchmark_token.json`, `benchmark_function.json`
165+
- Raw benchmark results
166+
- `project.toon`, `function.toon` (+ `*-schema.json`)
167+
- Self-analysis outputs used for size/token comparisons
168+
169+
Run all example scripts step-by-step:
170+
171+
```bash
172+
make examples
173+
```
174+
144175
## Detail Levels
145176

146177
| Level | Description | Use Case |

0 commit comments

Comments
 (0)