feat: add token counting, markdown format, and YAML escaping fixes

- Add token counting module with tiktoken support and fallback approximation
- Add o200k_harmony encoding for newer models
- Add warning when --token-encoding used without --tokens
- Fix YAML escaping for \n, \r, \0, \x85, \u2028, \u2029 in filenames
- Add markdown output format with language-aware code fences
- Add comprehensive tests for tokens (23), markdown (56), YAML escaping (11)

Author: nikolay-e

.github/workflows/cd.yml

@@ -40,7 +40,8 @@ jobs:
 
       - name: Check that we're on main branch
         run: |
-          CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          # Use GitHub context instead of git command to avoid detached HEAD issues
+          CURRENT_BRANCH="${{ github.ref_name }}"
           if [ "$CURRENT_BRANCH" != "main" ]; then
             echo "Error: Releases can only be created from the main branch. Current branch: $CURRENT_BRANCH"
             exit 1
@@ -87,6 +88,20 @@ jobs:
           echo "Commit SHA: $COMMIT_SHA"
           echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT
 
+      - name: Check tag doesn't already exist
+        run: |
+          TAG="v${{ github.event.inputs.version }}"
+          git fetch --tags origin 2>/dev/null || true
+          if git rev-parse "$TAG" >/dev/null 2>&1; then
+            echo "Error: Tag $TAG already exists locally"
+            exit 1
+          fi
+          if git ls-remote --tags origin | grep -q "refs/tags/$TAG$"; then
+            echo "Error: Tag $TAG already exists on remote"
+            exit 1
+          fi
+          echo "Tag $TAG does not exist, proceeding..."
+
       - name: Create local tag (no push yet)
         run: |
           git tag -a "v${{ github.event.inputs.version }}" -m "Release version ${{ github.event.inputs.version }}"
@@ -312,9 +327,10 @@ jobs:
         run: |
           git clone repo.bundle repo
           cd repo
-          git checkout main
-          echo "Current branch: $(git branch --show-current)"
+          # Checkout the version commit (not old main) to preserve version bump
+          git checkout ${{ needs.prepare-version.outputs.commit_sha }}
           echo "Current commit: $(git rev-parse HEAD)"
+          echo "Expected commit: ${{ needs.prepare-version.outputs.commit_sha }}"
 
           # Set up remote to point to the actual GitHub repository
           git remote set-url origin "https://x-access-token:${GH_TOKEN}@github.com/${{ github.repository }}.git"

.pre-commit-config.yaml

@@ -28,7 +28,7 @@ repos:
   # Order matters: isort must run before black to avoid conflicts
   # ============================================================================
   - repo: https://github.com/pycqa/isort
-    rev: 7.0.0
+    rev: 5.13.2
     hooks:
       - id: isort
 

CLAUDE.md

@@ -39,31 +39,49 @@ children:
 ## Usage
 
 ```bash
-treemapper .                          # YAML to stdout
+treemapper .                          # YAML to stdout + token count
 treemapper . -o tree.yaml             # save to file
+treemapper . -o                       # save to tree.yaml (default filename)
 treemapper . -o -                     # explicit stdout output
-treemapper . --format json            # JSON format
-treemapper . --format text            # tree-style text
+treemapper . -f json                  # JSON format
+treemapper . -f txt                   # plain text with indentation
+treemapper . -f md                    # Markdown with headings and fenced code blocks
+treemapper . -f yml                   # YAML format (alias for yaml)
 treemapper . --no-content             # structure only (no file contents)
 treemapper . --max-depth 3            # limit directory depth
 treemapper . --max-file-bytes 10000   # skip files larger than 10KB
+treemapper . --max-file-bytes 0       # no limit (include all files)
 treemapper . -i custom.ignore         # custom ignore patterns
 treemapper . --no-default-ignores     # disable .gitignore/.treemapperignore (custom -i still works)
-treemapper . -v 2                     # verbose output (0=ERROR, 1=WARNING, 2=INFO, 3=DEBUG)
-treemapper . -c                       # copy output to clipboard (also outputs to stdout)
-treemapper . --copy-only              # copy to clipboard only (no stdout output)
-treemapper --version                  # show version
+treemapper . --log-level info         # log level (error/warning/info/debug)
+treemapper . -c                       # copy to clipboard (no stdout)
+treemapper . -c -o tree.yaml          # copy to clipboard + save to file
+treemapper -v                         # show version
+```
+
+## Token Counting
+
+Token count and size are always displayed on stderr:
+
+```
+12,847 tokens (o200k_base), 52.3 KB
+Copied to clipboard
 ```
 
+For large outputs (>1MB), approximate counts are shown with `~` prefix:
+```
+~125,000 tokens (o200k_base), 5.2 MB
+```
+
+Uses tiktoken with `o200k_base` encoding (GPT-4o tokenizer).
+
 ## Clipboard Support
 
 Copy output directly to clipboard with `-c` or `--copy`:
 
 ```bash
-treemapper . -c                       # copy to clipboard + stdout
+treemapper . -c                       # copy to clipboard (no stdout)
 treemapper . -c -o tree.yaml          # copy to clipboard + save to file
-treemapper . --copy-only              # copy to clipboard only
-treemapper . --copy-only -o tree.yaml # copy to clipboard + save to file (no stdout)
 ```
 
 **System Requirements:**
@@ -72,10 +90,12 @@ treemapper . --copy-only -o tree.yaml # copy to clipboard + save to file (no std
 - **Linux/FreeBSD (Wayland):** `wl-copy` (install: `sudo apt install wl-clipboard`)
 - **Linux/FreeBSD (X11):** `xclip` or `xsel` (install: `sudo apt install xclip`)
 
+If clipboard is unavailable, output falls back to stdout with a warning on stderr.
+
 ## Python API
 
 ```python
-from treemapper import map_directory, to_yaml, to_json, to_text
+from treemapper import map_directory, to_yaml, to_json, to_text, to_markdown
 
 # Full function signature
 tree = map_directory(
@@ -95,7 +115,8 @@ tree = map_directory(".", max_file_bytes=50000, ignore_file="custom.ignore")
 # Serialize to string
 yaml_str = to_yaml(tree)
 json_str = to_json(tree)
-text_str = to_text(tree)
+text_str = to_text(tree)    # or to_txt(tree)
+md_str = to_markdown(tree)  # or to_md(tree)
 ```
 
 ## Ignore Patterns
@@ -111,8 +132,8 @@ Features:
 ## Content Placeholders
 
 When file content cannot be read normally, placeholders are used:
-- `<file too large: N bytes>` — file exceeds `--max-file-bytes` limit
-- `<binary file: N bytes>` — file detected as binary (contains null bytes)
+- `<file too large: N bytes>` — file exceeds `--max-file-bytes` limit (default: 10 MB)
+- `<binary file: N bytes>` — binary file (detected by extension or null bytes)
 - `<unreadable content: not utf-8>` — file is not valid UTF-8
 - `<unreadable content>` — file cannot be read (permission denied, I/O error)
 
@@ -134,8 +155,9 @@ Integration tests only - test against real filesystem. No mocking.
 src/treemapper/
 ├── cli.py        # argument parsing
 ├── ignore.py     # gitignore/treemapperignore handling
+├── tokens.py     # token counting (tiktoken)
 ├── tree.py       # directory traversal
-├── writer.py     # YAML/JSON/text output
+├── writer.py     # YAML/JSON/text/Markdown output
 └── treemapper.py # main entry point
 ```
 

docs/Token Counting.md

@@ -37,8 +37,8 @@ files = ["src"]
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version_provider = "pep621"
 tag_format = "v$version"
+version_files = ["src/treemapper/version.py:__version__"]
 
 [project]
 name = "treemapper"
@@ -50,7 +50,7 @@ description = "Export codebase structure and contents for AI/LLM context"
 readme = "README.md"
 requires-python = ">=3.9"
 license = { file = "LICENSE" }
-keywords = ["code-analysis", "directory-tree", "yaml", "json", "llm", "ai", "codebase", "context", "chatgpt", "claude", "code-context", "export", "tree"]
+keywords = ["code-analysis", "directory-tree", "yaml", "json", "llm", "ai", "codebase", "context", "chatgpt", "claude", "code-context", "export", "tree", "gpt-context", "llm-context", "code-to-prompt", "claude-context"]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Environment :: Console",
@@ -70,6 +70,7 @@ classifiers = [
 dependencies = [
     "pathspec>=0.11,<2.0",
     "pyyaml>=6.0.2,<8.0",
+    "tiktoken>=0.7,<1.0",
 ]
 
 [project.urls]

pyproject.toml

@@ -1,28 +1,33 @@
+from __future__ import annotations
+
 import io
 from pathlib import Path
-from typing import Any, Optional, Union
+from typing import Any
 
 from .ignore import get_ignore_specs
 from .tree import TreeBuildContext, build_tree
 from .version import __version__
-from .writer import write_tree_json, write_tree_text, write_tree_yaml
+from .writer import write_tree_json, write_tree_markdown, write_tree_text, write_tree_yaml
 
 __all__ = [
     "__version__",
     "map_directory",
     "to_json",
+    "to_markdown",
+    "to_md",
     "to_text",
+    "to_txt",
     "to_yaml",
 ]
 
 
 def map_directory(
-    path: Union[str, Path],
+    path: str | Path,
     *,
-    max_depth: Optional[int] = None,
+    max_depth: int | None = None,
     no_content: bool = False,
-    max_file_bytes: Optional[int] = None,
-    ignore_file: Optional[Union[str, Path]] = None,
+    max_file_bytes: int | None = None,
+    ignore_file: str | Path | None = None,
     no_default_ignores: bool = False,
 ) -> dict[str, Any]:
     root_dir = Path(path).resolve()
@@ -63,3 +68,13 @@ def to_text(tree: dict[str, Any]) -> str:
     buf = io.StringIO()
     write_tree_text(buf, tree)
     return buf.getvalue()
+
+
+def to_markdown(tree: dict[str, Any]) -> str:
+    buf = io.StringIO()
+    write_tree_markdown(buf, tree)
+    return buf.getvalue()
+
+
+to_md = to_markdown
+to_txt = to_text