CLI: global config, tqdm UX, vision dedupe+batch, shell completions

- Config: ~/.config/docproc/docproc.yml in search path; docproc init-config populates from .env
- CLI: tqdm progress bar, single-line output, quiet logs (httpx/openai/vision_llm), colors, full output path
- Spinner: background thread so spinner rotates constantly during extraction
- Vision: dedupe images by content hash (skip logos/repeats); batch 4-5 images per request; 3 batches in parallel; timeout per image
- Completions: docproc completions [bash|zsh]; shtab + tqdm deps
- Progress: bar only advances (no backward jump); refine phase shows 100% + refining

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: rithulkamesh

docproc/bin/cli.py

@@ -3,8 +3,13 @@
 import argparse
 import logging
 import sys
+import threading
+import time
 from pathlib import Path
 
+import shtab
+from tqdm import tqdm
+
 from docproc.doc.loaders import get_supported_extensions
 
 logging.basicConfig(
@@ -13,6 +18,59 @@
 logger = logging.getLogger(__name__)
 
 
+def _run_init_config():
+    """Populate ~/.config/docproc/docproc.yml from .env (one-time)."""
+    import os
+    import yaml
+    from dotenv import load_dotenv
+
+    parser = argparse.ArgumentParser(prog="docproc init-config")
+    parser.add_argument("--env", type=str, default=".env", help="Path to .env file")
+    args = parser.parse_args(sys.argv[2:])
+    load_dotenv(args.env)
+    cfg_dir = Path.home() / ".config" / "docproc"
+    cfg_dir.mkdir(parents=True, exist_ok=True)
+    out_path = cfg_dir / "docproc.yml"
+    raw = {}
+    if os.getenv("AZURE_OPENAI_API_KEY"):
+        raw["ai_providers"] = [
+            {
+                "provider": "azure",
+                "api_key": os.getenv("AZURE_OPENAI_API_KEY"),
+                "base_url": os.getenv("AZURE_OPENAI_ENDPOINT"),
+                "default_model": os.getenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o"),
+                "default_vision_model": os.getenv("AZURE_OPENAI_DEPLOYMENT", "gpt-4o"),
+                "extra": {
+                    "azure_deployment": os.getenv("AZURE_OPENAI_DEPLOYMENT"),
+                    "azure_embedding_deployment": os.getenv("AZURE_OPENAI_EMBEDDING_DEPLOYMENT"),
+                    "azure_vision_endpoint": os.getenv("AZURE_VISION_ENDPOINT"),
+                },
+            }
+        ]
+        raw["primary_ai"] = "azure"
+    elif os.getenv("OPENAI_API_KEY"):
+        raw["ai_providers"] = [{"provider": "openai", "api_key": os.getenv("OPENAI_API_KEY")}]
+        raw["primary_ai"] = "openai"
+    elif os.getenv("ANTHROPIC_API_KEY"):
+        raw["ai_providers"] = [{"provider": "anthropic", "api_key": os.getenv("ANTHROPIC_API_KEY")}]
+        raw["primary_ai"] = "anthropic"
+    else:
+        raw["ai_providers"] = [
+            {"provider": "ollama", "base_url": "http://localhost:11434", "default_vision_model": "llava"}
+        ]
+        raw["primary_ai"] = "ollama"
+    if os.getenv("DATABASE_URL"):
+        raw["database"] = {"provider": "pgvector", "connection_string": os.getenv("DATABASE_URL")}
+    else:
+        raw["database"] = {"provider": "memory"}
+    raw["rag"] = {"backend": "clara", "top_k": 5}
+    raw["ingest"] = {"use_vision": True, "use_llm_refine": True}
+    with open(out_path, "w") as f:
+        yaml.dump(raw, f, default_flow_style=False, sort_keys=False)
+    logger.info("Wrote %s", out_path)
+    return 0
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Extract document to markdown (vision + optional LLM refine)"
@@ -26,7 +84,31 @@ def parse_args():
     return parser.parse_args()
 
 
+def _get_completion_parser():
+    """Parser used for shtab completion generation (matches main extract args)."""
+    parser = argparse.ArgumentParser(prog="docproc")
+    parser.add_argument("--file", "-f", help="Input document").complete = shtab.FILE
+    parser.add_argument("-o", "--output", help="Output .md path").complete = shtab.FILE
+    parser.add_argument("--config", help="Config file path").complete = shtab.FILE
+    parser.add_argument("-v", "--verbose", action="store_true")
+    return parser
+
+
+def _run_completions():
+    """Print shell completion script. Usage: docproc completions [bash|zsh]."""
+    parser = _get_completion_parser()
+    shell = sys.argv[2] if len(sys.argv) > 2 else "bash"
+    if shell not in ("bash", "zsh"):
+        shell = "bash"
+    print(shtab.complete(parser, shell=shell))
+    return 0
+
+
 def main():
+    if len(sys.argv) > 1 and sys.argv[1] == "init-config":
+        return _run_init_config()
+    if len(sys.argv) > 1 and sys.argv[1] == "completions":
+        return _run_completions()
     args = parse_args()
     if args.verbose:
         logging.getLogger().setLevel(logging.DEBUG)
@@ -60,15 +142,76 @@ def main():
         else:
             load_config()
 
-        def progress(page: int, total: int, message: str):
-            logger.info("%s (%d/%d)", message, page, total)
+        # Single-line UX: suppress all logs during extraction
+        _log = logging.getLogger
+        _quiet = [_log("httpx"), _log("httpcore"), _log("openai"), _log("docproc.extractors.vision_llm")]
+        _saved = [g.level for g in _quiet]
+        for g in _quiet:
+            g.setLevel(logging.WARNING)
+
+        pbar: tqdm | None = None
+        spin_idx = [0]
+        SPINNER = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+        _C = "\033[36m"
+        _G = "\033[32m"
+        _Y = "\033[33m"
+        _R = "\033[0m"
+        stop_spinner = threading.Event()
+        spinner_thread: threading.Thread | None = None
+
+        def spinner_loop():
+            while not stop_spinner.wait(0.08):
+                if pbar is None:
+                    continue
+                spin_idx[0] = (spin_idx[0] + 1) % len(SPINNER)
+                pbar.set_description_str(f"{_C}docproc {SPINNER[spin_idx[0]]}{_R}")
+                pbar.refresh()
 
-        full_text = extract_document_to_text(
-            input_path, progress_callback=progress
-        )
-        Path(output_path).parent.mkdir(parents=True, exist_ok=True)
-        Path(output_path).write_text(full_text, encoding="utf-8")
-        logger.info("Wrote %s", output_path)
+        def progress(page: int, total: int, message: str):
+            nonlocal pbar, spinner_thread
+            if total == 1 and "Refining" in message:
+                if pbar is not None:
+                    pbar.n = pbar.total
+                    pbar.set_postfix_str("refining…", refresh=False)
+                    pbar.refresh()
+                return
+            if pbar is None:
+                pbar = tqdm(
+                    total=max(1, total),
+                    unit="",
+                    desc=f"{_C}docproc {SPINNER[0]}{_R}",
+                    bar_format=f"{{desc}} {_G}{{bar}}{_R} {_Y}{{n_fmt}}/{{total_fmt}}{_R} {{postfix}}",
+                    dynamic_ncols=True,
+                    leave=False,
+                    mininterval=0.2,
+                    maxinterval=0.5,
+                )
+                spinner_thread = threading.Thread(target=spinner_loop, daemon=True)
+                spinner_thread.start()
+            # Only advance; parallel batches complete out of order
+            new_n = min(page, pbar.total - 1) if pbar.total else page
+            if new_n > pbar.n:
+                pbar.n = new_n
+            pbar.set_postfix_str(message[:40].strip(), refresh=False)
+            pbar.refresh()
+
+        try:
+            full_text = extract_document_to_text(
+                input_path, progress_callback=progress
+            )
+        finally:
+            stop_spinner.set()
+            if spinner_thread is not None:
+                spinner_thread.join(timeout=0.5)
+            for g, level in zip(_quiet, _saved):
+                g.setLevel(level)
+            if pbar is not None:
+                pbar.close()
+        out = Path(output_path)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(full_text, encoding="utf-8")
+        full_path = str(out.resolve())
+        sys.stderr.write(f"\r\033[K{_G}✓ Wrote{_R} {full_path}\n")
         return 0
     except Exception as e:
         logger.error("Failed to extract: %s", e, exc_info=args.verbose)

docproc/config/loader.py

@@ -35,6 +35,7 @@ def load_config(path: Optional[str] = None) -> DocProcConfig:
             os.getenv("DOCPROC_CONFIG"),
             "docproc.yaml",
             "docproc.yml",
+            os.path.expanduser("~/.config/docproc/docproc.yml"),
             os.path.expanduser("~/.docproc.yaml"),
         ]
     config_path = None