Integrate with wcwidth 0.5.0 for proper Unicode handling (#166)

jquast · web-flow · commit a285b7e938c5 · 2026-01-29T15:59:25.000+01:00
- Simplify break_line() to use wcwidth.wrap() for word-boundary breaking
- Add proper support for grapheme clusters (emoji, flags, ZWJ sequences)
- Handle wide characters (CJK) correctly
- Preserve ANSI sequences across line breaks
- Fix word-wrap padding bug in containers
- Require wcwidth&gt;=0.5
- Add comprehensive tests for wcwidth integration
- Add interactive wcwidth demo example
diff --git a/examples/wcwidth_demo.py b/examples/wcwidth_demo.py
@@ -0,0 +1,191 @@
+#!/usr/bin/env python3
+"""Run: python examples/wcwidth_demo.py
+
+Demonstrates break_line() issues with wide characters and grapheme clusters.
+"""
+import pytermgui as ptg
+
+CJK = "中文字符"
+FAMILY = "\U0001F468\u200D\U0001F469\u200D\U0001F467"
+FLAG = "\U0001F1E8\U0001F1E6"  # Canada
+
+# Additional grapheme examples from wcwidth tests
+WAVE_SKIN = "\U0001F44B\U0001F3FB"  # Waving hand + skin tone modifier
+HEART_VS16 = "\u2764\uFE0F"  # Heart + variation selector-16
+CAFE = "cafe\u0301"  # café with combining acute accent
+WOMAN_TECH = "\U0001f469\U0001f3fb\u200d\U0001f4bb"  # Woman technologist (ZWJ)
+KISS = "\U0001F9D1\U0001F3FB\u200d\u2764\uFE0F\u200d\U0001F48B\u200d\U0001F9D1\U0001F3FD"  # Kiss
+
+# ANSI escape codes for color testing
+RST = "\x1b[0m"
+BOLD = "\x1b[1m"
+ITALIC = "\x1b[3m"
+RED = "\x1b[31m"
+GREEN = "\x1b[32m"
+ORANGE_256 = "\x1b[38;5;208m"
+PURPLE_256 = "\x1b[38;5;129m"
+CYAN_RGB = "\x1b[38;2;0;255;255m"
+PINK_RGB = "\x1b[38;2;255;105;180m"
+
+# Big word with 24-bit RGB gradient
+CHOOCHOO = (
+    "\x1b[38;2;255;0;0mc"
+    "\x1b[38;2;255;127;0mh"
+    "\x1b[38;2;255;255;0mo"
+    "\x1b[38;2;0;255;0mo"
+    "\x1b[38;2;0;255;127mc"
+    "\x1b[38;2;0;255;255mh"
+    "\x1b[38;2;0;127;255mo"
+    "\x1b[38;2;0;0;255mo"
+    "\x1b[38;2;127;0;255mp"
+    "\x1b[38;2;255;0;255mo"
+    "\x1b[38;2;255;0;127mn"
+    "\x1b[38;2;255;0;0my"
+    "\x1b[38;2;255;127;0me"
+    "\x1b[38;2;255;255;0mx"
+    "\x1b[38;2;0;255;0mp"
+    "\x1b[38;2;0;255;127mr"
+    "\x1b[38;2;0;255;255me"
+    "\x1b[38;2;0;127;255ms"
+    "\x1b[38;2;0;0;255ms"
+    "\x1b[0m"
+)
+
+ptg.WindowManager.autorun = False
+
+# Track adjustable containers and current width
+current_width = 19
+column_width = 25
+adjustable_containers = []
+column_containers = []
+main_window = None
+
+
+def adjust_width(delta):
+    """Adjust width of all tracked containers."""
+    global current_width, column_width, main_window
+    new_width = max(6, min(30, current_width + delta))
+    if new_width != current_width:
+        current_width = new_width
+        column_width = column_width + delta
+        for container in adjustable_containers:
+            container.static_width = current_width
+        for column in column_containers:
+            column.static_width = column_width
+        if main_window is not None:
+            main_window.width = main_window.width + delta * 4
+        width_label.value = f"[dim]Width: {current_width}  |  ←/→ adjust  |  Ctrl+C exit[/]"
+
+
+with ptg.WindowManager() as manager:
+    manager.mouse_enabled = False
+
+    # Bind arrow keys for width adjustment
+    manager.bind(ptg.keys.LEFT, lambda *_: adjust_width(-1), "Decrease width")
+    manager.bind(ptg.keys.RIGHT, lambda *_: adjust_width(1), "Increase width")
+
+    # Create containers and track the adjustable ones
+    sgr_box = ptg.Container(ptg.Label(f"{BOLD}bold{RST} {ITALIC}italic{RST}"),
+                            static_width=current_width)
+    colors_box = ptg.Container(ptg.Label(f"{RED}red{RST} {GREEN}green{RST}"),
+                               static_width=current_width)
+    c256_box = ptg.Container(ptg.Label(f"{ORANGE_256}orange{RST} {PURPLE_256}purple{RST}"),
+                             static_width=current_width)
+    rgb_box = ptg.Container(ptg.Label(f"{CYAN_RGB}cyan{RST} {PINK_RGB}pink{RST}"),
+                            static_width=current_width)
+
+    wrap_box = ptg.Container(ptg.Label("The quick-brown-fox"), static_width=current_width)
+    cjk_box = ptg.Container(ptg.Label(CJK + CJK), static_width=current_width)
+    family_box = ptg.Container(ptg.Label(f"Hi{FAMILY}!"), static_width=current_width)
+    flag_box = ptg.Container(ptg.Label(f"{FLAG}Canada"), static_width=current_width)
+
+    combine_box = ptg.Container(ptg.Label(f"A {CAFE}!"), static_width=current_width)
+    skin_box = ptg.Container(ptg.Label(f"Hi{WAVE_SKIN}!"), static_width=current_width)
+    heart_box = ptg.Container(ptg.Label(f"I{HEART_VS16}U"), static_width=current_width)
+    tech_box = ptg.Container(ptg.Label(f"{WOMAN_TECH}codes"), static_width=current_width)
+    lorem_box = ptg.Container(ptg.Label("Lorem ipsum dolor sit"), static_width=current_width)
+    choochoo_box = ptg.Container(ptg.Label(CHOOCHOO), static_width=current_width)
+
+    adjustable_containers = [
+        sgr_box, colors_box, c256_box, rgb_box,
+        wrap_box, cjk_box, family_box, flag_box,
+        combine_box, skin_box, heart_box, tech_box,
+        lorem_box, choochoo_box,
+    ]
+
+    left = ptg.Container(
+        ptg.Label("[bold]ANSI Sequences[/]"),
+        "",
+        ptg.Label("SGR:"),
+        sgr_box,
+        "",
+        ptg.Label("Colors:"),
+        colors_box,
+        "",
+        ptg.Label("256-color:"),
+        c256_box,
+        "",
+        ptg.Label("24-bit RGB:"),
+        rgb_box,
+        static_width=column_width,
+    )
+
+    middle = ptg.Container(
+        ptg.Label("[bold]Width & Graphemes[/]"),
+        "",
+        ptg.Label("Word wrap:"),
+        wrap_box,
+        "",
+        ptg.Label("CJK (2-cell):"),
+        cjk_box,
+        "",
+        ptg.Label("Family ZWJ:"),
+        family_box,
+        "",
+        ptg.Label("Flag:"),
+        flag_box,
+        static_width=column_width,
+    )
+
+    right = ptg.Container(
+        ptg.Label("[bold]Grapheme Clusters[/]"),
+        "",
+        ptg.Label("Combining:"),
+        combine_box,
+        "",
+        ptg.Label("Skin tone:"),
+        skin_box,
+        "",
+        ptg.Label("VS-16 heart:"),
+        heart_box,
+        "",
+        ptg.Label("Woman tech:"),
+        tech_box,
+        static_width=column_width,
+    )
+
+    far_right = ptg.Container(
+        ptg.Label("[bold]Long Text[/]"),
+        "",
+        ptg.Label("Lorem ipsum:"),
+        lorem_box,
+        "",
+        ptg.Label("RGB gradient:"),
+        choochoo_box,
+        static_width=column_width,
+    )
+
+    column_containers.extend([left, middle, right, far_right])
+
+    width_label = ptg.Label(f"[dim]Width: {current_width}  |  ←/→ adjust  |  Ctrl+C exit[/]")
+
+    main_window = ptg.Window(
+        ptg.Label("[bold]break_line() Demo[/]"),
+        "",
+        ptg.Splitter(left, middle, right, far_right),
+        "",
+        width_label,
+        width=113,
+    )
+    manager.add(main_window)
+    manager.run()
diff --git a/pyproject.toml b/pyproject.toml
@@ -13,7 +13,7 @@ description = """\
 license = {text = "MIT"}
 
 requires-python = ">=3.8"
-dependencies = ["wcwidth", "typing_extensions"]
+dependencies = ["wcwidth>=0.5", "typing_extensions"]
 
 keywords = [
     "terminal",
diff --git a/pytermgui/helpers.py b/pytermgui/helpers.py
@@ -4,110 +4,57 @@
 
 from typing import Iterator
 
-from .markup import tokenize_ansi
-from .markup.parsing import LINK_TEMPLATE, PARSERS
+from wcwidth import wrap as wcwidth_wrap
+
 from .regex import real_length
 
 __all__ = [
     "break_line",
 ]
 
 
-def break_line(  # pylint: disable=too-many-branches
+def break_line(
     line: str, limit: int, non_first_limit: int | None = None, fill: str | None = None
 ) -> Iterator[str]:
     """Breaks a line into a `list[str]` with maximum `limit` length per line.
 
-    It keeps ongoing ANSI sequences between lines, and inserts a reset sequence
-    at the end of each style-containing line.
-
-    At the moment it splits strings exactly on the limit, and not on word
-    boundaries. That functionality would be preferred, so it will end up being
-    implemented at some point.
+    Uses wcwidth.wrap() for proper word-boundary breaking, grapheme cluster
+    handling, and wide character support. ANSI sequences are preserved and
+    propagated across line breaks.
 
     Args:
         line: The line to split. May or may not contain ANSI sequences.
         limit: The maximum amount of characters allowed in each line, excluding
             non-printing sequences.
         non_first_limit: The limit after the first line. If not given, defaults
             to `limit`.
+        fill: Optional character to pad lines to the limit width.
     """
 
     if line in ["", "\x1b[0m"]:
         yield ""
         return
 
-    def _pad_and_link(line: str, link: str | None) -> str:
-        count = limit - real_length(line)
-
-        if link is not None:
-            line = LINK_TEMPLATE.format(uri=link, label=line)
-
+    def _pad_line(text: str, width: int) -> str:
         if fill is None:
-            return line
-
-        line += count * fill
-
-        return line
+            return text
 
-    used = 0
-    current = ""
-    sequences = ""
+        count = width - real_length(text)
+        if count > 0:
+            return text + count * fill
+        return text
 
     if non_first_limit is None:
         non_first_limit = limit
 
-    parsers = PARSERS
-    link = None
-
-    for token in tokenize_ansi(line):
-        if token.is_plain():
-            for char in token.value:
-                if char == "\n" or used >= limit:
-                    if sequences != "":
-                        current += "\x1b[0m"
-
-                    yield _pad_and_link(current, link)
-                    link = None
-
-                    current = sequences
-                    used = 0
-
-                    limit = non_first_limit
-
-                if char != "\n":
-                    current += char
-                    used += 1
-
-            # If the link wasn't yielded along with its token, remove and add it
-            # to current manually.
-            if link is not None:
-                current = current[: -len(token.value)]
-                current += LINK_TEMPLATE.format(uri=link, label=token.value)
-                link = None
-
-            continue
-
-        if token.value == "/":
-            sequences = "\x1b[0m"
-
-            if len(current) > 0:
-                current += sequences
-
+    for segment in line.split("\n"):
+        if not segment:
+            yield _pad_line("", limit)
+            limit = non_first_limit
             continue
 
-        if token.is_hyperlink():
-            link = token.value
-            continue
-
-        sequence = parsers[type(token)](token, {}, lambda: line)  # type: ignore
-        sequences += sequence
-        current += sequence
-
-    if current == "":
-        return
-
-    if sequences != "" and not current.endswith("\x1b[0m"):
-        current += "\x1b[0m"
+        wrapped = wcwidth_wrap(segment, limit)
 
-    yield _pad_and_link(current, link)
+        for wrapped_line in wrapped:
+            yield _pad_line(wrapped_line, limit)
+            limit = non_first_limit
diff --git a/pytermgui/widgets/containers.py b/pytermgui/widgets/containers.py
@@ -6,7 +6,6 @@
 
 from __future__ import annotations
 
-from itertools import zip_longest
 from typing import Any, Callable, Iterator, cast
 
 from ..ansi_interface import MouseAction, MouseEvent, clear, reset
@@ -1030,6 +1029,7 @@ def get_lines(self) -> list[str]:  # pylint: disable=too-many-locals
 
         self.positioned_line_buffer = []
         vertical_lines = []
+        column_widths = []
         total_offset = 0
 
         for widget in self._widgets:
@@ -1043,6 +1043,8 @@ def get_lines(self) -> list[str]:  # pylint: disable=too-many-locals
                 width = widget.width
                 error = 0
 
+            column_widths.append(width)
+
             aligned: str | None = None
             for line in widget.get_lines():
                 # See `enums.py` for information about this ignore
@@ -1073,8 +1075,16 @@ def get_lines(self) -> list[str]:  # pylint: disable=too-many-locals
 
             vertical_lines.append(inner)
 
+        # Pad columns to max height using each column's actual width.
+        # (target_width is mutated above, so we can't use it as a fillvalue)
+        max_height = max(len(col) for col in vertical_lines) if vertical_lines else 0
+        for i, col in enumerate(vertical_lines):
+            fill = " " * column_widths[i]
+            while len(col) < max_height:
+                col.append(fill)
+
         lines = []
-        for horizontal in zip_longest(*vertical_lines, fillvalue=" " * target_width):
+        for horizontal in zip(*vertical_lines):
             lines.append((reset() + separator).join(horizontal))
 
         self.height = max(widget.height for widget in self)
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
diff --git a/tests/test_wcwidth_integration.py b/tests/test_wcwidth_integration.py
