gh-154686: Add text_len to TextWrapper

Doc/library/textwrap.rst

@@ -19,7 +19,7 @@ functions should be good enough; otherwise, you should use an instance of
                    replace_whitespace=True, fix_sentence_endings=False, \
                    break_long_words=True, drop_whitespace=True, \
                    break_on_hyphens=True, tabsize=8, max_lines=None, \
-                   placeholder=' [...]')
+                   placeholder=' [...]', text_len=len)
 
    Wraps the single paragraph in *text* (a string) so every line is at most
    *width* characters long.  Returns a list of output lines, without final
@@ -37,7 +37,7 @@ functions should be good enough; otherwise, you should use an instance of
                    replace_whitespace=True, fix_sentence_endings=False, \
                    break_long_words=True, drop_whitespace=True, \
                    break_on_hyphens=True, tabsize=8, \
-                   max_lines=None, placeholder=' [...]')
+                   max_lines=None, placeholder=' [...]', text_len=len)
 
    Wraps the single paragraph in *text*, and returns a single string containing the
    wrapped paragraph.  :func:`fill` is shorthand for  ::
@@ -50,7 +50,7 @@ functions should be good enough; otherwise, you should use an instance of
 
 .. function:: shorten(text, width, *, fix_sentence_endings=False, \
                       break_long_words=True, break_on_hyphens=True, \
-                      placeholder=' [...]')
+                      placeholder=' [...]', text_len=len)
 
    Collapse and truncate the given *text* to fit in the given *width*.
 
@@ -293,6 +293,31 @@ hyphenated words; only then will long words be broken if necessary, unless
       .. versionadded:: 3.4
 
 
+   .. attribute:: text_len
+
+      (default: :func:`len`) Callable used to measure the width of a string
+      when deciding where to wrap.  Override the default to account for
+      characters that are not a single column wide, such as double-width East
+      Asian characters or zero-width combining characters::
+
+         >>> import textwrap, unicodedata
+         >>> def display_width(text):
+         ...     return sum(
+         ...         2 if unicodedata.east_asian_width(char) in 'FW' else 1
+         ...         for char in text)
+         ...
+         >>> textwrap.wrap('東京 大阪 京都 札幌', width=10)
+         ['東京 大阪 京都', '札幌']
+         >>> textwrap.wrap('東京 大阪 京都 札幌', width=10, text_len=display_width)
+         ['東京 大阪', '京都 札幌']
+
+      The callable must return a non-negative integer, and it must be
+      additive: the width of a string is the sum of the widths of its
+      characters.
+
+      .. versionadded:: next
+
+
    :class:`TextWrapper` also provides some public methods, analogous to the
    module-level convenience functions:
 

Doc/whatsnew/3.16.rst

@@ -758,6 +758,16 @@ symtable
   (Contributed by Irit Katriel in :gh:`124697`.)
 
 
+textwrap
+--------
+
+* Add a *text_len* parameter to :func:`textwrap.wrap`, :func:`textwrap.fill`,
+  :func:`textwrap.shorten`, and :class:`textwrap.TextWrapper`.  It customizes
+  how the width of a string is measured, so text that contains double-width
+  or zero-width characters can be wrapped correctly.
+  (Contributed by Kevin Deldycke in :gh:`154686`.)
+
+
 tkinter
 -------
 

Lib/idlelib/idle_test/test_calltip.py

@@ -105,7 +105,7 @@ def test_signature_wrap(self):
 (width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
     replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
     drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None,
-    placeholder=' [...]')
+    placeholder=' [...]', text_len=<built-in function len>)
 Object for wrapping/filling text.  The public interface consists of
 the wrap() and fill() methods; the other methods are just there for
 subclasses to override in order to tweak the default behaviour.

Lib/test/test_textwrap.py

@@ -8,6 +8,7 @@
 # $Id$
 #
 
+import unicodedata
 import unittest
 
 from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten
@@ -1133,5 +1134,90 @@ def test_first_word_too_long_but_placeholder_fits(self):
         self.check_shorten("Helloo", 5, "[...]")
 
 
+class TextLenTestCase(BaseTestCase):
+    # The text_len option measures the width of a string, for text with
+    # characters that are not one column wide.
+
+    @staticmethod
+    def display_width(text):
+        # East Asian wide characters take two columns, combining characters
+        # take none.
+        return sum(
+            0 if unicodedata.combining(char)
+            else 2 if unicodedata.east_asian_width(char) in "FW"
+            else 1
+            for char in text
+        )
+
+    def check_shorten(self, text, width, expect, **kwargs):
+        self.check(shorten(text, width, **kwargs), expect)
+
+    def test_default_text_len_is_len(self):
+        self.assertIs(TextWrapper().text_len, len)
+
+    def test_explicit_len_matches_default(self):
+        text = "Hello there, how are you this fine day?  I'm glad to hear it!"
+        self.check_wrap(text, 12, wrap(text, 12), text_len=len)
+
+    def test_wide_characters(self):
+        text = "東京 大阪 京都 札幌"
+        self.check_wrap(text, 10, ["東京 大阪 京都", "札幌"])
+        self.check_wrap(text, 10, ["東京 大阪", "京都 札幌"],
+                        text_len=self.display_width)
+
+    def test_zero_width_characters(self):
+        text = "cafe\u0301 cafe\u0301 cafe\u0301"
+        self.check_wrap(text, 10, ["cafe\u0301"] * 3)
+        self.check_wrap(text, 10, ["cafe\u0301 cafe\u0301", "cafe\u0301"],
+                        text_len=self.display_width)
+
+    def test_lines_fit_width(self):
+        text = "東京 大阪 名古屋 札幌 福岡 神戸 京都 横浜 仙台 広島"
+        for width in range(2, 30):
+            with self.subTest(width=width):
+                for line in wrap(text, width, text_len=self.display_width):
+                    self.assertLessEqual(self.display_width(line), width)
+
+    def test_indent_width(self):
+        self.check_wrap("aa bb cc", 7, ["東東aa bb", "東東cc"],
+                        initial_indent="東東", subsequent_indent="東東")
+        self.check_wrap("aa bb cc", 7, ["東東aa", "東東bb", "東東cc"],
+                        initial_indent="東東", subsequent_indent="東東",
+                        text_len=self.display_width)
+
+    def test_break_long_word_by_width(self):
+        word = "東" * 10
+        self.check_wrap(word, 8, ["東" * 8, "東" * 2])
+        self.check_wrap(word, 8, ["東" * 4, "東" * 4, "東" * 2],
+                        text_len=self.display_width)
+
+    def test_break_long_word_keeps_one_character(self):
+        # A character wider than the line still goes on a line of its own.
+        self.check_wrap("東東", 1, ["東", "東"], text_len=self.display_width)
+
+    def test_break_long_word_moves_wide_character(self):
+        # A wide character that does not fit at the end of a line moves to
+        # the next one.
+        self.check_wrap("aaaa 東京大阪", 6, ["aaaa", "東京大阪"])
+        self.check_wrap("aaaa 東京大阪", 6, ["aaaa", "東京大", "阪"],
+                        text_len=self.display_width)
+
+    def test_doubled_measure_and_width_keep_layout(self):
+        # Doubling both the measure and the width must not move any line
+        # break, including a hyphen break inside an over-long word.
+        double = lambda s: 2 * len(s)
+        for text in ("aa bb cc dd", "1234-5678-9012", "spam-egg-ham-bacon"):
+            for width in range(1, len(text) + 1):
+                with self.subTest(text=text, width=width):
+                    self.assertEqual(wrap(text, 2 * width, text_len=double),
+                                     wrap(text, width))
+
+    def test_shorten_with_text_len(self):
+        text = "東京 大阪 京都 札幌"
+        self.check_shorten(text, 12, text)
+        self.check_shorten(text, 12, "東京 [...]",
+                           text_len=self.display_width)
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/textwrap.py

@@ -61,6 +61,11 @@ class TextWrapper:
         Truncate wrapped lines.
       placeholder (default: ' [...]')
         Append to the last line of truncated text.
+      text_len (default: len)
+        Callable returning the width of a string.  Override the default to
+        account for characters that are not one column wide, such as
+        double-width or zero-width characters.  The width of a string must
+        be the sum of the widths of its characters.
     """
 
     unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(' '))
@@ -122,7 +127,8 @@ def __init__(self,
                  tabsize=8,
                  *,
                  max_lines=None,
-                 placeholder=' [...]'):
+                 placeholder=' [...]',
+                 text_len=len):
         self.width = width
         self.initial_indent = initial_indent
         self.subsequent_indent = subsequent_indent
@@ -135,6 +141,7 @@ def __init__(self,
         self.tabsize = tabsize
         self.max_lines = max_lines
         self.placeholder = placeholder
+        self.text_len = text_len
 
 
     # -- Private methods -----------------------------------------------
@@ -194,6 +201,26 @@ def _fix_sentence_endings(self, chunks):
             else:
                 i += 1
 
+    def _truncate_to_width(self, text, width):
+        """_truncate_to_width(text : string, width : int) -> string
+
+        Return the longest prefix of *text* whose width, as measured by
+        self.text_len, does not exceed *width*.  The prefix is empty when
+        the first character is wider than *width*.
+        """
+        # Fast path for the default len(): the width is the number of
+        # characters, so the prefix can be sliced directly.
+        if self.text_len is len:
+            return text[:width]
+        # text_len is additive: the width of a prefix is the sum of the
+        # widths of its characters.
+        prefix_width = 0
+        for i, char in enumerate(text):
+            prefix_width += self.text_len(char)
+            if prefix_width > width:
+                return text[:i]
+        return text
+
     def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
         """_handle_long_word(chunks : [string],
                              cur_line : [string],
@@ -212,16 +239,22 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width):
         # If we're allowed to break long words, then do so: put as much
         # of the next chunk onto the current line as will fit.
         if self.break_long_words and space_left > 0:
-            end = space_left
             chunk = reversed_chunks[-1]
-            if self.break_on_hyphens and len(chunk) > space_left:
+            # Keep as many leading characters as fit in the width.  A wide
+            # character that does not fit moves to the next line, but an
+            # empty line takes one anyway, so that wrapping makes progress.
+            end = len(self._truncate_to_width(chunk, space_left))
+            if not end and not cur_line:
+                end = 1
+            if self.break_on_hyphens and self.text_len(chunk) > space_left:
                 # break after last hyphen, but only if there are
                 # non-hyphens before it
-                hyphen = chunk.rfind('-', 0, space_left)
+                hyphen = chunk.rfind('-', 0, end)
                 if hyphen > 0 and any(c != '-' for c in chunk[:hyphen]):
                     end = hyphen + 1
-            cur_line.append(chunk[:end])
-            reversed_chunks[-1] = chunk[end:]
+            if end:
+                cur_line.append(chunk[:end])
+                reversed_chunks[-1] = chunk[end:]
 
         # Otherwise, we have to preserve the long word intact.  Only add
         # it to the current line if there's nothing already there --
@@ -256,7 +289,8 @@ def _wrap_chunks(self, chunks):
                 indent = self.subsequent_indent
             else:
                 indent = self.initial_indent
-            if len(indent) + len(self.placeholder.lstrip()) > self.width:
+            if (self.text_len(indent) +
+                    self.text_len(self.placeholder.lstrip()) > self.width):
                 raise ValueError("placeholder too large for max width")
 
         # Arrange in reverse order so items can be efficiently popped
@@ -277,15 +311,15 @@ def _wrap_chunks(self, chunks):
                 indent = self.initial_indent
 
             # Maximum width for this line.
-            width = self.width - len(indent)
+            width = self.width - self.text_len(indent)
 
             # First chunk on line is whitespace -- drop it, unless this
             # is the very beginning of the text (ie. no lines started yet).
             if self.drop_whitespace and chunks[-1].strip() == '' and lines:
                 del chunks[-1]
 
             while chunks:
-                l = len(chunks[-1])
+                l = self.text_len(chunks[-1])
 
                 # Can at least squeeze this chunk onto the current line.
                 if cur_len + l <= width:
@@ -298,13 +332,13 @@ def _wrap_chunks(self, chunks):
 
             # The current line is full, and the next chunk is too big to
             # fit on *any* line (not just this one).
-            if chunks and len(chunks[-1]) > width:
+            if chunks and self.text_len(chunks[-1]) > width:
                 self._handle_long_word(chunks, cur_line, cur_len, width)
-                cur_len = sum(map(len, cur_line))
+                cur_len = sum(map(self.text_len, cur_line))
 
             # If the last chunk on this line is all whitespace, drop it.
             if self.drop_whitespace and cur_line and cur_line[-1].strip() == '':
-                cur_len -= len(cur_line[-1])
+                cur_len -= self.text_len(cur_line[-1])
                 del cur_line[-1]
 
             if cur_line:
@@ -320,16 +354,17 @@ def _wrap_chunks(self, chunks):
                 else:
                     while cur_line:
                         if (cur_line[-1].strip() and
-                            cur_len + len(self.placeholder) <= width):
+                            cur_len + self.text_len(self.placeholder) <= width):
                             cur_line.append(self.placeholder)
                             lines.append(indent + ''.join(cur_line))
                             break
-                        cur_len -= len(cur_line[-1])
+                        cur_len -= self.text_len(cur_line[-1])
                         del cur_line[-1]
                     else:
                         if lines:
                             prev_line = lines[-1].rstrip()
-                            if (len(prev_line) + len(self.placeholder) <=
+                            if (self.text_len(prev_line) +
+                                    self.text_len(self.placeholder) <=
                                     self.width):
                                 lines[-1] = prev_line + self.placeholder
                                 break

Misc/NEWS.d/next/Library/2026-07-25-10-16-22.gh-issue-154686.Cz9Xpw.rst

@@ -0,0 +1,4 @@
+Add a *text_len* parameter to :func:`textwrap.wrap`, :func:`textwrap.fill`,
+:func:`textwrap.shorten`, and :class:`textwrap.TextWrapper`.  It customizes how
+the width of a string is measured, so that text containing double-width or
+zero-width characters can be wrapped correctly.