diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index 93a8c98f0667aa8..ce87fb36912c3eb 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -19,7 +19,7 @@ functions should be good enough; otherwise, you should use an instance of replace_whitespace=True, fix_sentence_endings=False, \ break_long_words=True, drop_whitespace=True, \ break_on_hyphens=True, tabsize=8, max_lines=None, \ - placeholder=' [...]') + placeholder=' [...]', text_len=len) Wraps the single paragraph in *text* (a string) so every line is at most *width* characters long. Returns a list of output lines, without final @@ -37,7 +37,7 @@ functions should be good enough; otherwise, you should use an instance of replace_whitespace=True, fix_sentence_endings=False, \ break_long_words=True, drop_whitespace=True, \ break_on_hyphens=True, tabsize=8, \ - max_lines=None, placeholder=' [...]') + max_lines=None, placeholder=' [...]', text_len=len) Wraps the single paragraph in *text*, and returns a single string containing the wrapped paragraph. :func:`fill` is shorthand for :: @@ -50,7 +50,7 @@ functions should be good enough; otherwise, you should use an instance of .. function:: shorten(text, width, *, fix_sentence_endings=False, \ break_long_words=True, break_on_hyphens=True, \ - placeholder=' [...]') + placeholder=' [...]', text_len=len) Collapse and truncate the given *text* to fit in the given *width*. @@ -293,6 +293,31 @@ hyphenated words; only then will long words be broken if necessary, unless .. versionadded:: 3.4 + .. attribute:: text_len + + (default: :func:`len`) Callable used to measure the width of a string + when deciding where to wrap. Override the default to account for + characters that are not a single column wide, such as double-width East + Asian characters or zero-width combining characters:: + + >>> import textwrap, unicodedata + >>> def display_width(text): + ... return sum( + ... 2 if unicodedata.east_asian_width(char) in 'FW' else 1 + ... for char in text) + ... + >>> textwrap.wrap('東京 大阪 京都 札幌', width=10) + ['東京 大阪 京都', '札幌'] + >>> textwrap.wrap('東京 大阪 京都 札幌', width=10, text_len=display_width) + ['東京 大阪', '京都 札幌'] + + The callable must return a non-negative integer, and it must be + additive: the width of a string is the sum of the widths of its + characters. + + .. versionadded:: next + + :class:`TextWrapper` also provides some public methods, analogous to the module-level convenience functions: diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 4af5d3c0bd072e2..8f4f3bc7289c135 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -758,6 +758,16 @@ symtable (Contributed by Irit Katriel in :gh:`124697`.) +textwrap +-------- + +* Add a *text_len* parameter to :func:`textwrap.wrap`, :func:`textwrap.fill`, + :func:`textwrap.shorten`, and :class:`textwrap.TextWrapper`. It customizes + how the width of a string is measured, so text that contains double-width + or zero-width characters can be wrapped correctly. + (Contributed by Kevin Deldycke in :gh:`154686`.) + + tkinter ------- diff --git a/Lib/idlelib/idle_test/test_calltip.py b/Lib/idlelib/idle_test/test_calltip.py index e216ae67f68b4d1..6b570f70be57ba8 100644 --- a/Lib/idlelib/idle_test/test_calltip.py +++ b/Lib/idlelib/idle_test/test_calltip.py @@ -105,7 +105,7 @@ def test_signature_wrap(self): (width=70, initial_indent='', subsequent_indent='', expand_tabs=True, replace_whitespace=True, fix_sentence_endings=False, break_long_words=True, drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None, - placeholder=' [...]') + placeholder=' [...]', text_len=) Object for wrapping/filling text. The public interface consists of the wrap() and fill() methods; the other methods are just there for subclasses to override in order to tweak the default behaviour. diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index aca1f427656bb50..bc5de4e8026c247 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -8,6 +8,7 @@ # $Id$ # +import unicodedata import unittest from textwrap import TextWrapper, wrap, fill, dedent, indent, shorten @@ -1133,5 +1134,90 @@ def test_first_word_too_long_but_placeholder_fits(self): self.check_shorten("Helloo", 5, "[...]") +class TextLenTestCase(BaseTestCase): + # The text_len option measures the width of a string, for text with + # characters that are not one column wide. + + @staticmethod + def display_width(text): + # East Asian wide characters take two columns, combining characters + # take none. + return sum( + 0 if unicodedata.combining(char) + else 2 if unicodedata.east_asian_width(char) in "FW" + else 1 + for char in text + ) + + def check_shorten(self, text, width, expect, **kwargs): + self.check(shorten(text, width, **kwargs), expect) + + def test_default_text_len_is_len(self): + self.assertIs(TextWrapper().text_len, len) + + def test_explicit_len_matches_default(self): + text = "Hello there, how are you this fine day? I'm glad to hear it!" + self.check_wrap(text, 12, wrap(text, 12), text_len=len) + + def test_wide_characters(self): + text = "東京 大阪 京都 札幌" + self.check_wrap(text, 10, ["東京 大阪 京都", "札幌"]) + self.check_wrap(text, 10, ["東京 大阪", "京都 札幌"], + text_len=self.display_width) + + def test_zero_width_characters(self): + text = "cafe\u0301 cafe\u0301 cafe\u0301" + self.check_wrap(text, 10, ["cafe\u0301"] * 3) + self.check_wrap(text, 10, ["cafe\u0301 cafe\u0301", "cafe\u0301"], + text_len=self.display_width) + + def test_lines_fit_width(self): + text = "東京 大阪 名古屋 札幌 福岡 神戸 京都 横浜 仙台 広島" + for width in range(2, 30): + with self.subTest(width=width): + for line in wrap(text, width, text_len=self.display_width): + self.assertLessEqual(self.display_width(line), width) + + def test_indent_width(self): + self.check_wrap("aa bb cc", 7, ["東東aa bb", "東東cc"], + initial_indent="東東", subsequent_indent="東東") + self.check_wrap("aa bb cc", 7, ["東東aa", "東東bb", "東東cc"], + initial_indent="東東", subsequent_indent="東東", + text_len=self.display_width) + + def test_break_long_word_by_width(self): + word = "東" * 10 + self.check_wrap(word, 8, ["東" * 8, "東" * 2]) + self.check_wrap(word, 8, ["東" * 4, "東" * 4, "東" * 2], + text_len=self.display_width) + + def test_break_long_word_keeps_one_character(self): + # A character wider than the line still goes on a line of its own. + self.check_wrap("東東", 1, ["東", "東"], text_len=self.display_width) + + def test_break_long_word_moves_wide_character(self): + # A wide character that does not fit at the end of a line moves to + # the next one. + self.check_wrap("aaaa 東京大阪", 6, ["aaaa", "東京大阪"]) + self.check_wrap("aaaa 東京大阪", 6, ["aaaa", "東京大", "阪"], + text_len=self.display_width) + + def test_doubled_measure_and_width_keep_layout(self): + # Doubling both the measure and the width must not move any line + # break, including a hyphen break inside an over-long word. + double = lambda s: 2 * len(s) + for text in ("aa bb cc dd", "1234-5678-9012", "spam-egg-ham-bacon"): + for width in range(1, len(text) + 1): + with self.subTest(text=text, width=width): + self.assertEqual(wrap(text, 2 * width, text_len=double), + wrap(text, width)) + + def test_shorten_with_text_len(self): + text = "東京 大阪 京都 札幌" + self.check_shorten(text, 12, text) + self.check_shorten(text, 12, "東京 [...]", + text_len=self.display_width) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 2f213e34c2c329f..fa7c328fd4d3023 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -61,6 +61,11 @@ class TextWrapper: Truncate wrapped lines. placeholder (default: ' [...]') Append to the last line of truncated text. + text_len (default: len) + Callable returning the width of a string. Override the default to + account for characters that are not one column wide, such as + double-width or zero-width characters. The width of a string must + be the sum of the widths of its characters. """ unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(' ')) @@ -122,7 +127,8 @@ def __init__(self, tabsize=8, *, max_lines=None, - placeholder=' [...]'): + placeholder=' [...]', + text_len=len): self.width = width self.initial_indent = initial_indent self.subsequent_indent = subsequent_indent @@ -135,6 +141,7 @@ def __init__(self, self.tabsize = tabsize self.max_lines = max_lines self.placeholder = placeholder + self.text_len = text_len # -- Private methods ----------------------------------------------- @@ -194,6 +201,26 @@ def _fix_sentence_endings(self, chunks): else: i += 1 + def _truncate_to_width(self, text, width): + """_truncate_to_width(text : string, width : int) -> string + + Return the longest prefix of *text* whose width, as measured by + self.text_len, does not exceed *width*. The prefix is empty when + the first character is wider than *width*. + """ + # Fast path for the default len(): the width is the number of + # characters, so the prefix can be sliced directly. + if self.text_len is len: + return text[:width] + # text_len is additive: the width of a prefix is the sum of the + # widths of its characters. + prefix_width = 0 + for i, char in enumerate(text): + prefix_width += self.text_len(char) + if prefix_width > width: + return text[:i] + return text + def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): """_handle_long_word(chunks : [string], cur_line : [string], @@ -212,16 +239,22 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. if self.break_long_words and space_left > 0: - end = space_left chunk = reversed_chunks[-1] - if self.break_on_hyphens and len(chunk) > space_left: + # Keep as many leading characters as fit in the width. A wide + # character that does not fit moves to the next line, but an + # empty line takes one anyway, so that wrapping makes progress. + end = len(self._truncate_to_width(chunk, space_left)) + if not end and not cur_line: + end = 1 + if self.break_on_hyphens and self.text_len(chunk) > space_left: # break after last hyphen, but only if there are # non-hyphens before it - hyphen = chunk.rfind('-', 0, space_left) + hyphen = chunk.rfind('-', 0, end) if hyphen > 0 and any(c != '-' for c in chunk[:hyphen]): end = hyphen + 1 - cur_line.append(chunk[:end]) - reversed_chunks[-1] = chunk[end:] + if end: + cur_line.append(chunk[:end]) + reversed_chunks[-1] = chunk[end:] # Otherwise, we have to preserve the long word intact. Only add # it to the current line if there's nothing already there -- @@ -256,7 +289,8 @@ def _wrap_chunks(self, chunks): indent = self.subsequent_indent else: indent = self.initial_indent - if len(indent) + len(self.placeholder.lstrip()) > self.width: + if (self.text_len(indent) + + self.text_len(self.placeholder.lstrip()) > self.width): raise ValueError("placeholder too large for max width") # Arrange in reverse order so items can be efficiently popped @@ -277,7 +311,7 @@ def _wrap_chunks(self, chunks): indent = self.initial_indent # Maximum width for this line. - width = self.width - len(indent) + width = self.width - self.text_len(indent) # First chunk on line is whitespace -- drop it, unless this # is the very beginning of the text (ie. no lines started yet). @@ -285,7 +319,7 @@ def _wrap_chunks(self, chunks): del chunks[-1] while chunks: - l = len(chunks[-1]) + l = self.text_len(chunks[-1]) # Can at least squeeze this chunk onto the current line. if cur_len + l <= width: @@ -298,13 +332,13 @@ def _wrap_chunks(self, chunks): # The current line is full, and the next chunk is too big to # fit on *any* line (not just this one). - if chunks and len(chunks[-1]) > width: + if chunks and self.text_len(chunks[-1]) > width: self._handle_long_word(chunks, cur_line, cur_len, width) - cur_len = sum(map(len, cur_line)) + cur_len = sum(map(self.text_len, cur_line)) # If the last chunk on this line is all whitespace, drop it. if self.drop_whitespace and cur_line and cur_line[-1].strip() == '': - cur_len -= len(cur_line[-1]) + cur_len -= self.text_len(cur_line[-1]) del cur_line[-1] if cur_line: @@ -320,16 +354,17 @@ def _wrap_chunks(self, chunks): else: while cur_line: if (cur_line[-1].strip() and - cur_len + len(self.placeholder) <= width): + cur_len + self.text_len(self.placeholder) <= width): cur_line.append(self.placeholder) lines.append(indent + ''.join(cur_line)) break - cur_len -= len(cur_line[-1]) + cur_len -= self.text_len(cur_line[-1]) del cur_line[-1] else: if lines: prev_line = lines[-1].rstrip() - if (len(prev_line) + len(self.placeholder) <= + if (self.text_len(prev_line) + + self.text_len(self.placeholder) <= self.width): lines[-1] = prev_line + self.placeholder break diff --git a/Misc/NEWS.d/next/Library/2026-07-25-10-16-22.gh-issue-154686.Cz9Xpw.rst b/Misc/NEWS.d/next/Library/2026-07-25-10-16-22.gh-issue-154686.Cz9Xpw.rst new file mode 100644 index 000000000000000..97c527c99b839cc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-25-10-16-22.gh-issue-154686.Cz9Xpw.rst @@ -0,0 +1,4 @@ +Add a *text_len* parameter to :func:`textwrap.wrap`, :func:`textwrap.fill`, +:func:`textwrap.shorten`, and :class:`textwrap.TextWrapper`. It customizes how +the width of a string is measured, so that text containing double-width or +zero-width characters can be wrapped correctly.