From 6aa88a2cb36240fe2b587f2e82043873270a27cf Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Mon, 31 Mar 2025 01:35:12 +0100 Subject: gh-130167: Optimise ``textwrap.dedent()`` (#131919) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Marius Juston Co-authored-by: Pieter Eendebak Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_textwrap.py | 50 ++++++++++++++++++++++ Lib/textwrap.py | 50 ++++++---------------- .../2025-03-30-19-55-10.gh-issue-131792.NNjzFA.rst | 5 +++ 3 files changed, 68 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-03-30-19-55-10.gh-issue-131792.NNjzFA.rst diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index dfbc2b9..7736698 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -769,6 +769,56 @@ class DedentTestCase(unittest.TestCase): """assert that dedent() has no effect on 'text'""" self.assertEqual(text, dedent(text)) + def test_dedent_whitespace(self): + # The empty string. + text = "" + self.assertUnchanged(text) + + # Only spaces. + text = " " + expect = "" + self.assertEqual(expect, dedent(text)) + + # Only tabs. + text = "\t\t\t\t" + expect = "" + self.assertEqual(expect, dedent(text)) + + # A mixture. + text = " \t \t\t \t " + expect = "" + self.assertEqual(expect, dedent(text)) + + # ASCII whitespace. + text = "\f\n\r\t\v " + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # One newline. + text = "\n" + expect = "\n" + self.assertEqual(expect, dedent(text)) + + # Windows-style newlines. + text = "\r\n" * 5 + expect = "\n" * 5 + self.assertEqual(expect, dedent(text)) + + # Whitespace mixture. + text = " \n\t\n \n\t\t\n\n\n " + expect = "\n\n\n\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Lines consisting only of whitespace are always normalised + text = "a\n \n\t\n" + expect = "a\n\n\n" + self.assertEqual(expect, dedent(text)) + + # Whitespace characters on non-empty lines are retained + text = "a\r\n\r\n\r\n" + expect = "a\r\n\n\n" + self.assertEqual(expect, dedent(text)) + def test_dedent_nomargin(self): # No lines indented. text = "Hello there.\nHow are you?\nOh good, I'm glad." diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 1bf07aa..bb6a118 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -413,9 +413,6 @@ def shorten(text, width, **kwargs): # -- Loosely related functionality ------------------------------------- -_whitespace_only_re = re.compile('^[ \t]+$', re.MULTILINE) -_leading_whitespace_re = re.compile('(^[ \t]*)(?:[^ \t\n])', re.MULTILINE) - def dedent(text): """Remove any common leading whitespace from every line in `text`. @@ -429,42 +426,21 @@ def dedent(text): Entirely blank lines are normalized to a newline character. """ - # Look for the longest leading string of spaces and tabs common to - # all lines. - margin = None - text = _whitespace_only_re.sub('', text) - indents = _leading_whitespace_re.findall(text) - for indent in indents: - if margin is None: - margin = indent - - # Current line more deeply indented than previous winner: - # no change (previous winner is still on top). - elif indent.startswith(margin): - pass - - # Current line consistent with and no deeper than previous winner: - # it's the new winner. - elif margin.startswith(indent): - margin = indent - - # Find the largest common whitespace between current line and previous - # winner. - else: - for i, (x, y) in enumerate(zip(margin, indent)): - if x != y: - margin = margin[:i] - break + if not text: + return text + + lines = text.split('\n') - # sanity check (testing/debugging only) - if 0 and margin: - for line in text.split("\n"): - assert not line or line.startswith(margin), \ - "line = %r, margin = %r" % (line, margin) + # Get length of leading whitespace, inspired by ``os.path.commonprefix()``. + non_blank_lines = [l for l in lines if l and not l.isspace()] + l1 = min(non_blank_lines, default='') + l2 = max(non_blank_lines, default='') + margin = 0 + for margin, c in enumerate(l1): + if c != l2[margin] or c not in ' \t': + break - if margin: - text = re.sub(r'(?m)^' + margin, '', text) - return text + return '\n'.join([l[margin:] if not l.isspace() else '' for l in lines]) def indent(text, prefix, predicate=None): diff --git a/Misc/NEWS.d/next/Library/2025-03-30-19-55-10.gh-issue-131792.NNjzFA.rst b/Misc/NEWS.d/next/Library/2025-03-30-19-55-10.gh-issue-131792.NNjzFA.rst new file mode 100644 index 0000000..62b619c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-30-19-55-10.gh-issue-131792.NNjzFA.rst @@ -0,0 +1,5 @@ +Improved performance of :func:`textwrap.dedent` by an average of ~2.4x, +(with improvements of up to 4x for large inputs), +and fixed a bug where blank lines with whitespace characters other than space +or horizontal tab were not normalised to the newline. +Patch by Adam Turner, Marius Juston, and Pieter Eendebak. -- cgit v0.12