bpo-32874: IDLE: add tests for pyparse (GH-5755)

There are no code changes other than comments and docstrings.

3 files changed, 619 insertions, 66 deletions

diff --git a/Lib/idlelib/idle_test/test_pyparse.py b/Lib/idlelib/idle_test/test_pyparse.py
new file mode 100644
index 0000000..b84e9f8
--- /dev/null
+++ b/Lib/idlelib/idle_test/test_pyparse.py
@@ -0,0 +1,523 @@
+"""Unittest for idlelib.pyparse.py."""
+
+from collections import namedtuple
+import unittest
+from idlelib import pyparse
+
+
+class StringTranslatePseudoMappingTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        whitespace_chars = ' \t\n\r'
+        cls.preserve_dict = {ord(c): ord(c) for c in whitespace_chars}
+        cls.default = ord('x')
+        cls.mapping = pyparse.StringTranslatePseudoMapping(
+                                cls.preserve_dict, default_value=ord('x'))
+
+    @classmethod
+    def tearDownClass(cls):
+        del cls.preserve_dict, cls.default, cls.mapping
+
+    def test__init__(self):
+        m = self.mapping
+        self.assertEqual(m._non_defaults, self.preserve_dict)
+        self.assertEqual(m._default_value, self.default)
+
+    def test__get_item__(self):
+        self.assertEqual(self.mapping[ord('\t')], ord('\t'))
+        self.assertEqual(self.mapping[ord('a')], self.default)
+
+    def test__len__(self):
+        self.assertEqual(len(self.mapping), len(self.preserve_dict))
+
+    def test__iter__(self):
+        count = 0
+        for key, value in self.mapping.items():
+            self.assertIn(key, self.preserve_dict)
+            count += 1
+        self.assertEqual(count, len(self.mapping))
+
+    def test_get(self):
+        self.assertEqual(self.mapping.get(ord('\t')), ord('\t'))
+        self.assertEqual(self.mapping.get('a'), self.default)
+        # Default is a parameter, but it isn't used.
+        self.assertEqual(self.mapping.get('a', default=500), self.default)
+
+
+class PyParseTest(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.parser = pyparse.Parser(indentwidth=4, tabwidth=4)
+
+    @classmethod
+    def tearDownClass(cls):
+        del cls.parser
+
+    def test_init(self):
+        self.assertEqual(self.parser.indentwidth, 4)
+        self.assertEqual(self.parser.tabwidth, 4)
+
+    def test_set_str(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+
+        # Not empty and doesn't end with newline.
+        with self.assertRaises(AssertionError):
+            setstr('a')
+
+        tests = ('',
+                 'a\n')
+
+        for string in tests:
+            with self.subTest(string=string):
+                setstr(string)
+                eq(p.str, string)
+                eq(p.study_level, 0)
+
+    def test_find_good_parse_start(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        start = p.find_good_parse_start
+
+        # Split def across lines.
+        setstr('"""This is a module docstring"""\n'
+               'class C():\n'
+               '    def __init__(self, a,\n'
+               '                 b=True):\n'
+               '        pass\n'
+               )
+
+        # No value sent for is_char_in_string().
+        self.assertIsNone(start())
+
+        # Make text look like a string.  This returns pos as the start
+        # position, but it's set to None.
+        self.assertIsNone(start(is_char_in_string=lambda index: True))
+
+        # Make all text look like it's not in a string.  This means that it
+        # found a good start position.
+        eq(start(is_char_in_string=lambda index: False), 44)
+
+        # If the beginning of the def line is not in a string, then it
+        # returns that as the index.
+        eq(start(is_char_in_string=lambda index: index > 44), 44)
+        # If the beginning of the def line is in a string, then it
+        # looks for a previous index.
+        eq(start(is_char_in_string=lambda index: index >= 44), 33)
+        # If everything before the 'def' is in a string, then returns None.
+        # The non-continuation def line returns 44 (see below).
+        eq(start(is_char_in_string=lambda index: index < 44), None)
+
+        # Code without extra line break in def line - mostly returns the same
+        # values.
+        setstr('"""This is a module docstring"""\n'
+               'class C():\n'
+               '    def __init__(self, a, b=True):\n'
+               '        pass\n'
+               )
+        eq(start(is_char_in_string=lambda index: False), 44)
+        eq(start(is_char_in_string=lambda index: index > 44), 44)
+        eq(start(is_char_in_string=lambda index: index >= 44), 33)
+        # When the def line isn't split, this returns which doesn't match the
+        # split line test.
+        eq(start(is_char_in_string=lambda index: index < 44), 44)
+
+    def test_set_lo(self):
+        code = (
+                '"""This is a module docstring"""\n'
+                'class C():\n'
+                '    def __init__(self, a,\n'
+                '                 b=True):\n'
+                '        pass\n'
+                )
+        p = self.parser
+        p.set_str(code)
+
+        # Previous character is not a newline.
+        with self.assertRaises(AssertionError):
+            p.set_lo(5)
+
+        # A value of 0 doesn't change self.str.
+        p.set_lo(0)
+        self.assertEqual(p.str, code)
+
+        # An index that is preceded by a newline.
+        p.set_lo(44)
+        self.assertEqual(p.str, code[44:])
+
+    def test_tran(self):
+        self.assertEqual('\t a([{b}])b"c\'d\n'.translate(self.parser._tran),
+                          'xxx(((x)))x"x\'x\n')
+
+    def test_study1(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        study = p._study1
+
+        (NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
+        TestInfo = namedtuple('TestInfo', ['string', 'goodlines',
+                                           'continuation'])
+        tests = (
+            TestInfo('', [0], NONE),
+            # Docstrings.
+            TestInfo('"""This is a complete docstring."""\n', [0, 1], NONE),
+            TestInfo("'''This is a complete docstring.'''\n", [0, 1], NONE),
+            TestInfo('"""This is a continued docstring.\n', [0, 1], FIRST),
+            TestInfo("'''This is a continued docstring.\n", [0, 1], FIRST),
+            TestInfo('"""Closing quote does not match."\n', [0, 1], FIRST),
+            TestInfo('"""Bracket in docstring [\n', [0, 1], FIRST),
+            TestInfo("'''Incomplete two line docstring.\n\n", [0, 2], NEXT),
+            # Single-quoted strings.
+            TestInfo('"This is a complete string."\n', [0, 1], NONE),
+            TestInfo('"This is an incomplete string.\n', [0, 1], NONE),
+            TestInfo("'This is more incomplete.\n\n", [0, 1, 2], NONE),
+            # Comment (backslash does not continue comments).
+            TestInfo('# Comment\\\n', [0, 1], NONE),
+            # Brackets.
+            TestInfo('("""Complete string in bracket"""\n', [0, 1], BRACKET),
+            TestInfo('("""Open string in bracket\n', [0, 1], FIRST),
+            TestInfo('a = (1 + 2) - 5 *\\\n', [0, 1], BACKSLASH),  # No bracket.
+            TestInfo('\n   def function1(self, a,\n                 b):\n',
+                     [0, 1, 3], NONE),
+            TestInfo('\n   def function1(self, a,\\\n', [0, 1, 2], BRACKET),
+            TestInfo('\n   def function1(self, a,\n', [0, 1, 2], BRACKET),
+            TestInfo('())\n', [0, 1], NONE),                    # Extra closer.
+            TestInfo(')(\n', [0, 1], BRACKET),                  # Extra closer.
+            # For the mismatched example, it doesn't look like contination.
+            TestInfo('{)(]\n', [0, 1], NONE),                   # Mismatched.
+            )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)  # resets study_level
+                study()
+                eq(p.study_level, 1)
+                eq(p.goodlines, test.goodlines)
+                eq(p.continuation, test.continuation)
+
+        # Called again, just returns without reprocessing.
+        self.assertIsNone(study())
+
+    def test_get_continuation_type(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        gettype = p.get_continuation_type
+
+        (NONE, BACKSLASH, FIRST, NEXT, BRACKET) = range(5)
+        TestInfo = namedtuple('TestInfo', ['string', 'continuation'])
+        tests = (
+            TestInfo('', NONE),
+            TestInfo('"""This is a continuation docstring.\n', FIRST),
+            TestInfo("'''This is a multiline-continued docstring.\n\n", NEXT),
+            TestInfo('a = (1 + 2) - 5 *\\\n', BACKSLASH),
+            TestInfo('\n   def function1(self, a,\\\n', BRACKET)
+            )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(gettype(), test.continuation)
+
+    def test_study2(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        study = p._study2
+
+        TestInfo = namedtuple('TestInfo', ['string', 'start', 'end', 'lastch',
+                                           'openbracket', 'bracketing'])
+        tests = (
+            TestInfo('', 0, 0, '', None, ((0, 0),)),
+            TestInfo("'''This is a multiline continutation docstring.\n\n",
+                     0, 49, "'", None, ((0, 0), (0, 1), (49, 0))),
+            TestInfo(' # Comment\\\n',
+                     0, 12, '', None, ((0, 0), (1, 1), (12, 0))),
+            # A comment without a space is a special case
+            TestInfo(' #Comment\\\n',
+                     0, 0, '', None, ((0, 0),)),
+            # Backslash continuation.
+            TestInfo('a = (1 + 2) - 5 *\\\n',
+                     0, 19, '*', None, ((0, 0), (4, 1), (11, 0))),
+            # Bracket continuation with close.
+            TestInfo('\n   def function1(self, a,\n                 b):\n',
+                     1, 48, ':', None, ((1, 0), (17, 1), (46, 0))),
+            # Bracket continuation with unneeded backslash.
+            TestInfo('\n   def function1(self, a,\\\n',
+                     1, 28, ',', 17, ((1, 0), (17, 1))),
+            # Bracket continuation.
+            TestInfo('\n   def function1(self, a,\n',
+                     1, 27, ',', 17, ((1, 0), (17, 1))),
+            # Bracket continuation with comment at end of line with text.
+            TestInfo('\n   def function1(self, a,  # End of line comment.\n',
+                     1, 51, ',', 17, ((1, 0), (17, 1), (28, 2), (51, 1))),
+            # Multi-line statement with comment line in between code lines.
+            TestInfo('  a = ["first item",\n  # Comment line\n    "next item",\n',
+                     0, 55, ',', 6, ((0, 0), (6, 1), (7, 2), (19, 1),
+                                     (23, 2), (38, 1), (42, 2), (53, 1))),
+            TestInfo('())\n',
+                     0, 4, ')', None, ((0, 0), (0, 1), (2, 0), (3, 0))),
+            TestInfo(')(\n', 0, 3, '(', 1, ((0, 0), (1, 0), (1, 1))),
+            # Wrong closers still decrement stack level.
+            TestInfo('{)(]\n',
+                     0, 5, ']', None, ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+            # Character after backslash.
+            TestInfo(':\\a\n', 0, 4, '\\a', None, ((0, 0),)),
+            TestInfo('\n', 0, 0, '', None, ((0, 0),)),
+            )
+
+        for test in tests:
+            # There is a bug where this is carried forward from last item.
+            p.lastopenbracketpos = None
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                study()
+                eq(p.study_level, 2)
+                eq(p.stmt_start, test.start)
+                eq(p.stmt_end, test.end)
+                eq(p.lastch, test.lastch)
+                eq(p.lastopenbracketpos, test.openbracket)
+                eq(p.stmt_bracketing, test.bracketing)
+
+        # Called again, just returns without reprocessing.
+        self.assertIsNone(study())
+
+    def test_get_num_lines_in_stmt(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        getlines = p.get_num_lines_in_stmt
+
+        TestInfo = namedtuple('TestInfo', ['string', 'lines'])
+        tests = (
+            TestInfo('[x for x in a]\n', 1),      # Closed on one line.
+            TestInfo('[x\nfor x in a\n', 2),      # Not closed.
+            TestInfo('[x\\\nfor x in a\\\n', 2),  # "", uneeded backslashes.
+            TestInfo('[x\nfor x in a\n]\n', 3),   # Closed on multi-line.
+            TestInfo('\n"""Docstring comment L1"""\nL2\nL3\nL4\n', 1),
+            TestInfo('\n"""Docstring comment L1\nL2"""\nL3\nL4\n', 1),
+            TestInfo('\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n', 4),
+            TestInfo('\n\n"""Docstring comment L1\\\nL2\\\nL3\\\nL4\\\n"""\n', 5)
+            )
+
+        # Blank string doesn't have enough elements in goodlines.
+        setstr('')
+        with self.assertRaises(IndexError):
+            getlines()
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(getlines(), test.lines)
+
+    def test_compute_bracket_indent(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        indent = p.compute_bracket_indent
+
+        TestInfo = namedtuple('TestInfo', ['string', 'spaces'])
+        tests = (
+            TestInfo('def function1(self, a,\n', 14),
+            # Characters after bracket.
+            TestInfo('\n    def function1(self, a,\n', 18),
+            TestInfo('\n\tdef function1(self, a,\n', 18),
+            # No characters after bracket.
+            TestInfo('\n    def function1(\n', 8),
+            TestInfo('\n\tdef function1(\n', 8),
+            TestInfo('\n    def function1(  \n', 8),  # Ignore extra spaces.
+            TestInfo('[\n"first item",\n  # Comment line\n    "next item",\n', 0),
+            TestInfo('[\n  "first item",\n  # Comment line\n    "next item",\n', 2),
+            TestInfo('["first item",\n  # Comment line\n    "next item",\n', 1),
+            TestInfo('(\n', 4),
+            TestInfo('(a\n', 1),
+             )
+
+        # Must be C_BRACKET continuation type.
+        setstr('def function1(self, a, b):\n')
+        with self.assertRaises(AssertionError):
+            indent()
+
+        for test in tests:
+            setstr(test.string)
+            eq(indent(), test.spaces)
+
+    def test_compute_backslash_indent(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        indent = p.compute_backslash_indent
+
+        # Must be C_BACKSLASH continuation type.
+        errors = (('def function1(self, a, b\\\n'),  # Bracket.
+                  ('    """ (\\\n'),                 # Docstring.
+                  ('a = #\\\n'),                     # Inline comment.
+                  )
+        for string in errors:
+            with self.subTest(string=string):
+                setstr(string)
+                with self.assertRaises(AssertionError):
+                    indent()
+
+        TestInfo = namedtuple('TestInfo', ('string', 'spaces'))
+        tests = (TestInfo('a = (1 + 2) - 5 *\\\n', 4),
+                 TestInfo('a = 1 + 2 - 5 *\\\n', 4),
+                 TestInfo('    a = 1 + 2 - 5 *\\\n', 8),
+                 TestInfo('  a = "spam"\\\n', 6),
+                 TestInfo('  a = \\\n"a"\\\n', 4),
+                 TestInfo('  a = #\\\n"a"\\\n', 5),
+                 TestInfo('a == \\\n', 2),
+                 TestInfo('a != \\\n', 2),
+                 # Difference between containing = and those not.
+                 TestInfo('\\\n', 2),
+                 TestInfo('    \\\n', 6),
+                 TestInfo('\t\\\n', 6),
+                 TestInfo('a\\\n', 3),
+                 TestInfo('{}\\\n', 4),
+                 TestInfo('(1 + 2) - 5 *\\\n', 3),
+                 )
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(indent(), test.spaces)
+
+    def test_get_base_indent_string(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        baseindent = p.get_base_indent_string
+
+        TestInfo = namedtuple('TestInfo', ['string', 'indent'])
+        tests = (TestInfo('', ''),
+                 TestInfo('def a():\n', ''),
+                 TestInfo('\tdef a():\n', '\t'),
+                 TestInfo('    def a():\n', '    '),
+                 TestInfo('    def a(\n', '    '),
+                 TestInfo('\t\n    def a(\n', '    '),
+                 TestInfo('\t\n    # Comment.\n', '    '),
+                 )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(baseindent(), test.indent)
+
+    def test_is_block_opener(self):
+        yes = self.assertTrue
+        no = self.assertFalse
+        p = self.parser
+        setstr = p.set_str
+        opener = p.is_block_opener
+
+        TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
+        tests = (
+            TestInfo('def a():\n', yes),
+            TestInfo('\n   def function1(self, a,\n                 b):\n', yes),
+            TestInfo(':\n', yes),
+            TestInfo('a:\n', yes),
+            TestInfo('):\n', yes),
+            TestInfo('(:\n', yes),
+            TestInfo('":\n', no),
+            TestInfo('\n   def function1(self, a,\n', no),
+            TestInfo('def function1(self, a):\n    pass\n', no),
+            TestInfo('# A comment:\n', no),
+            TestInfo('"""A docstring:\n', no),
+            TestInfo('"""A docstring:\n', no),
+            )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                test.assert_(opener())
+
+    def test_is_block_closer(self):
+        yes = self.assertTrue
+        no = self.assertFalse
+        p = self.parser
+        setstr = p.set_str
+        closer = p.is_block_closer
+
+        TestInfo = namedtuple('TestInfo', ['string', 'assert_'])
+        tests = (
+            TestInfo('return\n', yes),
+            TestInfo('\tbreak\n', yes),
+            TestInfo('  continue\n', yes),
+            TestInfo('     raise\n', yes),
+            TestInfo('pass    \n', yes),
+            TestInfo('pass\t\n', yes),
+            TestInfo('return #\n', yes),
+            TestInfo('raised\n', no),
+            TestInfo('returning\n', no),
+            TestInfo('# return\n', no),
+            TestInfo('"""break\n', no),
+            TestInfo('"continue\n', no),
+            TestInfo('def function1(self, a):\n    pass\n', yes),
+            )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                test.assert_(closer())
+
+    def test_get_last_open_bracket_pos(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        openbracket = p.get_last_open_bracket_pos
+
+        TestInfo = namedtuple('TestInfo', ['string', 'position'])
+        tests = (
+            TestInfo('', None),
+            TestInfo('a\n', None),
+            TestInfo('# (\n', None),
+            TestInfo('""" (\n', None),
+            TestInfo('a = (1 + 2) - 5 *\\\n', None),
+            TestInfo('\n   def function1(self, a,\n', 17),
+            TestInfo('\n   def function1(self, a,  # End of line comment.\n', 17),
+            TestInfo('{)(]\n', None),
+            TestInfo('(((((((((()))))))\n', 2),
+            TestInfo('(((((((((())\n)))\n))\n', 2),
+            )
+
+        for test in tests:
+            # There is a bug where the value is carried forward from last item.
+            p.lastopenbracketpos = None
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(openbracket(), test.position)
+
+    def test_get_last_stmt_bracketing(self):
+        eq = self.assertEqual
+        p = self.parser
+        setstr = p.set_str
+        bracketing = p.get_last_stmt_bracketing
+
+        TestInfo = namedtuple('TestInfo', ['string', 'bracket'])
+        tests = (
+            TestInfo('', ((0, 0),)),
+            TestInfo('a\n', ((0, 0),)),
+            TestInfo('()()\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+            TestInfo('(\n)()\n', ((0, 0), (0, 1), (3, 0), (3, 1), (5, 0))),
+            TestInfo('()\n()\n', ((3, 0), (3, 1), (5, 0))),
+            TestInfo('()(\n)\n', ((0, 0), (0, 1), (2, 0), (2, 1), (5, 0))),
+            TestInfo('(())\n', ((0, 0), (0, 1), (1, 2), (3, 1), (4, 0))),
+            TestInfo('(\n())\n', ((0, 0), (0, 1), (2, 2), (4, 1), (5, 0))),
+            # Same as matched test.
+            TestInfo('{)(]\n', ((0, 0), (0, 1), (2, 0), (2, 1), (4, 0))),
+            TestInfo('(((())\n',
+                     ((0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (5, 3), (6, 2))),
+            )
+
+        for test in tests:
+            with self.subTest(string=test.string):
+                setstr(test.string)
+                eq(bracketing(), test.bracket)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
diff --git a/Lib/idlelib/pyparse.py b/Lib/idlelib/pyparse.py
index 536b2d7..72bd9e0 100644
--- a/Lib/idlelib/pyparse.py
+++ b/Lib/idlelib/pyparse.py
@@ -1,8 +1,20 @@
+"""Define partial Python code Parser used by editor and hyperparser.
+
+Instances of StringTranslatePseudoMapping are used with str.translate.
+
+The following bound search and match functions are defined:
+_synchre - start of popular statement;
+_junkre - whitespace or comment line;
+_match_stringre: string, possibly without closer;
+_itemre - line that may have bracket structure start;
+_closere - line that must be followed by dedent.
+_chew_ordinaryre - non-special characters.
+"""
 from collections.abc import Mapping
 import re
 import sys
 
-# Reason last stmt is continued (or C_NONE if it's not).
+# Reason last statement is continued (or C_NONE if it's not).
 (C_NONE, C_BACKSLASH, C_STRING_FIRST_LINE,
  C_STRING_NEXT_LINES, C_BRACKET) = range(5)
 
@@ -10,7 +22,7 @@ if 0:   # for throwaway debugging output
     def dump(*stuff):
         sys.__stdout__.write(" ".join(map(str, stuff)) + "\n")
 
-# Find what looks like the start of a popular stmt.
+# Find what looks like the start of a popular statement.
 
 _synchre = re.compile(r"""
     ^
@@ -70,7 +82,7 @@ _itemre = re.compile(r"""
     [^\s#\\]    # if we match, m.end()-1 is the interesting char
 """, re.VERBOSE).match
 
-# Match start of stmts that should be followed by a dedent.
+# Match start of statements that should be followed by a dedent.
 
 _closere = re.compile(r"""
     \s*
@@ -146,19 +158,20 @@ class Parser:
         self.str = s
         self.study_level = 0
 
-    # Return index of a good place to begin parsing, as close to the
-    # end of the string as possible.  This will be the start of some
-    # popular stmt like "if" or "def".  Return None if none found:
-    # the caller should pass more prior context then, if possible, or
-    # if not (the entire program text up until the point of interest
-    # has already been tried) pass 0 to set_lo.
-    #
-    # This will be reliable iff given a reliable is_char_in_string
-    # function, meaning that when it says "no", it's absolutely
-    # guaranteed that the char is not in a string.
-
     def find_good_parse_start(self, is_char_in_string=None,
                               _synchre=_synchre):
+        """
+        Return index of a good place to begin parsing, as close to the
+        end of the string as possible.  This will be the start of some
+        popular stmt like "if" or "def".  Return None if none found:
+        the caller should pass more prior context then, if possible, or
+        if not (the entire program text up until the point of interest
+        has already been tried) pass 0 to set_lo().
+
+        This will be reliable iff given a reliable is_char_in_string()
+        function, meaning that when it says "no", it's absolutely
+        guaranteed that the char is not in a string.
+        """
         str, pos = self.str, None
 
         if not is_char_in_string:
@@ -173,7 +186,7 @@ class Parser:
             i = str.rfind(":\n", 0, limit)
             if i < 0:
                 break
-            i = str.rfind('\n', 0, i) + 1  # start of colon line
+            i = str.rfind('\n', 0, i) + 1  # start of colon line (-1+1=0)
             m = _synchre(str, i, limit)
             if m and not is_char_in_string(m.start()):
                 pos = m.start()
@@ -206,10 +219,11 @@ class Parser:
                 break
         return pos
 
-    # Throw away the start of the string.  Intended to be called with
-    # find_good_parse_start's result.
-
     def set_lo(self, lo):
+        """ Throw away the start of the string.
+
+        Intended to be called with the result of find_good_parse_start().
+        """
         assert lo == 0 or self.str[lo-1] == '\n'
         if lo > 0:
             self.str = self.str[lo:]
@@ -224,11 +238,13 @@ class Parser:
     _tran.update((ord(c), ord(c)) for c in "\"'\\\n#")
     _tran = StringTranslatePseudoMapping(_tran, default_value=ord('x'))
 
-    # As quickly as humanly possible <wink>, find the line numbers (0-
-    # based) of the non-continuation lines.
-    # Creates self.{goodlines, continuation}.
-
     def _study1(self):
+        """Find the line numbers of non-continuation lines.
+
+        As quickly as humanly possible <wink>, find the line numbers (0-
+        based) of the non-continuation lines.
+        Creates self.{goodlines, continuation}.
+        """
         if self.study_level >= 1:
             return
         self.study_level = 1
@@ -244,8 +260,8 @@ class Parser:
         str = str.replace('xx', 'x')
         str = str.replace('xx', 'x')
         str = str.replace('\nx', '\n')
-        # note that replacing x\n with \n would be incorrect, because
-        # x may be preceded by a backslash
+        # Replacing x\n with \n would be incorrect because
+        # x may be preceded by a backslash.
 
         # March over the squashed version of the program, accumulating
         # the line numbers of non-continued stmts, and determining
@@ -360,24 +376,25 @@ class Parser:
         self._study1()
         return self.continuation
 
-    # study1 was sufficient to determine the continuation status,
-    # but doing more requires looking at every character.  study2
-    # does this for the last interesting statement in the block.
-    # Creates:
-    #     self.stmt_start, stmt_end
-    #         slice indices of last interesting stmt
-    #     self.stmt_bracketing
-    #         the bracketing structure of the last interesting stmt;
-    #         for example, for the statement "say(boo) or die", stmt_bracketing
-    #         will be [(0, 0), (3, 1), (8, 0)]. Strings and comments are
-    #         treated as brackets, for the matter.
-    #     self.lastch
-    #         last non-whitespace character before optional trailing
-    #         comment
-    #     self.lastopenbracketpos
-    #         if continuation is C_BRACKET, index of last open bracket
-
     def _study2(self):
+        """
+        study1 was sufficient to determine the continuation status,
+        but doing more requires looking at every character.  study2
+        does this for the last interesting statement in the block.
+        Creates:
+            self.stmt_start, stmt_end
+                slice indices of last interesting stmt
+            self.stmt_bracketing
+                the bracketing structure of the last interesting stmt; for
+                example, for the statement "say(boo) or die",
+                stmt_bracketing will be ((0, 0), (0, 1), (2, 0), (2, 1),
+                (4, 0)). Strings and comments are treated as brackets, for
+                the matter.
+            self.lastch
+                last interesting character before optional trailing comment
+            self.lastopenbracketpos
+                if continuation is C_BRACKET, index of last open bracket
+        """
         if self.study_level >= 2:
             return
         self._study1()
@@ -385,11 +402,11 @@ class Parser:
 
         # Set p and q to slice indices of last interesting stmt.
         str, goodlines = self.str, self.goodlines
-        i = len(goodlines) - 1
-        p = len(str)    # index of newest line
+        i = len(goodlines) - 1  # Index of newest line.
+        p = len(str)  # End of goodlines[i]
         while i:
             assert p
-            # p is the index of the stmt at line number goodlines[i].
+            # Make p be the index of the stmt at line number goodlines[i].
             # Move p back to the stmt at line number goodlines[i-1].
             q = p
             for nothing in range(goodlines[i-1], goodlines[i]):
@@ -483,10 +500,11 @@ class Parser:
             self.lastopenbracketpos = stack[-1]
         self.stmt_bracketing = tuple(bracketing)
 
-    # Assuming continuation is C_BRACKET, return the number
-    # of spaces the next line should be indented.
-
     def compute_bracket_indent(self):
+        """Return number of spaces the next line should be indented.
+
+        Line continuation must be C_BRACKET.
+        """
         self._study2()
         assert self.continuation == C_BRACKET
         j = self.lastopenbracketpos
@@ -513,20 +531,22 @@ class Parser:
             extra = self.indentwidth
         return len(str[i:j].expandtabs(self.tabwidth)) + extra
 
-    # Return number of physical lines in last stmt (whether or not
-    # it's an interesting stmt!  this is intended to be called when
-    # continuation is C_BACKSLASH).
-
     def get_num_lines_in_stmt(self):
+        """Return number of physical lines in last stmt.
+
+        The statement doesn't have to be an interesting statement.  This is
+        intended to be called when continuation is C_BACKSLASH.
+        """
         self._study1()
         goodlines = self.goodlines
         return goodlines[-1] - goodlines[-2]
 
-    # Assuming continuation is C_BACKSLASH, return the number of spaces
-    # the next line should be indented.  Also assuming the new line is
-    # the first one following the initial line of the stmt.
-
     def compute_backslash_indent(self):
+        """Return number of spaces the next line should be indented.
+
+        Line continuation must be C_BACKSLASH.  Also assume that the new
+        line is the first one following the initial line of the stmt.
+        """
         self._study2()
         assert self.continuation == C_BACKSLASH
         str = self.str
@@ -551,6 +571,8 @@ class Parser:
             elif ch == '"' or ch == "'":
                 i = _match_stringre(str, i, endpos).end()
             elif ch == '#':
+                # This line is unreachable because the # makes a comment of
+                # everything after it.
                 break
             elif level == 0 and ch == '=' and \
                    (i == 0 or str[i-1] not in "=<>!") and \
@@ -576,10 +598,10 @@ class Parser:
         return len(str[self.stmt_start:i].expandtabs(\
                                      self.tabwidth)) + 1
 
-    # Return the leading whitespace on the initial line of the last
-    # interesting stmt.
-
     def get_base_indent_string(self):
+        """Return the leading whitespace on the initial line of the last
+        interesting stmt.
+        """
         self._study2()
         i, n = self.stmt_start, self.stmt_end
         j = i
@@ -588,30 +610,37 @@ class Parser:
             j = j + 1
         return str[i:j]
 
-    # Did the last interesting stmt open a block?
-
     def is_block_opener(self):
+        "Return True if the last interesting statemtent opens a block."
         self._study2()
         return self.lastch == ':'
 
-    # Did the last interesting stmt close a block?
-
     def is_block_closer(self):
+        "Return True if the last interesting statement closes a block."
         self._study2()
         return _closere(self.str, self.stmt_start) is not None
 
-    # index of last open bracket ({[, or None if none
+    # XXX - is this used?
     lastopenbracketpos = None
 
     def get_last_open_bracket_pos(self):
+        "Return index of last open bracket or None."
         self._study2()
         return self.lastopenbracketpos
 
-    # the structure of the bracketing of the last interesting statement,
-    # in the format defined in _study2, or None if the text didn't contain
-    # anything
+    # XXX - is this used?
     stmt_bracketing = None
 
     def get_last_stmt_bracketing(self):
+        """Return a tuple of the structure of the bracketing of the last
+        interesting statement.
+
+        Tuple is in the format defined in _study2().
+        """
         self._study2()
         return self.stmt_bracketing
+
+
+if __name__ == '__main__':  #pragma: nocover
+    import unittest
+    unittest.main('idlelib.idle_test.test_pyparse', verbosity=2)
diff --git a/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst b/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst
new file mode 100644
index 0000000..7965531
--- /dev/null
+++ b/Misc/NEWS.d/next/IDLE/2018-02-19-10-56-41.bpo-32874.6pZ9Gv.rst
@@ -0,0 +1 @@
+Add tests for pyparse.