New file by Tim Peters:

One new file in the attached, PyParse.py.  The LineStudier (whatever it was
called <wink>) class was removed from AutoIndent; PyParse subsumes its
functionality.

1 files changed, 470 insertions, 0 deletions

diff --git a/Tools/idle/PyParse.py b/Tools/idle/PyParse.py
new file mode 100644
index 0000000..e3b6e1f
--- /dev/null
+++ b/Tools/idle/PyParse.py
@@ -0,0 +1,470 @@
+import string
+import re
+import sys
+
+# Reason last stmt is continued (or C_NONE if it's not).
+C_NONE, C_BACKSLASH, C_STRING, C_BRACKET = range(4)
+
+if 0:   # for throwaway debugging output
+    def dump(*stuff):
+        import sys
+        sys.__stdout__.write(string.join(map(str, stuff), " ") + "\n")
+
+# find a def or class stmt
+_defclassre = re.compile(r"""
+    ^
+    [ \t]*
+    (?:
+        def   [ \t]+ [a-zA-Z_]\w* [ \t]* \(
+    |   class [ \t]+ [a-zA-Z_]\w* [ \t]*
+        (?: \( .* \) )?
+        [ \t]* :
+    )
+""", re.VERBOSE | re.MULTILINE).search
+
+# match blank line or non-indenting comment line
+_junkre = re.compile(r"""
+    [ \t]*
+    (?: \# [^ \t\n] .* )?
+    \n
+""", re.VERBOSE).match
+
+# match any flavor of string; the terminating quote is optional
+# so that we're robust in the face of incomplete program text
+_match_stringre = re.compile(r"""
+    \""" [^"\\]* (?:
+                     (?: \\. | "(?!"") )
+                     [^"\\]*
+                 )*
+    (?: \""" )?
+
+|   " [^"\\\n]* (?: \\. [^"\\\n]* )* "?
+
+|   ''' [^'\\]* (?:
+                   (?: \\. | '(?!'') )
+                   [^'\\]*
+                )*
+    (?: ''' )?
+
+|   ' [^'\\\n]* (?: \\. [^'\\\n]* )* '?
+""", re.VERBOSE | re.DOTALL).match
+
+# match a line that doesn't start with something interesting;
+# used to skip junk lines when searching for the first element
+# of a bracket structure
+_not_itemre = re.compile(r"""
+    [ \t]*
+    [#\n\\]
+""", re.VERBOSE).match
+
+# match start of stmts that should be followed by a dedent
+_closere = re.compile(r"""
+    \s*
+    (?: return
+    |   break
+    |   continue
+    |   raise
+    |   pass
+    )
+    \b
+""", re.VERBOSE).match
+
+# Build translation table to map uninteresting chars to "x", open
+# brackets to "(", and close brackets to ")".
+
+_tran = ['x'] * 256
+for ch in "({[":
+    _tran[ord(ch)] = '('
+for ch in ")}]":
+    _tran[ord(ch)] = ')'
+for ch in "\"'\\\n#":
+    _tran[ord(ch)] = ch
+_tran = string.join(_tran, '')
+del ch
+
+class Parser:
+
+    def __init__(self, indentwidth, tabwidth):
+        self.indentwidth = indentwidth
+        self.tabwidth = tabwidth
+
+    def set_str(self, str):
+        assert len(str) == 0 or str[-1] == '\n'
+        self.str = str
+        self.study_level = 0
+
+    # Return index of start of last (probable!) def or class stmt, or
+    # None if none found.  It's only probable because we can't know
+    # whether we're in a string without reparsing from the start of
+    # the file -- and that's too slow to bear.
+    #
+    # Ack, hack: in the shell window this kills us, because there's
+    # no way to tell the differences between output, >>> etc and
+    # user input.  Indeed, IDLE's first output line makes the rest
+    # look like it's in an unclosed paren!:
+    # Python 1.5.2 (#0, Apr 13 1999, ...
+
+    def find_last_def_or_class(self, _defclassre=_defclassre):
+        str, pos = self.str, None
+        i = 0
+        while 1:
+            m = _defclassre(str, i)
+            if m:
+                pos, i = m.span()
+            else:
+                break
+        if pos is None:
+            # hack for shell window
+            ps1 = '\n' + sys.ps1
+            i = string.rfind(str, ps1)
+            if i >= 0:
+                pos = i + len(ps1)
+                self.str = str[:pos-1] + '\n' + str[pos:]
+        return pos
+
+    # Throw away the start of the string.  Intended to be called with
+    # find_last_def_or_class's result.
+
+    def set_lo(self, lo):
+        assert lo == 0 or self.str[lo-1] == '\n'
+        if lo > 0:
+            self.str = self.str[lo:]
+
+    # As quickly as humanly possible <wink>, find the line numbers (0-
+    # based) of the non-continuation lines.
+    # Creates self.{stmts, continuation}.
+
+    def _study1(self, _replace=string.replace, _find=string.find):
+        if self.study_level >= 1:
+            return
+        self.study_level = 1
+
+        # Map all uninteresting characters to "x", all open brackets
+        # to "(", all close brackets to ")", then collapse runs of
+        # uninteresting characters.  This can cut the number of chars
+        # by a factor of 10-40, and so greatly speed the following loop.
+        str = self.str
+        str = string.translate(str, _tran)
+        str = _replace(str, 'xxxxxxxx', 'x')
+        str = _replace(str, 'xxxx', 'x')
+        str = _replace(str, 'xx', 'x')
+        str = _replace(str, 'xx', 'x')
+        str = _replace(str, '\nx', '\n')
+        # note that replacing x\n with \n would be incorrect, because
+        # x may be preceded by a backslash
+
+        # March over the squashed version of the program, accumulating
+        # the line numbers of non-continued stmts, and determining
+        # whether & why the last stmt is a continuation.
+        continuation = C_NONE
+        level = lno = 0     # level is nesting level; lno is line number
+        self.stmts = stmts = [0]
+        push_stmt = stmts.append
+        i, n = 0, len(str)
+        while i < n:
+            ch = str[i]
+            # cases are checked in decreasing order of frequency
+
+            if ch == 'x':
+                i = i+1
+                continue
+
+            if ch == '\n':
+                lno = lno + 1
+                if level == 0:
+                    push_stmt(lno)
+                    # else we're in an unclosed bracket structure
+                i = i+1
+                continue
+
+            if ch == '(':
+                level = level + 1
+                i = i+1
+                continue
+
+            if ch == ')':
+                if level:
+                    level = level - 1
+                    # else the program is invalid, but we can't complain
+                i = i+1
+                continue
+
+            if ch == '"' or ch == "'":
+                # consume the string
+                quote = ch
+                if str[i:i+3] == quote * 3:
+                    quote = quote * 3
+                w = len(quote)
+                i = i+w
+                while i < n:
+                    ch = str[i]
+                    if ch == 'x':
+                        i = i+1
+                        continue
+
+                    if str[i:i+w] == quote:
+                        i = i+w
+                        break
+
+                    if ch == '\n':
+                        lno = lno + 1
+                        i = i+1
+                        if w == 1:
+                            # unterminated single-quoted string
+                            if level == 0:
+                                push_stmt(lno)
+                            break
+                        continue
+
+                    if ch == '\\':
+                        assert i+1 < n
+                        if str[i+1] == '\n':
+                            lno = lno + 1
+                        i = i+2
+                        continue
+
+                    # else comment char or paren inside string
+                    i = i+1
+
+                else:
+                    # didn't break out of the loop, so it's an
+                    # unterminated triple-quoted string
+                    assert w == 3
+                    continuation = C_STRING
+                continue
+
+            if ch == '#':
+                # consume the comment
+                i = _find(str, '\n', i)
+                assert i >= 0
+                continue
+
+            assert ch == '\\'
+            assert i+1 < n
+            if str[i+1] == '\n':
+                lno = lno + 1
+                if i+2 == n:
+                    continuation = C_BACKSLASH
+            i = i+2
+
+        # Push the final line number as a sentinel value, regardless of
+        # whether it's continued.
+        if stmts[-1] != lno:
+            push_stmt(lno)
+
+        # The last stmt may be continued for all 3 reasons.
+        # String continuation takes precedence over bracket
+        # continuation, which beats backslash continuation.
+        if continuation != C_STRING and level > 0:
+            continuation = C_BRACKET
+        self.continuation = continuation
+
+    def get_continuation_type(self):
+        self._study1()
+        return self.continuation
+
+    # study1 was sufficient to determine the continuation status,
+    # but doing more requires looking at every character.  study2
+    # does this for the last interesting statement in the block.
+    # Creates:
+    #     self.stmt_start, stmt_end
+    #         slice indices of last interesting stmt
+    #     self.lastch
+    #         last non-whitespace character before optional trailing
+    #         comment
+    #     self.lastopenbracketpos
+    #         if continuation is C_BRACKET, index of last open bracket
+
+    def _study2(self, _rfind=string.rfind, _find=string.find,
+                      _ws=string.whitespace):
+        if self.study_level >= 2:
+            return
+        self._study1()
+        self.study_level = 2
+
+        self.lastch = ""
+
+        # Set p and q to slice indices of last interesting stmt.
+        str, stmts = self.str, self.stmts
+        i = len(stmts) - 1
+        p = len(str)    # index of newest line
+        found = 0
+        while i:
+            assert p
+            # p is the index of the stmt at line number stmts[i].
+            # Move p back to the stmt at line number stmts[i-1].
+            q = p
+            for nothing in range(stmts[i-1], stmts[i]):
+                # tricky: sets p to 0 if no preceding newline
+                p = _rfind(str, '\n', 0, p-1) + 1
+            # The stmt str[p:q] isn't a continuation, but may be blank
+            # or a non-indenting comment line.
+            if  _junkre(str, p):
+                i = i-1
+            else:
+                found = 1
+                break
+        self.stmt_start, self.stmt_end = p, q
+
+        # Analyze this stmt, to find the last open bracket (if any)
+        # and last interesting character (if any).
+        stack = []  # stack of open bracket indices
+        push_stack = stack.append
+        while p < q:
+            ch = str[p]
+            if ch == '"' or ch == "'":
+                # consume string
+                # Note that study1 did this with a Python loop, but
+                # we use a regexp here; the reason is speed in both
+                # cases; the string may be huge, but study1 pre-squashed
+                # strings to a couple of characters per line.  study1
+                # also needed to keep track of newlines, and we don't
+                # have to.
+                self.lastch = ch
+                p = _match_stringre(str, p, q).end()
+                continue
+
+            if ch == '#':
+                # consume comment and trailing newline
+                p = _find(str, '\n', p, q) + 1
+                assert p > 0
+                continue
+
+            if ch == '\\':
+                assert p+1 < q
+                if str[p+1] != '\n':
+                    # the program is invalid, but can't complain
+                    self.lastch = str[p:p+2]
+                p = p+2
+                continue
+
+            if ch not in _ws:
+                self.lastch = ch
+                if ch in "([{":
+                    push_stack(p)
+                elif ch in ")]}" and stack:
+                    del stack[-1]
+            p = p+1
+
+        # end while p < q:
+
+        if stack:
+            self.lastopenbracketpos = stack[-1]
+
+    # Assuming continuation is C_BRACKET, return the number
+    # of spaces the next line should be indented.
+
+    def compute_bracket_indent(self, _find=string.find):
+        self._study2()
+        assert self.continuation == C_BRACKET
+        j = self.lastopenbracketpos
+        str = self.str
+        n = len(str)
+        origi = i = string.rfind(str, '\n', 0, j) + 1
+        j = j+1
+        # find first list item
+        while _not_itemre(str, j):
+            # this line is junk; advance to the next line
+            i = _find(str, '\n', j)
+            if i < 0:
+                break
+            j = i = i+1
+        if i < 0 or j >= n:
+            # nothing interesting follows the bracket;
+            # reproduce the bracket line's indentation + a level
+            j = i = origi
+            extra = self.indentwidth
+        else:
+            # the first list item begins on this line; line up with
+            # the first interesting character
+            extra = 0
+        while str[j] in " \t":
+            j = j+1
+        return len(string.expandtabs(str[i:j],
+                                     self.tabwidth)) + extra
+
+    # Return number of physical lines in last stmt (whether or not
+    # it's an interesting stmt!  this is intended to be called when
+    # continuation is C_BACKSLASH).
+
+    def get_num_lines_in_stmt(self):
+        self._study1()
+        stmts = self.stmts
+        return stmts[-1] - stmts[-2]
+
+    # Assuming continuation is C_BACKSLASH, return the number of spaces
+    # the next line should be indented.  Also assuming the new line is
+    # the first one following the initial line of the stmt.
+
+    def compute_backslash_indent(self):
+        self._study2()
+        assert self.continuation == C_BACKSLASH
+        str = self.str
+        i = self.stmt_start
+        while str[i] in " \t":
+            i = i+1
+        startpos = i
+        endpos = string.find(str, '\n', startpos) + 1
+        found = level = 0
+        while i < endpos:
+            ch = str[i]
+            if ch in "([{":
+                level = level + 1
+                i = i+1
+            elif ch in ")]}":
+                if level:
+                    level = level - 1
+                i = i+1
+            elif ch == '"' or ch == "'":
+                i = _match_stringre(str, i, endpos).end()
+            elif ch == '#':
+                break
+            elif level == 0 and ch == '=' and \
+                 (i == 0 or str[i-1] not in "=<>!") and \
+                 str[i+1] != '=':
+                found = 1
+                break
+            else:
+                i = i+1
+
+        if found:
+            # found a legit =, but it may be the last interesting
+            # thing on the line
+            i = i+1     # move beyond the =
+            found = re.match(r"\s*\\", str[i:endpos]) is None
+
+        if not found:
+            # oh well ... settle for moving beyond the first chunk
+            # of non-whitespace chars
+            i = startpos
+            while str[i] not in " \t\n":
+                i = i+1
+
+        return len(string.expandtabs(str[self.stmt_start :
+                                         i],
+                                     self.tabwidth)) + 1
+
+    # Return the leading whitespace on the initial line of the last
+    # interesting stmt.
+
+    def get_base_indent_string(self):
+        self._study2()
+        i, n = self.stmt_start, self.stmt_end
+        assert i is not None
+        j = i
+        str = self.str
+        while j < n and str[j] in " \t":
+            j = j + 1
+        return str[i:j]
+
+    # Did the last interesting stmt open a block?
+
+    def is_block_opener(self):
+        self._study2()
+        return self.lastch == ':'
+
+    # Did the last interesting stmt close a block?
+
+    def is_block_closer(self):
+        self._study2()
+        return _closere(self.str, self.stmt_start) is not None