From bbaba8540200dae111961f799f690b2a0e95dfd9 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Tue, 1 Jun 1999 19:55:34 +0000 Subject: Tim Peters again: The new version (attached) is fast enough all the time in every real module I have . You can make it slow by, e.g., creating an open list with 5,000 90-character identifiers (+ trailing comma) each on its own line, then adding an item to the end -- but that still consumes less than a second on my P5-166. Response time in real code appears instantaneous. Fixed some bugs. New feature: when hitting ENTER and the cursor is beyond the line's leading indentation, whitespace is removed on both sides of the cursor; before whitespace was removed only on the left; e.g., assuming the cursor is between the comma and the space: def something(arg1, arg2): ^ cursor to the left of here, and hit ENTER arg2): # new line used to end up here arg2): # but now lines up the way you expect New hack: AutoIndent has grown a context_use_ps1 Boolean config option, defaulting to 0 (false) and set to 1 (only) by PyShell. Reason: handling the fancy stuff requires looking backward for a parsing synch point; ps1 lines are the only sensible thing to look for in a shell window, but are a bad thing to look for in a file window (ps1 lines show up in my module docstrings often). PythonWin's shell should set this true too. Persistent problem: strings containing def/class can still screw things up completely. No improvement. Simplest workaround is on the user's head, and consists of inserting e.g. def _(): pass (or any other def/class) after the end of the multiline string that's screwing them up. This is especially irksome because IDLE's syntax coloring is *not* confused, so when this happens the colors don't match the indentation behavior they see. --- Tools/idle/AutoIndent.py | 23 +++-- Tools/idle/PyParse.py | 228 +++++++++++++++++++++++++++-------------------- Tools/idle/PyShell.py | 2 +- 3 files changed, 146 insertions(+), 107 deletions(-) diff --git a/Tools/idle/AutoIndent.py b/Tools/idle/AutoIndent.py index b1a2167..2913cc1 100644 --- a/Tools/idle/AutoIndent.py +++ b/Tools/idle/AutoIndent.py @@ -101,6 +101,10 @@ class AutoIndent: indentwidth = 4 tabwidth = TK_TABWIDTH_DEFAULT + # If context_use_ps1 is true, parsing searches back for a ps1 line; + # else searches back for closest preceding def or class. + context_use_ps1 = 0 + # When searching backwards for the closest preceding def or class, # first start num_context_lines[0] lines back, then # num_context_lines[1] lines back if that didn't work, and so on. @@ -108,11 +112,10 @@ class AutoIndent: # conceivable file). # Making the initial values larger slows things down more often. # OTOH, if you happen to find a line that looks like a def or class - # in a multiline string, and the start of the string isn't in the - # chunk, the parsing is utterly hosed. Can't think of a way to - # stop that without always reparsing from the start of the file. - # doctest.py is a killer example of this (IDLE is useless for - # editing that!). + # in a multiline string, the parsing is utterly hosed. Can't think + # of a way to stop that without always reparsing from the start + # of the file. doctest.py is a killer example of this (IDLE is + # useless for editing that!). num_context_lines = 50, 500, 5000000 def __init__(self, editwin): @@ -126,6 +129,8 @@ class AutoIndent: self.indentwidth = value elif key == 'tabwidth': self.tabwidth = value + elif key == 'context_use_ps1': + self.context_use_ps1 = value else: raise KeyError, "bad option name: %s" % `key` @@ -240,13 +245,17 @@ class AutoIndent: text.insert("insert linestart", '\n') return "break" indent = line[:i] - # strip trailing whitespace + # strip whitespace before insert point i = 0 while line and line[-1] in " \t": line = line[:-1] i = i+1 if i: text.delete("insert - %d chars" % i, "insert") + # strip whitespace after insert point + while text.get("insert") in " \t": + text.delete("insert") + # start new line text.insert("insert", '\n') # adjust indentation for continuations and block open/close lno = index2line(text.index('insert')) @@ -255,7 +264,7 @@ class AutoIndent: startat = max(lno - context, 1) rawtext = text.get(`startat` + ".0", "insert") y.set_str(rawtext) - bod = y.find_last_def_or_class() + bod = y.find_last_def_or_class(self.context_use_ps1) if bod is not None or startat == 1: break y.set_lo(bod or 0) diff --git a/Tools/idle/PyParse.py b/Tools/idle/PyParse.py index e3b6e1f..ddafe39 100644 --- a/Tools/idle/PyParse.py +++ b/Tools/idle/PyParse.py @@ -7,10 +7,10 @@ C_NONE, C_BACKSLASH, C_STRING, C_BRACKET = range(4) if 0: # for throwaway debugging output def dump(*stuff): - import sys sys.__stdout__.write(string.join(map(str, stuff), " ") + "\n") -# find a def or class stmt +# Find a def or class stmt. + _defclassre = re.compile(r""" ^ [ \t]* @@ -22,15 +22,17 @@ _defclassre = re.compile(r""" ) """, re.VERBOSE | re.MULTILINE).search -# match blank line or non-indenting comment line +# Match blank line or non-indenting comment line. + _junkre = re.compile(r""" [ \t]* - (?: \# [^ \t\n] .* )? + (?: \# \S .* )? \n """, re.VERBOSE).match -# match any flavor of string; the terminating quote is optional -# so that we're robust in the face of incomplete program text +# Match any flavor of string; the terminating quote is optional +# so that we're robust in the face of incomplete program text. + _match_stringre = re.compile(r""" \""" [^"\\]* (?: (?: \\. | "(?!"") ) @@ -49,15 +51,16 @@ _match_stringre = re.compile(r""" | ' [^'\\\n]* (?: \\. [^'\\\n]* )* '? """, re.VERBOSE | re.DOTALL).match -# match a line that doesn't start with something interesting; -# used to skip junk lines when searching for the first element -# of a bracket structure -_not_itemre = re.compile(r""" +# Match a line that starts with something interesting; +# used to find the first item of a bracket structure. + +_itemre = re.compile(r""" [ \t]* - [#\n\\] + [^\s#\\] # if we match, m.end()-1 is the interesting char """, re.VERBOSE).match -# match start of stmts that should be followed by a dedent +# Match start of stmts that should be followed by a dedent. + _closere = re.compile(r""" \s* (?: return @@ -69,6 +72,17 @@ _closere = re.compile(r""" \b """, re.VERBOSE).match +# Chew up non-special chars as quickly as possible, but retaining +# enough info to determine the last non-ws char seen; if match is +# successful, and m.group(1) isn't None, m.end(1) less 1 is the +# index of the last non-ws char matched. + +_chew_ordinaryre = re.compile(r""" + (?: \s+ + | ( [^\s[\](){}#'"\\]+ ) + )+ +""", re.VERBOSE).match + # Build translation table to map uninteresting chars to "x", open # brackets to "(", and close brackets to ")". @@ -96,7 +110,7 @@ class Parser: # Return index of start of last (probable!) def or class stmt, or # None if none found. It's only probable because we can't know # whether we're in a string without reparsing from the start of - # the file -- and that's too slow to bear. + # the file -- and that's too slow in large files for routine use. # # Ack, hack: in the shell window this kills us, because there's # no way to tell the differences between output, >>> etc and @@ -104,22 +118,23 @@ class Parser: # look like it's in an unclosed paren!: # Python 1.5.2 (#0, Apr 13 1999, ... - def find_last_def_or_class(self, _defclassre=_defclassre): + def find_last_def_or_class(self, use_ps1, _defclassre=_defclassre): str, pos = self.str, None - i = 0 - while 1: - m = _defclassre(str, i) - if m: - pos, i = m.span() - else: - break - if pos is None: + if use_ps1: # hack for shell window ps1 = '\n' + sys.ps1 i = string.rfind(str, ps1) if i >= 0: pos = i + len(ps1) self.str = str[:pos-1] + '\n' + str[pos:] + else: + i = 0 + while 1: + m = _defclassre(str, i) + if m: + pos, i = m.span() + else: + break return pos # Throw away the start of the string. Intended to be called with @@ -132,7 +147,7 @@ class Parser: # As quickly as humanly possible , find the line numbers (0- # based) of the non-continuation lines. - # Creates self.{stmts, continuation}. + # Creates self.{goodlines, continuation}. def _study1(self, _replace=string.replace, _find=string.find): if self.study_level >= 1: @@ -158,80 +173,75 @@ class Parser: # whether & why the last stmt is a continuation. continuation = C_NONE level = lno = 0 # level is nesting level; lno is line number - self.stmts = stmts = [0] - push_stmt = stmts.append + self.goodlines = goodlines = [0] + push_good = goodlines.append i, n = 0, len(str) while i < n: ch = str[i] - # cases are checked in decreasing order of frequency + i = i+1 + # cases are checked in decreasing order of frequency if ch == 'x': - i = i+1 continue if ch == '\n': lno = lno + 1 if level == 0: - push_stmt(lno) + push_good(lno) # else we're in an unclosed bracket structure - i = i+1 continue if ch == '(': level = level + 1 - i = i+1 continue if ch == ')': if level: level = level - 1 # else the program is invalid, but we can't complain - i = i+1 continue if ch == '"' or ch == "'": # consume the string quote = ch - if str[i:i+3] == quote * 3: + if str[i-1:i+2] == quote * 3: quote = quote * 3 - w = len(quote) + w = len(quote) - 1 i = i+w while i < n: ch = str[i] + i = i+1 + if ch == 'x': - i = i+1 continue - if str[i:i+w] == quote: + if str[i-1:i+w] == quote: i = i+w break if ch == '\n': lno = lno + 1 - i = i+1 - if w == 1: + if w == 0: # unterminated single-quoted string if level == 0: - push_stmt(lno) + push_good(lno) break continue if ch == '\\': - assert i+1 < n - if str[i+1] == '\n': + assert i < n + if str[i] == '\n': lno = lno + 1 - i = i+2 + i = i+1 continue # else comment char or paren inside string - i = i+1 else: - # didn't break out of the loop, so it's an - # unterminated triple-quoted string - assert w == 3 + # didn't break out of the loop, so we're still + # inside a string continuation = C_STRING - continue + continue # with outer loop if ch == '#': # consume the comment @@ -240,17 +250,12 @@ class Parser: continue assert ch == '\\' - assert i+1 < n - if str[i+1] == '\n': + assert i < n + if str[i] == '\n': lno = lno + 1 - if i+2 == n: + if i+1 == n: continuation = C_BACKSLASH - i = i+2 - - # Push the final line number as a sentinel value, regardless of - # whether it's continued. - if stmts[-1] != lno: - push_stmt(lno) + i = i+1 # The last stmt may be continued for all 3 reasons. # String continuation takes precedence over bracket @@ -259,6 +264,12 @@ class Parser: continuation = C_BRACKET self.continuation = continuation + # Push the final line number as a sentinel value, regardless of + # whether it's continued. + assert (continuation == C_NONE) == (goodlines[-1] == lno) + if goodlines[-1] != lno: + push_good(lno) + def get_continuation_type(self): self._study1() return self.continuation @@ -282,19 +293,16 @@ class Parser: self._study1() self.study_level = 2 - self.lastch = "" - # Set p and q to slice indices of last interesting stmt. - str, stmts = self.str, self.stmts - i = len(stmts) - 1 + str, goodlines = self.str, self.goodlines + i = len(goodlines) - 1 p = len(str) # index of newest line - found = 0 while i: assert p - # p is the index of the stmt at line number stmts[i]. - # Move p back to the stmt at line number stmts[i-1]. + # p is the index of the stmt at line number goodlines[i]. + # Move p back to the stmt at line number goodlines[i-1]. q = p - for nothing in range(stmts[i-1], stmts[i]): + for nothing in range(goodlines[i-1], goodlines[i]): # tricky: sets p to 0 if no preceding newline p = _rfind(str, '\n', 0, p-1) + 1 # The stmt str[p:q] isn't a continuation, but may be blank @@ -302,16 +310,44 @@ class Parser: if _junkre(str, p): i = i-1 else: - found = 1 break + if i == 0: + # nothing but junk! + assert p == 0 + q = p self.stmt_start, self.stmt_end = p, q # Analyze this stmt, to find the last open bracket (if any) # and last interesting character (if any). + lastch = "" stack = [] # stack of open bracket indices push_stack = stack.append while p < q: + # suck up all except ()[]{}'"#\\ + m = _chew_ordinaryre(str, p, q) + if m: + i = m.end(1) - 1 # last non-ws (if any) + if i >= 0: + lastch = str[i] + p = m.end() + if p >= q: + break + ch = str[p] + + if ch in "([{": + push_stack(p) + lastch = ch + p = p+1 + continue + + if ch in ")]}": + if stack: + del stack[-1] + lastch = ch + p = p+1 + continue + if ch == '"' or ch == "'": # consume string # Note that study1 did this with a Python loop, but @@ -320,7 +356,7 @@ class Parser: # strings to a couple of characters per line. study1 # also needed to keep track of newlines, and we don't # have to. - self.lastch = ch + lastch = ch p = _match_stringre(str, p, q).end() continue @@ -330,24 +366,17 @@ class Parser: assert p > 0 continue - if ch == '\\': - assert p+1 < q - if str[p+1] != '\n': - # the program is invalid, but can't complain - self.lastch = str[p:p+2] - p = p+2 - continue - - if ch not in _ws: - self.lastch = ch - if ch in "([{": - push_stack(p) - elif ch in ")]}" and stack: - del stack[-1] - p = p+1 + assert ch == '\\' + p = p+1 # beyond backslash + assert p < q + if str[p] != '\n': + # the program is invalid, but can't complain + lastch = ch + str[p] + p = p+1 # beyond escaped char # end while p < q: + self.lastch = lastch if stack: self.lastopenbracketpos = stack[-1] @@ -361,25 +390,24 @@ class Parser: str = self.str n = len(str) origi = i = string.rfind(str, '\n', 0, j) + 1 - j = j+1 - # find first list item - while _not_itemre(str, j): - # this line is junk; advance to the next line - i = _find(str, '\n', j) - if i < 0: + j = j+1 # one beyond open bracket + # find first list item; set i to start of its line + while j < n: + m = _itemre(str, j) + if m: + j = m.end() - 1 # index of first interesting char + extra = 0 break - j = i = i+1 - if i < 0 or j >= n: + else: + # this line is junk; advance to next line + i = j = _find(str, '\n', j) + 1 + else: # nothing interesting follows the bracket; # reproduce the bracket line's indentation + a level j = i = origi + while str[j] in " \t": + j = j+1 extra = self.indentwidth - else: - # the first list item begins on this line; line up with - # the first interesting character - extra = 0 - while str[j] in " \t": - j = j+1 return len(string.expandtabs(str[i:j], self.tabwidth)) + extra @@ -389,8 +417,8 @@ class Parser: def get_num_lines_in_stmt(self): self._study1() - stmts = self.stmts - return stmts[-1] - stmts[-2] + goodlines = self.goodlines + return goodlines[-1] - goodlines[-2] # Assuming continuation is C_BACKSLASH, return the number of spaces # the next line should be indented. Also assuming the new line is @@ -404,6 +432,9 @@ class Parser: while str[i] in " \t": i = i+1 startpos = i + + # See whether the initial line starts an assignment stmt; i.e., + # look for an = operator endpos = string.find(str, '\n', startpos) + 1 found = level = 0 while i < endpos: @@ -420,8 +451,8 @@ class Parser: elif ch == '#': break elif level == 0 and ch == '=' and \ - (i == 0 or str[i-1] not in "=<>!") and \ - str[i+1] != '=': + (i == 0 or str[i-1] not in "=<>!") and \ + str[i+1] != '=': found = 1 break else: @@ -450,7 +481,6 @@ class Parser: def get_base_indent_string(self): self._study2() i, n = self.stmt_start, self.stmt_end - assert i is not None j = i str = self.str while j < n and str[j] in " \t": diff --git a/Tools/idle/PyShell.py b/Tools/idle/PyShell.py index bd095cf..a618ad1 100644 --- a/Tools/idle/PyShell.py +++ b/Tools/idle/PyShell.py @@ -291,7 +291,7 @@ class PyShell(OutputWindow): __builtin__.quit = __builtin__.exit = "To exit, type Ctrl-D." self.auto = self.extensions["AutoIndent"] # Required extension - self.auto.config(usetabs=1, indentwidth=8) + self.auto.config(usetabs=1, indentwidth=8, context_use_ps1=1) text = self.text text.configure(wrap="char") -- cgit v0.12