New offerings by Tim Peters; he writes:

IDLE is now the first Python editor in the Universe not confused by my
doctest.py <wink>.

As threatened, this defines IDLE's is_char_in_string function as a
method of EditorWindow.  You just need to define one similarly in
whatever it is you pass as editwin to AutoIndent; looking at the
EditorWindow.py part of the patch should make this clear.

3 files changed, 66 insertions, 26 deletions

diff --git a/Tools/idle/AutoIndent.py b/Tools/idle/AutoIndent.py
index cf32135..9c088fa 100644
--- a/Tools/idle/AutoIndent.py
+++ b/Tools/idle/AutoIndent.py
@@ -104,20 +104,15 @@ class AutoIndent:
     tabwidth = TK_TABWIDTH_DEFAULT
 
     # If context_use_ps1 is true, parsing searches back for a ps1 line;
-    # else searches back for closest preceding def or class.
+    # else searches for a popular (if, def, ...) Python stmt.
     context_use_ps1 = 0
 
-    # When searching backwards for the closest preceding def or class,
+    # When searching backwards for a reliable place to begin parsing,
     # first start num_context_lines[0] lines back, then
     # num_context_lines[1] lines back if that didn't work, and so on.
     # The last value should be huge (larger than the # of lines in a
     # conceivable file).
     # Making the initial values larger slows things down more often.
-    # OTOH, if you happen to find a line that looks like a def or class
-    # in a multiline string, the parsing is utterly hosed.  Can't think
-    # of a way to stop that without always reparsing from the start
-    # of the file.  doctest.py is a killer example of this (IDLE is
-    # useless for editing that!).
     num_context_lines = 50, 500, 5000000
 
     def __init__(self, editwin):
@@ -260,14 +255,19 @@ class AutoIndent:
                 text.delete("insert")
             # start new line
             text.insert("insert", '\n')
+
             # adjust indentation for continuations and block open/close
+            # first need to find the last stmt
             lno = index2line(text.index('insert'))
             y = PyParse.Parser(self.indentwidth, self.tabwidth)
             for context in self.num_context_lines:
                 startat = max(lno - context, 1)
-                rawtext = text.get(`startat` + ".0", "insert")
+                startatindex = `startat` + ".0"
+                rawtext = text.get(startatindex, "insert")
                 y.set_str(rawtext)
-                bod = y.find_last_def_or_class(self.context_use_ps1)
+                bod = y.find_good_parse_start(
+                          self.context_use_ps1,
+                          self._build_char_in_string_func(startatindex))
                 if bod is not None or startat == 1:
                     break
             y.set_lo(bod or 0)
@@ -313,6 +313,16 @@ class AutoIndent:
 
     auto_indent = newline_and_indent_event
 
+    # Our editwin provides a is_char_in_string function that works with
+    # a Tk text index, but PyParse only knows about offsets into a string.
+    # This builds a function for PyParse that accepts an offset.
+
+    def _build_char_in_string_func(self, startindex):
+        def inner(offset, _startindex=startindex,
+                  _icis=self.editwin.is_char_in_string):
+            return _icis(_startindex + "+%dc" % offset)
+        return inner
+
     def indent_region_event(self, event):
         head, tail, chars, lines = self.get_region()
         for pos in range(len(lines)):
diff --git a/Tools/idle/EditorWindow.py b/Tools/idle/EditorWindow.py
index 8bb8ad3..18bedc2 100644
--- a/Tools/idle/EditorWindow.py
+++ b/Tools/idle/EditorWindow.py
@@ -579,6 +579,25 @@ class EditorWindow:
             self.vars[name] = var = vartype(self.text)
         return var
 
+    # Tk implementations of "virtual text methods" -- each platform
+    # reusing IDLE's support code needs to define these for its GUI's
+    # flavor of widget.
+
+    # Is character at text_index in a Python string?  Return 0 for
+    # "guaranteed no", true for anything else.  This info is expensive to
+    # compute ab initio, but is probably already known by the platform's
+    # colorizer.
+
+    def is_char_in_string(self, text_index):
+        if self.color:
+            # return true iff colorizer hasn't (re)gotten this far yet, or
+            # the character is tagged as being in a string
+            return self.text.tag_prevrange("TODO", text_index) or \
+                   "STRING" in self.text.tag_names(text_index)
+        else:
+            # the colorizer is missing: assume the worst
+            return 1
+
 def prepstr(s):
     # Helper to extract the underscore from a string,
     # e.g. prepstr("Co_py") returns (2, "Copy").
diff --git a/Tools/idle/PyParse.py b/Tools/idle/PyParse.py
index ddafe39..43e10b3 100644
--- a/Tools/idle/PyParse.py
+++ b/Tools/idle/PyParse.py
@@ -9,17 +9,13 @@ if 0:   # for throwaway debugging output
     def dump(*stuff):
         sys.__stdout__.write(string.join(map(str, stuff), " ") + "\n")
 
-# Find a def or class stmt.
+# Find what looks like the start of a popular stmt.
 
-_defclassre = re.compile(r"""
+_synchre = re.compile(r"""
     ^
     [ \t]*
-    (?:
-        def   [ \t]+ [a-zA-Z_]\w* [ \t]* \(
-    |   class [ \t]+ [a-zA-Z_]\w* [ \t]*
-        (?: \( .* \) )?
-        [ \t]* :
-    )
+    (?: if | else | elif | while | def | class )
+    \b
 """, re.VERBOSE | re.MULTILINE).search
 
 # Match blank line or non-indenting comment line.
@@ -107,10 +103,13 @@ class Parser:
         self.str = str
         self.study_level = 0
 
-    # Return index of start of last (probable!) def or class stmt, or
-    # None if none found.  It's only probable because we can't know
-    # whether we're in a string without reparsing from the start of
-    # the file -- and that's too slow in large files for routine use.
+    # Return index of a good place to begin parsing, as close to the
+    # end of the string as possible.  This will be the start of some
+    # popular stmt like "if" or "def".  Return None if none found.
+    #
+    # This will be reliable iff given a reliable is_char_in_string
+    # function, meaning that when it says "no", it's absolutely guaranteed
+    # that the char is not in a string.
     #
     # Ack, hack: in the shell window this kills us, because there's
     # no way to tell the differences between output, >>> etc and
@@ -118,7 +117,9 @@ class Parser:
     # look like it's in an unclosed paren!:
     # Python 1.5.2 (#0, Apr 13 1999, ...
 
-    def find_last_def_or_class(self, use_ps1, _defclassre=_defclassre):
+    def find_good_parse_start(self, use_ps1,
+                              is_char_in_string=None,
+                              _synchre=_synchre):
         str, pos = self.str, None
         if use_ps1:
             # hack for shell window
@@ -127,18 +128,21 @@ class Parser:
             if i >= 0:
                 pos = i + len(ps1)
                 self.str = str[:pos-1] + '\n' + str[pos:]
-        else:
+        elif is_char_in_string:
+            # otherwise we can't be sure, so leave pos at None
             i = 0
             while 1:
-                m = _defclassre(str, i)
+                m = _synchre(str, i)
                 if m:
-                    pos, i = m.span()
+                    s, i = m.span()
+                    if not is_char_in_string(s):
+                        pos = s
                 else:
                     break
         return pos
 
     # Throw away the start of the string.  Intended to be called with
-    # find_last_def_or_class's result.
+    # find_good_parse_start's result.
 
     def set_lo(self, lo):
         assert lo == 0 or self.str[lo-1] == '\n'
@@ -498,3 +502,10 @@ class Parser:
     def is_block_closer(self):
         self._study2()
         return _closere(self.str, self.stmt_start) is not None
+
+    # index of last open bracket ({[, or None if none
+    lastopenbracketpos = None
+
+    def get_last_open_bracket_pos(self):
+        self._study2()
+        return self.lastopenbracketpos