Much improved autoindent and handling of tabs,

by Tim Peters.

3 files changed, 242 insertions, 65 deletions

diff --git a/Tools/idle/AutoIndent.py b/Tools/idle/AutoIndent.py
index 386490d..fa72eb0 100644
--- a/Tools/idle/AutoIndent.py
+++ b/Tools/idle/AutoIndent.py
@@ -1,5 +1,10 @@
 import string
 from Tkinter import TclError
+import tkMessageBox
+import tkSimpleDialog
+
+# The default tab setting for a Text widget, in average-width characters.
+TK_TABWIDTH_DEFAULT = 8
 
 ###$ event <<newline-and-indent>>
 ###$ win <Key-Return>
@@ -58,6 +63,9 @@ class AutoIndent:
             ('U_ncomment region', '<<uncomment-region>>'),
             ('Tabify region', '<<tabify-region>>'),
             ('Untabify region', '<<untabify-region>>'),
+            ('Toggle tabs', '<<toggle-tabs>>'),
+            ('New tab width', '<<change-tabwidth>>'),
+            ('New indent width', '<<change-indentwidth>>'),
         ]),
     ]
 
@@ -74,6 +82,9 @@ class AutoIndent:
         '<<uncomment-region>>': ['<Alt-Key-4>'],
         '<<tabify-region>>': ['<Alt-Key-5>'],
         '<<untabify-region>>': ['<Alt-Key-6>'],
+        '<<toggle-tabs>>': ['<Alt-Key-t>'],
+        '<<change-tabwidth>>': ['<Alt-Key-u>'],
+        '<<change-indentwidth>>': ['<Alt-Key-v>'],
     }
 
     unix_keydefs = {
@@ -89,21 +100,62 @@ class AutoIndent:
         '<<untabify-region>>': ['<Alt-Key-6>', '<Meta-Key-6>'],
     }
 
-    prefertabs = 0
-    spaceindent = 4*" "
+    # usetabs true  -> literal tab characters are used by indent and
+    #                  dedent cmds, possibly mixed with spaces if
+    #                  indentwidth is not a multiple of tabwidth
+    #         false -> tab characters are converted to spaces by indent
+    #                  and dedent cmds, and ditto TAB keystrokes
+    # indentwidth is the number of characters per logical indent level
+    # tabwidth is the display width of a literal tab character
+    usetabs = 0
+    indentwidth = 4
+    tabwidth = 8
 
     def __init__(self, editwin):
         self.text = editwin.text
 
     def config(self, **options):
         for key, value in options.items():
-            if key == 'prefertabs':
-                self.prefertabs = value
-            elif key == 'spaceindent':
-                self.spaceindent = value
+            if key == 'usetabs':
+                self.usetabs = value
+            elif key == 'indentwidth':
+                self.indentwidth = value
+            elif key == 'tabwidth':
+                self.tabwidth = value
             else:
                 raise KeyError, "bad option name: %s" % `key`
 
+    # If ispythonsource and guess are true, guess a good value for
+    # indentwidth based on file content (if possible), and if
+    # indentwidth != tabwidth set usetabs false.
+    # In any case, adjust the Text widget's view of what a tab
+    # character means.
+
+    def set_indentation_params(self, ispythonsource, guess=1):
+        text = self.text
+
+        if guess and ispythonsource:
+            i = self.guess_indent()
+            import sys
+            ##sys.__stdout__.write("indent %d\n" % i)
+            if 2 <= i <= 8:
+                self.indentwidth = i
+            if self.indentwidth != self.tabwidth:
+                self.usetabs = 0
+
+        current_tabs = text['tabs']
+        if current_tabs == "" and self.tabwidth == TK_TABWIDTH_DEFAULT:
+            pass
+        else:
+            # Reconfigure the Text widget by measuring the width
+            # of a tabwidth-length string in pixels, forcing the
+            # widget's tab stops to that.
+            need_tabs = text.tk.call("font", "measure", text['font'],
+                                     "-displayof", text.master,
+                                     "n" * self.tabwidth)
+            if current_tabs != need_tabs:
+                text.configure(tabs=need_tabs)
+
     def smart_backspace_event(self, event):
         text = self.text
         try:
@@ -115,16 +167,15 @@ class AutoIndent:
             text.delete(first, last)
             text.mark_set("insert", first)
             return "break"
-        # After Tim Peters
-        ndelete = 1
+        # If we're at the end of leading whitespace, nuke one indent
+        # level, else one character.
         chars = text.get("insert linestart", "insert")
-        i = 0
-        n = len(chars)
-        while i < n and chars[i] in " \t":
-            i = i+1
-        if i == n and chars[-4:] == "    ":
-            ndelete = 4
-        text.delete("insert - %d chars" % ndelete, "insert")
+        raw, effective = classifyws(chars, self.tabwidth)
+        if 0 < raw == len(chars):
+            if effective >= self.indentwidth:
+                self.reindent_to(effective - self.indentwidth)
+                return "break"
+        text.delete("insert-1c")
         return "break"
 
     def smart_indent_event(self, event):
@@ -132,10 +183,7 @@ class AutoIndent:
         #     delete it
         # elif multiline selection:
         #     do indent-region & return
-        # if tabs preferred:
-        #     insert a tab
-        # else:
-        #     insert spaces up to next higher multiple of indent level
+        # indent one level
         text = self.text
         try:
             first = text.index("sel.first")
@@ -149,13 +197,20 @@ class AutoIndent:
                     return self.indent_region_event(event)
                 text.delete(first, last)
                 text.mark_set("insert", first)
-            if self.prefertabs:
-                pad = '\t'
+            prefix = text.get("insert linestart", "insert")
+            raw, effective = classifyws(prefix, self.tabwidth)
+            if raw == len(prefix):
+                # only whitespace to the left
+                self.reindent_to(effective + self.indentwidth)
             else:
-                n = len(self.spaceindent)
-                prefix = text.get("insert linestart", "insert")
-                pad = ' ' * (n - len(prefix) % n)
-            text.insert("insert", pad)
+                if self.usetabs:
+                    pad = '\t'
+                else:
+                    effective = len(string.expandtabs(prefix,
+                                                      self.tabwidth))
+                    n = self.indentwidth
+                    pad = ' ' * (n - effective % n)
+                text.insert("insert", pad)
             text.see("insert")
             return "break"
         finally:
@@ -185,10 +240,13 @@ class AutoIndent:
                 i = i + 1
             if i:
                 text.delete("insert - %d chars" % i, "insert")
+            # XXX this reproduces the current line's indentation,
+            # without regard for usetabs etc; could instead insert
+            # "\n" + self._make_blanks(classifyws(indent)[1]).
             text.insert("insert", "\n" + indent)
             if _is_block_opener(line):
                 self.smart_indent_event(event)
-            elif indent and _is_block_closer(line) and line[-1:] != "\\":
+            elif indent and _is_block_closer(line) and line[-1] != "\\":
                 self.smart_backspace_event(event)
             text.see("insert")
             return "break"
@@ -202,11 +260,9 @@ class AutoIndent:
         for pos in range(len(lines)):
             line = lines[pos]
             if line:
-                i, n = 0, len(line)
-                while i < n and line[i] in " \t":
-                    i = i+1
-                line = line[:i] + "    " + line[i:]
-                lines[pos] = line
+                raw, effective = classifyws(line, self.tabwidth)
+                effective = effective + self.indentwidth
+                lines[pos] = self._make_blanks(effective) + line[raw:]
         self.set_region(head, tail, chars, lines)
         return "break"
 
@@ -215,20 +271,9 @@ class AutoIndent:
         for pos in range(len(lines)):
             line = lines[pos]
             if line:
-                i, n = 0, len(line)
-                while i < n and line[i] in " \t":
-                    i = i+1
-                indent, line = line[:i], line[i:]
-                if indent:
-                    if indent == "\t" or indent[-2:] == "\t\t":
-                        indent = indent[:-1] + "    "
-                    elif indent[-4:] == "    ":
-                        indent = indent[:-4]
-                    else:
-                        indent = string.expandtabs(indent, 8)
-                        indent = indent[:-4]
-                    line = indent + line
-                lines[pos] = line
+                raw, effective = classifyws(line, self.tabwidth)
+                effective = max(effective - self.indentwidth, 0)
+                lines[pos] = self._make_blanks(effective) + line[raw:]
         self.set_region(head, tail, chars, lines)
         return "break"
 
@@ -236,9 +281,8 @@ class AutoIndent:
         head, tail, chars, lines = self.get_region()
         for pos in range(len(lines)):
             line = lines[pos]
-            if not line:
-                continue
-            lines[pos] = '##' + line
+            if line:
+                lines[pos] = '##' + line
         self.set_region(head, tail, chars, lines)
 
     def uncomment_region_event(self, event):
@@ -256,14 +300,48 @@ class AutoIndent:
 
     def tabify_region_event(self, event):
         head, tail, chars, lines = self.get_region()
-        lines = map(tabify, lines)
+        for pos in range(len(lines)):
+            line = lines[pos]
+            if line:
+                raw, effective = classifyws(line, self.tabwidth)
+                ntabs, nspaces = divmod(effective, self.tabwidth)
+                lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:]
         self.set_region(head, tail, chars, lines)
 
     def untabify_region_event(self, event):
         head, tail, chars, lines = self.get_region()
-        lines = map(string.expandtabs, lines)
+        for pos in range(len(lines)):
+            lines[pos] = string.expandtabs(lines[pos], self.tabwidth)
         self.set_region(head, tail, chars, lines)
 
+    def toggle_tabs_event(self, event):
+        if tkMessageBox.askyesno("Toggle tabs",
+              "Turn tabs " + ("on", "off")[self.usetabs] + "?",
+              parent=self.text):
+            self.usetabs = not self.usetabs
+        return "break"
+
+    def change_tabwidth_event(self, event):
+        new = tkSimpleDialog.askinteger("Tab width",
+            "New tab width (2-16)",
+            parent=self.text,
+            initialvalue=self.tabwidth,
+            minvalue=2, maxvalue=16)
+        if new and new != self.tabwidth:
+            self.tabwidth = new
+            self.set_indentation_params(0, guess=0)
+        return "break"
+
+    def change_indentwidth_event(self, event):
+        new = tkSimpleDialog.askinteger("Indent width",
+            "New indent width (1-16)",
+            parent=self.text,
+            initialvalue=self.indentwidth,
+            minvalue=1, maxvalue=16)
+        if new and new != self.indentwidth:
+            self.indentwidth = new
+        return "break"
+
     def get_region(self):
         text = self.text
         head = text.index("sel.first linestart")
@@ -289,15 +367,110 @@ class AutoIndent:
         text.undo_block_stop()
         text.tag_add("sel", head, "insert")
 
-def tabify(line, tabsize=8):
-    spaces = tabsize * ' '
-    for i in range(0, len(line), tabsize):
-        if line[i:i+tabsize] != spaces:
-            break
-    else:
-        i = len(line)
-    return '\t' * (i/tabsize) + line[i:]
+    # Make string that displays as n leading blanks.
+
+    def _make_blanks(self, n):
+        if self.usetabs:
+            ntabs, nspaces = divmod(n, self.tabwidth)
+            return '\t' * ntabs + ' ' * nspaces
+        else:
+            return ' ' * n
+
+    # Delete from beginning of line to insert point, then reinsert
+    # column logical (meaning use tabs if appropriate) spaces.
+
+    def reindent_to(self, column):
+        text = self.text
+        text.undo_block_start()
+        text.delete("insert linestart", "insert")
+        if column:
+            text.insert("insert", self._make_blanks(column))
+        text.undo_block_stop()
+
+    # Guess indentwidth from text content.
+    # Return guessed indentwidth.  This should not be believed unless
+    # it's in a reasonable range (e.g., it will be 0 if no indented
+    # blocks are found).
+
+    def guess_indent(self):
+        opener, indented = IndentSearcher(self.text, self.tabwidth).run()
+        if opener and indented:
+            raw, indentsmall = classifyws(opener, self.tabwidth)
+            raw, indentlarge = classifyws(indented, self.tabwidth)
+        else:
+            indentsmall = indentlarge = 0
+        return indentlarge - indentsmall
 
 # "line.col" -> line, as an int
 def index2line(index):
     return int(float(index))
+
+# Look at the leading whitespace in s.
+# Return pair (# of leading ws characters,
+#              effective # of leading blanks after expanding
+#              tabs to width tabwidth)
+
+def classifyws(s, tabwidth):
+    raw = effective = 0
+    for ch in s:
+        if ch == ' ':
+            raw = raw + 1
+            effective = effective + 1
+        elif ch == '\t':
+            raw = raw + 1
+            effective = (effective / tabwidth + 1) * tabwidth
+        else:
+            break
+    return raw, effective
+
+import tokenize
+_tokenize = tokenize
+del tokenize
+
+class IndentSearcher:
+
+    # .run() chews over the Text widget, looking for a block opener
+    # and the stmt following it.  Returns a pair,
+    #     (line containing block opener, line containing stmt)
+    # Either or both may be None.
+
+    def __init__(self, text, tabwidth):
+        self.text = text
+        self.tabwidth = tabwidth
+        self.i = self.finished = 0
+        self.blkopenline = self.indentedline = None
+
+    def readline(self):
+        if self.finished:
+            return ""
+        i = self.i = self.i + 1
+        mark = `i` + ".0"
+        if self.text.compare(mark, ">=", "end"):
+            return ""
+        return self.text.get(mark, mark + " lineend+1c")
+
+    def tokeneater(self, type, token, start, end, line,
+                   INDENT=_tokenize.INDENT,
+                   NAME=_tokenize.NAME,
+                   OPENERS=('class', 'def', 'for', 'if', 'try', 'while')):
+        if self.finished:
+            pass
+        elif type == NAME and token in OPENERS:
+            self.blkopenline = line
+        elif type == INDENT and self.blkopenline:
+            self.indentedline = line
+            self.finished = 1
+
+    def run(self):
+        save_tabsize = _tokenize.tabsize
+        _tokenize.tabsize = self.tabwidth
+        try:
+            try:
+                _tokenize.tokenize(self.readline, self.tokeneater)
+            except _tokenize.TokenError:
+                # since we cut off the tokenizer early, we can trigger
+                # spurious errors
+                pass
+        finally:
+            _tokenize.tabsize = save_tabsize
+        return self.blkopenline, self.indentedline
diff --git a/Tools/idle/EditorWindow.py b/Tools/idle/EditorWindow.py
index 8b6a0b4..13cfc22 100644
--- a/Tools/idle/EditorWindow.py
+++ b/Tools/idle/EditorWindow.py
@@ -100,7 +100,7 @@ class EditorWindow:
         self.vbar = vbar = Scrollbar(top, name='vbar')
         self.text = text = Text(top, name='text', padx=5,
                                 foreground=cprefs.CNormal[0],
-                                background=cprefs.CNormal[1], 
+                                background=cprefs.CNormal[1],
                                 highlightcolor=cprefs.CHilite[0],
                                 highlightbackground=cprefs.CHilite[1],
                                 insertbackground=cprefs.CCursor[1],
@@ -134,6 +134,7 @@ class EditorWindow:
         text['yscrollcommand'] = vbar.set
         if sys.platform[:3] == 'win':
             text['font'] = ("lucida console", 8)
+#            text['font'] = ("courier new", 10)
         text.pack(side=LEFT, fill=BOTH, expand=1)
         text.focus_set()
 
@@ -173,6 +174,10 @@ class EditorWindow:
             self.wmenu_end = end
             WindowList.register_callback(self.postwindowsmenu)
 
+        if self.extensions.has_key('AutoIndent'):
+            self.extensions['AutoIndent'].set_indentation_params(
+                self.ispythonsource(filename))
+
     def wakeup(self):
         if self.top.wm_state() == "iconic":
             self.top.wm_deiconify()
@@ -323,7 +328,7 @@ class EditorWindow:
         import ClassBrowser
         ClassBrowser.ClassBrowser(self.flist, base, [head])
         self.text["cursor"] = save_cursor
-    
+
     def open_path_browser(self, event=None):
         import PathBrowser
         PathBrowser.PathBrowser(self.flist)
@@ -558,24 +563,23 @@ class EditorWindow:
                     else:
                         menu.add_command(label=label, underline=underline,
                             command=command, accelerator=accelerator)
-    
+
     def getvar(self, name):
         var = self.getrawvar(name)
         if var:
             return var.get()
-    
+
     def setvar(self, name, value, vartype=None):
         var = self.getrawvar(name, vartype)
         if var:
             var.set(value)
-    
+
     def getrawvar(self, name, vartype=None):
         var = self.vars.get(name)
         if not var and vartype:
             self.vars[name] = var = vartype(self.text)
         return var
 
-
 def prepstr(s):
     # Helper to extract the underscore from a string,
     # e.g. prepstr("Co_py") returns (2, "Copy").
diff --git a/Tools/idle/PyShell.py b/Tools/idle/PyShell.py
index 64ef2d1..e01cad8 100644
--- a/Tools/idle/PyShell.py
+++ b/Tools/idle/PyShell.py
@@ -291,7 +291,7 @@ class PyShell(OutputWindow):
         __builtin__.quit = __builtin__.exit = "To exit, type Ctrl-D."
 
         self.auto = self.extensions["AutoIndent"] # Required extension
-        self.auto.config(prefertabs=1)
+        self.auto.config(usetabs=1, indentwidth=8)
 
         text = self.text
         text.configure(wrap="char")