py-cvs-rel2_1 (Rev 1.6 - 1.8) merge

Fix autoindent bug and deflect Unicode from text.get()

1 files changed, 23 insertions, 4 deletions

diff --git a/Lib/idlelib/PyParse.py b/Lib/idlelib/PyParse.py
index 23b995c..a512356 100644
--- a/Lib/idlelib/PyParse.py
+++ b/Lib/idlelib/PyParse.py
@@ -105,6 +105,11 @@ for ch in "\"'\\\n#":
 _tran = string.join(_tran, '')
 del ch
 
+try:
+    UnicodeType = type(unicode(""))
+except NameError:
+    UnicodeType = None
+
 class Parser:
 
     def __init__(self, indentwidth, tabwidth):
@@ -113,6 +118,19 @@ class Parser:
 
     def set_str(self, str):
         assert len(str) == 0 or str[-1] == '\n'
+        if type(str) is UnicodeType:
+            # The parse functions have no idea what to do with Unicode, so
+            # replace all Unicode characters with "x".  This is "safe"
+            # so long as the only characters germane to parsing the structure
+            # of Python are 7-bit ASCII.  It's *necessary* because Unicode
+            # strings don't have a .translate() method that supports
+            # deletechars.
+            uniphooey = str
+            str = []
+            push = str.append
+            for raw in map(ord, uniphooey):
+                push(raw < 127 and chr(raw) or "x")
+            str = "".join(str)
         self.str = str
         self.study_level = 0
 
@@ -385,13 +403,14 @@ class Parser:
             m = _chew_ordinaryre(str, p, q)
             if m:
                 # we skipped at least one boring char
-                p = m.end()
+                newp = m.end()
                 # back up over totally boring whitespace
-                i = p-1    # index of last boring char
-                while i >= 0 and str[i] in " \t\n":
+                i = newp - 1    # index of last boring char
+                while i >= p and str[i] in " \t\n":
                     i = i-1
-                if i >= 0:
+                if i >= p:
                     lastch = str[i]
+                p = newp
                 if p >= q:
                     break