Issue #21840: Fixed expanding unicode variables of form $var in

posixpath.expandvars().  Fixed all os.path implementations on
unicode-disabled builds.

10 files changed, 52 insertions, 23 deletions

diff --git a/Lib/genericpath.py b/Lib/genericpath.py
index 7ddb94c..2648e54 100644
--- a/Lib/genericpath.py
+++ b/Lib/genericpath.py
@@ -10,6 +10,14 @@ __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime',
            'getsize', 'isdir', 'isfile']
 
 
+try:
+    _unicode = unicode
+except NameError:
+    # If Python is built without Unicode support, the unicode type
+    # will not exist. Fake one.
+    class _unicode(object):
+        pass
+
 # Does a path exist?
 # This is false for dangling symbolic links on systems that support them.
 def exists(path):
diff --git a/Lib/macpath.py b/Lib/macpath.py
index c31bdaa..9ebd83c 100644
--- a/Lib/macpath.py
+++ b/Lib/macpath.py
@@ -5,6 +5,7 @@ import warnings
 from stat import *
 import genericpath
 from genericpath import *
+from genericpath import _unicode
 
 __all__ = ["normcase","isabs","join","splitdrive","split","splitext",
            "basename","dirname","commonprefix","getsize","getmtime",
@@ -186,7 +187,7 @@ def walk(top, func, arg):
 def abspath(path):
     """Return an absolute path."""
     if not isabs(path):
-        if isinstance(path, unicode):
+        if isinstance(path, _unicode):
             cwd = os.getcwdu()
         else:
             cwd = os.getcwd()
diff --git a/Lib/ntpath.py b/Lib/ntpath.py
index fcaf21b..11e4470 100644
--- a/Lib/ntpath.py
+++ b/Lib/ntpath.py
@@ -12,6 +12,7 @@ import genericpath
 import warnings
 
 from genericpath import *
+from genericpath import _unicode
 
 __all__ = ["normcase","isabs","join","splitdrive","split","splitext",
            "basename","dirname","commonprefix","getsize","getmtime",
@@ -331,7 +332,7 @@ def expandvars(path):
         return path
     import string
     varchars = string.ascii_letters + string.digits + '_-'
-    if isinstance(path, unicode):
+    if isinstance(path, _unicode):
         encoding = sys.getfilesystemencoding()
         def getenv(var):
             return os.environ[var.encode(encoding)].decode(encoding)
@@ -414,7 +415,7 @@ def expandvars(path):
 def normpath(path):
     """Normalize path, eliminating double slashes, etc."""
     # Preserve unicode (if path is unicode)
-    backslash, dot = (u'\\', u'.') if isinstance(path, unicode) else ('\\', '.')
+    backslash, dot = (u'\\', u'.') if isinstance(path, _unicode) else ('\\', '.')
     if path.startswith(('\\\\.\\', '\\\\?\\')):
         # in the case of paths with these prefixes:
         # \\.\ -> device names
@@ -471,7 +472,7 @@ except ImportError: # not running on Windows - mock up something sensible
     def abspath(path):
         """Return the absolute version of a path."""
         if not isabs(path):
-            if isinstance(path, unicode):
+            if isinstance(path, _unicode):
                 cwd = os.getcwdu()
             else:
                 cwd = os.getcwd()
@@ -487,7 +488,7 @@ else:  # use native Windows method on Windows
                 path = _getfullpathname(path)
             except WindowsError:
                 pass # Bad path - return unchanged.
-        elif isinstance(path, unicode):
+        elif isinstance(path, _unicode):
             path = os.getcwdu()
         else:
             path = os.getcwd()
diff --git a/Lib/os2emxpath.py b/Lib/os2emxpath.py
index 1bed51d..0b32d63 100644
--- a/Lib/os2emxpath.py
+++ b/Lib/os2emxpath.py
@@ -8,6 +8,7 @@ module as os.path.
 import os
 import stat
 from genericpath import *
+from genericpath import _unicode
 from ntpath import (expanduser, expandvars, isabs, islink, splitdrive,
                     splitext, split, walk)
 
@@ -146,7 +147,7 @@ def normpath(path):
 def abspath(path):
     """Return the absolute version of a path"""
     if not isabs(path):
-        if isinstance(path, unicode):
+        if isinstance(path, _unicode):
             cwd = os.getcwdu()
         else:
             cwd = os.getcwd()
diff --git a/Lib/posixpath.py b/Lib/posixpath.py
index 0378004..6578481 100644
--- a/Lib/posixpath.py
+++ b/Lib/posixpath.py
@@ -16,14 +16,7 @@ import stat
 import genericpath
 import warnings
 from genericpath import *
-
-try:
-    _unicode = unicode
-except NameError:
-    # If Python is built without Unicode support, the unicode type
-    # will not exist. Fake one.
-    class _unicode(object):
-        pass
+from genericpath import _unicode
 
 __all__ = ["normcase","isabs","join","splitdrive","split","splitext",
            "basename","dirname","commonprefix","getsize","getmtime",
@@ -294,16 +287,16 @@ def expandvars(path):
     if '$' not in path:
         return path
     if isinstance(path, _unicode):
-        if not _varprog:
+        if not _uvarprog:
             import re
-            _varprog = re.compile(r'\$(\w+|\{[^}]*\})')
-        varprog = _varprog
+            _uvarprog = re.compile(ur'\$(\w+|\{[^}]*\})', re.UNICODE)
+        varprog = _uvarprog
         encoding = sys.getfilesystemencoding()
     else:
-        if not _uvarprog:
+        if not _varprog:
             import re
-            _uvarprog = re.compile(_unicode(r'\$(\w+|\{[^}]*\})'), re.UNICODE)
-        varprog = _uvarprog
+            _varprog = re.compile(r'\$(\w+|\{[^}]*\})')
+        varprog = _varprog
         encoding = None
     i = 0
     while True:
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index 94380b1..741b755 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -243,11 +243,13 @@ class CommonTest(GenericTest):
     def test_realpath(self):
         self.assertIn("foo", self.pathmodule.realpath("foo"))
 
+    @test_support.requires_unicode
     def test_normpath_issue5827(self):
         # Make sure normpath preserves unicode
         for path in (u'', u'.', u'/', u'\\', u'///foo/.//bar//'):
             self.assertIsInstance(self.pathmodule.normpath(path), unicode)
 
+    @test_support.requires_unicode
     def test_abspath_issue3426(self):
         # Check that abspath returns unicode when the arg is unicode
         # with both ASCII and non-ASCII cwds.
diff --git a/Lib/test/test_macpath.py b/Lib/test/test_macpath.py
index 96ad61f..be936de 100644
--- a/Lib/test/test_macpath.py
+++ b/Lib/test/test_macpath.py
@@ -59,6 +59,7 @@ class MacPathTestCase(unittest.TestCase):
         self.assertEqual(splitext(""), ('', ''))
         self.assertEqual(splitext("foo.bar.ext"), ('foo.bar', '.ext'))
 
+    @test_support.requires_unicode
     def test_normpath(self):
         # Issue 5827: Make sure normpath preserves unicode
         for path in (u'', u'.', u'/', u'\\', u':', u'///foo/.//bar//'):
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
index 0fbe6a1..55da7e1 100644
--- a/Lib/test/test_ntpath.py
+++ b/Lib/test/test_ntpath.py
@@ -69,8 +69,9 @@ class TestNtpath(unittest.TestCase):
                ('', '\\\\conky\\\\mountpoint\\foo\\bar'))
         tester('ntpath.splitunc("//conky//mountpoint/foo/bar")',
                ('', '//conky//mountpoint/foo/bar'))
-        self.assertEqual(ntpath.splitunc(u'//conky/MOUNTPO\u0130NT/foo/bar'),
-                         (u'//conky/MOUNTPO\u0130NT', u'/foo/bar'))
+        if test_support.have_unicode:
+            self.assertEqual(ntpath.splitunc(u'//conky/MOUNTPO%cNT/foo/bar' % 0x0130),
+                             (u'//conky/MOUNTPO%cNT' % 0x0130, u'/foo/bar'))
 
     def test_split(self):
         tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar'))
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
index f74dc14..295bf49 100644
--- a/Lib/test/test_posixpath.py
+++ b/Lib/test/test_posixpath.py
@@ -1,7 +1,9 @@
 import unittest
 from test import test_support, test_genericpath
 
-import posixpath, os
+import posixpath
+import os
+import sys
 from posixpath import realpath, abspath, dirname, basename
 
 # An absolute path to a temporary filename for testing. We can't rely on TESTFN
@@ -409,6 +411,21 @@ class PosixPathTest(unittest.TestCase):
         finally:
             os.getcwd = real_getcwd
 
+    @test_support.requires_unicode
+    def test_expandvars_nonascii_word(self):
+        encoding = sys.getfilesystemencoding()
+        # Non-ASCII word characters
+        letters = test_support.u(r'\xe6\u0130\u0141\u03c6\u041a\u05d0\u062a\u0e01')
+        uwnonascii = letters.encode(encoding, 'ignore').decode(encoding)[:3]
+        swnonascii = uwnonascii.encode(encoding)
+        if not swnonascii:
+            self.skip('Needs non-ASCII word characters')
+        with test_support.EnvironmentVarGuard() as env:
+            env.clear()
+            env[swnonascii] = 'baz' + swnonascii
+            self.assertEqual(posixpath.expandvars(u'$%s bar' % uwnonascii),
+                             u'baz%s bar' % uwnonascii)
+
 
 class PosixCommonTest(test_genericpath.CommonTest):
     pathmodule = posixpath
diff --git a/Misc/NEWS b/Misc/NEWS
index 4680a6f..60a44ee 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -18,6 +18,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #21840: Fixed expanding unicode variables of form $var in
+  posixpath.expandvars().  Fixed all os.path implementations on
+  unicode-disabled builds.
+
 - Issue #23363: Fix possible overflow in itertools.permutations.
 
 - Issue #23364: Fix possible overflow in itertools.product.