From 2bd8b22b6de76851a7e36dc4a92d20930335ff57 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 13 Feb 2015 12:02:05 +0200 Subject: Issue #21840: Fixed expanding unicode variables of form $var in posixpath.expandvars(). Fixed all os.path implementations on unicode-disabled builds. --- Lib/genericpath.py | 8 ++++++++ Lib/macpath.py | 3 ++- Lib/ntpath.py | 9 +++++---- Lib/os2emxpath.py | 3 ++- Lib/posixpath.py | 21 +++++++-------------- Lib/test/test_genericpath.py | 2 ++ Lib/test/test_macpath.py | 1 + Lib/test/test_ntpath.py | 5 +++-- Lib/test/test_posixpath.py | 19 ++++++++++++++++++- Misc/NEWS | 4 ++++ 10 files changed, 52 insertions(+), 23 deletions(-) diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 7ddb94c..2648e54 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -10,6 +10,14 @@ __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', 'getsize', 'isdir', 'isfile'] +try: + _unicode = unicode +except NameError: + # If Python is built without Unicode support, the unicode type + # will not exist. Fake one. + class _unicode(object): + pass + # Does a path exist? # This is false for dangling symbolic links on systems that support them. def exists(path): diff --git a/Lib/macpath.py b/Lib/macpath.py index c31bdaa..9ebd83c 100644 --- a/Lib/macpath.py +++ b/Lib/macpath.py @@ -5,6 +5,7 @@ import warnings from stat import * import genericpath from genericpath import * +from genericpath import _unicode __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -186,7 +187,7 @@ def walk(top, func, arg): def abspath(path): """Return an absolute path.""" if not isabs(path): - if isinstance(path, unicode): + if isinstance(path, _unicode): cwd = os.getcwdu() else: cwd = os.getcwd() diff --git a/Lib/ntpath.py b/Lib/ntpath.py index fcaf21b..11e4470 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -12,6 +12,7 @@ import genericpath import warnings from genericpath import * +from genericpath import _unicode __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -331,7 +332,7 @@ def expandvars(path): return path import string varchars = string.ascii_letters + string.digits + '_-' - if isinstance(path, unicode): + if isinstance(path, _unicode): encoding = sys.getfilesystemencoding() def getenv(var): return os.environ[var.encode(encoding)].decode(encoding) @@ -414,7 +415,7 @@ def expandvars(path): def normpath(path): """Normalize path, eliminating double slashes, etc.""" # Preserve unicode (if path is unicode) - backslash, dot = (u'\\', u'.') if isinstance(path, unicode) else ('\\', '.') + backslash, dot = (u'\\', u'.') if isinstance(path, _unicode) else ('\\', '.') if path.startswith(('\\\\.\\', '\\\\?\\')): # in the case of paths with these prefixes: # \\.\ -> device names @@ -471,7 +472,7 @@ except ImportError: # not running on Windows - mock up something sensible def abspath(path): """Return the absolute version of a path.""" if not isabs(path): - if isinstance(path, unicode): + if isinstance(path, _unicode): cwd = os.getcwdu() else: cwd = os.getcwd() @@ -487,7 +488,7 @@ else: # use native Windows method on Windows path = _getfullpathname(path) except WindowsError: pass # Bad path - return unchanged. - elif isinstance(path, unicode): + elif isinstance(path, _unicode): path = os.getcwdu() else: path = os.getcwd() diff --git a/Lib/os2emxpath.py b/Lib/os2emxpath.py index 1bed51d..0b32d63 100644 --- a/Lib/os2emxpath.py +++ b/Lib/os2emxpath.py @@ -8,6 +8,7 @@ module as os.path. import os import stat from genericpath import * +from genericpath import _unicode from ntpath import (expanduser, expandvars, isabs, islink, splitdrive, splitext, split, walk) @@ -146,7 +147,7 @@ def normpath(path): def abspath(path): """Return the absolute version of a path""" if not isabs(path): - if isinstance(path, unicode): + if isinstance(path, _unicode): cwd = os.getcwdu() else: cwd = os.getcwd() diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 0378004..6578481 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -16,14 +16,7 @@ import stat import genericpath import warnings from genericpath import * - -try: - _unicode = unicode -except NameError: - # If Python is built without Unicode support, the unicode type - # will not exist. Fake one. - class _unicode(object): - pass +from genericpath import _unicode __all__ = ["normcase","isabs","join","splitdrive","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", @@ -294,16 +287,16 @@ def expandvars(path): if '$' not in path: return path if isinstance(path, _unicode): - if not _varprog: + if not _uvarprog: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})') - varprog = _varprog + _uvarprog = re.compile(ur'\$(\w+|\{[^}]*\})', re.UNICODE) + varprog = _uvarprog encoding = sys.getfilesystemencoding() else: - if not _uvarprog: + if not _varprog: import re - _uvarprog = re.compile(_unicode(r'\$(\w+|\{[^}]*\})'), re.UNICODE) - varprog = _uvarprog + _varprog = re.compile(r'\$(\w+|\{[^}]*\})') + varprog = _varprog encoding = None i = 0 while True: diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 94380b1..741b755 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -243,11 +243,13 @@ class CommonTest(GenericTest): def test_realpath(self): self.assertIn("foo", self.pathmodule.realpath("foo")) + @test_support.requires_unicode def test_normpath_issue5827(self): # Make sure normpath preserves unicode for path in (u'', u'.', u'/', u'\\', u'///foo/.//bar//'): self.assertIsInstance(self.pathmodule.normpath(path), unicode) + @test_support.requires_unicode def test_abspath_issue3426(self): # Check that abspath returns unicode when the arg is unicode # with both ASCII and non-ASCII cwds. diff --git a/Lib/test/test_macpath.py b/Lib/test/test_macpath.py index 96ad61f..be936de 100644 --- a/Lib/test/test_macpath.py +++ b/Lib/test/test_macpath.py @@ -59,6 +59,7 @@ class MacPathTestCase(unittest.TestCase): self.assertEqual(splitext(""), ('', '')) self.assertEqual(splitext("foo.bar.ext"), ('foo.bar', '.ext')) + @test_support.requires_unicode def test_normpath(self): # Issue 5827: Make sure normpath preserves unicode for path in (u'', u'.', u'/', u'\\', u':', u'///foo/.//bar//'): diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 0fbe6a1..55da7e1 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -69,8 +69,9 @@ class TestNtpath(unittest.TestCase): ('', '\\\\conky\\\\mountpoint\\foo\\bar')) tester('ntpath.splitunc("//conky//mountpoint/foo/bar")', ('', '//conky//mountpoint/foo/bar')) - self.assertEqual(ntpath.splitunc(u'//conky/MOUNTPO\u0130NT/foo/bar'), - (u'//conky/MOUNTPO\u0130NT', u'/foo/bar')) + if test_support.have_unicode: + self.assertEqual(ntpath.splitunc(u'//conky/MOUNTPO%cNT/foo/bar' % 0x0130), + (u'//conky/MOUNTPO%cNT' % 0x0130, u'/foo/bar')) def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index f74dc14..295bf49 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,7 +1,9 @@ import unittest from test import test_support, test_genericpath -import posixpath, os +import posixpath +import os +import sys from posixpath import realpath, abspath, dirname, basename # An absolute path to a temporary filename for testing. We can't rely on TESTFN @@ -409,6 +411,21 @@ class PosixPathTest(unittest.TestCase): finally: os.getcwd = real_getcwd + @test_support.requires_unicode + def test_expandvars_nonascii_word(self): + encoding = sys.getfilesystemencoding() + # Non-ASCII word characters + letters = test_support.u(r'\xe6\u0130\u0141\u03c6\u041a\u05d0\u062a\u0e01') + uwnonascii = letters.encode(encoding, 'ignore').decode(encoding)[:3] + swnonascii = uwnonascii.encode(encoding) + if not swnonascii: + self.skip('Needs non-ASCII word characters') + with test_support.EnvironmentVarGuard() as env: + env.clear() + env[swnonascii] = 'baz' + swnonascii + self.assertEqual(posixpath.expandvars(u'$%s bar' % uwnonascii), + u'baz%s bar' % uwnonascii) + class PosixCommonTest(test_genericpath.CommonTest): pathmodule = posixpath diff --git a/Misc/NEWS b/Misc/NEWS index 4680a6f..60a44ee 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -18,6 +18,10 @@ Core and Builtins Library ------- +- Issue #21840: Fixed expanding unicode variables of form $var in + posixpath.expandvars(). Fixed all os.path implementations on + unicode-disabled builds. + - Issue #23363: Fix possible overflow in itertools.permutations. - Issue #23364: Fix possible overflow in itertools.product. -- cgit v0.12