diff options
Diffstat (limited to 'Lib')
28 files changed, 761 insertions, 251 deletions
diff --git a/Lib/compileall.py b/Lib/compileall.py index ae86292..d9d7816 100644 --- a/Lib/compileall.py +++ b/Lib/compileall.py @@ -12,6 +12,7 @@ See module py_compile for details of the actual byte-compilation. """ import os +import errno import sys import py_compile import struct @@ -20,7 +21,7 @@ import imp __all__ = ["compile_dir","compile_file","compile_path"] def compile_dir(dir, maxlevels=10, ddir=None, - force=0, rx=None, quiet=0): + force=False, rx=None, quiet=False, legacy=False): """Byte-compile all modules in the given directory tree. Arguments (only dir is required): @@ -29,8 +30,9 @@ def compile_dir(dir, maxlevels=10, ddir=None, maxlevels: maximum recursion level (default 10) ddir: if given, purported directory name (this is the directory name that will show up in error messages) - force: if 1, force compilation, even if timestamps are up-to-date - quiet: if 1, be quiet during compilation + force: if True, force compilation, even if timestamps are up-to-date + quiet: if True, be quiet during compilation + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths """ if not quiet: @@ -49,24 +51,26 @@ def compile_dir(dir, maxlevels=10, ddir=None, else: dfile = None if not os.path.isdir(fullname): - if not compile_file(fullname, ddir, force, rx, quiet): + if not compile_file(fullname, ddir, force, rx, quiet, legacy): success = 0 elif maxlevels > 0 and \ name != os.curdir and name != os.pardir and \ os.path.isdir(fullname) and \ not os.path.islink(fullname): if not compile_dir(fullname, maxlevels - 1, dfile, force, rx, - quiet): + quiet, legacy): success = 0 return success -def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): +def compile_file(fullname, ddir=None, force=0, rx=None, quiet=False, + legacy=False): """Byte-compile file. - file: the file to byte-compile + fullname: the file to byte-compile ddir: if given, purported directory name (this is the directory name that will show up in error messages) - force: if 1, force compilation, even if timestamps are up-to-date - quiet: if 1, be quiet during compilation + force: if True, force compilation, even if timestamps are up-to-date + quiet: if True, be quiet during compilation + legacy: if True, produce legacy pyc paths instead of PEP 3147 paths """ success = 1 @@ -80,13 +84,22 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): if mo: return success if os.path.isfile(fullname): + if legacy: + cfile = fullname + ('c' if __debug__ else 'o') + else: + cfile = imp.cache_from_source(fullname) + cache_dir = os.path.dirname(cfile) + try: + os.mkdir(cache_dir) + except OSError as error: + if error.errno != errno.EEXIST: + raise head, tail = name[:-3], name[-3:] if tail == '.py': if not force: try: mtime = int(os.stat(fullname).st_mtime) expect = struct.pack('<4sl', imp.get_magic(), mtime) - cfile = fullname + (__debug__ and 'c' or 'o') with open(cfile, 'rb') as chandle: actual = chandle.read(8) if expect == actual: @@ -96,14 +109,15 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): if not quiet: print('Compiling', fullname, '...') try: - ok = py_compile.compile(fullname, None, dfile, True) + ok = py_compile.compile(fullname, cfile, dfile, True) except py_compile.PyCompileError as err: if quiet: print('*** Error compiling', fullname, '...') else: print('*** ', end='') # escape non-printable characters in msg - msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace') + msg = err.msg.encode(sys.stdout.encoding, + errors='backslashreplace') msg = msg.decode(sys.stdout.encoding) print(msg) success = 0 @@ -119,15 +133,17 @@ def compile_file(fullname, ddir=None, force=0, rx=None, quiet=0): success = 0 return success -def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0): +def compile_path(skip_curdir=1, maxlevels=0, force=False, quiet=False, + legacy=False): """Byte-compile all module on sys.path. Arguments (all optional): skip_curdir: if true, skip current directory (default true) maxlevels: max recursion level (default 0) - force: as for compile_dir() (default 0) - quiet: as for compile_dir() (default 0) + force: as for compile_dir() (default False) + quiet: as for compile_dir() (default False) + legacy: as for compile_dir() (default False) """ success = 1 @@ -136,7 +152,8 @@ def compile_path(skip_curdir=1, maxlevels=0, force=0, quiet=0): print('Skipping current directory') else: success = success and compile_dir(dir, maxlevels, None, - force, quiet=quiet) + force, quiet=quiet, + legacy=legacy) return success def expand_args(args, flist): @@ -162,10 +179,10 @@ def main(): """Script main program.""" import getopt try: - opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:') + opts, args = getopt.getopt(sys.argv[1:], 'lfqd:x:i:b') except getopt.error as msg: print(msg) - print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] " \ + print("usage: python compileall.py [-l] [-f] [-q] [-d destdir] " "[-x regexp] [-i list] [directory|file ...]") print("-l: don't recurse down") print("-f: force rebuild even if timestamps are up-to-date") @@ -174,23 +191,27 @@ def main(): print(" if no directory arguments, -l sys.path is assumed") print("-x regexp: skip files matching the regular expression regexp") print(" the regexp is searched for in the full path of the file") - print("-i list: expand list with its content (file and directory names)") + print("-i list: expand list with its content " + "(file and directory names)") + print("-b: Produce legacy byte-compile file paths") sys.exit(2) maxlevels = 10 ddir = None - force = 0 - quiet = 0 + force = False + quiet = False rx = None flist = None + legacy = False for o, a in opts: if o == '-l': maxlevels = 0 if o == '-d': ddir = a - if o == '-f': force = 1 - if o == '-q': quiet = 1 + if o == '-f': force = True + if o == '-q': quiet = True if o == '-x': import re rx = re.compile(a) if o == '-i': flist = a + if o == '-b': legacy = True if ddir: if len(args) != 1 and not os.path.isdir(args[0]): print("-d destdir require exactly one directory argument") @@ -207,13 +228,14 @@ def main(): for arg in args: if os.path.isdir(arg): if not compile_dir(arg, maxlevels, ddir, - force, rx, quiet): + force, rx, quiet, legacy): success = 0 else: - if not compile_file(arg, ddir, force, rx, quiet): + if not compile_file(arg, ddir, force, rx, + quiet, legacy): success = 0 else: - success = compile_path() + success = compile_path(legacy=legacy) except KeyboardInterrupt: print("\n[interrupt]") success = 0 diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 330eb63..30d5251 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -488,6 +488,16 @@ class _PyPycFileLoader(PyPycLoader, _PyFileLoader): """Load a module from a source or bytecode file.""" + def _find_path(self, ext_type): + """Return PEP 3147 path if ext_type is PY_COMPILED, otherwise + super()._find_path() is called.""" + if ext_type == imp.PY_COMPILED: + # We don't really care what the extension on self._base_path is, + # as long as it has exactly one dot. + bytecode_path = imp.cache_from_source(self._base_path + '.py') + return (bytecode_path if _path_exists(bytecode_path) else None) + return super()._find_path(ext_type) + @_check_name def source_mtime(self, name): """Return the modification time of the source for the specified @@ -515,7 +525,16 @@ class _PyPycFileLoader(PyPycLoader, _PyFileLoader): """ bytecode_path = self.bytecode_path(name) if not bytecode_path: - bytecode_path = self._base_path + _suffix_list(imp.PY_COMPILED)[0] + source_path = self.source_path(name) + bytecode_path = imp.cache_from_source(source_path) + # Ensure that the __pycache__ directory exists. We can't use + # os.path.dirname() here. + dirname, sep, basename = bytecode_path.rpartition(path_sep) + try: + _os.mkdir(dirname) + except OSError as error: + if error.errno != errno.EEXIST: + raise try: # Assuming bytes. with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file: diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py index b97e382..8329264 100644 --- a/Lib/importlib/test/__main__.py +++ b/Lib/importlib/test/__main__.py @@ -13,7 +13,12 @@ import unittest def test_main(): - start_dir = os.path.dirname(__file__) + if '__pycache__' in __file__: + parts = __file__.split(os.path.sep) + start_dir = sep.join(parts[:-2]) + else: + start_dir = os.path.dirname(__file__) + # XXX 2010-03-18 barry: Fix __file__ top_dir = os.path.dirname(os.path.dirname(start_dir)) test_loader = unittest.TestLoader() if '--builtin' in sys.argv: diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index ae4b185..9059405 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -127,7 +127,7 @@ class BadBytecodeTest(unittest.TestCase): except KeyError: pass py_compile.compile(mapping[name]) - bytecode_path = source_util.bytecode_path(mapping[name]) + bytecode_path = imp.cache_from_source(mapping[name]) with open(bytecode_path, 'rb') as file: bc = file.read() new_bc = manipulator(bc) @@ -226,7 +226,7 @@ class BadBytecodeTest(unittest.TestCase): zeros = b'\x00\x00\x00\x00' with source_util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(4) bytecode_file.write(zeros) @@ -242,9 +242,10 @@ class BadBytecodeTest(unittest.TestCase): def test_bad_marshal(self): # Bad marshal data should raise a ValueError. with source_util.create_modules('_temp') as mapping: - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) source_mtime = os.path.getmtime(mapping['_temp']) source_timestamp = importlib._w_long(source_mtime) + source_util.ensure_bytecode_path(bytecode_path) with open(bytecode_path, 'wb') as bytecode_file: bytecode_file.write(imp.get_magic()) bytecode_file.write(source_timestamp) @@ -260,7 +261,7 @@ class BadBytecodeTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index 8f15f62..1673669 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,9 @@ from importlib import _bootstrap from .. import abc from . import util as source_util +from test.support import make_legacy_pyc import os +import errno import py_compile import unittest import warnings @@ -52,6 +54,14 @@ class FinderTests(abc.FinderTests): if unlink: for name in unlink: os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise loader = self.import_(mapping['.root'], test) self.assertTrue(hasattr(loader, 'load_module')) return loader @@ -60,7 +70,8 @@ class FinderTests(abc.FinderTests): # [top-level source] self.run_test('top_level') # [top-level bc] - self.run_test('top_level', compile_={'top_level'}, unlink={'top_level'}) + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) # [top-level both] self.run_test('top_level', compile_={'top_level'}) diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/importlib/test/source/test_source_encoding.py index fde355f..04aac24 100644 --- a/Lib/importlib/test/source/test_source_encoding.py +++ b/Lib/importlib/test/source/test_source_encoding.py @@ -33,7 +33,7 @@ class EncodingTest(unittest.TestCase): def run_test(self, source): with source_util.create_modules(self.module_name) as mapping: - with open(mapping[self.module_name], 'wb')as file: + with open(mapping[self.module_name], 'wb') as file: file.write(source) loader = _bootstrap._PyPycFileLoader(self.module_name, mapping[self.module_name], False) diff --git a/Lib/importlib/test/source/util.py b/Lib/importlib/test/source/util.py index 2b945c5..ae65663 100644 --- a/Lib/importlib/test/source/util.py +++ b/Lib/importlib/test/source/util.py @@ -1,5 +1,6 @@ from .. import util import contextlib +import errno import functools import imp import os @@ -26,14 +27,16 @@ def writes_bytecode_files(fxn): return wrapper -def bytecode_path(source_path): - for suffix, _, type_ in imp.get_suffixes(): - if type_ == imp.PY_COMPILED: - bc_suffix = suffix - break - else: - raise ValueError("no bytecode suffix is defined") - return os.path.splitext(source_path)[0] + bc_suffix +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise @contextlib.contextmanager diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3abc6a9..7b44fa1 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,4 +1,5 @@ """Utility code for constructing importers, etc.""" + from ._bootstrap import module_for_loader from ._bootstrap import set_loader from ._bootstrap import set_package diff --git a/Lib/inspect.py b/Lib/inspect.py index b9fcd74..ea30466 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -54,6 +54,7 @@ def ismodule(object): """Return true if the object is a module. Module objects provide these attributes: + __cached__ pathname to byte compiled file __doc__ documentation string __file__ filename (missing for built-in modules)""" return isinstance(object, types.ModuleType) diff --git a/Lib/py_compile.py b/Lib/py_compile.py index 9361875..f257770 100644 --- a/Lib/py_compile.py +++ b/Lib/py_compile.py @@ -4,6 +4,7 @@ This module has intimate knowledge of the format of .pyc files. """ import builtins +import errno import imp import marshal import os @@ -37,16 +38,18 @@ class PyCompileError(Exception): can be accesses as class variable 'file' msg: string message to be written as error message - If no value is given, a default exception message will be given, - consistent with 'standard' py_compile output. - message (or default) can be accesses as class variable 'msg' + If no value is given, a default exception message will be + given, consistent with 'standard' py_compile output. + message (or default) can be accesses as class variable + 'msg' """ def __init__(self, exc_type, exc_value, file, msg=''): exc_type_name = exc_type.__name__ if exc_type is SyntaxError: - tbtext = ''.join(traceback.format_exception_only(exc_type, exc_value)) + tbtext = ''.join(traceback.format_exception_only( + exc_type, exc_value)) errmsg = tbtext.replace('File "<string>"', 'File "%s"' % file) else: errmsg = "Sorry: %s: %s" % (exc_type_name,exc_value) @@ -64,7 +67,7 @@ class PyCompileError(Exception): def wr_long(f, x): """Internal; write a 32-bit int to a file in little-endian order.""" - f.write(bytes([x & 0xff, + f.write(bytes([x & 0xff, (x >> 8) & 0xff, (x >> 16) & 0xff, (x >> 24) & 0xff])) @@ -72,20 +75,18 @@ def wr_long(f, x): def compile(file, cfile=None, dfile=None, doraise=False): """Byte-compile one Python source file to Python bytecode. - Arguments: - - file: source filename - cfile: target filename; defaults to source with 'c' or 'o' appended - ('c' normally, 'o' in optimizing mode, giving .pyc or .pyo) - dfile: purported filename; defaults to source (this is the filename - that will show up in error messages) - doraise: flag indicating whether or not an exception should be - raised when a compile error is found. If an exception - occurs and this flag is set to False, a string - indicating the nature of the exception will be printed, - and the function will return to the caller. If an - exception occurs and this flag is set to True, a - PyCompileError exception will be raised. + :param file: The source file name. + :param cfile: The target byte compiled file name. When not given, this + defaults to the PEP 3147 location. + :param dfile: Purported file name, i.e. the file name that shows up in + error messages. Defaults to the source file name. + :param doraise: Flag indicating whether or not an exception should be + raised when a compile error is found. If an exception occurs and this + flag is set to False, a string indicating the nature of the exception + will be printed, and the function will return to the caller. If an + exception occurs and this flag is set to True, a PyCompileError + exception will be raised. + :return: Path to the resulting byte compiled file. Note that it isn't necessary to byte-compile Python modules for execution efficiency -- Python itself byte-compiles a module when @@ -102,7 +103,6 @@ def compile(file, cfile=None, dfile=None, doraise=False): See compileall.py for a script/module that uses this module to byte-compile all installed files (or all files in selected directories). - """ with open(file, "rb") as f: encoding = tokenize.detect_encoding(f.readline)[0] @@ -122,7 +122,12 @@ def compile(file, cfile=None, dfile=None, doraise=False): sys.stderr.write(py_exc.msg + '\n') return if cfile is None: - cfile = file + (__debug__ and 'c' or 'o') + cfile = imp.cache_from_source(file) + try: + os.mkdir(os.path.dirname(cfile)) + except OSError as error: + if error.errno != errno.EEXIST: + raise with open(cfile, 'wb') as fc: fc.write(b'\0\0\0\0') wr_long(fc, timestamp) @@ -130,6 +135,7 @@ def compile(file, cfile=None, dfile=None, doraise=False): fc.flush() fc.seek(0, 0) fc.write(MAGIC) + return cfile def main(args=None): """Compile several source files. diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 6b9dd3d..208740f 100755 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -159,7 +159,8 @@ def visiblename(name, all=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant. _hidden_names = ('__builtins__', '__doc__', '__file__', '__path__', - '__module__', '__name__', '__slots__', '__package__') + '__module__', '__name__', '__slots__', '__package__', + '__cached__') if name in _hidden_names: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 diff --git a/Lib/runpy.py b/Lib/runpy.py index 6e94d6b..f251081 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -67,6 +67,7 @@ def _run_code(code, run_globals, init_globals=None, run_globals.update(init_globals) run_globals.update(__name__ = mod_name, __file__ = mod_fname, + __cached__ = None, __loader__ = mod_loader, __package__ = pkg_name) exec(code, run_globals) @@ -130,6 +131,7 @@ def _run_module_as_main(mod_name, alter_argv=True): At the very least, these variables in __main__ will be overwritten: __name__ __file__ + __cached__ __loader__ __package__ """ diff --git a/Lib/site.py b/Lib/site.py index 55e662c..d99b538 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -74,15 +74,19 @@ def makepath(*paths): return dir, os.path.normcase(dir) -def abs__file__(): - """Set all module' __file__ attribute to an absolute path""" +def abs_paths(): + """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): if hasattr(m, '__loader__'): continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) except AttributeError: - continue + pass + try: + m.__cached__ = os.path.abspath(m.__cached__) + except AttributeError: + pass def removeduppaths(): @@ -518,7 +522,7 @@ def execusercustomize(): def main(): global ENABLE_USER_SITE - abs__file__() + abs_paths() known_paths = removeduppaths() if (os.name == "posix" and sys.path and os.path.basename(sys.path[-1]) == "Modules"): diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py index 144cf66..39874d9 100644 --- a/Lib/test/script_helper.py +++ b/Lib/test/script_helper.py @@ -11,6 +11,9 @@ import contextlib import shutil import zipfile +from imp import source_from_cache +from test.support import make_legacy_pyc + # Executing the interpreter in a subprocess def python_exit_code(*args): cmd_line = [sys.executable, '-E'] @@ -62,20 +65,18 @@ def make_script(script_dir, script_basename, source): script_file.close() return script_name -def compile_script(script_name): - py_compile.compile(script_name, doraise=True) - if __debug__: - compiled_name = script_name + 'c' - else: - compiled_name = script_name + 'o' - return compiled_name - def make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None): zip_filename = zip_basename+os.extsep+'zip' zip_name = os.path.join(zip_dir, zip_filename) zip_file = zipfile.ZipFile(zip_name, 'w') if name_in_zip is None: - name_in_zip = os.path.basename(script_name) + parts = script_name.split(os.sep) + if len(parts) >= 2 and parts[-2] == '__pycache__': + legacy_pyc = make_legacy_pyc(source_from_cache(script_name)) + name_in_zip = os.path.basename(legacy_pyc) + script_name = legacy_pyc + else: + name_in_zip = os.path.basename(script_name) zip_file.write(script_name, name_in_zip) zip_file.close() #if test.test_support.verbose: @@ -98,8 +99,8 @@ def make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, script_name = make_script(zip_dir, script_basename, source) unlink.append(script_name) if compiled: - init_name = compile_script(init_name) - script_name = compile_script(script_name) + init_name = py_compile(init_name, doraise=True) + script_name = py_compile(script_name, doraise=True) unlink.extend((init_name, script_name)) pkg_names = [os.sep.join([pkg_name]*i) for i in range(1, depth+1)] script_name_in_zip = os.path.join(pkg_names[-1], os.path.basename(script_name)) diff --git a/Lib/test/support.py b/Lib/test/support.py index 9f9292d..3c0002b 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -17,22 +17,25 @@ import unittest import importlib import collections import re +import imp import time -__all__ = ["Error", "TestFailed", "ResourceDenied", "import_module", - "verbose", "use_resources", "max_memuse", "record_original_stdout", - "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "find_unused_port", "bind_port", - "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", - "findfile", "sortdict", "check_syntax_error", "open_urlresource", - "check_warnings", "CleanImport", "EnvironmentVarGuard", - "TransientResource", "captured_output", "captured_stdout", - "time_out", "socket_peer_reset", "ioerror_peer_reset", - "run_with_locale", - "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", - "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", - "reap_children", "cpython_only", "check_impl_detail", "get_attribute", - "swap_item", "swap_attr"] +__all__ = [ + "Error", "TestFailed", "ResourceDenied", "import_module", + "verbose", "use_resources", "max_memuse", "record_original_stdout", + "get_original_stdout", "unload", "unlink", "rmtree", "forget", + "is_resource_enabled", "requires", "find_unused_port", "bind_port", + "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", + "findfile", "sortdict", "check_syntax_error", "open_urlresource", + "check_warnings", "CleanImport", "EnvironmentVarGuard", + "TransientResource", "captured_output", "captured_stdout", + "time_out", "socket_peer_reset", "ioerror_peer_reset", + "run_with_locale", 'temp_umask', + "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", + "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", + "reap_children", "cpython_only", "check_impl_detail", "get_attribute", + "swap_item", "swap_attr", + ] class Error(Exception): @@ -177,27 +180,50 @@ def unload(name): def unlink(filename): try: os.unlink(filename) - except OSError: - pass + except OSError as error: + # The filename need not exist. + if error.errno != errno.ENOENT: + raise def rmtree(path): try: shutil.rmtree(path) - except OSError as e: + except OSError as error: # Unix returns ENOENT, Windows returns ESRCH. - if e.errno not in (errno.ENOENT, errno.ESRCH): + if error.errno not in (errno.ENOENT, errno.ESRCH): raise +def make_legacy_pyc(source): + """Move a PEP 3147 pyc/pyo file to its legacy pyc/pyo location. + + The choice of .pyc or .pyo extension is done based on the __debug__ flag + value. + + :param source: The file system path to the source file. The source file + does not need to exist, however the PEP 3147 pyc file must exist. + :return: The file system path to the legacy pyc file. + """ + pyc_file = imp.cache_from_source(source) + up_one = os.path.dirname(os.path.abspath(source)) + legacy_pyc = os.path.join(up_one, source + ('c' if __debug__ else 'o')) + os.rename(pyc_file, legacy_pyc) + return legacy_pyc + def forget(modname): - '''"Forget" a module was ever imported by removing it from sys.modules and - deleting any .pyc and .pyo files.''' + """'Forget' a module was ever imported. + + This removes the module from sys.modules and deletes any PEP 3147 or + legacy .pyc and .pyo files. + """ unload(modname) for dirname in sys.path: - unlink(os.path.join(dirname, modname + '.pyc')) - # Deleting the .pyo file cannot be within the 'try' for the .pyc since - # the chance exists that there is no .pyc (and thus the 'try' statement - # is exited) but there is a .pyo file. - unlink(os.path.join(dirname, modname + '.pyo')) + source = os.path.join(dirname, modname + '.py') + # It doesn't matter if they exist or not, unlink all possible + # combinations of PEP 3147 and legacy pyc and pyo files. + unlink(source + 'c') + unlink(source + 'o') + unlink(imp.cache_from_source(source, debug_override=True)) + unlink(imp.cache_from_source(source, debug_override=False)) def is_resource_enabled(resource): """Test whether a resource is enabled. Known resources are set by @@ -208,7 +234,9 @@ def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available. If the caller's module is __main__ then automatically return True. The - possibility of False being returned occurs when regrtest.py is executing.""" + possibility of False being returned occurs when regrtest.py is + executing. + """ # see if the caller's module is __main__ - if so, treat as if # the resource was set if sys._getframe(1).f_globals.get("__name__") == "__main__": @@ -405,6 +433,16 @@ def temp_cwd(name='tempcwd', quiet=False): rmtree(name) +@contextlib.contextmanager +def temp_umask(umask): + """Context manager that temporarily sets the process umask.""" + oldmask = os.umask(umask) + try: + yield + finally: + os.umask(oldmask) + + def findfile(file, here=__file__, subdir=None): """Try to find a file on sys.path and the working directory. If it is not found the argument passed to the function is returned (this does not diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index f7c27a7..3f4dd6d 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -1,12 +1,14 @@ -# Tests command line execution of scripts +# tests command line execution of scripts import unittest import os import os.path +import py_compile + import test.support -from test.script_helper import (run_python, - temp_dir, make_script, compile_script, - make_pkg, make_zip_script, make_zip_pkg) +from test.script_helper import ( + make_pkg, make_script, make_zip_pkg, make_zip_script, run_python, + temp_dir) verbose = test.support.verbose @@ -28,6 +30,7 @@ assertEqual(result, ['Top level assignment', 'Lower level reference']) # Check population of magic variables assertEqual(__name__, '__main__') print('__file__==%r' % __file__) +assertEqual(__cached__, None) print('__package__==%r' % __package__) # Check the sys module import sys @@ -101,9 +104,10 @@ class CmdLineTest(unittest.TestCase): def test_script_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(compiled_name, compiled_name, compiled_name, None) + self._check_script(compiled_name, compiled_name, + compiled_name, None) def test_directory(self): with temp_dir() as script_dir: @@ -113,9 +117,10 @@ class CmdLineTest(unittest.TestCase): def test_directory_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(script_dir, compiled_name, script_dir, '') + pyc_file = test.support.make_legacy_pyc(script_name) + self._check_script(script_dir, pyc_file, script_dir, '') def test_directory_error(self): with temp_dir() as script_dir: @@ -131,7 +136,7 @@ class CmdLineTest(unittest.TestCase): def test_zipfile_compiled(self): with temp_dir() as script_dir: script_name = _make_test_script(script_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name) self._check_script(zip_name, run_name, zip_name, '') @@ -176,11 +181,12 @@ class CmdLineTest(unittest.TestCase): pkg_dir = os.path.join(script_dir, 'test_pkg') make_pkg(pkg_dir) script_name = _make_test_script(pkg_dir, '__main__') - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) + pyc_file = test.support.make_legacy_pyc(script_name) launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg') - self._check_script(launch_name, compiled_name, - compiled_name, 'test_pkg') + self._check_script(launch_name, pyc_file, + pyc_file, 'test_pkg') def test_package_error(self): with temp_dir() as script_dir: diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index 4b6feba..8b34587 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -5,22 +5,23 @@ import os import py_compile import shutil import struct +import subprocess import tempfile -from test import support import unittest import io +from test import support class CompileallTests(unittest.TestCase): def setUp(self): self.directory = tempfile.mkdtemp() self.source_path = os.path.join(self.directory, '_test.py') - self.bc_path = self.source_path + ('c' if __debug__ else 'o') + self.bc_path = imp.cache_from_source(self.source_path) with open(self.source_path, 'w') as file: file.write('x = 123\n') self.source_path2 = os.path.join(self.directory, '_test2.py') - self.bc_path2 = self.source_path2 + ('c' if __debug__ else 'o') + self.bc_path2 = imp.cache_from_source(self.source_path2) shutil.copyfile(self.source_path, self.source_path2) def tearDown(self): @@ -65,17 +66,19 @@ class CompileallTests(unittest.TestCase): except: pass compileall.compile_file(self.source_path, force=False, quiet=True) - self.assertTrue(os.path.isfile(self.bc_path) \ - and not os.path.isfile(self.bc_path2)) + self.assertTrue(os.path.isfile(self.bc_path) and + not os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) compileall.compile_dir(self.directory, force=False, quiet=True) - self.assertTrue(os.path.isfile(self.bc_path) \ - and os.path.isfile(self.bc_path2)) + self.assertTrue(os.path.isfile(self.bc_path) and + os.path.isfile(self.bc_path2)) os.unlink(self.bc_path) os.unlink(self.bc_path2) + class EncodingTest(unittest.TestCase): - 'Issue 6716: compileall should escape source code when printing errors to stdout.' + """Issue 6716: compileall should escape source code when printing errors + to stdout.""" def setUp(self): self.directory = tempfile.mkdtemp() @@ -95,9 +98,65 @@ class EncodingTest(unittest.TestCase): finally: sys.stdout = orig_stdout +class CommandLineTests(unittest.TestCase): + """Test some aspects of compileall's CLI.""" + + def setUp(self): + self.addCleanup(self._cleanup) + self.directory = tempfile.mkdtemp() + self.pkgdir = os.path.join(self.directory, 'foo') + os.mkdir(self.pkgdir) + # Touch the __init__.py and a package module. + with open(os.path.join(self.pkgdir, '__init__.py'), 'w'): + pass + with open(os.path.join(self.pkgdir, 'bar.py'), 'w'): + pass + sys.path.insert(0, self.directory) + + def _cleanup(self): + support.rmtree(self.directory) + assert sys.path[0] == self.directory, 'Missing path' + del sys.path[0] + + def test_pep3147_paths(self): + # Ensure that the default behavior of compileall's CLI is to create + # PEP 3147 pyc/pyo files. + retcode = subprocess.call( + (sys.executable, '-m', 'compileall', '-q', self.pkgdir)) + self.assertEqual(retcode, 0) + # Verify the __pycache__ directory contents. + cachedir = os.path.join(self.pkgdir, '__pycache__') + self.assertTrue(os.path.exists(cachedir)) + ext = ('pyc' if __debug__ else 'pyo') + expected = sorted(base.format(imp.get_tag(), ext) for base in + ('__init__.{}.{}', 'bar.{}.{}')) + self.assertEqual(sorted(os.listdir(cachedir)), expected) + # Make sure there are no .pyc files in the source directory. + self.assertFalse([pyc_file for pyc_file in os.listdir(self.pkgdir) + if pyc_file.endswith(ext)]) + + def test_legacy_paths(self): + # Ensure that with the proper switch, compileall leaves legacy + # pyc/pyo files, and no __pycache__ directory. + retcode = subprocess.call( + (sys.executable, '-m', 'compileall', '-b', '-q', self.pkgdir)) + self.assertEqual(retcode, 0) + # Verify the __pycache__ directory contents. + cachedir = os.path.join(self.pkgdir, '__pycache__') + self.assertFalse(os.path.exists(cachedir)) + ext = ('pyc' if __debug__ else 'pyo') + expected = [base.format(ext) for base in ('__init__.{}', 'bar.{}')] + expected.extend(['__init__.py', 'bar.py']) + expected.sort() + self.assertEqual(sorted(os.listdir(self.pkgdir)), expected) + + def test_main(): - support.run_unittest(CompileallTests, - EncodingTest) + support.run_unittest( + CommandLineTests, + CompileallTests, + EncodingTest, + ) if __name__ == "__main__": diff --git a/Lib/test/test_frozen.py b/Lib/test/test_frozen.py index 79cc1c3..28186bb 100644 --- a/Lib/test/test_frozen.py +++ b/Lib/test/test_frozen.py @@ -11,7 +11,7 @@ class FrozenTests(unittest.TestCase): except ImportError as x: self.fail("import __hello__ failed:" + str(x)) self.assertEqual(__hello__.initialized, True) - self.assertEqual(len(dir(__hello__)), 6, dir(__hello__)) + self.assertEqual(len(dir(__hello__)), 7, dir(__hello__)) try: import __phello__ @@ -19,9 +19,9 @@ class FrozenTests(unittest.TestCase): self.fail("import __phello__ failed:" + str(x)) self.assertEqual(__phello__.initialized, True) if not "__phello__.spam" in sys.modules: - self.assertEqual(len(dir(__phello__)), 7, dir(__phello__)) - else: self.assertEqual(len(dir(__phello__)), 8, dir(__phello__)) + else: + self.assertEqual(len(dir(__phello__)), 9, dir(__phello__)) self.assertEquals(__phello__.__path__, [__phello__.__name__]) try: @@ -29,8 +29,8 @@ class FrozenTests(unittest.TestCase): except ImportError as x: self.fail("import __phello__.spam failed:" + str(x)) self.assertEqual(__phello__.spam.initialized, True) - self.assertEqual(len(dir(__phello__.spam)), 6) - self.assertEqual(len(dir(__phello__)), 8) + self.assertEqual(len(dir(__phello__.spam)), 7) + self.assertEqual(len(dir(__phello__)), 9) try: import __phello__.foo diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index e995bf0..6412f3f 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,6 +1,7 @@ import imp import os import os.path +import shutil import sys import unittest from test import support @@ -139,7 +140,8 @@ class ImportTests(unittest.TestCase): mod = imp.load_source(temp_mod_name, temp_mod_name + '.py') self.assertEqual(mod.a, 1) - mod = imp.load_compiled(temp_mod_name, temp_mod_name + '.pyc') + mod = imp.load_compiled( + temp_mod_name, imp.cache_from_source(temp_mod_name + '.py')) self.assertEqual(mod.a, 1) if not os.path.exists(test_package_name): @@ -184,11 +186,132 @@ class ReloadTests(unittest.TestCase): imp.reload(marshal) +class PEP3147Tests(unittest.TestCase): + """Tests of PEP 3147.""" + + tag = imp.get_tag() + + def test_cache_from_source(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyc file (i.e. under __pycache__). + self.assertEqual( + imp.cache_from_source('/foo/bar/baz/qux.py', True), + '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)) + + def test_cache_from_source_optimized(self): + # Given the path to a .py file, return the path to its PEP 3147 + # defined .pyo file (i.e. under __pycache__). + self.assertEqual( + imp.cache_from_source('/foo/bar/baz/qux.py', False), + '/foo/bar/baz/__pycache__/qux.{}.pyo'.format(self.tag)) + + def test_cache_from_source_cwd(self): + self.assertEqual(imp.cache_from_source('foo.py', True), + os.sep.join(('__pycache__', + 'foo.{}.pyc'.format(self.tag)))) + + def test_cache_from_source_override(self): + # When debug_override is not None, it can be any true-ish or false-ish + # value. + self.assertEqual( + imp.cache_from_source('/foo/bar/baz.py', []), + '/foo/bar/__pycache__/baz.{}.pyo'.format(self.tag)) + self.assertEqual( + imp.cache_from_source('/foo/bar/baz.py', [17]), + '/foo/bar/__pycache__/baz.{}.pyc'.format(self.tag)) + # However if the bool-ishness can't be determined, the exception + # propagates. + class Bearish: + def __bool__(self): raise RuntimeError + self.assertRaises( + RuntimeError, + imp.cache_from_source, '/foo/bar/baz.py', Bearish()) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_altsep_cache_from_source(self): + # Windows path and PEP 3147. + self.assertEqual( + imp.cache_from_source('\\foo\\bar\\baz\\qux.py', True), + '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where altsep is right of sep. + self.assertEqual( + imp.cache_from_source('\\foo\\bar/baz\\qux.py', True), + '\\foo\\bar/baz\\__pycache__\\qux.{}.pyc'.format(self.tag)) + + @unittest.skipIf(os.altsep is None, + 'test meaningful only where os.altsep is defined') + def test_sep_altsep_and_sep_cache_from_source(self): + # Windows path and PEP 3147 where sep is right of altsep. + self.assertEqual( + imp.cache_from_source('\\foo\\bar\\baz/qux.py', True), + '\\foo\\bar\\baz/__pycache__/qux.{}.pyc'.format(self.tag)) + + def test_source_from_cache(self): + # Given the path to a PEP 3147 defined .pyc file, return the path to + # its source. This tests the good path. + self.assertEqual(imp.source_from_cache( + '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)), + '/foo/bar/baz/qux.py') + + def test_source_from_cache_bad_path(self): + # When the path to a pyc file is not in PEP 3147 format, a ValueError + # is raised. + self.assertRaises( + ValueError, imp.source_from_cache, '/foo/bar/bazqux.pyc') + + def test_source_from_cache_no_slash(self): + # No slashes at all in path -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, 'foo.cpython-32.pyc') + + def test_source_from_cache_too_few_dots(self): + # Too few dots in final path component -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, '__pycache__/foo.pyc') + + def test_source_from_cache_too_many_dots(self): + # Too many dots in final path component -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, + '__pycache__/foo.cpython-32.foo.pyc') + + def test_source_from_cache_no__pycache__(self): + # Another problem with the path -> ValueError + self.assertRaises( + ValueError, imp.source_from_cache, + '/foo/bar/foo.cpython-32.foo.pyc') + + def test_package___file__(self): + # Test that a package's __file__ points to the right source directory. + os.mkdir('pep3147') + sys.path.insert(0, os.curdir) + def cleanup(): + if sys.path[0] == os.curdir: + del sys.path[0] + shutil.rmtree('pep3147') + self.addCleanup(cleanup) + # Touch the __init__.py file. + with open('pep3147/__init__.py', 'w'): + pass + m = __import__('pep3147') + # Ensure we load the pyc file. + support.forget('pep3147') + m = __import__('pep3147') + self.assertEqual(m.__file__, + os.sep.join(('.', 'pep3147', '__init__.py'))) + + def test_main(): tests = [ ImportTests, + PEP3147Tests, ReloadTests, - ] + ] try: import _thread except ImportError: diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index 9b34467..0a21e18 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -1,4 +1,5 @@ import builtins +import errno import imp import marshal import os @@ -8,8 +9,11 @@ import shutil import stat import sys import unittest -from test.support import (unlink, TESTFN, unload, run_unittest, is_jython, - check_warnings, EnvironmentVarGuard, swap_attr, swap_item) + +from test.support import ( + EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, + make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, + unlink, unload) def remove_files(name): @@ -19,12 +23,18 @@ def remove_files(name): name + ".pyw", name + "$py.class"): unlink(f) + try: + shutil.rmtree('__pycache__') + except OSError as error: + if error.errno != errno.ENOENT: + raise class ImportTests(unittest.TestCase): def tearDown(self): unload(TESTFN) + setUp = tearDown def test_case_sensitivity(self): @@ -53,8 +63,8 @@ class ImportTests(unittest.TestCase): pyc = TESTFN + ".pyc" with open(source, "w") as f: - print("# This tests Python's ability to import a", ext, "file.", - file=f) + print("# This tests Python's ability to import a", + ext, "file.", file=f) a = random.randrange(1000) b = random.randrange(1000) print("a =", a, file=f) @@ -73,10 +83,10 @@ class ImportTests(unittest.TestCase): self.assertEqual(mod.b, b, "module loaded (%s) but contents invalid" % mod) finally: + forget(TESTFN) unlink(source) unlink(pyc) unlink(pyo) - unload(TESTFN) sys.path.insert(0, os.curdir) try: @@ -87,32 +97,31 @@ class ImportTests(unittest.TestCase): finally: del sys.path[0] - @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") + @unittest.skipUnless(os.name == 'posix', + "test meaningful only on posix systems") def test_execute_bit_not_copied(self): # Issue 6070: under posix .pyc files got their execute bit set if # the .py file had the execute bit set, but they aren't executable. - oldmask = os.umask(0o022) - sys.path.insert(0, os.curdir) - try: - fname = TESTFN + os.extsep + "py" - f = open(fname, 'w').close() - os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) - __import__(TESTFN) - fn = fname + 'c' - if not os.path.exists(fn): - fn = fname + 'o' + with temp_umask(0o022): + sys.path.insert(0, os.curdir) + try: + fname = TESTFN + os.extsep + "py" + f = open(fname, 'w').close() + os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | + stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) + __import__(TESTFN) + fn = imp.cache_from_source(fname) if not os.path.exists(fn): self.fail("__import__ did not result in creation of " "either a .pyc or .pyo file") - s = os.stat(fn) - self.assertEqual(stat.S_IMODE(s.st_mode), - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) - finally: - os.umask(oldmask) - remove_files(TESTFN) - unload(TESTFN) - del sys.path[0] + s = os.stat(fn) + self.assertEqual( + stat.S_IMODE(s.st_mode), + stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + finally: + del sys.path[0] + remove_files(TESTFN) + unload(TESTFN) def test_imp_module(self): # Verify that the imp module can correctly load and find .py files @@ -144,10 +153,12 @@ class ImportTests(unittest.TestCase): f.write('"",\n') f.write(']') - # Compile & remove .py file, we only need .pyc (or .pyo). + # Compile & remove .py file, we only need .pyc (or .pyo), but that + # must be relocated to the PEP 3147 bytecode-only location. with open(filename, 'r') as f: py_compile.compile(filename) unlink(filename) + make_legacy_pyc(filename) # Need to be able to load from current dir. sys.path.append('') @@ -247,8 +258,9 @@ class ImportTests(unittest.TestCase): self.assertTrue(mod.__file__.endswith('.py')) os.remove(source) del sys.modules[TESTFN] + make_legacy_pyc(source) mod = __import__(TESTFN) - ext = mod.__file__[-4:] + base, ext = os.path.splitext(mod.__file__) self.assertIn(ext, ('.pyc', '.pyo')) finally: del sys.path[0] @@ -298,7 +310,7 @@ func_filename = func.__code__.co_filename """ dir_name = os.path.abspath(TESTFN) file_name = os.path.join(dir_name, module_name) + os.extsep + "py" - compiled_name = file_name + ("c" if __debug__ else "o") + compiled_name = imp.cache_from_source(file_name) def setUp(self): self.sys_path = sys.path[:] @@ -346,8 +358,9 @@ func_filename = func.__code__.co_filename target = "another_module.py" py_compile.compile(self.file_name, dfile=target) os.remove(self.file_name) + pyc_file = make_legacy_pyc(self.file_name) mod = self.import_module() - self.assertEqual(mod.module_filename, self.compiled_name) + self.assertEqual(mod.module_filename, pyc_file) self.assertEqual(mod.code_filename, target) self.assertEqual(mod.func_filename, target) @@ -476,10 +489,143 @@ class OverridingImportBuiltinTests(unittest.TestCase): self.assertEqual(foo(), os) +class PycacheTests(unittest.TestCase): + # Test the various PEP 3147 related behaviors. + + tag = imp.get_tag() + + def _clean(self): + forget(TESTFN) + rmtree('__pycache__') + unlink(self.source) + + def setUp(self): + self.source = TESTFN + '.py' + self._clean() + with open(self.source, 'w') as fp: + print('# This is a test file written by test_import.py', file=fp) + sys.path.insert(0, os.curdir) + + def tearDown(self): + assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' + del sys.path[0] + self._clean() + + def test_import_pyc_path(self): + self.assertFalse(os.path.exists('__pycache__')) + __import__(TESTFN) + self.assertTrue(os.path.exists('__pycache__')) + self.assertTrue(os.path.exists(os.path.join( + '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag)))) + + @unittest.skipUnless(os.name == 'posix', + "test meaningful only on posix systems") + def test_unwritable_directory(self): + # When the umask causes the new __pycache__ directory to be + # unwritable, the import still succeeds but no .pyc file is written. + with temp_umask(0o222): + __import__(TESTFN) + self.assertTrue(os.path.exists('__pycache__')) + self.assertFalse(os.path.exists(os.path.join( + '__pycache__', '{}.{}.pyc'.format(TESTFN, self.tag)))) + + def test_missing_source(self): + # With PEP 3147 cache layout, removing the source but leaving the pyc + # file does not satisfy the import. + __import__(TESTFN) + pyc_file = imp.cache_from_source(self.source) + self.assertTrue(os.path.exists(pyc_file)) + os.remove(self.source) + forget(TESTFN) + self.assertRaises(ImportError, __import__, TESTFN) + + def test_missing_source_legacy(self): + # Like test_missing_source() except that for backward compatibility, + # when the pyc file lives where the py file would have been (and named + # without the tag), it is importable. The __file__ of the imported + # module is the pyc location. + __import__(TESTFN) + # pyc_file gets removed in _clean() via tearDown(). + pyc_file = make_legacy_pyc(self.source) + os.remove(self.source) + unload(TESTFN) + m = __import__(TESTFN) + self.assertEqual(m.__file__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + + def test___cached__(self): + # Modules now also have an __cached__ that points to the pyc file. + m = __import__(TESTFN) + pyc_file = imp.cache_from_source(TESTFN + '.py') + self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file)) + + def test___cached___legacy_pyc(self): + # Like test___cached__() except that for backward compatibility, + # when the pyc file lives where the py file would have been (and named + # without the tag), it is importable. The __cached__ of the imported + # module is the pyc location. + __import__(TESTFN) + # pyc_file gets removed in _clean() via tearDown(). + pyc_file = make_legacy_pyc(self.source) + os.remove(self.source) + unload(TESTFN) + m = __import__(TESTFN) + self.assertEqual(m.__cached__, + os.path.join(os.curdir, os.path.relpath(pyc_file))) + + def test_package___cached__(self): + # Like test___cached__ but for packages. + def cleanup(): + shutil.rmtree('pep3147') + os.mkdir('pep3147') + self.addCleanup(cleanup) + # Touch the __init__.py + with open(os.path.join('pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('pep3147', 'foo.py'), 'w'): + pass + unload('pep3147.foo') + unload('pep3147') + m = __import__('pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('pep3147', '__init__.py')) + self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) + foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + self.assertEqual(sys.modules['pep3147.foo'].__cached__, + os.path.join(os.curdir, foo_pyc)) + + def test_package___cached___from_pyc(self): + # Like test___cached__ but ensuring __cached__ when imported from a + # PEP 3147 pyc file. + def cleanup(): + shutil.rmtree('pep3147') + os.mkdir('pep3147') + self.addCleanup(cleanup) + unload('pep3147.foo') + unload('pep3147') + # Touch the __init__.py + with open(os.path.join('pep3147', '__init__.py'), 'w'): + pass + with open(os.path.join('pep3147', 'foo.py'), 'w'): + pass + m = __import__('pep3147.foo') + unload('pep3147.foo') + unload('pep3147') + m = __import__('pep3147.foo') + init_pyc = imp.cache_from_source( + os.path.join('pep3147', '__init__.py')) + self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc)) + foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py')) + self.assertEqual(sys.modules['pep3147.foo'].__cached__, + os.path.join(os.curdir, foo_pyc)) + + def test_main(verbose=None): - run_unittest(ImportTests, PycRewritingTests, PathsTests, RelativeImportTests, + run_unittest(ImportTests, PycacheTests, + PycRewritingTests, PathsTests, RelativeImportTests, OverridingImportBuiltinTests) + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. from test.test_import import test_main diff --git a/Lib/test/test_pkg.py b/Lib/test/test_pkg.py index 2c19589..a342f7a 100644 --- a/Lib/test/test_pkg.py +++ b/Lib/test/test_pkg.py @@ -196,14 +196,14 @@ class TestPkg(unittest.TestCase): import t5 self.assertEqual(fixdir(dir(t5)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'foo', 'string', 't5']) self.assertEqual(fixdir(dir(t5.foo)), - ['__doc__', '__file__', '__name__', '__package__', - 'string']) + ['__cached__', '__doc__', '__file__', '__name__', + '__package__', 'string']) self.assertEqual(fixdir(dir(t5.string)), - ['__doc__', '__file__', '__name__','__package__', - 'spam']) + ['__cached__', '__doc__', '__file__', '__name__', + '__package__', 'spam']) def test_6(self): hier = [ @@ -218,13 +218,13 @@ class TestPkg(unittest.TestCase): import t6 self.assertEqual(fixdir(dir(t6)), - ['__all__', '__doc__', '__file__', + ['__all__', '__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) s = """ import t6 from t6 import * self.assertEqual(fixdir(dir(t6)), - ['__all__', '__doc__', '__file__', + ['__all__', '__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'eggs', 'ham', 'spam']) self.assertEqual(dir(), ['eggs', 'ham', 'self', 'spam', 't6']) @@ -252,18 +252,18 @@ class TestPkg(unittest.TestCase): t7, sub, subsub = None, None, None import t7 as tas self.assertEqual(fixdir(dir(tas)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) self.assertFalse(t7) from t7 import sub as subpar self.assertEqual(fixdir(dir(subpar)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__']) self.assertFalse(t7) self.assertFalse(sub) from t7.sub import subsub as subsubsub self.assertEqual(fixdir(dir(subsubsub)), - ['__doc__', '__file__', '__name__', + ['__cached__', '__doc__', '__file__', '__name__', '__package__', '__path__', 'spam']) self.assertFalse(t7) self.assertFalse(sub) diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py index a9a475c..eab66fb 100644 --- a/Lib/test/test_pkgimport.py +++ b/Lib/test/test_pkgimport.py @@ -1,5 +1,12 @@ -import os, sys, string, random, tempfile, unittest - +import os +import sys +import shutil +import string +import random +import tempfile +import unittest + +from imp import cache_from_source from test.support import run_unittest class TestImport(unittest.TestCase): @@ -26,22 +33,17 @@ class TestImport(unittest.TestCase): self.module_path = os.path.join(self.package_dir, 'foo.py') def tearDown(self): - for file in os.listdir(self.package_dir): - os.remove(os.path.join(self.package_dir, file)) - os.rmdir(self.package_dir) - os.rmdir(self.test_dir) + shutil.rmtree(self.test_dir) self.assertNotEqual(sys.path.count(self.test_dir), 0) sys.path.remove(self.test_dir) self.remove_modules() def rewrite_file(self, contents): - for extension in "co": - compiled_path = self.module_path + extension - if os.path.exists(compiled_path): - os.remove(compiled_path) - f = open(self.module_path, 'w') - f.write(contents) - f.close() + compiled_path = cache_from_source(self.module_path) + if os.path.exists(compiled_path): + os.remove(compiled_path) + with open(self.module_path, 'w') as f: + f.write(contents) def test_package_import__semantics(self): diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index d0b81e3..603755a 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -19,8 +19,7 @@ from test import pydoc_mod if hasattr(pydoc_mod, "__loader__"): del pydoc_mod.__loader__ -expected_text_pattern = \ -""" +expected_text_pattern = """ NAME test.pydoc_mod - This is a test module for test_pydoc @@ -87,8 +86,7 @@ CREDITS Nobody """.strip() -expected_html_pattern = \ -""" +expected_html_pattern = """ <table width="100%%" cellspacing=0 cellpadding=2 border=0 summary="heading"> <tr bgcolor="#7799ee"> <td valign=bottom> <br> @@ -186,7 +184,7 @@ war</tt></dd></dl> \x20\x20\x20\x20 <tr><td bgcolor="#7799ee"><tt> </tt></td><td> </td> <td width="100%%">Nobody</td></tr></table> -""".strip() +""".strip() # ' <- emacs turd # output pattern for missing module @@ -287,7 +285,8 @@ class PyDocDocTest(unittest.TestCase): ('i_am_not_here', 'i_am_not_here'), ('test.i_am_not_here_either', 'i_am_not_here_either'), ('test.i_am_not_here.neither_am_i', 'i_am_not_here.neither_am_i'), - ('i_am_not_here.{}'.format(modname), 'i_am_not_here.{}'.format(modname)), + ('i_am_not_here.{}'.format(modname), + 'i_am_not_here.{}'.format(modname)), ('test.{}'.format(modname), modname), ) @@ -304,9 +303,8 @@ class PyDocDocTest(unittest.TestCase): fullmodname = os.path.join(TESTFN, modname) sourcefn = fullmodname + os.extsep + "py" for importstring, expectedinmsg in testpairs: - f = open(sourcefn, 'w') - f.write("import {}\n".format(importstring)) - f.close() + with open(sourcefn, 'w') as f: + f.write("import {}\n".format(importstring)) try: result = run_pydoc(modname).decode("ascii") finally: diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 995c891..068eca9 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -5,9 +5,10 @@ import os.path import sys import re import tempfile -from test.support import verbose, run_unittest, forget -from test.script_helper import (temp_dir, make_script, compile_script, - make_pkg, make_zip_script, make_zip_pkg) +import py_compile +from test.support import forget, make_legacy_pyc, run_unittest, verbose +from test.script_helper import ( + make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir) from runpy import _run_code, _run_module_code, run_module, run_path @@ -45,6 +46,7 @@ class RunModuleCodeTest(unittest.TestCase): self.assertEqual(d["result"], self.expected_result) self.assertIs(d["__name__"], None) self.assertIs(d["__file__"], None) + self.assertIs(d["__cached__"], None) self.assertIs(d["__loader__"], None) self.assertIs(d["__package__"], None) self.assertIs(d["run_argv0"], saved_argv0) @@ -73,6 +75,7 @@ class RunModuleCodeTest(unittest.TestCase): self.assertTrue(d2["run_name_in_sys_modules"]) self.assertTrue(d2["module_in_sys_modules"]) self.assertIs(d2["__file__"], file) + self.assertIs(d2["__cached__"], None) self.assertIs(d2["run_argv0"], file) self.assertIs(d2["__loader__"], loader) self.assertIs(d2["__package__"], package) @@ -170,6 +173,7 @@ class RunModuleTest(unittest.TestCase): del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", mod_name) d2 = run_module(mod_name) # Read from bytecode self.assertIn("x", d2) @@ -192,6 +196,7 @@ class RunModuleTest(unittest.TestCase): del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", pkg_name) d2 = run_module(pkg_name) # Read from bytecode self.assertIn("x", d2) @@ -246,6 +251,7 @@ from ..uncle.cousin import nephew del d1 # Ensure __loader__ entry doesn't keep file open __import__(mod_name) os.remove(mod_fname) + make_legacy_pyc(mod_fname) if verbose: print("Running from compiled:", mod_name) d2 = run_module(mod_name, run_name=run_name) # Read from bytecode self.assertIn("__package__", d2) @@ -313,6 +319,7 @@ argv0 = sys.argv[0] result = run_path(script_name) self.assertEqual(result["__name__"], expected_name) self.assertEqual(result["__file__"], expected_file) + self.assertEqual(result["__cached__"], None) self.assertIn("argv0", result) self.assertEqual(result["argv0"], expected_argv0) self.assertEqual(result["__package__"], expected_package) @@ -332,7 +339,7 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = 'script' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) self._check_script(compiled_name, "<run_path>", compiled_name, compiled_name, None) @@ -348,9 +355,10 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) + compiled_name = py_compile.compile(script_name, doraise=True) os.remove(script_name) - self._check_script(script_dir, "<run_path>", compiled_name, + legacy_pyc = make_legacy_pyc(script_name) + self._check_script(script_dir, "<run_path>", legacy_pyc, script_dir, '') def test_directory_error(self): @@ -371,8 +379,9 @@ argv0 = sys.argv[0] with temp_dir() as script_dir: mod_name = '__main__' script_name = self._make_test_script(script_dir, mod_name) - compiled_name = compile_script(script_name) - zip_name, fname = make_zip_script(script_dir, 'test_zip', compiled_name) + compiled_name = py_compile.compile(script_name, doraise=True) + zip_name, fname = make_zip_script(script_dir, 'test_zip', + compiled_name) self._check_script(zip_name, "<run_path>", fname, zip_name, '') def test_zipfile_error(self): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 931a166..1a50f19 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -258,19 +258,38 @@ class ImportSideEffectTests(unittest.TestCase): """Restore sys.path""" sys.path[:] = self.sys_path - def test_abs__file__(self): - # Make sure all imported modules have their __file__ attribute - # as an absolute path. - # Handled by abs__file__() - site.abs__file__() - for module in (sys, os, builtins): - try: - self.assertTrue(os.path.isabs(module.__file__), repr(module)) - except AttributeError: - continue - # We could try everything in sys.modules; however, when regrtest.py - # runs something like test_frozen before test_site, then we will - # be testing things loaded *after* test_site did path normalization + def test_abs_paths(self): + # Make sure all imported modules have their __file__ and __cached__ + # attributes as absolute paths. Arranging to put the Lib directory on + # PYTHONPATH would cause the os module to have a relative path for + # __file__ if abs_paths() does not get run. sys and builtins (the + # only other modules imported before site.py runs) do not have + # __file__ or __cached__ because they are built-in. + parent = os.path.relpath(os.path.dirname(os.__file__)) + env = os.environ.copy() + env['PYTHONPATH'] = parent + command = 'import os; print(os.__file__, os.__cached__)' + # First, prove that with -S (no 'import site'), the paths are + # relative. + proc = subprocess.Popen([sys.executable, '-S', '-c', command], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + self.assertEqual(proc.returncode, 0) + os__file__, os__cached__ = stdout.split() + self.assertFalse(os.path.isabs(os__file__)) + self.assertFalse(os.path.isabs(os__cached__)) + # Now, with 'import site', it works. + proc = subprocess.Popen([sys.executable, '-c', command], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = proc.communicate() + self.assertEqual(proc.returncode, 0) + os__file__, os__cached__ = stdout.split() + self.assertTrue(os.path.isabs(os__file__)) + self.assertTrue(os.path.isabs(os__cached__)) def test_no_duplicate_paths(self): # No duplicate paths should exist in sys.path diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 8e2cf55..e9a90e5 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -6,6 +6,7 @@ except ImportError: import io import os +import imp import time import shutil import struct @@ -587,7 +588,13 @@ class PyZipFileTests(unittest.TestCase): with zipfile.PyZipFile(TemporaryFile(), "w") as zipfp: fn = __file__ if fn.endswith('.pyc') or fn.endswith('.pyo'): - fn = fn[:-1] + path_split = fn.split(os.sep) + if os.altsep is not None: + path_split.extend(fn.split(os.altsep)) + if '__pycache__' in path_split: + fn = imp.source_from_cache(fn) + else: + fn = fn[:-1] zipfp.writepy(fn) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index c89aef5..ba4e34a 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -48,17 +48,14 @@ NOW = time.time() test_pyc = make_pyc(test_co, NOW) -if __debug__: - pyc_ext = ".pyc" -else: - pyc_ext = ".pyo" - - TESTMOD = "ziptestmodule" TESTPACK = "ziptestpackage" TESTPACK2 = "ziptestpackage2" TEMP_ZIP = os.path.abspath("junk95142.zip") +pyc_file = imp.cache_from_source(TESTMOD + '.py') +pyc_ext = ('.pyc' if __debug__ else '.pyo') + class UncompressedZipImportTestCase(ImportHooksBaseTestCase): @@ -83,14 +80,11 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): stuff = kw.get("stuff", None) if stuff is not None: # Prepend 'stuff' to the start of the zipfile - f = open(TEMP_ZIP, "rb") - data = f.read() - f.close() - - f = open(TEMP_ZIP, "wb") - f.write(stuff) - f.write(data) - f.close() + with open(TEMP_ZIP, "rb") as f: + data = f.read() + with open(TEMP_ZIP, "wb") as f: + f.write(stuff) + f.write(data) sys.path.insert(0, TEMP_ZIP) @@ -180,8 +174,9 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): def testBadMTime(self): badtime_pyc = bytearray(test_pyc) - badtime_pyc[7] ^= 0x02 # flip the second bit -- not the first as that one - # isn't stored in the .py's mtime in the zip archive. + # flip the second bit -- not the first as that one isn't stored in the + # .py's mtime in the zip archive. + badtime_pyc[7] ^= 0x02 files = {TESTMOD + ".py": (NOW, test_src), TESTMOD + pyc_ext: (NOW, badtime_pyc)} self.doTest(".py", files, TESTMOD) @@ -232,7 +227,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEquals(zi.get_source(TESTPACK), None) self.assertEquals(zi.get_source(mod_path), None) self.assertEquals(zi.get_filename(mod_path), mod.__file__) - # To pass in the module name instead of the path, we must use the right importer + # To pass in the module name instead of the path, we must use the + # right importer loader = mod.__loader__ self.assertEquals(loader.get_source(mod_name), None) self.assertEquals(loader.get_filename(mod_name), mod.__file__) @@ -266,8 +262,10 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): mod = zi.load_module(TESTPACK2) self.assertEquals(zi.get_filename(TESTPACK2), mod.__file__) - self.assertEquals(zi.is_package(TESTPACK2 + os.sep + '__init__'), False) - self.assertEquals(zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) + self.assertEquals( + zi.is_package(TESTPACK2 + os.sep + '__init__'), False) + self.assertEquals( + zi.is_package(TESTPACK2 + os.sep + TESTMOD), False) mod_path = TESTPACK2 + os.sep + TESTMOD mod_name = module_path_to_dotted_name(mod_path) @@ -276,7 +274,8 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): self.assertEquals(zi.get_source(TESTPACK2), None) self.assertEquals(zi.get_source(mod_path), None) self.assertEquals(zi.get_filename(mod_path), mod.__file__) - # To pass in the module name instead of the path, we must use the right importer + # To pass in the module name instead of the path, we must use the + # right importer loader = mod.__loader__ self.assertEquals(loader.get_source(mod_name), None) self.assertEquals(loader.get_filename(mod_name), mod.__file__) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 2982ec3..f81cc8b 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -3,10 +3,17 @@ Read and write ZIP files. XXX references to utf-8 need further investigation. """ -import struct, os, time, sys, shutil -import binascii, io, stat import io +import os import re +import imp +import sys +import time +import stat +import shutil +import struct +import binascii + try: import zlib # We may need its compression method @@ -1303,22 +1310,42 @@ class PyZipFile(ZipFile): file_py = pathname + ".py" file_pyc = pathname + ".pyc" file_pyo = pathname + ".pyo" - if os.path.isfile(file_pyo) and \ - os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime: - fname = file_pyo # Use .pyo file - elif not os.path.isfile(file_pyc) or \ - os.stat(file_pyc).st_mtime < os.stat(file_py).st_mtime: + pycache_pyc = imp.cache_from_source(file_py, True) + pycache_pyo = imp.cache_from_source(file_py, False) + if (os.path.isfile(file_pyo) and + os.stat(file_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyo file. + arcname = fname = file_pyo + elif (os.path.isfile(file_pyc) and + os.stat(file_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use .pyc file. + arcname = fname = file_pyc + elif (os.path.isfile(pycache_pyc) and + os.stat(pycache_pyc).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyc file, but write it to the legacy pyc + # file name in the archive. + fname = pycache_pyc + arcname = file_pyc + elif (os.path.isfile(pycache_pyo) and + os.stat(pycache_pyo).st_mtime >= os.stat(file_py).st_mtime): + # Use the __pycache__/*.pyo file, but write it to the legacy pyo + # file name in the archive. + fname = pycache_pyo + arcname = file_pyo + else: + # Compile py into PEP 3147 pyc file. import py_compile if self.debug: print("Compiling", file_py) try: - py_compile.compile(file_py, file_pyc, None, True) - except py_compile.PyCompileError as err: + py_compile.compile(file_py, doraise=True) + except py_compile.PyCompileError as error: print(err.msg) - fname = file_pyc - else: - fname = file_pyc - archivename = os.path.split(fname)[1] + fname = file_py + else: + fname = (pycache_pyc if __debug__ else pycache_pyo) + arcname = (file_pyc if __debug__ else file_pyo) + archivename = os.path.split(arcname)[1] if basename: archivename = "%s/%s" % (basename, archivename) return (fname, archivename) |