diff options
Diffstat (limited to 'Lib/importlib')
32 files changed, 1591 insertions, 690 deletions
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 37577ff..2baaf93 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -36,7 +36,7 @@ def _case_ok(directory, check): """ if 'PYTHONCASEOK' in os.environ: return True - elif check in os.listdir(directory): + elif check in os.listdir(directory if directory else os.getcwd()): return True return False diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 6f60843..425b8bf 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -22,7 +22,7 @@ work. One should use importlib as the public-facing version of this module. def _path_join(*args): """Replacement for os.path.join.""" return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x - for x in args) + for x in args if x) def _path_exists(path): @@ -53,6 +53,8 @@ def _path_isfile(path): # XXX Could also expose Modules/getpath.c:isdir() def _path_isdir(path): """Replacement for os.path.isdir.""" + if not path: + path = _os.getcwd() return _path_is_mode_type(path, 0o040000) @@ -78,20 +80,6 @@ def _path_absolute(path): return _path_join(_os.getcwd(), path) -class _closing: - - """Simple replacement for contextlib.closing.""" - - def __init__(self, obj): - self.obj = obj - - def __enter__(self): - return self.obj - - def __exit__(self, *args): - self.obj.close() - - def _wrap(new, old): """Simple substitute for functools.wraps.""" for replace in ['__module__', '__name__', '__doc__']: @@ -99,6 +87,8 @@ def _wrap(new, old): new.__dict__.update(old.__dict__) +code_type = type(_wrap.__code__) + # Finder/loader utility code ################################################## def set_package(fxn): @@ -138,7 +128,7 @@ def module_for_loader(fxn): the second argument. """ - def decorated(self, fullname): + def decorated(self, fullname, *args, **kwargs): module = sys.modules.get(fullname) is_reload = bool(module) if not is_reload: @@ -148,7 +138,7 @@ def module_for_loader(fxn): module = imp.new_module(fullname) sys.modules[fullname] = module try: - return fxn(self, module) + return fxn(self, module, *args, **kwargs) except: if not is_reload: del sys.modules[fullname] @@ -301,242 +291,238 @@ class FrozenImporter: return imp.is_frozen_package(fullname) -class PyLoader: +class _LoaderBasics: - """Loader base class for Python source code. + """Base class of common code needed by both SourceLoader and + _SourcelessFileLoader.""" - Subclasses need to implement the methods: + def is_package(self, fullname): + """Concrete implementation of InspectLoader.is_package by checking if + the path returned by get_filename has a filename of '__init__.py'.""" + filename = self.get_filename(fullname).rpartition(path_sep)[2] + return filename.rsplit('.', 1)[0] == '__init__' - - source_path - - get_data - - is_package + def _bytes_from_bytecode(self, fullname, data, source_mtime): + """Return the marshalled bytes from bytecode, verifying the magic + number and timestamp along the way. - """ + If source_mtime is None then skip the timestamp check. - @module_for_loader - def load_module(self, module): - """Load a source module.""" - return self._load_module(module) + """ + magic = data[:4] + raw_timestamp = data[4:8] + if len(magic) != 4 or magic != imp.get_magic(): + raise ImportError("bad magic number in {}".format(fullname)) + elif len(raw_timestamp) != 4: + raise EOFError("bad timestamp in {}".format(fullname)) + elif source_mtime is not None: + if marshal._r_long(raw_timestamp) != source_mtime: + raise ImportError("bytecode is stale for {}".format(fullname)) + # Can't return the code object as errors from marshal loading need to + # propagate even when source is available. + return data[8:] - def _load_module(self, module): - """Initialize a module from source.""" + @module_for_loader + def _load_module(self, module, *, sourceless=False): + """Helper for load_module able to handle either source or sourceless + loading.""" name = module.__name__ - code_object = self.get_code(module.__name__) - # __file__ may have been set by the caller, e.g. bytecode path. - if not hasattr(module, '__file__'): - module.__file__ = self.source_path(name) + code_object = self.get_code(name) + module.__file__ = self.get_filename(name) + if not sourceless: + module.__cached__ = imp.cache_from_source(module.__file__) + else: + module.__cached__ = module.__file__ + module.__package__ = name if self.is_package(name): - module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): + module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] + else: module.__package__ = module.__package__.rpartition('.')[0] module.__loader__ = self exec(code_object, module.__dict__) return module - def get_code(self, fullname): - """Get a code object from source.""" - source_path = self.source_path(fullname) - if source_path is None: - message = "a source path must exist to load {0}".format(fullname) - raise ImportError(message) - source = self.get_data(source_path) - # Convert to universal newlines. - line_endings = b'\n' - for index, c in enumerate(source): - if c == ord(b'\n'): - break - elif c == ord(b'\r'): - line_endings = b'\r' - try: - if source[index+1] == ord(b'\n'): - line_endings += b'\n' - except IndexError: - pass - break - if line_endings != b'\n': - source = source.replace(line_endings, b'\n') - return compile(source, source_path, 'exec', dont_inherit=True) - # Never use in implementing import! Imports code within the method. - def get_source(self, fullname): - """Return the source code for a module. +class SourceLoader(_LoaderBasics): - self.source_path() and self.get_data() are used to implement this - method. + def path_mtime(self, path): + """Optional method that returns the modification time (an int) for the + specified path, where path is a str. - """ - path = self.source_path(fullname) - if path is None: - return None - try: - source_bytes = self.get_data(path) - except IOError: - return ImportError("source not available through get_data()") - import io - import tokenize - encoding = tokenize.detect_encoding(io.BytesIO(source_bytes).readline) - return source_bytes.decode(encoding[0]) + Implementing this method allows the loader to read bytecode files. + """ + raise NotImplementedError -class PyPycLoader(PyLoader): + def set_data(self, path, data): + """Optional method which writes data (bytes) to a file path (a str). - """Loader base class for Python source and bytecode. + Implementing this method allows for the writing of bytecode files. - Requires implementing the methods needed for PyLoader as well as - source_mtime, bytecode_path, and write_bytecode. + """ + raise NotImplementedError - """ - @module_for_loader - def load_module(self, module): - """Load a module from source or bytecode.""" - name = module.__name__ - source_path = self.source_path(name) - bytecode_path = self.bytecode_path(name) - # get_code can worry about no viable paths existing. - module.__file__ = source_path or bytecode_path - return self._load_module(module) + def get_source(self, fullname): + """Concrete implementation of InspectLoader.get_source.""" + import tokenize + path = self.get_filename(fullname) + try: + source_bytes = self.get_data(path) + except IOError: + raise ImportError("source not available through get_data()") + encoding = tokenize.detect_encoding(_io.BytesIO(source_bytes).readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) def get_code(self, fullname): - """Get a code object from source or bytecode.""" - # XXX Care enough to make sure this call does not happen if the magic - # number is bad? - source_timestamp = self.source_mtime(fullname) - # Try to use bytecode if it is available. - bytecode_path = self.bytecode_path(fullname) - if bytecode_path: - data = self.get_data(bytecode_path) + """Concrete implementation of InspectLoader.get_code. + + Reading of bytecode requires path_mtime to be implemented. To write + bytecode, set_data must also be implemented. + + """ + source_path = self.get_filename(fullname) + bytecode_path = imp.cache_from_source(source_path) + source_mtime = None + if bytecode_path is not None: try: - magic = data[:4] - if len(magic) < 4: - raise ImportError("bad magic number in {}".format(fullname)) - raw_timestamp = data[4:8] - if len(raw_timestamp) < 4: - raise EOFError("bad timestamp in {}".format(fullname)) - pyc_timestamp = marshal._r_long(raw_timestamp) - bytecode = data[8:] - # Verify that the magic number is valid. - if imp.get_magic() != magic: - raise ImportError("bad magic number in {}".format(fullname)) - # Verify that the bytecode is not stale (only matters when - # there is source to fall back on. - if source_timestamp: - if pyc_timestamp < source_timestamp: - raise ImportError("bytecode is stale") - except (ImportError, EOFError): - # If source is available give it a shot. - if source_timestamp is not None: + source_mtime = self.path_mtime(source_path) + except NotImplementedError: + pass + else: + try: + data = self.get_data(bytecode_path) + except IOError: pass else: - raise - else: - # Bytecode seems fine, so try to use it. - # XXX If the bytecode is ill-formed, would it be beneficial to - # try for using source if available and issue a warning? - return marshal.loads(bytecode) - elif source_timestamp is None: - raise ImportError("no source or bytecode available to create code " - "object for {0!r}".format(fullname)) - # Use the source. - code_object = super().get_code(fullname) - # Generate bytecode and write it out. - if not sys.dont_write_bytecode: + try: + bytes_data = self._bytes_from_bytecode(fullname, data, + source_mtime) + except (ImportError, EOFError): + pass + else: + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + return found + else: + msg = "Non-code object in {}" + raise ImportError(msg.format(bytecode_path)) + source_bytes = self.get_data(source_path) + code_object = compile(source_bytes, source_path, 'exec', + dont_inherit=True) + if (not sys.dont_write_bytecode and bytecode_path is not None and + source_mtime is not None): + # If e.g. Jython ever implements imp.cache_from_source to have + # their own cached file format, this block of code will most likely + # throw an exception. data = bytearray(imp.get_magic()) - data.extend(marshal._w_long(source_timestamp)) + data.extend(marshal._w_long(source_mtime)) data.extend(marshal.dumps(code_object)) - self.write_bytecode(fullname, data) + try: + self.set_data(bytecode_path, data) + except NotImplementedError: + pass return code_object + def load_module(self, fullname): + """Concrete implementation of Loader.load_module. -class _PyFileLoader(PyLoader): + Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be + implemented to load source code. Use of bytecode is dictated by whether + get_code uses/writes bytecode. - """Load a Python source file.""" + """ + return self._load_module(fullname) - def __init__(self, name, path, is_pkg): - self._name = name - self._is_pkg = is_pkg - # Figure out the base path based on whether it was source or bytecode - # that was found. - try: - self._base_path = _path_without_ext(path, imp.PY_SOURCE) - except ValueError: - self._base_path = _path_without_ext(path, imp.PY_COMPILED) - - def _find_path(self, ext_type): - """Find a path from the base path and the specified extension type that - exists, returning None if one is not found.""" - for suffix in _suffix_list(ext_type): - path = self._base_path + suffix - if _path_exists(path): - return path - else: - return None + +class _FileLoader: + + """Base file loader class which implements the loader protocol methods that + require file system usage.""" + + def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" + self._name = fullname + self._path = path @_check_name - def source_path(self, fullname): - """Return the path to an existing source file for the module, or None - if one cannot be found.""" - # Not a property so that it is easy to override. - return self._find_path(imp.PY_SOURCE) + def get_filename(self, fullname): + """Return the path to the source file as found by the finder.""" + return self._path def get_data(self, path): """Return the data from path as raw bytes.""" - return _io.FileIO(path, 'r').read() # Assuming bytes. + with _io.FileIO(path, 'r') as file: + return file.read() - @_check_name - def is_package(self, fullname): - """Return a boolean based on whether the module is a package. - Raises ImportError (like get_source) if the loader cannot handle the - package. +class _SourceFileLoader(_FileLoader, SourceLoader): - """ - return self._is_pkg + """Concrete implementation of SourceLoader using the file system.""" + def path_mtime(self, path): + """Return the modification time for the path.""" + return int(_os.stat(path).st_mtime) -class _PyPycFileLoader(PyPycLoader, _PyFileLoader): + def set_data(self, path, data): + """Write bytes data to a file.""" + parent, _, filename = path.rpartition(path_sep) + path_parts = [] + # Figure out what directories are missing. + while parent and not _path_isdir(parent): + parent, _, part = parent.rpartition(path_sep) + path_parts.append(part) + # Create needed directories. + for part in reversed(path_parts): + parent = _path_join(parent, part) + try: + _os.mkdir(parent) + except OSError as exc: + # Probably another Python process already created the dir. + if exc.errno == errno.EEXIST: + continue + else: + raise + except IOError as exc: + # If can't get proper access, then just forget about writing + # the data. + if exc.errno == errno.EACCES: + return + else: + raise + try: + with _io.FileIO(path, 'wb') as file: + file.write(data) + except IOError as exc: + # Don't worry if you can't write bytecode. + if exc.errno == errno.EACCES: + return + else: + raise - """Load a module from a source or bytecode file.""" - @_check_name - def source_mtime(self, name): - """Return the modification time of the source for the specified - module.""" - source_path = self.source_path(name) - if not source_path: - return None - return int(_os.stat(source_path).st_mtime) +class _SourcelessFileLoader(_FileLoader, _LoaderBasics): - @_check_name - def bytecode_path(self, fullname): - """Return the path to a bytecode file, or None if one does not - exist.""" - # Not a property for easy overriding. - return self._find_path(imp.PY_COMPILED) + """Loader which handles sourceless file imports.""" - @_check_name - def write_bytecode(self, name, data): - """Write out 'data' for the specified module, returning a boolean - signifying if the write-out actually occurred. + def load_module(self, fullname): + return self._load_module(fullname, sourceless=True) - Raises ImportError (just like get_source) if the specified module - cannot be handled by the loader. + def get_code(self, fullname): + path = self.get_filename(fullname) + data = self.get_data(path) + bytes_data = self._bytes_from_bytecode(fullname, data, None) + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + return found + else: + raise ImportError("Non-code object in {}".format(path)) - """ - bytecode_path = self.bytecode_path(name) - if not bytecode_path: - bytecode_path = self._base_path + _suffix_list(imp.PY_COMPILED)[0] - try: - # Assuming bytes. - with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file: - bytecode_file.write(data) - return True - except IOError as exc: - if exc.errno == errno.EACCES: - return False - else: - raise + def get_source(self, fullname): + """Return None as there is no source code.""" + return None class _ExtensionFileLoader: @@ -547,7 +533,7 @@ class _ExtensionFileLoader: """ - def __init__(self, name, path, is_pkg): + def __init__(self, name, path): """Initialize the loader. If is_pkg is True then an exception is raised as extension modules @@ -556,8 +542,6 @@ class _ExtensionFileLoader: """ self._name = name self._path = path - if is_pkg: - raise ValueError("extension modules cannot be packages") @_check_name @set_package @@ -655,147 +639,88 @@ class PathFinder: return None -class _ChainedFinder: - - """Finder that sequentially calls other finders.""" - - def __init__(self, *finders): - self._finders = finders - - def find_module(self, fullname, path=None): - for finder in self._finders: - result = finder.find_module(fullname, path) - if result: - return result - else: - return None - - class _FileFinder: - """Base class for file finders. - - Subclasses are expected to define the following attributes: - - * _suffixes - Sequence of file suffixes whose order will be followed. - - * _possible_package - True if importer should check for packages. + """File-based finder. - * _loader - A callable that takes the module name, a file path, and whether - the path points to a package and returns a loader for the module - found at that path. + Constructor takes a list of objects detailing what file extensions their + loader supports along with whether it can be used for a package. """ - def __init__(self, path_entry): - """Initialize an importer for the passed-in sys.path entry (which is - assumed to have already been verified as an existing directory). - - Can be used as an entry on sys.path_hook. - - """ - absolute_path = _path_absolute(path_entry) - if not _path_isdir(absolute_path): - raise ImportError("only directories are supported") - self._path_entry = absolute_path - - def find_module(self, fullname, path=None): + def __init__(self, path, *details): + """Initialize with finder details.""" + packages = [] + modules = [] + for detail in details: + modules.extend((suffix, detail.loader) for suffix in detail.suffixes) + if detail.supports_packages: + packages.extend((suffix, detail.loader) + for suffix in detail.suffixes) + self.packages = packages + self.modules = modules + self.path = path + + def find_module(self, fullname): + """Try to find a loader for the specified module.""" tail_module = fullname.rpartition('.')[2] - package_directory = None - if self._possible_package: - for ext in self._suffixes: - package_directory = _path_join(self._path_entry, tail_module) - init_filename = '__init__' + ext - package_init = _path_join(package_directory, init_filename) - if (_path_isfile(package_init) and - _case_ok(self._path_entry, tail_module) and - _case_ok(package_directory, init_filename)): - return self._loader(fullname, package_init, True) - for ext in self._suffixes: - file_name = tail_module + ext - file_path = _path_join(self._path_entry, file_name) - if (_path_isfile(file_path) and - _case_ok(self._path_entry, file_name)): - return self._loader(fullname, file_path, False) - else: - # Raise a warning if it matches a directory w/o an __init__ file. - if (package_directory is not None and - _path_isdir(package_directory) and - _case_ok(self._path_entry, tail_module)): - _warnings.warn("Not importing directory %s: missing __init__" - % package_directory, ImportWarning) - return None - - -class _PyFileFinder(_FileFinder): - - """Importer for source/bytecode files.""" - - _possible_package = True - _loader = _PyFileLoader - - def __init__(self, path_entry): - # Lack of imp during class creation means _suffixes is set here. - # Make sure that Python source files are listed first! Needed for an - # optimization by the loader. - self._suffixes = _suffix_list(imp.PY_SOURCE) - super().__init__(path_entry) - - -class _PyPycFileFinder(_PyFileFinder): + base_path = _path_join(self.path, tail_module) + if _path_isdir(base_path) and _case_ok(self.path, tail_module): + for suffix, loader in self.packages: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if (_path_isfile(full_path) and + _case_ok(base_path, init_filename)): + return loader(fullname, full_path) + else: + msg = "Not importing directory {}: missing __init__" + _warnings.warn(msg.format(base_path), ImportWarning) + for suffix, loader in self.modules: + mod_filename = tail_module + suffix + full_path = _path_join(self.path, mod_filename) + if _path_isfile(full_path) and _case_ok(self.path, mod_filename): + return loader(fullname, full_path) + return None - """Finder for source and bytecode files.""" +class _SourceFinderDetails: - _loader = _PyPycFileLoader + loader = _SourceFileLoader + supports_packages = True - def __init__(self, path_entry): - super().__init__(path_entry) - self._suffixes += _suffix_list(imp.PY_COMPILED) + def __init__(self): + self.suffixes = _suffix_list(imp.PY_SOURCE) +class _SourcelessFinderDetails: + loader = _SourcelessFileLoader + supports_packages = True + def __init__(self): + self.suffixes = _suffix_list(imp.PY_COMPILED) -class _ExtensionFileFinder(_FileFinder): - """Importer for extension files.""" +class _ExtensionFinderDetails: - _possible_package = False - _loader = _ExtensionFileLoader + loader = _ExtensionFileLoader + supports_packages = False - def __init__(self, path_entry): - # Assigning to _suffixes here instead of at the class level because - # imp is not imported at the time of class creation. - self._suffixes = _suffix_list(imp.C_EXTENSION) - super().__init__(path_entry) + def __init__(self): + self.suffixes = _suffix_list(imp.C_EXTENSION) # Import itself ############################################################### -def _chained_path_hook(*path_hooks): - """Create a closure which sequentially checks path hooks to see which ones - (if any) can work with a path.""" - def path_hook(entry): - """Check to see if 'entry' matches any of the enclosed path hooks.""" - finders = [] - for hook in path_hooks: - try: - finder = hook(entry) - except ImportError: - continue - else: - finders.append(finder) - if not finders: - raise ImportError("no finder found") - else: - return _ChainedFinder(*finders) - - return path_hook +def _file_path_hook(path): + """If the path is a directory, return a file-based finder.""" + if _path_isdir(path): + return _FileFinder(path, _ExtensionFinderDetails(), + _SourceFinderDetails(), + _SourcelessFinderDetails()) + else: + raise ImportError("only directories are supported") -_DEFAULT_PATH_HOOK = _chained_path_hook(_ExtensionFileFinder, _PyPycFileFinder) +_DEFAULT_PATH_HOOK = _file_path_hook class _DefaultPathFinder(PathFinder): @@ -833,6 +758,8 @@ class _ImportLockContext: _IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder] +_ERR_MSG = 'No module named {}' + def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. @@ -880,7 +807,11 @@ def _gcd_import(name, package=None, level=0): _gcd_import(parent) # Backwards-compatibility; be nicer to skip the dict lookup. parent_module = sys.modules[parent] - path = parent_module.__path__ + try: + path = parent_module.__path__ + except AttributeError: + msg = (_ERR_MSG + '; {} is not a package').format(name, parent) + raise ImportError(msg) meta_path = sys.meta_path + _IMPLICIT_META_PATH for finder in meta_path: loader = finder.find_module(name, path) @@ -888,7 +819,7 @@ def _gcd_import(name, package=None, level=0): loader.load_module(name) break else: - raise ImportError("No module named {0}".format(name)) + raise ImportError(_ERR_MSG.format(name)) # Backwards-compatibility; be nicer to skip the dict lookup. module = sys.modules[name] if parent: @@ -918,13 +849,15 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0): import (e.g. ``from ..pkg import mod`` would have a 'level' of 2). """ + if not hasattr(name, 'rpartition'): + raise TypeError("module name must be str, not {}".format(type(name))) if level == 0: module = _gcd_import(name) else: - # __package__ is not guaranteed to be defined. - try: - package = globals['__package__'] - except KeyError: + # __package__ is not guaranteed to be defined or could be set to None + # to represent that it's proper value is unknown + package = globals.get('__package__') + if package is None: package = globals['__name__'] if '__path__' not in globals: package = package.rpartition('.')[0] @@ -944,6 +877,7 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0): # If a package was imported, try to import stuff from fromlist. if hasattr(module, '__path__'): if '*' in fromlist and hasattr(module, '__all__'): + fromlist = list(fromlist) fromlist.remove('*') fromlist.extend(module.__all__) for x in (y for y in fromlist if not hasattr(module,y)): diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 7b89d0b..fa343f8 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -1,8 +1,16 @@ """Abstract base classes related to import.""" from . import _bootstrap from . import machinery +from . import util import abc +import imp +import io +import marshal +import os.path +import sys +import tokenize import types +import warnings class Loader(metaclass=abc.ABCMeta): @@ -10,8 +18,9 @@ class Loader(metaclass=abc.ABCMeta): """Abstract base class for import loaders.""" @abc.abstractmethod - def load_module(self, fullname:str) -> types.ModuleType: - """Abstract method which when implemented should load a module.""" + def load_module(self, fullname): + """Abstract method which when implemented should load a module. + The fullname is a str.""" raise NotImplementedError @@ -20,8 +29,11 @@ class Finder(metaclass=abc.ABCMeta): """Abstract base class for import finders.""" @abc.abstractmethod - def find_module(self, fullname:str, path:[str]=None) -> Loader: - """Abstract method which when implemented should find a module.""" + def find_module(self, fullname, path=None): + """Abstract method which when implemented should find a module. + The fullname is a str and the optional path is a str or None. + Returns a Loader object. + """ raise NotImplementedError Finder.register(machinery.BuiltinImporter) @@ -39,9 +51,9 @@ class ResourceLoader(Loader): """ @abc.abstractmethod - def get_data(self, path:str) -> bytes: + def get_data(self, path): """Abstract method which when implemented should return the bytes for - the specified path.""" + the specified path. The path must be a str.""" raise NotImplementedError @@ -55,68 +67,238 @@ class InspectLoader(Loader): """ @abc.abstractmethod - def is_package(self, fullname:str) -> bool: + def is_package(self, fullname): """Abstract method which when implemented should return whether the - module is a package.""" - return NotImplementedError + module is a package. The fullname is a str. Returns a bool.""" + raise NotImplementedError @abc.abstractmethod - def get_code(self, fullname:str) -> types.CodeType: + def get_code(self, fullname): """Abstract method which when implemented should return the code object - for the module""" - return NotImplementedError + for the module. The fullname is a str. Returns a types.CodeType.""" + raise NotImplementedError @abc.abstractmethod - def get_source(self, fullname:str) -> str: + def get_source(self, fullname): """Abstract method which should return the source code for the - module.""" - return NotImplementedError + module. The fullname is a str. Returns a str.""" + raise NotImplementedError InspectLoader.register(machinery.BuiltinImporter) InspectLoader.register(machinery.FrozenImporter) -class PyLoader(_bootstrap.PyLoader, ResourceLoader, InspectLoader): +class ExecutionLoader(InspectLoader): - """Abstract base class to assist in loading source code by requiring only - back-end storage methods to be implemented. + """Abstract base class for loaders that wish to support the execution of + modules as scripts. - The methods get_code, get_source, and load_module are implemented for the - user. + This ABC represents one of the optional protocols specified in PEP 302. + + """ + + @abc.abstractmethod + def get_filename(self, fullname): + """Abstract method which should return the value that __file__ is to be + set to.""" + raise NotImplementedError + + +class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): + + """Abstract base class for loading source code (and optionally any + corresponding bytecode). + + To support loading from source code, the abstractmethods inherited from + ResourceLoader and ExecutionLoader need to be implemented. To also support + loading from bytecode, the optional methods specified directly by this ABC + is required. + + Inherited abstractmethods not implemented in this ABC: + + * ResourceLoader.get_data + * ExecutionLoader.get_filename + + """ + + def path_mtime(self, path): + """Return the (int) modification time for the path (str).""" + raise NotImplementedError + + def set_data(self, path, data): + """Write the bytes to the path (if possible). + + Accepts a str path and data as bytes. + + Any needed intermediary directories are to be created. If for some + reason the file cannot be written because of permissions, fail + silently. + + """ + raise NotImplementedError + + +class PyLoader(SourceLoader): + + """Implement the deprecated PyLoader ABC in terms of SourceLoader. + + This class has been deprecated! It is slated for removal in Python 3.4. + If compatibility with Python 3.1 is not needed then implement the + SourceLoader ABC instead of this class. If Python 3.1 compatibility is + needed, then use the following idiom to have a single class that is + compatible with Python 3.1 onwards:: + + try: + from importlib.abc import SourceLoader + except ImportError: + from importlib.abc import PyLoader as SourceLoader + + + class CustomLoader(SourceLoader): + def get_filename(self, fullname): + # Implement ... + + def source_path(self, fullname): + '''Implement source_path in terms of get_filename.''' + try: + return self.get_filename(fullname) + except ImportError: + return None + + def is_package(self, fullname): + filename = os.path.basename(self.get_filename(fullname)) + return os.path.splitext(filename)[0] == '__init__' """ @abc.abstractmethod - def source_path(self, fullname:str) -> object: - """Abstract method which when implemented should return the path to the - sourced code for the module.""" + def is_package(self, fullname): + raise NotImplementedError + + @abc.abstractmethod + def source_path(self, fullname): + """Abstract method. Accepts a str module name and returns the path to + the source code for the module.""" raise NotImplementedError + def get_filename(self, fullname): + """Implement get_filename in terms of source_path. -class PyPycLoader(_bootstrap.PyPycLoader, PyLoader): + As get_filename should only return a source file path there is no + chance of the path not existing but loading still being possible, so + ImportError should propagate instead of being turned into returning + None. + + """ + warnings.warn("importlib.abc.PyLoader is deprecated and is " + "slated for removal in Python 3.4; " + "use SourceLoader instead. " + "See the importlib documentation on how to be " + "compatible with Python 3.1 onwards.", + PendingDeprecationWarning) + path = self.source_path(fullname) + if path is None: + raise ImportError + else: + return path + + +class PyPycLoader(PyLoader): """Abstract base class to assist in loading source and bytecode by requiring only back-end storage methods to be implemented. + This class has been deprecated! Removal is slated for Python 3.4. Implement + the SourceLoader ABC instead. If Python 3.1 compatibility is needed, see + PyLoader. + The methods get_code, get_source, and load_module are implemented for the user. """ + def get_filename(self, fullname): + """Return the source or bytecode file path.""" + path = self.source_path(fullname) + if path is not None: + return path + path = self.bytecode_path(fullname) + if path is not None: + return path + raise ImportError("no source or bytecode path available for " + "{0!r}".format(fullname)) + + def get_code(self, fullname): + """Get a code object from source or bytecode.""" + warnings.warn("importlib.abc.PyPycLoader is deprecated and slated for " + "removal in Python 3.4; use SourceLoader instead. " + "If Python 3.1 compatibility is required, see the " + "latest documentation for PyLoader.", + PendingDeprecationWarning) + source_timestamp = self.source_mtime(fullname) + # Try to use bytecode if it is available. + bytecode_path = self.bytecode_path(fullname) + if bytecode_path: + data = self.get_data(bytecode_path) + try: + magic = data[:4] + if len(magic) < 4: + raise ImportError("bad magic number in {}".format(fullname)) + raw_timestamp = data[4:8] + if len(raw_timestamp) < 4: + raise EOFError("bad timestamp in {}".format(fullname)) + pyc_timestamp = marshal._r_long(raw_timestamp) + bytecode = data[8:] + # Verify that the magic number is valid. + if imp.get_magic() != magic: + raise ImportError("bad magic number in {}".format(fullname)) + # Verify that the bytecode is not stale (only matters when + # there is source to fall back on. + if source_timestamp: + if pyc_timestamp < source_timestamp: + raise ImportError("bytecode is stale") + except (ImportError, EOFError): + # If source is available give it a shot. + if source_timestamp is not None: + pass + else: + raise + else: + # Bytecode seems fine, so try to use it. + return marshal.loads(bytecode) + elif source_timestamp is None: + raise ImportError("no source or bytecode available to create code " + "object for {0!r}".format(fullname)) + # Use the source. + source_path = self.source_path(fullname) + if source_path is None: + message = "a source path must exist to load {0}".format(fullname) + raise ImportError(message) + source = self.get_data(source_path) + code_object = compile(source, source_path, 'exec', dont_inherit=True) + # Generate bytecode and write it out. + if not sys.dont_write_bytecode: + data = bytearray(imp.get_magic()) + data.extend(marshal._w_long(source_timestamp)) + data.extend(marshal.dumps(code_object)) + self.write_bytecode(fullname, data) + return code_object + @abc.abstractmethod - def source_mtime(self, fullname:str) -> int: - """Abstract method which when implemented should return the + def source_mtime(self, fullname): + """Abstract method. Accepts a str filename and returns an int modification time for the source of the module.""" raise NotImplementedError @abc.abstractmethod - def bytecode_path(self, fullname:str) -> object: - """Abstract method which when implemented should return the path to the - bytecode for the module.""" + def bytecode_path(self, fullname): + """Abstract method. Accepts a str filename and returns the str pathname + to the bytecode for the module.""" raise NotImplementedError @abc.abstractmethod - def write_bytecode(self, fullname:str, bytecode:bytes): - """Abstract method which when implemented should attempt to write the - bytecode for the module.""" + def write_bytecode(self, fullname, bytecode): + """Abstract method. Accepts a str filename and bytes object + representing the bytecode for the module. Returns a boolean + representing whether the bytecode was written or not.""" raise NotImplementedError diff --git a/Lib/importlib/test/__init__.py b/Lib/importlib/test/__init__.py index bda33e6..e69de29 100644 --- a/Lib/importlib/test/__init__.py +++ b/Lib/importlib/test/__init__.py @@ -1,31 +0,0 @@ -import os.path -import sys -import unittest - - -def test_suite(package=__package__, directory=os.path.dirname(__file__)): - suite = unittest.TestSuite() - for name in os.listdir(directory): - if name.startswith('.'): - continue - path = os.path.join(directory, name) - if (os.path.isfile(path) and name.startswith('test_') and - name.endswith('.py')): - submodule_name = os.path.splitext(name)[0] - module_name = "{0}.{1}".format(package, submodule_name) - __import__(module_name, level=0) - module_tests = unittest.findTestCases(sys.modules[module_name]) - suite.addTest(module_tests) - elif os.path.isdir(path): - package_name = "{0}.{1}".format(package, name) - __import__(package_name, level=0) - package_tests = getattr(sys.modules[package_name], 'test_suite')() - suite.addTest(package_tests) - else: - continue - return suite - - -if __name__ == '__main__': - from test.support import run_unittest - run_unittest(test_suite('importlib.test')) diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py new file mode 100644 index 0000000..decc53d --- /dev/null +++ b/Lib/importlib/test/__main__.py @@ -0,0 +1,29 @@ +"""Run importlib's test suite. + +Specifying the ``--builtin`` flag will run tests, where applicable, with +builtins.__import__ instead of importlib.__import__. + +""" +import importlib +from importlib.test.import_ import util +import os.path +from test.support import run_unittest +import sys +import unittest + + +def test_main(): + if '__pycache__' in __file__: + parts = __file__.split(os.path.sep) + start_dir = sep.join(parts[:-2]) + else: + start_dir = os.path.dirname(__file__) + top_dir = os.path.dirname(os.path.dirname(start_dir)) + test_loader = unittest.TestLoader() + if '--builtin' in sys.argv: + util.using___import__ = True + run_unittest(test_loader.discover(start_dir, top_level_dir=top_dir)) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py index f709a3c..b5de6c6 100644 --- a/Lib/importlib/test/benchmark.py +++ b/Lib/importlib/test/benchmark.py @@ -1,69 +1,159 @@ +"""Benchmark some basic import use-cases. + +The assumption is made that this benchmark is run in a fresh interpreter and +thus has no external changes made to import-related attributes in sys. + +""" from . import util from .source import util as source_util -import gc import decimal import imp import importlib +import os +import py_compile import sys import timeit -def bench_cache(import_, repeat, number): - """Measure the time it takes to pull from sys.modules.""" +def bench(name, cleanup=lambda: None, *, seconds=1, repeat=3): + """Bench the given statement as many times as necessary until total + executions take one second.""" + stmt = "__import__({!r})".format(name) + timer = timeit.Timer(stmt) + for x in range(repeat): + total_time = 0 + count = 0 + while total_time < seconds: + try: + total_time += timer.timeit(1) + finally: + cleanup() + count += 1 + else: + # One execution too far + if total_time > seconds: + count -= 1 + yield count // seconds + +def from_cache(seconds, repeat): + """sys.modules""" name = '<benchmark import>' + module = imp.new_module(name) + module.__file__ = '<test>' + module.__package__ = '' with util.uncache(name): - module = imp.new_module(name) sys.modules[name] = module - runs = [] - for x in range(repeat): - start_time = timeit.default_timer() - for y in range(number): - import_(name) - end_time = timeit.default_timer() - runs.append(end_time - start_time) - return min(runs) + for result in bench(name, repeat=repeat, seconds=seconds): + yield result -def bench_importing_source(import_, repeat, number, loc=100000): - """Measure importing source from disk. +def builtin_mod(seconds, repeat): + """Built-in module""" + name = 'errno' + if name in sys.modules: + del sys.modules[name] + # Relying on built-in importer being implicit. + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result - For worst-case scenario, the line endings are \\r\\n and thus require - universal newline translation. - """ - name = '__benchmark' - with source_util.create_modules(name) as mapping: - with open(mapping[name], 'w') as file: - for x in range(loc): - file.write("{0}\r\n".format(x)) - with util.import_state(path=[mapping['.root']]): - runs = [] - for x in range(repeat): - start_time = timeit.default_timer() - for y in range(number): - try: - import_(name) - finally: - del sys.modules[name] - end_time = timeit.default_timer() - runs.append(end_time - start_time) - return min(runs) +def source_wo_bytecode(seconds, repeat): + """Source w/o bytecode: simple""" + sys.dont_write_bytecode = True + try: + name = '__importlib_test_benchmark__' + # Clears out sys.modules and puts an entry at the front of sys.path. + with source_util.create_modules(name) as mapping: + assert not os.path.exists(imp.cache_from_source(mapping[name])) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + finally: + sys.dont_write_bytecode = False -def main(import_): - args = [('sys.modules', bench_cache, 5, 500000), - ('source', bench_importing_source, 5, 10000)] - test_msg = "{test}, {number} times (best of {repeat}):" - result_msg = "{result:.2f} secs" - gc.disable() +def decimal_wo_bytecode(seconds, repeat): + """Source w/o bytecode: decimal""" + name = 'decimal' + decimal_bytecode = imp.cache_from_source(decimal.__file__) + if os.path.exists(decimal_bytecode): + os.unlink(decimal_bytecode) + sys.dont_write_bytecode = True try: - for name, meth, repeat, number in args: - result = meth(import_, repeat, number) - print(test_msg.format(test=name, repeat=repeat, - number=number).ljust(40), - result_msg.format(result=result).rjust(10)) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result finally: - gc.enable() + sys.dont_write_bytecode = False + + +def source_writing_bytecode(seconds, repeat): + """Source writing bytecode: simple""" + assert not sys.dont_write_bytecode + name = '__importlib_test_benchmark__' + with source_util.create_modules(name) as mapping: + def cleanup(): + sys.modules.pop(name) + os.unlink(imp.cache_from_source(mapping[name])) + for result in bench(name, cleanup, repeat=repeat, seconds=seconds): + assert not os.path.exists(imp.cache_from_source(mapping[name])) + yield result + + +def decimal_writing_bytecode(seconds, repeat): + """Source writing bytecode: decimal""" + assert not sys.dont_write_bytecode + name = 'decimal' + def cleanup(): + sys.modules.pop(name) + os.unlink(imp.cache_from_source(decimal.__file__)) + for result in bench(name, cleanup, repeat=repeat, seconds=seconds): + yield result + + +def source_using_bytecode(seconds, repeat): + """Bytecode w/ source: simple""" + name = '__importlib_test_benchmark__' + with source_util.create_modules(name) as mapping: + py_compile.compile(mapping[name]) + assert os.path.exists(imp.cache_from_source(mapping[name])) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + + +def decimal_using_bytecode(seconds, repeat): + """Bytecode w/ source: decimal""" + name = 'decimal' + py_compile.compile(decimal.__file__) + for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat, + seconds=seconds): + yield result + + +def main(import_): + __builtins__.__import__ = import_ + benchmarks = (from_cache, builtin_mod, + source_using_bytecode, source_wo_bytecode, + source_writing_bytecode, + decimal_using_bytecode, decimal_writing_bytecode, + decimal_wo_bytecode,) + seconds = 1 + seconds_plural = 's' if seconds > 1 else '' + repeat = 3 + header = "Measuring imports/second over {} second{}, best out of {}\n" + print(header.format(seconds, seconds_plural, repeat)) + for benchmark in benchmarks: + print(benchmark.__doc__, "[", end=' ') + sys.stdout.flush() + results = [] + for result in benchmark(seconds=seconds, repeat=repeat): + results.append(result) + print(result, end=' ') + sys.stdout.flush() + assert not sys.dont_write_bytecode + print("]", "best is", format(max(results), ',d')) if __name__ == '__main__': @@ -74,7 +164,7 @@ if __name__ == '__main__': default=False, help="use the built-in __import__") options, args = parser.parse_args() if args: - raise RuntimeError("unrecognized args: {0}".format(args)) + raise RuntimeError("unrecognized args: {}".format(args)) import_ = __import__ if not options.builtin: import_ = importlib.__import__ diff --git a/Lib/importlib/test/builtin/test_loader.py b/Lib/importlib/test/builtin/test_loader.py index dff00ce..1a8539b 100644 --- a/Lib/importlib/test/builtin/test_loader.py +++ b/Lib/importlib/test/builtin/test_loader.py @@ -54,13 +54,15 @@ class LoaderTests(abc.LoaderTests): def test_unloadable(self): name = 'dssdsdfff' assert name not in sys.builtin_module_names - self.assertRaises(ImportError, self.load_module, name) + with self.assertRaises(ImportError): + self.load_module(name) def test_already_imported(self): # Using the name of a module already imported but not a built-in should # still fail. assert hasattr(importlib, '__file__') # Not a built-in. - self.assertRaises(ImportError, self.load_module, 'importlib') + with self.assertRaises(ImportError): + self.load_module('importlib') class InspectLoaderTests(unittest.TestCase): @@ -86,7 +88,8 @@ class InspectLoaderTests(unittest.TestCase): # Modules not built-in should raise ImportError. for meth_name in ('get_code', 'get_source', 'is_package'): method = getattr(machinery.BuiltinImporter, meth_name) - self.assertRaises(ImportError, method, builtin_util.BAD_NAME) + with self.assertRaises(ImportError): + method(builtin_util.BAD_NAME) diff --git a/Lib/importlib/test/extension/test_case_sensitivity.py b/Lib/importlib/test/extension/test_case_sensitivity.py index 3865539..e062fb6 100644 --- a/Lib/importlib/test/extension/test_case_sensitivity.py +++ b/Lib/importlib/test/extension/test_case_sensitivity.py @@ -13,7 +13,8 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase): good_name = ext_util.NAME bad_name = good_name.upper() assert good_name != bad_name - finder = _bootstrap._ExtensionFileFinder(ext_util.PATH) + finder = _bootstrap._FileFinder(ext_util.PATH, + _bootstrap._ExtensionFinderDetails()) return finder.find_module(bad_name) def test_case_sensitive(self): diff --git a/Lib/importlib/test/extension/test_finder.py b/Lib/importlib/test/extension/test_finder.py index 546a176..ea97483 100644 --- a/Lib/importlib/test/extension/test_finder.py +++ b/Lib/importlib/test/extension/test_finder.py @@ -9,7 +9,8 @@ class FinderTests(abc.FinderTests): """Test the finder for extension modules.""" def find_module(self, fullname): - importer = _bootstrap._ExtensionFileFinder(util.PATH) + importer = _bootstrap._FileFinder(util.PATH, + _bootstrap._ExtensionFinderDetails()) return importer.find_module(fullname) def test_module(self): diff --git a/Lib/importlib/test/extension/test_loader.py b/Lib/importlib/test/extension/test_loader.py index 71841c6..4a783db 100644 --- a/Lib/importlib/test/extension/test_loader.py +++ b/Lib/importlib/test/extension/test_loader.py @@ -13,7 +13,7 @@ class LoaderTests(abc.LoaderTests): def load_module(self, fullname): loader = _bootstrap._ExtensionFileLoader(ext_util.NAME, - ext_util.FILEPATH, False) + ext_util.FILEPATH) return loader.load_module(fullname) def test_module(self): @@ -46,7 +46,8 @@ class LoaderTests(abc.LoaderTests): pass def test_unloadable(self): - self.assertRaises(ImportError, self.load_module, 'asdfjkl;') + with self.assertRaises(ImportError): + self.load_module('asdfjkl;') def test_main(): diff --git a/Lib/importlib/test/extension/test_path_hook.py b/Lib/importlib/test/extension/test_path_hook.py index bf2f411..4610420 100644 --- a/Lib/importlib/test/extension/test_path_hook.py +++ b/Lib/importlib/test/extension/test_path_hook.py @@ -14,7 +14,7 @@ class PathHookTests(unittest.TestCase): # XXX Should it only work for directories containing an extension module? def hook(self, entry): - return _bootstrap._ExtensionFileFinder(entry) + return _bootstrap._file_path_hook(entry) def test_success(self): # Path hook should handle a directory where a known extension module diff --git a/Lib/importlib/test/frozen/test_loader.py b/Lib/importlib/test/frozen/test_loader.py index 06532f1..b685ef5 100644 --- a/Lib/importlib/test/frozen/test_loader.py +++ b/Lib/importlib/test/frozen/test_loader.py @@ -57,8 +57,8 @@ class LoaderTests(abc.LoaderTests): def test_unloadable(self): assert machinery.FrozenImporter.find_module('_not_real') is None - self.assertRaises(ImportError, machinery.FrozenImporter.load_module, - '_not_real') + with self.assertRaises(ImportError): + machinery.FrozenImporter.load_module('_not_real') class InspectLoaderTests(unittest.TestCase): @@ -92,7 +92,8 @@ class InspectLoaderTests(unittest.TestCase): # Raise ImportError for modules that are not frozen. for meth_name in ('get_code', 'get_source', 'is_package'): method = getattr(machinery.FrozenImporter, meth_name) - self.assertRaises(ImportError, method, 'importlib') + with self.assertRaises(ImportError): + method('importlib') def test_main(): diff --git a/Lib/importlib/test/import_/test___package__.py b/Lib/importlib/test/import_/test___package__.py index 4dc6901..5056ae5 100644 --- a/Lib/importlib/test/import_/test___package__.py +++ b/Lib/importlib/test/import_/test___package__.py @@ -19,8 +19,9 @@ class Using__package__(unittest.TestCase): base = package.rsplit('.', level)[0] return '{0}.{1}'.format(base, name) - But since there is no guarantee that __package__ has been set, there has to - be a way to calculate the attribute's value [__name__]:: + But since there is no guarantee that __package__ has been set (or not been + set to None [None]), there has to be a way to calculate the attribute's value + [__name__]:: def calc_package(caller_name, has___path__): if has__path__: @@ -43,28 +44,34 @@ class Using__package__(unittest.TestCase): fromlist=['attr'], level=2) self.assertEqual(module.__name__, 'pkg') - def test_using___name__(self): + def test_using___name__(self, package_as_None=False): # [__name__] + globals_ = {'__name__': 'pkg.fake', '__path__': []} + if package_as_None: + globals_['__package__'] = None with util.mock_modules('pkg.__init__', 'pkg.fake') as importer: with util.import_state(meta_path=[importer]): import_util.import_('pkg.fake') - module = import_util.import_('', - globals={'__name__': 'pkg.fake', - '__path__': []}, - fromlist=['attr'], level=2) + module = import_util.import_('', globals= globals_, + fromlist=['attr'], level=2) self.assertEqual(module.__name__, 'pkg') + def test_None_as___package__(self): + # [None] + self.test_using___name__(package_as_None=True) + def test_bad__package__(self): globals = {'__package__': '<not real>'} - self.assertRaises(SystemError, import_util.import_,'', globals, {}, - ['relimport'], 1) + with self.assertRaises(SystemError): + import_util.import_('', globals, {}, ['relimport'], 1) def test_bunk__package__(self): globals = {'__package__': 42} - self.assertRaises(ValueError, import_util.import_, '', globals, {}, - ['relimport'], 1) + with self.assertRaises(ValueError): + import_util.import_('', globals, {}, ['relimport'], 1) +@import_util.importlib_only class Setting__package__(unittest.TestCase): """Because __package__ is a new feature, it is not always set by a loader. diff --git a/Lib/importlib/test/import_/test_api.py b/Lib/importlib/test/import_/test_api.py new file mode 100644 index 0000000..9075d42 --- /dev/null +++ b/Lib/importlib/test/import_/test_api.py @@ -0,0 +1,22 @@ +from . import util +import unittest + + +class APITest(unittest.TestCase): + + """Test API-specific details for __import__ (e.g. raising the right + exception when passing in an int for the module name).""" + + def test_name_requires_rparition(self): + # Raise TypeError if a non-string is passed in for the module name. + with self.assertRaises(TypeError): + util.import_(42) + + +def test_main(): + from test.support import run_unittest + run_unittest(APITest) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/importlib/test/import_/test_fromlist.py b/Lib/importlib/test/import_/test_fromlist.py index ddd355e..b903e8e 100644 --- a/Lib/importlib/test/import_/test_fromlist.py +++ b/Lib/importlib/test/import_/test_fromlist.py @@ -84,16 +84,23 @@ class HandlingFromlist(unittest.TestCase): module = import_util.import_('pkg.mod', fromlist=['']) self.assertEqual(module.__name__, 'pkg.mod') - def test_using_star(self): + def basic_star_test(self, fromlist=['*']): # [using *] with util.mock_modules('pkg.__init__', 'pkg.module') as mock: with util.import_state(meta_path=[mock]): mock['pkg'].__all__ = ['module'] - module = import_util.import_('pkg', fromlist=['*']) + module = import_util.import_('pkg', fromlist=fromlist) self.assertEqual(module.__name__, 'pkg') self.assertTrue(hasattr(module, 'module')) self.assertEqual(module.module.__name__, 'pkg.module') + def test_using_star(self): + # [using *] + self.basic_star_test() + + def test_fromlist_as_tuple(self): + self.basic_star_test(('*',)) + def test_star_with_others(self): # [using * with others] context = util.mock_modules('pkg.__init__', 'pkg.module1', 'pkg.module2') diff --git a/Lib/importlib/test/import_/test_packages.py b/Lib/importlib/test/import_/test_packages.py index b41c36f..faadc32 100644 --- a/Lib/importlib/test/import_/test_packages.py +++ b/Lib/importlib/test/import_/test_packages.py @@ -18,8 +18,14 @@ class ParentModuleTests(unittest.TestCase): def test_bad_parent(self): with util.mock_modules('pkg.module') as mock: with util.import_state(meta_path=[mock]): - self.assertRaises(ImportError, - import_util.import_, 'pkg.module') + with self.assertRaises(ImportError): + import_util.import_('pkg.module') + + def test_module_not_package(self): + # Try to import a submodule from a non-package should raise ImportError. + assert not hasattr(sys, '__path__') + with self.assertRaises(ImportError): + import_util.import_('sys.no_submodules_here') def test_main(): diff --git a/Lib/importlib/test/import_/test_path.py b/Lib/importlib/test/import_/test_path.py index 0e055b1..2faa231 100644 --- a/Lib/importlib/test/import_/test_path.py +++ b/Lib/importlib/test/import_/test_path.py @@ -2,10 +2,10 @@ from importlib import _bootstrap from importlib import machinery from .. import util from . import util as import_util -from contextlib import nested import imp import os import sys +import tempfile from test import support from types import MethodType import unittest @@ -81,23 +81,28 @@ class DefaultPathFinderTests(unittest.TestCase): def test_implicit_hooks(self): # Test that the implicit path hooks are used. - existing_path = os.path.dirname(support.TESTFN) bad_path = '<path>' module = '<module>' assert not os.path.exists(bad_path) - with util.import_state(): - nothing = _bootstrap._DefaultPathFinder.find_module(module, - path=[existing_path]) - self.assertTrue(nothing is None) - self.assertTrue(existing_path in sys.path_importer_cache) - self.assertTrue(not isinstance(sys.path_importer_cache[existing_path], - imp.NullImporter)) - nothing = _bootstrap._DefaultPathFinder.find_module(module, - path=[bad_path]) - self.assertTrue(nothing is None) - self.assertTrue(bad_path in sys.path_importer_cache) - self.assertTrue(isinstance(sys.path_importer_cache[bad_path], - imp.NullImporter)) + existing_path = tempfile.mkdtemp() + try: + with util.import_state(): + nothing = _bootstrap._DefaultPathFinder.find_module(module, + path=[existing_path]) + self.assertTrue(nothing is None) + self.assertTrue(existing_path in sys.path_importer_cache) + result = isinstance(sys.path_importer_cache[existing_path], + imp.NullImporter) + self.assertFalse(result) + nothing = _bootstrap._DefaultPathFinder.find_module(module, + path=[bad_path]) + self.assertTrue(nothing is None) + self.assertTrue(bad_path in sys.path_importer_cache) + self.assertTrue(isinstance(sys.path_importer_cache[bad_path], + imp.NullImporter)) + finally: + os.rmdir(existing_path) + def test_path_importer_cache_has_None(self): # Test that the default hook is used when sys.path_importer_cache diff --git a/Lib/importlib/test/import_/test_relative_imports.py b/Lib/importlib/test/import_/test_relative_imports.py index 5547d4c..a0f6b2d 100644 --- a/Lib/importlib/test/import_/test_relative_imports.py +++ b/Lib/importlib/test/import_/test_relative_imports.py @@ -154,8 +154,9 @@ class RelativeImports(unittest.TestCase): {'__name__': 'pkg', '__path__': ['blah']}) def callback(global_): import_util.import_('pkg') - self.assertRaises(ValueError, import_util.import_, '', global_, - fromlist=['top_level'], level=2) + with self.assertRaises(ValueError): + import_util.import_('', global_, fromlist=['top_level'], + level=2) self.relative_import_test(create, globals_, callback) def test_too_high_from_module(self): @@ -164,13 +165,15 @@ class RelativeImports(unittest.TestCase): globals_ = {'__package__': 'pkg'}, {'__name__': 'pkg.module'} def callback(global_): import_util.import_('pkg') - self.assertRaises(ValueError, import_util.import_, '', global_, - fromlist=['top_level'], level=2) + with self.assertRaises(ValueError): + import_util.import_('', global_, fromlist=['top_level'], + level=2) self.relative_import_test(create, globals_, callback) def test_empty_name_w_level_0(self): # [empty name] - self.assertRaises(ValueError, import_util.import_, '') + with self.assertRaises(ValueError): + import_util.import_('') def test_import_from_different_package(self): # Test importing from a different package than the caller. diff --git a/Lib/importlib/test/import_/util.py b/Lib/importlib/test/import_/util.py index 5a1b727..649c5ed 100644 --- a/Lib/importlib/test/import_/util.py +++ b/Lib/importlib/test/import_/util.py @@ -1,5 +1,7 @@ import functools +import importlib import importlib._bootstrap +import unittest using___import__ = False @@ -10,19 +12,12 @@ def import_(*args, **kwargs): if using___import__: return __import__(*args, **kwargs) else: - return importlib._bootstrap.__import__(*args, **kwargs) + return importlib.__import__(*args, **kwargs) def importlib_only(fxn): - """Decorator to mark which tests are not supported by the current - implementation of __import__().""" - def inner(*args, **kwargs): - if using___import__: - return - else: - return fxn(*args, **kwargs) - functools.update_wrapper(inner, fxn) - return inner + """Decorator to skip a test if using __builtins__.__import__.""" + return unittest.skipIf(using___import__, "importlib-specific test")(fxn) def mock_path_hook(*entries, importer): diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py new file mode 100644 index 0000000..b103ae7d --- /dev/null +++ b/Lib/importlib/test/regrtest.py @@ -0,0 +1,35 @@ +"""Run Python's standard test suite using importlib.__import__. + +Tests known to fail because of assumptions that importlib (properly) +invalidates are automatically skipped if the entire test suite is run. +Otherwise all command-line options valid for test.regrtest are also valid for +this script. + +XXX FAILING + * test_import + - test_incorrect_code_name + file name differing between __file__ and co_filename (r68360 on trunk) + - test_import_by_filename + exception for trying to import by file name does not match + +""" +import importlib +import sys +from test import regrtest + +if __name__ == '__main__': + __builtins__.__import__ = importlib.__import__ + + exclude = ['--exclude', + 'test_frozen', # Does not expect __loader__ attribute + 'test_pkg', # Does not expect __loader__ attribute + 'test_pydoc', # Does not expect __loader__ attribute + ] + + # Switching on --exclude implies running all test but the ones listed, so + # only use it when one is not running an explicit test + if len(sys.argv) == 1: + # No programmatic way to specify tests to exclude + sys.argv.extend(exclude) + + regrtest.main(quiet=True, verbose2=True) diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py index 1ce83fb..3245907 100644 --- a/Lib/importlib/test/source/test_abc_loader.py +++ b/Lib/importlib/test/source/test_abc_loader.py @@ -1,14 +1,68 @@ import importlib from importlib import abc + from .. import abc as testing_abc from .. import util from . import util as source_util + import imp +import inspect +import io import marshal import os import sys import types import unittest +import warnings + + +class SourceOnlyLoaderMock(abc.SourceLoader): + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " + b"repr(__loader__)])") + + def __init__(self, path): + self.path = path + + def get_data(self, path): + assert self.path == path + return self.source + + def get_filename(self, fullname): + return self.path + + +class SourceLoaderMock(SourceOnlyLoaderMock): + + source_mtime = 1 + + def __init__(self, path, magic=imp.get_magic()): + super().__init__(path) + self.bytecode_path = imp.cache_from_source(self.path) + data = bytearray(magic) + data.extend(marshal._w_long(self.source_mtime)) + code_object = compile(self.source, self.path, 'exec', + dont_inherit=True) + data.extend(marshal.dumps(code_object)) + self.bytecode = bytes(data) + self.written = {} + + def get_data(self, path): + if path == self.path: + return super().get_data(path) + elif path == self.bytecode_path: + return self.bytecode + else: + raise IOError + + def path_mtime(self, path): + assert path == self.path + return self.source_mtime + + def set_data(self, path, data): + self.written[path] = bytes(data) + return path == self.bytecode_path class PyLoaderMock(abc.PyLoader): @@ -33,17 +87,42 @@ class PyLoaderMock(abc.PyLoader): return self.source def is_package(self, name): + filename = os.path.basename(self.get_filename(name)) + return os.path.splitext(filename)[0] == '__init__' + + def source_path(self, name): try: - return '__init__' in self.module_paths[name] + return self.module_paths[name] except KeyError: raise ImportError - def source_path(self, name): + def get_filename(self, name): + """Silence deprecation warning.""" + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + path = super().get_filename(name) + assert len(w) == 1 + assert issubclass(w[0].category, PendingDeprecationWarning) + return path + + +class PyLoaderCompatMock(PyLoaderMock): + + """Mock that matches what is suggested to have a loader that is compatible + from Python 3.1 onwards.""" + + def get_filename(self, fullname): try: - return self.module_paths[name] + return self.module_paths[fullname] except KeyError: raise ImportError + def source_path(self, fullname): + try: + return self.get_filename(fullname) + except ImportError: + return None + class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): @@ -114,6 +193,13 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): except TypeError: return '__init__' in self.bytecode_to_path[name] + def get_code(self, name): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + code_object = super().get_code(name) + assert len(w) == 1 + assert issubclass(w[0].category, PendingDeprecationWarning) + return code_object class PyLoaderTests(testing_abc.LoaderTests): @@ -183,7 +269,8 @@ class PyLoaderTests(testing_abc.LoaderTests): mock.source = b"1/0" with util.uncache(name): sys.modules[name] = module - self.assertRaises(ZeroDivisionError, mock.load_module, name) + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) self.assertTrue(sys.modules[name] is module) self.assertTrue(hasattr(module, 'blah')) return mock @@ -193,11 +280,20 @@ class PyLoaderTests(testing_abc.LoaderTests): mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) mock.source = b"1/0" with util.uncache(name): - self.assertRaises(ZeroDivisionError, mock.load_module, name) + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) self.assertTrue(name not in sys.modules) return mock +class PyLoaderCompatTests(PyLoaderTests): + + """Test that the suggested code to make a loader that is compatible from + Python 3.1 forward works.""" + + mocker = PyLoaderCompatMock + + class PyLoaderInterfaceTests(unittest.TestCase): """Tests for importlib.abc.PyLoader to make sure that when source_path() @@ -207,38 +303,29 @@ class PyLoaderInterfaceTests(unittest.TestCase): # No source path should lead to ImportError. name = 'mod' mock = PyLoaderMock({}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_source_path_is_None(self): name = 'mod' mock = PyLoaderMock({name: None}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) - - -class PyLoaderGetSourceTests(unittest.TestCase): - - """Tests for importlib.abc.PyLoader.get_source().""" + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) - def test_default_encoding(self): - # Should have no problems with UTF-8 text. + def test_get_filename_with_source_path(self): + # get_filename() should return what source_path() returns. name = 'mod' - mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')}) - source = 'x = "ü"' - mock.source = source.encode('utf-8') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) + path = os.path.join('path', 'to', 'source') + mock = PyLoaderMock({name: path}) + with util.uncache(name): + self.assertEqual(mock.get_filename(name), path) - def test_decoded_source(self): - # Decoding should work. + def test_get_filename_no_source_path(self): + # get_filename() should raise ImportError if source_path returns None. name = 'mod' - mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')}) - source = "# coding: Latin-1\nx='ü'" - assert source.encode('latin-1') != source.encode('utf-8') - mock.source = source.encode('latin-1') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) + mock = PyLoaderMock({name: None}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) class PyPycLoaderTests(PyLoaderTests): @@ -281,6 +368,38 @@ class PyPycLoaderTests(PyLoaderTests): super().test_unloadable() +class PyPycLoaderInterfaceTests(unittest.TestCase): + + """Test for the interface of importlib.abc.PyPycLoader.""" + + def get_filename_check(self, src_path, bc_path, expect): + name = 'mod' + mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}}) + with util.uncache(name): + assert mock.source_path(name) == src_path + assert mock.bytecode_path(name) == bc_path + self.assertEqual(mock.get_filename(name), expect) + + def test_filename_with_source_bc(self): + # When source and bytecode paths present, return the source path. + self.get_filename_check('source_path', 'bc_path', 'source_path') + + def test_filename_with_source_no_bc(self): + # With source but no bc, return source path. + self.get_filename_check('source_path', None, 'source_path') + + def test_filename_with_no_source_bc(self): + # With not source but bc, return the bc path. + self.get_filename_check(None, 'bc_path', 'bc_path') + + def test_filename_with_no_source_or_bc(self): + # With no source or bc, raise ImportError. + name = 'mod' + mock = PyPycLoaderMock({name: None}, {name: {'path': None}}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) + + class SkipWritingBytecodeTests(unittest.TestCase): """Test that bytecode is properly handled based on @@ -346,20 +465,28 @@ class BadBytecodeFailureTests(unittest.TestCase): # A bad magic number should lead to an ImportError. name = 'mod' bad_magic = b'\x00\x00\x00\x00' - mock = PyPycLoaderMock({name: None}, - {name: {'path': os.path.join('path', 'to', 'mod'), - 'magic': bad_magic}}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + bc = {name: + {'path': os.path.join('path', 'to', 'mod'), + 'magic': bad_magic}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_no_bytecode(self): + # Missing code object bytecode should lead to an EOFError. + name = 'mod' + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(EOFError): + mock.load_module(name) def test_bad_bytecode(self): - # Bad code object bytecode should lead to an ImportError. + # Malformed code object bytecode should lead to a ValueError. name = 'mod' - mock = PyPycLoaderMock({name: None}, - {name: {'path': os.path.join('path', 'to', 'mod'), - 'bc': b''}}) - with util.uncache(name): - self.assertRaises(EOFError, mock.load_module, name) + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ValueError): + mock.load_module(name) def raise_ImportError(*args, **kwargs): @@ -387,16 +514,16 @@ class MissingPathsTests(unittest.TestCase): # If all *_path methods return None, raise ImportError. name = 'mod' mock = PyPycLoaderMock({name: None}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_source_path_ImportError(self): # An ImportError from source_path should trigger an ImportError. name = 'mod' mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to', 'mod')}}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_bytecode_path_ImportError(self): # An ImportError from bytecode_path should trigger an ImportError. @@ -404,16 +531,345 @@ class MissingPathsTests(unittest.TestCase): mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) bad_meth = types.MethodType(raise_ImportError, mock) mock.bytecode_path = bad_meth - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + +class SourceLoaderTestHarness(unittest.TestCase): + + def setUp(self, *, is_package=True, **kwargs): + self.package = 'pkg' + if is_package: + self.path = os.path.join(self.package, '__init__.py') + self.name = self.package + else: + module_name = 'mod' + self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) + self.name = '.'.join([self.package, module_name]) + self.cached = imp.cache_from_source(self.path) + self.loader = self.loader_mock(self.path, **kwargs) + + def verify_module(self, module): + self.assertEqual(module.__name__, self.name) + self.assertEqual(module.__file__, self.path) + self.assertEqual(module.__cached__, self.cached) + self.assertEqual(module.__package__, self.package) + self.assertEqual(module.__loader__, self.loader) + values = module._.split('::') + self.assertEqual(values[0], self.name) + self.assertEqual(values[1], self.path) + self.assertEqual(values[2], self.cached) + self.assertEqual(values[3], self.package) + self.assertEqual(values[4], repr(self.loader)) + + def verify_code(self, code_object): + module = imp.new_module(self.name) + module.__file__ = self.path + module.__cached__ = self.cached + module.__package__ = self.package + module.__loader__ = self.loader + module.__path__ = [] + exec(code_object, module.__dict__) + self.verify_module(module) + + +class SourceOnlyLoaderTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader for source-only loading. + + Reload testing is subsumed by the tests for + importlib.util.module_for_loader. + + """ + + loader_mock = SourceOnlyLoaderMock + + def test_get_source(self): + # Verify the source code is returned as a string. + # If an IOError is raised by get_data then raise ImportError. + expected_source = self.loader.source.decode('utf-8') + self.assertEqual(self.loader.get_source(self.name), expected_source) + def raise_IOError(path): + raise IOError + self.loader.get_data = raise_IOError + with self.assertRaises(ImportError): + self.loader.get_source(self.name) + + def test_is_package(self): + # Properly detect when loading a package. + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.setUp(is_package=False) + self.assertFalse(self.loader.is_package(self.name)) + + def test_get_code(self): + # Verify the code object is created. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_load_module(self): + # Loading a module should set __name__, __loader__, __package__, + # __path__ (for packages), __file__, and __cached__. + # The module should also be put into sys.modules. + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertTrue(self.name in sys.modules) + + def test_package_settings(self): + # __package__ needs to be set, while __path__ is set on if the module + # is a package. + # Testing the values for a package are covered by test_load_module. + self.setUp(is_package=False) + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertTrue(not hasattr(module, '__path__')) + + def test_get_source_encoding(self): + # Source is considered encoded in UTF-8 by default unless otherwise + # specified by an encoding line. + source = "_ = 'ü'" + self.loader.source = source.encode('utf-8') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + source = "# coding: latin-1\n_ = ü" + self.loader.source = source.encode('latin-1') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + + +@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +class SourceLoaderBytecodeTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader's use of bytecode. + + Source-only testing handled by SourceOnlyLoaderTests. + + """ + + loader_mock = SourceLoaderMock + + def verify_code(self, code_object, *, bytecode_written=False): + super().verify_code(code_object) + if bytecode_written: + self.assertIn(self.cached, self.loader.written) + data = bytearray(imp.get_magic()) + data.extend(marshal._w_long(self.loader.source_mtime)) + data.extend(marshal.dumps(code_object)) + self.assertEqual(self.loader.written[self.cached], bytes(data)) + + def test_code_with_everything(self): + # When everything should work. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_no_bytecode(self): + # If no bytecode exists then move on to the source. + self.loader.bytecode_path = "<does not exist>" + # Sanity check + with self.assertRaises(IOError): + bytecode_path = imp.cache_from_source(self.path) + self.loader.get_data(bytecode_path) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_code_bad_timestamp(self): + # Bytecode is only used when the timestamp matches the source EXACTLY. + for source_mtime in (0, 2): + assert source_mtime != self.loader.source_mtime + original = self.loader.source_mtime + self.loader.source_mtime = source_mtime + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + self.loader.source_mtime = original + + def test_code_bad_magic(self): + # Skip over bytecode with a bad magic number. + self.setUp(magic=b'0000') + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_dont_write_bytecode(self): + # Bytecode is not written if sys.dont_write_bytecode is true. + # Can assume it is false already thanks to the skipIf class decorator. + try: + sys.dont_write_bytecode = True + self.loader.bytecode_path = "<does not exist>" + code_object = self.loader.get_code(self.name) + self.assertNotIn(self.cached, self.loader.written) + finally: + sys.dont_write_bytecode = False + + def test_no_set_data(self): + # If set_data is not defined, one can still read bytecode. + self.setUp(magic=b'0000') + original_set_data = self.loader.__class__.set_data + try: + del self.loader.__class__.set_data + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + finally: + self.loader.__class__.set_data = original_set_data + + def test_set_data_raises_exceptions(self): + # Raising NotImplementedError or IOError is okay for set_data. + def raise_exception(exc): + def closure(*args, **kwargs): + raise exc + return closure + + self.setUp(magic=b'0000') + self.loader.set_data = raise_exception(NotImplementedError) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + +class SourceLoaderGetSourceTests(unittest.TestCase): + + """Tests for importlib.abc.SourceLoader.get_source().""" + + def test_default_encoding(self): + # Should have no problems with UTF-8 text. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = 'x = "ü"' + mock.source = source.encode('utf-8') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_decoded_source(self): + # Decoding should work. + name = 'mod' + mock = SourceOnlyLoaderMock("mod.file") + source = "# coding: Latin-1\nx='ü'" + assert source.encode('latin-1') != source.encode('utf-8') + mock.source = source.encode('latin-1') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_universal_newlines(self): + # PEP 302 says universal newlines should be used. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = "x = 42\r\ny = -13\r\n" + mock.source = source.encode('utf-8') + expect = io.IncrementalNewlineDecoder(None, True).decode(source) + self.assertEqual(mock.get_source(name), expect) + +class AbstractMethodImplTests(unittest.TestCase): + + """Test the concrete abstractmethod implementations.""" + + class Loader(abc.Loader): + def load_module(self, fullname): + super().load_module(fullname) + + class Finder(abc.Finder): + def find_module(self, _): + super().find_module(_) + + class ResourceLoader(Loader, abc.ResourceLoader): + def get_data(self, _): + super().get_data(_) + + class InspectLoader(Loader, abc.InspectLoader): + def is_package(self, _): + super().is_package(_) + + def get_code(self, _): + super().get_code(_) + + def get_source(self, _): + super().get_source(_) + + class ExecutionLoader(InspectLoader, abc.ExecutionLoader): + def get_filename(self, _): + super().get_filename(_) + + class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader): + pass + + class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader): + def source_path(self, _): + super().source_path(_) + + class PyPycLoader(PyLoader, abc.PyPycLoader): + def bytecode_path(self, _): + super().bytecode_path(_) + + def source_mtime(self, _): + super().source_mtime(_) + + def write_bytecode(self, _, _2): + super().write_bytecode(_, _2) + + def raises_NotImplementedError(self, ins, *args): + for method_name in args: + method = getattr(ins, method_name) + arg_count = len(inspect.getfullargspec(method)[0]) - 1 + args = [''] * arg_count + try: + method(*args) + except NotImplementedError: + pass + else: + msg = "{}.{} did not raise NotImplementedError" + self.fail(msg.format(ins.__class__.__name__, method_name)) + + def test_Loader(self): + self.raises_NotImplementedError(self.Loader(), 'load_module') + + # XXX misplaced; should be somewhere else + def test_Finder(self): + self.raises_NotImplementedError(self.Finder(), 'find_module') + + def test_ResourceLoader(self): + self.raises_NotImplementedError(self.ResourceLoader(), 'load_module', + 'get_data') + + def test_InspectLoader(self): + self.raises_NotImplementedError(self.InspectLoader(), 'load_module', + 'is_package', 'get_code', 'get_source') + + def test_ExecutionLoader(self): + self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module', + 'is_package', 'get_code', 'get_source', + 'get_filename') + + def test_SourceLoader(self): + ins = self.SourceLoader() + # Required abstractmethods. + self.raises_NotImplementedError(ins, 'get_filename', 'get_data') + # Optional abstractmethods. + self.raises_NotImplementedError(ins,'path_mtime', 'set_data') + + def test_PyLoader(self): + self.raises_NotImplementedError(self.PyLoader(), 'source_path', + 'get_data', 'is_package') + + def test_PyPycLoader(self): + self.raises_NotImplementedError(self.PyPycLoader(), 'source_path', + 'source_mtime', 'bytecode_path', + 'write_bytecode') def test_main(): from test.support import run_unittest - run_unittest(PyLoaderTests, PyLoaderInterfaceTests, PyLoaderGetSourceTests, - PyPycLoaderTests, SkipWritingBytecodeTests, - RegeneratedBytecodeTests, BadBytecodeFailureTests, - MissingPathsTests) + run_unittest(PyLoaderTests, PyLoaderCompatTests, + PyLoaderInterfaceTests, + PyPycLoaderTests, PyPycLoaderInterfaceTests, + SkipWritingBytecodeTests, RegeneratedBytecodeTests, + BadBytecodeFailureTests, MissingPathsTests, + SourceOnlyLoaderTests, + SourceLoaderBytecodeTests, + SourceLoaderGetSourceTests, + AbstractMethodImplTests) if __name__ == '__main__': diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py index 6fad881..73777de 100644 --- a/Lib/importlib/test/source/test_case_sensitivity.py +++ b/Lib/importlib/test/source/test_case_sensitivity.py @@ -19,7 +19,9 @@ class CaseSensitivityTest(unittest.TestCase): assert name != name.lower() def find(self, path): - finder = _bootstrap._PyPycFileFinder(path) + finder = _bootstrap._FileFinder(path, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) return finder.find_module(self.name) def sensitivity_test(self): @@ -27,7 +29,7 @@ class CaseSensitivityTest(unittest.TestCase): sensitive_pkg = 'sensitive.{0}'.format(self.name) insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) context = source_util.create_modules(insensitive_pkg, sensitive_pkg) - with context as mapping: + with context as mapping: sensitive_path = os.path.join(mapping['.root'], 'sensitive') insensitive_path = os.path.join(mapping['.root'], 'insensitive') return self.find(sensitive_path), self.find(insensitive_path) @@ -37,7 +39,7 @@ class CaseSensitivityTest(unittest.TestCase): env.unset('PYTHONCASEOK') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive._base_path) + self.assertIn(self.name, sensitive.get_filename(self.name)) self.assertIsNone(insensitive) def test_insensitive(self): @@ -45,9 +47,9 @@ class CaseSensitivityTest(unittest.TestCase): env.set('PYTHONCASEOK', '1') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive._base_path) + self.assertIn(self.name, sensitive.get_filename(self.name)) self.assertTrue(hasattr(insensitive, 'load_module')) - self.assertIn(self.name, insensitive._base_path) + self.assertIn(self.name, insensitive.get_filename(self.name)) def test_main(): diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index 145d076..0ffe78d 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -1,15 +1,20 @@ import importlib from importlib import _bootstrap from .. import abc +from .. import util from . import util as source_util import imp +import marshal import os import py_compile +import shutil import stat import sys import unittest +from test.support import make_legacy_pyc + class SimpleTest(unittest.TestCase): @@ -21,8 +26,7 @@ class SimpleTest(unittest.TestCase): # [basic] def test_module(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') self.assertTrue('_temp' in sys.modules) check = {'__name__': '_temp', '__file__': mapping['_temp'], @@ -32,9 +36,8 @@ class SimpleTest(unittest.TestCase): def test_package(self): with source_util.create_modules('_pkg.__init__') as mapping: - loader = _bootstrap._PyPycFileLoader('_pkg', - mapping['_pkg.__init__'], - True) + loader = _bootstrap._SourceFileLoader('_pkg', + mapping['_pkg.__init__']) module = loader.load_module('_pkg') self.assertTrue('_pkg' in sys.modules) check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], @@ -46,8 +49,8 @@ class SimpleTest(unittest.TestCase): def test_lacking_parent(self): with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: - loader = _bootstrap._PyPycFileLoader('_pkg.mod', - mapping['_pkg.mod'], False) + loader = _bootstrap._SourceFileLoader('_pkg.mod', + mapping['_pkg.mod']) module = loader.load_module('_pkg.mod') self.assertTrue('_pkg.mod' in sys.modules) check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], @@ -61,8 +64,7 @@ class SimpleTest(unittest.TestCase): def test_module_reuse(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') module_id = id(module) module_dict_id = id(module.__dict__) @@ -72,7 +74,7 @@ class SimpleTest(unittest.TestCase): # everything that has happened above can be too fast; # force an mtime on the source that is guaranteed to be different # than the original mtime. - loader.source_mtime = self.fake_mtime(loader.source_mtime) + loader.path_mtime = self.fake_mtime(loader.path_mtime) module = loader.load_module('_temp') self.assertTrue('testing_var' in module.__dict__, "'testing_var' not in " @@ -92,9 +94,9 @@ class SimpleTest(unittest.TestCase): setattr(orig_module, attr, value) with open(mapping[name], 'w') as file: file.write('+++ bad syntax +++') - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) - self.assertRaises(SyntaxError, loader.load_module, name) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module(name) for attr in attributes: self.assertEqual(getattr(orig_module, attr), value) @@ -103,16 +105,34 @@ class SimpleTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: with open(mapping['_temp'], 'w') as file: file.write('=') - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) - self.assertRaises(SyntaxError, loader.load_module, '_temp') + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module('_temp') self.assertTrue('_temp' not in sys.modules) + def test_file_from_empty_string_dir(self): + # Loading a module found from an empty string entry on sys.path should + # not only work, but keep all attributes relative. + file_path = '_temp.py' + with open(file_path, 'w') as file: + file.write("# test file for importlib") + try: + with util.uncache('_temp'): + loader = _bootstrap._SourceFileLoader('_temp', file_path) + mod = loader.load_module('_temp') + self.assertEqual(file_path, mod.__file__) + self.assertEqual(imp.cache_from_source(file_path), + mod.__cached__) + finally: + os.unlink(file_path) + pycache = os.path.dirname(imp.cache_from_source(file_path)) + shutil.rmtree(pycache) + class BadBytecodeTest(unittest.TestCase): def import_(self, file, module_name): - loader = _bootstrap._PyPycFileLoader(module_name, file, False) + loader = self.loader(module_name, file) module = loader.load_module(module_name) self.assertTrue(module_name in sys.modules) @@ -125,106 +145,162 @@ class BadBytecodeTest(unittest.TestCase): except KeyError: pass py_compile.compile(mapping[name]) - bytecode_path = source_util.bytecode_path(mapping[name]) - with open(bytecode_path, 'rb') as file: - bc = file.read() - new_bc = manipulator(bc) - with open(bytecode_path, 'wb') as file: - if new_bc: - file.write(new_bc) - if del_source: + if not del_source: + bytecode_path = imp.cache_from_source(mapping[name]) + else: os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) + if manipulator: + with open(bytecode_path, 'rb') as file: + bc = file.read() + new_bc = manipulator(bc) + with open(bytecode_path, 'wb') as file: + if new_bc is not None: + file.write(new_bc) return bytecode_path - @source_util.writes_bytecode_files - def test_empty_file(self): - # When a .pyc is empty, regenerate it if possible, else raise - # ImportError. + def _test_empty_file(self, test, *, del_source=False): with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: None) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: None, - del_source=True) - with self.assertRaises(ImportError): - self.import_(mapping['_temp'], '_temp') + lambda bc: b'', + del_source=del_source) + test('_temp', mapping, bc_path) @source_util.writes_bytecode_files - def test_partial_magic(self): + def _test_partial_magic(self, test, *, del_source=False): # When their are less than 4 bytes to a .pyc, regenerate it if # possible, else raise ImportError. with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:3]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3], - del_source=True) + lambda bc: bc[:3], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_magic_only(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:4], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_timestamp(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:7], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_no_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8], + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bc_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_non_code_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8] + marshal.dumps(b'abcd'), + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(ImportError): - self.import_(mapping['_temp'], '_temp') + self.import_(file_path, '_temp') + + def _test_bad_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8] + b'<test>', + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(ValueError): + self.import_(file_path, '_temp') + + def _test_bad_magic(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'\x00\x00\x00\x00' + bc[4:]) + test('_temp', mapping, bc_path) + + +class SourceLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourceFileLoader + + @source_util.writes_bytecode_files + def test_empty_file(self): + # When a .pyc is empty, regenerate it if possible, else raise + # ImportError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 8) + + self._test_empty_file(test) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 8) + + self._test_partial_magic(test) @source_util.writes_bytecode_files def test_magic_only(self): # When there is only the magic number, regenerate the .pyc if possible, # else raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:4]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:4], - del_source=True) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') + + @source_util.writes_bytecode_files + def test_bad_magic(self): + # When the magic number is different, the bytecode should be + # regenerated. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as bytecode_file: + self.assertEqual(bytecode_file.read(4), imp.get_magic()) + + self._test_bad_magic(test) @source_util.writes_bytecode_files def test_partial_timestamp(self): # When the timestamp is partial, regenerate the .pyc, else # raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:7]) - self.import_(mapping['_temp'], '_temp') + def test(name, mapping, bc_path): + self.import_(mapping[name], name) with open(bc_path, 'rb') as file: self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:7], - del_source=True) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') @source_util.writes_bytecode_files def test_no_marshal(self): # When there is only the magic number and timestamp, raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8]) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') + self._test_no_marshal() @source_util.writes_bytecode_files - def test_bad_magic(self): - # When the magic number is different, the bytecode should be - # regenerated. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: b'\x00\x00\x00\x00' + bc[4:]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as bytecode_file: - self.assertEqual(bytecode_file.read(4), imp.get_magic()) + def test_non_code_marshal(self): + self._test_non_code_marshal() + # XXX ImportError when sourceless + + # [bad marshal] + @source_util.writes_bytecode_files + def test_bad_marshal(self): + # Bad marshal data should raise a ValueError. + self._test_bad_marshal() # [bad timestamp] @source_util.writes_bytecode_files - def test_bad_bytecode(self): + def test_old_timestamp(self): # When the timestamp is older than the source, bytecode should be # regenerated. zeros = b'\x00\x00\x00\x00' with source_util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(4) bytecode_file.write(zeros) @@ -235,22 +311,6 @@ class BadBytecodeTest(unittest.TestCase): bytecode_file.seek(4) self.assertEqual(bytecode_file.read(4), source_timestamp) - # [bad marshal] - @source_util.writes_bytecode_files - def test_bad_marshal(self): - # Bad marshal data should raise a ValueError. - with source_util.create_modules('_temp') as mapping: - bytecode_path = source_util.bytecode_path(mapping['_temp']) - source_mtime = os.path.getmtime(mapping['_temp']) - source_timestamp = importlib._w_long(source_mtime) - with open(bytecode_path, 'wb') as bytecode_file: - bytecode_file.write(imp.get_magic()) - bytecode_file.write(source_timestamp) - bytecode_file.write(b'AAAA') - self.assertRaises(ValueError, self.import_, mapping['_temp'], - '_temp') - self.assertTrue('_temp' not in sys.modules) - # [bytecode read-only] @source_util.writes_bytecode_files def test_read_only_bytecode(self): @@ -258,7 +318,7 @@ class BadBytecodeTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') @@ -273,9 +333,57 @@ class BadBytecodeTest(unittest.TestCase): os.chmod(bytecode_path, stat.S_IWUSR) +class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourcelessFileLoader + + def test_empty_file(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_empty_file(test, del_source=True) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + self._test_partial_magic(test, del_source=True) + + def test_magic_only(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_magic_only(test, del_source=True) + + def test_bad_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_bad_magic(test, del_source=True) + + def test_partial_timestamp(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_timestamp(test, del_source=True) + + def test_no_marshal(self): + self._test_no_marshal(del_source=True) + + def test_non_code_marshal(self): + self._test_non_code_marshal(del_source=True) + + def test_main(): from test.support import run_unittest - run_unittest(SimpleTest, BadBytecodeTest) + run_unittest(SimpleTest, + SourceLoaderBadBytecodeTest, + SourcelessLoaderBadBytecodeTest + ) if __name__ == '__main__': diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index c495c5a..7b9088d 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,9 @@ from importlib import _bootstrap from .. import abc from . import util as source_util +from test.support import make_legacy_pyc import os +import errno import py_compile import unittest import warnings @@ -32,7 +34,9 @@ class FinderTests(abc.FinderTests): """ def import_(self, root, module): - finder = _bootstrap._PyPycFileFinder(root) + finder = _bootstrap._FileFinder(root, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) return finder.find_module(module) def run_test(self, test, create=None, *, compile_=None, unlink=None): @@ -52,6 +56,14 @@ class FinderTests(abc.FinderTests): if unlink: for name in unlink: os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise loader = self.import_(mapping['.root'], test) self.assertTrue(hasattr(loader, 'load_module')) return loader @@ -60,7 +72,8 @@ class FinderTests(abc.FinderTests): # [top-level source] self.run_test('top_level') # [top-level bc] - self.run_test('top_level', compile_={'top_level'}, unlink={'top_level'}) + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) # [top-level both] self.run_test('top_level', compile_={'top_level'}) @@ -97,15 +110,14 @@ class FinderTests(abc.FinderTests): with context as mapping: os.unlink(mapping['pkg.sub.__init__']) pkg_dir = os.path.dirname(mapping['pkg.__init__']) - self.assertRaises(ImportWarning, self.import_, pkg_dir, - 'pkg.sub') + with self.assertRaises(ImportWarning): + self.import_(pkg_dir, 'pkg.sub') # [package over modules] def test_package_over_module(self): - # XXX This is not a blackbox test! name = '_temp' loader = self.run_test(name, {'{0}.__init__'.format(name), name}) - self.assertTrue('__init__' in loader._base_path) + self.assertTrue('__init__' in loader.get_filename(name)) def test_failure(self): @@ -117,8 +129,19 @@ class FinderTests(abc.FinderTests): def test_empty_dir(self): with warnings.catch_warnings(): warnings.simplefilter("error", ImportWarning) - self.assertRaises(ImportWarning, self.run_test, 'pkg', - {'pkg.__init__'}, unlink={'pkg.__init__'}) + with self.assertRaises(ImportWarning): + self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'}) + + def test_empty_string_for_dir(self): + # The empty string from sys.path means to search in the cwd. + finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails()) + with open('mod.py', 'w') as file: + file.write("# test file for importlib") + try: + loader = finder.find_module('mod') + self.assertTrue(hasattr(loader, 'load_module')) + finally: + os.unlink('mod.py') def test_main(): diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py index 3efb3be..374f7b6 100644 --- a/Lib/importlib/test/source/test_path_hook.py +++ b/Lib/importlib/test/source/test_path_hook.py @@ -8,11 +8,14 @@ class PathHookTest(unittest.TestCase): """Test the path hook for source.""" def test_success(self): - # XXX Only work on existing directories? with source_util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(_bootstrap._FileFinder(mapping['.root']), + self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']), 'find_module')) + def test_empty_string(self): + # The empty string represents the cwd. + self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module')) + def test_main(): from test.support import run_unittest diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/importlib/test/source/test_source_encoding.py index 3734712..794a3df 100644 --- a/Lib/importlib/test/source/test_source_encoding.py +++ b/Lib/importlib/test/source/test_source_encoding.py @@ -33,10 +33,10 @@ class EncodingTest(unittest.TestCase): def run_test(self, source): with source_util.create_modules(self.module_name) as mapping: - with open(mapping[self.module_name], 'wb')as file: + with open(mapping[self.module_name], 'wb') as file: file.write(source) - loader = _bootstrap._PyPycFileLoader(self.module_name, - mapping[self.module_name], False) + loader = _bootstrap._SourceFileLoader(self.module_name, + mapping[self.module_name]) return loader.load_module(self.module_name) def create_source(self, encoding): @@ -81,7 +81,8 @@ class EncodingTest(unittest.TestCase): # [BOM conflict] def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') - self.assertRaises(SyntaxError, self.run_test, source) + with self.assertRaises(SyntaxError): + self.run_test(source) class LineEndingTest(unittest.TestCase): @@ -96,8 +97,8 @@ class LineEndingTest(unittest.TestCase): with source_util.create_modules(module_name) as mapping: with open(mapping[module_name], 'wb') as file: file.write(source) - loader = _bootstrap._PyPycFileLoader(module_name, - mapping[module_name], False) + loader = _bootstrap._SourceFileLoader(module_name, + mapping[module_name]) return loader.load_module(module_name) # [cr] diff --git a/Lib/importlib/test/source/util.py b/Lib/importlib/test/source/util.py index 2b945c5..ae65663 100644 --- a/Lib/importlib/test/source/util.py +++ b/Lib/importlib/test/source/util.py @@ -1,5 +1,6 @@ from .. import util import contextlib +import errno import functools import imp import os @@ -26,14 +27,16 @@ def writes_bytecode_files(fxn): return wrapper -def bytecode_path(source_path): - for suffix, _, type_ in imp.get_suffixes(): - if type_ == imp.PY_COMPILED: - bc_suffix = suffix - break - else: - raise ValueError("no bytecode suffix is defined") - return os.path.splitext(source_path)[0] + bc_suffix +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise @contextlib.contextmanager diff --git a/Lib/importlib/test/test_abc.py b/Lib/importlib/test/test_abc.py index 6e09534..0ecbe39 100644 --- a/Lib/importlib/test/test_abc.py +++ b/Lib/importlib/test/test_abc.py @@ -53,9 +53,20 @@ class InspectLoader(InheritanceTests, unittest.TestCase): machinery.FrozenImporter] +class ExecutionLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.InspectLoader] + subclasses = [abc.PyLoader] + + +class SourceLoader(InheritanceTests, unittest.TestCase): + + superclasses = [abc.ResourceLoader, abc.ExecutionLoader] + + class PyLoader(InheritanceTests, unittest.TestCase): - superclasses = [abc.Loader, abc.ResourceLoader, abc.InspectLoader] + superclasses = [abc.Loader, abc.ResourceLoader, abc.ExecutionLoader] class PyPycLoader(InheritanceTests, unittest.TestCase): diff --git a/Lib/importlib/test/test_api.py b/Lib/importlib/test/test_api.py index 65f8d04..0ffa3c4 100644 --- a/Lib/importlib/test/test_api.py +++ b/Lib/importlib/test/test_api.py @@ -27,7 +27,7 @@ class ImportModuleTests(unittest.TestCase): self.assertEqual(module.__name__, name) def test_shallow_relative_package_import(self): - # Test importing a module from a package through a relatve import. + # Test importing a module from a package through a relative import. pkg_name = 'pkg' pkg_long_name = '{0}.__init__'.format(pkg_name) module_name = 'mod' @@ -63,7 +63,8 @@ class ImportModuleTests(unittest.TestCase): def test_relative_import_wo_package(self): # Relative imports cannot happen without the 'package' argument being # set. - self.assertRaises(TypeError, importlib.import_module, '.support') + with self.assertRaises(TypeError): + importlib.import_module('.support') def test_main(): diff --git a/Lib/importlib/test/test_util.py b/Lib/importlib/test/test_util.py index 406477d..602447f 100644 --- a/Lib/importlib/test/test_util.py +++ b/Lib/importlib/test/test_util.py @@ -40,7 +40,7 @@ class ModuleForLoaderTests(unittest.TestCase): with test_util.uncache(name): sys.modules[name] = module returned_module = self.return_module(name) - self.assertTrue(sys.modules[name] is returned_module) + self.assertIs(returned_module, sys.modules[name]) def test_new_module_failure(self): # Test that a module is removed from sys.modules if added but an @@ -57,7 +57,7 @@ class ModuleForLoaderTests(unittest.TestCase): with test_util.uncache(name): sys.modules[name] = module self.raise_exception(name) - self.assertTrue(sys.modules[name] is module) + self.assertIs(module, sys.modules[name]) class SetPackageTests(unittest.TestCase): diff --git a/Lib/importlib/test/util.py b/Lib/importlib/test/util.py index 845e380..0c0c84c 100644 --- a/Lib/importlib/test/util.py +++ b/Lib/importlib/test/util.py @@ -6,21 +6,22 @@ import unittest import sys -def case_insensitive_tests(class_): +CASE_INSENSITIVE_FS = True +# Windows is the only OS that is *always* case-insensitive +# (OS X *can* be case-sensitive). +if sys.platform not in ('win32', 'cygwin'): + changed_name = __file__.upper() + if changed_name == __file__: + changed_name = __file__.lower() + if not os.path.exists(changed_name): + CASE_INSENSITIVE_FS = False + + +def case_insensitive_tests(test): """Class decorator that nullifies tests requiring a case-insensitive file system.""" - # Windows is the only OS that is *always* case-insensitive - # (OS X *can* be case-sensitive). - if sys.platform not in ('win32', 'cygwin'): - changed_name = __file__.upper() - if changed_name == __file__: - changed_name = __file__.lower() - if os.path.exists(changed_name): - return class_ - else: - return unittest.TestCase - else: - return class_ + return unittest.skipIf(not CASE_INSENSITIVE_FS, + "requires a case-insensitive filesystem")(test) @contextmanager diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py index 3abc6a9..7b44fa1 100644 --- a/Lib/importlib/util.py +++ b/Lib/importlib/util.py @@ -1,4 +1,5 @@ """Utility code for constructing importers, etc.""" + from ._bootstrap import module_for_loader from ._bootstrap import set_loader from ._bootstrap import set_package |