diff options
Diffstat (limited to 'Lib/importlib/_bootstrap.py')
| -rw-r--r-- | Lib/importlib/_bootstrap.py | 814 |
1 files changed, 419 insertions, 395 deletions
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index f4fdc82..fad91ca 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -9,7 +9,7 @@ work. One should use importlib as the public-facing version of this module. # # IMPORTANT: Whenever making changes to this module, be sure to run # a top-level make in order to get the frozen version of the module -# update. Not doing so, will result in the Makefile to fail for +# update. Not doing so will result in the Makefile to fail for # all others who don't have a ./python around to freeze the module # in the early stages of compilation. # @@ -20,10 +20,6 @@ work. One should use importlib as the public-facing version of this module. # reference any injected objects! This includes not only global code but also # anything specified at the class level. -# XXX Make sure all public names have no single leading underscore and all -# others do. - - # Bootstrap-related code ###################################################### _CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin' @@ -44,58 +40,32 @@ def _make_relax_case(): return _relax_case -# TODO: Expose from marshal def _w_long(x): - """Convert a 32-bit integer to little-endian. - - XXX Temporary until marshal's long functions are exposed. - - """ - x = int(x) - int_bytes = [] - int_bytes.append(x & 0xFF) - int_bytes.append((x >> 8) & 0xFF) - int_bytes.append((x >> 16) & 0xFF) - int_bytes.append((x >> 24) & 0xFF) - return bytearray(int_bytes) + """Convert a 32-bit integer to little-endian.""" + return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little') -# TODO: Expose from marshal def _r_long(int_bytes): - """Convert 4 bytes in little-endian to an integer. - - XXX Temporary until marshal's long function are exposed. - - """ - x = int_bytes[0] - x |= int_bytes[1] << 8 - x |= int_bytes[2] << 16 - x |= int_bytes[3] << 24 - return x + """Convert 4 bytes in little-endian to an integer.""" + return int.from_bytes(int_bytes, 'little') def _path_join(*path_parts): """Replacement for os.path.join().""" - new_parts = [] - for part in path_parts: - if not part: - continue - new_parts.append(part) - if part[-1] not in path_separators: - new_parts.append(path_sep) - return ''.join(new_parts[:-1]) # Drop superfluous path separator. + return path_sep.join([part.rstrip(path_separators) + for part in path_parts if part]) def _path_split(path): """Replacement for os.path.split().""" + if len(path_separators) == 1: + front, _, tail = path.rpartition(path_sep) + return front, tail for x in reversed(path): if x in path_separators: - sep = x - break - else: - sep = path_sep - front, _, tail = path.rpartition(sep) - return front, tail + front, tail = path.rsplit(x, maxsplit=1) + return front, tail + return '', path def _path_is_mode_type(path, mode): @@ -154,15 +124,6 @@ def _wrap(new, old): _code_type = type(_wrap.__code__) -def new_module(name): - """Create a new module. - - The module is not entered into sys.modules. - - """ - return type(_io)(name) - - # Module-level locking ######################################################## # A dict mapping module names to weakrefs of _ModuleLock instances @@ -217,7 +178,7 @@ class _ModuleLock: self.count += 1 return True if self.has_deadlock(): - raise _DeadlockError("deadlock detected by %r" % self) + raise _DeadlockError('deadlock detected by %r' % self) if self.wakeup.acquire(False): self.waiters += 1 # Wait for a release() call @@ -230,7 +191,7 @@ class _ModuleLock: tid = _thread.get_ident() with self.lock: if self.owner != tid: - raise RuntimeError("cannot release un-acquired lock") + raise RuntimeError('cannot release un-acquired lock') assert self.count > 0 self.count -= 1 if self.count == 0: @@ -240,7 +201,7 @@ class _ModuleLock: self.wakeup.release() def __repr__(self): - return "_ModuleLock(%r) at %d" % (self.name, id(self)) + return '_ModuleLock({!r}) at {}'.format(self.name, id(self)) class _DummyModuleLock: @@ -257,11 +218,11 @@ class _DummyModuleLock: def release(self): if self.count == 0: - raise RuntimeError("cannot release un-acquired lock") + raise RuntimeError('cannot release un-acquired lock') self.count -= 1 def __repr__(self): - return "_DummyModuleLock(%r) at %d" % (self.name, id(self)) + return '_DummyModuleLock({!r}) at {}'.format(self.name, id(self)) # The following two functions are for consumption by Python/import.c. @@ -318,95 +279,106 @@ def _call_with_frames_removed(f, *args, **kwds): # Finder/loader utility code ############################################### -"""Magic word to reject .pyc files generated by other Python versions. -It should change for each incompatible change to the bytecode. - -The value of CR and LF is incorporated so if you ever read or write -a .pyc file in text mode the magic number will be wrong; also, the -Apple MPW compiler swaps their values, botching string constants. - -The magic numbers must be spaced apart at least 2 values, as the --U interpeter flag will cause MAGIC+1 being used. They have been -odd numbers for some time now. - -There were a variety of old schemes for setting the magic number. -The current working scheme is to increment the previous value by -10. - -Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic -number also includes a new "magic tag", i.e. a human readable string used -to represent the magic number in __pycache__ directories. When you change -the magic number, you must also set a new unique magic tag. Generally this -can be named after the Python major version of the magic number bump, but -it can really be anything, as long as it's different than anything else -that's come before. The tags are included in the following table, starting -with Python 3.2a0. - -Known values: - Python 1.5: 20121 - Python 1.5.1: 20121 - Python 1.5.2: 20121 - Python 1.6: 50428 - Python 2.0: 50823 - Python 2.0.1: 50823 - Python 2.1: 60202 - Python 2.1.1: 60202 - Python 2.1.2: 60202 - Python 2.2: 60717 - Python 2.3a0: 62011 - Python 2.3a0: 62021 - Python 2.3a0: 62011 (!) - Python 2.4a0: 62041 - Python 2.4a3: 62051 - Python 2.4b1: 62061 - Python 2.5a0: 62071 - Python 2.5a0: 62081 (ast-branch) - Python 2.5a0: 62091 (with) - Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) - Python 2.5b3: 62101 (fix wrong code: for x, in ...) - Python 2.5b3: 62111 (fix wrong code: x += yield) - Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) - Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) - Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) - Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 3000: 3000 - 3010 (removed UNARY_CONVERT) - 3020 (added BUILD_SET) - 3030 (added keyword-only parameters) - 3040 (added signature annotations) - 3050 (print becomes a function) - 3060 (PEP 3115 metaclass syntax) - 3061 (string literals become unicode) - 3071 (PEP 3109 raise changes) - 3081 (PEP 3137 make __file__ and __name__ unicode) - 3091 (kill str8 interning) - 3101 (merge from 2.6a0, see 62151) - 3103 (__file__ points to source file) - Python 3.0a4: 3111 (WITH_CLEANUP optimization). - Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) - Python 3.1a0: 3141 (optimize list, set and dict comprehensions: - change LIST_APPEND and SET_ADD, add MAP_ADD) - Python 3.1a0: 3151 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 3.2a0: 3160 (add SETUP_WITH) - tag: cpython-32 - Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) - tag: cpython-32 - Python 3.2a2 3180 (add DELETE_DEREF) - Python 3.3a0 3190 __class__ super closure changed - Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) - Python 3.3a1 3220 (changed PEP 380 implementation) - Python 3.3a4 3230 (revert changes to implicit __class__ closure) - -MAGIC must change whenever the bytecode emitted by the compiler may no -longer be understood by older implementations of the eval loop (usually -due to the addition of new opcodes). +# Magic word to reject .pyc files generated by other Python versions. +# It should change for each incompatible change to the bytecode. +# +# The value of CR and LF is incorporated so if you ever read or write +# a .pyc file in text mode the magic number will be wrong; also, the +# Apple MPW compiler swaps their values, botching string constants. +# +# The magic numbers must be spaced apart at least 2 values, as the +# -U interpeter flag will cause MAGIC+1 being used. They have been +# odd numbers for some time now. +# +# There were a variety of old schemes for setting the magic number. +# The current working scheme is to increment the previous value by +# 10. +# +# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic +# number also includes a new "magic tag", i.e. a human readable string used +# to represent the magic number in __pycache__ directories. When you change +# the magic number, you must also set a new unique magic tag. Generally this +# can be named after the Python major version of the magic number bump, but +# it can really be anything, as long as it's different than anything else +# that's come before. The tags are included in the following table, starting +# with Python 3.2a0. +# +# Known values: +# Python 1.5: 20121 +# Python 1.5.1: 20121 +# Python 1.5.2: 20121 +# Python 1.6: 50428 +# Python 2.0: 50823 +# Python 2.0.1: 50823 +# Python 2.1: 60202 +# Python 2.1.1: 60202 +# Python 2.1.2: 60202 +# Python 2.2: 60717 +# Python 2.3a0: 62011 +# Python 2.3a0: 62021 +# Python 2.3a0: 62011 (!) +# Python 2.4a0: 62041 +# Python 2.4a3: 62051 +# Python 2.4b1: 62061 +# Python 2.5a0: 62071 +# Python 2.5a0: 62081 (ast-branch) +# Python 2.5a0: 62091 (with) +# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) +# Python 2.5b3: 62101 (fix wrong code: for x, in ...) +# Python 2.5b3: 62111 (fix wrong code: x += yield) +# Python 2.5c1: 62121 (fix wrong lnotab with for loops and +# storing constants that should have been removed) +# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) +# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) +# Python 2.6a1: 62161 (WITH_CLEANUP optimization) +# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND) +# Python 2.7a0: 62181 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 2.7a0 62191 (introduce SETUP_WITH) +# Python 2.7a0 62201 (introduce BUILD_SET) +# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD) +# Python 3000: 3000 +# 3010 (removed UNARY_CONVERT) +# 3020 (added BUILD_SET) +# 3030 (added keyword-only parameters) +# 3040 (added signature annotations) +# 3050 (print becomes a function) +# 3060 (PEP 3115 metaclass syntax) +# 3061 (string literals become unicode) +# 3071 (PEP 3109 raise changes) +# 3081 (PEP 3137 make __file__ and __name__ unicode) +# 3091 (kill str8 interning) +# 3101 (merge from 2.6a0, see 62151) +# 3103 (__file__ points to source file) +# Python 3.0a4: 3111 (WITH_CLEANUP optimization). +# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) +# Python 3.1a0: 3141 (optimize list, set and dict comprehensions: +# change LIST_APPEND and SET_ADD, add MAP_ADD) +# Python 3.1a0: 3151 (optimize conditional branches: +# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) +# Python 3.2a0: 3160 (add SETUP_WITH) +# tag: cpython-32 +# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) +# tag: cpython-32 +# Python 3.2a2 3180 (add DELETE_DEREF) +# Python 3.3a0 3190 __class__ super closure changed +# Python 3.3a0 3200 (__qualname__ added) +# 3210 (added size modulo 2**32 to the pyc header) +# Python 3.3a1 3220 (changed PEP 380 implementation) +# Python 3.3a4 3230 (revert changes to implicit __class__ closure) +# Python 3.4a1 3250 (evaluate positional default arguments before +# keyword-only defaults) +# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override +# free vars) +# Python 3.4a1 3270 (various tweaks to the __class__ closure) +# Python 3.4a1 3280 (remove implicit class argument) +# +# MAGIC must change whenever the bytecode emitted by the compiler may no +# longer be understood by older implementations of the eval loop (usually +# due to the addition of new opcodes). -""" -_RAW_MAGIC_NUMBER = 3230 | ord('\r') << 16 | ord('\n') << 24 -_MAGIC_BYTES = bytes(_RAW_MAGIC_NUMBER >> n & 0xff for n in range(0, 25, 8)) +MAGIC_NUMBER = (3280).to_bytes(2, 'little') + b'\r\n' +_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c _PYCACHE = '__pycache__' @@ -484,6 +456,18 @@ def _get_sourcefile(bytecode_path): return source_path if _path_isfile(source_path) else bytecode_path +def _calc_mode(path): + """Calculate the mode permissions for a bytecode file.""" + try: + mode = _os.stat(path).st_mode + except OSError: + mode = 0o666 + # We always ensure write access so we can update cached files + # later even when the source files are read-only on Windows (#6074) + mode |= 0o200 + return mode + + def _verbose_message(message, *args, verbosity=1): """Print the message to stderr if -v/PYTHONVERBOSE is turned on.""" if sys.flags.verbose >= verbosity: @@ -492,6 +476,100 @@ def _verbose_message(message, *args, verbosity=1): print(message.format(*args), file=sys.stderr) +class _ManageReload: + + def __init__(self, name): + self._name = name + + def __enter__(self): + self._is_reload = self._name in sys.modules + + def __exit__(self, *args): + if any(arg is not None for arg in args) and not self._is_reload: + try: + del sys.modules[self._name] + except KeyError: + pass + + +# Written as a class only because contextlib is not available. +class _ModuleManager(_ManageReload): + + """Context manager which returns the module to be loaded. + + Does the proper unloading from sys.modules upon failure. + + """ + + def __init__(self, name, *, reset_name=True): + """Prepare the context manager. + + The reset_name argument specifies whether to unconditionally reset + the __name__ attribute if the module is found to be a reload. + """ + super().__init__(name) + self._reset_name = reset_name + + def __enter__(self): + super().__enter__() + self._module = sys.modules.get(self._name) + if not self._is_reload: + # This must be done before open() is called as the 'io' module + # implicitly imports 'locale' and would otherwise trigger an + # infinite loop. + self._module = type(_io)(self._name) + # This must be done before putting the module in sys.modules + # (otherwise an optimization shortcut in import.c becomes wrong) + self._module.__initializing__ = True + sys.modules[self._name] = self._module + elif self._reset_name: + try: + self._module.__name__ = self._name + except AttributeError: + pass + return self._module + + def __exit__(self, *args): + self._module.__initializing__ = False + del self._module + super().__exit__(*args) + + +def module_to_load(name, *, reset_name=True): + """Return a context manager which provides the module object to load. + + If reset_name is true, reset the module's __name__ to 'name'. + """ + # Hiding _ModuleManager behind a function for better naming. + return _ModuleManager(name, reset_name=reset_name) + + +def _init_package_attrs(loader, module): + """Set __package__ and __path__ based on what loader.is_package() says.""" + name = module.__name__ + try: + is_package = loader.is_package(name) + except ImportError: + pass + else: + if is_package: + module.__package__ = name + module.__path__ = [] + else: + module.__package__ = name.rpartition('.')[0] + + +def _init_file_attrs(loader, module): + """Set __file__ and __path__ based on loader.get_filename().""" + try: + module.__file__ = loader.get_filename(module.__name__) + except ImportError: + pass + else: + if module.__name__ == module.__package__: + module.__path__.append(_path_split(module.__file__)[0]) + + def set_package(fxn): """Set __package__ on the returned module.""" def set_package_wrapper(*args, **kwargs): @@ -509,68 +587,13 @@ def set_loader(fxn): """Set __loader__ on the returned module.""" def set_loader_wrapper(self, *args, **kwargs): module = fxn(self, *args, **kwargs) - if not hasattr(module, '__loader__'): + if getattr(module, '__loader__', None) is None: module.__loader__ = self return module _wrap(set_loader_wrapper, fxn) return set_loader_wrapper -def module_for_loader(fxn): - """Decorator to handle selecting the proper module for loaders. - - The decorated function is passed the module to use instead of the module - name. The module passed in to the function is either from sys.modules if - it already exists or is a new module. If the module is new, then __name__ - is set the first argument to the method, __loader__ is set to self, and - __package__ is set accordingly (if self.is_package() is defined) will be set - before it is passed to the decorated function (if self.is_package() does - not work for the module it will be set post-load). - - If an exception is raised and the decorator created the module it is - subsequently removed from sys.modules. - - The decorator assumes that the decorated function takes the module name as - the second argument. - - """ - def module_for_loader_wrapper(self, fullname, *args, **kwargs): - module = sys.modules.get(fullname) - is_reload = module is not None - if not is_reload: - # This must be done before open() is called as the 'io' module - # implicitly imports 'locale' and would otherwise trigger an - # infinite loop. - module = new_module(fullname) - # This must be done before putting the module in sys.modules - # (otherwise an optimization shortcut in import.c becomes wrong) - module.__initializing__ = True - sys.modules[fullname] = module - module.__loader__ = self - try: - is_package = self.is_package(fullname) - except (ImportError, AttributeError): - pass - else: - if is_package: - module.__package__ = fullname - else: - module.__package__ = fullname.rpartition('.')[0] - else: - module.__initializing__ = True - try: - # If __package__ was not set above, __import__() will do it later. - return fxn(self, module, *args, **kwargs) - except: - if not is_reload: - del sys.modules[fullname] - raise - finally: - module.__initializing__ = False - _wrap(module_for_loader_wrapper, fxn) - return module_for_loader_wrapper - - def _check_name(method): """Decorator to verify that the module being requested matches the one the loader can handle. @@ -583,7 +606,7 @@ def _check_name(method): if name is None: name = self.name elif self.name != name: - raise ImportError("loader cannot handle %s" % name, name=name) + raise ImportError('loader cannot handle %s' % name, name=name) return method(self, name, *args, **kwargs) _wrap(_check_name_wrapper, method) return _check_name_wrapper @@ -593,7 +616,7 @@ def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" def _requires_builtin_wrapper(self, fullname): if fullname not in sys.builtin_module_names: - raise ImportError("{} is not a built-in module".format(fullname), + raise ImportError('{} is not a built-in module'.format(fullname), name=fullname) return fxn(self, fullname) _wrap(_requires_builtin_wrapper, fxn) @@ -604,7 +627,7 @@ def _requires_frozen(fxn): """Decorator to verify the named module is frozen.""" def _requires_frozen_wrapper(self, fullname): if not _imp.is_frozen(fullname): - raise ImportError("{} is not a frozen module".format(fullname), + raise ImportError('{} is not a frozen module'.format(fullname), name=fullname) return fxn(self, fullname) _wrap(_requires_frozen_wrapper, fxn) @@ -619,11 +642,98 @@ def _find_module_shim(self, fullname): # return None. loader, portions = self.find_loader(fullname) if loader is None and len(portions): - msg = "Not importing directory {}: missing __init__" + msg = 'Not importing directory {}: missing __init__' _warnings.warn(msg.format(portions[0]), ImportWarning) return loader +def _validate_bytecode_header(data, source_stats=None, name=None, path=None): + """Validate the header of the passed-in bytecode against source_stats (if + given) and returning the bytecode that can be compiled by compile(). + + All other arguments are used to enhance error reporting. + + ImportError is raised when the magic number is incorrect or the bytecode is + found to be stale. EOFError is raised when the data is found to be + truncated. + + """ + exc_details = {} + if name is not None: + exc_details['name'] = name + else: + # To prevent having to make all messages have a conditional name. + name = '<bytecode>' + if path is not None: + exc_details['path'] = path + magic = data[:4] + raw_timestamp = data[4:8] + raw_size = data[8:12] + if magic != MAGIC_NUMBER: + message = 'bad magic number in {!r}: {!r}'.format(name, magic) + _verbose_message(message) + raise ImportError(message, **exc_details) + elif len(raw_timestamp) != 4: + message = 'reached EOF while reading timestamp in {!r}'.format(name) + _verbose_message(message) + raise EOFError(message) + elif len(raw_size) != 4: + message = 'reached EOF while reading size of source in {!r}'.format(name) + _verbose_message(message) + raise EOFError(message) + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if _r_long(raw_timestamp) != source_mtime: + message = 'bytecode is stale for {!r}'.format(name) + _verbose_message(message) + raise ImportError(message, **exc_details) + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if _r_long(raw_size) != source_size: + raise ImportError('bytecode is stale for {!r}'.format(name), + **exc_details) + return data[12:] + + +def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None): + """Compile bytecode as returned by _validate_bytecode_header().""" + code = marshal.loads(data) + if isinstance(code, _code_type): + _verbose_message('code object from {!r}', bytecode_path) + if source_path is not None: + _imp._fix_co_filename(code, source_path) + return code + else: + raise ImportError('Non-code object in {!r}'.format(bytecode_path), + name=name, path=bytecode_path) + +def _code_to_bytecode(code, mtime=0, source_size=0): + """Compile a code object into bytecode for writing out to a byte-compiled + file.""" + data = bytearray(MAGIC_NUMBER) + data.extend(_w_long(mtime)) + data.extend(_w_long(source_size)) + data.extend(marshal.dumps(code)) + return data + + +def decode_source(source_bytes): + """Decode bytes representing source code and return the string. + + Universal newline support is used in the decoding. + """ + import tokenize # To avoid bootstrap issues. + source_bytes_readline = _io.BytesIO(source_bytes).readline + encoding = tokenize.detect_encoding(source_bytes_readline) + newline_decoder = _io.IncrementalNewlineDecoder(None, True) + return newline_decoder.decode(source_bytes.decode(encoding[0])) # Loaders ##################################################################### @@ -639,7 +749,7 @@ class BuiltinImporter: @classmethod def module_repr(cls, module): - return "<module '{}' (built-in)>".format(module.__name__) + return '<module {!r} (built-in)>'.format(module.__name__) @classmethod def find_module(cls, fullname, path=None): @@ -658,13 +768,8 @@ class BuiltinImporter: @_requires_builtin def load_module(cls, fullname): """Load a built-in module.""" - is_reload = fullname in sys.modules - try: + with _ManageReload(fullname): return _call_with_frames_removed(_imp.init_builtin, fullname) - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise @classmethod @_requires_builtin @@ -696,7 +801,7 @@ class FrozenImporter: @classmethod def module_repr(cls, m): - return "<module '{}' (frozen)>".format(m.__name__) + return '<module {!r} (frozen)>'.format(m.__name__) @classmethod def find_module(cls, fullname, path=None): @@ -709,16 +814,11 @@ class FrozenImporter: @_requires_frozen def load_module(cls, fullname): """Load a frozen module.""" - is_reload = fullname in sys.modules - try: + with _ManageReload(fullname): m = _call_with_frames_removed(_imp.init_frozen, fullname) # Let our own module_repr() method produce a suitable repr. del m.__file__ return m - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise @classmethod @_requires_frozen @@ -745,18 +845,18 @@ class WindowsRegistryFinder: """ REGISTRY_KEY = ( - "Software\\Python\\PythonCore\\{sys_version}" - "\\Modules\\{fullname}") + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}') REGISTRY_KEY_DEBUG = ( - "Software\\Python\\PythonCore\\{sys_version}" - "\\Modules\\{fullname}\\Debug") + 'Software\\Python\\PythonCore\\{sys_version}' + '\\Modules\\{fullname}\\Debug') DEBUG_BUILD = False # Changed in _setup() @classmethod def _open_registry(cls, key): try: return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key) - except WindowsError: + except OSError: return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key) @classmethod @@ -769,8 +869,8 @@ class WindowsRegistryFinder: sys_version=sys.version[:3]) try: with cls._open_registry(key) as hkey: - filepath = _winreg.QueryValue(hkey, "") - except WindowsError: + filepath = _winreg.QueryValue(hkey, '') + except OSError: return None return filepath @@ -802,74 +902,32 @@ class _LoaderBasics: tail_name = fullname.rpartition('.')[2] return filename_base == '__init__' and tail_name != '__init__' - def _bytes_from_bytecode(self, fullname, data, bytecode_path, source_stats): - """Return the marshalled bytes from bytecode, verifying the magic - number, timestamp and source size along the way. - - If source_stats is None then skip the timestamp check. + def init_module_attrs(self, module): + """Set various attributes on the module. + ExecutionLoader.init_module_attrs() is used to set __loader__, + __package__, __file__, and optionally __path__. The __cached__ attribute + is set using imp.cache_from_source() and __file__. """ - magic = data[:4] - raw_timestamp = data[4:8] - raw_size = data[8:12] - if magic != _MAGIC_BYTES: - msg = 'bad magic number in {!r}: {!r}'.format(fullname, magic) - _verbose_message(msg) - raise ImportError(msg, name=fullname, path=bytecode_path) - elif len(raw_timestamp) != 4: - message = 'bad timestamp in {}'.format(fullname) - _verbose_message(message) - raise EOFError(message) - elif len(raw_size) != 4: - message = 'bad size in {}'.format(fullname) - _verbose_message(message) - raise EOFError(message) - if source_stats is not None: - try: - source_mtime = int(source_stats['mtime']) - except KeyError: - pass - else: - if _r_long(raw_timestamp) != source_mtime: - message = 'bytecode is stale for {}'.format(fullname) - _verbose_message(message) - raise ImportError(message, name=fullname, - path=bytecode_path) - try: - source_size = source_stats['size'] & 0xFFFFFFFF - except KeyError: - pass - else: - if _r_long(raw_size) != source_size: - raise ImportError( - "bytecode is stale for {}".format(fullname), - name=fullname, path=bytecode_path) - # Can't return the code object as errors from marshal loading need to - # propagate even when source is available. - return data[12:] - - @module_for_loader - def _load_module(self, module, *, sourceless=False): - """Helper for load_module able to handle either source or sourceless - loading.""" - name = module.__name__ - code_object = self.get_code(name) - module.__file__ = self.get_filename(name) - if not sourceless: + module.__loader__ = self # Loader + _init_package_attrs(self, module) # InspectLoader + _init_file_attrs(self, module) # ExecutionLoader + if hasattr(module, '__file__'): # SourceLoader try: module.__cached__ = cache_from_source(module.__file__) except NotImplementedError: - module.__cached__ = module.__file__ - else: - module.__cached__ = module.__file__ - module.__package__ = name - if self.is_package(name): - module.__path__ = [_path_split(module.__file__)[0]] - else: - module.__package__ = module.__package__.rpartition('.')[0] - module.__loader__ = self - _call_with_frames_removed(exec, code_object, module.__dict__) - return module + pass + + def load_module(self, fullname): + """Load the specified module into sys.modules and return it.""" + with module_to_load(fullname) as module: + self.init_module_attrs(module) + code = self.get_code(fullname) + if code is None: + raise ImportError('cannot load module {!r} when get_code() ' + 'returns None'.format(fullname)) + _call_with_frames_removed(exec, code, module.__dict__) + return module class SourceLoader(_LoaderBasics): @@ -877,8 +935,10 @@ class SourceLoader(_LoaderBasics): def path_mtime(self, path): """Optional method that returns the modification time (an int) for the specified path, where path is a str. + + Raises IOError when the path cannot be handled. """ - raise NotImplementedError + raise IOError def path_stats(self, path): """Optional method returning a metadata dict for the specified path @@ -889,6 +949,7 @@ class SourceLoader(_LoaderBasics): - 'size' (optional) is the size in bytes of the source code. Implementing this method allows the loader to read bytecode files. + Raises IOError when the path cannot be handled. """ return {'mtime': self.path_mtime(path)} @@ -906,32 +967,26 @@ class SourceLoader(_LoaderBasics): """Optional method which writes data (bytes) to a file path (a str). Implementing this method allows for the writing of bytecode files. - """ - raise NotImplementedError def get_source(self, fullname): """Concrete implementation of InspectLoader.get_source.""" - import tokenize path = self.get_filename(fullname) try: source_bytes = self.get_data(path) - except IOError as exc: - raise ImportError("source not available through get_data()", - name=fullname) from exc - readsource = _io.BytesIO(source_bytes).readline - try: - encoding = tokenize.detect_encoding(readsource) - except SyntaxError as exc: - raise ImportError("Failed to detect encoding", - name=fullname) from exc - newline_decoder = _io.IncrementalNewlineDecoder(None, True) - try: - return newline_decoder.decode(source_bytes.decode(encoding[0])) - except UnicodeDecodeError as exc: - raise ImportError("Failed to decode source file", + except OSError as exc: + raise ImportError('source not available through get_data()', name=fullname) from exc + return decode_source(source_bytes) + + def source_to_code(self, data, path, *, _optimize=-1): + """Return the code object compiled from source. + + The 'data' argument can be any object type that compile() supports. + """ + return _call_with_frames_removed(compile, data, path, 'exec', + dont_inherit=True, optimize=_optimize) def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. @@ -949,45 +1004,34 @@ class SourceLoader(_LoaderBasics): else: try: st = self.path_stats(source_path) - except NotImplementedError: + except IOError: pass else: source_mtime = int(st['mtime']) try: data = self.get_data(bytecode_path) - except IOError: + except OSError: pass else: try: - bytes_data = self._bytes_from_bytecode(fullname, data, - bytecode_path, - st) + bytes_data = _validate_bytecode_header(data, + source_stats=st, name=fullname, + path=bytecode_path) except (ImportError, EOFError): pass else: _verbose_message('{} matches {}', bytecode_path, source_path) - found = marshal.loads(bytes_data) - if isinstance(found, _code_type): - _imp._fix_co_filename(found, source_path) - _verbose_message('code object from {}', - bytecode_path) - return found - else: - msg = "Non-code object in {}" - raise ImportError(msg.format(bytecode_path), - name=fullname, path=bytecode_path) + return _compile_bytecode(bytes_data, name=fullname, + bytecode_path=bytecode_path, + source_path=source_path) source_bytes = self.get_data(source_path) - code_object = _call_with_frames_removed(compile, - source_bytes, source_path, 'exec', - dont_inherit=True) + code_object = self.source_to_code(source_bytes, source_path) _verbose_message('code object from {}', source_path) if (not sys.dont_write_bytecode and bytecode_path is not None and - source_mtime is not None): - data = bytearray(_MAGIC_BYTES) - data.extend(_w_long(source_mtime)) - data.extend(_w_long(len(source_bytes))) - data.extend(marshal.dumps(code_object)) + source_mtime is not None): + data = _code_to_bytecode(code_object, source_mtime, + len(source_bytes)) try: self._cache_bytecode(source_path, bytecode_path, data) _verbose_message('wrote {!r}', bytecode_path) @@ -995,16 +1039,6 @@ class SourceLoader(_LoaderBasics): pass return code_object - def load_module(self, fullname): - """Concrete implementation of Loader.load_module. - - Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be - implemented to load source code. Use of bytecode is dictated by whether - get_code uses/writes bytecode. - - """ - return self._load_module(fullname) - class FileLoader: @@ -1046,13 +1080,7 @@ class SourceFileLoader(FileLoader, SourceLoader): def _cache_bytecode(self, source_path, bytecode_path, data): # Adapt between the two APIs - try: - mode = _os.stat(source_path).st_mode - except OSError: - mode = 0o666 - # We always ensure write access so we can update cached files - # later even when the source files are read-only on Windows (#6074) - mode |= 0o200 + mode = _calc_mode(source_path) return self.set_data(bytecode_path, data, _mode=mode) def set_data(self, path, data, *, _mode=0o666): @@ -1088,20 +1116,15 @@ class SourcelessFileLoader(FileLoader, _LoaderBasics): """Loader which handles sourceless file imports.""" - def load_module(self, fullname): - return self._load_module(fullname, sourceless=True) + def init_module_attrs(self, module): + super().init_module_attrs(module) + module.__cached__ = module.__file__ def get_code(self, fullname): path = self.get_filename(fullname) data = self.get_data(path) - bytes_data = self._bytes_from_bytecode(fullname, data, path, None) - found = marshal.loads(bytes_data) - if isinstance(found, _code_type): - _verbose_message('code object from {!r}', path) - return found - else: - raise ImportError("Non-code object in {}".format(path), - name=fullname, path=path) + bytes_data = _validate_bytecode_header(data, name=fullname, path=path) + return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path) def get_source(self, fullname): """Return None as there is no source code.""" @@ -1129,18 +1152,13 @@ class ExtensionFileLoader: @set_loader def load_module(self, fullname): """Load an extension module.""" - is_reload = fullname in sys.modules - try: + with _ManageReload(fullname): module = _call_with_frames_removed(_imp.load_dynamic, fullname, self.path) _verbose_message('extension module loaded from {!r}', self.path) if self.is_package(fullname) and not hasattr(module, '__path__'): module.__path__ = [_path_split(self.path)[0]] return module - except: - if not is_reload and fullname in sys.modules: - del sys.modules[fullname] - raise def is_package(self, fullname): """Return True if the extension module is a package.""" @@ -1203,7 +1221,7 @@ class _NamespacePath: return len(self._recalculate()) def __repr__(self): - return "_NamespacePath({!r})".format(self._path) + return '_NamespacePath({!r})'.format(self._path) def __contains__(self, item): return item in self._recalculate() @@ -1218,14 +1236,28 @@ class NamespaceLoader: @classmethod def module_repr(cls, module): - return "<module '{}' (namespace)>".format(module.__name__) + return '<module {!r} (namespace)>'.format(module.__name__) + + def is_package(self, fullname): + return True + + def get_source(self, fullname): + return '' + + def get_code(self, fullname): + return compile('', '<string>', 'exec', dont_inherit=True) - @module_for_loader - def load_module(self, module): + def init_module_attrs(self, module): + module.__loader__ = self + module.__package__ = module.__name__ + + def load_module(self, fullname): """Load a namespace module.""" _verbose_message('namespace module loaded with path {!r}', self._path) - module.__path__ = self._path - return module + with module_to_load(fullname) as module: + self.init_module_attrs(module) + module.__path__ = self._path + return module # Finders ##################################################################### @@ -1423,7 +1455,7 @@ class FileFinder: lower_suffix_contents.add(new_name) self._path_cache = lower_suffix_contents if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS): - self._relaxed_path_cache = set(fn.lower() for fn in contents) + self._relaxed_path_cache = {fn.lower() for fn in contents} @classmethod def path_hook(cls, *loader_details): @@ -1438,13 +1470,13 @@ class FileFinder: def path_hook_for_FileFinder(path): """Path hook for importlib.machinery.FileFinder.""" if not _path_isdir(path): - raise ImportError("only directories are supported", path=path) + raise ImportError('only directories are supported', path=path) return cls(path, *loader_details) return path_hook_for_FileFinder def __repr__(self): - return "FileFinder(%r)" % (self.path,) + return 'FileFinder({!r})'.format(self.path) # Import itself ############################################################### @@ -1491,21 +1523,22 @@ def _find_module(name, path): def _sanity_check(name, package, level): """Verify arguments are "sane".""" if not isinstance(name, str): - raise TypeError("module name must be str, not {}".format(type(name))) + raise TypeError('module name must be str, not {}'.format(type(name))) if level < 0: raise ValueError('level must be >= 0') if package: if not isinstance(package, str): - raise TypeError("__package__ not set to a string") + raise TypeError('__package__ not set to a string') elif package not in sys.modules: - msg = ("Parent module {!r} not loaded, cannot perform relative " - "import") + msg = ('Parent module {!r} not loaded, cannot perform relative ' + 'import') raise SystemError(msg.format(package)) if not name and level == 0: - raise ValueError("Empty module name") + raise ValueError('Empty module name') -_ERR_MSG = 'No module named {!r}' +_ERR_MSG_PREFIX = 'No module named ' +_ERR_MSG = _ERR_MSG_PREFIX + '{!r}' def _find_and_load_unlocked(name, import_): path = None @@ -1525,11 +1558,7 @@ def _find_and_load_unlocked(name, import_): raise ImportError(msg, name=name) loader = _find_module(name, path) if loader is None: - exc = ImportError(_ERR_MSG.format(name), name=name) - # TODO(brett): switch to a proper ModuleNotFound exception in Python - # 3.4. - exc._not_found = True - raise exc + raise ImportError(_ERR_MSG.format(name), name=name) elif name not in sys.modules: # The parent import may have already imported this module. loader.load_module(name) @@ -1549,7 +1578,7 @@ def _find_and_load_unlocked(name, import_): except AttributeError: pass # Set loader if need be. - if not hasattr(module, '__loader__'): + if getattr(module, '__loader__', None) is None: try: module.__loader__ = loader except AttributeError: @@ -1588,8 +1617,8 @@ def _gcd_import(name, package=None, level=0): module = sys.modules[name] if module is None: _imp.release_lock() - message = ("import of {} halted; " - "None in sys.modules".format(name)) + message = ('import of {} halted; ' + 'None in sys.modules'.format(name)) raise ImportError(message, name=name) _lock_unlock_module(name) return module @@ -1619,9 +1648,7 @@ def _handle_fromlist(module, fromlist, import_): # Backwards-compatibility dictates we ignore failed # imports triggered by fromlist for modules that don't # exist. - # TODO(brett): In Python 3.4, have import raise - # ModuleNotFound and catch that. - if getattr(exc, '_not_found', False): + if str(exc).startswith(_ERR_MSG_PREFIX): if exc.name == from_name: continue raise @@ -1710,7 +1737,7 @@ def _setup(sys_module, _imp_module): module_type = type(sys) for name, module in sys.modules.items(): if isinstance(module, module_type): - if not hasattr(module, '__loader__'): + if getattr(module, '__loader__', None) is None: if name in sys.builtin_module_names: module.__loader__ = BuiltinImporter elif _imp.is_frozen(name): @@ -1724,7 +1751,7 @@ def _setup(sys_module, _imp_module): builtin_module = sys.modules[builtin_name] setattr(self_module, builtin_name, builtin_module) - os_details = ('posix', ['/']), ('nt', ['\\', '/']), ('os2', ['\\', '/']) + os_details = ('posix', ['/']), ('nt', ['\\', '/']) for builtin_os, path_separators in os_details: # Assumption made in _path_join() assert all(len(sep) == 1 for sep in path_separators) @@ -1735,9 +1762,6 @@ def _setup(sys_module, _imp_module): else: try: os_module = BuiltinImporter.load_module(builtin_os) - # TODO: rip out os2 code after 3.3 is released as per PEP 11 - if builtin_os == 'os2' and 'EMX GCC' in sys.version: - path_sep = path_separators[1] break except ImportError: continue @@ -1759,7 +1783,7 @@ def _setup(sys_module, _imp_module): setattr(self_module, '_thread', thread_module) setattr(self_module, '_weakref', weakref_module) setattr(self_module, 'path_sep', path_sep) - setattr(self_module, 'path_separators', set(path_separators)) + setattr(self_module, 'path_separators', ''.join(path_separators)) # Constants setattr(self_module, '_relax_case', _make_relax_case()) EXTENSION_SUFFIXES.extend(_imp.extension_suffixes()) |