summaryrefslogtreecommitdiffstats
path: root/Lib
diff options
context:
space:
mode:
Diffstat (limited to 'Lib')
-rw-r--r--Lib/ctypes/test/test_values.py19
-rw-r--r--Lib/imp.py3
-rw-r--r--Lib/importlib/__init__.py20
-rw-r--r--Lib/importlib/_bootstrap.py1343
-rw-r--r--Lib/importlib/_bootstrap_external.py1462
-rw-r--r--Lib/importlib/abc.py24
-rw-r--r--Lib/importlib/machinery.py18
-rw-r--r--Lib/importlib/util.py12
-rw-r--r--Lib/modulefinder.py4
-rw-r--r--Lib/py_compile.py8
-rwxr-xr-xLib/pydoc.py5
-rw-r--r--Lib/runpy.py2
-rw-r--r--Lib/site.py4
-rw-r--r--Lib/test/test_import/__init__.py20
-rw-r--r--Lib/test/test_importlib/extension/test_case_sensitivity.py6
-rw-r--r--Lib/test/test_importlib/import_/test_path.py2
-rw-r--r--Lib/test/test_importlib/source/test_case_sensitivity.py4
-rw-r--r--Lib/test/test_site.py6
18 files changed, 1572 insertions, 1390 deletions
diff --git a/Lib/ctypes/test/test_values.py b/Lib/ctypes/test/test_values.py
index 1c1fd7d..9a40b71 100644
--- a/Lib/ctypes/test/test_values.py
+++ b/Lib/ctypes/test/test_values.py
@@ -59,8 +59,13 @@ class Win_ValuesTestCase(unittest.TestCase):
items = []
# _frozen_importlib changes size whenever importlib._bootstrap
# changes, so it gets a special case. We should make sure it's
- # found, but don't worry about its size too much.
- _fzn_implib_seen = False
+ # found, but don't worry about its size too much. The same
+ # applies to _frozen_importlib_external.
+ bootstrap_seen = []
+ bootstrap_expected = (
+ b'_frozen_importlib',
+ b'_frozen_importlib_external',
+ )
for entry in ft:
# This is dangerous. We *can* iterate over a pointer, but
# the loop will not terminate (maybe with an access
@@ -68,10 +73,10 @@ class Win_ValuesTestCase(unittest.TestCase):
if entry.name is None:
break
- if entry.name == b'_frozen_importlib':
- _fzn_implib_seen = True
+ if entry.name in bootstrap_expected:
+ bootstrap_seen.append(entry.name)
self.assertTrue(entry.size,
- "_frozen_importlib was reported as having no size")
+ "{} was reported as having no size".format(entry.name))
continue
items.append((entry.name, entry.size))
@@ -81,8 +86,8 @@ class Win_ValuesTestCase(unittest.TestCase):
]
self.assertEqual(items, expected)
- self.assertTrue(_fzn_implib_seen,
- "_frozen_importlib wasn't found in PyImport_FrozenModules")
+ self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
+ "frozen bootstrap modules did not match PyImport_FrozenModules")
from ctypes import _pointer_type_cache
del _pointer_type_cache[struct_frozen]
diff --git a/Lib/imp.py b/Lib/imp.py
index f1810f5..3177b28 100644
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -16,7 +16,8 @@ except ImportError:
# Platform doesn't support dynamic loading.
load_dynamic = None
-from importlib._bootstrap import SourcelessFileLoader, _ERR_MSG, _exec, _load
+from importlib._bootstrap import _ERR_MSG, _exec, _load
+from importlib._bootstrap_external import SourcelessFileLoader
from importlib import machinery
from importlib import util
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index e99f50e..349b846 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -30,9 +30,25 @@ else:
pass
sys.modules['importlib._bootstrap'] = _bootstrap
+try:
+ import _frozen_importlib_external as _bootstrap_external
+except ImportError:
+ from . import _bootstrap_external
+ _bootstrap_external._setup(_bootstrap)
+else:
+ _bootstrap_external.__name__ = 'importlib._bootstrap_external'
+ _bootstrap_external.__package__ = 'importlib'
+ try:
+ _bootstrap_external.__file__ = __file__.replace('__init__.py', '_bootstrap_external.py')
+ except NameError:
+ # __file__ is not guaranteed to be defined, e.g. if this code gets
+ # frozen by a tool like cx_Freeze.
+ pass
+ sys.modules['importlib._bootstrap_external'] = _bootstrap_external
+
# To simplify imports in test code
-_w_long = _bootstrap._w_long
-_r_long = _bootstrap._r_long
+_w_long = _bootstrap_external._w_long
+_r_long = _bootstrap_external._r_long
# Fully bootstrapped at this point, import whatever you like, circular
# dependencies and startup overhead minimisation permitting :)
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index c6019ad..cfe6ff9 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -22,102 +22,6 @@ work. One should use importlib as the public-facing version of this module.
# Bootstrap-related code ######################################################
-_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
-
-
-def _make_relax_case():
- if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
- def _relax_case():
- """True if filenames must be checked case-insensitively."""
- return b'PYTHONCASEOK' in _os.environ
- else:
- def _relax_case():
- """True if filenames must be checked case-insensitively."""
- return False
- return _relax_case
-
-
-def _w_long(x):
- """Convert a 32-bit integer to little-endian."""
- return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
-
-
-def _r_long(int_bytes):
- """Convert 4 bytes in little-endian to an integer."""
- return int.from_bytes(int_bytes, 'little')
-
-
-def _path_join(*path_parts):
- """Replacement for os.path.join()."""
- return path_sep.join([part.rstrip(path_separators)
- for part in path_parts if part])
-
-
-def _path_split(path):
- """Replacement for os.path.split()."""
- if len(path_separators) == 1:
- front, _, tail = path.rpartition(path_sep)
- return front, tail
- for x in reversed(path):
- if x in path_separators:
- front, tail = path.rsplit(x, maxsplit=1)
- return front, tail
- return '', path
-
-
-def _path_stat(path):
- """Stat the path.
-
- Made a separate function to make it easier to override in experiments
- (e.g. cache stat results).
-
- """
- return _os.stat(path)
-
-
-def _path_is_mode_type(path, mode):
- """Test whether the path is the specified mode type."""
- try:
- stat_info = _path_stat(path)
- except OSError:
- return False
- return (stat_info.st_mode & 0o170000) == mode
-
-
-def _path_isfile(path):
- """Replacement for os.path.isfile."""
- return _path_is_mode_type(path, 0o100000)
-
-
-def _path_isdir(path):
- """Replacement for os.path.isdir."""
- if not path:
- path = _os.getcwd()
- return _path_is_mode_type(path, 0o040000)
-
-
-def _write_atomic(path, data, mode=0o666):
- """Best-effort function to write data to a path atomically.
- Be prepared to handle a FileExistsError if concurrent writing of the
- temporary file is attempted."""
- # id() is used to generate a pseudo-random filename.
- path_tmp = '{}.{}'.format(path, id(path))
- fd = _os.open(path_tmp,
- _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
- try:
- # We first write data to a temporary file, and then use os.replace() to
- # perform an atomic rename.
- with _io.FileIO(fd, 'wb') as file:
- file.write(data)
- _os.replace(path_tmp, path)
- except OSError:
- try:
- _os.unlink(path_tmp)
- except OSError:
- pass
- raise
-
-
def _wrap(new, old):
"""Simple substitute for functools.update_wrapper."""
for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
@@ -130,10 +34,6 @@ def _new_module(name):
return type(sys)(name)
-_code_type = type(_wrap.__code__)
-
-
-
class _ManageReload:
"""Manages the possible clean-up of sys.modules for load_module()."""
@@ -309,7 +209,6 @@ def _lock_unlock_module(name):
lock.release()
# Frame stripping magic ###############################################
-
def _call_with_frames_removed(f, *args, **kwds):
"""remove_importlib_frames in import.c will always remove sequences
of importlib frames that end with a call to this function
@@ -321,230 +220,6 @@ def _call_with_frames_removed(f, *args, **kwds):
return f(*args, **kwds)
-# Finder/loader utility code ###############################################
-
-# Magic word to reject .pyc files generated by other Python versions.
-# It should change for each incompatible change to the bytecode.
-#
-# The value of CR and LF is incorporated so if you ever read or write
-# a .pyc file in text mode the magic number will be wrong; also, the
-# Apple MPW compiler swaps their values, botching string constants.
-#
-# The magic numbers must be spaced apart at least 2 values, as the
-# -U interpeter flag will cause MAGIC+1 being used. They have been
-# odd numbers for some time now.
-#
-# There were a variety of old schemes for setting the magic number.
-# The current working scheme is to increment the previous value by
-# 10.
-#
-# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
-# number also includes a new "magic tag", i.e. a human readable string used
-# to represent the magic number in __pycache__ directories. When you change
-# the magic number, you must also set a new unique magic tag. Generally this
-# can be named after the Python major version of the magic number bump, but
-# it can really be anything, as long as it's different than anything else
-# that's come before. The tags are included in the following table, starting
-# with Python 3.2a0.
-#
-# Known values:
-# Python 1.5: 20121
-# Python 1.5.1: 20121
-# Python 1.5.2: 20121
-# Python 1.6: 50428
-# Python 2.0: 50823
-# Python 2.0.1: 50823
-# Python 2.1: 60202
-# Python 2.1.1: 60202
-# Python 2.1.2: 60202
-# Python 2.2: 60717
-# Python 2.3a0: 62011
-# Python 2.3a0: 62021
-# Python 2.3a0: 62011 (!)
-# Python 2.4a0: 62041
-# Python 2.4a3: 62051
-# Python 2.4b1: 62061
-# Python 2.5a0: 62071
-# Python 2.5a0: 62081 (ast-branch)
-# Python 2.5a0: 62091 (with)
-# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
-# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
-# Python 2.5b3: 62111 (fix wrong code: x += yield)
-# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
-# storing constants that should have been removed)
-# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
-# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
-# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
-# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
-# Python 2.7a0: 62181 (optimize conditional branches:
-# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
-# Python 2.7a0 62191 (introduce SETUP_WITH)
-# Python 2.7a0 62201 (introduce BUILD_SET)
-# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
-# Python 3000: 3000
-# 3010 (removed UNARY_CONVERT)
-# 3020 (added BUILD_SET)
-# 3030 (added keyword-only parameters)
-# 3040 (added signature annotations)
-# 3050 (print becomes a function)
-# 3060 (PEP 3115 metaclass syntax)
-# 3061 (string literals become unicode)
-# 3071 (PEP 3109 raise changes)
-# 3081 (PEP 3137 make __file__ and __name__ unicode)
-# 3091 (kill str8 interning)
-# 3101 (merge from 2.6a0, see 62151)
-# 3103 (__file__ points to source file)
-# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
-# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
-# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
-# change LIST_APPEND and SET_ADD, add MAP_ADD)
-# Python 3.1a0: 3151 (optimize conditional branches:
-# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
-# Python 3.2a0: 3160 (add SETUP_WITH)
-# tag: cpython-32
-# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
-# tag: cpython-32
-# Python 3.2a2 3180 (add DELETE_DEREF)
-# Python 3.3a0 3190 __class__ super closure changed
-# Python 3.3a0 3200 (__qualname__ added)
-# 3210 (added size modulo 2**32 to the pyc header)
-# Python 3.3a1 3220 (changed PEP 380 implementation)
-# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
-# Python 3.4a1 3250 (evaluate positional default arguments before
-# keyword-only defaults)
-# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
-# free vars)
-# Python 3.4a1 3270 (various tweaks to the __class__ closure)
-# Python 3.4a1 3280 (remove implicit class argument)
-# Python 3.4a4 3290 (changes to __qualname__ computation)
-# Python 3.4a4 3300 (more changes to __qualname__ computation)
-# Python 3.4rc2 3310 (alter __qualname__ computation)
-# Python 3.5a0 3320 (matrix multiplication operator)
-#
-# MAGIC must change whenever the bytecode emitted by the compiler may no
-# longer be understood by older implementations of the eval loop (usually
-# due to the addition of new opcodes).
-
-MAGIC_NUMBER = (3320).to_bytes(2, 'little') + b'\r\n'
-_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
-
-_PYCACHE = '__pycache__'
-_OPT = 'opt-'
-
-SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
-
-BYTECODE_SUFFIXES = ['.pyc']
-# Deprecated.
-DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
-
-def cache_from_source(path, debug_override=None, *, optimization=None):
- """Given the path to a .py file, return the path to its .pyc file.
-
- The .py file does not need to exist; this simply returns the path to the
- .pyc file calculated as if the .py file were imported.
-
- The 'optimization' parameter controls the presumed optimization level of
- the bytecode file. If 'optimization' is not None, the string representation
- of the argument is taken and verified to be alphanumeric (else ValueError
- is raised).
-
- The debug_override parameter is deprecated. If debug_override is not None,
- a True value is the same as setting 'optimization' to the empty string
- while a False value is equivalent to setting 'optimization' to '1'.
-
- If sys.implementation.cache_tag is None then NotImplementedError is raised.
-
- """
- if debug_override is not None:
- _warnings.warn('the debug_override parameter is deprecated; use '
- "'optimization' instead", DeprecationWarning)
- if optimization is not None:
- message = 'debug_override or optimization must be set to None'
- raise TypeError(message)
- optimization = '' if debug_override else 1
- head, tail = _path_split(path)
- base, sep, rest = tail.rpartition('.')
- tag = sys.implementation.cache_tag
- if tag is None:
- raise NotImplementedError('sys.implementation.cache_tag is None')
- almost_filename = ''.join([(base if base else rest), sep, tag])
- if optimization is None:
- if sys.flags.optimize == 0:
- optimization = ''
- else:
- optimization = sys.flags.optimize
- optimization = str(optimization)
- if optimization != '':
- if not optimization.isalnum():
- raise ValueError('{!r} is not alphanumeric'.format(optimization))
- almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
- return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
-
-
-def source_from_cache(path):
- """Given the path to a .pyc. file, return the path to its .py file.
-
- The .pyc file does not need to exist; this simply returns the path to
- the .py file calculated to correspond to the .pyc file. If path does
- not conform to PEP 3147/488 format, ValueError will be raised. If
- sys.implementation.cache_tag is None then NotImplementedError is raised.
-
- """
- if sys.implementation.cache_tag is None:
- raise NotImplementedError('sys.implementation.cache_tag is None')
- head, pycache_filename = _path_split(path)
- head, pycache = _path_split(head)
- if pycache != _PYCACHE:
- raise ValueError('{} not bottom-level directory in '
- '{!r}'.format(_PYCACHE, path))
- dot_count = pycache_filename.count('.')
- if dot_count not in {2, 3}:
- raise ValueError('expected only 2 or 3 dots in '
- '{!r}'.format(pycache_filename))
- elif dot_count == 3:
- optimization = pycache_filename.rsplit('.', 2)[-2]
- if not optimization.startswith(_OPT):
- raise ValueError("optimization portion of filename does not start "
- "with {!r}".format(_OPT))
- opt_level = optimization[len(_OPT):]
- if not opt_level.isalnum():
- raise ValueError("optimization level {!r} is not an alphanumeric "
- "value".format(optimization))
- base_filename = pycache_filename.partition('.')[0]
- return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
-
-
-def _get_sourcefile(bytecode_path):
- """Convert a bytecode file path to a source path (if possible).
-
- This function exists purely for backwards-compatibility for
- PyImport_ExecCodeModuleWithFilenames() in the C API.
-
- """
- if len(bytecode_path) == 0:
- return None
- rest, _, extension = bytecode_path.rpartition('.')
- if not rest or extension.lower()[-3:-1] != 'py':
- return bytecode_path
- try:
- source_path = source_from_cache(bytecode_path)
- except (NotImplementedError, ValueError):
- source_path = bytecode_path[:-1]
- return source_path if _path_isfile(source_path) else bytecode_path
-
-
-def _calc_mode(path):
- """Calculate the mode permissions for a bytecode file."""
- try:
- mode = _path_stat(path).st_mode
- except OSError:
- mode = 0o666
- # We always ensure write access so we can update cached files
- # later even when the source files are read-only on Windows (#6074)
- mode |= 0o200
- return mode
-
-
def _verbose_message(message, *args, verbosity=1):
"""Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
if sys.flags.verbose >= verbosity:
@@ -553,24 +228,6 @@ def _verbose_message(message, *args, verbosity=1):
print(message.format(*args), file=sys.stderr)
-def _check_name(method):
- """Decorator to verify that the module being requested matches the one the
- loader can handle.
-
- The first argument (self) must define _name which the second argument is
- compared against. If the comparison fails then ImportError is raised.
-
- """
- def _check_name_wrapper(self, name=None, *args, **kwargs):
- if name is None:
- name = self.name
- elif self.name != name:
- raise ImportError('loader cannot handle %s' % name, name=name)
- return method(self, name, *args, **kwargs)
- _wrap(_check_name_wrapper, method)
- return _check_name_wrapper
-
-
def _requires_builtin(fxn):
"""Decorator to verify the named module is built-in."""
def _requires_builtin_wrapper(self, fullname):
@@ -593,23 +250,6 @@ def _requires_frozen(fxn):
return _requires_frozen_wrapper
-def _find_module_shim(self, fullname):
- """Try to find a loader for the specified module by delegating to
- self.find_loader().
-
- This method is deprecated in favor of finder.find_spec().
-
- """
- # Call find_loader(). If it returns a string (indicating this
- # is a namespace package portion), generate a warning and
- # return None.
- loader, portions = self.find_loader(fullname)
- if loader is None and len(portions):
- msg = 'Not importing directory {}: missing __init__'
- _warnings.warn(msg.format(portions[0]), ImportWarning)
- return loader
-
-
# Typically used by loader classes as a method replacement.
def _load_module_shim(self, fullname):
"""Load the specified module into sys.modules and return it.
@@ -625,96 +265,6 @@ def _load_module_shim(self, fullname):
else:
return _load(spec)
-
-def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
- """Validate the header of the passed-in bytecode against source_stats (if
- given) and returning the bytecode that can be compiled by compile().
-
- All other arguments are used to enhance error reporting.
-
- ImportError is raised when the magic number is incorrect or the bytecode is
- found to be stale. EOFError is raised when the data is found to be
- truncated.
-
- """
- exc_details = {}
- if name is not None:
- exc_details['name'] = name
- else:
- # To prevent having to make all messages have a conditional name.
- name = '<bytecode>'
- if path is not None:
- exc_details['path'] = path
- magic = data[:4]
- raw_timestamp = data[4:8]
- raw_size = data[8:12]
- if magic != MAGIC_NUMBER:
- message = 'bad magic number in {!r}: {!r}'.format(name, magic)
- _verbose_message(message)
- raise ImportError(message, **exc_details)
- elif len(raw_timestamp) != 4:
- message = 'reached EOF while reading timestamp in {!r}'.format(name)
- _verbose_message(message)
- raise EOFError(message)
- elif len(raw_size) != 4:
- message = 'reached EOF while reading size of source in {!r}'.format(name)
- _verbose_message(message)
- raise EOFError(message)
- if source_stats is not None:
- try:
- source_mtime = int(source_stats['mtime'])
- except KeyError:
- pass
- else:
- if _r_long(raw_timestamp) != source_mtime:
- message = 'bytecode is stale for {!r}'.format(name)
- _verbose_message(message)
- raise ImportError(message, **exc_details)
- try:
- source_size = source_stats['size'] & 0xFFFFFFFF
- except KeyError:
- pass
- else:
- if _r_long(raw_size) != source_size:
- raise ImportError('bytecode is stale for {!r}'.format(name),
- **exc_details)
- return data[12:]
-
-
-def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
- """Compile bytecode as returned by _validate_bytecode_header()."""
- code = marshal.loads(data)
- if isinstance(code, _code_type):
- _verbose_message('code object from {!r}', bytecode_path)
- if source_path is not None:
- _imp._fix_co_filename(code, source_path)
- return code
- else:
- raise ImportError('Non-code object in {!r}'.format(bytecode_path),
- name=name, path=bytecode_path)
-
-def _code_to_bytecode(code, mtime=0, source_size=0):
- """Compile a code object into bytecode for writing out to a byte-compiled
- file."""
- data = bytearray(MAGIC_NUMBER)
- data.extend(_w_long(mtime))
- data.extend(_w_long(source_size))
- data.extend(marshal.dumps(code))
- return data
-
-
-def decode_source(source_bytes):
- """Decode bytes representing source code and return the string.
-
- Universal newline support is used in the decoding.
- """
- import tokenize # To avoid bootstrap issues.
- source_bytes_readline = _io.BytesIO(source_bytes).readline
- encoding = tokenize.detect_encoding(source_bytes_readline)
- newline_decoder = _io.IncrementalNewlineDecoder(None, True)
- return newline_decoder.decode(source_bytes.decode(encoding[0]))
-
-
# Module specifications #######################################################
def _module_repr(module):
@@ -855,14 +405,8 @@ class ModuleSpec:
def cached(self):
if self._cached is None:
if self.origin is not None and self._set_fileattr:
- filename = self.origin
- if filename.endswith(tuple(SOURCE_SUFFIXES)):
- try:
- self._cached = cache_from_source(filename)
- except NotImplementedError:
- pass
- elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
- self._cached = filename
+ import _frozen_importlib_external as _bootstrap_external # XXX yuck
+ self._cached = _bootstrap_external._get_cached(self.origin)
return self._cached
@cached.setter
@@ -889,6 +433,7 @@ class ModuleSpec:
def spec_from_loader(name, loader, *, origin=None, is_package=None):
"""Return a module spec based on various loader methods."""
if hasattr(loader, 'get_filename'):
+ from ._bootstrap_external import spec_from_file_location # XXX yuck
if is_package is None:
return spec_from_file_location(name, loader=loader)
search = [] if is_package else None
@@ -911,70 +456,6 @@ def spec_from_loader(name, loader, *, origin=None, is_package=None):
_POPULATE = object()
-def spec_from_file_location(name, location=None, *, loader=None,
- submodule_search_locations=_POPULATE):
- """Return a module spec based on a file location.
-
- To indicate that the module is a package, set
- submodule_search_locations to a list of directory paths. An
- empty list is sufficient, though its not otherwise useful to the
- import system.
-
- The loader must take a spec as its only __init__() arg.
-
- """
- if location is None:
- # The caller may simply want a partially populated location-
- # oriented spec. So we set the location to a bogus value and
- # fill in as much as we can.
- location = '<unknown>'
- if hasattr(loader, 'get_filename'):
- # ExecutionLoader
- try:
- location = loader.get_filename(name)
- except ImportError:
- pass
-
- # If the location is on the filesystem, but doesn't actually exist,
- # we could return None here, indicating that the location is not
- # valid. However, we don't have a good way of testing since an
- # indirect location (e.g. a zip file or URL) will look like a
- # non-existent file relative to the filesystem.
-
- spec = ModuleSpec(name, loader, origin=location)
- spec._set_fileattr = True
-
- # Pick a loader if one wasn't provided.
- if loader is None:
- for loader_class, suffixes in _get_supported_file_loaders():
- if location.endswith(tuple(suffixes)):
- loader = loader_class(name, location)
- spec.loader = loader
- break
- else:
- return None
-
- # Set submodule_search_paths appropriately.
- if submodule_search_locations is _POPULATE:
- # Check the loader.
- if hasattr(loader, 'is_package'):
- try:
- is_package = loader.is_package(name)
- except ImportError:
- pass
- else:
- if is_package:
- spec.submodule_search_locations = []
- else:
- spec.submodule_search_locations = submodule_search_locations
- if spec.submodule_search_locations == []:
- if location:
- dirname = _path_split(location)[0]
- spec.submodule_search_locations.append(dirname)
-
- return spec
-
-
def _spec_from_module(module, loader=None, origin=None):
# This function is meant for use in _setup().
try:
@@ -1035,6 +516,7 @@ def _init_module_attrs(spec, module, *, override=False):
if loader is None:
# A backward compatibility hack.
if spec.submodule_search_locations is not None:
+ from ._bootstrap_external import _NamespaceLoader # XXX yuck
loader = _NamespaceLoader.__new__(_NamespaceLoader)
loader._path = spec.submodule_search_locations
try:
@@ -1202,29 +684,6 @@ def _load(spec):
return _load_unlocked(spec)
-def _fix_up_module(ns, name, pathname, cpathname=None):
- # This function is used by PyImport_ExecCodeModuleObject().
- loader = ns.get('__loader__')
- spec = ns.get('__spec__')
- if not loader:
- if spec:
- loader = spec.loader
- elif pathname == cpathname:
- loader = SourcelessFileLoader(name, pathname)
- else:
- loader = SourceFileLoader(name, pathname)
- if not spec:
- spec = spec_from_file_location(name, pathname, loader=loader)
- try:
- ns['__spec__'] = spec
- ns['__loader__'] = loader
- ns['__file__'] = pathname
- ns['__cached__'] = cpathname
- except Exception:
- # Not important enough to report.
- pass
-
-
# Loaders #####################################################################
class BuiltinImporter:
@@ -1351,6 +810,7 @@ class FrozenImporter:
This method is deprecated. Use exec_module() instead.
"""
+ from ._bootstrap_external import _load_module_shim # XXX yuck
return _load_module_shim(cls, fullname)
@classmethod
@@ -1372,742 +832,6 @@ class FrozenImporter:
return _imp.is_frozen_package(fullname)
-class WindowsRegistryFinder:
-
- """Meta path finder for modules declared in the Windows registry."""
-
- REGISTRY_KEY = (
- 'Software\\Python\\PythonCore\\{sys_version}'
- '\\Modules\\{fullname}')
- REGISTRY_KEY_DEBUG = (
- 'Software\\Python\\PythonCore\\{sys_version}'
- '\\Modules\\{fullname}\\Debug')
- DEBUG_BUILD = False # Changed in _setup()
-
- @classmethod
- def _open_registry(cls, key):
- try:
- return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
- except OSError:
- return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
-
- @classmethod
- def _search_registry(cls, fullname):
- if cls.DEBUG_BUILD:
- registry_key = cls.REGISTRY_KEY_DEBUG
- else:
- registry_key = cls.REGISTRY_KEY
- key = registry_key.format(fullname=fullname,
- sys_version=sys.version[:3])
- try:
- with cls._open_registry(key) as hkey:
- filepath = _winreg.QueryValue(hkey, '')
- except OSError:
- return None
- return filepath
-
- @classmethod
- def find_spec(cls, fullname, path=None, target=None):
- filepath = cls._search_registry(fullname)
- if filepath is None:
- return None
- try:
- _path_stat(filepath)
- except OSError:
- return None
- for loader, suffixes in _get_supported_file_loaders():
- if filepath.endswith(tuple(suffixes)):
- spec = spec_from_loader(fullname, loader(fullname, filepath),
- origin=filepath)
- return spec
-
- @classmethod
- def find_module(cls, fullname, path=None):
- """Find module named in the registry.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- spec = cls.find_spec(fullname, path)
- if spec is not None:
- return spec.loader
- else:
- return None
-
-
-class _LoaderBasics:
-
- """Base class of common code needed by both SourceLoader and
- SourcelessFileLoader."""
-
- def is_package(self, fullname):
- """Concrete implementation of InspectLoader.is_package by checking if
- the path returned by get_filename has a filename of '__init__.py'."""
- filename = _path_split(self.get_filename(fullname))[1]
- filename_base = filename.rsplit('.', 1)[0]
- tail_name = fullname.rpartition('.')[2]
- return filename_base == '__init__' and tail_name != '__init__'
-
- def create_module(self, spec):
- """Use default semantics for module creation."""
-
- def exec_module(self, module):
- """Execute the module."""
- code = self.get_code(module.__name__)
- if code is None:
- raise ImportError('cannot load module {!r} when get_code() '
- 'returns None'.format(module.__name__))
- _call_with_frames_removed(exec, code, module.__dict__)
-
- load_module = _load_module_shim
-
-
-class SourceLoader(_LoaderBasics):
-
- def path_mtime(self, path):
- """Optional method that returns the modification time (an int) for the
- specified path, where path is a str.
-
- Raises IOError when the path cannot be handled.
- """
- raise IOError
-
- def path_stats(self, path):
- """Optional method returning a metadata dict for the specified path
- to by the path (str).
- Possible keys:
- - 'mtime' (mandatory) is the numeric timestamp of last source
- code modification;
- - 'size' (optional) is the size in bytes of the source code.
-
- Implementing this method allows the loader to read bytecode files.
- Raises IOError when the path cannot be handled.
- """
- return {'mtime': self.path_mtime(path)}
-
- def _cache_bytecode(self, source_path, cache_path, data):
- """Optional method which writes data (bytes) to a file path (a str).
-
- Implementing this method allows for the writing of bytecode files.
-
- The source path is needed in order to correctly transfer permissions
- """
- # For backwards compatibility, we delegate to set_data()
- return self.set_data(cache_path, data)
-
- def set_data(self, path, data):
- """Optional method which writes data (bytes) to a file path (a str).
-
- Implementing this method allows for the writing of bytecode files.
- """
-
-
- def get_source(self, fullname):
- """Concrete implementation of InspectLoader.get_source."""
- path = self.get_filename(fullname)
- try:
- source_bytes = self.get_data(path)
- except OSError as exc:
- raise ImportError('source not available through get_data()',
- name=fullname) from exc
- return decode_source(source_bytes)
-
- def source_to_code(self, data, path, *, _optimize=-1):
- """Return the code object compiled from source.
-
- The 'data' argument can be any object type that compile() supports.
- """
- return _call_with_frames_removed(compile, data, path, 'exec',
- dont_inherit=True, optimize=_optimize)
-
- def get_code(self, fullname):
- """Concrete implementation of InspectLoader.get_code.
-
- Reading of bytecode requires path_stats to be implemented. To write
- bytecode, set_data must also be implemented.
-
- """
- source_path = self.get_filename(fullname)
- source_mtime = None
- try:
- bytecode_path = cache_from_source(source_path)
- except NotImplementedError:
- bytecode_path = None
- else:
- try:
- st = self.path_stats(source_path)
- except IOError:
- pass
- else:
- source_mtime = int(st['mtime'])
- try:
- data = self.get_data(bytecode_path)
- except OSError:
- pass
- else:
- try:
- bytes_data = _validate_bytecode_header(data,
- source_stats=st, name=fullname,
- path=bytecode_path)
- except (ImportError, EOFError):
- pass
- else:
- _verbose_message('{} matches {}', bytecode_path,
- source_path)
- return _compile_bytecode(bytes_data, name=fullname,
- bytecode_path=bytecode_path,
- source_path=source_path)
- source_bytes = self.get_data(source_path)
- code_object = self.source_to_code(source_bytes, source_path)
- _verbose_message('code object from {}', source_path)
- if (not sys.dont_write_bytecode and bytecode_path is not None and
- source_mtime is not None):
- data = _code_to_bytecode(code_object, source_mtime,
- len(source_bytes))
- try:
- self._cache_bytecode(source_path, bytecode_path, data)
- _verbose_message('wrote {!r}', bytecode_path)
- except NotImplementedError:
- pass
- return code_object
-
-
-class FileLoader:
-
- """Base file loader class which implements the loader protocol methods that
- require file system usage."""
-
- def __init__(self, fullname, path):
- """Cache the module name and the path to the file found by the
- finder."""
- self.name = fullname
- self.path = path
-
- def __eq__(self, other):
- return (self.__class__ == other.__class__ and
- self.__dict__ == other.__dict__)
-
- def __hash__(self):
- return hash(self.name) ^ hash(self.path)
-
- @_check_name
- def load_module(self, fullname):
- """Load a module from a file.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- # The only reason for this method is for the name check.
- # Issue #14857: Avoid the zero-argument form of super so the implementation
- # of that form can be updated without breaking the frozen module
- return super(FileLoader, self).load_module(fullname)
-
- @_check_name
- def get_filename(self, fullname):
- """Return the path to the source file as found by the finder."""
- return self.path
-
- def get_data(self, path):
- """Return the data from path as raw bytes."""
- with _io.FileIO(path, 'r') as file:
- return file.read()
-
-
-class SourceFileLoader(FileLoader, SourceLoader):
-
- """Concrete implementation of SourceLoader using the file system."""
-
- def path_stats(self, path):
- """Return the metadata for the path."""
- st = _path_stat(path)
- return {'mtime': st.st_mtime, 'size': st.st_size}
-
- def _cache_bytecode(self, source_path, bytecode_path, data):
- # Adapt between the two APIs
- mode = _calc_mode(source_path)
- return self.set_data(bytecode_path, data, _mode=mode)
-
- def set_data(self, path, data, *, _mode=0o666):
- """Write bytes data to a file."""
- parent, filename = _path_split(path)
- path_parts = []
- # Figure out what directories are missing.
- while parent and not _path_isdir(parent):
- parent, part = _path_split(parent)
- path_parts.append(part)
- # Create needed directories.
- for part in reversed(path_parts):
- parent = _path_join(parent, part)
- try:
- _os.mkdir(parent)
- except FileExistsError:
- # Probably another Python process already created the dir.
- continue
- except OSError as exc:
- # Could be a permission error, read-only filesystem: just forget
- # about writing the data.
- _verbose_message('could not create {!r}: {!r}', parent, exc)
- return
- try:
- _write_atomic(path, data, _mode)
- _verbose_message('created {!r}', path)
- except OSError as exc:
- # Same as above: just don't write the bytecode.
- _verbose_message('could not create {!r}: {!r}', path, exc)
-
-
-class SourcelessFileLoader(FileLoader, _LoaderBasics):
-
- """Loader which handles sourceless file imports."""
-
- def get_code(self, fullname):
- path = self.get_filename(fullname)
- data = self.get_data(path)
- bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
- return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
-
- def get_source(self, fullname):
- """Return None as there is no source code."""
- return None
-
-
-# Filled in by _setup().
-EXTENSION_SUFFIXES = []
-
-
-class ExtensionFileLoader:
-
- """Loader for extension modules.
-
- The constructor is designed to work with FileFinder.
-
- """
-
- def __init__(self, name, path):
- self.name = name
- self.path = path
-
- def __eq__(self, other):
- return (self.__class__ == other.__class__ and
- self.__dict__ == other.__dict__)
-
- def __hash__(self):
- return hash(self.name) ^ hash(self.path)
-
- @_check_name
- def load_module(self, fullname):
- """Load an extension module."""
- # Once an exec_module() implementation is added we can also
- # add a deprecation warning here.
- with _ManageReload(fullname):
- module = _call_with_frames_removed(_imp.load_dynamic,
- fullname, self.path)
- _verbose_message('extension module loaded from {!r}', self.path)
- is_package = self.is_package(fullname)
- if is_package and not hasattr(module, '__path__'):
- module.__path__ = [_path_split(self.path)[0]]
- module.__loader__ = self
- module.__package__ = module.__name__
- if not is_package:
- module.__package__ = module.__package__.rpartition('.')[0]
- return module
-
- def is_package(self, fullname):
- """Return True if the extension module is a package."""
- file_name = _path_split(self.path)[1]
- return any(file_name == '__init__' + suffix
- for suffix in EXTENSION_SUFFIXES)
-
- def get_code(self, fullname):
- """Return None as an extension module cannot create a code object."""
- return None
-
- def get_source(self, fullname):
- """Return None as extension modules have no source code."""
- return None
-
- @_check_name
- def get_filename(self, fullname):
- """Return the path to the source file as found by the finder."""
- return self.path
-
-
-class _NamespacePath:
- """Represents a namespace package's path. It uses the module name
- to find its parent module, and from there it looks up the parent's
- __path__. When this changes, the module's own path is recomputed,
- using path_finder. For top-level modules, the parent module's path
- is sys.path."""
-
- def __init__(self, name, path, path_finder):
- self._name = name
- self._path = path
- self._last_parent_path = tuple(self._get_parent_path())
- self._path_finder = path_finder
-
- def _find_parent_path_names(self):
- """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
- parent, dot, me = self._name.rpartition('.')
- if dot == '':
- # This is a top-level module. sys.path contains the parent path.
- return 'sys', 'path'
- # Not a top-level module. parent-module.__path__ contains the
- # parent path.
- return parent, '__path__'
-
- def _get_parent_path(self):
- parent_module_name, path_attr_name = self._find_parent_path_names()
- return getattr(sys.modules[parent_module_name], path_attr_name)
-
- def _recalculate(self):
- # If the parent's path has changed, recalculate _path
- parent_path = tuple(self._get_parent_path()) # Make a copy
- if parent_path != self._last_parent_path:
- spec = self._path_finder(self._name, parent_path)
- # Note that no changes are made if a loader is returned, but we
- # do remember the new parent path
- if spec is not None and spec.loader is None:
- if spec.submodule_search_locations:
- self._path = spec.submodule_search_locations
- self._last_parent_path = parent_path # Save the copy
- return self._path
-
- def __iter__(self):
- return iter(self._recalculate())
-
- def __len__(self):
- return len(self._recalculate())
-
- def __repr__(self):
- return '_NamespacePath({!r})'.format(self._path)
-
- def __contains__(self, item):
- return item in self._recalculate()
-
- def append(self, item):
- self._path.append(item)
-
-
-# We use this exclusively in module_from_spec() for backward-compatibility.
-class _NamespaceLoader:
- def __init__(self, name, path, path_finder):
- self._path = _NamespacePath(name, path, path_finder)
-
- @classmethod
- def module_repr(cls, module):
- """Return repr for the module.
-
- The method is deprecated. The import machinery does the job itself.
-
- """
- return '<module {!r} (namespace)>'.format(module.__name__)
-
- def is_package(self, fullname):
- return True
-
- def get_source(self, fullname):
- return ''
-
- def get_code(self, fullname):
- return compile('', '<string>', 'exec', dont_inherit=True)
-
- def create_module(self, spec):
- """Use default semantics for module creation."""
-
- def exec_module(self, module):
- pass
-
- def load_module(self, fullname):
- """Load a namespace module.
-
- This method is deprecated. Use exec_module() instead.
-
- """
- # The import system never calls this method.
- _verbose_message('namespace module loaded with path {!r}', self._path)
- return _load_module_shim(self, fullname)
-
-
-# Finders #####################################################################
-
-class PathFinder:
-
- """Meta path finder for sys.path and package __path__ attributes."""
-
- @classmethod
- def invalidate_caches(cls):
- """Call the invalidate_caches() method on all path entry finders
- stored in sys.path_importer_caches (where implemented)."""
- for finder in sys.path_importer_cache.values():
- if hasattr(finder, 'invalidate_caches'):
- finder.invalidate_caches()
-
- @classmethod
- def _path_hooks(cls, path):
- """Search sequence of hooks for a finder for 'path'.
-
- If 'hooks' is false then use sys.path_hooks.
-
- """
- if sys.path_hooks is not None and not sys.path_hooks:
- _warnings.warn('sys.path_hooks is empty', ImportWarning)
- for hook in sys.path_hooks:
- try:
- return hook(path)
- except ImportError:
- continue
- else:
- return None
-
- @classmethod
- def _path_importer_cache(cls, path):
- """Get the finder for the path entry from sys.path_importer_cache.
-
- If the path entry is not in the cache, find the appropriate finder
- and cache it. If no finder is available, store None.
-
- """
- if path == '':
- try:
- path = _os.getcwd()
- except FileNotFoundError:
- # Don't cache the failure as the cwd can easily change to
- # a valid directory later on.
- return None
- try:
- finder = sys.path_importer_cache[path]
- except KeyError:
- finder = cls._path_hooks(path)
- sys.path_importer_cache[path] = finder
- return finder
-
- @classmethod
- def _legacy_get_spec(cls, fullname, finder):
- # This would be a good place for a DeprecationWarning if
- # we ended up going that route.
- if hasattr(finder, 'find_loader'):
- loader, portions = finder.find_loader(fullname)
- else:
- loader = finder.find_module(fullname)
- portions = []
- if loader is not None:
- return spec_from_loader(fullname, loader)
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = portions
- return spec
-
- @classmethod
- def _get_spec(cls, fullname, path, target=None):
- """Find the loader or namespace_path for this module/package name."""
- # If this ends up being a namespace package, namespace_path is
- # the list of paths that will become its __path__
- namespace_path = []
- for entry in path:
- if not isinstance(entry, (str, bytes)):
- continue
- finder = cls._path_importer_cache(entry)
- if finder is not None:
- if hasattr(finder, 'find_spec'):
- spec = finder.find_spec(fullname, target)
- else:
- spec = cls._legacy_get_spec(fullname, finder)
- if spec is None:
- continue
- if spec.loader is not None:
- return spec
- portions = spec.submodule_search_locations
- if portions is None:
- raise ImportError('spec missing loader')
- # This is possibly part of a namespace package.
- # Remember these path entries (if any) for when we
- # create a namespace package, and continue iterating
- # on path.
- namespace_path.extend(portions)
- else:
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = namespace_path
- return spec
-
- @classmethod
- def find_spec(cls, fullname, path=None, target=None):
- """find the module on sys.path or 'path' based on sys.path_hooks and
- sys.path_importer_cache."""
- if path is None:
- path = sys.path
- spec = cls._get_spec(fullname, path, target)
- if spec is None:
- return None
- elif spec.loader is None:
- namespace_path = spec.submodule_search_locations
- if namespace_path:
- # We found at least one namespace path. Return a
- # spec which can create the namespace package.
- spec.origin = 'namespace'
- spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
- return spec
- else:
- return None
- else:
- return spec
-
- @classmethod
- def find_module(cls, fullname, path=None):
- """find the module on sys.path or 'path' based on sys.path_hooks and
- sys.path_importer_cache.
-
- This method is deprecated. Use find_spec() instead.
-
- """
- spec = cls.find_spec(fullname, path)
- if spec is None:
- return None
- return spec.loader
-
-
-class FileFinder:
-
- """File-based finder.
-
- Interactions with the file system are cached for performance, being
- refreshed when the directory the finder is handling has been modified.
-
- """
-
- def __init__(self, path, *loader_details):
- """Initialize with the path to search on and a variable number of
- 2-tuples containing the loader and the file suffixes the loader
- recognizes."""
- loaders = []
- for loader, suffixes in loader_details:
- loaders.extend((suffix, loader) for suffix in suffixes)
- self._loaders = loaders
- # Base (directory) path
- self.path = path or '.'
- self._path_mtime = -1
- self._path_cache = set()
- self._relaxed_path_cache = set()
-
- def invalidate_caches(self):
- """Invalidate the directory mtime."""
- self._path_mtime = -1
-
- find_module = _find_module_shim
-
- def find_loader(self, fullname):
- """Try to find a loader for the specified module, or the namespace
- package portions. Returns (loader, list-of-portions).
-
- This method is deprecated. Use find_spec() instead.
-
- """
- spec = self.find_spec(fullname)
- if spec is None:
- return None, []
- return spec.loader, spec.submodule_search_locations or []
-
- def _get_spec(self, loader_class, fullname, path, smsl, target):
- loader = loader_class(fullname, path)
- return spec_from_file_location(fullname, path, loader=loader,
- submodule_search_locations=smsl)
-
- def find_spec(self, fullname, target=None):
- """Try to find a loader for the specified module, or the namespace
- package portions. Returns (loader, list-of-portions)."""
- is_namespace = False
- tail_module = fullname.rpartition('.')[2]
- try:
- mtime = _path_stat(self.path or _os.getcwd()).st_mtime
- except OSError:
- mtime = -1
- if mtime != self._path_mtime:
- self._fill_cache()
- self._path_mtime = mtime
- # tail_module keeps the original casing, for __file__ and friends
- if _relax_case():
- cache = self._relaxed_path_cache
- cache_module = tail_module.lower()
- else:
- cache = self._path_cache
- cache_module = tail_module
- # Check if the module is the name of a directory (and thus a package).
- if cache_module in cache:
- base_path = _path_join(self.path, tail_module)
- for suffix, loader_class in self._loaders:
- init_filename = '__init__' + suffix
- full_path = _path_join(base_path, init_filename)
- if _path_isfile(full_path):
- return self._get_spec(loader_class, fullname, full_path, [base_path], target)
- else:
- # If a namespace package, return the path if we don't
- # find a module in the next section.
- is_namespace = _path_isdir(base_path)
- # Check for a file w/ a proper suffix exists.
- for suffix, loader_class in self._loaders:
- full_path = _path_join(self.path, tail_module + suffix)
- _verbose_message('trying {}'.format(full_path), verbosity=2)
- if cache_module + suffix in cache:
- if _path_isfile(full_path):
- return self._get_spec(loader_class, fullname, full_path, None, target)
- if is_namespace:
- _verbose_message('possible namespace for {}'.format(base_path))
- spec = ModuleSpec(fullname, None)
- spec.submodule_search_locations = [base_path]
- return spec
- return None
-
- def _fill_cache(self):
- """Fill the cache of potential modules and packages for this directory."""
- path = self.path
- try:
- contents = _os.listdir(path or _os.getcwd())
- except (FileNotFoundError, PermissionError, NotADirectoryError):
- # Directory has either been removed, turned into a file, or made
- # unreadable.
- contents = []
- # We store two cached versions, to handle runtime changes of the
- # PYTHONCASEOK environment variable.
- if not sys.platform.startswith('win'):
- self._path_cache = set(contents)
- else:
- # Windows users can import modules with case-insensitive file
- # suffixes (for legacy reasons). Make the suffix lowercase here
- # so it's done once instead of for every import. This is safe as
- # the specified suffixes to check against are always specified in a
- # case-sensitive manner.
- lower_suffix_contents = set()
- for item in contents:
- name, dot, suffix = item.partition('.')
- if dot:
- new_name = '{}.{}'.format(name, suffix.lower())
- else:
- new_name = name
- lower_suffix_contents.add(new_name)
- self._path_cache = lower_suffix_contents
- if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
- self._relaxed_path_cache = {fn.lower() for fn in contents}
-
- @classmethod
- def path_hook(cls, *loader_details):
- """A class method which returns a closure to use on sys.path_hook
- which will return an instance using the specified loaders and the path
- called on the closure.
-
- If the path called on the closure is not a directory, ImportError is
- raised.
-
- """
- def path_hook_for_FileFinder(path):
- """Path hook for importlib.machinery.FileFinder."""
- if not _path_isdir(path):
- raise ImportError('only directories are supported', path=path)
- return cls(path, *loader_details)
-
- return path_hook_for_FileFinder
-
- def __repr__(self):
- return 'FileFinder({!r})'.format(self.path)
-
-
# Import itself ###############################################################
class _ImportLockContext:
@@ -2305,17 +1029,6 @@ def _calc___package__(globals):
return package
-def _get_supported_file_loaders():
- """Returns a list of file-based module loaders.
-
- Each item is a tuple (loader, suffixes).
- """
- extensions = ExtensionFileLoader, _imp.extension_suffixes()
- source = SourceFileLoader, SOURCE_SUFFIXES
- bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
- return [extensions, source, bytecode]
-
-
def __import__(name, globals=None, locals=None, fromlist=(), level=0):
"""Import a module.
@@ -2385,34 +1098,13 @@ def _setup(sys_module, _imp_module):
# Directly load built-in modules needed during bootstrap.
self_module = sys.modules[__name__]
- for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+ for builtin_name in ('_warnings',):
if builtin_name not in sys.modules:
builtin_module = _builtin_from_name(builtin_name)
else:
builtin_module = sys.modules[builtin_name]
setattr(self_module, builtin_name, builtin_module)
- # Directly load the os module (needed during bootstrap).
- os_details = ('posix', ['/']), ('nt', ['\\', '/'])
- for builtin_os, path_separators in os_details:
- # Assumption made in _path_join()
- assert all(len(sep) == 1 for sep in path_separators)
- path_sep = path_separators[0]
- if builtin_os in sys.modules:
- os_module = sys.modules[builtin_os]
- break
- else:
- try:
- os_module = _builtin_from_name(builtin_os)
- break
- except ImportError:
- continue
- else:
- raise ImportError('importlib requires posix or nt')
- setattr(self_module, '_os', os_module)
- setattr(self_module, 'path_sep', path_sep)
- setattr(self_module, 'path_separators', ''.join(path_separators))
-
# Directly load the _thread module (needed during bootstrap).
try:
thread_module = _builtin_from_name('_thread')
@@ -2425,27 +1117,14 @@ def _setup(sys_module, _imp_module):
weakref_module = _builtin_from_name('_weakref')
setattr(self_module, '_weakref', weakref_module)
- # Directly load the winreg module (needed during bootstrap).
- if builtin_os == 'nt':
- winreg_module = _builtin_from_name('winreg')
- setattr(self_module, '_winreg', winreg_module)
-
- # Constants
- setattr(self_module, '_relax_case', _make_relax_case())
- EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
- if builtin_os == 'nt':
- SOURCE_SUFFIXES.append('.pyw')
- if '_d.pyd' in EXTENSION_SUFFIXES:
- WindowsRegistryFinder.DEBUG_BUILD = True
-
def _install(sys_module, _imp_module):
"""Install importlib as the implementation of import."""
_setup(sys_module, _imp_module)
- supported_loaders = _get_supported_file_loaders()
- sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
+
sys.meta_path.append(BuiltinImporter)
sys.meta_path.append(FrozenImporter)
- if _os.__name__ == 'nt':
- sys.meta_path.append(WindowsRegistryFinder)
- sys.meta_path.append(PathFinder)
+
+ import _frozen_importlib_external
+ _frozen_importlib_external._install(sys.modules[__name__])
+ sys.modules[__name__]._bootstrap_external = _frozen_importlib_external
diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py
new file mode 100644
index 0000000..9385d11
--- /dev/null
+++ b/Lib/importlib/_bootstrap_external.py
@@ -0,0 +1,1462 @@
+"""Core implementation of path-based import.
+
+This module is NOT meant to be directly imported! It has been designed such
+that it can be bootstrapped into Python as the implementation of import. As
+such it requires the injection of specific modules and attributes in order to
+work. One should use importlib as the public-facing version of this module.
+
+"""
+#
+# IMPORTANT: Whenever making changes to this module, be sure to run
+# a top-level make in order to get the frozen version of the module
+# updated. Not doing so will result in the Makefile to fail for
+# all others who don't have a ./python around to freeze the module
+# in the early stages of compilation.
+#
+
+# See importlib._setup() for what is injected into the global namespace.
+
+# When editing this code be aware that code executed at import time CANNOT
+# reference any injected objects! This includes not only global code but also
+# anything specified at the class level.
+
+# Bootstrap-related code ######################################################
+
+_CASE_INSENSITIVE_PLATFORMS = 'win', 'cygwin', 'darwin'
+
+
+def _make_relax_case():
+ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+ def _relax_case():
+ """True if filenames must be checked case-insensitively."""
+ return b'PYTHONCASEOK' in _os.environ
+ else:
+ def _relax_case():
+ """True if filenames must be checked case-insensitively."""
+ return False
+ return _relax_case
+
+
+def _w_long(x):
+ """Convert a 32-bit integer to little-endian."""
+ return (int(x) & 0xFFFFFFFF).to_bytes(4, 'little')
+
+
+def _r_long(int_bytes):
+ """Convert 4 bytes in little-endian to an integer."""
+ return int.from_bytes(int_bytes, 'little')
+
+
+def _path_join(*path_parts):
+ """Replacement for os.path.join()."""
+ return path_sep.join([part.rstrip(path_separators)
+ for part in path_parts if part])
+
+
+def _path_split(path):
+ """Replacement for os.path.split()."""
+ if len(path_separators) == 1:
+ front, _, tail = path.rpartition(path_sep)
+ return front, tail
+ for x in reversed(path):
+ if x in path_separators:
+ front, tail = path.rsplit(x, maxsplit=1)
+ return front, tail
+ return '', path
+
+
+def _path_stat(path):
+ """Stat the path.
+
+ Made a separate function to make it easier to override in experiments
+ (e.g. cache stat results).
+
+ """
+ return _os.stat(path)
+
+
+def _path_is_mode_type(path, mode):
+ """Test whether the path is the specified mode type."""
+ try:
+ stat_info = _path_stat(path)
+ except OSError:
+ return False
+ return (stat_info.st_mode & 0o170000) == mode
+
+
+def _path_isfile(path):
+ """Replacement for os.path.isfile."""
+ return _path_is_mode_type(path, 0o100000)
+
+
+def _path_isdir(path):
+ """Replacement for os.path.isdir."""
+ if not path:
+ path = _os.getcwd()
+ return _path_is_mode_type(path, 0o040000)
+
+
+def _write_atomic(path, data, mode=0o666):
+ """Best-effort function to write data to a path atomically.
+ Be prepared to handle a FileExistsError if concurrent writing of the
+ temporary file is attempted."""
+ # id() is used to generate a pseudo-random filename.
+ path_tmp = '{}.{}'.format(path, id(path))
+ fd = _os.open(path_tmp,
+ _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, mode & 0o666)
+ try:
+ # We first write data to a temporary file, and then use os.replace() to
+ # perform an atomic rename.
+ with _io.FileIO(fd, 'wb') as file:
+ file.write(data)
+ _os.replace(path_tmp, path)
+ except OSError:
+ try:
+ _os.unlink(path_tmp)
+ except OSError:
+ pass
+ raise
+
+
+_code_type = type(_write_atomic.__code__)
+
+
+# Finder/loader utility code ###############################################
+
+# Magic word to reject .pyc files generated by other Python versions.
+# It should change for each incompatible change to the bytecode.
+#
+# The value of CR and LF is incorporated so if you ever read or write
+# a .pyc file in text mode the magic number will be wrong; also, the
+# Apple MPW compiler swaps their values, botching string constants.
+#
+# The magic numbers must be spaced apart at least 2 values, as the
+# -U interpeter flag will cause MAGIC+1 being used. They have been
+# odd numbers for some time now.
+#
+# There were a variety of old schemes for setting the magic number.
+# The current working scheme is to increment the previous value by
+# 10.
+#
+# Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic
+# number also includes a new "magic tag", i.e. a human readable string used
+# to represent the magic number in __pycache__ directories. When you change
+# the magic number, you must also set a new unique magic tag. Generally this
+# can be named after the Python major version of the magic number bump, but
+# it can really be anything, as long as it's different than anything else
+# that's come before. The tags are included in the following table, starting
+# with Python 3.2a0.
+#
+# Known values:
+# Python 1.5: 20121
+# Python 1.5.1: 20121
+# Python 1.5.2: 20121
+# Python 1.6: 50428
+# Python 2.0: 50823
+# Python 2.0.1: 50823
+# Python 2.1: 60202
+# Python 2.1.1: 60202
+# Python 2.1.2: 60202
+# Python 2.2: 60717
+# Python 2.3a0: 62011
+# Python 2.3a0: 62021
+# Python 2.3a0: 62011 (!)
+# Python 2.4a0: 62041
+# Python 2.4a3: 62051
+# Python 2.4b1: 62061
+# Python 2.5a0: 62071
+# Python 2.5a0: 62081 (ast-branch)
+# Python 2.5a0: 62091 (with)
+# Python 2.5a0: 62092 (changed WITH_CLEANUP opcode)
+# Python 2.5b3: 62101 (fix wrong code: for x, in ...)
+# Python 2.5b3: 62111 (fix wrong code: x += yield)
+# Python 2.5c1: 62121 (fix wrong lnotab with for loops and
+# storing constants that should have been removed)
+# Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp)
+# Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode)
+# Python 2.6a1: 62161 (WITH_CLEANUP optimization)
+# Python 2.7a0: 62171 (optimize list comprehensions/change LIST_APPEND)
+# Python 2.7a0: 62181 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 2.7a0 62191 (introduce SETUP_WITH)
+# Python 2.7a0 62201 (introduce BUILD_SET)
+# Python 2.7a0 62211 (introduce MAP_ADD and SET_ADD)
+# Python 3000: 3000
+# 3010 (removed UNARY_CONVERT)
+# 3020 (added BUILD_SET)
+# 3030 (added keyword-only parameters)
+# 3040 (added signature annotations)
+# 3050 (print becomes a function)
+# 3060 (PEP 3115 metaclass syntax)
+# 3061 (string literals become unicode)
+# 3071 (PEP 3109 raise changes)
+# 3081 (PEP 3137 make __file__ and __name__ unicode)
+# 3091 (kill str8 interning)
+# 3101 (merge from 2.6a0, see 62151)
+# 3103 (__file__ points to source file)
+# Python 3.0a4: 3111 (WITH_CLEANUP optimization).
+# Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT)
+# Python 3.1a0: 3141 (optimize list, set and dict comprehensions:
+# change LIST_APPEND and SET_ADD, add MAP_ADD)
+# Python 3.1a0: 3151 (optimize conditional branches:
+# introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE)
+# Python 3.2a0: 3160 (add SETUP_WITH)
+# tag: cpython-32
+# Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR)
+# tag: cpython-32
+# Python 3.2a2 3180 (add DELETE_DEREF)
+# Python 3.3a0 3190 __class__ super closure changed
+# Python 3.3a0 3200 (__qualname__ added)
+# 3210 (added size modulo 2**32 to the pyc header)
+# Python 3.3a1 3220 (changed PEP 380 implementation)
+# Python 3.3a4 3230 (revert changes to implicit __class__ closure)
+# Python 3.4a1 3250 (evaluate positional default arguments before
+# keyword-only defaults)
+# Python 3.4a1 3260 (add LOAD_CLASSDEREF; allow locals of class to override
+# free vars)
+# Python 3.4a1 3270 (various tweaks to the __class__ closure)
+# Python 3.4a1 3280 (remove implicit class argument)
+# Python 3.4a4 3290 (changes to __qualname__ computation)
+# Python 3.4a4 3300 (more changes to __qualname__ computation)
+# Python 3.4rc2 3310 (alter __qualname__ computation)
+# Python 3.5a0 3320 (matrix multiplication operator)
+#
+# MAGIC must change whenever the bytecode emitted by the compiler may no
+# longer be understood by older implementations of the eval loop (usually
+# due to the addition of new opcodes).
+
+MAGIC_NUMBER = (3320).to_bytes(2, 'little') + b'\r\n'
+_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
+
+_PYCACHE = '__pycache__'
+_OPT = 'opt-'
+
+SOURCE_SUFFIXES = ['.py'] # _setup() adds .pyw as needed.
+
+BYTECODE_SUFFIXES = ['.pyc']
+# Deprecated.
+DEBUG_BYTECODE_SUFFIXES = OPTIMIZED_BYTECODE_SUFFIXES = BYTECODE_SUFFIXES
+
+def cache_from_source(path, debug_override=None, *, optimization=None):
+ """Given the path to a .py file, return the path to its .pyc file.
+
+ The .py file does not need to exist; this simply returns the path to the
+ .pyc file calculated as if the .py file were imported.
+
+ The 'optimization' parameter controls the presumed optimization level of
+ the bytecode file. If 'optimization' is not None, the string representation
+ of the argument is taken and verified to be alphanumeric (else ValueError
+ is raised).
+
+ The debug_override parameter is deprecated. If debug_override is not None,
+ a True value is the same as setting 'optimization' to the empty string
+ while a False value is equivalent to setting 'optimization' to '1'.
+
+ If sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ if debug_override is not None:
+ _warnings.warn('the debug_override parameter is deprecated; use '
+ "'optimization' instead", DeprecationWarning)
+ if optimization is not None:
+ message = 'debug_override or optimization must be set to None'
+ raise TypeError(message)
+ optimization = '' if debug_override else 1
+ head, tail = _path_split(path)
+ base, sep, rest = tail.rpartition('.')
+ tag = sys.implementation.cache_tag
+ if tag is None:
+ raise NotImplementedError('sys.implementation.cache_tag is None')
+ almost_filename = ''.join([(base if base else rest), sep, tag])
+ if optimization is None:
+ if sys.flags.optimize == 0:
+ optimization = ''
+ else:
+ optimization = sys.flags.optimize
+ optimization = str(optimization)
+ if optimization != '':
+ if not optimization.isalnum():
+ raise ValueError('{!r} is not alphanumeric'.format(optimization))
+ almost_filename = '{}.{}{}'.format(almost_filename, _OPT, optimization)
+ return _path_join(head, _PYCACHE, almost_filename + BYTECODE_SUFFIXES[0])
+
+
+def source_from_cache(path):
+ """Given the path to a .pyc. file, return the path to its .py file.
+
+ The .pyc file does not need to exist; this simply returns the path to
+ the .py file calculated to correspond to the .pyc file. If path does
+ not conform to PEP 3147/488 format, ValueError will be raised. If
+ sys.implementation.cache_tag is None then NotImplementedError is raised.
+
+ """
+ if sys.implementation.cache_tag is None:
+ raise NotImplementedError('sys.implementation.cache_tag is None')
+ head, pycache_filename = _path_split(path)
+ head, pycache = _path_split(head)
+ if pycache != _PYCACHE:
+ raise ValueError('{} not bottom-level directory in '
+ '{!r}'.format(_PYCACHE, path))
+ dot_count = pycache_filename.count('.')
+ if dot_count not in {2, 3}:
+ raise ValueError('expected only 2 or 3 dots in '
+ '{!r}'.format(pycache_filename))
+ elif dot_count == 3:
+ optimization = pycache_filename.rsplit('.', 2)[-2]
+ if not optimization.startswith(_OPT):
+ raise ValueError("optimization portion of filename does not start "
+ "with {!r}".format(_OPT))
+ opt_level = optimization[len(_OPT):]
+ if not opt_level.isalnum():
+ raise ValueError("optimization level {!r} is not an alphanumeric "
+ "value".format(optimization))
+ base_filename = pycache_filename.partition('.')[0]
+ return _path_join(head, base_filename + SOURCE_SUFFIXES[0])
+
+
+def _get_sourcefile(bytecode_path):
+ """Convert a bytecode file path to a source path (if possible).
+
+ This function exists purely for backwards-compatibility for
+ PyImport_ExecCodeModuleWithFilenames() in the C API.
+
+ """
+ if len(bytecode_path) == 0:
+ return None
+ rest, _, extension = bytecode_path.rpartition('.')
+ if not rest or extension.lower()[-3:-1] != 'py':
+ return bytecode_path
+ try:
+ source_path = source_from_cache(bytecode_path)
+ except (NotImplementedError, ValueError):
+ source_path = bytecode_path[:-1]
+ return source_path if _path_isfile(source_path) else bytecode_path
+
+
+def _get_cached(filename):
+ if filename.endswith(tuple(SOURCE_SUFFIXES)):
+ try:
+ return cache_from_source(filename)
+ except NotImplementedError:
+ pass
+ elif filename.endswith(tuple(BYTECODE_SUFFIXES)):
+ return filename
+ else:
+ return None
+
+
+def _calc_mode(path):
+ """Calculate the mode permissions for a bytecode file."""
+ try:
+ mode = _path_stat(path).st_mode
+ except OSError:
+ mode = 0o666
+ # We always ensure write access so we can update cached files
+ # later even when the source files are read-only on Windows (#6074)
+ mode |= 0o200
+ return mode
+
+
+def _verbose_message(message, *args, verbosity=1):
+ """Print the message to stderr if -v/PYTHONVERBOSE is turned on."""
+ if sys.flags.verbose >= verbosity:
+ if not message.startswith(('#', 'import ')):
+ message = '# ' + message
+ print(message.format(*args), file=sys.stderr)
+
+
+def _check_name(method):
+ """Decorator to verify that the module being requested matches the one the
+ loader can handle.
+
+ The first argument (self) must define _name which the second argument is
+ compared against. If the comparison fails then ImportError is raised.
+
+ """
+ def _check_name_wrapper(self, name=None, *args, **kwargs):
+ if name is None:
+ name = self.name
+ elif self.name != name:
+ raise ImportError('loader cannot handle %s' % name, name=name)
+ return method(self, name, *args, **kwargs)
+ try:
+ _wrap = _bootstrap._wrap
+ except NameError:
+ # XXX yuck
+ def _wrap(new, old):
+ for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
+ if hasattr(old, replace):
+ setattr(new, replace, getattr(old, replace))
+ new.__dict__.update(old.__dict__)
+ _wrap(_check_name_wrapper, method)
+ return _check_name_wrapper
+
+
+def _find_module_shim(self, fullname):
+ """Try to find a loader for the specified module by delegating to
+ self.find_loader().
+
+ This method is deprecated in favor of finder.find_spec().
+
+ """
+ # Call find_loader(). If it returns a string (indicating this
+ # is a namespace package portion), generate a warning and
+ # return None.
+ loader, portions = self.find_loader(fullname)
+ if loader is None and len(portions):
+ msg = 'Not importing directory {}: missing __init__'
+ _warnings.warn(msg.format(portions[0]), ImportWarning)
+ return loader
+
+
+# Typically used by loader classes as a method replacement.
+def _load_module_shim(self, fullname):
+ """Load the specified module into sys.modules and return it.
+
+ This method is deprecated. Use loader.exec_module instead.
+
+ """
+ spec = spec_from_loader(fullname, self)
+ if fullname in sys.modules:
+ module = sys.modules[fullname]
+ _bootstrap._exec(spec, module)
+ return sys.modules[fullname]
+ else:
+ return _bootstrap._load(spec)
+
+
+def _validate_bytecode_header(data, source_stats=None, name=None, path=None):
+ """Validate the header of the passed-in bytecode against source_stats (if
+ given) and returning the bytecode that can be compiled by compile().
+
+ All other arguments are used to enhance error reporting.
+
+ ImportError is raised when the magic number is incorrect or the bytecode is
+ found to be stale. EOFError is raised when the data is found to be
+ truncated.
+
+ """
+ exc_details = {}
+ if name is not None:
+ exc_details['name'] = name
+ else:
+ # To prevent having to make all messages have a conditional name.
+ name = '<bytecode>'
+ if path is not None:
+ exc_details['path'] = path
+ magic = data[:4]
+ raw_timestamp = data[4:8]
+ raw_size = data[8:12]
+ if magic != MAGIC_NUMBER:
+ message = 'bad magic number in {!r}: {!r}'.format(name, magic)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ elif len(raw_timestamp) != 4:
+ message = 'reached EOF while reading timestamp in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ elif len(raw_size) != 4:
+ message = 'reached EOF while reading size of source in {!r}'.format(name)
+ _verbose_message(message)
+ raise EOFError(message)
+ if source_stats is not None:
+ try:
+ source_mtime = int(source_stats['mtime'])
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_timestamp) != source_mtime:
+ message = 'bytecode is stale for {!r}'.format(name)
+ _verbose_message(message)
+ raise ImportError(message, **exc_details)
+ try:
+ source_size = source_stats['size'] & 0xFFFFFFFF
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_size) != source_size:
+ raise ImportError('bytecode is stale for {!r}'.format(name),
+ **exc_details)
+ return data[12:]
+
+
+def _compile_bytecode(data, name=None, bytecode_path=None, source_path=None):
+ """Compile bytecode as returned by _validate_bytecode_header()."""
+ code = marshal.loads(data)
+ if isinstance(code, _code_type):
+ _verbose_message('code object from {!r}', bytecode_path)
+ if source_path is not None:
+ _imp._fix_co_filename(code, source_path)
+ return code
+ else:
+ raise ImportError('Non-code object in {!r}'.format(bytecode_path),
+ name=name, path=bytecode_path)
+
+def _code_to_bytecode(code, mtime=0, source_size=0):
+ """Compile a code object into bytecode for writing out to a byte-compiled
+ file."""
+ data = bytearray(MAGIC_NUMBER)
+ data.extend(_w_long(mtime))
+ data.extend(_w_long(source_size))
+ data.extend(marshal.dumps(code))
+ return data
+
+
+def decode_source(source_bytes):
+ """Decode bytes representing source code and return the string.
+
+ Universal newline support is used in the decoding.
+ """
+ import tokenize # To avoid bootstrap issues.
+ source_bytes_readline = _io.BytesIO(source_bytes).readline
+ encoding = tokenize.detect_encoding(source_bytes_readline)
+ newline_decoder = _io.IncrementalNewlineDecoder(None, True)
+ return newline_decoder.decode(source_bytes.decode(encoding[0]))
+
+
+# Module specifications #######################################################
+
+def spec_from_loader(name, loader, *, origin=None, is_package=None):
+ """Return a module spec based on various loader methods."""
+ if hasattr(loader, 'get_filename'):
+ if is_package is None:
+ return spec_from_file_location(name, loader=loader)
+ search = [] if is_package else None
+ return spec_from_file_location(name, loader=loader,
+ submodule_search_locations=search)
+
+ if is_package is None:
+ if hasattr(loader, 'is_package'):
+ try:
+ is_package = loader.is_package(name)
+ except ImportError:
+ is_package = None # aka, undefined
+ else:
+ # the default
+ is_package = False
+
+ return _bootstrap.ModuleSpec(name, loader, origin=origin, is_package=is_package)
+
+
+_POPULATE = object()
+
+
+def spec_from_file_location(name, location=None, *, loader=None,
+ submodule_search_locations=_POPULATE):
+ """Return a module spec based on a file location.
+
+ To indicate that the module is a package, set
+ submodule_search_locations to a list of directory paths. An
+ empty list is sufficient, though its not otherwise useful to the
+ import system.
+
+ The loader must take a spec as its only __init__() arg.
+
+ """
+ if location is None:
+ # The caller may simply want a partially populated location-
+ # oriented spec. So we set the location to a bogus value and
+ # fill in as much as we can.
+ location = '<unknown>'
+ if hasattr(loader, 'get_filename'):
+ # ExecutionLoader
+ try:
+ location = loader.get_filename(name)
+ except ImportError:
+ pass
+
+ # If the location is on the filesystem, but doesn't actually exist,
+ # we could return None here, indicating that the location is not
+ # valid. However, we don't have a good way of testing since an
+ # indirect location (e.g. a zip file or URL) will look like a
+ # non-existent file relative to the filesystem.
+
+ spec = _bootstrap.ModuleSpec(name, loader, origin=location)
+ spec._set_fileattr = True
+
+ # Pick a loader if one wasn't provided.
+ if loader is None:
+ for loader_class, suffixes in _get_supported_file_loaders():
+ if location.endswith(tuple(suffixes)):
+ loader = loader_class(name, location)
+ spec.loader = loader
+ break
+ else:
+ return None
+
+ # Set submodule_search_paths appropriately.
+ if submodule_search_locations is _POPULATE:
+ # Check the loader.
+ if hasattr(loader, 'is_package'):
+ try:
+ is_package = loader.is_package(name)
+ except ImportError:
+ pass
+ else:
+ if is_package:
+ spec.submodule_search_locations = []
+ else:
+ spec.submodule_search_locations = submodule_search_locations
+ if spec.submodule_search_locations == []:
+ if location:
+ dirname = _path_split(location)[0]
+ spec.submodule_search_locations.append(dirname)
+
+ return spec
+
+
+# Loaders #####################################################################
+
+class WindowsRegistryFinder:
+
+ """Meta path finder for modules declared in the Windows registry."""
+
+ REGISTRY_KEY = (
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}')
+ REGISTRY_KEY_DEBUG = (
+ 'Software\\Python\\PythonCore\\{sys_version}'
+ '\\Modules\\{fullname}\\Debug')
+ DEBUG_BUILD = False # Changed in _setup()
+
+ @classmethod
+ def _open_registry(cls, key):
+ try:
+ return _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, key)
+ except OSError:
+ return _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, key)
+
+ @classmethod
+ def _search_registry(cls, fullname):
+ if cls.DEBUG_BUILD:
+ registry_key = cls.REGISTRY_KEY_DEBUG
+ else:
+ registry_key = cls.REGISTRY_KEY
+ key = registry_key.format(fullname=fullname,
+ sys_version=sys.version[:3])
+ try:
+ with cls._open_registry(key) as hkey:
+ filepath = _winreg.QueryValue(hkey, '')
+ except OSError:
+ return None
+ return filepath
+
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ filepath = cls._search_registry(fullname)
+ if filepath is None:
+ return None
+ try:
+ _path_stat(filepath)
+ except OSError:
+ return None
+ for loader, suffixes in _get_supported_file_loaders():
+ if filepath.endswith(tuple(suffixes)):
+ spec = spec_from_loader(fullname, loader(fullname, filepath),
+ origin=filepath)
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """Find module named in the registry.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is not None:
+ return spec.loader
+ else:
+ return None
+
+
+class _LoaderBasics:
+
+ """Base class of common code needed by both SourceLoader and
+ SourcelessFileLoader."""
+
+ def is_package(self, fullname):
+ """Concrete implementation of InspectLoader.is_package by checking if
+ the path returned by get_filename has a filename of '__init__.py'."""
+ filename = _path_split(self.get_filename(fullname))[1]
+ filename_base = filename.rsplit('.', 1)[0]
+ tail_name = fullname.rpartition('.')[2]
+ return filename_base == '__init__' and tail_name != '__init__'
+
+ def create_module(self, spec):
+ """Use default semantics for module creation."""
+
+ def exec_module(self, module):
+ """Execute the module."""
+ code = self.get_code(module.__name__)
+ if code is None:
+ raise ImportError('cannot load module {!r} when get_code() '
+ 'returns None'.format(module.__name__))
+ _bootstrap._call_with_frames_removed(exec, code, module.__dict__)
+
+ load_module = _load_module_shim
+
+
+class SourceLoader(_LoaderBasics):
+
+ def path_mtime(self, path):
+ """Optional method that returns the modification time (an int) for the
+ specified path, where path is a str.
+
+ Raises IOError when the path cannot be handled.
+ """
+ raise IOError
+
+ def path_stats(self, path):
+ """Optional method returning a metadata dict for the specified path
+ to by the path (str).
+ Possible keys:
+ - 'mtime' (mandatory) is the numeric timestamp of last source
+ code modification;
+ - 'size' (optional) is the size in bytes of the source code.
+
+ Implementing this method allows the loader to read bytecode files.
+ Raises IOError when the path cannot be handled.
+ """
+ return {'mtime': self.path_mtime(path)}
+
+ def _cache_bytecode(self, source_path, cache_path, data):
+ """Optional method which writes data (bytes) to a file path (a str).
+
+ Implementing this method allows for the writing of bytecode files.
+
+ The source path is needed in order to correctly transfer permissions
+ """
+ # For backwards compatibility, we delegate to set_data()
+ return self.set_data(cache_path, data)
+
+ def set_data(self, path, data):
+ """Optional method which writes data (bytes) to a file path (a str).
+
+ Implementing this method allows for the writing of bytecode files.
+ """
+
+
+ def get_source(self, fullname):
+ """Concrete implementation of InspectLoader.get_source."""
+ path = self.get_filename(fullname)
+ try:
+ source_bytes = self.get_data(path)
+ except OSError as exc:
+ raise ImportError('source not available through get_data()',
+ name=fullname) from exc
+ return decode_source(source_bytes)
+
+ def source_to_code(self, data, path, *, _optimize=-1):
+ """Return the code object compiled from source.
+
+ The 'data' argument can be any object type that compile() supports.
+ """
+ return _bootstrap._call_with_frames_removed(compile, data, path, 'exec',
+ dont_inherit=True, optimize=_optimize)
+
+ def get_code(self, fullname):
+ """Concrete implementation of InspectLoader.get_code.
+
+ Reading of bytecode requires path_stats to be implemented. To write
+ bytecode, set_data must also be implemented.
+
+ """
+ source_path = self.get_filename(fullname)
+ source_mtime = None
+ try:
+ bytecode_path = cache_from_source(source_path)
+ except NotImplementedError:
+ bytecode_path = None
+ else:
+ try:
+ st = self.path_stats(source_path)
+ except IOError:
+ pass
+ else:
+ source_mtime = int(st['mtime'])
+ try:
+ data = self.get_data(bytecode_path)
+ except OSError:
+ pass
+ else:
+ try:
+ bytes_data = _validate_bytecode_header(data,
+ source_stats=st, name=fullname,
+ path=bytecode_path)
+ except (ImportError, EOFError):
+ pass
+ else:
+ _verbose_message('{} matches {}', bytecode_path,
+ source_path)
+ return _compile_bytecode(bytes_data, name=fullname,
+ bytecode_path=bytecode_path,
+ source_path=source_path)
+ source_bytes = self.get_data(source_path)
+ code_object = self.source_to_code(source_bytes, source_path)
+ _verbose_message('code object from {}', source_path)
+ if (not sys.dont_write_bytecode and bytecode_path is not None and
+ source_mtime is not None):
+ data = _code_to_bytecode(code_object, source_mtime,
+ len(source_bytes))
+ try:
+ self._cache_bytecode(source_path, bytecode_path, data)
+ _verbose_message('wrote {!r}', bytecode_path)
+ except NotImplementedError:
+ pass
+ return code_object
+
+
+class FileLoader:
+
+ """Base file loader class which implements the loader protocol methods that
+ require file system usage."""
+
+ def __init__(self, fullname, path):
+ """Cache the module name and the path to the file found by the
+ finder."""
+ self.name = fullname
+ self.path = path
+
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
+ @_check_name
+ def load_module(self, fullname):
+ """Load a module from a file.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The only reason for this method is for the name check.
+ # Issue #14857: Avoid the zero-argument form of super so the implementation
+ # of that form can be updated without breaking the frozen module
+ return super(FileLoader, self).load_module(fullname)
+
+ @_check_name
+ def get_filename(self, fullname):
+ """Return the path to the source file as found by the finder."""
+ return self.path
+
+ def get_data(self, path):
+ """Return the data from path as raw bytes."""
+ with _io.FileIO(path, 'r') as file:
+ return file.read()
+
+
+class SourceFileLoader(FileLoader, SourceLoader):
+
+ """Concrete implementation of SourceLoader using the file system."""
+
+ def path_stats(self, path):
+ """Return the metadata for the path."""
+ st = _path_stat(path)
+ return {'mtime': st.st_mtime, 'size': st.st_size}
+
+ def _cache_bytecode(self, source_path, bytecode_path, data):
+ # Adapt between the two APIs
+ mode = _calc_mode(source_path)
+ return self.set_data(bytecode_path, data, _mode=mode)
+
+ def set_data(self, path, data, *, _mode=0o666):
+ """Write bytes data to a file."""
+ parent, filename = _path_split(path)
+ path_parts = []
+ # Figure out what directories are missing.
+ while parent and not _path_isdir(parent):
+ parent, part = _path_split(parent)
+ path_parts.append(part)
+ # Create needed directories.
+ for part in reversed(path_parts):
+ parent = _path_join(parent, part)
+ try:
+ _os.mkdir(parent)
+ except FileExistsError:
+ # Probably another Python process already created the dir.
+ continue
+ except OSError as exc:
+ # Could be a permission error, read-only filesystem: just forget
+ # about writing the data.
+ _verbose_message('could not create {!r}: {!r}', parent, exc)
+ return
+ try:
+ _write_atomic(path, data, _mode)
+ _verbose_message('created {!r}', path)
+ except OSError as exc:
+ # Same as above: just don't write the bytecode.
+ _verbose_message('could not create {!r}: {!r}', path, exc)
+
+
+class SourcelessFileLoader(FileLoader, _LoaderBasics):
+
+ """Loader which handles sourceless file imports."""
+
+ def get_code(self, fullname):
+ path = self.get_filename(fullname)
+ data = self.get_data(path)
+ bytes_data = _validate_bytecode_header(data, name=fullname, path=path)
+ return _compile_bytecode(bytes_data, name=fullname, bytecode_path=path)
+
+ def get_source(self, fullname):
+ """Return None as there is no source code."""
+ return None
+
+
+# Filled in by _setup().
+EXTENSION_SUFFIXES = []
+
+
+class ExtensionFileLoader:
+
+ """Loader for extension modules.
+
+ The constructor is designed to work with FileFinder.
+
+ """
+
+ def __init__(self, name, path):
+ self.name = name
+ self.path = path
+
+ def __eq__(self, other):
+ return (self.__class__ == other.__class__ and
+ self.__dict__ == other.__dict__)
+
+ def __hash__(self):
+ return hash(self.name) ^ hash(self.path)
+
+ @_check_name
+ def load_module(self, fullname):
+ """Load an extension module."""
+ # Once an exec_module() implementation is added we can also
+ # add a deprecation warning here.
+ with _bootstrap._ManageReload(fullname):
+ module = _bootstrap._call_with_frames_removed(_imp.load_dynamic,
+ fullname, self.path)
+ _verbose_message('extension module loaded from {!r}', self.path)
+ is_package = self.is_package(fullname)
+ if is_package and not hasattr(module, '__path__'):
+ module.__path__ = [_path_split(self.path)[0]]
+ module.__loader__ = self
+ module.__package__ = module.__name__
+ if not is_package:
+ module.__package__ = module.__package__.rpartition('.')[0]
+ return module
+
+ def is_package(self, fullname):
+ """Return True if the extension module is a package."""
+ file_name = _path_split(self.path)[1]
+ return any(file_name == '__init__' + suffix
+ for suffix in EXTENSION_SUFFIXES)
+
+ def get_code(self, fullname):
+ """Return None as an extension module cannot create a code object."""
+ return None
+
+ def get_source(self, fullname):
+ """Return None as extension modules have no source code."""
+ return None
+
+ @_check_name
+ def get_filename(self, fullname):
+ """Return the path to the source file as found by the finder."""
+ return self.path
+
+
+class _NamespacePath:
+ """Represents a namespace package's path. It uses the module name
+ to find its parent module, and from there it looks up the parent's
+ __path__. When this changes, the module's own path is recomputed,
+ using path_finder. For top-level modules, the parent module's path
+ is sys.path."""
+
+ def __init__(self, name, path, path_finder):
+ self._name = name
+ self._path = path
+ self._last_parent_path = tuple(self._get_parent_path())
+ self._path_finder = path_finder
+
+ def _find_parent_path_names(self):
+ """Returns a tuple of (parent-module-name, parent-path-attr-name)"""
+ parent, dot, me = self._name.rpartition('.')
+ if dot == '':
+ # This is a top-level module. sys.path contains the parent path.
+ return 'sys', 'path'
+ # Not a top-level module. parent-module.__path__ contains the
+ # parent path.
+ return parent, '__path__'
+
+ def _get_parent_path(self):
+ parent_module_name, path_attr_name = self._find_parent_path_names()
+ return getattr(sys.modules[parent_module_name], path_attr_name)
+
+ def _recalculate(self):
+ # If the parent's path has changed, recalculate _path
+ parent_path = tuple(self._get_parent_path()) # Make a copy
+ if parent_path != self._last_parent_path:
+ spec = self._path_finder(self._name, parent_path)
+ # Note that no changes are made if a loader is returned, but we
+ # do remember the new parent path
+ if spec is not None and spec.loader is None:
+ if spec.submodule_search_locations:
+ self._path = spec.submodule_search_locations
+ self._last_parent_path = parent_path # Save the copy
+ return self._path
+
+ def __iter__(self):
+ return iter(self._recalculate())
+
+ def __len__(self):
+ return len(self._recalculate())
+
+ def __repr__(self):
+ return '_NamespacePath({!r})'.format(self._path)
+
+ def __contains__(self, item):
+ return item in self._recalculate()
+
+ def append(self, item):
+ self._path.append(item)
+
+
+# We use this exclusively in module_from_spec() for backward-compatibility.
+class _NamespaceLoader:
+ def __init__(self, name, path, path_finder):
+ self._path = _NamespacePath(name, path, path_finder)
+
+ @classmethod
+ def module_repr(cls, module):
+ """Return repr for the module.
+
+ The method is deprecated. The import machinery does the job itself.
+
+ """
+ return '<module {!r} (namespace)>'.format(module.__name__)
+
+ def is_package(self, fullname):
+ return True
+
+ def get_source(self, fullname):
+ return ''
+
+ def get_code(self, fullname):
+ return compile('', '<string>', 'exec', dont_inherit=True)
+
+ def create_module(self, spec):
+ """Use default semantics for module creation."""
+
+ def exec_module(self, module):
+ pass
+
+ def load_module(self, fullname):
+ """Load a namespace module.
+
+ This method is deprecated. Use exec_module() instead.
+
+ """
+ # The import system never calls this method.
+ _verbose_message('namespace module loaded with path {!r}', self._path)
+ return _load_module_shim(self, fullname)
+
+
+# Finders #####################################################################
+
+class PathFinder:
+
+ """Meta path finder for sys.path and package __path__ attributes."""
+
+ @classmethod
+ def invalidate_caches(cls):
+ """Call the invalidate_caches() method on all path entry finders
+ stored in sys.path_importer_caches (where implemented)."""
+ for finder in sys.path_importer_cache.values():
+ if hasattr(finder, 'invalidate_caches'):
+ finder.invalidate_caches()
+
+ @classmethod
+ def _path_hooks(cls, path):
+ """Search sequence of hooks for a finder for 'path'.
+
+ If 'hooks' is false then use sys.path_hooks.
+
+ """
+ if sys.path_hooks is not None and not sys.path_hooks:
+ _warnings.warn('sys.path_hooks is empty', ImportWarning)
+ for hook in sys.path_hooks:
+ try:
+ return hook(path)
+ except ImportError:
+ continue
+ else:
+ return None
+
+ @classmethod
+ def _path_importer_cache(cls, path):
+ """Get the finder for the path entry from sys.path_importer_cache.
+
+ If the path entry is not in the cache, find the appropriate finder
+ and cache it. If no finder is available, store None.
+
+ """
+ if path == '':
+ try:
+ path = _os.getcwd()
+ except FileNotFoundError:
+ # Don't cache the failure as the cwd can easily change to
+ # a valid directory later on.
+ return None
+ try:
+ finder = sys.path_importer_cache[path]
+ except KeyError:
+ finder = cls._path_hooks(path)
+ sys.path_importer_cache[path] = finder
+ return finder
+
+ @classmethod
+ def _legacy_get_spec(cls, fullname, finder):
+ # This would be a good place for a DeprecationWarning if
+ # we ended up going that route.
+ if hasattr(finder, 'find_loader'):
+ loader, portions = finder.find_loader(fullname)
+ else:
+ loader = finder.find_module(fullname)
+ portions = []
+ if loader is not None:
+ return spec_from_loader(fullname, loader)
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = portions
+ return spec
+
+ @classmethod
+ def _get_spec(cls, fullname, path, target=None):
+ """Find the loader or namespace_path for this module/package name."""
+ # If this ends up being a namespace package, namespace_path is
+ # the list of paths that will become its __path__
+ namespace_path = []
+ for entry in path:
+ if not isinstance(entry, (str, bytes)):
+ continue
+ finder = cls._path_importer_cache(entry)
+ if finder is not None:
+ if hasattr(finder, 'find_spec'):
+ spec = finder.find_spec(fullname, target)
+ else:
+ spec = cls._legacy_get_spec(fullname, finder)
+ if spec is None:
+ continue
+ if spec.loader is not None:
+ return spec
+ portions = spec.submodule_search_locations
+ if portions is None:
+ raise ImportError('spec missing loader')
+ # This is possibly part of a namespace package.
+ # Remember these path entries (if any) for when we
+ # create a namespace package, and continue iterating
+ # on path.
+ namespace_path.extend(portions)
+ else:
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = namespace_path
+ return spec
+
+ @classmethod
+ def find_spec(cls, fullname, path=None, target=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
+ sys.path_importer_cache."""
+ if path is None:
+ path = sys.path
+ spec = cls._get_spec(fullname, path, target)
+ if spec is None:
+ return None
+ elif spec.loader is None:
+ namespace_path = spec.submodule_search_locations
+ if namespace_path:
+ # We found at least one namespace path. Return a
+ # spec which can create the namespace package.
+ spec.origin = 'namespace'
+ spec.submodule_search_locations = _NamespacePath(fullname, namespace_path, cls._get_spec)
+ return spec
+ else:
+ return None
+ else:
+ return spec
+
+ @classmethod
+ def find_module(cls, fullname, path=None):
+ """find the module on sys.path or 'path' based on sys.path_hooks and
+ sys.path_importer_cache.
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = cls.find_spec(fullname, path)
+ if spec is None:
+ return None
+ return spec.loader
+
+
+class FileFinder:
+
+ """File-based finder.
+
+ Interactions with the file system are cached for performance, being
+ refreshed when the directory the finder is handling has been modified.
+
+ """
+
+ def __init__(self, path, *loader_details):
+ """Initialize with the path to search on and a variable number of
+ 2-tuples containing the loader and the file suffixes the loader
+ recognizes."""
+ loaders = []
+ for loader, suffixes in loader_details:
+ loaders.extend((suffix, loader) for suffix in suffixes)
+ self._loaders = loaders
+ # Base (directory) path
+ self.path = path or '.'
+ self._path_mtime = -1
+ self._path_cache = set()
+ self._relaxed_path_cache = set()
+
+ def invalidate_caches(self):
+ """Invalidate the directory mtime."""
+ self._path_mtime = -1
+
+ find_module = _find_module_shim
+
+ def find_loader(self, fullname):
+ """Try to find a loader for the specified module, or the namespace
+ package portions. Returns (loader, list-of-portions).
+
+ This method is deprecated. Use find_spec() instead.
+
+ """
+ spec = self.find_spec(fullname)
+ if spec is None:
+ return None, []
+ return spec.loader, spec.submodule_search_locations or []
+
+ def _get_spec(self, loader_class, fullname, path, smsl, target):
+ loader = loader_class(fullname, path)
+ return spec_from_file_location(fullname, path, loader=loader,
+ submodule_search_locations=smsl)
+
+ def find_spec(self, fullname, target=None):
+ """Try to find a loader for the specified module, or the namespace
+ package portions. Returns (loader, list-of-portions)."""
+ is_namespace = False
+ tail_module = fullname.rpartition('.')[2]
+ try:
+ mtime = _path_stat(self.path or _os.getcwd()).st_mtime
+ except OSError:
+ mtime = -1
+ if mtime != self._path_mtime:
+ self._fill_cache()
+ self._path_mtime = mtime
+ # tail_module keeps the original casing, for __file__ and friends
+ if _relax_case():
+ cache = self._relaxed_path_cache
+ cache_module = tail_module.lower()
+ else:
+ cache = self._path_cache
+ cache_module = tail_module
+ # Check if the module is the name of a directory (and thus a package).
+ if cache_module in cache:
+ base_path = _path_join(self.path, tail_module)
+ for suffix, loader_class in self._loaders:
+ init_filename = '__init__' + suffix
+ full_path = _path_join(base_path, init_filename)
+ if _path_isfile(full_path):
+ return self._get_spec(loader_class, fullname, full_path, [base_path], target)
+ else:
+ # If a namespace package, return the path if we don't
+ # find a module in the next section.
+ is_namespace = _path_isdir(base_path)
+ # Check for a file w/ a proper suffix exists.
+ for suffix, loader_class in self._loaders:
+ full_path = _path_join(self.path, tail_module + suffix)
+ _verbose_message('trying {}'.format(full_path), verbosity=2)
+ if cache_module + suffix in cache:
+ if _path_isfile(full_path):
+ return self._get_spec(loader_class, fullname, full_path, None, target)
+ if is_namespace:
+ _verbose_message('possible namespace for {}'.format(base_path))
+ spec = _bootstrap.ModuleSpec(fullname, None)
+ spec.submodule_search_locations = [base_path]
+ return spec
+ return None
+
+ def _fill_cache(self):
+ """Fill the cache of potential modules and packages for this directory."""
+ path = self.path
+ try:
+ contents = _os.listdir(path or _os.getcwd())
+ except (FileNotFoundError, PermissionError, NotADirectoryError):
+ # Directory has either been removed, turned into a file, or made
+ # unreadable.
+ contents = []
+ # We store two cached versions, to handle runtime changes of the
+ # PYTHONCASEOK environment variable.
+ if not sys.platform.startswith('win'):
+ self._path_cache = set(contents)
+ else:
+ # Windows users can import modules with case-insensitive file
+ # suffixes (for legacy reasons). Make the suffix lowercase here
+ # so it's done once instead of for every import. This is safe as
+ # the specified suffixes to check against are always specified in a
+ # case-sensitive manner.
+ lower_suffix_contents = set()
+ for item in contents:
+ name, dot, suffix = item.partition('.')
+ if dot:
+ new_name = '{}.{}'.format(name, suffix.lower())
+ else:
+ new_name = name
+ lower_suffix_contents.add(new_name)
+ self._path_cache = lower_suffix_contents
+ if sys.platform.startswith(_CASE_INSENSITIVE_PLATFORMS):
+ self._relaxed_path_cache = {fn.lower() for fn in contents}
+
+ @classmethod
+ def path_hook(cls, *loader_details):
+ """A class method which returns a closure to use on sys.path_hook
+ which will return an instance using the specified loaders and the path
+ called on the closure.
+
+ If the path called on the closure is not a directory, ImportError is
+ raised.
+
+ """
+ def path_hook_for_FileFinder(path):
+ """Path hook for importlib.machinery.FileFinder."""
+ if not _path_isdir(path):
+ raise ImportError('only directories are supported', path=path)
+ return cls(path, *loader_details)
+
+ return path_hook_for_FileFinder
+
+ def __repr__(self):
+ return 'FileFinder({!r})'.format(self.path)
+
+
+# Import setup ###############################################################
+
+def _fix_up_module(ns, name, pathname, cpathname=None):
+ # This function is used by PyImport_ExecCodeModuleObject().
+ loader = ns.get('__loader__')
+ spec = ns.get('__spec__')
+ if not loader:
+ if spec:
+ loader = spec.loader
+ elif pathname == cpathname:
+ loader = SourcelessFileLoader(name, pathname)
+ else:
+ loader = SourceFileLoader(name, pathname)
+ if not spec:
+ spec = spec_from_file_location(name, pathname, loader=loader)
+ try:
+ ns['__spec__'] = spec
+ ns['__loader__'] = loader
+ ns['__file__'] = pathname
+ ns['__cached__'] = cpathname
+ except Exception:
+ # Not important enough to report.
+ pass
+
+
+def _get_supported_file_loaders():
+ """Returns a list of file-based module loaders.
+
+ Each item is a tuple (loader, suffixes).
+ """
+ extensions = ExtensionFileLoader, _imp.extension_suffixes()
+ source = SourceFileLoader, SOURCE_SUFFIXES
+ bytecode = SourcelessFileLoader, BYTECODE_SUFFIXES
+ return [extensions, source, bytecode]
+
+
+def _setup(_bootstrap_module):
+ """Setup the path-based importers for importlib by importing needed
+ built-in modules and injecting them into the global namespace.
+
+ Other components are extracted from the core bootstrap module.
+
+ """
+ global sys, _imp, _bootstrap
+ _bootstrap = _bootstrap_module
+ sys = _bootstrap.sys
+ _imp = _bootstrap._imp
+
+ # Directly load built-in modules needed during bootstrap.
+ self_module = sys.modules[__name__]
+ for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+ if builtin_name not in sys.modules:
+ builtin_module = _bootstrap._builtin_from_name(builtin_name)
+ else:
+ builtin_module = sys.modules[builtin_name]
+ setattr(self_module, builtin_name, builtin_module)
+
+ # Directly load the os module (needed during bootstrap).
+ os_details = ('posix', ['/']), ('nt', ['\\', '/'])
+ for builtin_os, path_separators in os_details:
+ # Assumption made in _path_join()
+ assert all(len(sep) == 1 for sep in path_separators)
+ path_sep = path_separators[0]
+ if builtin_os in sys.modules:
+ os_module = sys.modules[builtin_os]
+ break
+ else:
+ try:
+ os_module = _bootstrap._builtin_from_name(builtin_os)
+ break
+ except ImportError:
+ continue
+ else:
+ raise ImportError('importlib requires posix or nt')
+ setattr(self_module, '_os', os_module)
+ setattr(self_module, 'path_sep', path_sep)
+ setattr(self_module, 'path_separators', ''.join(path_separators))
+
+ # Directly load the _thread module (needed during bootstrap).
+ try:
+ thread_module = _bootstrap._builtin_from_name('_thread')
+ except ImportError:
+ # Python was built without threads
+ thread_module = None
+ setattr(self_module, '_thread', thread_module)
+
+ # Directly load the _weakref module (needed during bootstrap).
+ weakref_module = _bootstrap._builtin_from_name('_weakref')
+ setattr(self_module, '_weakref', weakref_module)
+
+ # Directly load the winreg module (needed during bootstrap).
+ if builtin_os == 'nt':
+ winreg_module = _bootstrap._builtin_from_name('winreg')
+ setattr(self_module, '_winreg', winreg_module)
+
+ # Constants
+ setattr(self_module, '_relax_case', _make_relax_case())
+ EXTENSION_SUFFIXES.extend(_imp.extension_suffixes())
+ if builtin_os == 'nt':
+ SOURCE_SUFFIXES.append('.pyw')
+ if '_d.pyd' in EXTENSION_SUFFIXES:
+ WindowsRegistryFinder.DEBUG_BUILD = True
+
+
+def _install(_bootstrap_module):
+ """Install the path-based import components."""
+ _setup(_bootstrap_module)
+ supported_loaders = _get_supported_file_loaders()
+ sys.path_hooks.extend([FileFinder.path_hook(*supported_loaders)])
+ if _os.__name__ == 'nt':
+ sys.meta_path.append(WindowsRegistryFinder)
+ sys.meta_path.append(PathFinder)
+
+ # XXX We expose a couple of classes in _bootstrap for the sake of
+ # a setuptools bug (https://bitbucket.org/pypa/setuptools/issue/378).
+ _bootstrap_module.FileFinder = FileFinder
+ _bootstrap_module.SourceFileLoader = SourceFileLoader
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index 2878488..d1f0364 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -1,12 +1,17 @@
"""Abstract base classes related to import."""
-from . import _bootstrap
+from . import _bootstrap_external
from . import machinery
try:
import _frozen_importlib
+# import _frozen_importlib_external
except ImportError as exc:
if exc.name != '_frozen_importlib':
raise
_frozen_importlib = None
+try:
+ import _frozen_importlib_external
+except ImportError as exc:
+ _frozen_importlib_external = _bootstrap_external
import abc
@@ -14,7 +19,10 @@ def _register(abstract_cls, *classes):
for cls in classes:
abstract_cls.register(cls)
if _frozen_importlib is not None:
- frozen_cls = getattr(_frozen_importlib, cls.__name__)
+ try:
+ frozen_cls = getattr(_frozen_importlib, cls.__name__)
+ except AttributeError:
+ frozen_cls = getattr(_frozen_importlib_external, cls.__name__)
abstract_cls.register(frozen_cls)
@@ -102,7 +110,7 @@ class PathEntryFinder(Finder):
else:
return None, []
- find_module = _bootstrap._find_module_shim
+ find_module = _bootstrap_external._find_module_shim
def invalidate_caches(self):
"""An optional method for clearing the finder's cache, if any.
@@ -144,7 +152,7 @@ class Loader(metaclass=abc.ABCMeta):
"""
if not hasattr(self, 'exec_module'):
raise ImportError
- return _bootstrap._load_module_shim(self, fullname)
+ return _bootstrap_external._load_module_shim(self, fullname)
def module_repr(self, module):
"""Return a module's repr.
@@ -222,8 +230,8 @@ class InspectLoader(Loader):
argument should be where the data was retrieved (when applicable)."""
return compile(data, path, 'exec', dont_inherit=True)
- exec_module = _bootstrap._LoaderBasics.exec_module
- load_module = _bootstrap._LoaderBasics.load_module
+ exec_module = _bootstrap_external._LoaderBasics.exec_module
+ load_module = _bootstrap_external._LoaderBasics.load_module
_register(InspectLoader, machinery.BuiltinImporter, machinery.FrozenImporter)
@@ -265,7 +273,7 @@ class ExecutionLoader(InspectLoader):
_register(ExecutionLoader, machinery.ExtensionFileLoader)
-class FileLoader(_bootstrap.FileLoader, ResourceLoader, ExecutionLoader):
+class FileLoader(_bootstrap_external.FileLoader, ResourceLoader, ExecutionLoader):
"""Abstract base class partially implementing the ResourceLoader and
ExecutionLoader ABCs."""
@@ -274,7 +282,7 @@ _register(FileLoader, machinery.SourceFileLoader,
machinery.SourcelessFileLoader)
-class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
+class SourceLoader(_bootstrap_external.SourceLoader, ResourceLoader, ExecutionLoader):
"""Abstract base class for loading source code (and optionally any
corresponding bytecode).
diff --git a/Lib/importlib/machinery.py b/Lib/importlib/machinery.py
index 2e1b2d7..1b2b5c9 100644
--- a/Lib/importlib/machinery.py
+++ b/Lib/importlib/machinery.py
@@ -2,18 +2,18 @@
import _imp
-from ._bootstrap import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
- OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
- EXTENSION_SUFFIXES)
from ._bootstrap import ModuleSpec
from ._bootstrap import BuiltinImporter
from ._bootstrap import FrozenImporter
-from ._bootstrap import WindowsRegistryFinder
-from ._bootstrap import PathFinder
-from ._bootstrap import FileFinder
-from ._bootstrap import SourceFileLoader
-from ._bootstrap import SourcelessFileLoader
-from ._bootstrap import ExtensionFileLoader
+from ._bootstrap_external import (SOURCE_SUFFIXES, DEBUG_BYTECODE_SUFFIXES,
+ OPTIMIZED_BYTECODE_SUFFIXES, BYTECODE_SUFFIXES,
+ EXTENSION_SUFFIXES)
+from ._bootstrap_external import WindowsRegistryFinder
+from ._bootstrap_external import PathFinder
+from ._bootstrap_external import FileFinder
+from ._bootstrap_external import SourceFileLoader
+from ._bootstrap_external import SourcelessFileLoader
+from ._bootstrap_external import ExtensionFileLoader
def all_suffixes():
diff --git a/Lib/importlib/util.py b/Lib/importlib/util.py
index c42ef14..1dbff26 100644
--- a/Lib/importlib/util.py
+++ b/Lib/importlib/util.py
@@ -1,14 +1,14 @@
"""Utility code for constructing importers, etc."""
from . import abc
-from ._bootstrap import MAGIC_NUMBER
-from ._bootstrap import cache_from_source
-from ._bootstrap import decode_source
from ._bootstrap import module_from_spec
-from ._bootstrap import source_from_cache
-from ._bootstrap import spec_from_loader
-from ._bootstrap import spec_from_file_location
from ._bootstrap import _resolve_name
+from ._bootstrap import spec_from_loader
from ._bootstrap import _find_spec
+from ._bootstrap_external import MAGIC_NUMBER
+from ._bootstrap_external import cache_from_source
+from ._bootstrap_external import decode_source
+from ._bootstrap_external import source_from_cache
+from ._bootstrap_external import spec_from_file_location
from contextlib import contextmanager
import functools
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index 5d2c06d..50f2462 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -1,7 +1,7 @@
"""Find modules used by a script, using introspection."""
import dis
-import importlib._bootstrap
+import importlib._bootstrap_external
import importlib.machinery
import marshal
import os
@@ -289,7 +289,7 @@ class ModuleFinder:
co = compile(fp.read()+'\n', pathname, 'exec')
elif type == imp.PY_COMPILED:
try:
- marshal_data = importlib._bootstrap._validate_bytecode_header(fp.read())
+ marshal_data = importlib._bootstrap_external._validate_bytecode_header(fp.read())
except ImportError as exc:
self.msgout(2, "raise ImportError: " + str(exc), pathname)
raise
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index 10c1ef5..11c5b50 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -3,7 +3,7 @@
This module has intimate knowledge of the format of .pyc files.
"""
-import importlib._bootstrap
+import importlib._bootstrap_external
import importlib.machinery
import importlib.util
import os
@@ -137,10 +137,10 @@ def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1):
except FileExistsError:
pass
source_stats = loader.path_stats(file)
- bytecode = importlib._bootstrap._code_to_bytecode(
+ bytecode = importlib._bootstrap_external._code_to_bytecode(
code, source_stats['mtime'], source_stats['size'])
- mode = importlib._bootstrap._calc_mode(file)
- importlib._bootstrap._write_atomic(cfile, bytecode, mode)
+ mode = importlib._bootstrap_external._calc_mode(file)
+ importlib._bootstrap_external._write_atomic(cfile, bytecode, mode)
return cfile
diff --git a/Lib/pydoc.py b/Lib/pydoc.py
index 306dcaf..ee558bf 100755
--- a/Lib/pydoc.py
+++ b/Lib/pydoc.py
@@ -53,6 +53,7 @@ Richard Chamberlain, for the first implementation of textdoc.
import builtins
import importlib._bootstrap
+import importlib._bootstrap_external
import importlib.machinery
import importlib.util
import inspect
@@ -292,9 +293,9 @@ def importfile(path):
filename = os.path.basename(path)
name, ext = os.path.splitext(filename)
if is_bytecode:
- loader = importlib._bootstrap.SourcelessFileLoader(name, path)
+ loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
else:
- loader = importlib._bootstrap.SourceFileLoader(name, path)
+ loader = importlib._bootstrap_external.SourceFileLoader(name, path)
# XXX We probably don't need to pass in the loader here.
spec = importlib.util.spec_from_file_location(name, path, loader=loader)
try:
diff --git a/Lib/runpy.py b/Lib/runpy.py
index d9c643d..1c5729d 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -58,7 +58,7 @@ class _ModifiedArgv0(object):
self.value = self._sentinel
sys.argv[0] = self._saved_value
-# TODO: Replace these helpers with importlib._bootstrap functions
+# TODO: Replace these helpers with importlib._bootstrap_external functions.
def _run_code(code, run_globals, init_globals=None,
mod_name=None, mod_spec=None,
pkg_name=None, script_name=None):
diff --git a/Lib/site.py b/Lib/site.py
index 4959cfc..b7d0331 100644
--- a/Lib/site.py
+++ b/Lib/site.py
@@ -98,8 +98,8 @@ def makepath(*paths):
def abs_paths():
"""Set all module __file__ and __cached__ attributes to an absolute path"""
for m in set(sys.modules.values()):
- if (getattr(getattr(m, '__loader__', None), '__module__', None) !=
- '_frozen_importlib'):
+ if (getattr(getattr(m, '__loader__', None), '__module__', None) not in
+ ('_frozen_importlib', '_frozen_importlib_external')):
continue # don't mess with a PEP 302-supplied __file__
try:
m.__file__ = os.path.abspath(m.__file__)
diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py
index ac228ff..9b4cac9 100644
--- a/Lib/test/test_import/__init__.py
+++ b/Lib/test/test_import/__init__.py
@@ -1,7 +1,7 @@
# We import importlib *ASAP* in order to test #15386
import importlib
import importlib.util
-from importlib._bootstrap import _get_sourcefile
+from importlib._bootstrap_external import _get_sourcefile
import builtins
import marshal
import os
@@ -845,19 +845,27 @@ class ImportlibBootstrapTests(unittest.TestCase):
self.assertEqual(mod.__package__, 'importlib')
self.assertTrue(mod.__file__.endswith('_bootstrap.py'), mod.__file__)
+ def test_frozen_importlib_external_is_bootstrap_external(self):
+ from importlib import _bootstrap_external
+ mod = sys.modules['_frozen_importlib_external']
+ self.assertIs(mod, _bootstrap_external)
+ self.assertEqual(mod.__name__, 'importlib._bootstrap_external')
+ self.assertEqual(mod.__package__, 'importlib')
+ self.assertTrue(mod.__file__.endswith('_bootstrap_external.py'), mod.__file__)
+
def test_there_can_be_only_one(self):
# Issue #15386 revealed a tricky loophole in the bootstrapping
# This test is technically redundant, since the bug caused importing
# this test module to crash completely, but it helps prove the point
from importlib import machinery
mod = sys.modules['_frozen_importlib']
- self.assertIs(machinery.FileFinder, mod.FileFinder)
+ self.assertIs(machinery.ModuleSpec, mod.ModuleSpec)
@cpython_only
class GetSourcefileTests(unittest.TestCase):
- """Test importlib._bootstrap._get_sourcefile() as used by the C API.
+ """Test importlib._bootstrap_external._get_sourcefile() as used by the C API.
Because of the peculiarities of the need of this function, the tests are
knowingly whitebox tests.
@@ -867,7 +875,7 @@ class GetSourcefileTests(unittest.TestCase):
def test_get_sourcefile(self):
# Given a valid bytecode path, return the path to the corresponding
# source file if it exists.
- with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+ with mock.patch('importlib._bootstrap_external._path_isfile') as _path_isfile:
_path_isfile.return_value = True;
path = TESTFN + '.pyc'
expect = TESTFN + '.py'
@@ -876,7 +884,7 @@ class GetSourcefileTests(unittest.TestCase):
def test_get_sourcefile_no_source(self):
# Given a valid bytecode path without a corresponding source path,
# return the original bytecode path.
- with mock.patch('importlib._bootstrap._path_isfile') as _path_isfile:
+ with mock.patch('importlib._bootstrap_external._path_isfile') as _path_isfile:
_path_isfile.return_value = False;
path = TESTFN + '.pyc'
self.assertEqual(_get_sourcefile(path), path)
@@ -1031,7 +1039,7 @@ class ImportTracebackTests(unittest.TestCase):
# We simulate a bug in importlib and check that it's not stripped
# away from the traceback.
self.create_module("foo", "")
- importlib = sys.modules['_frozen_importlib']
+ importlib = sys.modules['_frozen_importlib_external']
if 'load_module' in vars(importlib.SourceLoader):
old_exec_module = importlib.SourceLoader.exec_module
else:
diff --git a/Lib/test/test_importlib/extension/test_case_sensitivity.py b/Lib/test/test_importlib/extension/test_case_sensitivity.py
index c7d6ca6..706c3e4 100644
--- a/Lib/test/test_importlib/extension/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/extension/test_case_sensitivity.py
@@ -1,4 +1,4 @@
-from importlib import _bootstrap
+from importlib import _bootstrap_external
import sys
from test import support
import unittest
@@ -26,7 +26,7 @@ class ExtensionModuleCaseSensitivityTest:
def test_case_sensitive(self):
with support.EnvironmentVarGuard() as env:
env.unset('PYTHONCASEOK')
- if b'PYTHONCASEOK' in _bootstrap._os.environ:
+ if b'PYTHONCASEOK' in _bootstrap_external._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
loader = self.find_module()
@@ -35,7 +35,7 @@ class ExtensionModuleCaseSensitivityTest:
def test_case_insensitivity(self):
with support.EnvironmentVarGuard() as env:
env.set('PYTHONCASEOK', '1')
- if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+ if b'PYTHONCASEOK' not in _bootstrap_external._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
loader = self.find_module()
diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py
index c246d69..4359dd9 100644
--- a/Lib/test/test_importlib/import_/test_path.py
+++ b/Lib/test/test_importlib/import_/test_path.py
@@ -99,7 +99,7 @@ class FinderTests:
new_path_importer_cache.pop(None, None)
new_path_hooks = [zipimport.zipimporter,
self.machinery.FileFinder.path_hook(
- *self.importlib._bootstrap._get_supported_file_loaders())]
+ *self.importlib._bootstrap_external._get_supported_file_loaders())]
missing = object()
email = sys.modules.pop('email', missing)
try:
diff --git a/Lib/test/test_importlib/source/test_case_sensitivity.py b/Lib/test/test_importlib/source/test_case_sensitivity.py
index 29e95b2..c274b38 100644
--- a/Lib/test/test_importlib/source/test_case_sensitivity.py
+++ b/Lib/test/test_importlib/source/test_case_sensitivity.py
@@ -42,7 +42,7 @@ class CaseSensitivityTest:
def test_sensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.unset('PYTHONCASEOK')
- if b'PYTHONCASEOK' in self.importlib._bootstrap._os.environ:
+ if b'PYTHONCASEOK' in self.importlib._bootstrap_external._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
sensitive, insensitive = self.sensitivity_test()
@@ -53,7 +53,7 @@ class CaseSensitivityTest:
def test_insensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.set('PYTHONCASEOK', '1')
- if b'PYTHONCASEOK' not in self.importlib._bootstrap._os.environ:
+ if b'PYTHONCASEOK' not in self.importlib._bootstrap_external._os.environ:
self.skipTest('os.environ changes not reflected in '
'_os.environ')
sensitive, insensitive = self.sensitivity_test()
diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py
index e234164..9a99dfa 100644
--- a/Lib/test/test_site.py
+++ b/Lib/test/test_site.py
@@ -355,8 +355,10 @@ class ImportSideEffectTests(unittest.TestCase):
stdout, stderr = proc.communicate()
self.assertEqual(proc.returncode, 0)
os__file__, os__cached__ = stdout.splitlines()[:2]
- self.assertTrue(os.path.isabs(os__file__))
- self.assertTrue(os.path.isabs(os__cached__))
+ self.assertTrue(os.path.isabs(os__file__),
+ "expected absolute path, got {}".format(os__file__))
+ self.assertTrue(os.path.isabs(os__cached__),
+ "expected absolute path, got {}".format(os__cached__))
def test_no_duplicate_paths(self):
# No duplicate paths should exist in sys.path
generated by cgit v0.12 at 2024-11-24 00:47:12 (GMT)