summaryrefslogtreecommitdiffstats
path: root/Lib/importlib
diff options
context:
space:
mode:
Diffstat (limited to 'Lib/importlib')
-rw-r--r--Lib/importlib/__init__.py79
-rw-r--r--Lib/importlib/_bootstrap.py270
-rw-r--r--Lib/importlib/abc.py23
-rw-r--r--Lib/importlib/test/__main__.py7
-rw-r--r--Lib/importlib/test/benchmark.py159
-rw-r--r--Lib/importlib/test/extension/test_case_sensitivity.py6
-rw-r--r--Lib/importlib/test/import_/test_path.py10
-rw-r--r--Lib/importlib/test/regrtest.py7
-rw-r--r--Lib/importlib/test/source/test_abc_loader.py17
-rw-r--r--Lib/importlib/test/source/test_case_sensitivity.py6
-rw-r--r--Lib/importlib/test/source/test_file_loader.py44
-rw-r--r--Lib/importlib/test/test_util.py10
12 files changed, 424 insertions, 214 deletions
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py
index 2baaf93..940a9a2 100644
--- a/Lib/importlib/__init__.py
+++ b/Lib/importlib/__init__.py
@@ -22,82 +22,17 @@ __all__ = ['__import__', 'import_module']
from . import _bootstrap
-import os
-import re
-import tokenize
-# Bootstrap help #####################################################
-
-def _case_ok(directory, check):
- """Check if the directory contains something matching 'check'.
-
- No check is done if the file/directory exists or not.
-
- """
- if 'PYTHONCASEOK' in os.environ:
- return True
- elif check in os.listdir(directory if directory else os.getcwd()):
- return True
- return False
-
-
-def _w_long(x):
- """Convert a 32-bit integer to little-endian.
+# To simplify imports in test code
+_w_long = _bootstrap._w_long
+_r_long = _bootstrap._r_long
- XXX Temporary until marshal's long functions are exposed.
- """
- x = int(x)
- int_bytes = []
- int_bytes.append(x & 0xFF)
- int_bytes.append((x >> 8) & 0xFF)
- int_bytes.append((x >> 16) & 0xFF)
- int_bytes.append((x >> 24) & 0xFF)
- return bytearray(int_bytes)
-
-
-def _r_long(int_bytes):
- """Convert 4 bytes in little-endian to an integer.
-
- XXX Temporary until marshal's long function are exposed.
+# Bootstrap help #####################################################
+import imp
+import sys
- """
- x = int_bytes[0]
- x |= int_bytes[1] << 8
- x |= int_bytes[2] << 16
- x |= int_bytes[3] << 24
- return x
-
-
-# Required built-in modules.
-try:
- import posix as _os
-except ImportError:
- try:
- import nt as _os
- except ImportError:
- try:
- import os2 as _os
- except ImportError:
- raise ImportError('posix, nt, or os2 module required for importlib')
-_bootstrap._os = _os
-import imp, sys, marshal, errno, _io
-_bootstrap.imp = imp
-_bootstrap.sys = sys
-_bootstrap.marshal = marshal
-_bootstrap.errno = errno
-_bootstrap._io = _io
-import _warnings
-_bootstrap._warnings = _warnings
-
-
-from os import sep
-# For os.path.join replacement; pull from Include/osdefs.h:SEP .
-_bootstrap.path_sep = sep
-
-_bootstrap._case_ok = _case_ok
-marshal._w_long = _w_long
-marshal._r_long = _r_long
+_bootstrap._setup(sys, imp)
# Public API #########################################################
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 90eb1a7..9f7bd11 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -7,7 +7,7 @@ work. One should use importlib as the public-facing version of this module.
"""
-# Injected modules are '_warnings', 'imp', 'sys', 'marshal', 'errno', '_io',
+# Injected modules are '_warnings', 'imp', 'sys', 'marshal', '_io',
# and '_os' (a.k.a. 'posix', 'nt' or 'os2').
# Injected attribute is path_sep.
#
@@ -18,6 +18,64 @@ work. One should use importlib as the public-facing version of this module.
# Bootstrap-related code ######################################################
+# TODO: when not on any of these platforms, replace _case_ok() w/
+# ``lambda x,y: True``.
+CASE_OK_PLATFORMS = 'win', 'cygwin', 'darwin'
+
+def _case_ok(directory, check):
+ """Check if the directory contains something matching 'check'
+ case-sensitively when running on Windows or OS X.
+
+ If running on Window or OS X and PYTHONCASEOK is a defined environment
+ variable then no case-sensitive check is performed. No check is done to see
+ if what is being checked for exists, so if the platform is not Windows or
+ OS X then assume the case is fine.
+
+ """
+ if (any(map(sys.platform.startswith, CASE_OK_PLATFORMS)) and
+ b'PYTHONCASEOK' not in _os.environ):
+ if not directory:
+ directory = '.'
+ if check in _os.listdir(directory):
+ return True
+ else:
+ return False
+ else:
+ return True
+
+
+
+# TODO: Expose from marshal
+def _w_long(x):
+ """Convert a 32-bit integer to little-endian.
+
+ XXX Temporary until marshal's long functions are exposed.
+
+ """
+ x = int(x)
+ int_bytes = []
+ int_bytes.append(x & 0xFF)
+ int_bytes.append((x >> 8) & 0xFF)
+ int_bytes.append((x >> 16) & 0xFF)
+ int_bytes.append((x >> 24) & 0xFF)
+ return bytearray(int_bytes)
+
+
+# TODO: Expose from marshal
+def _r_long(int_bytes):
+ """Convert 4 bytes in little-endian to an integer.
+
+ XXX Temporary until marshal's long function are exposed.
+
+ """
+ x = int_bytes[0]
+ x |= int_bytes[1] << 8
+ x |= int_bytes[2] << 16
+ x |= int_bytes[3] << 24
+ return x
+
+
+
# XXX Could also expose Modules/getpath.c:joinpath()
def _path_join(*args):
"""Replacement for os.path.join."""
@@ -80,10 +138,40 @@ def _path_absolute(path):
return _path_join(_os.getcwd(), path)
+def _write_atomic(path, data):
+ """Best-effort function to write data to a path atomically.
+ Be prepared to handle a FileExistsError if concurrent writing of the
+ temporary file is attempted."""
+ # Renaming should be atomic on most platforms (including Windows).
+ # Under Windows, the limitation is that we can't rename() to an existing
+ # path, while POSIX will overwrite it. But here we don't really care
+ # if there is a glimpse of time during which the final pyc file doesn't
+ # exist.
+ # id() is used to generate a pseudo-random filename.
+ path_tmp = '{}.{}'.format(path, id(path))
+ fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, 0o666)
+ try:
+ with _io.FileIO(fd, 'wb') as file:
+ file.write(data)
+ try:
+ _os.rename(path_tmp, path)
+ except FileExistsError:
+ # Windows (if we had access to MoveFileEx, we could overwrite)
+ _os.unlink(path)
+ _os.rename(path_tmp, path)
+ except OSError:
+ try:
+ _os.unlink(path_tmp)
+ except OSError:
+ pass
+ raise
+
+
def _wrap(new, old):
"""Simple substitute for functools.wraps."""
- for replace in ['__module__', '__name__', '__doc__']:
- setattr(new, replace, getattr(old, replace))
+ for replace in ['__module__', '__name__', '__qualname__', '__doc__']:
+ if hasattr(old, replace):
+ setattr(new, replace, getattr(old, replace))
new.__dict__.update(old.__dict__)
@@ -93,26 +181,26 @@ code_type = type(_wrap.__code__)
def set_package(fxn):
"""Set __package__ on the returned module."""
- def wrapper(*args, **kwargs):
+ def set_package_wrapper(*args, **kwargs):
module = fxn(*args, **kwargs)
if not hasattr(module, '__package__') or module.__package__ is None:
module.__package__ = module.__name__
if not hasattr(module, '__path__'):
module.__package__ = module.__package__.rpartition('.')[0]
return module
- _wrap(wrapper, fxn)
- return wrapper
+ _wrap(set_package_wrapper, fxn)
+ return set_package_wrapper
def set_loader(fxn):
"""Set __loader__ on the returned module."""
- def wrapper(self, *args, **kwargs):
+ def set_loader_wrapper(self, *args, **kwargs):
module = fxn(self, *args, **kwargs)
if not hasattr(module, '__loader__'):
module.__loader__ = self
return module
- _wrap(wrapper, fxn)
- return wrapper
+ _wrap(set_loader_wrapper, fxn)
+ return set_loader_wrapper
def module_for_loader(fxn):
@@ -128,7 +216,7 @@ def module_for_loader(fxn):
the second argument.
"""
- def decorated(self, fullname, *args, **kwargs):
+ def module_for_loader_wrapper(self, fullname, *args, **kwargs):
module = sys.modules.get(fullname)
is_reload = bool(module)
if not is_reload:
@@ -143,8 +231,8 @@ def module_for_loader(fxn):
if not is_reload:
del sys.modules[fullname]
raise
- _wrap(decorated, fxn)
- return decorated
+ _wrap(module_for_loader_wrapper, fxn)
+ return module_for_loader_wrapper
def _check_name(method):
@@ -155,32 +243,32 @@ def _check_name(method):
compared against. If the comparison fails then ImportError is raised.
"""
- def inner(self, name, *args, **kwargs):
+ def _check_name_wrapper(self, name, *args, **kwargs):
if self._name != name:
raise ImportError("loader cannot handle %s" % name)
return method(self, name, *args, **kwargs)
- _wrap(inner, method)
- return inner
+ _wrap(_check_name_wrapper, method)
+ return _check_name_wrapper
def _requires_builtin(fxn):
"""Decorator to verify the named module is built-in."""
- def wrapper(self, fullname):
+ def _requires_builtin_wrapper(self, fullname):
if fullname not in sys.builtin_module_names:
raise ImportError("{0} is not a built-in module".format(fullname))
return fxn(self, fullname)
- _wrap(wrapper, fxn)
- return wrapper
+ _wrap(_requires_builtin_wrapper, fxn)
+ return _requires_builtin_wrapper
def _requires_frozen(fxn):
"""Decorator to verify the named module is frozen."""
- def wrapper(self, fullname):
+ def _requires_frozen_wrapper(self, fullname):
if not imp.is_frozen(fullname):
raise ImportError("{0} is not a frozen module".format(fullname))
return fxn(self, fullname)
- _wrap(wrapper, fxn)
- return wrapper
+ _wrap(_requires_frozen_wrapper, fxn)
+ return _requires_frozen_wrapper
def _suffix_list(suffix_type):
@@ -240,7 +328,7 @@ class BuiltinImporter:
@classmethod
@_requires_builtin
def is_package(cls, fullname):
- """Return None as built-in module are never packages."""
+ """Return None as built-in modules are never packages."""
return False
@@ -302,25 +390,40 @@ class _LoaderBasics:
filename = self.get_filename(fullname).rpartition(path_sep)[2]
return filename.rsplit('.', 1)[0] == '__init__'
- def _bytes_from_bytecode(self, fullname, data, source_mtime):
+ def _bytes_from_bytecode(self, fullname, data, source_stats):
"""Return the marshalled bytes from bytecode, verifying the magic
- number and timestamp along the way.
+ number, timestamp and source size along the way.
- If source_mtime is None then skip the timestamp check.
+ If source_stats is None then skip the timestamp check.
"""
magic = data[:4]
raw_timestamp = data[4:8]
+ raw_size = data[8:12]
if len(magic) != 4 or magic != imp.get_magic():
raise ImportError("bad magic number in {}".format(fullname))
elif len(raw_timestamp) != 4:
raise EOFError("bad timestamp in {}".format(fullname))
- elif source_mtime is not None:
- if marshal._r_long(raw_timestamp) != source_mtime:
- raise ImportError("bytecode is stale for {}".format(fullname))
+ elif len(raw_size) != 4:
+ raise EOFError("bad size in {}".format(fullname))
+ if source_stats is not None:
+ try:
+ source_mtime = int(source_stats['mtime'])
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_timestamp) != source_mtime:
+ raise ImportError("bytecode is stale for {}".format(fullname))
+ try:
+ source_size = source_stats['size'] & 0xFFFFFFFF
+ except KeyError:
+ pass
+ else:
+ if _r_long(raw_size) != source_size:
+ raise ImportError("bytecode is stale for {}".format(fullname))
# Can't return the code object as errors from marshal loading need to
# propagate even when source is available.
- return data[8:]
+ return data[12:]
@module_for_loader
def _load_module(self, module, *, sourceless=False):
@@ -348,11 +451,20 @@ class SourceLoader(_LoaderBasics):
def path_mtime(self, path):
"""Optional method that returns the modification time (an int) for the
specified path, where path is a str.
+ """
+ raise NotImplementedError
- Implementing this method allows the loader to read bytecode files.
+ def path_stats(self, path):
+ """Optional method returning a metadata dict for the specified path
+ to by the path (str).
+ Possible keys:
+ - 'mtime' (mandatory) is the numeric timestamp of last source
+ code modification;
+ - 'size' (optional) is the size in bytes of the source code.
+ Implementing this method allows the loader to read bytecode files.
"""
- raise NotImplementedError
+ return {'mtime': self.path_mtime(path)}
def set_data(self, path, data):
"""Optional method which writes data (bytes) to a file path (a str).
@@ -378,7 +490,7 @@ class SourceLoader(_LoaderBasics):
def get_code(self, fullname):
"""Concrete implementation of InspectLoader.get_code.
- Reading of bytecode requires path_mtime to be implemented. To write
+ Reading of bytecode requires path_stats to be implemented. To write
bytecode, set_data must also be implemented.
"""
@@ -387,10 +499,11 @@ class SourceLoader(_LoaderBasics):
source_mtime = None
if bytecode_path is not None:
try:
- source_mtime = self.path_mtime(source_path)
+ st = self.path_stats(source_path)
except NotImplementedError:
pass
else:
+ source_mtime = int(st['mtime'])
try:
data = self.get_data(bytecode_path)
except IOError:
@@ -398,12 +511,13 @@ class SourceLoader(_LoaderBasics):
else:
try:
bytes_data = self._bytes_from_bytecode(fullname, data,
- source_mtime)
+ st)
except (ImportError, EOFError):
pass
else:
found = marshal.loads(bytes_data)
if isinstance(found, code_type):
+ imp._fix_co_filename(found, source_path)
return found
else:
msg = "Non-code object in {}"
@@ -417,7 +531,8 @@ class SourceLoader(_LoaderBasics):
# their own cached file format, this block of code will most likely
# throw an exception.
data = bytearray(imp.get_magic())
- data.extend(marshal._w_long(source_mtime))
+ data.extend(_w_long(source_mtime))
+ data.extend(_w_long(len(source_bytes)))
data.extend(marshal.dumps(code_object))
try:
self.set_data(bytecode_path, data)
@@ -462,9 +577,10 @@ class _SourceFileLoader(_FileLoader, SourceLoader):
"""Concrete implementation of SourceLoader using the file system."""
- def path_mtime(self, path):
- """Return the modification time for the path."""
- return int(_os.stat(path).st_mtime)
+ def path_stats(self, path):
+ """Return the metadat for the path."""
+ st = _os.stat(path)
+ return {'mtime': st.st_mtime, 'size': st.st_size}
def set_data(self, path, data):
"""Write bytes data to a file."""
@@ -479,28 +595,19 @@ class _SourceFileLoader(_FileLoader, SourceLoader):
parent = _path_join(parent, part)
try:
_os.mkdir(parent)
- except OSError as exc:
+ except FileExistsError:
# Probably another Python process already created the dir.
- if exc.errno == errno.EEXIST:
- continue
- else:
- raise
- except IOError as exc:
+ continue
+ except PermissionError:
# If can't get proper access, then just forget about writing
# the data.
- if exc.errno == errno.EACCES:
- return
- else:
- raise
- try:
- with _io.FileIO(path, 'wb') as file:
- file.write(data)
- except IOError as exc:
- # Don't worry if you can't write bytecode.
- if exc.errno == errno.EACCES:
return
- else:
- raise
+ try:
+ _write_atomic(path, data)
+ except (PermissionError, FileExistsError):
+ # Don't worry if you can't write bytecode or someone is writing
+ # it at the same time.
+ pass
class _SourcelessFileLoader(_FileLoader, _LoaderBasics):
@@ -611,7 +718,7 @@ class PathFinder:
try:
finder = sys.path_importer_cache[path]
except KeyError:
- finder = cls._path_hooks(path)
+ finder = cls._path_hooks(path if path != '' else _os.getcwd())
sys.path_importer_cache[path] = finder
else:
if finder is None and default:
@@ -758,14 +865,14 @@ class _ImportLockContext:
_IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder]
-_ERR_MSG = 'No module named {}'
+_ERR_MSG = 'No module named {!r}'
def _gcd_import(name, package=None, level=0):
"""Import and return the module based on its name, the package the call is
being made from, and the level adjustment.
This function represents the greatest common denominator of functionality
- between import_module and __import__. This includes settting __package__ if
+ between import_module and __import__. This includes setting __package__ if
the loader did not.
"""
@@ -857,7 +964,7 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
module = _gcd_import(name)
else:
# __package__ is not guaranteed to be defined or could be set to None
- # to represent that it's proper value is unknown
+ # to represent that its proper value is unknown
package = globals.get('__package__')
if package is None:
package = globals['__name__']
@@ -888,3 +995,46 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0):
except ImportError:
pass
return module
+
+
+def _setup(sys_module, imp_module):
+ """Setup importlib by importing needed built-in modules and injecting them
+ into the global namespace.
+
+ As sys is needed for sys.modules access and imp is needed to load built-in
+ modules those two modules must be explicitly passed in.
+
+ """
+ global imp, sys
+ imp = imp_module
+ sys = sys_module
+
+ for module in (imp, sys):
+ if not hasattr(module, '__loader__'):
+ module.__loader__ = BuiltinImporter
+
+ self_module = sys.modules[__name__]
+ for builtin_name in ('_io', '_warnings', 'builtins', 'marshal'):
+ if builtin_name not in sys.modules:
+ builtin_module = BuiltinImporter.load_module(builtin_name)
+ else:
+ builtin_module = sys.modules[builtin_name]
+ setattr(self_module, builtin_name, builtin_module)
+
+ for builtin_os, path_sep in [('posix', '/'), ('nt', '\\'), ('os2', '\\')]:
+ if builtin_os in sys.modules:
+ os_module = sys.modules[builtin_os]
+ break
+ else:
+ try:
+ os_module = BuiltinImporter.load_module(builtin_os)
+ # TODO: rip out os2 code after 3.3 is released as per PEP 11
+ if builtin_os == 'os2' and 'EMX GCC' in sys.version:
+ path_sep = '/'
+ break
+ except ImportError:
+ continue
+ else:
+ raise ImportError('importlib requires posix or nt')
+ setattr(self_module, '_os', os_module)
+ setattr(self_module, 'path_sep', path_sep)
diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py
index fa343f8..c4b7187 100644
--- a/Lib/importlib/abc.py
+++ b/Lib/importlib/abc.py
@@ -123,7 +123,20 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader):
def path_mtime(self, path):
"""Return the (int) modification time for the path (str)."""
- raise NotImplementedError
+ if self.path_stats.__func__ is SourceLoader.path_stats:
+ raise NotImplementedError
+ return int(self.path_stats(path)['mtime'])
+
+ def path_stats(self, path):
+ """Return a metadata dict for the source pointed to by the path (str).
+ Possible keys:
+ - 'mtime' (mandatory) is the numeric timestamp of last source
+ code modification;
+ - 'size' (optional) is the size in bytes of the source code.
+ """
+ if self.path_mtime.__func__ is SourceLoader.path_mtime:
+ raise NotImplementedError
+ return {'mtime': self.path_mtime(path)}
def set_data(self, path, data):
"""Write the bytes to the path (if possible).
@@ -195,7 +208,7 @@ class PyLoader(SourceLoader):
"use SourceLoader instead. "
"See the importlib documentation on how to be "
"compatible with Python 3.1 onwards.",
- PendingDeprecationWarning)
+ DeprecationWarning)
path = self.source_path(fullname)
if path is None:
raise ImportError
@@ -234,7 +247,7 @@ class PyPycLoader(PyLoader):
"removal in Python 3.4; use SourceLoader instead. "
"If Python 3.1 compatibility is required, see the "
"latest documentation for PyLoader.",
- PendingDeprecationWarning)
+ DeprecationWarning)
source_timestamp = self.source_mtime(fullname)
# Try to use bytecode if it is available.
bytecode_path = self.bytecode_path(fullname)
@@ -247,7 +260,7 @@ class PyPycLoader(PyLoader):
raw_timestamp = data[4:8]
if len(raw_timestamp) < 4:
raise EOFError("bad timestamp in {}".format(fullname))
- pyc_timestamp = marshal._r_long(raw_timestamp)
+ pyc_timestamp = _bootstrap._r_long(raw_timestamp)
bytecode = data[8:]
# Verify that the magic number is valid.
if imp.get_magic() != magic:
@@ -279,7 +292,7 @@ class PyPycLoader(PyLoader):
# Generate bytecode and write it out.
if not sys.dont_write_bytecode:
data = bytearray(imp.get_magic())
- data.extend(marshal._w_long(source_timestamp))
+ data.extend(_bootstrap._w_long(source_timestamp))
data.extend(marshal.dumps(code_object))
self.write_bytecode(fullname, data)
return code_object
diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py
index decc53d..a1990b1 100644
--- a/Lib/importlib/test/__main__.py
+++ b/Lib/importlib/test/__main__.py
@@ -4,7 +4,6 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with
builtins.__import__ instead of importlib.__import__.
"""
-import importlib
from importlib.test.import_ import util
import os.path
from test.support import run_unittest
@@ -13,11 +12,7 @@ import unittest
def test_main():
- if '__pycache__' in __file__:
- parts = __file__.split(os.path.sep)
- start_dir = sep.join(parts[:-2])
- else:
- start_dir = os.path.dirname(__file__)
+ start_dir = os.path.dirname(__file__)
top_dir = os.path.dirname(os.path.dirname(start_dir))
test_loader = unittest.TestLoader()
if '--builtin' in sys.argv:
diff --git a/Lib/importlib/test/benchmark.py b/Lib/importlib/test/benchmark.py
index b5de6c6..ed88f04 100644
--- a/Lib/importlib/test/benchmark.py
+++ b/Lib/importlib/test/benchmark.py
@@ -9,9 +9,11 @@ from .source import util as source_util
import decimal
import imp
import importlib
+import json
import os
import py_compile
import sys
+import tabnanny
import timeit
@@ -59,7 +61,7 @@ def builtin_mod(seconds, repeat):
def source_wo_bytecode(seconds, repeat):
- """Source w/o bytecode: simple"""
+ """Source w/o bytecode: small"""
sys.dont_write_bytecode = True
try:
name = '__importlib_test_benchmark__'
@@ -73,23 +75,30 @@ def source_wo_bytecode(seconds, repeat):
sys.dont_write_bytecode = False
-def decimal_wo_bytecode(seconds, repeat):
- """Source w/o bytecode: decimal"""
- name = 'decimal'
- decimal_bytecode = imp.cache_from_source(decimal.__file__)
- if os.path.exists(decimal_bytecode):
- os.unlink(decimal_bytecode)
- sys.dont_write_bytecode = True
- try:
- for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
- seconds=seconds):
- yield result
- finally:
- sys.dont_write_bytecode = False
+def _wo_bytecode(module):
+ name = module.__name__
+ def benchmark_wo_bytecode(seconds, repeat):
+ """Source w/o bytecode: {}"""
+ bytecode_path = imp.cache_from_source(module.__file__)
+ if os.path.exists(bytecode_path):
+ os.unlink(bytecode_path)
+ sys.dont_write_bytecode = True
+ try:
+ for result in bench(name, lambda: sys.modules.pop(name),
+ repeat=repeat, seconds=seconds):
+ yield result
+ finally:
+ sys.dont_write_bytecode = False
+
+ benchmark_wo_bytecode.__doc__ = benchmark_wo_bytecode.__doc__.format(name)
+ return benchmark_wo_bytecode
+
+tabnanny_wo_bytecode = _wo_bytecode(tabnanny)
+decimal_wo_bytecode = _wo_bytecode(decimal)
def source_writing_bytecode(seconds, repeat):
- """Source writing bytecode: simple"""
+ """Source writing bytecode: small"""
assert not sys.dont_write_bytecode
name = '__importlib_test_benchmark__'
with source_util.create_modules(name) as mapping:
@@ -101,19 +110,27 @@ def source_writing_bytecode(seconds, repeat):
yield result
-def decimal_writing_bytecode(seconds, repeat):
- """Source writing bytecode: decimal"""
- assert not sys.dont_write_bytecode
- name = 'decimal'
- def cleanup():
- sys.modules.pop(name)
- os.unlink(imp.cache_from_source(decimal.__file__))
- for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
- yield result
+def _writing_bytecode(module):
+ name = module.__name__
+ def writing_bytecode_benchmark(seconds, repeat):
+ """Source writing bytecode: {}"""
+ assert not sys.dont_write_bytecode
+ def cleanup():
+ sys.modules.pop(name)
+ os.unlink(imp.cache_from_source(module.__file__))
+ for result in bench(name, cleanup, repeat=repeat, seconds=seconds):
+ yield result
+
+ writing_bytecode_benchmark.__doc__ = (
+ writing_bytecode_benchmark.__doc__.format(name))
+ return writing_bytecode_benchmark
+
+tabnanny_writing_bytecode = _writing_bytecode(tabnanny)
+decimal_writing_bytecode = _writing_bytecode(decimal)
def source_using_bytecode(seconds, repeat):
- """Bytecode w/ source: simple"""
+ """Source w/ bytecode: small"""
name = '__importlib_test_benchmark__'
with source_util.create_modules(name) as mapping:
py_compile.compile(mapping[name])
@@ -123,27 +140,55 @@ def source_using_bytecode(seconds, repeat):
yield result
-def decimal_using_bytecode(seconds, repeat):
- """Bytecode w/ source: decimal"""
- name = 'decimal'
- py_compile.compile(decimal.__file__)
- for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
- seconds=seconds):
- yield result
+def _using_bytecode(module):
+ name = module.__name__
+ def using_bytecode_benchmark(seconds, repeat):
+ """Source w/ bytecode: {}"""
+ py_compile.compile(module.__file__)
+ for result in bench(name, lambda: sys.modules.pop(name), repeat=repeat,
+ seconds=seconds):
+ yield result
+ using_bytecode_benchmark.__doc__ = (
+ using_bytecode_benchmark.__doc__.format(name))
+ return using_bytecode_benchmark
-def main(import_):
+tabnanny_using_bytecode = _using_bytecode(tabnanny)
+decimal_using_bytecode = _using_bytecode(decimal)
+
+
+def main(import_, filename=None, benchmark=None):
+ if filename and os.path.exists(filename):
+ with open(filename, 'r') as file:
+ prev_results = json.load(file)
+ else:
+ prev_results = {}
__builtins__.__import__ = import_
benchmarks = (from_cache, builtin_mod,
- source_using_bytecode, source_wo_bytecode,
source_writing_bytecode,
- decimal_using_bytecode, decimal_writing_bytecode,
- decimal_wo_bytecode,)
+ source_wo_bytecode, source_using_bytecode,
+ tabnanny_writing_bytecode,
+ tabnanny_wo_bytecode, tabnanny_using_bytecode,
+ decimal_writing_bytecode,
+ decimal_wo_bytecode, decimal_using_bytecode,
+ )
+ if benchmark:
+ for b in benchmarks:
+ if b.__doc__ == benchmark:
+ benchmarks = [b]
+ break
+ else:
+ print('Unknown benchmark: {!r}'.format(benchmark, file=sys.stderr))
+ sys.exit(1)
seconds = 1
seconds_plural = 's' if seconds > 1 else ''
repeat = 3
- header = "Measuring imports/second over {} second{}, best out of {}\n"
- print(header.format(seconds, seconds_plural, repeat))
+ header = ('Measuring imports/second over {} second{}, best out of {}\n'
+ 'Entire benchmark run should take about {} seconds\n'
+ 'Using {!r} as __import__\n')
+ print(header.format(seconds, seconds_plural, repeat,
+ len(benchmarks) * seconds * repeat, __import__))
+ new_results = {}
for benchmark in benchmarks:
print(benchmark.__doc__, "[", end=' ')
sys.stdout.flush()
@@ -154,19 +199,43 @@ def main(import_):
sys.stdout.flush()
assert not sys.dont_write_bytecode
print("]", "best is", format(max(results), ',d'))
+ new_results[benchmark.__doc__] = results
+ prev_results[import_.__module__] = new_results
+ if 'importlib._bootstrap' in prev_results and 'builtins' in prev_results:
+ print('\n\nComparing importlib vs. __import__\n')
+ importlib_results = prev_results['importlib._bootstrap']
+ builtins_results = prev_results['builtins']
+ for benchmark in benchmarks:
+ benchmark_name = benchmark.__doc__
+ importlib_result = max(importlib_results[benchmark_name])
+ builtins_result = max(builtins_results[benchmark_name])
+ result = '{:,d} vs. {:,d} ({:%})'.format(importlib_result,
+ builtins_result,
+ importlib_result/builtins_result)
+ print(benchmark_name, ':', result)
+ if filename:
+ with open(filename, 'w') as file:
+ json.dump(prev_results, file, indent=2)
if __name__ == '__main__':
- import optparse
+ import argparse
- parser = optparse.OptionParser()
- parser.add_option('-b', '--builtin', dest='builtin', action='store_true',
+ parser = argparse.ArgumentParser()
+ parser.add_argument('-b', '--builtin', dest='builtin', action='store_true',
default=False, help="use the built-in __import__")
- options, args = parser.parse_args()
- if args:
- raise RuntimeError("unrecognized args: {}".format(args))
+ parser.add_argument('-f', '--file', dest='filename', default=None,
+ help='file to read/write results from/to'
+ '(incompatible w/ --benchmark)')
+ parser.add_argument('--benchmark', dest='benchmark',
+ help='specific benchmark to run '
+ '(incompatible w/ --file)')
+ options = parser.parse_args()
+ if options.filename and options.benchmark:
+ print('Cannot specify a benchmark *and* read/write results')
+ sys.exit(1)
import_ = __import__
if not options.builtin:
import_ = importlib.__import__
- main(import_)
+ main(import_, options.filename, options.benchmark)
diff --git a/Lib/importlib/test/extension/test_case_sensitivity.py b/Lib/importlib/test/extension/test_case_sensitivity.py
index e062fb6..add830d 100644
--- a/Lib/importlib/test/extension/test_case_sensitivity.py
+++ b/Lib/importlib/test/extension/test_case_sensitivity.py
@@ -20,12 +20,18 @@ class ExtensionModuleCaseSensitivityTest(unittest.TestCase):
def test_case_sensitive(self):
with support.EnvironmentVarGuard() as env:
env.unset('PYTHONCASEOK')
+ if b'PYTHONCASEOK' in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
loader = self.find_module()
self.assertIsNone(loader)
def test_case_insensitivity(self):
with support.EnvironmentVarGuard() as env:
env.set('PYTHONCASEOK', '1')
+ if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
loader = self.find_module()
self.assertTrue(hasattr(loader, 'load_module'))
diff --git a/Lib/importlib/test/import_/test_path.py b/Lib/importlib/test/import_/test_path.py
index 2faa231..b28f25d 100644
--- a/Lib/importlib/test/import_/test_path.py
+++ b/Lib/importlib/test/import_/test_path.py
@@ -73,6 +73,16 @@ class FinderTests(unittest.TestCase):
loader = machinery.PathFinder.find_module(module)
self.assertTrue(loader is importer)
+ def test_path_importer_cache_empty_string(self):
+ # The empty string should create a finder using the cwd.
+ path = ''
+ module = '<test module>'
+ importer = util.mock_modules(module)
+ hook = import_util.mock_path_hook(os.getcwd(), importer=importer)
+ with util.import_state(path=[path], path_hooks=[hook]):
+ loader = machinery.PathFinder.find_module(module)
+ self.assertIs(loader, importer)
+ self.assertIn('', sys.path_importer_cache)
class DefaultPathFinderTests(unittest.TestCase):
diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py
index b103ae7d..dc0eb97 100644
--- a/Lib/importlib/test/regrtest.py
+++ b/Lib/importlib/test/regrtest.py
@@ -5,13 +5,6 @@ invalidates are automatically skipped if the entire test suite is run.
Otherwise all command-line options valid for test.regrtest are also valid for
this script.
-XXX FAILING
- * test_import
- - test_incorrect_code_name
- file name differing between __file__ and co_filename (r68360 on trunk)
- - test_import_by_filename
- exception for trying to import by file name does not match
-
"""
import importlib
import sys
diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py
index 3245907..01acda4 100644
--- a/Lib/importlib/test/source/test_abc_loader.py
+++ b/Lib/importlib/test/source/test_abc_loader.py
@@ -40,8 +40,10 @@ class SourceLoaderMock(SourceOnlyLoaderMock):
def __init__(self, path, magic=imp.get_magic()):
super().__init__(path)
self.bytecode_path = imp.cache_from_source(self.path)
+ self.source_size = len(self.source)
data = bytearray(magic)
- data.extend(marshal._w_long(self.source_mtime))
+ data.extend(importlib._w_long(self.source_mtime))
+ data.extend(importlib._w_long(self.source_size))
code_object = compile(self.source, self.path, 'exec',
dont_inherit=True)
data.extend(marshal.dumps(code_object))
@@ -56,9 +58,9 @@ class SourceLoaderMock(SourceOnlyLoaderMock):
else:
raise IOError
- def path_mtime(self, path):
+ def path_stats(self, path):
assert path == self.path
- return self.source_mtime
+ return {'mtime': self.source_mtime, 'size': self.source_size}
def set_data(self, path, data):
self.written[path] = bytes(data)
@@ -102,7 +104,7 @@ class PyLoaderMock(abc.PyLoader):
warnings.simplefilter("always")
path = super().get_filename(name)
assert len(w) == 1
- assert issubclass(w[0].category, PendingDeprecationWarning)
+ assert issubclass(w[0].category, DeprecationWarning)
return path
@@ -198,7 +200,7 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock):
warnings.simplefilter("always")
code_object = super().get_code(name)
assert len(w) == 1
- assert issubclass(w[0].category, PendingDeprecationWarning)
+ assert issubclass(w[0].category, DeprecationWarning)
return code_object
class PyLoaderTests(testing_abc.LoaderTests):
@@ -656,7 +658,8 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
if bytecode_written:
self.assertIn(self.cached, self.loader.written)
data = bytearray(imp.get_magic())
- data.extend(marshal._w_long(self.loader.source_mtime))
+ data.extend(importlib._w_long(self.loader.source_mtime))
+ data.extend(importlib._w_long(self.loader.source_size))
data.extend(marshal.dumps(code_object))
self.assertEqual(self.loader.written[self.cached], bytes(data))
@@ -847,7 +850,7 @@ class AbstractMethodImplTests(unittest.TestCase):
# Required abstractmethods.
self.raises_NotImplementedError(ins, 'get_filename', 'get_data')
# Optional abstractmethods.
- self.raises_NotImplementedError(ins,'path_mtime', 'set_data')
+ self.raises_NotImplementedError(ins,'path_stats', 'set_data')
def test_PyLoader(self):
self.raises_NotImplementedError(self.PyLoader(), 'source_path',
diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py
index 73777de..569f516 100644
--- a/Lib/importlib/test/source/test_case_sensitivity.py
+++ b/Lib/importlib/test/source/test_case_sensitivity.py
@@ -37,6 +37,9 @@ class CaseSensitivityTest(unittest.TestCase):
def test_sensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.unset('PYTHONCASEOK')
+ if b'PYTHONCASEOK' in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
self.assertIn(self.name, sensitive.get_filename(self.name))
@@ -45,6 +48,9 @@ class CaseSensitivityTest(unittest.TestCase):
def test_insensitive(self):
with test_support.EnvironmentVarGuard() as env:
env.set('PYTHONCASEOK', '1')
+ if b'PYTHONCASEOK' not in _bootstrap._os.environ:
+ self.skipTest('os.environ changes not reflected in '
+ '_os.environ')
sensitive, insensitive = self.sensitivity_test()
self.assertTrue(hasattr(sensitive, 'load_module'))
self.assertIn(self.name, sensitive.get_filename(self.name))
diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py
index c7a7d8f..21e718f 100644
--- a/Lib/importlib/test/source/test_file_loader.py
+++ b/Lib/importlib/test/source/test_file_loader.py
@@ -71,11 +71,6 @@ class SimpleTest(unittest.TestCase):
module_dict_id = id(module.__dict__)
with open(mapping['_temp'], 'w') as file:
file.write("testing_var = 42\n")
- # For filesystems where the mtime is only to a second granularity,
- # everything that has happened above can be too fast;
- # force an mtime on the source that is guaranteed to be different
- # than the original mtime.
- loader.path_mtime = self.fake_mtime(loader.path_mtime)
module = loader.load_module('_temp')
self.assertTrue('testing_var' in module.__dict__,
"'testing_var' not in "
@@ -215,10 +210,17 @@ class BadBytecodeTest(unittest.TestCase):
del_source=del_source)
test('_temp', mapping, bc_path)
+ def _test_partial_size(self, test, *, del_source=False):
+ with source_util.create_modules('_temp') as mapping:
+ bc_path = self.manipulate_bytecode('_temp', mapping,
+ lambda bc: bc[:11],
+ del_source=del_source)
+ test('_temp', mapping, bc_path)
+
def _test_no_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bc_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:8],
+ lambda bc: bc[:12],
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bc_path
with self.assertRaises(EOFError):
@@ -227,7 +229,7 @@ class BadBytecodeTest(unittest.TestCase):
def _test_non_code_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bytecode_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:8] + marshal.dumps(b'abcd'),
+ lambda bc: bc[:12] + marshal.dumps(b'abcd'),
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(ImportError):
@@ -236,7 +238,7 @@ class BadBytecodeTest(unittest.TestCase):
def _test_bad_marshal(self, *, del_source=False):
with source_util.create_modules('_temp') as mapping:
bytecode_path = self.manipulate_bytecode('_temp', mapping,
- lambda bc: bc[:8] + b'<test>',
+ lambda bc: bc[:12] + b'<test>',
del_source=del_source)
file_path = mapping['_temp'] if not del_source else bytecode_path
with self.assertRaises(EOFError):
@@ -260,7 +262,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
+ self.assertGreater(len(file.read()), 12)
self._test_empty_file(test)
@@ -268,7 +270,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
+ self.assertGreater(len(file.read()), 12)
self._test_partial_magic(test)
@@ -279,7 +281,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
def test(name, mapping, bytecode_path):
self.import_(mapping[name], name)
with open(bytecode_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
+ self.assertGreater(len(file.read()), 12)
self._test_magic_only(test)
@@ -301,11 +303,22 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest):
def test(name, mapping, bc_path):
self.import_(mapping[name], name)
with open(bc_path, 'rb') as file:
- self.assertGreater(len(file.read()), 8)
+ self.assertGreater(len(file.read()), 12)
self._test_partial_timestamp(test)
@source_util.writes_bytecode_files
+ def test_partial_size(self):
+ # When the size is partial, regenerate the .pyc, else
+ # raise EOFError.
+ def test(name, mapping, bc_path):
+ self.import_(mapping[name], name)
+ with open(bc_path, 'rb') as file:
+ self.assertGreater(len(file.read()), 12)
+
+ self._test_partial_size(test)
+
+ @source_util.writes_bytecode_files
def test_no_marshal(self):
# When there is only the magic number and timestamp, raise EOFError.
self._test_no_marshal()
@@ -400,6 +413,13 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest):
self._test_partial_timestamp(test, del_source=True)
+ def test_partial_size(self):
+ def test(name, mapping, bytecode_path):
+ with self.assertRaises(EOFError):
+ self.import_(bytecode_path, name)
+
+ self._test_partial_size(test, del_source=True)
+
def test_no_marshal(self):
self._test_no_marshal(del_source=True)
diff --git a/Lib/importlib/test/test_util.py b/Lib/importlib/test/test_util.py
index 602447f..c7cdad1 100644
--- a/Lib/importlib/test/test_util.py
+++ b/Lib/importlib/test/test_util.py
@@ -59,6 +59,11 @@ class ModuleForLoaderTests(unittest.TestCase):
self.raise_exception(name)
self.assertIs(module, sys.modules[name])
+ def test_decorator_attrs(self):
+ def fxn(self, module): pass
+ wrapped = util.module_for_loader(fxn)
+ self.assertEqual(wrapped.__name__, fxn.__name__)
+ self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
class SetPackageTests(unittest.TestCase):
@@ -108,6 +113,11 @@ class SetPackageTests(unittest.TestCase):
module.__package__ = value
self.verify(module, value)
+ def test_decorator_attrs(self):
+ def fxn(module): pass
+ wrapped = util.set_package(fxn)
+ self.assertEqual(wrapped.__name__, fxn.__name__)
+ self.assertEqual(wrapped.__qualname__, fxn.__qualname__)
def test_main():
from test import support
generated by cgit v0.12 at 2025-11-21 21:32:21 (GMT)