diff options
Diffstat (limited to 'Lib/importlib')
| -rw-r--r-- | Lib/importlib/__init__.py | 3 | ||||
| -rw-r--r-- | Lib/importlib/_bootstrap.py | 162 | ||||
| -rw-r--r-- | Lib/importlib/abc.py | 19 | ||||
| -rw-r--r-- | Lib/importlib/test/__main__.py | 7 | ||||
| -rw-r--r-- | Lib/importlib/test/regrtest.py | 7 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_abc_loader.py | 13 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_file_loader.py | 44 | ||||
| -rw-r--r-- | Lib/importlib/test/test_util.py | 10 |
8 files changed, 173 insertions, 92 deletions
diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 2baaf93..9b20367 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -81,11 +81,10 @@ except ImportError: except ImportError: raise ImportError('posix, nt, or os2 module required for importlib') _bootstrap._os = _os -import imp, sys, marshal, errno, _io +import imp, sys, marshal, _io _bootstrap.imp = imp _bootstrap.sys = sys _bootstrap.marshal = marshal -_bootstrap.errno = errno _bootstrap._io = _io import _warnings _bootstrap._warnings = _warnings diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 90eb1a7..11d7d0a 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -7,7 +7,7 @@ work. One should use importlib as the public-facing version of this module. """ -# Injected modules are '_warnings', 'imp', 'sys', 'marshal', 'errno', '_io', +# Injected modules are '_warnings', 'imp', 'sys', 'marshal', '_io', # and '_os' (a.k.a. 'posix', 'nt' or 'os2'). # Injected attribute is path_sep. # @@ -80,9 +80,38 @@ def _path_absolute(path): return _path_join(_os.getcwd(), path) +def _write_atomic(path, data): + """Best-effort function to write data to a path atomically. + Be prepared to handle a FileExistsError if concurrent writing of the + temporary file is attempted.""" + # Renaming should be atomic on most platforms (including Windows). + # Under Windows, the limitation is that we can't rename() to an existing + # path, while POSIX will overwrite it. But here we don't really care + # if there is a glimpse of time during which the final pyc file doesn't + # exist. + # id() is used to generate a pseudo-random filename. + path_tmp = '{}.{}'.format(path, id(path)) + fd = _os.open(path_tmp, _os.O_EXCL | _os.O_CREAT | _os.O_WRONLY, 0o666) + try: + with _io.FileIO(fd, 'wb') as file: + file.write(data) + try: + _os.rename(path_tmp, path) + except FileExistsError: + # Windows (if we had access to MoveFileEx, we could overwrite) + _os.unlink(path) + _os.rename(path_tmp, path) + except OSError: + try: + _os.unlink(path_tmp) + except OSError: + pass + raise + + def _wrap(new, old): """Simple substitute for functools.wraps.""" - for replace in ['__module__', '__name__', '__doc__']: + for replace in ['__module__', '__name__', '__qualname__', '__doc__']: setattr(new, replace, getattr(old, replace)) new.__dict__.update(old.__dict__) @@ -93,26 +122,26 @@ code_type = type(_wrap.__code__) def set_package(fxn): """Set __package__ on the returned module.""" - def wrapper(*args, **kwargs): + def set_package_wrapper(*args, **kwargs): module = fxn(*args, **kwargs) if not hasattr(module, '__package__') or module.__package__ is None: module.__package__ = module.__name__ if not hasattr(module, '__path__'): module.__package__ = module.__package__.rpartition('.')[0] return module - _wrap(wrapper, fxn) - return wrapper + _wrap(set_package_wrapper, fxn) + return set_package_wrapper def set_loader(fxn): """Set __loader__ on the returned module.""" - def wrapper(self, *args, **kwargs): + def set_loader_wrapper(self, *args, **kwargs): module = fxn(self, *args, **kwargs) if not hasattr(module, '__loader__'): module.__loader__ = self return module - _wrap(wrapper, fxn) - return wrapper + _wrap(set_loader_wrapper, fxn) + return set_loader_wrapper def module_for_loader(fxn): @@ -128,7 +157,7 @@ def module_for_loader(fxn): the second argument. """ - def decorated(self, fullname, *args, **kwargs): + def module_for_loader_wrapper(self, fullname, *args, **kwargs): module = sys.modules.get(fullname) is_reload = bool(module) if not is_reload: @@ -143,8 +172,8 @@ def module_for_loader(fxn): if not is_reload: del sys.modules[fullname] raise - _wrap(decorated, fxn) - return decorated + _wrap(module_for_loader_wrapper, fxn) + return module_for_loader_wrapper def _check_name(method): @@ -155,32 +184,32 @@ def _check_name(method): compared against. If the comparison fails then ImportError is raised. """ - def inner(self, name, *args, **kwargs): + def _check_name_wrapper(self, name, *args, **kwargs): if self._name != name: raise ImportError("loader cannot handle %s" % name) return method(self, name, *args, **kwargs) - _wrap(inner, method) - return inner + _wrap(_check_name_wrapper, method) + return _check_name_wrapper def _requires_builtin(fxn): """Decorator to verify the named module is built-in.""" - def wrapper(self, fullname): + def _requires_builtin_wrapper(self, fullname): if fullname not in sys.builtin_module_names: raise ImportError("{0} is not a built-in module".format(fullname)) return fxn(self, fullname) - _wrap(wrapper, fxn) - return wrapper + _wrap(_requires_builtin_wrapper, fxn) + return _requires_builtin_wrapper def _requires_frozen(fxn): """Decorator to verify the named module is frozen.""" - def wrapper(self, fullname): + def _requires_frozen_wrapper(self, fullname): if not imp.is_frozen(fullname): raise ImportError("{0} is not a frozen module".format(fullname)) return fxn(self, fullname) - _wrap(wrapper, fxn) - return wrapper + _wrap(_requires_frozen_wrapper, fxn) + return _requires_frozen_wrapper def _suffix_list(suffix_type): @@ -240,7 +269,7 @@ class BuiltinImporter: @classmethod @_requires_builtin def is_package(cls, fullname): - """Return None as built-in module are never packages.""" + """Return None as built-in modules are never packages.""" return False @@ -302,25 +331,40 @@ class _LoaderBasics: filename = self.get_filename(fullname).rpartition(path_sep)[2] return filename.rsplit('.', 1)[0] == '__init__' - def _bytes_from_bytecode(self, fullname, data, source_mtime): + def _bytes_from_bytecode(self, fullname, data, source_stats): """Return the marshalled bytes from bytecode, verifying the magic - number and timestamp along the way. + number, timestamp and source size along the way. - If source_mtime is None then skip the timestamp check. + If source_stats is None then skip the timestamp check. """ magic = data[:4] raw_timestamp = data[4:8] + raw_size = data[8:12] if len(magic) != 4 or magic != imp.get_magic(): raise ImportError("bad magic number in {}".format(fullname)) elif len(raw_timestamp) != 4: raise EOFError("bad timestamp in {}".format(fullname)) - elif source_mtime is not None: - if marshal._r_long(raw_timestamp) != source_mtime: - raise ImportError("bytecode is stale for {}".format(fullname)) + elif len(raw_size) != 4: + raise EOFError("bad size in {}".format(fullname)) + if source_stats is not None: + try: + source_mtime = int(source_stats['mtime']) + except KeyError: + pass + else: + if marshal._r_long(raw_timestamp) != source_mtime: + raise ImportError("bytecode is stale for {}".format(fullname)) + try: + source_size = source_stats['size'] & 0xFFFFFFFF + except KeyError: + pass + else: + if marshal._r_long(raw_size) != source_size: + raise ImportError("bytecode is stale for {}".format(fullname)) # Can't return the code object as errors from marshal loading need to # propagate even when source is available. - return data[8:] + return data[12:] @module_for_loader def _load_module(self, module, *, sourceless=False): @@ -348,11 +392,20 @@ class SourceLoader(_LoaderBasics): def path_mtime(self, path): """Optional method that returns the modification time (an int) for the specified path, where path is a str. + """ + raise NotImplementedError - Implementing this method allows the loader to read bytecode files. + def path_stats(self, path): + """Optional method returning a metadata dict for the specified path + to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + Implementing this method allows the loader to read bytecode files. """ - raise NotImplementedError + return {'mtime': self.path_mtime(path)} def set_data(self, path, data): """Optional method which writes data (bytes) to a file path (a str). @@ -378,7 +431,7 @@ class SourceLoader(_LoaderBasics): def get_code(self, fullname): """Concrete implementation of InspectLoader.get_code. - Reading of bytecode requires path_mtime to be implemented. To write + Reading of bytecode requires path_stats to be implemented. To write bytecode, set_data must also be implemented. """ @@ -387,10 +440,11 @@ class SourceLoader(_LoaderBasics): source_mtime = None if bytecode_path is not None: try: - source_mtime = self.path_mtime(source_path) + st = self.path_stats(source_path) except NotImplementedError: pass else: + source_mtime = int(st['mtime']) try: data = self.get_data(bytecode_path) except IOError: @@ -398,12 +452,13 @@ class SourceLoader(_LoaderBasics): else: try: bytes_data = self._bytes_from_bytecode(fullname, data, - source_mtime) + st) except (ImportError, EOFError): pass else: found = marshal.loads(bytes_data) if isinstance(found, code_type): + imp._fix_co_filename(found, source_path) return found else: msg = "Non-code object in {}" @@ -418,6 +473,7 @@ class SourceLoader(_LoaderBasics): # throw an exception. data = bytearray(imp.get_magic()) data.extend(marshal._w_long(source_mtime)) + data.extend(marshal._w_long(len(source_bytes))) data.extend(marshal.dumps(code_object)) try: self.set_data(bytecode_path, data) @@ -462,9 +518,10 @@ class _SourceFileLoader(_FileLoader, SourceLoader): """Concrete implementation of SourceLoader using the file system.""" - def path_mtime(self, path): - """Return the modification time for the path.""" - return int(_os.stat(path).st_mtime) + def path_stats(self, path): + """Return the metadat for the path.""" + st = _os.stat(path) + return {'mtime': st.st_mtime, 'size': st.st_size} def set_data(self, path, data): """Write bytes data to a file.""" @@ -479,28 +536,19 @@ class _SourceFileLoader(_FileLoader, SourceLoader): parent = _path_join(parent, part) try: _os.mkdir(parent) - except OSError as exc: + except FileExistsError: # Probably another Python process already created the dir. - if exc.errno == errno.EEXIST: - continue - else: - raise - except IOError as exc: + continue + except PermissionError: # If can't get proper access, then just forget about writing # the data. - if exc.errno == errno.EACCES: - return - else: - raise - try: - with _io.FileIO(path, 'wb') as file: - file.write(data) - except IOError as exc: - # Don't worry if you can't write bytecode. - if exc.errno == errno.EACCES: return - else: - raise + try: + _write_atomic(path, data) + except (PermissionError, FileExistsError): + # Don't worry if you can't write bytecode or someone is writing + # it at the same time. + pass class _SourcelessFileLoader(_FileLoader, _LoaderBasics): @@ -758,14 +806,14 @@ class _ImportLockContext: _IMPLICIT_META_PATH = [BuiltinImporter, FrozenImporter, _DefaultPathFinder] -_ERR_MSG = 'No module named {}' +_ERR_MSG = 'No module named {!r}' def _gcd_import(name, package=None, level=0): """Import and return the module based on its name, the package the call is being made from, and the level adjustment. This function represents the greatest common denominator of functionality - between import_module and __import__. This includes settting __package__ if + between import_module and __import__. This includes setting __package__ if the loader did not. """ @@ -857,7 +905,7 @@ def __import__(name, globals={}, locals={}, fromlist=[], level=0): module = _gcd_import(name) else: # __package__ is not guaranteed to be defined or could be set to None - # to represent that it's proper value is unknown + # to represent that its proper value is unknown package = globals.get('__package__') if package is None: package = globals['__name__'] diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index fa343f8..0d37629 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -123,7 +123,20 @@ class SourceLoader(_bootstrap.SourceLoader, ResourceLoader, ExecutionLoader): def path_mtime(self, path): """Return the (int) modification time for the path (str).""" - raise NotImplementedError + if self.path_stats.__func__ is SourceLoader.path_stats: + raise NotImplementedError + return int(self.path_stats(path)['mtime']) + + def path_stats(self, path): + """Return a metadata dict for the source pointed to by the path (str). + Possible keys: + - 'mtime' (mandatory) is the numeric timestamp of last source + code modification; + - 'size' (optional) is the size in bytes of the source code. + """ + if self.path_mtime.__func__ is SourceLoader.path_mtime: + raise NotImplementedError + return {'mtime': self.path_mtime(path)} def set_data(self, path, data): """Write the bytes to the path (if possible). @@ -195,7 +208,7 @@ class PyLoader(SourceLoader): "use SourceLoader instead. " "See the importlib documentation on how to be " "compatible with Python 3.1 onwards.", - PendingDeprecationWarning) + DeprecationWarning) path = self.source_path(fullname) if path is None: raise ImportError @@ -234,7 +247,7 @@ class PyPycLoader(PyLoader): "removal in Python 3.4; use SourceLoader instead. " "If Python 3.1 compatibility is required, see the " "latest documentation for PyLoader.", - PendingDeprecationWarning) + DeprecationWarning) source_timestamp = self.source_mtime(fullname) # Try to use bytecode if it is available. bytecode_path = self.bytecode_path(fullname) diff --git a/Lib/importlib/test/__main__.py b/Lib/importlib/test/__main__.py index decc53d..a1990b1 100644 --- a/Lib/importlib/test/__main__.py +++ b/Lib/importlib/test/__main__.py @@ -4,7 +4,6 @@ Specifying the ``--builtin`` flag will run tests, where applicable, with builtins.__import__ instead of importlib.__import__. """ -import importlib from importlib.test.import_ import util import os.path from test.support import run_unittest @@ -13,11 +12,7 @@ import unittest def test_main(): - if '__pycache__' in __file__: - parts = __file__.split(os.path.sep) - start_dir = sep.join(parts[:-2]) - else: - start_dir = os.path.dirname(__file__) + start_dir = os.path.dirname(__file__) top_dir = os.path.dirname(os.path.dirname(start_dir)) test_loader = unittest.TestLoader() if '--builtin' in sys.argv: diff --git a/Lib/importlib/test/regrtest.py b/Lib/importlib/test/regrtest.py index b103ae7d..dc0eb97 100644 --- a/Lib/importlib/test/regrtest.py +++ b/Lib/importlib/test/regrtest.py @@ -5,13 +5,6 @@ invalidates are automatically skipped if the entire test suite is run. Otherwise all command-line options valid for test.regrtest are also valid for this script. -XXX FAILING - * test_import - - test_incorrect_code_name - file name differing between __file__ and co_filename (r68360 on trunk) - - test_import_by_filename - exception for trying to import by file name does not match - """ import importlib import sys diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py index 3245907..be81992 100644 --- a/Lib/importlib/test/source/test_abc_loader.py +++ b/Lib/importlib/test/source/test_abc_loader.py @@ -40,8 +40,10 @@ class SourceLoaderMock(SourceOnlyLoaderMock): def __init__(self, path, magic=imp.get_magic()): super().__init__(path) self.bytecode_path = imp.cache_from_source(self.path) + self.source_size = len(self.source) data = bytearray(magic) data.extend(marshal._w_long(self.source_mtime)) + data.extend(marshal._w_long(self.source_size)) code_object = compile(self.source, self.path, 'exec', dont_inherit=True) data.extend(marshal.dumps(code_object)) @@ -56,9 +58,9 @@ class SourceLoaderMock(SourceOnlyLoaderMock): else: raise IOError - def path_mtime(self, path): + def path_stats(self, path): assert path == self.path - return self.source_mtime + return {'mtime': self.source_mtime, 'size': self.source_size} def set_data(self, path, data): self.written[path] = bytes(data) @@ -102,7 +104,7 @@ class PyLoaderMock(abc.PyLoader): warnings.simplefilter("always") path = super().get_filename(name) assert len(w) == 1 - assert issubclass(w[0].category, PendingDeprecationWarning) + assert issubclass(w[0].category, DeprecationWarning) return path @@ -198,7 +200,7 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): warnings.simplefilter("always") code_object = super().get_code(name) assert len(w) == 1 - assert issubclass(w[0].category, PendingDeprecationWarning) + assert issubclass(w[0].category, DeprecationWarning) return code_object class PyLoaderTests(testing_abc.LoaderTests): @@ -657,6 +659,7 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness): self.assertIn(self.cached, self.loader.written) data = bytearray(imp.get_magic()) data.extend(marshal._w_long(self.loader.source_mtime)) + data.extend(marshal._w_long(self.loader.source_size)) data.extend(marshal.dumps(code_object)) self.assertEqual(self.loader.written[self.cached], bytes(data)) @@ -847,7 +850,7 @@ class AbstractMethodImplTests(unittest.TestCase): # Required abstractmethods. self.raises_NotImplementedError(ins, 'get_filename', 'get_data') # Optional abstractmethods. - self.raises_NotImplementedError(ins,'path_mtime', 'set_data') + self.raises_NotImplementedError(ins,'path_stats', 'set_data') def test_PyLoader(self): self.raises_NotImplementedError(self.PyLoader(), 'source_path', diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index de1c4d8..0809077 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -70,11 +70,6 @@ class SimpleTest(unittest.TestCase): module_dict_id = id(module.__dict__) with open(mapping['_temp'], 'w') as file: file.write("testing_var = 42\n") - # For filesystems where the mtime is only to a second granularity, - # everything that has happened above can be too fast; - # force an mtime on the source that is guaranteed to be different - # than the original mtime. - loader.path_mtime = self.fake_mtime(loader.path_mtime) module = loader.load_module('_temp') self.assertTrue('testing_var' in module.__dict__, "'testing_var' not in " @@ -190,10 +185,17 @@ class BadBytecodeTest(unittest.TestCase): del_source=del_source) test('_temp', mapping, bc_path) + def _test_partial_size(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:11], + del_source=del_source) + test('_temp', mapping, bc_path) + def _test_no_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8], + lambda bc: bc[:12], del_source=del_source) file_path = mapping['_temp'] if not del_source else bc_path with self.assertRaises(EOFError): @@ -202,7 +204,7 @@ class BadBytecodeTest(unittest.TestCase): def _test_non_code_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bytecode_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8] + marshal.dumps(b'abcd'), + lambda bc: bc[:12] + marshal.dumps(b'abcd'), del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(ImportError): @@ -211,7 +213,7 @@ class BadBytecodeTest(unittest.TestCase): def _test_bad_marshal(self, *, del_source=False): with source_util.create_modules('_temp') as mapping: bytecode_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8] + b'<test>', + lambda bc: bc[:12] + b'<test>', del_source=del_source) file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(EOFError): @@ -235,7 +237,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_empty_file(test) @@ -243,7 +245,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_partial_magic(test) @@ -254,7 +256,7 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bytecode_path): self.import_(mapping[name], name) with open(bytecode_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_magic_only(test) @@ -276,11 +278,22 @@ class SourceLoaderBadBytecodeTest(BadBytecodeTest): def test(name, mapping, bc_path): self.import_(mapping[name], name) with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) + self.assertGreater(len(file.read()), 12) self._test_partial_timestamp(test) @source_util.writes_bytecode_files + def test_partial_size(self): + # When the size is partial, regenerate the .pyc, else + # raise EOFError. + def test(name, mapping, bc_path): + self.import_(mapping[name], name) + with open(bc_path, 'rb') as file: + self.assertGreater(len(file.read()), 12) + + self._test_partial_size(test) + + @source_util.writes_bytecode_files def test_no_marshal(self): # When there is only the magic number and timestamp, raise EOFError. self._test_no_marshal() @@ -375,6 +388,13 @@ class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): self._test_partial_timestamp(test, del_source=True) + def test_partial_size(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_size(test, del_source=True) + def test_no_marshal(self): self._test_no_marshal(del_source=True) diff --git a/Lib/importlib/test/test_util.py b/Lib/importlib/test/test_util.py index 602447f..c7cdad1 100644 --- a/Lib/importlib/test/test_util.py +++ b/Lib/importlib/test/test_util.py @@ -59,6 +59,11 @@ class ModuleForLoaderTests(unittest.TestCase): self.raise_exception(name) self.assertIs(module, sys.modules[name]) + def test_decorator_attrs(self): + def fxn(self, module): pass + wrapped = util.module_for_loader(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) class SetPackageTests(unittest.TestCase): @@ -108,6 +113,11 @@ class SetPackageTests(unittest.TestCase): module.__package__ = value self.verify(module, value) + def test_decorator_attrs(self): + def fxn(module): pass + wrapped = util.set_package(fxn) + self.assertEqual(wrapped.__name__, fxn.__name__) + self.assertEqual(wrapped.__qualname__, fxn.__qualname__) def test_main(): from test import support |