diff options
Diffstat (limited to 'Lib/importlib/test/source')
| -rw-r--r-- | Lib/importlib/test/source/test_abc_loader.py | 556 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_case_sensitivity.py | 12 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_file_loader.py | 300 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_finder.py | 39 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_path_hook.py | 7 | ||||
| -rw-r--r-- | Lib/importlib/test/source/test_source_encoding.py | 13 | ||||
| -rw-r--r-- | Lib/importlib/test/source/util.py | 19 |
7 files changed, 771 insertions, 175 deletions
diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py index 1ce83fb..3245907 100644 --- a/Lib/importlib/test/source/test_abc_loader.py +++ b/Lib/importlib/test/source/test_abc_loader.py @@ -1,14 +1,68 @@ import importlib from importlib import abc + from .. import abc as testing_abc from .. import util from . import util as source_util + import imp +import inspect +import io import marshal import os import sys import types import unittest +import warnings + + +class SourceOnlyLoaderMock(abc.SourceLoader): + + # Globals that should be defined for all modules. + source = (b"_ = '::'.join([__name__, __file__, __cached__, __package__, " + b"repr(__loader__)])") + + def __init__(self, path): + self.path = path + + def get_data(self, path): + assert self.path == path + return self.source + + def get_filename(self, fullname): + return self.path + + +class SourceLoaderMock(SourceOnlyLoaderMock): + + source_mtime = 1 + + def __init__(self, path, magic=imp.get_magic()): + super().__init__(path) + self.bytecode_path = imp.cache_from_source(self.path) + data = bytearray(magic) + data.extend(marshal._w_long(self.source_mtime)) + code_object = compile(self.source, self.path, 'exec', + dont_inherit=True) + data.extend(marshal.dumps(code_object)) + self.bytecode = bytes(data) + self.written = {} + + def get_data(self, path): + if path == self.path: + return super().get_data(path) + elif path == self.bytecode_path: + return self.bytecode + else: + raise IOError + + def path_mtime(self, path): + assert path == self.path + return self.source_mtime + + def set_data(self, path, data): + self.written[path] = bytes(data) + return path == self.bytecode_path class PyLoaderMock(abc.PyLoader): @@ -33,17 +87,42 @@ class PyLoaderMock(abc.PyLoader): return self.source def is_package(self, name): + filename = os.path.basename(self.get_filename(name)) + return os.path.splitext(filename)[0] == '__init__' + + def source_path(self, name): try: - return '__init__' in self.module_paths[name] + return self.module_paths[name] except KeyError: raise ImportError - def source_path(self, name): + def get_filename(self, name): + """Silence deprecation warning.""" + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + path = super().get_filename(name) + assert len(w) == 1 + assert issubclass(w[0].category, PendingDeprecationWarning) + return path + + +class PyLoaderCompatMock(PyLoaderMock): + + """Mock that matches what is suggested to have a loader that is compatible + from Python 3.1 onwards.""" + + def get_filename(self, fullname): try: - return self.module_paths[name] + return self.module_paths[fullname] except KeyError: raise ImportError + def source_path(self, fullname): + try: + return self.get_filename(fullname) + except ImportError: + return None + class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): @@ -114,6 +193,13 @@ class PyPycLoaderMock(abc.PyPycLoader, PyLoaderMock): except TypeError: return '__init__' in self.bytecode_to_path[name] + def get_code(self, name): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + code_object = super().get_code(name) + assert len(w) == 1 + assert issubclass(w[0].category, PendingDeprecationWarning) + return code_object class PyLoaderTests(testing_abc.LoaderTests): @@ -183,7 +269,8 @@ class PyLoaderTests(testing_abc.LoaderTests): mock.source = b"1/0" with util.uncache(name): sys.modules[name] = module - self.assertRaises(ZeroDivisionError, mock.load_module, name) + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) self.assertTrue(sys.modules[name] is module) self.assertTrue(hasattr(module, 'blah')) return mock @@ -193,11 +280,20 @@ class PyLoaderTests(testing_abc.LoaderTests): mock = self.mocker({name: os.path.join('path', 'to', 'mod')}) mock.source = b"1/0" with util.uncache(name): - self.assertRaises(ZeroDivisionError, mock.load_module, name) + with self.assertRaises(ZeroDivisionError): + mock.load_module(name) self.assertTrue(name not in sys.modules) return mock +class PyLoaderCompatTests(PyLoaderTests): + + """Test that the suggested code to make a loader that is compatible from + Python 3.1 forward works.""" + + mocker = PyLoaderCompatMock + + class PyLoaderInterfaceTests(unittest.TestCase): """Tests for importlib.abc.PyLoader to make sure that when source_path() @@ -207,38 +303,29 @@ class PyLoaderInterfaceTests(unittest.TestCase): # No source path should lead to ImportError. name = 'mod' mock = PyLoaderMock({}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_source_path_is_None(self): name = 'mod' mock = PyLoaderMock({name: None}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) - - -class PyLoaderGetSourceTests(unittest.TestCase): - - """Tests for importlib.abc.PyLoader.get_source().""" + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) - def test_default_encoding(self): - # Should have no problems with UTF-8 text. + def test_get_filename_with_source_path(self): + # get_filename() should return what source_path() returns. name = 'mod' - mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')}) - source = 'x = "ü"' - mock.source = source.encode('utf-8') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) + path = os.path.join('path', 'to', 'source') + mock = PyLoaderMock({name: path}) + with util.uncache(name): + self.assertEqual(mock.get_filename(name), path) - def test_decoded_source(self): - # Decoding should work. + def test_get_filename_no_source_path(self): + # get_filename() should raise ImportError if source_path returns None. name = 'mod' - mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')}) - source = "# coding: Latin-1\nx='ü'" - assert source.encode('latin-1') != source.encode('utf-8') - mock.source = source.encode('latin-1') - returned_source = mock.get_source(name) - self.assertEqual(returned_source, source) + mock = PyLoaderMock({name: None}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) class PyPycLoaderTests(PyLoaderTests): @@ -281,6 +368,38 @@ class PyPycLoaderTests(PyLoaderTests): super().test_unloadable() +class PyPycLoaderInterfaceTests(unittest.TestCase): + + """Test for the interface of importlib.abc.PyPycLoader.""" + + def get_filename_check(self, src_path, bc_path, expect): + name = 'mod' + mock = PyPycLoaderMock({name: src_path}, {name: {'path': bc_path}}) + with util.uncache(name): + assert mock.source_path(name) == src_path + assert mock.bytecode_path(name) == bc_path + self.assertEqual(mock.get_filename(name), expect) + + def test_filename_with_source_bc(self): + # When source and bytecode paths present, return the source path. + self.get_filename_check('source_path', 'bc_path', 'source_path') + + def test_filename_with_source_no_bc(self): + # With source but no bc, return source path. + self.get_filename_check('source_path', None, 'source_path') + + def test_filename_with_no_source_bc(self): + # With not source but bc, return the bc path. + self.get_filename_check(None, 'bc_path', 'bc_path') + + def test_filename_with_no_source_or_bc(self): + # With no source or bc, raise ImportError. + name = 'mod' + mock = PyPycLoaderMock({name: None}, {name: {'path': None}}) + with util.uncache(name), self.assertRaises(ImportError): + mock.get_filename(name) + + class SkipWritingBytecodeTests(unittest.TestCase): """Test that bytecode is properly handled based on @@ -346,20 +465,28 @@ class BadBytecodeFailureTests(unittest.TestCase): # A bad magic number should lead to an ImportError. name = 'mod' bad_magic = b'\x00\x00\x00\x00' - mock = PyPycLoaderMock({name: None}, - {name: {'path': os.path.join('path', 'to', 'mod'), - 'magic': bad_magic}}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + bc = {name: + {'path': os.path.join('path', 'to', 'mod'), + 'magic': bad_magic}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + def test_no_bytecode(self): + # Missing code object bytecode should lead to an EOFError. + name = 'mod' + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b''}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(EOFError): + mock.load_module(name) def test_bad_bytecode(self): - # Bad code object bytecode should lead to an ImportError. + # Malformed code object bytecode should lead to a ValueError. name = 'mod' - mock = PyPycLoaderMock({name: None}, - {name: {'path': os.path.join('path', 'to', 'mod'), - 'bc': b''}}) - with util.uncache(name): - self.assertRaises(EOFError, mock.load_module, name) + bc = {name: {'path': os.path.join('path', 'to', 'mod'), 'bc': b'1234'}} + mock = PyPycLoaderMock({name: None}, bc) + with util.uncache(name), self.assertRaises(ValueError): + mock.load_module(name) def raise_ImportError(*args, **kwargs): @@ -387,16 +514,16 @@ class MissingPathsTests(unittest.TestCase): # If all *_path methods return None, raise ImportError. name = 'mod' mock = PyPycLoaderMock({name: None}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_source_path_ImportError(self): # An ImportError from source_path should trigger an ImportError. name = 'mod' mock = PyPycLoaderMock({}, {name: {'path': os.path.join('path', 'to', 'mod')}}) - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) def test_bytecode_path_ImportError(self): # An ImportError from bytecode_path should trigger an ImportError. @@ -404,16 +531,345 @@ class MissingPathsTests(unittest.TestCase): mock = PyPycLoaderMock({name: os.path.join('path', 'to', 'mod')}) bad_meth = types.MethodType(raise_ImportError, mock) mock.bytecode_path = bad_meth - with util.uncache(name): - self.assertRaises(ImportError, mock.load_module, name) + with util.uncache(name), self.assertRaises(ImportError): + mock.load_module(name) + + +class SourceLoaderTestHarness(unittest.TestCase): + + def setUp(self, *, is_package=True, **kwargs): + self.package = 'pkg' + if is_package: + self.path = os.path.join(self.package, '__init__.py') + self.name = self.package + else: + module_name = 'mod' + self.path = os.path.join(self.package, '.'.join(['mod', 'py'])) + self.name = '.'.join([self.package, module_name]) + self.cached = imp.cache_from_source(self.path) + self.loader = self.loader_mock(self.path, **kwargs) + + def verify_module(self, module): + self.assertEqual(module.__name__, self.name) + self.assertEqual(module.__file__, self.path) + self.assertEqual(module.__cached__, self.cached) + self.assertEqual(module.__package__, self.package) + self.assertEqual(module.__loader__, self.loader) + values = module._.split('::') + self.assertEqual(values[0], self.name) + self.assertEqual(values[1], self.path) + self.assertEqual(values[2], self.cached) + self.assertEqual(values[3], self.package) + self.assertEqual(values[4], repr(self.loader)) + + def verify_code(self, code_object): + module = imp.new_module(self.name) + module.__file__ = self.path + module.__cached__ = self.cached + module.__package__ = self.package + module.__loader__ = self.loader + module.__path__ = [] + exec(code_object, module.__dict__) + self.verify_module(module) + + +class SourceOnlyLoaderTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader for source-only loading. + + Reload testing is subsumed by the tests for + importlib.util.module_for_loader. + + """ + + loader_mock = SourceOnlyLoaderMock + + def test_get_source(self): + # Verify the source code is returned as a string. + # If an IOError is raised by get_data then raise ImportError. + expected_source = self.loader.source.decode('utf-8') + self.assertEqual(self.loader.get_source(self.name), expected_source) + def raise_IOError(path): + raise IOError + self.loader.get_data = raise_IOError + with self.assertRaises(ImportError): + self.loader.get_source(self.name) + + def test_is_package(self): + # Properly detect when loading a package. + self.setUp(is_package=True) + self.assertTrue(self.loader.is_package(self.name)) + self.setUp(is_package=False) + self.assertFalse(self.loader.is_package(self.name)) + + def test_get_code(self): + # Verify the code object is created. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_load_module(self): + # Loading a module should set __name__, __loader__, __package__, + # __path__ (for packages), __file__, and __cached__. + # The module should also be put into sys.modules. + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertEqual(module.__path__, [os.path.dirname(self.path)]) + self.assertTrue(self.name in sys.modules) + + def test_package_settings(self): + # __package__ needs to be set, while __path__ is set on if the module + # is a package. + # Testing the values for a package are covered by test_load_module. + self.setUp(is_package=False) + with util.uncache(self.name): + module = self.loader.load_module(self.name) + self.verify_module(module) + self.assertTrue(not hasattr(module, '__path__')) + + def test_get_source_encoding(self): + # Source is considered encoded in UTF-8 by default unless otherwise + # specified by an encoding line. + source = "_ = 'ü'" + self.loader.source = source.encode('utf-8') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + source = "# coding: latin-1\n_ = ü" + self.loader.source = source.encode('latin-1') + returned_source = self.loader.get_source(self.name) + self.assertEqual(returned_source, source) + + +@unittest.skipIf(sys.dont_write_bytecode, "sys.dont_write_bytecode is true") +class SourceLoaderBytecodeTests(SourceLoaderTestHarness): + + """Test importlib.abc.SourceLoader's use of bytecode. + + Source-only testing handled by SourceOnlyLoaderTests. + + """ + + loader_mock = SourceLoaderMock + + def verify_code(self, code_object, *, bytecode_written=False): + super().verify_code(code_object) + if bytecode_written: + self.assertIn(self.cached, self.loader.written) + data = bytearray(imp.get_magic()) + data.extend(marshal._w_long(self.loader.source_mtime)) + data.extend(marshal.dumps(code_object)) + self.assertEqual(self.loader.written[self.cached], bytes(data)) + + def test_code_with_everything(self): + # When everything should work. + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + def test_no_bytecode(self): + # If no bytecode exists then move on to the source. + self.loader.bytecode_path = "<does not exist>" + # Sanity check + with self.assertRaises(IOError): + bytecode_path = imp.cache_from_source(self.path) + self.loader.get_data(bytecode_path) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_code_bad_timestamp(self): + # Bytecode is only used when the timestamp matches the source EXACTLY. + for source_mtime in (0, 2): + assert source_mtime != self.loader.source_mtime + original = self.loader.source_mtime + self.loader.source_mtime = source_mtime + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + self.loader.source_mtime = original + + def test_code_bad_magic(self): + # Skip over bytecode with a bad magic number. + self.setUp(magic=b'0000') + # If bytecode is used then EOFError would be raised by marshal. + self.loader.bytecode = self.loader.bytecode[8:] + code_object = self.loader.get_code(self.name) + self.verify_code(code_object, bytecode_written=True) + + def test_dont_write_bytecode(self): + # Bytecode is not written if sys.dont_write_bytecode is true. + # Can assume it is false already thanks to the skipIf class decorator. + try: + sys.dont_write_bytecode = True + self.loader.bytecode_path = "<does not exist>" + code_object = self.loader.get_code(self.name) + self.assertNotIn(self.cached, self.loader.written) + finally: + sys.dont_write_bytecode = False + + def test_no_set_data(self): + # If set_data is not defined, one can still read bytecode. + self.setUp(magic=b'0000') + original_set_data = self.loader.__class__.set_data + try: + del self.loader.__class__.set_data + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + finally: + self.loader.__class__.set_data = original_set_data + + def test_set_data_raises_exceptions(self): + # Raising NotImplementedError or IOError is okay for set_data. + def raise_exception(exc): + def closure(*args, **kwargs): + raise exc + return closure + + self.setUp(magic=b'0000') + self.loader.set_data = raise_exception(NotImplementedError) + code_object = self.loader.get_code(self.name) + self.verify_code(code_object) + + +class SourceLoaderGetSourceTests(unittest.TestCase): + + """Tests for importlib.abc.SourceLoader.get_source().""" + + def test_default_encoding(self): + # Should have no problems with UTF-8 text. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = 'x = "ü"' + mock.source = source.encode('utf-8') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_decoded_source(self): + # Decoding should work. + name = 'mod' + mock = SourceOnlyLoaderMock("mod.file") + source = "# coding: Latin-1\nx='ü'" + assert source.encode('latin-1') != source.encode('utf-8') + mock.source = source.encode('latin-1') + returned_source = mock.get_source(name) + self.assertEqual(returned_source, source) + + def test_universal_newlines(self): + # PEP 302 says universal newlines should be used. + name = 'mod' + mock = SourceOnlyLoaderMock('mod.file') + source = "x = 42\r\ny = -13\r\n" + mock.source = source.encode('utf-8') + expect = io.IncrementalNewlineDecoder(None, True).decode(source) + self.assertEqual(mock.get_source(name), expect) + +class AbstractMethodImplTests(unittest.TestCase): + + """Test the concrete abstractmethod implementations.""" + + class Loader(abc.Loader): + def load_module(self, fullname): + super().load_module(fullname) + + class Finder(abc.Finder): + def find_module(self, _): + super().find_module(_) + + class ResourceLoader(Loader, abc.ResourceLoader): + def get_data(self, _): + super().get_data(_) + + class InspectLoader(Loader, abc.InspectLoader): + def is_package(self, _): + super().is_package(_) + + def get_code(self, _): + super().get_code(_) + + def get_source(self, _): + super().get_source(_) + + class ExecutionLoader(InspectLoader, abc.ExecutionLoader): + def get_filename(self, _): + super().get_filename(_) + + class SourceLoader(ResourceLoader, ExecutionLoader, abc.SourceLoader): + pass + + class PyLoader(ResourceLoader, InspectLoader, abc.PyLoader): + def source_path(self, _): + super().source_path(_) + + class PyPycLoader(PyLoader, abc.PyPycLoader): + def bytecode_path(self, _): + super().bytecode_path(_) + + def source_mtime(self, _): + super().source_mtime(_) + + def write_bytecode(self, _, _2): + super().write_bytecode(_, _2) + + def raises_NotImplementedError(self, ins, *args): + for method_name in args: + method = getattr(ins, method_name) + arg_count = len(inspect.getfullargspec(method)[0]) - 1 + args = [''] * arg_count + try: + method(*args) + except NotImplementedError: + pass + else: + msg = "{}.{} did not raise NotImplementedError" + self.fail(msg.format(ins.__class__.__name__, method_name)) + + def test_Loader(self): + self.raises_NotImplementedError(self.Loader(), 'load_module') + + # XXX misplaced; should be somewhere else + def test_Finder(self): + self.raises_NotImplementedError(self.Finder(), 'find_module') + + def test_ResourceLoader(self): + self.raises_NotImplementedError(self.ResourceLoader(), 'load_module', + 'get_data') + + def test_InspectLoader(self): + self.raises_NotImplementedError(self.InspectLoader(), 'load_module', + 'is_package', 'get_code', 'get_source') + + def test_ExecutionLoader(self): + self.raises_NotImplementedError(self.ExecutionLoader(), 'load_module', + 'is_package', 'get_code', 'get_source', + 'get_filename') + + def test_SourceLoader(self): + ins = self.SourceLoader() + # Required abstractmethods. + self.raises_NotImplementedError(ins, 'get_filename', 'get_data') + # Optional abstractmethods. + self.raises_NotImplementedError(ins,'path_mtime', 'set_data') + + def test_PyLoader(self): + self.raises_NotImplementedError(self.PyLoader(), 'source_path', + 'get_data', 'is_package') + + def test_PyPycLoader(self): + self.raises_NotImplementedError(self.PyPycLoader(), 'source_path', + 'source_mtime', 'bytecode_path', + 'write_bytecode') def test_main(): from test.support import run_unittest - run_unittest(PyLoaderTests, PyLoaderInterfaceTests, PyLoaderGetSourceTests, - PyPycLoaderTests, SkipWritingBytecodeTests, - RegeneratedBytecodeTests, BadBytecodeFailureTests, - MissingPathsTests) + run_unittest(PyLoaderTests, PyLoaderCompatTests, + PyLoaderInterfaceTests, + PyPycLoaderTests, PyPycLoaderInterfaceTests, + SkipWritingBytecodeTests, RegeneratedBytecodeTests, + BadBytecodeFailureTests, MissingPathsTests, + SourceOnlyLoaderTests, + SourceLoaderBytecodeTests, + SourceLoaderGetSourceTests, + AbstractMethodImplTests) if __name__ == '__main__': diff --git a/Lib/importlib/test/source/test_case_sensitivity.py b/Lib/importlib/test/source/test_case_sensitivity.py index 6fad881..73777de 100644 --- a/Lib/importlib/test/source/test_case_sensitivity.py +++ b/Lib/importlib/test/source/test_case_sensitivity.py @@ -19,7 +19,9 @@ class CaseSensitivityTest(unittest.TestCase): assert name != name.lower() def find(self, path): - finder = _bootstrap._PyPycFileFinder(path) + finder = _bootstrap._FileFinder(path, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) return finder.find_module(self.name) def sensitivity_test(self): @@ -27,7 +29,7 @@ class CaseSensitivityTest(unittest.TestCase): sensitive_pkg = 'sensitive.{0}'.format(self.name) insensitive_pkg = 'insensitive.{0}'.format(self.name.lower()) context = source_util.create_modules(insensitive_pkg, sensitive_pkg) - with context as mapping: + with context as mapping: sensitive_path = os.path.join(mapping['.root'], 'sensitive') insensitive_path = os.path.join(mapping['.root'], 'insensitive') return self.find(sensitive_path), self.find(insensitive_path) @@ -37,7 +39,7 @@ class CaseSensitivityTest(unittest.TestCase): env.unset('PYTHONCASEOK') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive._base_path) + self.assertIn(self.name, sensitive.get_filename(self.name)) self.assertIsNone(insensitive) def test_insensitive(self): @@ -45,9 +47,9 @@ class CaseSensitivityTest(unittest.TestCase): env.set('PYTHONCASEOK', '1') sensitive, insensitive = self.sensitivity_test() self.assertTrue(hasattr(sensitive, 'load_module')) - self.assertIn(self.name, sensitive._base_path) + self.assertIn(self.name, sensitive.get_filename(self.name)) self.assertTrue(hasattr(insensitive, 'load_module')) - self.assertIn(self.name, insensitive._base_path) + self.assertIn(self.name, insensitive.get_filename(self.name)) def test_main(): diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py index 145d076..0ffe78d 100644 --- a/Lib/importlib/test/source/test_file_loader.py +++ b/Lib/importlib/test/source/test_file_loader.py @@ -1,15 +1,20 @@ import importlib from importlib import _bootstrap from .. import abc +from .. import util from . import util as source_util import imp +import marshal import os import py_compile +import shutil import stat import sys import unittest +from test.support import make_legacy_pyc + class SimpleTest(unittest.TestCase): @@ -21,8 +26,7 @@ class SimpleTest(unittest.TestCase): # [basic] def test_module(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') self.assertTrue('_temp' in sys.modules) check = {'__name__': '_temp', '__file__': mapping['_temp'], @@ -32,9 +36,8 @@ class SimpleTest(unittest.TestCase): def test_package(self): with source_util.create_modules('_pkg.__init__') as mapping: - loader = _bootstrap._PyPycFileLoader('_pkg', - mapping['_pkg.__init__'], - True) + loader = _bootstrap._SourceFileLoader('_pkg', + mapping['_pkg.__init__']) module = loader.load_module('_pkg') self.assertTrue('_pkg' in sys.modules) check = {'__name__': '_pkg', '__file__': mapping['_pkg.__init__'], @@ -46,8 +49,8 @@ class SimpleTest(unittest.TestCase): def test_lacking_parent(self): with source_util.create_modules('_pkg.__init__', '_pkg.mod')as mapping: - loader = _bootstrap._PyPycFileLoader('_pkg.mod', - mapping['_pkg.mod'], False) + loader = _bootstrap._SourceFileLoader('_pkg.mod', + mapping['_pkg.mod']) module = loader.load_module('_pkg.mod') self.assertTrue('_pkg.mod' in sys.modules) check = {'__name__': '_pkg.mod', '__file__': mapping['_pkg.mod'], @@ -61,8 +64,7 @@ class SimpleTest(unittest.TestCase): def test_module_reuse(self): with source_util.create_modules('_temp') as mapping: - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) module = loader.load_module('_temp') module_id = id(module) module_dict_id = id(module.__dict__) @@ -72,7 +74,7 @@ class SimpleTest(unittest.TestCase): # everything that has happened above can be too fast; # force an mtime on the source that is guaranteed to be different # than the original mtime. - loader.source_mtime = self.fake_mtime(loader.source_mtime) + loader.path_mtime = self.fake_mtime(loader.path_mtime) module = loader.load_module('_temp') self.assertTrue('testing_var' in module.__dict__, "'testing_var' not in " @@ -92,9 +94,9 @@ class SimpleTest(unittest.TestCase): setattr(orig_module, attr, value) with open(mapping[name], 'w') as file: file.write('+++ bad syntax +++') - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) - self.assertRaises(SyntaxError, loader.load_module, name) + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module(name) for attr in attributes: self.assertEqual(getattr(orig_module, attr), value) @@ -103,16 +105,34 @@ class SimpleTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: with open(mapping['_temp'], 'w') as file: file.write('=') - loader = _bootstrap._PyPycFileLoader('_temp', mapping['_temp'], - False) - self.assertRaises(SyntaxError, loader.load_module, '_temp') + loader = _bootstrap._SourceFileLoader('_temp', mapping['_temp']) + with self.assertRaises(SyntaxError): + loader.load_module('_temp') self.assertTrue('_temp' not in sys.modules) + def test_file_from_empty_string_dir(self): + # Loading a module found from an empty string entry on sys.path should + # not only work, but keep all attributes relative. + file_path = '_temp.py' + with open(file_path, 'w') as file: + file.write("# test file for importlib") + try: + with util.uncache('_temp'): + loader = _bootstrap._SourceFileLoader('_temp', file_path) + mod = loader.load_module('_temp') + self.assertEqual(file_path, mod.__file__) + self.assertEqual(imp.cache_from_source(file_path), + mod.__cached__) + finally: + os.unlink(file_path) + pycache = os.path.dirname(imp.cache_from_source(file_path)) + shutil.rmtree(pycache) + class BadBytecodeTest(unittest.TestCase): def import_(self, file, module_name): - loader = _bootstrap._PyPycFileLoader(module_name, file, False) + loader = self.loader(module_name, file) module = loader.load_module(module_name) self.assertTrue(module_name in sys.modules) @@ -125,106 +145,162 @@ class BadBytecodeTest(unittest.TestCase): except KeyError: pass py_compile.compile(mapping[name]) - bytecode_path = source_util.bytecode_path(mapping[name]) - with open(bytecode_path, 'rb') as file: - bc = file.read() - new_bc = manipulator(bc) - with open(bytecode_path, 'wb') as file: - if new_bc: - file.write(new_bc) - if del_source: + if not del_source: + bytecode_path = imp.cache_from_source(mapping[name]) + else: os.unlink(mapping[name]) + bytecode_path = make_legacy_pyc(mapping[name]) + if manipulator: + with open(bytecode_path, 'rb') as file: + bc = file.read() + new_bc = manipulator(bc) + with open(bytecode_path, 'wb') as file: + if new_bc is not None: + file.write(new_bc) return bytecode_path - @source_util.writes_bytecode_files - def test_empty_file(self): - # When a .pyc is empty, regenerate it if possible, else raise - # ImportError. + def _test_empty_file(self, test, *, del_source=False): with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: None) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: None, - del_source=True) - with self.assertRaises(ImportError): - self.import_(mapping['_temp'], '_temp') + lambda bc: b'', + del_source=del_source) + test('_temp', mapping, bc_path) @source_util.writes_bytecode_files - def test_partial_magic(self): + def _test_partial_magic(self, test, *, del_source=False): # When their are less than 4 bytes to a .pyc, regenerate it if # possible, else raise ImportError. with source_util.create_modules('_temp') as mapping: bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:3]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: - self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:3], - del_source=True) + lambda bc: bc[:3], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_magic_only(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:4], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_partial_timestamp(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:7], + del_source=del_source) + test('_temp', mapping, bc_path) + + def _test_no_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8], + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bc_path + with self.assertRaises(EOFError): + self.import_(file_path, '_temp') + + def _test_non_code_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8] + marshal.dumps(b'abcd'), + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path with self.assertRaises(ImportError): - self.import_(mapping['_temp'], '_temp') + self.import_(file_path, '_temp') + + def _test_bad_marshal(self, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bytecode_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: bc[:8] + b'<test>', + del_source=del_source) + file_path = mapping['_temp'] if not del_source else bytecode_path + with self.assertRaises(ValueError): + self.import_(file_path, '_temp') + + def _test_bad_magic(self, test, *, del_source=False): + with source_util.create_modules('_temp') as mapping: + bc_path = self.manipulate_bytecode('_temp', mapping, + lambda bc: b'\x00\x00\x00\x00' + bc[4:]) + test('_temp', mapping, bc_path) + + +class SourceLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourceFileLoader + + @source_util.writes_bytecode_files + def test_empty_file(self): + # When a .pyc is empty, regenerate it if possible, else raise + # ImportError. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 8) + + self._test_empty_file(test) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: + self.assertGreater(len(file.read()), 8) + + self._test_partial_magic(test) @source_util.writes_bytecode_files def test_magic_only(self): # When there is only the magic number, regenerate the .pyc if possible, # else raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:4]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as file: + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as file: self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:4], - del_source=True) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') + + @source_util.writes_bytecode_files + def test_bad_magic(self): + # When the magic number is different, the bytecode should be + # regenerated. + def test(name, mapping, bytecode_path): + self.import_(mapping[name], name) + with open(bytecode_path, 'rb') as bytecode_file: + self.assertEqual(bytecode_file.read(4), imp.get_magic()) + + self._test_bad_magic(test) @source_util.writes_bytecode_files def test_partial_timestamp(self): # When the timestamp is partial, regenerate the .pyc, else # raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:7]) - self.import_(mapping['_temp'], '_temp') + def test(name, mapping, bc_path): + self.import_(mapping[name], name) with open(bc_path, 'rb') as file: self.assertGreater(len(file.read()), 8) - self.manipulate_bytecode('_temp', mapping, lambda bc: bc[:7], - del_source=True) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') @source_util.writes_bytecode_files def test_no_marshal(self): # When there is only the magic number and timestamp, raise EOFError. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: bc[:8]) - with self.assertRaises(EOFError): - self.import_(mapping['_temp'], '_temp') + self._test_no_marshal() @source_util.writes_bytecode_files - def test_bad_magic(self): - # When the magic number is different, the bytecode should be - # regenerated. - with source_util.create_modules('_temp') as mapping: - bc_path = self.manipulate_bytecode('_temp', mapping, - lambda bc: b'\x00\x00\x00\x00' + bc[4:]) - self.import_(mapping['_temp'], '_temp') - with open(bc_path, 'rb') as bytecode_file: - self.assertEqual(bytecode_file.read(4), imp.get_magic()) + def test_non_code_marshal(self): + self._test_non_code_marshal() + # XXX ImportError when sourceless + + # [bad marshal] + @source_util.writes_bytecode_files + def test_bad_marshal(self): + # Bad marshal data should raise a ValueError. + self._test_bad_marshal() # [bad timestamp] @source_util.writes_bytecode_files - def test_bad_bytecode(self): + def test_old_timestamp(self): # When the timestamp is older than the source, bytecode should be # regenerated. zeros = b'\x00\x00\x00\x00' with source_util.create_modules('_temp') as mapping: py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(4) bytecode_file.write(zeros) @@ -235,22 +311,6 @@ class BadBytecodeTest(unittest.TestCase): bytecode_file.seek(4) self.assertEqual(bytecode_file.read(4), source_timestamp) - # [bad marshal] - @source_util.writes_bytecode_files - def test_bad_marshal(self): - # Bad marshal data should raise a ValueError. - with source_util.create_modules('_temp') as mapping: - bytecode_path = source_util.bytecode_path(mapping['_temp']) - source_mtime = os.path.getmtime(mapping['_temp']) - source_timestamp = importlib._w_long(source_mtime) - with open(bytecode_path, 'wb') as bytecode_file: - bytecode_file.write(imp.get_magic()) - bytecode_file.write(source_timestamp) - bytecode_file.write(b'AAAA') - self.assertRaises(ValueError, self.import_, mapping['_temp'], - '_temp') - self.assertTrue('_temp' not in sys.modules) - # [bytecode read-only] @source_util.writes_bytecode_files def test_read_only_bytecode(self): @@ -258,7 +318,7 @@ class BadBytecodeTest(unittest.TestCase): with source_util.create_modules('_temp') as mapping: # Create bytecode that will need to be re-created. py_compile.compile(mapping['_temp']) - bytecode_path = source_util.bytecode_path(mapping['_temp']) + bytecode_path = imp.cache_from_source(mapping['_temp']) with open(bytecode_path, 'r+b') as bytecode_file: bytecode_file.seek(0) bytecode_file.write(b'\x00\x00\x00\x00') @@ -273,9 +333,57 @@ class BadBytecodeTest(unittest.TestCase): os.chmod(bytecode_path, stat.S_IWUSR) +class SourcelessLoaderBadBytecodeTest(BadBytecodeTest): + + loader = _bootstrap._SourcelessFileLoader + + def test_empty_file(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_empty_file(test, del_source=True) + + def test_partial_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + self._test_partial_magic(test, del_source=True) + + def test_magic_only(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_magic_only(test, del_source=True) + + def test_bad_magic(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(ImportError): + self.import_(bytecode_path, name) + + self._test_bad_magic(test, del_source=True) + + def test_partial_timestamp(self): + def test(name, mapping, bytecode_path): + with self.assertRaises(EOFError): + self.import_(bytecode_path, name) + + self._test_partial_timestamp(test, del_source=True) + + def test_no_marshal(self): + self._test_no_marshal(del_source=True) + + def test_non_code_marshal(self): + self._test_non_code_marshal(del_source=True) + + def test_main(): from test.support import run_unittest - run_unittest(SimpleTest, BadBytecodeTest) + run_unittest(SimpleTest, + SourceLoaderBadBytecodeTest, + SourcelessLoaderBadBytecodeTest + ) if __name__ == '__main__': diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py index c495c5a..7b9088d 100644 --- a/Lib/importlib/test/source/test_finder.py +++ b/Lib/importlib/test/source/test_finder.py @@ -1,7 +1,9 @@ from importlib import _bootstrap from .. import abc from . import util as source_util +from test.support import make_legacy_pyc import os +import errno import py_compile import unittest import warnings @@ -32,7 +34,9 @@ class FinderTests(abc.FinderTests): """ def import_(self, root, module): - finder = _bootstrap._PyPycFileFinder(root) + finder = _bootstrap._FileFinder(root, + _bootstrap._SourceFinderDetails(), + _bootstrap._SourcelessFinderDetails()) return finder.find_module(module) def run_test(self, test, create=None, *, compile_=None, unlink=None): @@ -52,6 +56,14 @@ class FinderTests(abc.FinderTests): if unlink: for name in unlink: os.unlink(mapping[name]) + try: + make_legacy_pyc(mapping[name]) + except OSError as error: + # Some tests do not set compile_=True so the source + # module will not get compiled and there will be no + # PEP 3147 pyc file to rename. + if error.errno != errno.ENOENT: + raise loader = self.import_(mapping['.root'], test) self.assertTrue(hasattr(loader, 'load_module')) return loader @@ -60,7 +72,8 @@ class FinderTests(abc.FinderTests): # [top-level source] self.run_test('top_level') # [top-level bc] - self.run_test('top_level', compile_={'top_level'}, unlink={'top_level'}) + self.run_test('top_level', compile_={'top_level'}, + unlink={'top_level'}) # [top-level both] self.run_test('top_level', compile_={'top_level'}) @@ -97,15 +110,14 @@ class FinderTests(abc.FinderTests): with context as mapping: os.unlink(mapping['pkg.sub.__init__']) pkg_dir = os.path.dirname(mapping['pkg.__init__']) - self.assertRaises(ImportWarning, self.import_, pkg_dir, - 'pkg.sub') + with self.assertRaises(ImportWarning): + self.import_(pkg_dir, 'pkg.sub') # [package over modules] def test_package_over_module(self): - # XXX This is not a blackbox test! name = '_temp' loader = self.run_test(name, {'{0}.__init__'.format(name), name}) - self.assertTrue('__init__' in loader._base_path) + self.assertTrue('__init__' in loader.get_filename(name)) def test_failure(self): @@ -117,8 +129,19 @@ class FinderTests(abc.FinderTests): def test_empty_dir(self): with warnings.catch_warnings(): warnings.simplefilter("error", ImportWarning) - self.assertRaises(ImportWarning, self.run_test, 'pkg', - {'pkg.__init__'}, unlink={'pkg.__init__'}) + with self.assertRaises(ImportWarning): + self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'}) + + def test_empty_string_for_dir(self): + # The empty string from sys.path means to search in the cwd. + finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails()) + with open('mod.py', 'w') as file: + file.write("# test file for importlib") + try: + loader = finder.find_module('mod') + self.assertTrue(hasattr(loader, 'load_module')) + finally: + os.unlink('mod.py') def test_main(): diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py index 3efb3be..374f7b6 100644 --- a/Lib/importlib/test/source/test_path_hook.py +++ b/Lib/importlib/test/source/test_path_hook.py @@ -8,11 +8,14 @@ class PathHookTest(unittest.TestCase): """Test the path hook for source.""" def test_success(self): - # XXX Only work on existing directories? with source_util.create_modules('dummy') as mapping: - self.assertTrue(hasattr(_bootstrap._FileFinder(mapping['.root']), + self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']), 'find_module')) + def test_empty_string(self): + # The empty string represents the cwd. + self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module')) + def test_main(): from test.support import run_unittest diff --git a/Lib/importlib/test/source/test_source_encoding.py b/Lib/importlib/test/source/test_source_encoding.py index 3734712..794a3df 100644 --- a/Lib/importlib/test/source/test_source_encoding.py +++ b/Lib/importlib/test/source/test_source_encoding.py @@ -33,10 +33,10 @@ class EncodingTest(unittest.TestCase): def run_test(self, source): with source_util.create_modules(self.module_name) as mapping: - with open(mapping[self.module_name], 'wb')as file: + with open(mapping[self.module_name], 'wb') as file: file.write(source) - loader = _bootstrap._PyPycFileLoader(self.module_name, - mapping[self.module_name], False) + loader = _bootstrap._SourceFileLoader(self.module_name, + mapping[self.module_name]) return loader.load_module(self.module_name) def create_source(self, encoding): @@ -81,7 +81,8 @@ class EncodingTest(unittest.TestCase): # [BOM conflict] def test_bom_conflict(self): source = codecs.BOM_UTF8 + self.create_source('latin-1') - self.assertRaises(SyntaxError, self.run_test, source) + with self.assertRaises(SyntaxError): + self.run_test(source) class LineEndingTest(unittest.TestCase): @@ -96,8 +97,8 @@ class LineEndingTest(unittest.TestCase): with source_util.create_modules(module_name) as mapping: with open(mapping[module_name], 'wb') as file: file.write(source) - loader = _bootstrap._PyPycFileLoader(module_name, - mapping[module_name], False) + loader = _bootstrap._SourceFileLoader(module_name, + mapping[module_name]) return loader.load_module(module_name) # [cr] diff --git a/Lib/importlib/test/source/util.py b/Lib/importlib/test/source/util.py index 2b945c5..ae65663 100644 --- a/Lib/importlib/test/source/util.py +++ b/Lib/importlib/test/source/util.py @@ -1,5 +1,6 @@ from .. import util import contextlib +import errno import functools import imp import os @@ -26,14 +27,16 @@ def writes_bytecode_files(fxn): return wrapper -def bytecode_path(source_path): - for suffix, _, type_ in imp.get_suffixes(): - if type_ == imp.PY_COMPILED: - bc_suffix = suffix - break - else: - raise ValueError("no bytecode suffix is defined") - return os.path.splitext(source_path)[0] + bc_suffix +def ensure_bytecode_path(bytecode_path): + """Ensure that the __pycache__ directory for PEP 3147 pyc file exists. + + :param bytecode_path: File system path to PEP 3147 pyc file. + """ + try: + os.mkdir(os.path.dirname(bytecode_path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise @contextlib.contextmanager |