bpo-32248 - Implement importlib.resources (#4911)

Port importlib_resources to importlib.resources

24 files changed, 473 insertions, 0 deletions

diff --git a/Lib/test/test_importlib/data01/__init__.py b/Lib/test/test_importlib/data01/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data01/__init__.py
diff --git a/Lib/test/test_importlib/data01/binary.file b/Lib/test/test_importlib/data01/binary.file
new file mode 100644
index 0000000..eaf36c1
--- /dev/null
+++ b/Lib/test/test_importlib/data01/binary.file
diff --git a/Lib/test/test_importlib/data01/subdirectory/__init__.py b/Lib/test/test_importlib/data01/subdirectory/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data01/subdirectory/__init__.py
diff --git a/Lib/test/test_importlib/data01/subdirectory/binary.file b/Lib/test/test_importlib/data01/subdirectory/binary.file
new file mode 100644
index 0000000..eaf36c1
--- /dev/null
+++ b/Lib/test/test_importlib/data01/subdirectory/binary.file
diff --git a/Lib/test/test_importlib/data01/utf-16.file b/Lib/test/test_importlib/data01/utf-16.file
new file mode 100644
index 0000000..2cb7722
--- /dev/null
+++ b/Lib/test/test_importlib/data01/utf-16.file
diff --git a/Lib/test/test_importlib/data01/utf-8.file b/Lib/test/test_importlib/data01/utf-8.file
new file mode 100644
index 0000000..1c0132a
--- /dev/null
+++ b/Lib/test/test_importlib/data01/utf-8.file
@@ -0,0 +1 @@
+Hello, UTF-8 world!
diff --git a/Lib/test/test_importlib/data02/__init__.py b/Lib/test/test_importlib/data02/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data02/__init__.py
diff --git a/Lib/test/test_importlib/data02/one/__init__.py b/Lib/test/test_importlib/data02/one/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data02/one/__init__.py
diff --git a/Lib/test/test_importlib/data02/one/resource1.txt b/Lib/test/test_importlib/data02/one/resource1.txt
new file mode 100644
index 0000000..61a813e
--- /dev/null
+++ b/Lib/test/test_importlib/data02/one/resource1.txt
@@ -0,0 +1 @@
+one resource
diff --git a/Lib/test/test_importlib/data02/two/__init__.py b/Lib/test/test_importlib/data02/two/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data02/two/__init__.py
diff --git a/Lib/test/test_importlib/data02/two/resource2.txt b/Lib/test/test_importlib/data02/two/resource2.txt
new file mode 100644
index 0000000..a80ce46
--- /dev/null
+++ b/Lib/test/test_importlib/data02/two/resource2.txt
@@ -0,0 +1 @@
+two resource
diff --git a/Lib/test/test_importlib/data03/__init__.py b/Lib/test/test_importlib/data03/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data03/__init__.py
diff --git a/Lib/test/test_importlib/data03/namespace/portion1/__init__.py b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data03/namespace/portion1/__init__.py
diff --git a/Lib/test/test_importlib/data03/namespace/portion2/__init__.py b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data03/namespace/portion2/__init__.py
diff --git a/Lib/test/test_importlib/data03/namespace/resource1.txt b/Lib/test/test_importlib/data03/namespace/resource1.txt
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/data03/namespace/resource1.txt
diff --git a/Lib/test/test_importlib/test_open.py b/Lib/test/test_importlib/test_open.py
new file mode 100644
index 0000000..ad236c6
--- /dev/null
+++ b/Lib/test/test_importlib/test_open.py
@@ -0,0 +1,72 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.open_binary(package, path):
+            pass
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.open_text(package, path):
+            pass
+
+
+class OpenTests:
+    def test_open_binary(self):
+        with resources.open_binary(self.data, 'utf-8.file') as fp:
+            result = fp.read()
+            self.assertEqual(result, b'Hello, UTF-8 world!\n')
+
+    def test_open_text_default_encoding(self):
+        with resources.open_text(self.data, 'utf-8.file') as fp:
+            result = fp.read()
+            self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+    def test_open_text_given_encoding(self):
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-16', 'strict') as fp:
+            result = fp.read()
+        self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+    def test_open_text_with_errors(self):
+        # Raises UnicodeError without the 'errors' argument.
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-8', 'strict') as fp:
+            self.assertRaises(UnicodeError, fp.read)
+        with resources.open_text(
+                self.data, 'utf-16.file', 'utf-8', 'ignore') as fp:
+            result = fp.read()
+        self.assertEqual(
+            result,
+            'H\x00e\x00l\x00l\x00o\x00,\x00 '
+            '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+            '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+    def test_open_binary_FileNotFoundError(self):
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_binary, self.data, 'does-not-exist')
+
+    def test_open_text_FileNotFoundError(self):
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_text, self.data, 'does-not-exist')
+
+
+class OpenDiskTests(OpenTests, unittest.TestCase):
+    def setUp(self):
+        self.data = data01
+
+
+class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase):
+    pass
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_importlib/test_path.py b/Lib/test/test_importlib/test_path.py
new file mode 100644
index 0000000..2d3dcda
--- /dev/null
+++ b/Lib/test/test_importlib/test_path.py
@@ -0,0 +1,39 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        with resources.path(package, path):
+            pass
+
+
+class PathTests:
+    def test_reading(self):
+        # Path should be readable.
+        # Test also implicitly verifies the returned object is a pathlib.Path
+        # instance.
+        with resources.path(self.data, 'utf-8.file') as path:
+            # pathlib.Path.read_text() was introduced in Python 3.5.
+            with path.open('r', encoding='utf-8') as file:
+                text = file.read()
+            self.assertEqual('Hello, UTF-8 world!\n', text)
+
+
+class PathDiskTests(PathTests, unittest.TestCase):
+    data = data01
+
+
+class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase):
+    def test_remove_in_context_manager(self):
+        # It is not an error if the file that was temporarily stashed on the
+        # file system is removed inside the `with` stanza.
+        with resources.path(self.data, 'utf-8.file') as path:
+            path.unlink()
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_importlib/test_read.py b/Lib/test/test_importlib/test_read.py
new file mode 100644
index 0000000..231f501
--- /dev/null
+++ b/Lib/test/test_importlib/test_read.py
@@ -0,0 +1,53 @@
+import unittest
+
+from importlib import resources
+from . import data01
+from . import util
+
+
+class CommonBinaryTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        resources.read_binary(package, path)
+
+
+class CommonTextTests(util.CommonResourceTests, unittest.TestCase):
+    def execute(self, package, path):
+        resources.read_text(package, path)
+
+
+class ReadTests:
+    def test_read_binary(self):
+        result = resources.read_binary(self.data, 'binary.file')
+        self.assertEqual(result, b'\0\1\2\3')
+
+    def test_read_text_default_encoding(self):
+        result = resources.read_text(self.data, 'utf-8.file')
+        self.assertEqual(result, 'Hello, UTF-8 world!\n')
+
+    def test_read_text_given_encoding(self):
+        result = resources.read_text(
+            self.data, 'utf-16.file', encoding='utf-16')
+        self.assertEqual(result, 'Hello, UTF-16 world!\n')
+
+    def test_read_text_with_errors(self):
+        # Raises UnicodeError without the 'errors' argument.
+        self.assertRaises(
+            UnicodeError, resources.read_text, self.data, 'utf-16.file')
+        result = resources.read_text(self.data, 'utf-16.file', errors='ignore')
+        self.assertEqual(
+            result,
+            'H\x00e\x00l\x00l\x00o\x00,\x00 '
+            '\x00U\x00T\x00F\x00-\x001\x006\x00 '
+            '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00')
+
+
+class ReadDiskTests(ReadTests, unittest.TestCase):
+    data = data01
+
+
+class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase):
+    pass
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_importlib/test_resource.py b/Lib/test/test_importlib/test_resource.py
new file mode 100644
index 0000000..c35f789
--- /dev/null
+++ b/Lib/test/test_importlib/test_resource.py
@@ -0,0 +1,143 @@
+import sys
+import unittest
+
+from importlib import resources
+from . import data01
+from . import zipdata02
+from . import util
+
+
+class ResourceTests:
+    # Subclasses are expected to set the `data` attribute.
+
+    def test_is_resource_good_path(self):
+        self.assertTrue(resources.is_resource(self.data, 'binary.file'))
+
+    def test_is_resource_missing(self):
+        self.assertFalse(resources.is_resource(self.data, 'not-a-file'))
+
+    def test_is_resource_subresource_directory(self):
+        # Directories are not resources.
+        self.assertFalse(resources.is_resource(self.data, 'subdirectory'))
+
+    def test_contents(self):
+        contents = set(resources.contents(self.data))
+        # There may be cruft in the directory listing of the data directory.
+        # Under Python 3 we could have a __pycache__ directory, and under
+        # Python 2 we could have .pyc files.  These are both artifacts of the
+        # test suite importing these modules and writing these caches.  They
+        # aren't germane to this test, so just filter them out.
+        contents.discard('__pycache__')
+        contents.discard('__init__.pyc')
+        contents.discard('__init__.pyo')
+        self.assertEqual(contents, {
+            '__init__.py',
+            'subdirectory',
+            'utf-8.file',
+            'binary.file',
+            'utf-16.file',
+            })
+
+
+class ResourceDiskTests(ResourceTests, unittest.TestCase):
+    def setUp(self):
+        self.data = data01
+
+
+class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase):
+    pass
+
+
+class ResourceLoaderTests(unittest.TestCase):
+    def test_resource_contents(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+        self.assertEqual(
+            set(resources.contents(package)),
+            {'A', 'B', 'C'})
+
+    def test_resource_is_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertTrue(resources.is_resource(package, 'B'))
+
+    def test_resource_directory_is_not_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertFalse(resources.is_resource(package, 'D'))
+
+    def test_resource_missing_is_not_resource(self):
+        package = util.create_package(
+            file=data01, path=data01.__file__,
+            contents=['A', 'B', 'C', 'D/E', 'D/F'])
+        self.assertFalse(resources.is_resource(package, 'Z'))
+
+
+class ResourceCornerCaseTests(unittest.TestCase):
+    def test_package_has_no_reader_fallback(self):
+        # Test odd ball packages which:
+        # 1. Do not have a ResourceReader as a loader
+        # 2. Are not on the file system
+        # 3. Are not in a zip file
+        module = util.create_package(
+            file=data01, path=data01.__file__, contents=['A', 'B', 'C'])
+        # Give the module a dummy loader.
+        module.__loader__ = object()
+        # Give the module a dummy origin.
+        module.__file__ = '/path/which/shall/not/be/named'
+        if sys.version_info >= (3,):
+            module.__spec__.loader = module.__loader__
+            module.__spec__.origin = module.__file__
+        self.assertFalse(resources.is_resource(module, 'A'))
+
+
+class ResourceFromZipsTest(util.ZipSetupBase, unittest.TestCase):
+    ZIP_MODULE = zipdata02                          # type: ignore
+
+    def test_unrelated_contents(self):
+        # https://gitlab.com/python-devs/importlib_resources/issues/44
+        #
+        # Here we have a zip file with two unrelated subpackages.  The bug
+        # reports that getting the contents of a resource returns unrelated
+        # files.
+        self.assertEqual(
+            set(resources.contents('ziptestdata.one')),
+            {'__init__.py', 'resource1.txt'})
+        self.assertEqual(
+            set(resources.contents('ziptestdata.two')),
+            {'__init__.py', 'resource2.txt'})
+
+
+class NamespaceTest(unittest.TestCase):
+    def test_namespaces_cant_have_resources(self):
+        contents = set(resources.contents(
+            'test.test_importlib.data03.namespace'))
+        self.assertEqual(len(contents), 0)
+        # Even though there is a file in the namespace directory, it is not
+        # considered a resource, since namespace packages can't have them.
+        self.assertFalse(resources.is_resource(
+            'test.test_importlib.data03.namespace',
+            'resource1.txt'))
+        # We should get an exception if we try to read it or open it.
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_text,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.open_binary,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.read_text,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+        self.assertRaises(
+            FileNotFoundError,
+            resources.read_binary,
+            'test.test_importlib.data03.namespace', 'resource1.txt')
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py
index 64e039e..bfb7cad 100644
--- a/Lib/test/test_importlib/util.py
+++ b/Lib/test/test_importlib/util.py
@@ -1,17 +1,24 @@
+import abc
 import builtins
 import contextlib
 import errno
 import functools
 import importlib
 from importlib import machinery, util, invalidate_caches
+from importlib.abc import ResourceReader
+import io
 import os
 import os.path
+from pathlib import Path, PurePath
 from test import support
 import unittest
 import sys
 import tempfile
 import types
 
+from . import data01
+from . import zipdata01
+
 
 BUILTINS = types.SimpleNamespace()
 BUILTINS.good_name = None
@@ -386,3 +393,159 @@ class CASEOKTestBase:
         if any(x in self.importlib._bootstrap_external._os.environ
                     for x in possibilities) != should_exist:
             self.skipTest('os.environ changes not reflected in _os.environ')
+
+
+def create_package(file, path, is_package=True, contents=()):
+    class Reader(ResourceReader):
+        def open_resource(self, path):
+            self._path = path
+            if isinstance(file, Exception):
+                raise file
+            else:
+                return file
+
+        def resource_path(self, path_):
+            self._path = path_
+            if isinstance(path, Exception):
+                raise path
+            else:
+                return path
+
+        def is_resource(self, path_):
+            self._path = path_
+            if isinstance(path, Exception):
+                raise path
+            for entry in contents:
+                parts = entry.split('/')
+                if len(parts) == 1 and parts[0] == path_:
+                    return True
+            return False
+
+        def contents(self):
+            if isinstance(path, Exception):
+                raise path
+            # There's no yield from in baseball, er, Python 2.
+            for entry in contents:
+                yield entry
+
+    name = 'testingpackage'
+    # Unforunately importlib.util.module_from_spec() was not introduced until
+    # Python 3.5.
+    module = types.ModuleType(name)
+    loader = Reader()
+    spec = machinery.ModuleSpec(
+        name, loader,
+        origin='does-not-exist',
+        is_package=is_package)
+    module.__spec__ = spec
+    module.__loader__ = loader
+    return module
+
+
+class CommonResourceTests(abc.ABC):
+    @abc.abstractmethod
+    def execute(self, package, path):
+        raise NotImplementedError
+
+    def test_package_name(self):
+        # Passing in the package name should succeed.
+        self.execute(data01.__name__, 'utf-8.file')
+
+    def test_package_object(self):
+        # Passing in the package itself should succeed.
+        self.execute(data01, 'utf-8.file')
+
+    def test_string_path(self):
+        # Passing in a string for the path should succeed.
+        path = 'utf-8.file'
+        self.execute(data01, path)
+
+    @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support')
+    def test_pathlib_path(self):
+        # Passing in a pathlib.PurePath object for the path should succeed.
+        path = PurePath('utf-8.file')
+        self.execute(data01, path)
+
+    def test_absolute_path(self):
+        # An absolute path is a ValueError.
+        path = Path(__file__)
+        full_path = path.parent/'utf-8.file'
+        with self.assertRaises(ValueError):
+            self.execute(data01, full_path)
+
+    def test_relative_path(self):
+        # A reative path is a ValueError.
+        with self.assertRaises(ValueError):
+            self.execute(data01, '../data01/utf-8.file')
+
+    def test_importing_module_as_side_effect(self):
+        # The anchor package can already be imported.
+        del sys.modules[data01.__name__]
+        self.execute(data01.__name__, 'utf-8.file')
+
+    def test_non_package_by_name(self):
+        # The anchor package cannot be a module.
+        with self.assertRaises(TypeError):
+            self.execute(__name__, 'utf-8.file')
+
+    def test_non_package_by_package(self):
+        # The anchor package cannot be a module.
+        with self.assertRaises(TypeError):
+            module = sys.modules['test.test_importlib.util']
+            self.execute(module, 'utf-8.file')
+
+    @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2')
+    def test_resource_opener(self):
+        bytes_data = io.BytesIO(b'Hello, world!')
+        package = create_package(file=bytes_data, path=FileNotFoundError())
+        self.execute(package, 'utf-8.file')
+        self.assertEqual(package.__loader__._path, 'utf-8.file')
+
+    @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2')
+    def test_resource_path(self):
+        bytes_data = io.BytesIO(b'Hello, world!')
+        path = __file__
+        package = create_package(file=bytes_data, path=path)
+        self.execute(package, 'utf-8.file')
+        self.assertEqual(package.__loader__._path, 'utf-8.file')
+
+    def test_useless_loader(self):
+        package = create_package(file=FileNotFoundError(),
+                                 path=FileNotFoundError())
+        with self.assertRaises(FileNotFoundError):
+            self.execute(package, 'utf-8.file')
+
+
+class ZipSetupBase:
+    ZIP_MODULE = None
+
+    @classmethod
+    def setUpClass(cls):
+        data_path = Path(cls.ZIP_MODULE.__file__)
+        data_dir = data_path.parent
+        cls._zip_path = str(data_dir / 'ziptestdata.zip')
+        sys.path.append(cls._zip_path)
+        cls.data = importlib.import_module('ziptestdata')
+
+    @classmethod
+    def tearDownClass(cls):
+        try:
+            sys.path.remove(cls._zip_path)
+        except ValueError:
+            pass
+
+        try:
+            del sys.path_importer_cache[cls._zip_path]
+            del sys.modules[cls.data.__name__]
+        except KeyError:
+            pass
+
+        try:
+            del cls.data
+            del cls._zip_path
+        except AttributeError:
+            pass
+
+
+class ZipSetup(ZipSetupBase):
+    ZIP_MODULE = zipdata01                          # type: ignore
diff --git a/Lib/test/test_importlib/zipdata01/__init__.py b/Lib/test/test_importlib/zipdata01/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/zipdata01/__init__.py
diff --git a/Lib/test/test_importlib/zipdata01/ziptestdata.zip b/Lib/test/test_importlib/zipdata01/ziptestdata.zip
new file mode 100644
index 0000000..8d8fa97
--- /dev/null
+++ b/Lib/test/test_importlib/zipdata01/ziptestdata.zip
diff --git a/Lib/test/test_importlib/zipdata02/__init__.py b/Lib/test/test_importlib/zipdata02/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Lib/test/test_importlib/zipdata02/__init__.py
diff --git a/Lib/test/test_importlib/zipdata02/ziptestdata.zip b/Lib/test/test_importlib/zipdata02/ziptestdata.zip
new file mode 100644
index 0000000..6f34889
--- /dev/null
+++ b/Lib/test/test_importlib/zipdata02/ziptestdata.zip