Issue #17177: The imp module is pending deprecation.

To make sure there is no issue with code that is both Python 2 and 3
compatible, there are no plans to remove the module any sooner than
Python 4 (unless the community moves to Python 3 solidly before then).

15 files changed, 103 insertions, 85 deletions

diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst
index ab79be3..eae2c70 100644
--- a/Doc/library/imp.rst
+++ b/Doc/library/imp.rst
@@ -2,7 +2,7 @@
 ================================================================
 
 .. deprecated:: 3.4
-   The :mod:`imp` package has been deprecated in favor of :mod:`importlib`.
+   The :mod:`imp` package is pending deprecation in favor of :mod:`importlib`.
 
 .. module:: imp
    :synopsis: Access the implementation of the import statement.
@@ -232,7 +232,7 @@ file paths.
    Return the :pep:`3147` magic tag string matching this version of Python's
    magic number, as returned by :func:`get_magic`.
 
-   .. note::
+   .. deprecated:: 3.4
       You may use :attr:`sys.implementation.cache_tag` directly starting
       in Python 3.3.
 
@@ -355,6 +355,9 @@ to indicate the search result of :func:`find_module`.
       ``None`` is inserted into ``sys.path_importer_cache`` instead of an
       instance of :class:`NullImporter`.
 
+   .. deprecated:: 3.4
+      Insert ``None`` into ``sys.path_importer_cache`` instead.
+
 
 .. _examples-imp:
 
diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst
index 92b2058..fbe4b77 100644
--- a/Doc/whatsnew/3.4.rst
+++ b/Doc/whatsnew/3.4.rst
@@ -230,6 +230,9 @@ Deprecated Python modules, functions and methods
   :meth:`importlib.abc.Loader.init_module_attrs` allows subclasses of a loader
   to more easily customize module loading.
 
+* The :mod:`imp` module is pending deprecation. To keep compatibility with
+  Python 2/3 code bases, the module's removal is currently not scheduled.
+
 
 Deprecated functions and types of the C API
 -------------------------------------------
diff --git a/Lib/imp.py b/Lib/imp.py
index 1872b29..458d370 100644
--- a/Lib/imp.py
+++ b/Lib/imp.py
@@ -27,6 +27,9 @@ import tokenize
 import types
 import warnings
 
+warnings.warn("the imp module is deprecated in favour of importlib; "
+              "see the module's documentation for alternative uses",
+              PendingDeprecationWarning)
 
 # DEPRECATED
 SEARCH_ERROR = 0
@@ -98,9 +101,7 @@ def source_from_cache(path):
 
 
 def get_suffixes():
-    warnings.warn('imp.get_suffixes() is deprecated; use the constants '
-                  'defined on importlib.machinery instead',
-                  DeprecationWarning, 2)
+    """**DEPRECATED**"""
     extensions = [(s, 'rb', C_EXTENSION) for s in machinery.EXTENSION_SUFFIXES]
     source = [(s, 'U', PY_SOURCE) for s in machinery.SOURCE_SUFFIXES]
     bytecode = [(s, 'rb', PY_COMPILED) for s in machinery.BYTECODE_SUFFIXES]
@@ -110,7 +111,11 @@ def get_suffixes():
 
 class NullImporter:
 
-    """Null import object."""
+    """**DEPRECATED**
+
+    Null import object.
+
+    """
 
     def __init__(self, path):
         if path == '':
@@ -152,10 +157,6 @@ class _LoadSourceCompatibility(_HackedGetData, machinery.SourceFileLoader):
 
 
 def load_source(name, pathname, file=None):
-    msg = ('imp.load_source() is deprecated; use '
-           'importlib.machinery.SourceFileLoader(name, pathname).load_module()'
-           ' instead')
-    warnings.warn(msg, DeprecationWarning, 2)
     _LoadSourceCompatibility(name, pathname, file).load_module(name)
     module = sys.modules[name]
     # To allow reloading to potentially work, use a non-hacked loader which
@@ -170,10 +171,7 @@ class _LoadCompiledCompatibility(_HackedGetData, SourcelessFileLoader):
 
 
 def load_compiled(name, pathname, file=None):
-    msg = ('imp.load_compiled() is deprecated; use '
-           'importlib.machinery.SourcelessFileLoader(name, pathname).'
-           'load_module() instead ')
-    warnings.warn(msg, DeprecationWarning, 2)
+    """**DEPRECATED**"""
     _LoadCompiledCompatibility(name, pathname, file).load_module(name)
     module = sys.modules[name]
     # To allow reloading to potentially work, use a non-hacked loader which
@@ -183,10 +181,7 @@ def load_compiled(name, pathname, file=None):
 
 
 def load_package(name, path):
-    msg = ('imp.load_package() is deprecated; use either '
-           'importlib.machinery.SourceFileLoader() or '
-           'importlib.machinery.SourcelessFileLoader() instead')
-    warnings.warn(msg, DeprecationWarning, 2)
+    """**DEPRECATED**"""
     if os.path.isdir(path):
         extensions = (machinery.SOURCE_SUFFIXES[:] +
                       machinery.BYTECODE_SUFFIXES[:])
@@ -208,32 +203,30 @@ def load_module(name, file, filename, details):
 
     """
     suffix, mode, type_ = details
-    with warnings.catch_warnings():
-        warnings.simplefilter('ignore')
-        if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
-            raise ValueError('invalid file open mode {!r}'.format(mode))
-        elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
-            msg = 'file object required for import (type code {})'.format(type_)
-            raise ValueError(msg)
-        elif type_ == PY_SOURCE:
-            return load_source(name, filename, file)
-        elif type_ == PY_COMPILED:
-            return load_compiled(name, filename, file)
-        elif type_ == C_EXTENSION and load_dynamic is not None:
-            if file is None:
-                with open(filename, 'rb') as opened_file:
-                    return load_dynamic(name, filename, opened_file)
-            else:
-                return load_dynamic(name, filename, file)
-        elif type_ == PKG_DIRECTORY:
-            return load_package(name, filename)
-        elif type_ == C_BUILTIN:
-            return init_builtin(name)
-        elif type_ == PY_FROZEN:
-            return init_frozen(name)
+    if mode and (not mode.startswith(('r', 'U')) or '+' in mode):
+        raise ValueError('invalid file open mode {!r}'.format(mode))
+    elif file is None and type_ in {PY_SOURCE, PY_COMPILED}:
+        msg = 'file object required for import (type code {})'.format(type_)
+        raise ValueError(msg)
+    elif type_ == PY_SOURCE:
+        return load_source(name, filename, file)
+    elif type_ == PY_COMPILED:
+        return load_compiled(name, filename, file)
+    elif type_ == C_EXTENSION and load_dynamic is not None:
+        if file is None:
+            with open(filename, 'rb') as opened_file:
+                return load_dynamic(name, filename, opened_file)
         else:
-            msg =  "Don't know how to import {} (type code {})".format(name, type_)
-            raise ImportError(msg, name=name)
+            return load_dynamic(name, filename, file)
+    elif type_ == PKG_DIRECTORY:
+        return load_package(name, filename)
+    elif type_ == C_BUILTIN:
+        return init_builtin(name)
+    elif type_ == PY_FROZEN:
+        return init_frozen(name)
+    else:
+        msg =  "Don't know how to import {} (type code {})".format(name, type_)
+        raise ImportError(msg, name=name)
 
 
 def find_module(name, path=None):
@@ -269,16 +262,14 @@ def find_module(name, path=None):
             file_path = os.path.join(package_directory, package_file_name)
             if os.path.isfile(file_path):
                 return None, package_directory, ('', '', PKG_DIRECTORY)
-        with warnings.catch_warnings():
-            warnings.simplefilter('ignore')
-            for suffix, mode, type_ in get_suffixes():
-                file_name = name + suffix
-                file_path = os.path.join(entry, file_name)
-                if os.path.isfile(file_path):
-                    break
-            else:
-                continue
-            break  # Break out of outer loop when breaking out of inner loop.
+        for suffix, mode, type_ in get_suffixes():
+            file_name = name + suffix
+            file_path = os.path.join(entry, file_name)
+            if os.path.isfile(file_path):
+                break
+        else:
+            continue
+        break  # Break out of outer loop when breaking out of inner loop.
     else:
         raise ImportError(_ERR_MSG.format(name), name=name)
 
diff --git a/Lib/inspect.py b/Lib/inspect.py
index 378e22f..d788525 100644
--- a/Lib/inspect.py
+++ b/Lib/inspect.py
@@ -31,7 +31,6 @@ Here are some of the useful functions provided by this module:
 __author__ = ('Ka-Ping Yee <ping@lfw.org>',
               'Yury Selivanov <yselivanov@sprymix.com>')
 
-import imp
 import importlib.machinery
 import itertools
 import linecache
@@ -440,6 +439,9 @@ def getmoduleinfo(path):
     """Get the module name, suffix, mode, and module type for a given file."""
     warnings.warn('inspect.getmoduleinfo() is deprecated', DeprecationWarning,
                   2)
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        import imp
     filename = os.path.basename(path)
     suffixes = [(-len(suffix), suffix, mode, mtype)
                     for suffix, mode, mtype in imp.get_suffixes()]
diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py
index 4996d7a..e3ad7d6 100644
--- a/Lib/modulefinder.py
+++ b/Lib/modulefinder.py
@@ -1,13 +1,16 @@
 """Find modules used by a script, using introspection."""
 
 import dis
-import imp
 import importlib.machinery
 import marshal
 import os
 import sys
 import types
 import struct
+import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', PendingDeprecationWarning)
+    import imp
 
 # XXX Clean up once str8's cstor matches bytes.
 LOAD_CONST = bytes([dis.opname.index('LOAD_CONST')])
diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index d2a6f1e..4678bb8 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -1,13 +1,13 @@
 """Utilities to support packages."""
 
 from functools import singledispatch as simplegeneric
-import imp
 import importlib
+import importlib.util
 import os
 import os.path
 import sys
 from types import ModuleType
-from warnings import warn
+import warnings
 
 __all__ = [
     'get_importer', 'iter_importers', 'get_loader', 'find_loader',
@@ -21,7 +21,7 @@ def read_code(stream):
     import marshal
 
     magic = stream.read(4)
-    if magic != imp.get_magic():
+    if magic != importlib.util.MAGIC_NUMBER:
         return None
 
     stream.read(8) # Skip timestamp and size
@@ -160,6 +160,13 @@ def _iter_file_finder_modules(importer, prefix=''):
 iter_importer_modules.register(
     importlib.machinery.FileFinder, _iter_file_finder_modules)
 
+
+def _import_imp():
+    global imp
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', PendingDeprecationWarning)
+        imp = importlib.import_module('imp')
+
 class ImpImporter:
     """PEP 302 Importer that wraps Python's "classic" import algorithm
 
@@ -172,8 +179,10 @@ class ImpImporter:
     """
 
     def __init__(self, path=None):
-        warn("This emulation is deprecated, use 'importlib' instead",
+        global imp
+        warnings.warn("This emulation is deprecated, use 'importlib' instead",
              DeprecationWarning)
+        _import_imp()
         self.path = path
 
     def find_module(self, fullname, path=None):
@@ -238,8 +247,9 @@ class ImpLoader:
     code = source = None
 
     def __init__(self, fullname, file, filename, etc):
-        warn("This emulation is deprecated, use 'importlib' instead",
-             DeprecationWarning)
+        warnings.warn("This emulation is deprecated, use 'importlib' instead",
+                      DeprecationWarning)
+        _import_imp()
         self.file = file
         self.filename = filename
         self.fullname = fullname
diff --git a/Lib/runpy.py b/Lib/runpy.py
index 6d0954f..1e0a2be 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -13,7 +13,6 @@ importers when locating support scripts as well as when importing modules.
 import os
 import sys
 import importlib.machinery # importlib first so we can test #15386 via -m
-import imp
 import types
 from pkgutil import read_code, get_loader, get_importer
 
@@ -224,7 +223,12 @@ def run_path(path_name, init_globals=None, run_name=None):
         run_name = "<run_path>"
     pkg_name = run_name.rpartition(".")[0]
     importer = get_importer(path_name)
-    if isinstance(importer, (type(None), imp.NullImporter)):
+    # Trying to avoid importing imp so as to not consume the deprecation warning.
+    is_NullImporter = False
+    if type(importer).__module__ == 'imp':
+        if type(importer).__name__ == 'NullImporter':
+            is_NullImporter = True
+    if isinstance(importer, type(None)) or is_NullImporter:
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
         code, mod_loader = _get_code_from_file(run_name, path_name)
diff --git a/Lib/test/test_fork1.py b/Lib/test/test_fork1.py
index 8192c38..e0626df 100644
--- a/Lib/test/test_fork1.py
+++ b/Lib/test/test_fork1.py
@@ -1,7 +1,7 @@
 """This test checks for correct fork() behavior.
 """
 
-import imp
+import _imp as imp
 import os
 import signal
 import sys
diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py
index dc62423..cf50ea4 100644
--- a/Lib/test/test_imp.py
+++ b/Lib/test/test_imp.py
@@ -2,7 +2,6 @@ try:
     import _thread
 except ImportError:
     _thread = None
-import imp
 import importlib
 import os
 import os.path
@@ -11,6 +10,9 @@ import sys
 from test import support
 import unittest
 import warnings
+with warnings.catch_warnings():
+    warnings.simplefilter('ignore', PendingDeprecationWarning)
+    import imp
 
 
 def requires_load_dynamic(meth):
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
index 673fb41..26b5806 100644
--- a/Lib/test/test_import.py
+++ b/Lib/test/test_import.py
@@ -1,7 +1,7 @@
 # We import importlib *ASAP* in order to test #15386
 import importlib
+import importlib.util
 import builtins
-import imp
 from test.test_importlib.import_ import util as importlib_util
 import marshal
 import os
@@ -221,7 +221,7 @@ class ImportTests(unittest.TestCase):
             with open(source, "w") as f:
                 f.write("a = 10\nb=20//0\n")
 
-            self.assertRaises(ZeroDivisionError, imp.reload, mod)
+            self.assertRaises(ZeroDivisionError, importlib.reload, mod)
             # But we still expect the module to be in sys.modules.
             mod = sys.modules.get(TESTFN)
             self.assertIsNot(mod, None, "expected module to be in sys.modules")
@@ -287,7 +287,7 @@ class ImportTests(unittest.TestCase):
             import sys
             class C:
                def __del__(self):
-                  import imp
+                  import importlib
             sys.argv.insert(0, C())
             """))
         script_helper.assert_python_ok(testfn)
@@ -298,7 +298,7 @@ class ImportTests(unittest.TestCase):
         sys.path.insert(0, os.curdir)
         try:
             source = TESTFN + ".py"
-            compiled = imp.cache_from_source(source)
+            compiled = importlib.util.cache_from_source(source)
             with open(source, 'w') as f:
                 pass
             try:
@@ -339,7 +339,7 @@ class FilePermissionTests(unittest.TestCase):
     def test_creation_mode(self):
         mask = 0o022
         with temp_umask(mask), _ready_to_import() as (name, path):
-            cached_path = imp.cache_from_source(path)
+            cached_path = importlib.util.cache_from_source(path)
             module = __import__(name)
             if not os.path.exists(cached_path):
                 self.fail("__import__ did not result in creation of "
@@ -357,7 +357,7 @@ class FilePermissionTests(unittest.TestCase):
         # permissions of .pyc should match those of .py, regardless of mask
         mode = 0o600
         with temp_umask(0o022), _ready_to_import() as (name, path):
-            cached_path = imp.cache_from_source(path)
+            cached_path = importlib.util.cache_from_source(path)
             os.chmod(path, mode)
             __import__(name)
             if not os.path.exists(cached_path):
@@ -372,7 +372,7 @@ class FilePermissionTests(unittest.TestCase):
     def test_cached_readonly(self):
         mode = 0o400
         with temp_umask(0o022), _ready_to_import() as (name, path):
-            cached_path = imp.cache_from_source(path)
+            cached_path = importlib.util.cache_from_source(path)
             os.chmod(path, mode)
             __import__(name)
             if not os.path.exists(cached_path):
@@ -412,7 +412,7 @@ class FilePermissionTests(unittest.TestCase):
                 bytecode_only = path + "c"
             else:
                 bytecode_only = path + "o"
-            os.rename(imp.cache_from_source(path), bytecode_only)
+            os.rename(importlib.util.cache_from_source(path), bytecode_only)
             m = __import__(name)
             self.assertEqual(m.x, 'rewritten')
 
@@ -434,7 +434,7 @@ func_filename = func.__code__.co_filename
 """
     dir_name = os.path.abspath(TESTFN)
     file_name = os.path.join(dir_name, module_name) + os.extsep + "py"
-    compiled_name = imp.cache_from_source(file_name)
+    compiled_name = importlib.util.cache_from_source(file_name)
 
     def setUp(self):
         self.sys_path = sys.path[:]
@@ -637,7 +637,7 @@ class OverridingImportBuiltinTests(unittest.TestCase):
 class PycacheTests(unittest.TestCase):
     # Test the various PEP 3147 related behaviors.
 
-    tag = imp.get_tag()
+    tag = sys.implementation.cache_tag
 
     def _clean(self):
         forget(TESTFN)
@@ -685,7 +685,7 @@ class PycacheTests(unittest.TestCase):
         # With PEP 3147 cache layout, removing the source but leaving the pyc
         # file does not satisfy the import.
         __import__(TESTFN)
-        pyc_file = imp.cache_from_source(self.source)
+        pyc_file = importlib.util.cache_from_source(self.source)
         self.assertTrue(os.path.exists(pyc_file))
         os.remove(self.source)
         forget(TESTFN)
@@ -710,7 +710,7 @@ class PycacheTests(unittest.TestCase):
     def test___cached__(self):
         # Modules now also have an __cached__ that points to the pyc file.
         m = __import__(TESTFN)
-        pyc_file = imp.cache_from_source(TESTFN + '.py')
+        pyc_file = importlib.util.cache_from_source(TESTFN + '.py')
         self.assertEqual(m.__cached__, os.path.join(os.curdir, pyc_file))
 
     @skip_if_dont_write_bytecode
@@ -745,10 +745,10 @@ class PycacheTests(unittest.TestCase):
             pass
         importlib.invalidate_caches()
         m = __import__('pep3147.foo')
-        init_pyc = imp.cache_from_source(
+        init_pyc = importlib.util.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
         self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
-        foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+        foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py'))
         self.assertEqual(sys.modules['pep3147.foo'].__cached__,
                          os.path.join(os.curdir, foo_pyc))
 
@@ -772,10 +772,10 @@ class PycacheTests(unittest.TestCase):
         unload('pep3147')
         importlib.invalidate_caches()
         m = __import__('pep3147.foo')
-        init_pyc = imp.cache_from_source(
+        init_pyc = importlib.util.cache_from_source(
             os.path.join('pep3147', '__init__.py'))
         self.assertEqual(m.__cached__, os.path.join(os.curdir, init_pyc))
-        foo_pyc = imp.cache_from_source(os.path.join('pep3147', 'foo.py'))
+        foo_pyc = importlib.util.cache_from_source(os.path.join('pep3147', 'foo.py'))
         self.assertEqual(sys.modules['pep3147.foo'].__cached__,
                          os.path.join(os.curdir, foo_pyc))
 
diff --git a/Lib/test/test_importlib/import_/test_api.py b/Lib/test/test_importlib/import_/test_api.py
index 98f79d6..b83cead 100644
--- a/Lib/test/test_importlib/import_/test_api.py
+++ b/Lib/test/test_importlib/import_/test_api.py
@@ -1,6 +1,5 @@
 from .. import util as importlib_test_util
 from . import util
-import imp
 import sys
 import types
 import unittest
diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py
index 616e775..66ad96e 100644
--- a/Lib/test/test_importlib/source/test_file_loader.py
+++ b/Lib/test/test_importlib/source/test_file_loader.py
@@ -7,7 +7,6 @@ from .. import util
 from . import util as source_util
 
 import errno
-import imp
 import marshal
 import os
 import py_compile
diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py
index 84a5e7b..02c6c1e 100644
--- a/Lib/test/test_socketserver.py
+++ b/Lib/test/test_socketserver.py
@@ -2,8 +2,8 @@
 Test suite for socketserver.
 """
 
+import _imp as imp
 import contextlib
-import imp
 import os
 import select
 import signal
diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py
index 6c2965b..3d961b5 100644
--- a/Lib/test/test_threaded_import.py
+++ b/Lib/test/test_threaded_import.py
@@ -5,8 +5,8 @@
 # complains several times about module random having no attribute
 # randrange, and then Python hangs.
 
+import _imp as imp
 import os
-import imp
 import importlib
 import sys
 import time
diff --git a/Misc/NEWS b/Misc/NEWS
index 2563d71..be0fe5a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -123,6 +123,8 @@ Core and Builtins
 Library
 -------
 
+- Issue #17177: The imp module is pending deprecation.
+
 - subprocess: Prevent a possible double close of parent pipe fds when the
   subprocess exec runs into an error.  Prevent a regular multi-close of the
   /dev/null fd when any of stdin, stdout and stderr was set to DEVNULL.