Issue #14285: Do not catch __init__.py exceptions in runpy

Initialize package before calling find_spec() for __main__, so that we do not
incorrectly handle exceptions from __init__.py. When runpy is used from the
Python CLI, use an internal exception rather than ImportError, to avoid
catching unexpected exceptions.

Also remove exception message rewriting in _run_module_as_main(), because it
seems to be redundant with the _get_main_module_details() function.

4 files changed, 110 insertions, 32 deletions

diff --git a/Lib/runpy.py b/Lib/runpy.py
index 1c5729d..bbcbf3a 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -99,7 +99,7 @@ def _run_module_code(code, init_globals=None,
     return mod_globals.copy()
 
 # Helper to get the loader, code and filename for a module
-def _get_module_details(mod_name):
+def _get_module_details(mod_name, error=ImportError):
     try:
         spec = importlib.util.find_spec(mod_name)
     except (ImportError, AttributeError, TypeError, ValueError) as ex:
@@ -107,27 +107,34 @@ def _get_module_details(mod_name):
         # importlib, where the latter raises other errors for cases where
         # pkgutil previously raised ImportError
         msg = "Error while finding spec for {!r} ({}: {})"
-        raise ImportError(msg.format(mod_name, type(ex), ex)) from ex
+        raise error(msg.format(mod_name, type(ex), ex)) from ex
     if spec is None:
-        raise ImportError("No module named %s" % mod_name)
+        raise error("No module named %s" % mod_name)
     if spec.submodule_search_locations is not None:
         if mod_name == "__main__" or mod_name.endswith(".__main__"):
-            raise ImportError("Cannot use package as __main__ module")
+            raise error("Cannot use package as __main__ module")
+        __import__(mod_name)  # Do not catch exceptions initializing package
         try:
             pkg_main_name = mod_name + ".__main__"
             return _get_module_details(pkg_main_name)
         except ImportError as e:
-            raise ImportError(("%s; %r is a package and cannot " +
+            raise error(("%s; %r is a package and cannot " +
                                "be directly executed") %(e, mod_name))
     loader = spec.loader
     if loader is None:
-        raise ImportError("%r is a namespace package and cannot be executed"
+        raise error("%r is a namespace package and cannot be executed"
                                                                  % mod_name)
-    code = loader.get_code(mod_name)
+    try:
+        code = loader.get_code(mod_name)
+    except ImportError as e:
+        raise error(format(e)) from e
     if code is None:
-        raise ImportError("No code object available for %s" % mod_name)
+        raise error("No code object available for %s" % mod_name)
     return mod_name, spec, code
 
+class _Error(Exception):
+    """Error that _run_module_as_main() should report without a traceback"""
+
 # XXX ncoghlan: Should this be documented and made public?
 # (Current thoughts: don't repeat the mistake that lead to its
 # creation when run_module() no longer met the needs of
@@ -148,20 +155,11 @@ def _run_module_as_main(mod_name, alter_argv=True):
     """
     try:
         if alter_argv or mod_name != "__main__": # i.e. -m switch
-            mod_name, mod_spec, code = _get_module_details(mod_name)
+            mod_name, mod_spec, code = _get_module_details(mod_name, _Error)
         else:          # i.e. directory or zipfile execution
-            mod_name, mod_spec, code = _get_main_module_details()
-    except ImportError as exc:
-        # Try to provide a good error message
-        # for directories, zip files and the -m switch
-        if alter_argv:
-            # For -m switch, just display the exception
-            info = str(exc)
-        else:
-            # For directories/zipfiles, let the user
-            # know what the code was looking for
-            info = "can't find '__main__' module in %r" % sys.argv[0]
-        msg = "%s: %s" % (sys.executable, info)
+            mod_name, mod_spec, code = _get_main_module_details(_Error)
+    except _Error as exc:
+        msg = "%s: %s" % (sys.executable, exc)
         sys.exit(msg)
     main_globals = sys.modules["__main__"].__dict__
     if alter_argv:
@@ -184,7 +182,7 @@ def run_module(mod_name, init_globals=None,
         # Leave the sys module alone
         return _run_code(code, {}, init_globals, run_name, mod_spec)
 
-def _get_main_module_details():
+def _get_main_module_details(error=ImportError):
     # Helper that gives a nicer error message when attempting to
     # execute a zipfile or directory by invoking __main__.py
     # Also moves the standard __main__ out of the way so that the
@@ -196,7 +194,7 @@ def _get_main_module_details():
         return _get_module_details(main_name)
     except ImportError as exc:
         if main_name in str(exc):
-            raise ImportError("can't find %r module in %r" %
+            raise error("can't find %r module in %r" %
                               (main_name, sys.path[0])) from exc
         raise
     finally:
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index fda3e62..77cb95c 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -397,20 +397,73 @@ class CmdLineTest(unittest.TestCase):
                                       script_name, script_name, '', '',
                                       importlib.machinery.SourceFileLoader)
 
+    @contextlib.contextmanager
+    def setup_test_pkg(self, *args):
+        with support.temp_dir() as script_dir, \
+                support.change_cwd(path=script_dir):
+            pkg_dir = os.path.join(script_dir, 'test_pkg')
+            make_pkg(pkg_dir, *args)
+            yield pkg_dir
+
+    def check_dash_m_failure(self, *args):
+        rc, out, err = assert_python_failure('-m', *args, __isolated=False)
+        if verbose > 1:
+            print(repr(out))
+        self.assertEqual(rc, 1)
+        return err
+
     def test_dash_m_error_code_is_one(self):
         # If a module is invoked with the -m command line flag
         # and results in an error that the return code to the
         # shell is '1'
-        with support.temp_dir() as script_dir:
-            with support.change_cwd(path=script_dir):
-                pkg_dir = os.path.join(script_dir, 'test_pkg')
-                make_pkg(pkg_dir)
-                script_name = _make_test_script(pkg_dir, 'other',
-                                                "if __name__ == '__main__': raise ValueError")
-                rc, out, err = assert_python_failure('-m', 'test_pkg.other', *example_args)
-                if verbose > 1:
-                    print(repr(out))
+        with self.setup_test_pkg() as pkg_dir:
+            script_name = _make_test_script(pkg_dir, 'other',
+                                            "if __name__ == '__main__': raise ValueError")
+            err = self.check_dash_m_failure('test_pkg.other', *example_args)
+            self.assertIn(b'ValueError', err)
+
+    def test_dash_m_errors(self):
+        # Exercise error reporting for various invalid package executions
+        tests = (
+            ('builtins', br'No code object available'),
+            ('builtins.x', br'Error while finding spec.*AttributeError'),
+            ('builtins.x.y', br'Error while finding spec.*'
+                br'ImportError.*No module named.*not a package'),
+            ('os.path', br'loader.*cannot handle'),
+            ('importlib', br'No module named.*'
+                br'is a package and cannot be directly executed'),
+            ('importlib.nonexistant', br'No module named'),
+        )
+        for name, regex in tests:
+            with self.subTest(name):
+                rc, _, err = assert_python_failure('-m', name)
                 self.assertEqual(rc, 1)
+                self.assertRegex(err, regex)
+                self.assertNotIn(b'Traceback', err)
+
+    def test_dash_m_init_traceback(self):
+        # These were wrapped in an ImportError and tracebacks were
+        # suppressed; see Issue 14285
+        exceptions = (ImportError, AttributeError, TypeError, ValueError)
+        for exception in exceptions:
+            exception = exception.__name__
+            init = "raise {0}('Exception in __init__.py')".format(exception)
+            with self.subTest(exception), \
+                    self.setup_test_pkg(init) as pkg_dir:
+                err = self.check_dash_m_failure('test_pkg')
+                self.assertIn(exception.encode('ascii'), err)
+                self.assertIn(b'Exception in __init__.py', err)
+                self.assertIn(b'Traceback', err)
+
+    def test_dash_m_main_traceback(self):
+        # Ensure that an ImportError's traceback is reported
+        with self.setup_test_pkg() as pkg_dir:
+            main = "raise ImportError('Exception in __main__ module')"
+            _make_test_script(pkg_dir, '__main__', main)
+            err = self.check_dash_m_failure('test_pkg')
+            self.assertIn(b'ImportError', err)
+            self.assertIn(b'Exception in __main__ module', err)
+            self.assertIn(b'Traceback', err)
 
     def test_pep_409_verbiage(self):
         # Make sure PEP 409 syntax properly suppresses
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
index 4bae949..01f6abd 100644
--- a/Lib/test/test_runpy.py
+++ b/Lib/test/test_runpy.py
@@ -439,6 +439,28 @@ from ..uncle.cousin import nephew
             if verbose > 1: print("Testing package depth:", depth)
             self._check_package(depth)
 
+    def test_run_package_init_exceptions(self):
+        # These were previously wrapped in an ImportError; see Issue 14285
+        result = self._make_pkg("", 1, "__main__")
+        pkg_dir, _, mod_name, _ = result
+        mod_name = mod_name.replace(".__main__", "")
+        self.addCleanup(self._del_pkg, pkg_dir, 1, mod_name)
+        init = os.path.join(pkg_dir, "__runpy_pkg__", "__init__.py")
+
+        exceptions = (ImportError, AttributeError, TypeError, ValueError)
+        for exception in exceptions:
+            name = exception.__name__
+            with self.subTest(name):
+                source = "raise {0}('{0} in __init__.py.')".format(name)
+                with open(init, "wt", encoding="ascii") as mod_file:
+                    mod_file.write(source)
+                try:
+                    run_module(mod_name)
+                except exception as err:
+                    self.assertNotIn("finding spec", format(err))
+                else:
+                    self.fail("Nothing raised; expected {}".format(name))
+
     def test_run_package_in_namespace_package(self):
         for depth in range(1, 4):
             if verbose > 1: print("Testing package depth:", depth)
diff --git a/Misc/NEWS b/Misc/NEWS
index 166864a..c286183 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -22,6 +22,11 @@ Core and Builtins
 Library
 -------
 
+- Issue #14285: When executing a package with the "python -m package" option,
+  and package initialization fails, a proper traceback is now reported.  The
+  "runpy" module now lets exceptions from package initialization pass back to
+  the caller, rather than raising ImportError.
+
 - Issue #25177: Fixed problem with the mean of very small and very large
   numbers. As a side effect, statistics.mean and statistics.variance should
   be significantly faster.