Make importlib.abc.SourceLoader the primary mechanism for importlib.

This required moving the class from importlib/abc.py into
importlib/_bootstrap.py and jiggering some code to work better with the class.
This included changing how the file finder worked to better meet import
semantics. This also led to fixing importlib to handle the empty string from
sys.path as import currently does (and making me wish we didn't support that
instead just required people to insert '.' instead to represent cwd).

It also required making the new set_data abstractmethod create
any needed subdirectories implicitly thanks to __pycache__ (it was either this
or grow the SourceLoader ABC to gain an 'exists' method and either a mkdir
method or have set_data with no data arg mean to create a directory).

Lastly, as an optimization the file loaders cache the file path where the
finder found something to use for loading (this is thanks to having a
sourceless loader separate from the source loader to simplify the code and
cut out stat calls).
Unfortunately test_runpy assumed a loader would always work for a module, even
if you changed from underneath it what it was expected to work with. By simply
dropping the previous loader in test_runpy so the proper loader can be returned
by the finder fixed the failure.

At this point importlib deviates from import on two points:

1. The exception raised when trying to import a file is different (import does
an explicit file check to print a special message, importlib just says the path
cannot be imported as if it was just some module name).

2. the co_filename on a code object is not being set to where bytecode was
actually loaded from instead of where the marshalled code object originally
came from (a solution for this has already been agreed upon on python-dev but has
not been implemented yet; issue8611).

3 files changed, 32 insertions, 1 deletions

diff --git a/Lib/importlib/test/source/test_file_loader.py b/Lib/importlib/test/source/test_file_loader.py
index 0a85bc4..ce6f5d4 100644
--- a/Lib/importlib/test/source/test_file_loader.py
+++ b/Lib/importlib/test/source/test_file_loader.py
@@ -1,6 +1,7 @@
 import importlib
 from importlib import _bootstrap
 from .. import abc
+from .. import util
 from . import util as source_util
 
 import imp
@@ -108,6 +109,22 @@ class SimpleTest(unittest.TestCase):
                 loader.load_module('_temp')
             self.assertTrue('_temp' not in sys.modules)
 
+    def test_file_from_empty_string_dir(self):
+        # Loading a module found from an empty string entry on sys.path should
+        # not only work, but keep all attributes relative.
+        with open('_temp.py', 'w') as file:
+            file.write("# test file for importlib")
+        try:
+            with util.uncache('_temp'):
+                loader = _bootstrap._SourceFileLoader('_temp', '_temp.py')
+                mod = loader.load_module('_temp')
+                self.assertEqual('_temp.py', mod.__file__)
+                self.assertEqual(imp.cache_from_source('_temp.py'),
+                                 mod.__cached__)
+
+        finally:
+            os.unlink('_temp.py')
+
 
 class BadBytecodeTest(unittest.TestCase):
 
diff --git a/Lib/importlib/test/source/test_finder.py b/Lib/importlib/test/source/test_finder.py
index 12bab8f..7b9088d 100644
--- a/Lib/importlib/test/source/test_finder.py
+++ b/Lib/importlib/test/source/test_finder.py
@@ -115,7 +115,6 @@ class FinderTests(abc.FinderTests):
 
     # [package over modules]
     def test_package_over_module(self):
-        # XXX This is not a blackbox test!
         name = '_temp'
         loader = self.run_test(name, {'{0}.__init__'.format(name), name})
         self.assertTrue('__init__' in loader.get_filename(name))
@@ -133,6 +132,17 @@ class FinderTests(abc.FinderTests):
             with self.assertRaises(ImportWarning):
                 self.run_test('pkg', {'pkg.__init__'}, unlink={'pkg.__init__'})
 
+    def test_empty_string_for_dir(self):
+        # The empty string from sys.path means to search in the cwd.
+        finder = _bootstrap._FileFinder('', _bootstrap._SourceFinderDetails())
+        with open('mod.py', 'w') as file:
+            file.write("# test file for importlib")
+        try:
+            loader = finder.find_module('mod')
+            self.assertTrue(hasattr(loader, 'load_module'))
+        finally:
+            os.unlink('mod.py')
+
 
 def test_main():
     from test.support import run_unittest
diff --git a/Lib/importlib/test/source/test_path_hook.py b/Lib/importlib/test/source/test_path_hook.py
index 8697f0c..374f7b6 100644
--- a/Lib/importlib/test/source/test_path_hook.py
+++ b/Lib/importlib/test/source/test_path_hook.py
@@ -12,6 +12,10 @@ class PathHookTest(unittest.TestCase):
             self.assertTrue(hasattr(_bootstrap._file_path_hook(mapping['.root']),
                                  'find_module'))
 
+    def test_empty_string(self):
+        # The empty string represents the cwd.
+        self.assertTrue(hasattr(_bootstrap._file_path_hook(''), 'find_module'))
+
 
 def test_main():
     from test.support import run_unittest