Make importlib.abc.SourceLoader the primary mechanism for importlib.

This required moving the class from importlib/abc.py into
importlib/_bootstrap.py and jiggering some code to work better with the class.
This included changing how the file finder worked to better meet import
semantics. This also led to fixing importlib to handle the empty string from
sys.path as import currently does (and making me wish we didn't support that
instead just required people to insert '.' instead to represent cwd).

It also required making the new set_data abstractmethod create
any needed subdirectories implicitly thanks to __pycache__ (it was either this
or grow the SourceLoader ABC to gain an 'exists' method and either a mkdir
method or have set_data with no data arg mean to create a directory).

Lastly, as an optimization the file loaders cache the file path where the
finder found something to use for loading (this is thanks to having a
sourceless loader separate from the source loader to simplify the code and
cut out stat calls).
Unfortunately test_runpy assumed a loader would always work for a module, even
if you changed from underneath it what it was expected to work with. By simply
dropping the previous loader in test_runpy so the proper loader can be returned
by the finder fixed the failure.

At this point importlib deviates from import on two points:

1. The exception raised when trying to import a file is different (import does
an explicit file check to print a special message, importlib just says the path
cannot be imported as if it was just some module name).

2. the co_filename on a code object is not being set to where bytecode was
actually loaded from instead of where the marshalled code object originally
came from (a solution for this has already been agreed upon on python-dev but has
not been implemented yet; issue8611).

2 files changed, 38 insertions, 27 deletions

diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py
index 2fac499..04d9bff 100644
--- a/Lib/importlib/_bootstrap.py
+++ b/Lib/importlib/_bootstrap.py
@@ -386,8 +386,8 @@ class SourceLoader(_LoaderBasics):
         except IOError:
             raise ImportError("source not available through get_data()")
         encoding = tokenize.detect_encoding(_io.BytesIO(source_bytes).readline)
-        # XXX Universal newlines?
-        return source_bytes.decode(encoding[0])
+        newline_decoder = _io.IncrementalNewlineDecoder(None, True)
+        return newline_decoder.decode(source_bytes.decode(encoding[0]))
 
     def get_code(self, fullname):
         """Concrete implementation of InspectLoader.get_code.
diff --git a/Lib/importlib/test/source/test_abc_loader.py b/Lib/importlib/test/source/test_abc_loader.py
index ce0021f..3245907 100644
--- a/Lib/importlib/test/source/test_abc_loader.py
+++ b/Lib/importlib/test/source/test_abc_loader.py
@@ -7,6 +7,7 @@ from . import util as source_util
 
 import imp
 import inspect
+import io
 import marshal
 import os
 import sys
@@ -327,30 +328,6 @@ class PyLoaderInterfaceTests(unittest.TestCase):
             mock.get_filename(name)
 
 
-class PyLoaderGetSourceTests(unittest.TestCase):
-
-    """Tests for importlib.abc.PyLoader.get_source()."""
-
-    def test_default_encoding(self):
-        # Should have no problems with UTF-8 text.
-        name = 'mod'
-        mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')})
-        source = 'x = "ü"'
-        mock.source = source.encode('utf-8')
-        returned_source = mock.get_source(name)
-        self.assertEqual(returned_source, source)
-
-    def test_decoded_source(self):
-        # Decoding should work.
-        name = 'mod'
-        mock = PyLoaderMock({name: os.path.join('path', 'to', 'mod')})
-        source = "# coding: Latin-1\nx='ü'"
-        assert source.encode('latin-1') != source.encode('utf-8')
-        mock.source = source.encode('latin-1')
-        returned_source = mock.get_source(name)
-        self.assertEqual(returned_source, source)
-
-
 class PyPycLoaderTests(PyLoaderTests):
 
     """Tests for importlib.abc.PyPycLoader."""
@@ -752,6 +729,39 @@ class SourceLoaderBytecodeTests(SourceLoaderTestHarness):
         code_object = self.loader.get_code(self.name)
         self.verify_code(code_object)
 
+
+class SourceLoaderGetSourceTests(unittest.TestCase):
+
+    """Tests for importlib.abc.SourceLoader.get_source()."""
+
+    def test_default_encoding(self):
+        # Should have no problems with UTF-8 text.
+        name = 'mod'
+        mock = SourceOnlyLoaderMock('mod.file')
+        source = 'x = "ü"'
+        mock.source = source.encode('utf-8')
+        returned_source = mock.get_source(name)
+        self.assertEqual(returned_source, source)
+
+    def test_decoded_source(self):
+        # Decoding should work.
+        name = 'mod'
+        mock = SourceOnlyLoaderMock("mod.file")
+        source = "# coding: Latin-1\nx='ü'"
+        assert source.encode('latin-1') != source.encode('utf-8')
+        mock.source = source.encode('latin-1')
+        returned_source = mock.get_source(name)
+        self.assertEqual(returned_source, source)
+
+    def test_universal_newlines(self):
+        # PEP 302 says universal newlines should be used.
+        name = 'mod'
+        mock = SourceOnlyLoaderMock('mod.file')
+        source = "x = 42\r\ny = -13\r\n"
+        mock.source = source.encode('utf-8')
+        expect = io.IncrementalNewlineDecoder(None, True).decode(source)
+        self.assertEqual(mock.get_source(name), expect)
+
 class AbstractMethodImplTests(unittest.TestCase):
 
     """Test the concrete abstractmethod implementations."""
@@ -852,12 +862,13 @@ class AbstractMethodImplTests(unittest.TestCase):
 def test_main():
     from test.support import run_unittest
     run_unittest(PyLoaderTests, PyLoaderCompatTests,
-                    PyLoaderInterfaceTests, PyLoaderGetSourceTests,
+                    PyLoaderInterfaceTests,
                     PyPycLoaderTests, PyPycLoaderInterfaceTests,
                     SkipWritingBytecodeTests, RegeneratedBytecodeTests,
                     BadBytecodeFailureTests, MissingPathsTests,
                     SourceOnlyLoaderTests,
                     SourceLoaderBytecodeTests,
+                    SourceLoaderGetSourceTests,
                     AbstractMethodImplTests)