bpo-40260: Update modulefinder to use io.open_code() and respect coding comments (GH-19488)

1 files changed, 60 insertions, 5 deletions

diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py
index ebd96e1..1aa4501 100644
--- a/Lib/test/test_modulefinder.py
+++ b/Lib/test/test_modulefinder.py
@@ -40,7 +40,8 @@ a/module.py
                                 from c import something
 b/__init__.py
                                 from sys import *
-"""]
+""",
+]
 
 maybe_test_new = [
     "a.module",
@@ -245,6 +246,48 @@ b/__init__.py
 b/c.py
 """]
 
+coding_default_utf8_test = [
+    "a_utf8",
+    ["a_utf8", "b_utf8"],
+    [], [],
+    """\
+a_utf8.py
+                                # use the default of utf8
+                                print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
+                                import b_utf8
+b_utf8.py
+                                # use the default of utf8
+                                print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
+"""]
+
+coding_explicit_utf8_test = [
+    "a_utf8",
+    ["a_utf8", "b_utf8"],
+    [], [],
+    """\
+a_utf8.py
+                                # coding=utf8
+                                print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
+                                import b_utf8
+b_utf8.py
+                                # use the default of utf8
+                                print('Unicode test B code point 2090 \u2090 that is not valid in cp1252')
+"""]
+
+coding_explicit_cp1252_test = [
+    "a_cp1252",
+    ["a_cp1252", "b_utf8"],
+    [], [],
+    b"""\
+a_cp1252.py
+                                # coding=cp1252
+                                # 0xe2 is not allowed in utf8
+                                print('CP1252 test P\xe2t\xe9')
+                                import b_utf8
+b_utf8.py
+                                # use the default of utf8
+                                print('Unicode test A code point 2090 \u2090 that is not valid in cp1252')
+"""]
 
 def open_file(path):
     dirname = os.path.dirname(path)
@@ -253,18 +296,22 @@ def open_file(path):
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
-    return open(path, "w")
+    return open(path, 'wb')
 
 
 def create_package(source):
     ofi = None
     try:
         for line in source.splitlines():
-            if line.startswith(" ") or line.startswith("\t"):
-                ofi.write(line.strip() + "\n")
+            if type(line) != bytes:
+                line = line.encode('utf-8')
+            if line.startswith(b' ') or line.startswith(b'\t'):
+                ofi.write(line.strip() + b'\n')
             else:
                 if ofi:
                     ofi.close()
+                if type(line) == bytes:
+                    line = line.decode('utf-8')
                 ofi = open_file(os.path.join(TEST_DIR, line.strip()))
     finally:
         if ofi:
@@ -337,7 +384,7 @@ class ModuleFinderTest(unittest.TestCase):
         source_path = base_path + importlib.machinery.SOURCE_SUFFIXES[0]
         bytecode_path = base_path + importlib.machinery.BYTECODE_SUFFIXES[0]
         with open_file(source_path) as file:
-            file.write('testing_modulefinder = True\n')
+            file.write('testing_modulefinder = True\n'.encode('utf-8'))
         py_compile.compile(source_path, cfile=bytecode_path)
         os.remove(source_path)
         self._do_test(bytecode_test)
@@ -365,6 +412,14 @@ b.py
 """ % list(range(2**16))]  # 2**16 constants
         self._do_test(extended_opargs_test)
 
+    def test_coding_default_utf8(self):
+        self._do_test(coding_default_utf8_test)
+
+    def test_coding_explicit_utf8(self):
+        self._do_test(coding_explicit_utf8_test)
+
+    def test_coding_explicit_cp1252(self):
+        self._do_test(coding_explicit_cp1252_test)
 
 if __name__ == "__main__":
     unittest.main()