Issue #16444, #16218: Use TESTFN_UNDECODABLE on UNIX

Check if data is decoded by os.fsdecode() (filesystem encoding with
surrogateescape error handler, PEP 383), not by UTF-8 or the filesystem
encoding in strict mode.

Use TESTFN_UNDECODABLE in test_cmd_line_script.test_non_ascii() on UNIX.

3 files changed, 42 insertions, 29 deletions

diff --git a/Lib/test/support.py b/Lib/test/support.py
index ec4b47d..d0a37ea 100644
--- a/Lib/test/support.py
+++ b/Lib/test/support.py
@@ -692,17 +692,29 @@ elif sys.platform != 'darwin':
 
 # TESTFN_UNDECODABLE is a filename (bytes type) that should *not* be able to be
 # decoded from the filesystem encoding (in strict mode). It can be None if we
-# cannot generate such filename.
+# cannot generate such filename (ex: the latin1 encoding can decode any byte
+# sequence). On UNIX, TESTFN_UNDECODABLE can be decoded by os.fsdecode() thanks
+# to the surrogateescape error handler (PEP 383), but not from the filesystem
+# encoding in strict mode.
 TESTFN_UNDECODABLE = None
-# b'\xff' is not decodable by os.fsdecode() with code page 932. Windows
-# accepts it to create a file or a directory, or don't accept to enter to
-# such directory (when the bytes name is used). So test b'\xe7' first: it is
-# not decodable from cp932.
-for name in (b'\xe7w\xf0', b'abc\xff'):
+for name in (
+    # b'\xff' is not decodable by os.fsdecode() with code page 932. Windows
+    # accepts it to create a file or a directory, or don't accept to enter to
+    # such directory (when the bytes name is used). So test b'\xe7' first: it is
+    # not decodable from cp932.
+    b'\xe7w\xf0',
+    # undecodable from ASCII, UTF-8
+    b'\xff',
+    # undecodable from iso8859-3, iso8859-6, iso8859-7, cp424, iso8859-8, cp856
+    # and cp857
+    b'\xae\xd5'
+    # undecodable from UTF-8 (UNIX and Mac OS X)
+    b'\xed\xb2\x80', b'\xed\xb4\x80',
+):
     try:
-        os.fsdecode(name)
+        name.decode(TESTFN_ENCODING)
     except UnicodeDecodeError:
-        TESTFN_UNDECODABLE = name
+        TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
         break
 
 if FS_NONASCII:
diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index 77152b3..6051e18 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -363,11 +363,21 @@ class CmdLineTest(unittest.TestCase):
             self.assertTrue(text[1].startswith('  File '))
             self.assertTrue(text[3].startswith('NameError'))
 
-    @unittest.skipUnless(support.TESTFN_NONASCII, 'need support.TESTFN_NONASCII')
     def test_non_ascii(self):
+        # Mac OS X denies the creation of a file with an invalid UTF-8 name.
+        # Windows allows to create a name with an arbitrary bytes name, but
+        # Python cannot a undecodable bytes argument to a subprocess.
+        if (support.TESTFN_UNDECODABLE
+        and sys.platform not in ('win32', 'darwin')):
+            name = os.fsdecode(support.TESTFN_UNDECODABLE)
+        elif support.TESTFN_NONASCII:
+            name = support.TESTFN_NONASCII
+        else:
+            self.skipTest("need support.TESTFN_NONASCII")
+
         # Issue #16218
         source = 'print(ascii(__file__))\n'
-        script_name = _make_test_script(os.curdir, support.TESTFN_NONASCII, source)
+        script_name = _make_test_script(os.curdir, name, source)
         self.addCleanup(support.unlink, script_name)
         rc, stdout, stderr = assert_python_ok(script_name)
         self.assertEqual(
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
index 9d1d2bc..084dbb2 100644
--- a/Lib/test/test_genericpath.py
+++ b/Lib/test/test_genericpath.py
@@ -309,26 +309,17 @@ class CommonTest(GenericTest):
                     self.assertIsInstance(abspath(path), str)
 
     def test_nonascii_abspath(self):
-        # Test non-ASCII in the path
-        if sys.platform in ('win32', 'darwin'):
-            if support.TESTFN_NONASCII:
-                name = support.TESTFN_NONASCII
-            else:
-                # Mac OS X denies the creation of a directory with an invalid
-                # UTF-8 name. Windows allows to create a directory with an
-                # arbitrary bytes name, but fails to enter this directory
-                # (when the bytes name is used).
-                self.skipTest("need support.TESTFN_NONASCII")
+        if (support.TESTFN_UNDECODABLE
+        # Mac OS X denies the creation of a directory with an invalid
+        # UTF-8 name. Windows allows to create a directory with an
+        # arbitrary bytes name, but fails to enter this directory
+        # (when the bytes name is used).
+        and sys.platform not in ('win32', 'darwin')):
+            name = support.TESTFN_UNDECODABLE
+        elif support.TESTFN_NONASCII:
+            name = support.TESTFN_NONASCII
         else:
-            if support.TESTFN_UNDECODABLE:
-                name = support.TESTFN_UNDECODABLE
-            elif support.TESTFN_NONASCII:
-                name = support.TESTFN_NONASCII
-            else:
-                # On UNIX, the surrogateescape error handler is used to
-                # decode paths, so any byte is allowed, it does not depend
-                # on the locale
-                name = b'a\xffb\xe7w\xf0'
+            self.skipTest("need support.TESTFN_NONASCII")
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)