Issue #22286: The "backslashreplace" error handlers now works with

decoding and translating.

10 files changed, 196 insertions, 83 deletions

diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst
index 5ea311e..ee31a9c 100644
--- a/Doc/howto/unicode.rst
+++ b/Doc/howto/unicode.rst
@@ -280,8 +280,9 @@ and optionally an *errors* argument.
 The *errors* argument specifies the response when the input string can't be
 converted according to the encoding's rules.  Legal values for this argument are
 ``'strict'`` (raise a :exc:`UnicodeDecodeError` exception), ``'replace'`` (use
-``U+FFFD``, ``REPLACEMENT CHARACTER``), or ``'ignore'`` (just leave the
-character out of the Unicode result).
+``U+FFFD``, ``REPLACEMENT CHARACTER``), ``'ignore'`` (just leave the
+character out of the Unicode result), or ``'backslashreplace'`` (inserts a
+``\xNN`` escape sequence).
 The following examples show the differences::
 
     >>> b'\x80abc'.decode("utf-8", "strict")  #doctest: +NORMALIZE_WHITESPACE
@@ -291,6 +292,8 @@ The following examples show the differences::
       invalid start byte
     >>> b'\x80abc'.decode("utf-8", "replace")
     '\ufffdabc'
+    >>> b'\x80abc'.decode("utf-8", "backslashreplace")
+    '\\x80abc'
     >>> b'\x80abc'.decode("utf-8", "ignore")
     'abc'
 
diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst
index 3510f69..048f0e9 100644
--- a/Doc/library/codecs.rst
+++ b/Doc/library/codecs.rst
@@ -314,8 +314,8 @@ The following error handlers are only applicable to
 |                         | reference (only for encoding).  Implemented   |
 |                         | in :func:`xmlcharrefreplace_errors`.          |
 +-------------------------+-----------------------------------------------+
-| ``'backslashreplace'``  | Replace with backslashed escape sequences     |
-|                         | (only for encoding).  Implemented in          |
+| ``'backslashreplace'``  | Replace with backslashed escape sequences.    |
+|                         | Implemented in                                |
 |                         | :func:`backslashreplace_errors`.              |
 +-------------------------+-----------------------------------------------+
 | ``'namereplace'``       | Replace with ``\N{...}`` escape sequences     |
@@ -350,6 +350,10 @@ In addition, the following error handler is specific to the given codecs:
 .. versionadded:: 3.5
    The ``'namereplace'`` error handler.
 
+.. versionchanged:: 3.5
+   The ``'backslashreplace'`` error handlers now works with decoding and
+   translating.
+
 The set of allowed values can be extended by registering a new named error
 handler:
 
@@ -417,9 +421,9 @@ functions:
 
 .. function:: backslashreplace_errors(exception)
 
-   Implements the ``'backslashreplace'`` error handling (for encoding with
-   :term:`text encodings <text encoding>` only): the
-   unencodable character is replaced by a backslashed escape sequence.
+   Implements the ``'backslashreplace'`` error handling (for
+   :term:`text encodings <text encoding>` only): malformed data is
+   replaced by a backslashed escape sequence.
 
 .. function:: namereplace_errors(exception)
 
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index c6b66b5..eb28513 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -973,9 +973,8 @@ are always available.  They are listed here in alphabetical order.
      Characters not supported by the encoding are replaced with the
      appropriate XML character reference ``&#nnn;``.
 
-   * ``'backslashreplace'`` (also only supported when writing)
-     replaces unsupported characters with Python's backslashed escape
-     sequences.
+   * ``'backslashreplace'`` replaces malformed data by Python's backslashed
+     escape sequences.
 
    * ``'namereplace'`` (also only supported when writing)
      replaces unsupported characters with ``\N{...}`` escape sequences.
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index c77db90..b0b1af3 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -825,11 +825,12 @@ Text I/O
    exception if there is an encoding error (the default of ``None`` has the same
    effect), or pass ``'ignore'`` to ignore errors.  (Note that ignoring encoding
    errors can lead to data loss.)  ``'replace'`` causes a replacement marker
-   (such as ``'?'``) to be inserted where there is malformed data.  When
-   writing, ``'xmlcharrefreplace'`` (replace with the appropriate XML character
-   reference), ``'backslashreplace'`` (replace with backslashed escape
-   sequences) or ``'namereplace'`` (replace with ``\N{...}`` escape sequences)
-   can be used.  Any other error handling name that has been registered with
+   (such as ``'?'``) to be inserted where there is malformed data.
+   ``'backslashreplace'`` causes malformed data to be replaced by a
+   backslashed escape sequence.  When writing, ``'xmlcharrefreplace'``
+   (replace with the appropriate XML character reference)  or ``'namereplace'``
+   (replace with ``\N{...}`` escape sequences) can be used.  Any other error
+   handling name that has been registered with
    :func:`codecs.register_error` is also valid.
 
    .. index::
diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst
index bea6a8a..ae18276 100644
--- a/Doc/whatsnew/3.5.rst
+++ b/Doc/whatsnew/3.5.rst
@@ -118,7 +118,9 @@ Other Language Changes
 
 Some smaller changes made to the core Python language are:
 
-* None yet.
+* Added the ``'namereplace'`` error handlers.  The ``'backslashreplace'``
+  error handlers now works with decoding and translating.
+  (Contributed by Serhiy Storchaka in :issue:`19676` and :issue:`22286`.)
 
 
 
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 7439b9f..c2b5d6c 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -127,7 +127,8 @@ class Codec:
          'surrogateescape' - replace with private code points U+DCnn.
          'xmlcharrefreplace' - Replace with the appropriate XML
                                character reference (only for encoding).
-         'backslashreplace'  - Replace with backslashed escape sequences
+         'backslashreplace'  - Replace with backslashed escape sequences.
+         'namereplace'       - Replace with \\N{...} escape sequences
                                (only for encoding).
 
         The set of allowed values can be extended via register_error.
@@ -359,7 +360,8 @@ class StreamWriter(Codec):
              'xmlcharrefreplace' - Replace with the appropriate XML
                                    character reference.
              'backslashreplace'  - Replace with backslashed escape
-                                   sequences (only for encoding).
+                                   sequences.
+             'namereplace'       - Replace with \\N{...} escape sequences.
 
             The set of allowed parameter values can be extended via
             register_error.
@@ -429,7 +431,8 @@ class StreamReader(Codec):
 
              'strict' - raise a ValueError (or a subclass)
              'ignore' - ignore the character and continue with the next
-             'replace'- replace with a suitable replacement character;
+             'replace'- replace with a suitable replacement character
+             'backslashreplace' - Replace with backslashed escape sequences;
 
             The set of allowed parameter values can be extended via
             register_error.
diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py
index 9743791..e29ac53 100644
--- a/Lib/test/test_codeccallbacks.py
+++ b/Lib/test/test_codeccallbacks.py
@@ -246,6 +246,11 @@ class CodecCallbackTest(unittest.TestCase):
                     "\u0000\ufffd"
                 )
 
+                self.assertEqual(
+                    b"\x00\x00\x00\x00\x00".decode("unicode-internal", "backslashreplace"),
+                    "\u0000\\x00"
+                )
+
                 codecs.register_error("test.hui", handler_unicodeinternal)
 
                 self.assertEqual(
@@ -565,17 +570,6 @@ class CodecCallbackTest(unittest.TestCase):
            codecs.backslashreplace_errors,
            UnicodeError("ouch")
         )
-        # "backslashreplace" can only be used for encoding
-        self.assertRaises(
-            TypeError,
-            codecs.backslashreplace_errors,
-            UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")
-        )
-        self.assertRaises(
-            TypeError,
-            codecs.backslashreplace_errors,
-            UnicodeTranslateError("\u3042", 0, 1, "ouch")
-        )
         # Use the correct exception
         self.assertEqual(
             codecs.backslashreplace_errors(
@@ -701,6 +695,16 @@ class CodecCallbackTest(unittest.TestCase):
                     UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")),
                 ("\\udfff", 1)
             )
+        self.assertEqual(
+            codecs.backslashreplace_errors(
+                UnicodeDecodeError("ascii", bytearray(b"\xff"), 0, 1, "ouch")),
+            ("\\xff", 1)
+        )
+        self.assertEqual(
+            codecs.backslashreplace_errors(
+                UnicodeTranslateError("\u3042", 0, 1, "ouch")),
+            ("\\u3042", 1)
+        )
 
     def test_badhandlerresults(self):
         results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) )
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 7fed1f7..353a850 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -378,6 +378,10 @@ class ReadTest(MixInCheckStateHandling):
                              before + after)
             self.assertEqual(test_sequence.decode(self.encoding, "replace"),
                              before + self.ill_formed_sequence_replace + after)
+            backslashreplace = ''.join('\\x%02x' % b
+                                       for b in self.ill_formed_sequence)
+            self.assertEqual(test_sequence.decode(self.encoding, "backslashreplace"),
+                             before + backslashreplace + after)
 
 class UTF32Test(ReadTest, unittest.TestCase):
     encoding = "utf-32"
@@ -1300,14 +1304,19 @@ class UnicodeInternalTest(unittest.TestCase):
                                   "unicode_internal")
         if sys.byteorder == "little":
             invalid = b"\x00\x00\x11\x00"
+            invalid_backslashreplace = r"\x00\x00\x11\x00"
         else:
             invalid = b"\x00\x11\x00\x00"
+            invalid_backslashreplace = r"\x00\x11\x00\x00"
         with support.check_warnings():
             self.assertRaises(UnicodeDecodeError,
                               invalid.decode, "unicode_internal")
         with support.check_warnings():
             self.assertEqual(invalid.decode("unicode_internal", "replace"),
                              '\ufffd')
+        with support.check_warnings():
+            self.assertEqual(invalid.decode("unicode_internal", "backslashreplace"),
+                             invalid_backslashreplace)
 
     @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t')
     def test_decode_error_attributes(self):
@@ -2043,6 +2052,16 @@ class CharmapTest(unittest.TestCase):
         )
 
         self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab"),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace", "ab\ufffe"),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
             codecs.charmap_decode(b"\x00\x01\x02", "ignore", "ab"),
             ("ab", 3)
         )
@@ -2119,6 +2138,25 @@ class CharmapTest(unittest.TestCase):
         )
 
         self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+                                  {0: 'a', 1: 'b'}),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+                                  {0: 'a', 1: 'b', 2: None}),
+            ("ab\\x02", 3)
+        )
+
+        # Issue #14850
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+                                  {0: 'a', 1: 'b', 2: '\ufffe'}),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
             codecs.charmap_decode(b"\x00\x01\x02", "ignore",
                                   {0: 'a', 1: 'b'}),
             ("ab", 3)
@@ -2195,6 +2233,18 @@ class CharmapTest(unittest.TestCase):
         )
 
         self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+                                  {0: a, 1: b}),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
+            codecs.charmap_decode(b"\x00\x01\x02", "backslashreplace",
+                                  {0: a, 1: b, 2: 0xFFFE}),
+            ("ab\\x02", 3)
+        )
+
+        self.assertEqual(
             codecs.charmap_decode(b"\x00\x01\x02", "ignore",
                                   {0: a, 1: b}),
             ("ab", 3)
@@ -2253,9 +2303,13 @@ class TypesTest(unittest.TestCase):
 
         self.assertRaises(UnicodeDecodeError, codecs.unicode_escape_decode, br"\U00110000")
         self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+        self.assertEqual(codecs.unicode_escape_decode(r"\U00110000", "backslashreplace"),
+                         (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
 
         self.assertRaises(UnicodeDecodeError, codecs.raw_unicode_escape_decode, br"\U00110000")
         self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "replace"), ("\ufffd", 10))
+        self.assertEqual(codecs.raw_unicode_escape_decode(r"\U00110000", "backslashreplace"),
+                         (r"\x5c\x55\x30\x30\x31\x31\x30\x30\x30\x30", 10))
 
 
 class UnicodeEscapeTest(unittest.TestCase):
@@ -2894,11 +2948,13 @@ class CodePageTest(unittest.TestCase):
             (b'[\xff]', 'strict', None),
             (b'[\xff]', 'ignore', '[]'),
             (b'[\xff]', 'replace', '[\ufffd]'),
+            (b'[\xff]', 'backslashreplace', '[\\xff]'),
             (b'[\xff]', 'surrogateescape', '[\udcff]'),
             (b'[\xff]', 'surrogatepass', None),
             (b'\x81\x00abc', 'strict', None),
             (b'\x81\x00abc', 'ignore', '\x00abc'),
             (b'\x81\x00abc', 'replace', '\ufffd\x00abc'),
+            (b'\x81\x00abc', 'backslashreplace', '\\xff\x00abc'),
         ))
 
     def test_cp1252(self):
diff --git a/Misc/NEWS b/Misc/NEWS
index 50e4bbe..163aff4 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@ Release date: TBA
 Core and Builtins
 -----------------
 
+- Issue #22286: The "backslashreplace" error handlers now works with
+  decoding and translating.
+
 - Issue #23253: Delay-load ShellExecute[AW] in os.startfile for reduced
   startup overhead on Windows.
 
diff --git a/Python/codecs.c b/Python/codecs.c
index a0a5403..a558859 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -864,74 +864,112 @@ PyObject *PyCodec_XMLCharRefReplaceErrors(PyObject *exc)
 
 PyObject *PyCodec_BackslashReplaceErrors(PyObject *exc)
 {
-    if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
-        PyObject *restuple;
-        PyObject *object;
-        Py_ssize_t i;
-        Py_ssize_t start;
-        Py_ssize_t end;
-        PyObject *res;
-        unsigned char *outp;
-        Py_ssize_t ressize;
-        Py_UCS4 c;
-        if (PyUnicodeEncodeError_GetStart(exc, &start))
+    PyObject *object;
+    Py_ssize_t i;
+    Py_ssize_t start;
+    Py_ssize_t end;
+    PyObject *res;
+    unsigned char *outp;
+    int ressize;
+    Py_UCS4 c;
+
+    if (PyObject_IsInstance(exc, PyExc_UnicodeDecodeError)) {
+        unsigned char *p;
+        if (PyUnicodeDecodeError_GetStart(exc, &start))
             return NULL;
-        if (PyUnicodeEncodeError_GetEnd(exc, &end))
+        if (PyUnicodeDecodeError_GetEnd(exc, &end))
             return NULL;
-        if (!(object = PyUnicodeEncodeError_GetObject(exc)))
+        if (!(object = PyUnicodeDecodeError_GetObject(exc)))
+            return NULL;
+        if (!(p = (unsigned char*)PyBytes_AsString(object))) {
+            Py_DECREF(object);
             return NULL;
-        if (end - start > PY_SSIZE_T_MAX / (1+1+8))
-            end = start + PY_SSIZE_T_MAX / (1+1+8);
-        for (i = start, ressize = 0; i < end; ++i) {
-            /* object is guaranteed to be "ready" */
-            c = PyUnicode_READ_CHAR(object, i);
-            if (c >= 0x10000) {
-                ressize += 1+1+8;
-            }
-            else if (c >= 0x100) {
-                ressize += 1+1+4;
-            }
-            else
-                ressize += 1+1+2;
         }
-        res = PyUnicode_New(ressize, 127);
+        res = PyUnicode_New(4 * (end - start), 127);
         if (res == NULL) {
             Py_DECREF(object);
             return NULL;
         }
-        for (i = start, outp = PyUnicode_1BYTE_DATA(res);
-            i < end; ++i) {
-            c = PyUnicode_READ_CHAR(object, i);
-            *outp++ = '\\';
-            if (c >= 0x00010000) {
-                *outp++ = 'U';
-                *outp++ = Py_hexdigits[(c>>28)&0xf];
-                *outp++ = Py_hexdigits[(c>>24)&0xf];
-                *outp++ = Py_hexdigits[(c>>20)&0xf];
-                *outp++ = Py_hexdigits[(c>>16)&0xf];
-                *outp++ = Py_hexdigits[(c>>12)&0xf];
-                *outp++ = Py_hexdigits[(c>>8)&0xf];
-            }
-            else if (c >= 0x100) {
-                *outp++ = 'u';
-                *outp++ = Py_hexdigits[(c>>12)&0xf];
-                *outp++ = Py_hexdigits[(c>>8)&0xf];
-            }
-            else
-                *outp++ = 'x';
-            *outp++ = Py_hexdigits[(c>>4)&0xf];
-            *outp++ = Py_hexdigits[c&0xf];
+        outp = PyUnicode_1BYTE_DATA(res);
+        for (i = start; i < end; i++, outp += 4) {
+            unsigned char c = p[i];
+            outp[0] = '\\';
+            outp[1] = 'x';
+            outp[2] = Py_hexdigits[(c>>4)&0xf];
+            outp[3] = Py_hexdigits[c&0xf];
         }
 
         assert(_PyUnicode_CheckConsistency(res, 1));
-        restuple = Py_BuildValue("(Nn)", res, end);
         Py_DECREF(object);
-        return restuple;
+        return Py_BuildValue("(Nn)", res, end);
+    }
+    if (PyObject_IsInstance(exc, PyExc_UnicodeEncodeError)) {
+        if (PyUnicodeEncodeError_GetStart(exc, &start))
+            return NULL;
+        if (PyUnicodeEncodeError_GetEnd(exc, &end))
+            return NULL;
+        if (!(object = PyUnicodeEncodeError_GetObject(exc)))
+            return NULL;
+    }
+    else if (PyObject_IsInstance(exc, PyExc_UnicodeTranslateError)) {
+        if (PyUnicodeTranslateError_GetStart(exc, &start))
+            return NULL;
+        if (PyUnicodeTranslateError_GetEnd(exc, &end))
+            return NULL;
+        if (!(object = PyUnicodeTranslateError_GetObject(exc)))
+            return NULL;
     }
     else {
         wrong_exception_type(exc);
         return NULL;
     }
+
+    if (end - start > PY_SSIZE_T_MAX / (1+1+8))
+        end = start + PY_SSIZE_T_MAX / (1+1+8);
+    for (i = start, ressize = 0; i < end; ++i) {
+        /* object is guaranteed to be "ready" */
+        c = PyUnicode_READ_CHAR(object, i);
+        if (c >= 0x10000) {
+            ressize += 1+1+8;
+        }
+        else if (c >= 0x100) {
+            ressize += 1+1+4;
+        }
+        else
+            ressize += 1+1+2;
+    }
+    res = PyUnicode_New(ressize, 127);
+    if (res == NULL) {
+        Py_DECREF(object);
+        return NULL;
+    }
+    outp = PyUnicode_1BYTE_DATA(res);
+    for (i = start; i < end; ++i) {
+        c = PyUnicode_READ_CHAR(object, i);
+        *outp++ = '\\';
+        if (c >= 0x00010000) {
+            *outp++ = 'U';
+            *outp++ = Py_hexdigits[(c>>28)&0xf];
+            *outp++ = Py_hexdigits[(c>>24)&0xf];
+            *outp++ = Py_hexdigits[(c>>20)&0xf];
+            *outp++ = Py_hexdigits[(c>>16)&0xf];
+            *outp++ = Py_hexdigits[(c>>12)&0xf];
+            *outp++ = Py_hexdigits[(c>>8)&0xf];
+        }
+        else if (c >= 0x100) {
+            *outp++ = 'u';
+            *outp++ = Py_hexdigits[(c>>12)&0xf];
+            *outp++ = Py_hexdigits[(c>>8)&0xf];
+        }
+        else
+            *outp++ = 'x';
+        *outp++ = Py_hexdigits[(c>>4)&0xf];
+        *outp++ = Py_hexdigits[c&0xf];
+    }
+
+    assert(_PyUnicode_CheckConsistency(res, 1));
+    Py_DECREF(object);
+    return Py_BuildValue("(Nn)", res, end);
 }
 
 static _PyUnicode_Name_CAPI *ucnhash_CAPI = NULL;
@@ -1444,8 +1482,8 @@ static int _PyCodecRegistry_Init(void)
                 backslashreplace_errors,
                 METH_O,
                 PyDoc_STR("Implements the 'backslashreplace' error handling, "
-                          "which replaces an unencodable character with a "
-                          "backslashed escape sequence.")
+                          "which replaces malformed data with a backslashed "
+                          "escape sequence.")
             }
         },
         {