This patch changes the default behaviour of the builtin charmap

codec to not apply Latin-1 mappings for keys which are not found
in the mapping dictionaries, but instead treat them as undefined
mappings.

The patch was originally written by Martin v. Loewis with some
additional (cosmetic) changes and an updated test script
by Marc-Andre Lemburg.

The standard codecs were recreated from the most current files
available at the Unicode.org site using the Tools/scripts/gencodec.py
tool.

This patch closes the bugs #116285 and #119960.

1 files changed, 8 insertions, 13 deletions

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index fe591b5..b9e457d 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -1970,11 +1970,11 @@ PyObject *PyUnicode_DecodeCharmap(const char *s,
 	Py_DECREF(w);
 	if (x == NULL) {
 	    if (PyErr_ExceptionMatches(PyExc_LookupError)) {
-		/* No mapping found: default to Latin-1 mapping */
+		/* No mapping found means: mapping is undefined. */
 		PyErr_Clear();
-		*p++ = (Py_UNICODE)ch;
-		continue;
-	    }
+		x = Py_None;
+		Py_INCREF(x);
+	    } else
 	    goto onError;
 	}
 
@@ -2086,16 +2086,11 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
 	Py_DECREF(w);
 	if (x == NULL) {
 	    if (PyErr_ExceptionMatches(PyExc_LookupError)) {
-		/* No mapping found: default to Latin-1 mapping if possible */
+		/* No mapping found means: mapping is undefined. */
 		PyErr_Clear();
-		if (ch < 256) {
-		    *s++ = (char)ch;
-		    continue;
-		}
-		else if (!charmap_encoding_error(&p, &s, errors,
-				     "missing character mapping"))
-		    continue;
-	    }
+		x = Py_None;
+		Py_INCREF(x);
+	    } else
 	    goto onError;
 	}