- Fix segfault with invalid coding.

- SF Bug #772896, unknown encoding results in MemoryError, which is not helpful

I will only backport the segfault fix.  I'll let Anthony decide if he wants
the other changes backported.  I will do the backport if asked.

3 files changed, 12 insertions, 2 deletions

diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 1fa9739..1d25437 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -42,7 +42,7 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
 	initerr(err_ret, filename);
 
 	if ((tok = PyTokenizer_FromString(s)) == NULL) {
-		err_ret->error = E_NOMEM;
+		err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
 		return NULL;
 	}
 
diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
index 64485eb..695e2b7 100644
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -116,6 +116,13 @@ getgrammar(char *filename)
 	return g;
 }
 
+/* Can't happen in pgen */
+PyObject*
+PyErr_Occurred()
+{
+	return 0;
+}
+
 void
 Py_FatalError(const char *msg)
 {
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 6957cc9..ce61322 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -603,8 +603,11 @@ decode_str(const char *str, struct tok_state *tok)
 	if (tok->enc != NULL) {
 		assert(utf8 == NULL);
 		utf8 = translate_into_utf8(str, tok->enc);
-		if (utf8 == NULL)
+		if (utf8 == NULL) {
+			PyErr_Format(PyExc_SyntaxError,
+				"unknown encoding: %s", tok->enc);
 			return NULL;
+		}
 		str = PyString_AsString(utf8);
 	}
 #endif