Implement PEP 3131. Add isidentifier to str.

2 files changed, 24 insertions, 1 deletions

diff --git a/Python/ast.c b/Python/ast.c
index 5426c02..c13d093 100644
--- a/Python/ast.c
+++ b/Python/ast.c
@@ -47,8 +47,27 @@ static PyObject *parsestrplus(struct compiling *, const node *n,
 #define COMP_SETCOMP  2
 
 static identifier
-new_identifier(const char* n, PyArena *arena) {
+new_identifier(const char* n, PyArena *arena)
+{
     PyObject* id = PyUnicode_DecodeUTF8(n, strlen(n), NULL);
+    Py_UNICODE *u = PyUnicode_AS_UNICODE(id);
+    /* Check whether there are non-ASCII characters in the
+       identifier; if so, normalize to NFKC. */
+    for (; *u; u++) {
+	if (*u >= 128) {
+	    PyObject *m = PyImport_ImportModule("unicodedata");
+	    PyObject *id2;
+	    if (!m)
+		return NULL;
+	    id2 = PyObject_CallMethod(m, "normalize", "sO", "NFKC", id);
+	    Py_DECREF(m);
+	    if (!id2)
+		return NULL;
+	    Py_DECREF(id);
+	    id = id2;
+	    break;
+	}
+    }
     PyUnicode_InternInPlace(&id);
     PyArena_AddPyObject(arena, id);
     return id;
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index cc0926a..32bc6f7 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -1530,6 +1530,10 @@ err_input(perrdetail *err)
 	case E_LINECONT:
 		msg = "unexpected character after line continuation character";
 		break;
+
+	case E_IDENTIFIER:
+		msg = "invalid character in identifier";
+		break;
 	default:
 		fprintf(stderr, "error=%d\n", err->error);
 		msg = "unknown parsing error";