1 files changed, 11 insertions, 1 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0e2dc3a..be2940c 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok)
 
 #else /* PGEN */
 
+/* Ensure that the locale does not interfere with tokenization. */
+
+static int
+ascii_isalnum(int c)
+{
+	return (('a' <= c && c <= 'z') ||
+		('A' <= c && c <= 'Z') ||
+		('0' <= c && c <= '9'));
+}
+
 static char *
 error_ret(struct tok_state *tok) /* XXX */
 {
@@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
 			} while (t[0] == '\x20' || t[0] == '\t');
 
 			begin = t;
-			while (isalnum(Py_CHARMASK(t[0])) ||
+			while (ascii_isalnum(Py_CHARMASK(t[0])) ||
 			       t[0] == '-' || t[0] == '_' || t[0] == '.')
 				t++;