diff options
| -rw-r--r-- | Makefile.pre.in | 1 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 12 |
2 files changed, 2 insertions, 11 deletions
diff --git a/Makefile.pre.in b/Makefile.pre.in index 898c4f0..c704b14 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -239,6 +239,7 @@ PARSER_OBJS= $(POBJS) Parser/myreadline.o Parser/tokenizer.o PGOBJS= \ Objects/obmalloc.o \ Python/mysnprintf.o \ + Python/pyctype.o \ Parser/tokenizer_pgen.o \ Parser/printgrammar.o \ Parser/pgenmain.o diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index be2940c..77fec74 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -179,16 +179,6 @@ decode_str(const char *str, int exec_input, struct tok_state *tok) #else /* PGEN */ -/* Ensure that the locale does not interfere with tokenization. */ - -static int -ascii_isalnum(int c) -{ - return (('a' <= c && c <= 'z') || - ('A' <= c && c <= 'Z') || - ('0' <= c && c <= '9')); -} - static char * error_ret(struct tok_state *tok) /* XXX */ { @@ -255,7 +245,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (ascii_isalnum(Py_CHARMASK(t[0])) || + while (Py_ISALNUM(t[0]) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; |