From 88623d76b49a553445fff037bc9cf2e79a24ceef Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 3 Apr 2010 23:03:35 +0000 Subject: use our own locale independent ctype macros requires building pyctype.o into pgen --- Makefile.pre.in | 1 + Parser/tokenizer.c | 22 +++------------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 36b0a56..a332b6b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -230,6 +230,7 @@ PARSER_OBJS= $(POBJS) Parser/myreadline.o Parser/tokenizer.o PGOBJS= \ Objects/obmalloc.o \ Python/mysnprintf.o \ + Python/pyctype.o \ Parser/tokenizer_pgen.o \ Parser/printgrammar.o \ Parser/pgenmain.o diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index fbbd0bc..495182f 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -92,22 +92,6 @@ char *_PyParser_TokenNames[] = { "" }; - -/* Ensure that the locale does not interfere with tokenization. */ - -static int -ascii_isalpha(int c) -{ - return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z'); -} - -static int -ascii_isalnum(int c) -{ - return ascii_isalpha(c) || ('0' <= c && c <= '9'); -} - - /* Create and initialize a new tok_state structure */ static struct tok_state * @@ -245,7 +229,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (ascii_isalnum(Py_CHARMASK(t[0])) || + while (Py_ISALNUM(t[0]) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; @@ -1355,7 +1339,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end) } /* Identifier (most frequent token!) */ - if (ascii_isalpha(c) || c == '_') { + if (Py_ISALPHA(c) || c == '_') { /* Process r"", u"" and ur"" */ switch (c) { case 'b': @@ -1381,7 +1365,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end) goto letter_quote; break; } - while (ascii_isalnum(c) || c == '_') { + while (Py_ISALNUM(c) || c == '_') { c = tok_nextc(tok); } tok_backup(tok, c); -- cgit v0.12