summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.c
diff options
context:
space:
mode:
authorGuido van Rossum <guido@python.org>1998-01-19 22:07:46 (GMT)
committerGuido van Rossum <guido@python.org>1998-01-19 22:07:46 (GMT)
commitcf57d8b8c92143342b4d978c6134d307114fbf44 (patch)
tree6d3ad951d848e577e11a4f41a7f179d774e94f29 /Parser/tokenizer.c
parentd29806c37e69042bb6dcaded85beaeef790db605 (diff)
downloadcpython-cf57d8b8c92143342b4d978c6134d307114fbf44.zip
cpython-cf57d8b8c92143342b4d978c6134d307114fbf44.tar.gz
cpython-cf57d8b8c92143342b4d978c6134d307114fbf44.tar.bz2
tok_nextc() should return unsigned characters, to avoid mistaking
'\377' for EOF.
Diffstat (limited to 'Parser/tokenizer.c')
-rw-r--r--Parser/tokenizer.c12
1 files changed, 10 insertions, 2 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 3dc6c82..134f00f 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -46,6 +46,14 @@ extern char *PyOS_Readline Py_PROTO((char *));
/* Don't ever change this -- it would break the portability of Python code */
#define TABSIZE 8
+/* Convert a possibly signed character to a nonnegative int */
+/* XXX This assumes characters are 8 bits wide */
+#ifdef __CHAR_UNSIGNED__
+#define Py_CHARMASK(c) (c)
+#else
+#define Py_CHARMASK(c) ((c) & 0xff)
+#endif
+
/* Forward */
static struct tok_state *tok_new Py_PROTO((void));
static int tok_nextc Py_PROTO((struct tok_state *tok));
@@ -178,7 +186,7 @@ tok_nextc(tok)
{
for (;;) {
if (tok->cur != tok->inp) {
- return *tok->cur++; /* Fast path */
+ return Py_CHARMASK(*tok->cur++); /* Fast path */
}
if (tok->done != E_OK)
return EOF;
@@ -197,7 +205,7 @@ tok_nextc(tok)
tok->buf = tok->cur;
tok->lineno++;
tok->inp = end;
- return *tok->cur++;
+ return Py_CHARMASK(*tok->cur++);
}
if (tok->prompt != NULL) {
char *new = PyOS_Readline(tok->prompt);