summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.c
diff options
context:
space:
mode:
authorJust van Rossum <just@letterror.com>2003-02-09 20:38:48 (GMT)
committerJust van Rossum <just@letterror.com>2003-02-09 20:38:48 (GMT)
commitf032f86e9e828dfe1147852783aa6784e3ddf610 (patch)
treeef5d3464d068b930514170f5a4b1acdeabd08a13 /Parser/tokenizer.c
parentcf117b0b4060a532e13980100c6f94ee98ec2ab1 (diff)
downloadcpython-f032f86e9e828dfe1147852783aa6784e3ddf610.zip
cpython-f032f86e9e828dfe1147852783aa6784e3ddf610.tar.gz
cpython-f032f86e9e828dfe1147852783aa6784e3ddf610.tar.bz2
patch 680474 that fixes bug 679880: compile/eval/exec refused utf-8 bom
mark. Added unit test.
Diffstat (limited to 'Parser/tokenizer.c')
-rw-r--r--Parser/tokenizer.c4
1 files changed, 2 insertions, 2 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index aaed637..4952a3c 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -506,14 +506,14 @@ decoding_feof(struct tok_state *tok)
/* Fetch a byte from TOK, using the string buffer. */
static int buf_getc(struct tok_state *tok) {
- return *tok->str++;
+ return Py_CHARMASK(*tok->str++);
}
/* Unfetch a byte from TOK, using the string buffer. */
static void buf_ungetc(int c, struct tok_state *tok) {
tok->str--;
- assert(*tok->str == c); /* tok->cur may point to read-only segment */
+ assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
}
/* Set the readline function for TOK to ENC. For the string-based