diff options
| author | Benjamin Peterson <benjamin@python.org> | 2009-11-13 00:17:59 (GMT) |
|---|---|---|
| committer | Benjamin Peterson <benjamin@python.org> | 2009-11-13 00:17:59 (GMT) |
| commit | aeaa592516c8ea8a0a6318f69635baa817ced82f (patch) | |
| tree | 7b082d501188cc6acc44469e7f33a766c51d9d55 /Parser/tokenizer.h | |
| parent | a1d23326b19a0182ef74aae32386c5119b1a6e39 (diff) | |
| download | cpython-aeaa592516c8ea8a0a6318f69635baa817ced82f.zip cpython-aeaa592516c8ea8a0a6318f69635baa817ced82f.tar.gz cpython-aeaa592516c8ea8a0a6318f69635baa817ced82f.tar.bz2 | |
Merged revisions 76230 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r76230 | benjamin.peterson | 2009-11-12 17:39:44 -0600 (Thu, 12 Nov 2009) | 2 lines
fix several compile() issues by translating newlines in the tokenizer
........
Diffstat (limited to 'Parser/tokenizer.h')
| -rw-r--r-- | Parser/tokenizer.h | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index e3328f1..1a81e33 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -58,10 +58,11 @@ struct tok_state { #endif const char* enc; /* Encoding for the current str. */ const char* str; + const char* input; /* Tokenizer's newline translated copy of the string. */ }; -extern struct tok_state *PyTokenizer_FromString(const char *); -extern struct tok_state *PyTokenizer_FromUTF8(const char *); +extern struct tok_state *PyTokenizer_FromString(const char *, int); +extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); extern struct tok_state *PyTokenizer_FromFile(FILE *, char*, char *, char *); extern void PyTokenizer_Free(struct tok_state *); |