diff options
Diffstat (limited to 'Parser/tokenizer.h')
| -rw-r--r-- | Parser/tokenizer.h | 5 |
1 files changed, 3 insertions, 2 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index e3328f1..1a81e33 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -58,10 +58,11 @@ struct tok_state { #endif const char* enc; /* Encoding for the current str. */ const char* str; + const char* input; /* Tokenizer's newline translated copy of the string. */ }; -extern struct tok_state *PyTokenizer_FromString(const char *); -extern struct tok_state *PyTokenizer_FromUTF8(const char *); +extern struct tok_state *PyTokenizer_FromString(const char *, int); +extern struct tok_state *PyTokenizer_FromUTF8(const char *, int); extern struct tok_state *PyTokenizer_FromFile(FILE *, char*, char *, char *); extern void PyTokenizer_Free(struct tok_state *); |