diff options
Diffstat (limited to 'Parser/tokenizer.h')
| -rw-r--r-- | Parser/tokenizer.h | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 5e7ebf7..8482cdd 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -58,6 +58,8 @@ extern struct tok_state *PyTokenizer_FromString(const char *); extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *); extern void PyTokenizer_Free(struct tok_state *); extern int PyTokenizer_Get(struct tok_state *, char **, char **); +extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, + int len, int *offset); #ifdef __cplusplus } |