diff options
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 2 | ||||
| -rw-r--r-- | Parser/tokenizer.h | 2 |
2 files changed, 4 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index f59ab59..7fcff20 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1533,6 +1533,7 @@ PyTokenizer_Get(struct tok_state *tok, char **p_start, char **p_end) return result; } +#ifdef Py_USING_UNICODE /* This function is only called from parsetok. However, it cannot live there, as it must be empty for PGEN, and we can check for PGEN only in this file. */ @@ -1593,6 +1594,7 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) } #endif +#endif /* defined(Py_USING_UNICODE) */ #ifdef Py_DEBUG diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 8482cdd..feccd4e 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -58,8 +58,10 @@ extern struct tok_state *PyTokenizer_FromString(const char *); extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *); extern void PyTokenizer_Free(struct tok_state *); extern int PyTokenizer_Get(struct tok_state *, char **, char **); +#ifdef Py_USING_UNICODE extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset); +#endif #ifdef __cplusplus } |