summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.h
diff options
context:
space:
mode:
authorVictor Stinner <victor.stinner@haypocalc.com>2011-04-04 22:39:01 (GMT)
committerVictor Stinner <victor.stinner@haypocalc.com>2011-04-04 22:39:01 (GMT)
commit7f2fee36401f7b987a368fe043637b3ae7116600 (patch)
treed21cec17c440b8255cf9bc4c79269d6bfb3253f0 /Parser/tokenizer.h
parent9bdb43e43f9f5d80699e297c2f73c106179b33d5 (diff)
downloadcpython-7f2fee36401f7b987a368fe043637b3ae7116600.zip
cpython-7f2fee36401f7b987a368fe043637b3ae7116600.tar.gz
cpython-7f2fee36401f7b987a368fe043637b3ae7116600.tar.bz2
Issue #10785: Store the filename as Unicode in the Python parser.
Diffstat (limited to 'Parser/tokenizer.h')
-rw-r--r--Parser/tokenizer.h8
1 files changed, 7 insertions, 1 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 2be3bf2..3a0d3cb 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -40,7 +40,13 @@ struct tok_state {
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
/* Stuff for checking on different tab sizes */
- const char *filename; /* encoded to the filesystem encoding */
+#ifndef PGEN
+ /* pgen doesn't have access to Python codecs, it cannot decode the input
+ filename. The bytes filename might be kept, but it is only used by
+ indenterror() and it is not really needed: pgen only compiles one file
+ (Grammar/Grammar). */
+ PyObject *filename;
+#endif
int altwarning; /* Issue warning if alternate tabs don't match */
int alterror; /* Issue error if alternate tabs don't match */
int alttabsize; /* Alternate tab spacing */