Latin-1 source code was not being properly decoded when passed through

compile(). This was due to left-over special-casing before UTF-8 became the
default source encoding.

Closes issue #3574. Thanks to Victor Stinner for help with the patch.

1 files changed, 2 insertions, 2 deletions

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index c45dea1..df9cbc7 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -49,14 +49,14 @@ struct tok_state {
 	enum decoding_state decoding_state;
 	int decoding_erred;	/* whether erred in decoding  */
 	int read_coding_spec;	/* whether 'coding:...' has been read  */
-	char *encoding;
+	char *encoding;         /* Source encoding. */
 	int cont_line;          /* whether we are in a continuation line. */
 	const char* line_start;	/* pointer to start of current line */
 #ifndef PGEN
 	PyObject *decoding_readline; /* codecs.open(...).readline */
 	PyObject *decoding_buffer;
 #endif
-	const char* enc;
+	const char* enc;        /* Encoding for the current str. */
 	const char* str;
 };