Patch #534304: Implement phase 1 of PEP 263.

1 files changed, 11 insertions, 0 deletions

diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 8fded37..9782666 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -4,6 +4,7 @@
 extern "C" {
 #endif
 
+#include "object.h"
 
 /* Tokenizer interface */
 
@@ -38,6 +39,16 @@ struct tok_state {
 	int alterror;	/* Issue error if alternate tabs don't match */
 	int alttabsize;	/* Alternate tab spacing */
 	int altindstack[MAXINDENT];	/* Stack of alternate indents */
+	/* Stuff for PEP 0263 */
+	int decoding_state;	/* -1:decoding, 0:init, 1:raw */
+	int decoding_erred;	/* whether erred in decoding  */
+	int read_coding_spec;	/* whether 'coding:...' has been read  */
+	int issued_encoding_warning; /* whether non-ASCII warning was issued */
+	char *encoding;
+	PyObject *decoding_readline; /* codecs.open(...).readline */
+	PyObject *decoding_buffer;
+	const char* enc;
+	const char* str;
 };
 
 extern struct tok_state *PyTokenizer_FromString(char *);