summaryrefslogtreecommitdiffstats
path: root/Parser/tokenizer.h
diff options
context:
space:
mode:
authorGuido van Rossum <guido@python.org>1990-10-14 12:07:46 (GMT)
committerGuido van Rossum <guido@python.org>1990-10-14 12:07:46 (GMT)
commit85a5fbbdfea617f6cc8fae82c9e8c2b5c424436d (patch)
treea1bf57db1c75e2a7029c8f2fad5f8dba4b9ba25c /Parser/tokenizer.h
parentc636014c430620325f8d213e9ba10d925991b8d7 (diff)
downloadcpython-85a5fbbdfea617f6cc8fae82c9e8c2b5c424436d.zip
cpython-85a5fbbdfea617f6cc8fae82c9e8c2b5c424436d.tar.gz
cpython-85a5fbbdfea617f6cc8fae82c9e8c2b5c424436d.tar.bz2
Initial revision
Diffstat (limited to 'Parser/tokenizer.h')
-rw-r--r--Parser/tokenizer.h29
1 files changed, 29 insertions, 0 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
new file mode 100644
index 0000000..8950c62
--- /dev/null
+++ b/Parser/tokenizer.h
@@ -0,0 +1,29 @@
+/* Tokenizer interface */
+
+#include "token.h" /* For token types */
+
+#define MAXINDENT 100 /* Max indentation level */
+
+/* Tokenizer state */
+struct tok_state {
+ /* Input state; buf <= cur <= inp <= end */
+ /* NB an entire token must fit in the buffer */
+ char *buf; /* Input buffer */
+ char *cur; /* Next character in buffer */
+ char *inp; /* End of data in buffer */
+ char *end; /* End of input buffer */
+ int done; /* 0 normally, 1 at EOF, -1 after error */
+ FILE *fp; /* Rest of input; NULL if tokenizing a string */
+ int tabsize; /* Tab spacing */
+ int indent; /* Current indentation index */
+ int indstack[MAXINDENT]; /* Stack of indents */
+ int atbol; /* Nonzero if at begin of new line */
+ int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
+ char *prompt, *nextprompt; /* For interactive prompting */
+ int lineno; /* Current line number */
+};
+
+extern struct tok_state *tok_setups PROTO((char *));
+extern struct tok_state *tok_setupf PROTO((FILE *, char *ps1, char *ps2));
+extern void tok_free PROTO((struct tok_state *));
+extern int tok_get PROTO((struct tok_state *, char **, char **));