diff options
Diffstat (limited to 'Parser/tokenizer.h')
| -rw-r--r-- | Parser/tokenizer.h | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 636295a..9abc598 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -31,12 +31,13 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. /* Tokenizer state */ struct tok_state { /* Input state; buf <= cur <= inp <= end */ - /* NB an entire token must fit in the buffer */ - char *buf; /* Input buffer */ + /* NB an entire line is held in the buffer */ + char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */ char *cur; /* Next character in buffer */ char *inp; /* End of data in buffer */ - char *end; /* End of input buffer */ - int done; /* 0 normally, 1 at EOF, -1 after error */ + char *end; /* End of input buffer if buf != NULL */ + int done; /* E_OK normally, E_EOF at EOF, otherwise error code */ + /* NB If done != E_OK, cur must be == inp!!! */ FILE *fp; /* Rest of input; NULL if tokenizing a string */ int tabsize; /* Tab spacing */ int indent; /* Current indentation index */ |