diff options
| author | Pablo Galindo <Pablogsal@gmail.com> | 2021-03-28 22:48:05 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-03-28 22:48:05 (GMT) |
| commit | 261a452a1300eeeae1428ffd6e6623329c085e2c (patch) | |
| tree | f3b34dab530366537eda73354d8e28e05c51dd62 /Parser/tokenizer.h | |
| parent | fb1d01b9630b5069fe975f16e07a027d90b89434 (diff) | |
| download | cpython-261a452a1300eeeae1428ffd6e6623329c085e2c.zip cpython-261a452a1300eeeae1428ffd6e6623329c085e2c.tar.gz cpython-261a452a1300eeeae1428ffd6e6623329c085e2c.tar.bz2 | |
bpo-25643: Refactor the C tokenizer into smaller, logical units (GH-25050)
Diffstat (limited to 'Parser/tokenizer.h')
| -rw-r--r-- | Parser/tokenizer.h | 5 |
1 files changed, 2 insertions, 3 deletions
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 111126c..aaa31f3 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -15,8 +15,8 @@ extern "C" { enum decoding_state { STATE_INIT, - STATE_RAW, - STATE_NORMAL /* have a codec associated with input */ + STATE_SEEK_CODING, + STATE_NORMAL }; /* Tokenizer state */ @@ -54,7 +54,6 @@ struct tok_state { /* Stuff for PEP 0263 */ enum decoding_state decoding_state; int decoding_erred; /* whether erred in decoding */ - int read_coding_spec; /* whether 'coding:...' has been read */ char *encoding; /* Source encoding. */ int cont_line; /* whether we are in a continuation line. */ const char* line_start; /* pointer to start of current line */ |