diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2023-06-06 11:52:16 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-06-06 11:52:16 (GMT) |
| commit | c0a6ed39343b6dc355607fbff108c515e6c103bf (patch) | |
| tree | 39b12fa4a80178944cba90020076b336f4c9f1f4 /Parser | |
| parent | 0202aa002e06acef9aa55ace0d939103df19cadd (diff) | |
| download | cpython-c0a6ed39343b6dc355607fbff108c515e6c103bf.zip cpython-c0a6ed39343b6dc355607fbff108c515e6c103bf.tar.gz cpython-c0a6ed39343b6dc355607fbff108c515e6c103bf.tar.bz2 | |
gh-105259: Ensure we don't show newline characters for trailing NEWLINE tokens (#105364)
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 7 | ||||
| -rw-r--r-- | Parser/tokenizer.h | 1 |
2 files changed, 8 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index db12828..b349f59 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -114,6 +114,7 @@ tok_new(void) tok->report_warnings = 1; tok->tok_extra_tokens = 0; tok->comment_newline = 0; + tok->implicit_newline = 0; tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0}; tok->tok_mode_stack_index = 0; tok->tok_report_warnings = 1; @@ -355,10 +356,12 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) { return -1; } strcpy(new_str + current_size, line); + tok->implicit_newline = 0; if (last_char != '\n') { /* Last line does not end in \n, fake one */ new_str[current_size + line_size - 1] = '\n'; new_str[current_size + line_size] = '\0'; + tok->implicit_newline = 1; } tok->interactive_src_start = new_str; tok->interactive_src_end = new_str + current_size + line_size; @@ -1262,11 +1265,13 @@ tok_underflow_file(struct tok_state *tok) { tok->done = E_EOF; return 0; } + tok->implicit_newline = 0; if (tok->inp[-1] != '\n') { assert(tok->inp + 1 < tok->end); /* Last line does not end in \n, fake one */ *tok->inp++ = '\n'; *tok->inp = '\0'; + tok->implicit_newline = 1; } ADVANCE_LINENO(); @@ -1304,11 +1309,13 @@ tok_underflow_readline(struct tok_state* tok) { tok->done = E_EOF; return 0; } + tok->implicit_newline = 0; if (tok->inp[-1] != '\n') { assert(tok->inp + 1 < tok->end); /* Last line does not end in \n, fake one */ *tok->inp++ = '\n'; *tok->inp = '\0'; + tok->implicit_newline = 1; } ADVANCE_LINENO(); diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 600d429..16e919a 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -131,6 +131,7 @@ struct tok_state { int tok_report_warnings; int tok_extra_tokens; int comment_newline; + int implicit_newline; #ifdef Py_DEBUG int debug; #endif |