bpo-40176: Improve error messages for unclosed string literals (GH-19346)

Automerge-Triggered-By: GH:isidentical

2 files changed, 16 insertions, 16 deletions

diff --git a/Parser/pegen.c b/Parser/pegen.c
index 0d39030..0e7f86b 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -327,12 +327,6 @@ tokenizer_error(Parser *p)
         case E_TOKEN:
             msg = "invalid token";
             break;
-        case E_EOFS:
-            RAISE_SYNTAX_ERROR("EOF while scanning triple-quoted string literal");
-            return -1;
-        case E_EOLS:
-            RAISE_SYNTAX_ERROR("EOL while scanning string literal");
-            return -1;
         case E_EOF:
             if (p->tok->level) {
                 raise_unclosed_parentheses_error(p);
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index d3e846c..d9334aa 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1739,20 +1739,26 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
         /* Get rest of string */
         while (end_quote_size != quote_size) {
             c = tok_nextc(tok);
-            if (c == EOF) {
+            if (c == EOF || (quote_size == 1 && c == '\n')) {
+                // shift the tok_state's location into
+                // the start of string, and report the error
+                // from the initial quote character
+                tok->cur = (char *)tok->start;
+                tok->cur++;
+                tok->line_start = tok->multi_line_start;
+                int start = tok->lineno;
+                tok->lineno = tok->first_lineno;
+
                 if (quote_size == 3) {
-                    tok->done = E_EOFS;
+                    return syntaxerror(tok,
+                                       "unterminated triple-quoted string literal"
+                                       " (detected at line %d)", start);
                 }
                 else {
-                    tok->done = E_EOLS;
+                    return syntaxerror(tok,
+                                       "unterminated string literal (detected at"
+                                       " line %d)", start);
                 }
-                tok->cur = tok->inp;
-                return ERRORTOKEN;
-            }
-            if (quote_size == 1 && c == '\n') {
-                tok->done = E_EOLS;
-                tok->cur = tok->inp;
-                return ERRORTOKEN;
             }
             if (c == quote) {
                 end_quote_size += 1;