From e3ce3bba9277a7c4cfde5aaf6269b6c68f334176 Mon Sep 17 00:00:00 2001
From: "Miss Islington (bot)"
 <31488909+miss-islington@users.noreply.github.com>
Date: Wed, 10 Jun 2020 17:14:16 -0700
Subject: bpo-40847: Consider a line with only a LINECONT a blank line
 (GH-20769)

A line with only a line continuation character should be considered
a blank line at tokenizer level so that only a single NEWLINE token
gets emitted. The old parser was working around the issue, but the
new parser threw a `SyntaxError` for valid input. For example,
an empty line following a line continuation character was interpreted
as a `SyntaxError`.

Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
(cherry picked from commit 896f4cf63f9ab93e30572d879a5719d5aa2499fb)

Co-authored-by: Lysandros Nikolaou <lisandrosnik@gmail.com>
---
 Lib/test/test_peg_parser.py                                |  7 +++++++
 Lib/test/test_syntax.py                                    | 14 ++++++++++++++
 .../2020-06-09-23-52-32.bpo-40847.4XAACw.rst               |  4 ++++
 Parser/tokenizer.c                                         |  3 ++-
 4 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst

diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py
index 6ccb257..fae85e3 100644
--- a/Lib/test/test_peg_parser.py
+++ b/Lib/test/test_peg_parser.py
@@ -153,6 +153,13 @@ TEST_CASES = [
     ('dict_comp', '{x:1 for x in a}'),
     ('dict_comp_if', '{x:1+2 for x in a if b}'),
     ('dict_empty', '{}'),
+    ('empty_line_after_linecont',
+     r'''
+        pass
+        \
+
+        pass
+     '''),
     ('for',
      '''
         for i in a:
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
index c6f53eb..1d10a20 100644
--- a/Lib/test/test_syntax.py
+++ b/Lib/test/test_syntax.py
@@ -854,6 +854,20 @@ class SyntaxTestCase(unittest.TestCase):
                           "iterable argument unpacking follows "
                           "keyword argument unpacking")
 
+    def test_empty_line_after_linecont(self):
+        # See issue-40847
+        s = r"""\
+pass
+        \
+
+pass
+"""
+        try:
+            compile(s, '<string>', 'exec')
+        except SyntaxError:
+            self.fail("Empty line after a line continuation character is valid.")
+
+
 def test_main():
     support.run_unittest(SyntaxTestCase)
     from test import test_syntax
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst
new file mode 100644
index 0000000..0b489f2
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst	
@@ -0,0 +1,4 @@
+Fix a bug where a line with only a line continuation character is not considered a blank line at tokenizer level.
+In such cases, more than a single `NEWLINE` token was emitted. The old parser was working around the issue,
+but the new parser threw a :exc:`SyntaxError` for valid input due to this. For example, an empty line following
+a line continuation character was interpreted as a :exc:`SyntaxError`. 
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index cebfadc..d461e4e 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -1203,8 +1203,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
             }
         }
         tok_backup(tok, c);
-        if (c == '#' || c == '\n') {
+        if (c == '#' || c == '\n' || c == '\\') {
             /* Lines with only whitespace and/or comments
+               and/or a line continuation character
                shouldn't affect the indentation and are
                not passed to the parser as NEWLINE tokens,
                except *totally* empty lines in interactive
-- 
cgit v0.12