Issue #14629: Raise SyntaxError in tokenizer.detect_encoding

if the first two lines have non-UTF-8 characters without an encoding declaration.

1 files changed, 5 insertions, 2 deletions

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index f575e9b..f283c6d 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -292,9 +292,12 @@ def detect_encoding(readline):
 
     def find_cookie(line):
         try:
-            line_string = line.decode('ascii')
+            # Decode as UTF-8. Either the line is an encoding declaration,
+            # in which case it should be pure ASCII, or it must be UTF-8
+            # per default encoding.
+            line_string = line.decode('utf-8')
         except UnicodeDecodeError:
-            return None
+            raise SyntaxError("invalid or missing encoding declaration")
 
         matches = cookie_re.findall(line_string)
         if not matches: