From 674c1882fe59c5d8d8ee33d404dcdaa7d848cbe0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= <martin@v.loewis.de>
Date: Wed, 24 Aug 2005 07:38:36 +0000
Subject: Return complete lines from codec stream readers even if there is an
 exception in later lines, resulting in correct line numbers for decoding
 errors in source code. Fixes #1178484.

---
 Doc/lib/libcodecs.tex |  6 +++++-
 Lib/codecs.py         | 20 +++++++++++++++++---
 Misc/NEWS             |  4 ++++
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/Doc/lib/libcodecs.tex b/Doc/lib/libcodecs.tex
index d98f474..7e22386 100644
--- a/Doc/lib/libcodecs.tex
+++ b/Doc/lib/libcodecs.tex
@@ -394,7 +394,7 @@ order to be compatible to the Python codec registry.
   be extended with \function{register_error()}.
 \end{classdesc}
 
-\begin{methoddesc}{read}{\optional{size\optional{, chars}}}
+\begin{methoddesc}{read}{\optional{size\optional{, chars, \optional{firstline}}}}
   Decodes data from the stream and returns the resulting object.
 
   \var{chars} indicates the number of characters to read from the
@@ -408,12 +408,16 @@ order to be compatible to the Python codec registry.
   decode as much as possible.  \var{size} is intended to prevent having
   to decode huge files in one step.
 
+  \var{firstline} indicates that it would be sufficient to only return
+  the first line, if there are decoding errors on later lines.
+
   The method should use a greedy read strategy meaning that it should
   read as much data as is allowed within the definition of the encoding
   and the given size, e.g.  if optional encoding endings or state
   markers are available on the stream, these should be read too.
 
   \versionchanged[\var{chars} argument added]{2.4}
+  \versionchanged[\var{firstline} argument added]{2.4.2}
 \end{methoddesc}
 
 \begin{methoddesc}{readline}{\optional{size\optional{, keepends}}}
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 33b7481..761cc06 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -236,7 +236,7 @@ class StreamReader(Codec):
     def decode(self, input, errors='strict'):
         raise NotImplementedError
 
-    def read(self, size=-1, chars=-1):
+    def read(self, size=-1, chars=-1, firstline=False):
 
         """ Decodes data from the stream self.stream and returns the
             resulting object.
@@ -253,6 +253,11 @@ class StreamReader(Codec):
             is intended to prevent having to decode huge files in one
             step.
 
+            If firstline is true, and a UnicodeDecodeError happens
+            after the first line terminator in the input only the first line
+            will be returned, the rest of the input will be kept until the
+            next call to read().
+
             The method should use a greedy read strategy meaning that
             it should read as much data as is allowed within the
             definition of the encoding and the given size, e.g.  if
@@ -275,7 +280,16 @@ class StreamReader(Codec):
                 newdata = self.stream.read(size)
             # decode bytes (those remaining from the last call included)
             data = self.bytebuffer + newdata
-            newchars, decodedbytes = self.decode(data, self.errors)
+            try:
+                newchars, decodedbytes = self.decode(data, self.errors)
+            except UnicodeDecodeError, exc:
+                if firstline:
+                    newchars, decodedbytes = self.decode(data[:exc.start], self.errors)
+                    lines = newchars.splitlines(True)
+                    if len(lines)<=1:
+                        raise
+                else:
+                    raise
             # keep undecoded bytes until the next call
             self.bytebuffer = data[decodedbytes:]
             # put new characters in the character buffer
@@ -306,7 +320,7 @@ class StreamReader(Codec):
         line = ""
         # If size is given, we call read() only once
         while True:
-            data = self.read(readsize)
+            data = self.read(readsize, firstline=True)
             if data:
                 # If we're at a "\r" read one extra character (which might
                 # be a "\n") to get a proper line ending. If the stream is
diff --git a/Misc/NEWS b/Misc/NEWS
index 48a7d89..1626d6a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -66,6 +66,10 @@ Extension Modules
 Library
 -------
 
+- Bug #1178484: Return complete lines from codec stream readers
+  even if there is an exception in later lines, resulting in
+  correct line numbers for decoding errors in source code. 
+
 - Bug #1266283: "lexists" is now in os.path.__all__.
 
 - The sets module can now properly compute s-=s and s^=s as an empty set.
-- 
cgit v0.12