diff options
Diffstat (limited to 'Lib/_pyio.py')
| -rw-r--r-- | Lib/_pyio.py | 103 |
1 files changed, 70 insertions, 33 deletions
diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 3fa9325..fec1c6f 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -14,7 +14,6 @@ except ImportError: import io from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END) -from errno import EINTR # open() uses st_blksize whenever we can DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes @@ -23,20 +22,12 @@ DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes # defined in io.py. We don't use real inheritance though, because we don't # want to inherit the C implementations. - -class BlockingIOError(IOError): - - """Exception raised when I/O would block on a non-blocking I/O stream.""" - - def __init__(self, errno, strerror, characters_written=0): - super().__init__(errno, strerror) - if not isinstance(characters_written, int): - raise TypeError("characters_written must be a integer") - self.characters_written = characters_written +# Rebind for compatibility +BlockingIOError = BlockingIOError def open(file, mode="r", buffering=-1, encoding=None, errors=None, - newline=None, closefd=True): + newline=None, closefd=True, opener=None): r"""Open file and return a stream. Raise IOError upon failure. @@ -131,6 +122,12 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, be kept open when the file is closed. This does not work when a file name is given and must be True in that case. + A custom opener can be used by passing a callable as *opener*. The + underlying file descriptor for the file object is then obtained by calling + *opener* with (*file*, *flags*). *opener* must return an open file + descriptor (passing os.open as *opener* results in functionality similar to + passing None). + open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing are performed. When open() is used to open a file in a text mode ('w', @@ -185,7 +182,7 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, (writing and "w" or "") + (appending and "a" or "") + (updating and "+" or ""), - closefd) + closefd, opener=opener) line_buffering = False if buffering == 1 or buffering < 0 and raw.isatty(): buffering = -1 @@ -944,15 +941,19 @@ class BufferedReader(_BufferedIOMixin): # Special case for when the number of bytes to read is unspecified. if n is None or n == -1: self._reset_read_buf() + if hasattr(self.raw, 'readall'): + chunk = self.raw.readall() + if chunk is None: + return buf[pos:] or None + else: + return buf[pos:] + chunk chunks = [buf[pos:]] # Strip the consumed bytes. current_size = 0 while True: # Read until EOF or until read() would block. try: chunk = self.raw.read() - except IOError as e: - if e.errno != EINTR: - raise + except InterruptedError: continue if chunk in empty_values: nodata_val = chunk @@ -974,9 +975,7 @@ class BufferedReader(_BufferedIOMixin): while avail < n: try: chunk = self.raw.read(wanted) - except IOError as e: - if e.errno != EINTR: - raise + except InterruptedError: continue if chunk in empty_values: nodata_val = chunk @@ -1009,9 +1008,7 @@ class BufferedReader(_BufferedIOMixin): while True: try: current = self.raw.read(to_read) - except IOError as e: - if e.errno != EINTR: - raise + except InterruptedError: continue break if current: @@ -1122,9 +1119,7 @@ class BufferedWriter(_BufferedIOMixin): while self._write_buf: try: n = self.raw.write(self._write_buf) - except IOError as e: - if e.errno != EINTR: - raise + except InterruptedError: continue if n > len(self._write_buf) or n < 0: raise IOError("write() returned incorrect number of bytes") @@ -1516,6 +1511,7 @@ class TextIOWrapper(TextIOBase): self._snapshot = None # info for reconstructing decoder state self._seekable = self._telling = self.buffer.seekable() self._has_read1 = hasattr(self.buffer, 'read1') + self._b2cratio = 0.0 if self._seekable and self.writable(): position = self.buffer.tell() @@ -1686,7 +1682,12 @@ class TextIOWrapper(TextIOBase): else: input_chunk = self.buffer.read(self._CHUNK_SIZE) eof = not input_chunk - self._set_decoded_chars(self._decoder.decode(input_chunk, eof)) + decoded_chars = self._decoder.decode(input_chunk, eof) + self._set_decoded_chars(decoded_chars) + if decoded_chars: + self._b2cratio = len(input_chunk) / len(self._decoded_chars) + else: + self._b2cratio = 0.0 if self._telling: # At the snapshot point, len(dec_buffer) bytes before the read, @@ -1740,20 +1741,56 @@ class TextIOWrapper(TextIOBase): # forward until it gives us enough decoded characters. saved_state = decoder.getstate() try: + # Fast search for an acceptable start point, close to our + # current pos. + # Rationale: calling decoder.decode() has a large overhead + # regardless of chunk size; we want the number of such calls to + # be O(1) in most situations (common decoders, non-crazy input). + # Actually, it will be exactly 1 for fixed-size codecs (all + # 8-bit codecs, also UTF-16 and UTF-32). + skip_bytes = int(self._b2cratio * chars_to_skip) + skip_back = 1 + assert skip_bytes <= len(next_input) + while skip_bytes > 0: + decoder.setstate((b'', dec_flags)) + # Decode up to temptative start point + n = len(decoder.decode(next_input[:skip_bytes])) + if n <= chars_to_skip: + b, d = decoder.getstate() + if not b: + # Before pos and no bytes buffered in decoder => OK + dec_flags = d + chars_to_skip -= n + break + # Skip back by buffered amount and reset heuristic + skip_bytes -= len(b) + skip_back = 1 + else: + # We're too far ahead, skip back a bit + skip_bytes -= skip_back + skip_back = skip_back * 2 + else: + skip_bytes = 0 + decoder.setstate((b'', dec_flags)) + # Note our initial start point. - decoder.setstate((b'', dec_flags)) - start_pos = position - start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 - need_eof = 0 + start_pos = position + skip_bytes + start_flags = dec_flags + if chars_to_skip == 0: + # We haven't moved from the start point. + return self._pack_cookie(start_pos, start_flags) # Feed the decoder one byte at a time. As we go, note the # nearest "safe start point" before the current location # (a point where the decoder has nothing buffered, so seek() # can safely start from there and advance to this location). - next_byte = bytearray(1) - for next_byte[0] in next_input: + bytes_fed = 0 + need_eof = 0 + # Chars decoded since `start_pos` + chars_decoded = 0 + for i in range(skip_bytes, len(next_input)): bytes_fed += 1 - chars_decoded += len(decoder.decode(next_byte)) + chars_decoded += len(decoder.decode(next_input[i:i+1])) dec_buffer, dec_flags = decoder.getstate() if not dec_buffer and chars_decoded <= chars_to_skip: # Decoder buffer is empty, so this is a safe start point. |