diff options
| -rw-r--r-- | Include/Python.h | 1 | ||||
| -rw-r--r-- | Include/bytes_methods.h | 84 | ||||
| -rw-r--r-- | Include/bytesobject.h | 53 | ||||
| -rw-r--r-- | Include/pydebug.h | 1 | ||||
| -rw-r--r-- | Include/pyerrors.h | 1 | ||||
| -rw-r--r-- | Include/pythonrun.h | 2 | ||||
| -rw-r--r-- | Lib/codecs.py | 48 | ||||
| -rw-r--r-- | Lib/io.py | 1601 | ||||
| -rw-r--r-- | Lib/test/buffer_tests.py | 206 | ||||
| -rw-r--r-- | Lib/test/exception_hierarchy.txt | 1 | ||||
| -rw-r--r-- | Lib/test/string_tests.py | 8 | ||||
| -rw-r--r-- | Lib/test/test_bytes.py | 982 | ||||
| -rw-r--r-- | Lib/test/test_io.py | 1162 | ||||
| -rw-r--r-- | Lib/test/test_print.py | 4 | ||||
| -rw-r--r-- | Makefile.pre.in | 9 | ||||
| -rw-r--r-- | Modules/main.c | 5 | ||||
| -rw-r--r-- | Objects/bytes_methods.c | 610 | ||||
| -rw-r--r-- | Objects/bytesobject.c | 3374 | ||||
| -rw-r--r-- | Objects/exceptions.c | 8 | ||||
| -rw-r--r-- | Objects/object.c | 3 | ||||
| -rw-r--r-- | Objects/stringlib/ctype.h | 110 | ||||
| -rw-r--r-- | Objects/stringlib/transmogrify.h | 362 | ||||
| -rw-r--r-- | Objects/stringobject.c | 14 | ||||
| -rw-r--r-- | Objects/typeobject.c | 2 | ||||
| -rw-r--r-- | Objects/unicodeobject.c | 8 | ||||
| -rw-r--r-- | PCbuild/pythoncore.vcproj | 16 | ||||
| -rw-r--r-- | Python/bltinmodule.c | 8 | ||||
| -rw-r--r-- | Python/pythonrun.c | 27 |
28 files changed, 8702 insertions, 8 deletions
diff --git a/Include/Python.h b/Include/Python.h index 763b144..a5e2853 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -92,6 +92,7 @@ #include "stringobject.h" /* #include "memoryobject.h" */ #include "bufferobject.h" +#include "bytesobject.h" #include "tupleobject.h" #include "listobject.h" #include "dictobject.h" diff --git a/Include/bytes_methods.h b/Include/bytes_methods.h new file mode 100644 index 0000000..59873f2 --- /dev/null +++ b/Include/bytes_methods.h @@ -0,0 +1,84 @@ +#ifndef Py_BYTES_CTYPE_H +#define Py_BYTES_CTYPE_H + +/* + * The internal implementation behind PyString (bytes) and PyBytes (buffer) + * methods of the given names, they operate on ASCII byte strings. + */ +extern PyObject* _Py_bytes_isspace(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_isalpha(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_isalnum(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_isdigit(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_islower(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_isupper(const char *cptr, Py_ssize_t len); +extern PyObject* _Py_bytes_istitle(const char *cptr, Py_ssize_t len); + +/* These store their len sized answer in the given preallocated *result arg. */ +extern void _Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len); +extern void _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len); +extern void _Py_bytes_title(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_capitalize(char *result, char *s, Py_ssize_t len); +extern void _Py_bytes_swapcase(char *result, char *s, Py_ssize_t len); + +/* Shared __doc__ strings. */ +extern const char _Py_isspace__doc__[]; +extern const char _Py_isalpha__doc__[]; +extern const char _Py_isalnum__doc__[]; +extern const char _Py_isdigit__doc__[]; +extern const char _Py_islower__doc__[]; +extern const char _Py_isupper__doc__[]; +extern const char _Py_istitle__doc__[]; +extern const char _Py_lower__doc__[]; +extern const char _Py_upper__doc__[]; +extern const char _Py_title__doc__[]; +extern const char _Py_capitalize__doc__[]; +extern const char _Py_swapcase__doc__[]; + +#define FLAG_LOWER 0x01 +#define FLAG_UPPER 0x02 +#define FLAG_ALPHA (FLAG_LOWER|FLAG_UPPER) +#define FLAG_DIGIT 0x04 +#define FLAG_ALNUM (FLAG_ALPHA|FLAG_DIGIT) +#define FLAG_SPACE 0x08 +#define FLAG_XDIGIT 0x10 + +extern const unsigned int _Py_ctype_table[256]; + +#define ISLOWER(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_LOWER) +#define ISUPPER(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_UPPER) +#define ISALPHA(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_ALPHA) +#define ISDIGIT(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_DIGIT) +#define ISXDIGIT(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_XDIGIT) +#define ISALNUM(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_ALNUM) +#define ISSPACE(c) (_Py_ctype_table[Py_CHARMASK(c)] & FLAG_SPACE) + +#undef islower +#define islower(c) undefined_islower(c) +#undef isupper +#define isupper(c) undefined_isupper(c) +#undef isalpha +#define isalpha(c) undefined_isalpha(c) +#undef isdigit +#define isdigit(c) undefined_isdigit(c) +#undef isxdigit +#define isxdigit(c) undefined_isxdigit(c) +#undef isalnum +#define isalnum(c) undefined_isalnum(c) +#undef isspace +#define isspace(c) undefined_isspace(c) + +extern const unsigned char _Py_ctype_tolower[256]; +extern const unsigned char _Py_ctype_toupper[256]; + +#define TOLOWER(c) (_Py_ctype_tolower[Py_CHARMASK(c)]) +#define TOUPPER(c) (_Py_ctype_toupper[Py_CHARMASK(c)]) + +#undef tolower +#define tolower(c) undefined_tolower(c) +#undef toupper +#define toupper(c) undefined_toupper(c) + +/* this is needed because some docs are shared from the .o, not static */ +#define PyDoc_STRVAR_shared(name,str) const char name[] = PyDoc_STR(str) + +#endif /* !Py_BYTES_CTYPE_H */ diff --git a/Include/bytesobject.h b/Include/bytesobject.h new file mode 100644 index 0000000..49d1d38 --- /dev/null +++ b/Include/bytesobject.h @@ -0,0 +1,53 @@ +/* Bytes object interface */ + +#ifndef Py_BYTESOBJECT_H +#define Py_BYTESOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdarg.h> + +/* Type PyBytesObject represents a mutable array of bytes. + * The Python API is that of a sequence; + * the bytes are mapped to ints in [0, 256). + * Bytes are not characters; they may be used to encode characters. + * The only way to go between bytes and str/unicode is via encoding + * and decoding. + * For the convenience of C programmers, the bytes type is considered + * to contain a char pointer, not an unsigned char pointer. + */ + +/* Object layout */ +typedef struct { + PyObject_VAR_HEAD + /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ + int ob_exports; /* how many buffer exports */ + Py_ssize_t ob_alloc; /* How many bytes allocated */ + char *ob_bytes; +} PyBytesObject; + +/* Type object */ +PyAPI_DATA(PyTypeObject) PyBytes_Type; +PyAPI_DATA(PyTypeObject) PyBytesIter_Type; + +/* Type check macros */ +#define PyBytes_Check(self) PyObject_TypeCheck(self, &PyBytes_Type) +#define PyBytes_CheckExact(self) (Py_TYPE(self) == &PyBytes_Type) + +/* Direct API functions */ +PyAPI_FUNC(PyObject *) PyBytes_FromObject(PyObject *); +PyAPI_FUNC(PyObject *) PyBytes_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); +PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); +PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); +PyAPI_FUNC(int) PyBytes_Resize(PyObject *, Py_ssize_t); + +/* Macros, trading safety for speed */ +#define PyBytes_AS_STRING(self) (assert(PyBytes_Check(self)),((PyBytesObject *)(self))->ob_bytes) +#define PyBytes_GET_SIZE(self) (assert(PyBytes_Check(self)),Py_SIZE(self)) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_BYTESOBJECT_H */ diff --git a/Include/pydebug.h b/Include/pydebug.h index 1d24853..1174f0c 100644 --- a/Include/pydebug.h +++ b/Include/pydebug.h @@ -11,6 +11,7 @@ PyAPI_DATA(int) Py_InteractiveFlag; PyAPI_DATA(int) Py_InspectFlag; PyAPI_DATA(int) Py_OptimizeFlag; PyAPI_DATA(int) Py_NoSiteFlag; +PyAPI_DATA(int) Py_BytesWarningFlag; PyAPI_DATA(int) Py_UseClassExceptionsFlag; PyAPI_DATA(int) Py_FrozenFlag; PyAPI_DATA(int) Py_TabcheckFlag; diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 9c2bc67..b687733 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -175,6 +175,7 @@ PyAPI_DATA(PyObject *) PyExc_RuntimeWarning; PyAPI_DATA(PyObject *) PyExc_FutureWarning; PyAPI_DATA(PyObject *) PyExc_ImportWarning; PyAPI_DATA(PyObject *) PyExc_UnicodeWarning; +PyAPI_DATA(PyObject *) PyExc_BytesWarning; /* Convenience functions */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index f2105b8..a4dd914 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -123,6 +123,7 @@ PyAPI_FUNC(void) _PyImportHooks_Init(void); PyAPI_FUNC(int) _PyFrame_Init(void); PyAPI_FUNC(int) _PyInt_Init(void); PyAPI_FUNC(void) _PyFloat_Init(void); +PyAPI_FUNC(int) PyBytes_Init(void); /* Various internal finalizers */ PyAPI_FUNC(void) _PyExc_Fini(void); @@ -138,6 +139,7 @@ PyAPI_FUNC(void) PyString_Fini(void); PyAPI_FUNC(void) PyInt_Fini(void); PyAPI_FUNC(void) PyFloat_Fini(void); PyAPI_FUNC(void) PyOS_FiniInterrupts(void); +PyAPI_FUNC(void) PyBytes_Fini(void); /* Stuff with no proper home (yet) */ PyAPI_FUNC(char *) PyOS_Readline(FILE *, FILE *, char *); diff --git a/Lib/codecs.py b/Lib/codecs.py index f834b8d..557ccf7 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -181,6 +181,18 @@ class IncrementalEncoder(object): Resets the encoder to the initial state. """ + def getstate(self): + """ + Return the current state of the encoder. + """ + return 0 + + def setstate(self, state): + """ + Set the current state of the encoder. state must have been + returned by getstate(). + """ + class BufferedIncrementalEncoder(IncrementalEncoder): """ This subclass of IncrementalEncoder can be used as the baseclass for an @@ -208,6 +220,12 @@ class BufferedIncrementalEncoder(IncrementalEncoder): IncrementalEncoder.reset(self) self.buffer = "" + def getstate(self): + return self.buffer or 0 + + def setstate(self, state): + self.buffer = state or "" + class IncrementalDecoder(object): """ An IncrementalDecoder decodes an input in multiple steps. The input can be @@ -235,6 +253,28 @@ class IncrementalDecoder(object): Resets the decoder to the initial state. """ + def getstate(self): + """ + Return the current state of the decoder. + + This must be a (buffered_input, additional_state_info) tuple. + buffered_input must be a bytes object containing bytes that + were passed to decode() that have not yet been converted. + additional_state_info must be a non-negative integer + representing the state of the decoder WITHOUT yet having + processed the contents of buffered_input. In the initial state + and after reset(), getstate() must return (b"", 0). + """ + return (b"", 0) + + def setstate(self, state): + """ + Set the current state of the decoder. + + state must have been returned by getstate(). The effect of + setstate((b"", 0)) must be equivalent to reset(). + """ + class BufferedIncrementalDecoder(IncrementalDecoder): """ This subclass of IncrementalDecoder can be used as the baseclass for an @@ -262,6 +302,14 @@ class BufferedIncrementalDecoder(IncrementalDecoder): IncrementalDecoder.reset(self) self.buffer = "" + def getstate(self): + # additional state info is always 0 + return (self.buffer, 0) + + def setstate(self, state): + # ignore additional state info + self.buffer = state[0] + # # The StreamWriter and StreamReader class provide generic working # interfaces which can be used to implement new encoding submodules diff --git a/Lib/io.py b/Lib/io.py new file mode 100644 index 0000000..334b34c --- /dev/null +++ b/Lib/io.py @@ -0,0 +1,1601 @@ +"""New I/O library conforming to PEP 3116. + +This is a prototype; hopefully eventually some of this will be +reimplemented in C. + +Conformance of alternative implementations: all arguments are intended +to be positional-only except the arguments of the open() function. +Argument names except those of the open() function are not part of the +specification. Instance variables and methods whose name starts with +a leading underscore are not part of the specification (except "magic" +names like __iter__). Only the top-level names listed in the __all__ +variable are part of the specification. + +XXX edge cases when switching between reading/writing +XXX need to support 1 meaning line-buffered +XXX whenever an argument is None, use the default value +XXX read/write ops should check readable/writable +XXX buffered readinto should work with arbitrary buffer objects +XXX use incremental encoder for text output, at least for UTF-16 and UTF-8-SIG +XXX check writable, readable and seekable in appropriate places +""" + +__author__ = ("Guido van Rossum <guido@python.org>, " + "Mike Verdone <mike.verdone@gmail.com>, " + "Mark Russell <mark.russell@zen.co.uk>") + +__all__ = ["BlockingIOError", "open", "IOBase", "RawIOBase", "FileIO", + "BytesIO", "StringIO", "BufferedIOBase", + "BufferedReader", "BufferedWriter", "BufferedRWPair", + "BufferedRandom", "TextIOBase", "TextIOWrapper"] + +import os +import abc +import sys +import codecs +import _fileio +import warnings + +# open() uses st_blksize whenever we can +DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes + +# py3k has only new style classes +__metaclass__ = type + +class BlockingIOError(IOError): + + """Exception raised when I/O would block on a non-blocking I/O stream.""" + + def __init__(self, errno, strerror, characters_written=0): + IOError.__init__(self, errno, strerror) + self.characters_written = characters_written + + +def open(file, mode="r", buffering=None, encoding=None, errors=None, + newline=None, closefd=True): + r"""Replacement for the built-in open function. + + Args: + file: string giving the name of the file to be opened; + or integer file descriptor of the file to be wrapped (*). + mode: optional mode string; see below. + buffering: optional int >= 0 giving the buffer size; values + can be: 0 = unbuffered, 1 = line buffered, + larger = fully buffered. + encoding: optional string giving the text encoding. + errors: optional string giving the encoding error handling. + newline: optional newlines specifier; must be None, '', '\n', '\r' + or '\r\n'; all other values are illegal. It controls the + handling of line endings. It works as follows: + + * On input, if `newline` is `None`, universal newlines + mode is enabled. Lines in the input can end in `'\n'`, + `'\r'`, or `'\r\n'`, and these are translated into + `'\n'` before being returned to the caller. If it is + `''`, universal newline mode is enabled, but line endings + are returned to the caller untranslated. If it has any of + the other legal values, input lines are only terminated by + the given string, and the line ending is returned to the + caller untranslated. + + * On output, if `newline` is `None`, any `'\n'` + characters written are translated to the system default + line separator, `os.linesep`. If `newline` is `''`, + no translation takes place. If `newline` is any of the + other legal values, any `'\n'` characters written are + translated to the given string. + + closefd: optional argument to keep the underlying file descriptor + open when the file is closed. It must not be false when + a filename is given. + + (*) If a file descriptor is given, it is closed when the returned + I/O object is closed, unless closefd=False is given. + + Mode strings characters: + 'r': open for reading (default) + 'w': open for writing, truncating the file first + 'a': open for writing, appending to the end if the file exists + 'b': binary mode + 't': text mode (default) + '+': open a disk file for updating (implies reading and writing) + 'U': universal newline mode (for backwards compatibility) + + Constraints: + - encoding or errors must not be given when a binary mode is given + - buffering must not be zero when a text mode is given + + Returns: + Depending on the mode and buffering arguments, either a raw + binary stream, a buffered binary stream, or a buffered text + stream, open for reading and/or writing. + """ + if not isinstance(file, (str, unicode, int)): + raise TypeError("invalid file: %r" % file) + if not isinstance(mode, str): + raise TypeError("invalid mode: %r" % mode) + if buffering is not None and not isinstance(buffering, int): + raise TypeError("invalid buffering: %r" % buffering) + if encoding is not None and not isinstance(encoding, str): + raise TypeError("invalid encoding: %r" % encoding) + if errors is not None and not isinstance(errors, str): + raise TypeError("invalid errors: %r" % errors) + modes = set(mode) + if modes - set("arwb+tU") or len(mode) > len(modes): + raise ValueError("invalid mode: %r" % mode) + reading = "r" in modes + writing = "w" in modes + appending = "a" in modes + updating = "+" in modes + text = "t" in modes + binary = "b" in modes + if "U" in modes: + if writing or appending: + raise ValueError("can't use U and writing mode at once") + reading = True + if text and binary: + raise ValueError("can't have text and binary mode at once") + if reading + writing + appending > 1: + raise ValueError("can't have read/write/append mode at once") + if not (reading or writing or appending): + raise ValueError("must have exactly one of read/write/append mode") + if binary and encoding is not None: + raise ValueError("binary mode doesn't take an encoding argument") + if binary and errors is not None: + raise ValueError("binary mode doesn't take an errors argument") + if binary and newline is not None: + raise ValueError("binary mode doesn't take a newline argument") + raw = FileIO(file, + (reading and "r" or "") + + (writing and "w" or "") + + (appending and "a" or "") + + (updating and "+" or ""), + closefd) + if buffering is None: + buffering = -1 + line_buffering = False + if buffering == 1 or buffering < 0 and raw.isatty(): + buffering = -1 + line_buffering = True + if buffering < 0: + buffering = DEFAULT_BUFFER_SIZE + try: + bs = os.fstat(raw.fileno()).st_blksize + except (os.error, AttributeError): + pass + else: + if bs > 1: + buffering = bs + if buffering < 0: + raise ValueError("invalid buffering size") + if buffering == 0: + if binary: + raw._name = file + raw._mode = mode + return raw + raise ValueError("can't have unbuffered text I/O") + if updating: + buffer = BufferedRandom(raw, buffering) + elif writing or appending: + buffer = BufferedWriter(raw, buffering) + elif reading: + buffer = BufferedReader(raw, buffering) + else: + raise ValueError("unknown mode: %r" % mode) + if binary: + buffer.name = file + buffer.mode = mode + return buffer + text = TextIOWrapper(buffer, encoding, errors, newline, line_buffering) + text.name = file + text.mode = mode + return text + +class _DocDescriptor: + """Helper for builtins.open.__doc__ + """ + def __get__(self, obj, typ): + return ( + "open(file, mode='r', buffering=None, encoding=None, " + "errors=None, newline=None, closefd=True)\n\n" + + open.__doc__) + +class OpenWrapper: + """Wrapper for builtins.open + + Trick so that open won't become a bound method when stored + as a class variable (as dumbdbm does). + + See initstdio() in Python/pythonrun.c. + """ + __doc__ = _DocDescriptor() + + def __new__(cls, *args, **kwargs): + return open(*args, **kwargs) + + +class UnsupportedOperation(ValueError, IOError): + pass + + +class IOBase(object): + + """Base class for all I/O classes. + + This class provides dummy implementations for many methods that + derived classes can override selectively; the default + implementations represent a file that cannot be read, written or + seeked. + + This does not define read(), readinto() and write(), nor + readline() and friends, since their signatures vary per layer. + + Not that calling any method (even inquiries) on a closed file is + undefined. Implementations may raise IOError in this case. + """ + + __metaclass__ = abc.ABCMeta + + ### Internal ### + + def _unsupported(self, name): + """Internal: raise an exception for unsupported operations.""" + raise UnsupportedOperation("%s.%s() not supported" % + (self.__class__.__name__, name)) + + ### Positioning ### + + def seek(self, pos, whence = 0): + """seek(pos: int, whence: int = 0) -> int. Change stream position. + + Seek to byte offset pos relative to position indicated by whence: + 0 Start of stream (the default). pos should be >= 0; + 1 Current position - whence may be negative; + 2 End of stream - whence usually negative. + Returns the new absolute position. + """ + self._unsupported("seek") + + def tell(self): + """tell() -> int. Return current stream position.""" + return self.seek(0, 1) + + def truncate(self, pos = None): + """truncate(size: int = None) -> int. Truncate file to size bytes. + + Size defaults to the current IO position as reported by tell(). + Returns the new size. + """ + self._unsupported("truncate") + + ### Flush and close ### + + def flush(self): + """flush() -> None. Flushes write buffers, if applicable. + + This is a no-op for read-only and non-blocking streams. + """ + # XXX Should this return the number of bytes written??? + + __closed = False + + def close(self): + """close() -> None. Flushes and closes the IO object. + + This must be idempotent. It should also set a flag for the + 'closed' property (see below) to test. + """ + if not self.__closed: + try: + self.flush() + except IOError: + pass # If flush() fails, just give up + self.__closed = True + + def __del__(self): + """Destructor. Calls close().""" + # The try/except block is in case this is called at program + # exit time, when it's possible that globals have already been + # deleted, and then the close() call might fail. Since + # there's nothing we can do about such failures and they annoy + # the end users, we suppress the traceback. + try: + self.close() + except: + pass + + ### Inquiries ### + + def seekable(self): + """seekable() -> bool. Return whether object supports random access. + + If False, seek(), tell() and truncate() will raise IOError. + This method may need to do a test seek(). + """ + return False + + def _checkSeekable(self, msg=None): + """Internal: raise an IOError if file is not seekable + """ + if not self.seekable(): + raise IOError("File or stream is not seekable." + if msg is None else msg) + + + def readable(self): + """readable() -> bool. Return whether object was opened for reading. + + If False, read() will raise IOError. + """ + return False + + def _checkReadable(self, msg=None): + """Internal: raise an IOError if file is not readable + """ + if not self.readable(): + raise IOError("File or stream is not readable." + if msg is None else msg) + + def writable(self): + """writable() -> bool. Return whether object was opened for writing. + + If False, write() and truncate() will raise IOError. + """ + return False + + def _checkWritable(self, msg=None): + """Internal: raise an IOError if file is not writable + """ + if not self.writable(): + raise IOError("File or stream is not writable." + if msg is None else msg) + + @property + def closed(self): + """closed: bool. True iff the file has been closed. + + For backwards compatibility, this is a property, not a predicate. + """ + return self.__closed + + def _checkClosed(self, msg=None): + """Internal: raise an ValueError if file is closed + """ + if self.closed: + raise ValueError("I/O operation on closed file." + if msg is None else msg) + + ### Context manager ### + + def __enter__(self): + """Context management protocol. Returns self.""" + self._checkClosed() + return self + + def __exit__(self, *args): + """Context management protocol. Calls close()""" + self.close() + + ### Lower-level APIs ### + + # XXX Should these be present even if unimplemented? + + def fileno(self): + """fileno() -> int. Returns underlying file descriptor if one exists. + + Raises IOError if the IO object does not use a file descriptor. + """ + self._unsupported("fileno") + + def isatty(self): + """isatty() -> int. Returns whether this is an 'interactive' stream. + + Returns False if we don't know. + """ + self._checkClosed() + return False + + ### Readline[s] and writelines ### + + def readline(self, limit = -1): + """For backwards compatibility, a (slowish) readline().""" + if hasattr(self, "peek"): + def nreadahead(): + readahead = self.peek(1) + if not readahead: + return 1 + n = (readahead.find(b"\n") + 1) or len(readahead) + if limit >= 0: + n = min(n, limit) + return n + else: + def nreadahead(): + return 1 + if limit is None: + limit = -1 + res = bytearray() + while limit < 0 or len(res) < limit: + b = self.read(nreadahead()) + if not b: + break + res += b + if res.endswith(b"\n"): + break + return bytes(res) + + def __iter__(self): + self._checkClosed() + return self + + def next(self): + line = self.readline() + if not line: + raise StopIteration + return line + + def readlines(self, hint=None): + if hint is None: + return list(self) + n = 0 + lines = [] + for line in self: + lines.append(line) + n += len(line) + if n >= hint: + break + return lines + + def writelines(self, lines): + self._checkClosed() + for line in lines: + self.write(line) + + +class RawIOBase(IOBase): + + """Base class for raw binary I/O. + + The read() method is implemented by calling readinto(); derived + classes that want to support read() only need to implement + readinto() as a primitive operation. In general, readinto() + can be more efficient than read(). + + (It would be tempting to also provide an implementation of + readinto() in terms of read(), in case the latter is a more + suitable primitive operation, but that would lead to nasty + recursion in case a subclass doesn't implement either.) + """ + + def read(self, n = -1): + """read(n: int) -> bytes. Read and return up to n bytes. + + Returns an empty bytes array on EOF, or None if the object is + set not to block and has no data to read. + """ + if n is None: + n = -1 + if n < 0: + return self.readall() + b = bytearray(n.__index__()) + n = self.readinto(b) + del b[n:] + return bytes(b) + + def readall(self): + """readall() -> bytes. Read until EOF, using multiple read() call.""" + res = bytearray() + while True: + data = self.read(DEFAULT_BUFFER_SIZE) + if not data: + break + res += data + return bytes(res) + + def readinto(self, b): + """readinto(b: bytes) -> int. Read up to len(b) bytes into b. + + Returns number of bytes read (0 for EOF), or None if the object + is set not to block as has no data to read. + """ + self._unsupported("readinto") + + def write(self, b): + """write(b: bytes) -> int. Write the given buffer to the IO stream. + + Returns the number of bytes written, which may be less than len(b). + """ + self._unsupported("write") + + +class FileIO(_fileio._FileIO, RawIOBase): + + """Raw I/O implementation for OS files. + + This multiply inherits from _FileIO and RawIOBase to make + isinstance(io.FileIO(), io.RawIOBase) return True without + requiring that _fileio._FileIO inherits from io.RawIOBase (which + would be hard to do since _fileio.c is written in C). + """ + + def close(self): + _fileio._FileIO.close(self) + RawIOBase.close(self) + + @property + def name(self): + return self._name + + @property + def mode(self): + return self._mode + + +class BufferedIOBase(IOBase): + + """Base class for buffered IO objects. + + The main difference with RawIOBase is that the read() method + supports omitting the size argument, and does not have a default + implementation that defers to readinto(). + + In addition, read(), readinto() and write() may raise + BlockingIOError if the underlying raw stream is in non-blocking + mode and not ready; unlike their raw counterparts, they will never + return None. + + A typical implementation should not inherit from a RawIOBase + implementation, but wrap one. + """ + + def read(self, n = None): + """read(n: int = None) -> bytes. Read and return up to n bytes. + + If the argument is omitted, None, or negative, reads and + returns all data until EOF. + + If the argument is positive, and the underlying raw stream is + not 'interactive', multiple raw reads may be issued to satisfy + the byte count (unless EOF is reached first). But for + interactive raw streams (XXX and for pipes?), at most one raw + read will be issued, and a short result does not imply that + EOF is imminent. + + Returns an empty bytes array on EOF. + + Raises BlockingIOError if the underlying raw stream has no + data at the moment. + """ + self._unsupported("read") + + def readinto(self, b): + """readinto(b: bytes) -> int. Read up to len(b) bytes into b. + + Like read(), this may issue multiple reads to the underlying + raw stream, unless the latter is 'interactive' (XXX or a + pipe?). + + Returns the number of bytes read (0 for EOF). + + Raises BlockingIOError if the underlying raw stream has no + data at the moment. + """ + # XXX This ought to work with anything that supports the buffer API + data = self.read(len(b)) + n = len(data) + try: + b[:n] = data + except TypeError as err: + import array + if not isinstance(b, array.array): + raise err + b[:n] = array.array('b', data) + return n + + def write(self, b): + """write(b: bytes) -> int. Write the given buffer to the IO stream. + + Returns the number of bytes written, which is never less than + len(b). + + Raises BlockingIOError if the buffer is full and the + underlying raw stream cannot accept more data at the moment. + """ + self._unsupported("write") + + +class _BufferedIOMixin(BufferedIOBase): + + """A mixin implementation of BufferedIOBase with an underlying raw stream. + + This passes most requests on to the underlying raw stream. It + does *not* provide implementations of read(), readinto() or + write(). + """ + + def __init__(self, raw): + self.raw = raw + + ### Positioning ### + + def seek(self, pos, whence=0): + return self.raw.seek(pos, whence) + + def tell(self): + return self.raw.tell() + + def truncate(self, pos=None): + # Flush the stream. We're mixing buffered I/O with lower-level I/O, + # and a flush may be necessary to synch both views of the current + # file state. + self.flush() + + if pos is None: + pos = self.tell() + return self.raw.truncate(pos) + + ### Flush and close ### + + def flush(self): + self.raw.flush() + + def close(self): + if not self.closed: + try: + self.flush() + except IOError: + pass # If flush() fails, just give up + self.raw.close() + + ### Inquiries ### + + def seekable(self): + return self.raw.seekable() + + def readable(self): + return self.raw.readable() + + def writable(self): + return self.raw.writable() + + @property + def closed(self): + return self.raw.closed + + ### Lower-level APIs ### + + def fileno(self): + return self.raw.fileno() + + def isatty(self): + return self.raw.isatty() + + +class BytesIO(BufferedIOBase): + + """Buffered I/O implementation using an in-memory bytes buffer.""" + + # XXX More docs + + def __init__(self, initial_bytes=None): + buf = bytearray() + if initial_bytes is not None: + buf += initial_bytes + self._buffer = buf + self._pos = 0 + + def getvalue(self): + return bytes(self._buffer) + + def read(self, n=None): + if n is None: + n = -1 + if n < 0: + n = len(self._buffer) + newpos = min(len(self._buffer), self._pos + n) + b = self._buffer[self._pos : newpos] + self._pos = newpos + return bytes(b) + + def read1(self, n): + return self.read(n) + + def write(self, b): + if self.closed: + raise ValueError("write to closed file") + if isinstance(b, unicode): + raise TypeError("can't write unicode to binary stream") + n = len(b) + newpos = self._pos + n + if newpos > len(self._buffer): + # Inserts null bytes between the current end of the file + # and the new write position. + padding = b'\x00' * (newpos - len(self._buffer) - n) + self._buffer[self._pos:newpos - n] = padding + self._buffer[self._pos:newpos] = b + self._pos = newpos + return n + + def seek(self, pos, whence=0): + try: + pos = pos.__index__() + except AttributeError as err: + raise TypeError("an integer is required") # from err + if whence == 0: + self._pos = max(0, pos) + elif whence == 1: + self._pos = max(0, self._pos + pos) + elif whence == 2: + self._pos = max(0, len(self._buffer) + pos) + else: + raise IOError("invalid whence value") + return self._pos + + def tell(self): + return self._pos + + def truncate(self, pos=None): + if pos is None: + pos = self._pos + del self._buffer[pos:] + return pos + + def readable(self): + return True + + def writable(self): + return True + + def seekable(self): + return True + + +class BufferedReader(_BufferedIOMixin): + + """Buffer for a readable sequential RawIO object.""" + + def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): + """Create a new buffered reader using the given readable raw IO object. + """ + raw._checkReadable() + _BufferedIOMixin.__init__(self, raw) + self._read_buf = b"" + self.buffer_size = buffer_size + + def read(self, n=None): + """Read n bytes. + + Returns exactly n bytes of data unless the underlying raw IO + stream reaches EOF or if the call would block in non-blocking + mode. If n is negative, read until EOF or until read() would + block. + """ + if n is None: + n = -1 + nodata_val = b"" + while n < 0 or len(self._read_buf) < n: + to_read = max(self.buffer_size, + n if n is not None else 2*len(self._read_buf)) + current = self.raw.read(to_read) + if current in (b"", None): + nodata_val = current + break + self._read_buf += current + if self._read_buf: + if n < 0: + n = len(self._read_buf) + out = self._read_buf[:n] + self._read_buf = self._read_buf[n:] + else: + out = nodata_val + return out + + def peek(self, n=0): + """Returns buffered bytes without advancing the position. + + The argument indicates a desired minimal number of bytes; we + do at most one raw read to satisfy it. We never return more + than self.buffer_size. + """ + want = min(n, self.buffer_size) + have = len(self._read_buf) + if have < want: + to_read = self.buffer_size - have + current = self.raw.read(to_read) + if current: + self._read_buf += current + return self._read_buf + + def read1(self, n): + """Reads up to n bytes, with at most one read() system call. + + Returns up to n bytes. If at least one byte is buffered, we + only return buffered bytes. Otherwise, we do one raw read. + """ + if n <= 0: + return b"" + self.peek(1) + return self.read(min(n, len(self._read_buf))) + + def tell(self): + return self.raw.tell() - len(self._read_buf) + + def seek(self, pos, whence=0): + if whence == 1: + pos -= len(self._read_buf) + pos = self.raw.seek(pos, whence) + self._read_buf = b"" + return pos + + +class BufferedWriter(_BufferedIOMixin): + + # XXX docstring + + def __init__(self, raw, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): + raw._checkWritable() + _BufferedIOMixin.__init__(self, raw) + self.buffer_size = buffer_size + self.max_buffer_size = (2*buffer_size + if max_buffer_size is None + else max_buffer_size) + self._write_buf = bytearray() + + def write(self, b): + if self.closed: + raise ValueError("write to closed file") + if isinstance(b, unicode): + raise TypeError("can't write unicode to binary stream") + # XXX we can implement some more tricks to try and avoid partial writes + if len(self._write_buf) > self.buffer_size: + # We're full, so let's pre-flush the buffer + try: + self.flush() + except BlockingIOError as e: + # We can't accept anything else. + # XXX Why not just let the exception pass through? + raise BlockingIOError(e.errno, e.strerror, 0) + before = len(self._write_buf) + self._write_buf.extend(b) + written = len(self._write_buf) - before + if len(self._write_buf) > self.buffer_size: + try: + self.flush() + except BlockingIOError as e: + if (len(self._write_buf) > self.max_buffer_size): + # We've hit max_buffer_size. We have to accept a partial + # write and cut back our buffer. + overage = len(self._write_buf) - self.max_buffer_size + self._write_buf = self._write_buf[:self.max_buffer_size] + raise BlockingIOError(e.errno, e.strerror, overage) + return written + + def flush(self): + if self.closed: + raise ValueError("flush of closed file") + written = 0 + try: + while self._write_buf: + n = self.raw.write(self._write_buf) + del self._write_buf[:n] + written += n + except BlockingIOError as e: + n = e.characters_written + del self._write_buf[:n] + written += n + raise BlockingIOError(e.errno, e.strerror, written) + + def tell(self): + return self.raw.tell() + len(self._write_buf) + + def seek(self, pos, whence=0): + self.flush() + return self.raw.seek(pos, whence) + + +class BufferedRWPair(BufferedIOBase): + + """A buffered reader and writer object together. + + A buffered reader object and buffered writer object put together + to form a sequential IO object that can read and write. + + This is typically used with a socket or two-way pipe. + + XXX The usefulness of this (compared to having two separate IO + objects) is questionable. + """ + + def __init__(self, reader, writer, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): + """Constructor. + + The arguments are two RawIO instances. + """ + reader._checkReadable() + writer._checkWritable() + self.reader = BufferedReader(reader, buffer_size) + self.writer = BufferedWriter(writer, buffer_size, max_buffer_size) + + def read(self, n=None): + if n is None: + n = -1 + return self.reader.read(n) + + def readinto(self, b): + return self.reader.readinto(b) + + def write(self, b): + return self.writer.write(b) + + def peek(self, n=0): + return self.reader.peek(n) + + def read1(self, n): + return self.reader.read1(n) + + def readable(self): + return self.reader.readable() + + def writable(self): + return self.writer.writable() + + def flush(self): + return self.writer.flush() + + def close(self): + self.writer.close() + self.reader.close() + + def isatty(self): + return self.reader.isatty() or self.writer.isatty() + + @property + def closed(self): + return self.writer.closed() + + +class BufferedRandom(BufferedWriter, BufferedReader): + + # XXX docstring + + def __init__(self, raw, + buffer_size=DEFAULT_BUFFER_SIZE, max_buffer_size=None): + raw._checkSeekable() + BufferedReader.__init__(self, raw, buffer_size) + BufferedWriter.__init__(self, raw, buffer_size, max_buffer_size) + + def seek(self, pos, whence=0): + self.flush() + # First do the raw seek, then empty the read buffer, so that + # if the raw seek fails, we don't lose buffered data forever. + pos = self.raw.seek(pos, whence) + self._read_buf = b"" + return pos + + def tell(self): + if (self._write_buf): + return self.raw.tell() + len(self._write_buf) + else: + return self.raw.tell() - len(self._read_buf) + + def read(self, n=None): + if n is None: + n = -1 + self.flush() + return BufferedReader.read(self, n) + + def readinto(self, b): + self.flush() + return BufferedReader.readinto(self, b) + + def peek(self, n=0): + self.flush() + return BufferedReader.peek(self, n) + + def read1(self, n): + self.flush() + return BufferedReader.read1(self, n) + + def write(self, b): + if self._read_buf: + self.raw.seek(-len(self._read_buf), 1) # Undo readahead + self._read_buf = b"" + return BufferedWriter.write(self, b) + + +class TextIOBase(IOBase): + + """Base class for text I/O. + + This class provides a character and line based interface to stream I/O. + + There is no readinto() method, as character strings are immutable. + """ + + def read(self, n = -1): + """read(n: int = -1) -> unicode. Read at most n characters from stream. + + Read from underlying buffer until we have n characters or we hit EOF. + If n is negative or omitted, read until EOF. + """ + self._unsupported("read") + + def write(self, s): + """write(s: unicode) -> int. Write string s to stream.""" + self._unsupported("write") + + def truncate(self, pos = None): + """truncate(pos: int = None) -> int. Truncate size to pos.""" + self.flush() + if pos is None: + pos = self.tell() + self.seek(pos) + return self.buffer.truncate() + + def readline(self): + """readline() -> unicode. Read until newline or EOF. + + Returns an empty string if EOF is hit immediately. + """ + self._unsupported("readline") + + @property + def encoding(self): + """Subclasses should override.""" + return None + + @property + def newlines(self): + """newlines -> None | unicode | tuple of unicode. Line endings translated + so far. + + Only line endings translated during reading are considered. + + Subclasses should override. + """ + return None + + +class IncrementalNewlineDecoder(codecs.IncrementalDecoder): + """Codec used when reading a file in universal newlines mode. + It wraps another incremental decoder, translating \\r\\n and \\r into \\n. + It also records the types of newlines encountered. + When used with translate=False, it ensures that the newline sequence is + returned in one piece. + """ + def __init__(self, decoder, translate, errors='strict'): + codecs.IncrementalDecoder.__init__(self, errors=errors) + self.buffer = b'' + self.translate = translate + self.decoder = decoder + self.seennl = 0 + + def decode(self, input, final=False): + # decode input (with the eventual \r from a previous pass) + if self.buffer: + input = self.buffer + input + + output = self.decoder.decode(input, final=final) + + # retain last \r even when not translating data: + # then readline() is sure to get \r\n in one pass + if output.endswith("\r") and not final: + output = output[:-1] + self.buffer = b'\r' + else: + self.buffer = b'' + + # Record which newlines are read + crlf = output.count('\r\n') + cr = output.count('\r') - crlf + lf = output.count('\n') - crlf + self.seennl |= (lf and self._LF) | (cr and self._CR) \ + | (crlf and self._CRLF) + + if self.translate: + if crlf: + output = output.replace("\r\n", "\n") + if cr: + output = output.replace("\r", "\n") + + return output + + def getstate(self): + buf, flag = self.decoder.getstate() + return buf + self.buffer, flag + + def setstate(self, state): + buf, flag = state + if buf.endswith(b'\r'): + self.buffer = b'\r' + buf = buf[:-1] + else: + self.buffer = b'' + self.decoder.setstate((buf, flag)) + + def reset(self): + self.seennl = 0 + self.buffer = b'' + self.decoder.reset() + + _LF = 1 + _CR = 2 + _CRLF = 4 + + @property + def newlines(self): + return (None, + "\n", + "\r", + ("\r", "\n"), + "\r\n", + ("\n", "\r\n"), + ("\r", "\r\n"), + ("\r", "\n", "\r\n") + )[self.seennl] + + +class TextIOWrapper(TextIOBase): + + """Buffered text stream. + + Character and line based layer over a BufferedIOBase object. + """ + + _CHUNK_SIZE = 128 + + def __init__(self, buffer, encoding=None, errors=None, newline=None, + line_buffering=False): + if newline not in (None, "", "\n", "\r", "\r\n"): + raise ValueError("illegal newline value: %r" % (newline,)) + if encoding is None: + try: + encoding = os.device_encoding(buffer.fileno()) + except (AttributeError, UnsupportedOperation): + pass + if encoding is None: + try: + import locale + except ImportError: + # Importing locale may fail if Python is being built + encoding = "ascii" + else: + encoding = locale.getpreferredencoding() + + if not isinstance(encoding, str): + raise ValueError("invalid encoding: %r" % encoding) + + if errors is None: + errors = "strict" + else: + if not isinstance(errors, str): + raise ValueError("invalid errors: %r" % errors) + + self.buffer = buffer + self._line_buffering = line_buffering + self._encoding = encoding + self._errors = errors + self._readuniversal = not newline + self._readtranslate = newline is None + self._readnl = newline + self._writetranslate = newline != '' + self._writenl = newline or os.linesep + self._encoder = None + self._decoder = None + self._decoded_chars = '' # buffer for text returned from decoder + self._decoded_chars_used = 0 # offset into _decoded_chars for read() + self._snapshot = None # info for reconstructing decoder state + self._seekable = self._telling = self.buffer.seekable() + + # self._snapshot is either None, or a tuple (dec_flags, next_input) + # where dec_flags is the second (integer) item of the decoder state + # and next_input is the chunk of input bytes that comes next after the + # snapshot point. We use this to reconstruct decoder states in tell(). + + # Naming convention: + # - "bytes_..." for integer variables that count input bytes + # - "chars_..." for integer variables that count decoded characters + + def __repr__(self): + return '<TIOW %x>' % id(self) + + @property + def encoding(self): + return self._encoding + + @property + def errors(self): + return self._errors + + @property + def line_buffering(self): + return self._line_buffering + + def seekable(self): + return self._seekable + + def flush(self): + self.buffer.flush() + self._telling = self._seekable + + def close(self): + try: + self.flush() + except: + pass # If flush() fails, just give up + self.buffer.close() + + @property + def closed(self): + return self.buffer.closed + + def fileno(self): + return self.buffer.fileno() + + def isatty(self): + return self.buffer.isatty() + + def write(self, s): + if self.closed: + raise ValueError("write to closed file") + if not isinstance(s, unicode): + raise TypeError("can't write %s to text stream" % + s.__class__.__name__) + length = len(s) + haslf = (self._writetranslate or self._line_buffering) and "\n" in s + if haslf and self._writetranslate and self._writenl != "\n": + s = s.replace("\n", self._writenl) + encoder = self._encoder or self._get_encoder() + # XXX What if we were just reading? + b = encoder.encode(s) + self.buffer.write(b) + if self._line_buffering and (haslf or "\r" in s): + self.flush() + self._snapshot = None + if self._decoder: + self._decoder.reset() + return length + + def _get_encoder(self): + make_encoder = codecs.getincrementalencoder(self._encoding) + self._encoder = make_encoder(self._errors) + return self._encoder + + def _get_decoder(self): + make_decoder = codecs.getincrementaldecoder(self._encoding) + decoder = make_decoder(self._errors) + if self._readuniversal: + decoder = IncrementalNewlineDecoder(decoder, self._readtranslate) + self._decoder = decoder + return decoder + + # The following three methods implement an ADT for _decoded_chars. + # Text returned from the decoder is buffered here until the client + # requests it by calling our read() or readline() method. + def _set_decoded_chars(self, chars): + """Set the _decoded_chars buffer.""" + self._decoded_chars = chars + self._decoded_chars_used = 0 + + def _get_decoded_chars(self, n=None): + """Advance into the _decoded_chars buffer.""" + offset = self._decoded_chars_used + if n is None: + chars = self._decoded_chars[offset:] + else: + chars = self._decoded_chars[offset:offset + n] + self._decoded_chars_used += len(chars) + return chars + + def _rewind_decoded_chars(self, n): + """Rewind the _decoded_chars buffer.""" + if self._decoded_chars_used < n: + raise AssertionError("rewind decoded_chars out of bounds") + self._decoded_chars_used -= n + + def _read_chunk(self): + """ + Read and decode the next chunk of data from the BufferedReader. + + The return value is True unless EOF was reached. The decoded string + is placed in self._decoded_chars (replacing its previous value). + The entire input chunk is sent to the decoder, though some of it + may remain buffered in the decoder, yet to be converted. + """ + + if self._decoder is None: + raise ValueError("no decoder") + + if self._telling: + # To prepare for tell(), we need to snapshot a point in the + # file where the decoder's input buffer is empty. + + dec_buffer, dec_flags = self._decoder.getstate() + # Given this, we know there was a valid snapshot point + # len(dec_buffer) bytes ago with decoder state (b'', dec_flags). + + # Read a chunk, decode it, and put the result in self._decoded_chars. + input_chunk = self.buffer.read1(self._CHUNK_SIZE) + eof = not input_chunk + self._set_decoded_chars(self._decoder.decode(input_chunk, eof)) + + if self._telling: + # At the snapshot point, len(dec_buffer) bytes before the read, + # the next input to be decoded is dec_buffer + input_chunk. + self._snapshot = (dec_flags, dec_buffer + input_chunk) + + return not eof + + def _pack_cookie(self, position, dec_flags=0, + bytes_to_feed=0, need_eof=0, chars_to_skip=0): + # The meaning of a tell() cookie is: seek to position, set the + # decoder flags to dec_flags, read bytes_to_feed bytes, feed them + # into the decoder with need_eof as the EOF flag, then skip + # chars_to_skip characters of the decoded result. For most simple + # decoders, tell() will often just give a byte offset in the file. + return (position | (dec_flags<<64) | (bytes_to_feed<<128) | + (chars_to_skip<<192) | bool(need_eof)<<256) + + def _unpack_cookie(self, bigint): + rest, position = divmod(bigint, 1<<64) + rest, dec_flags = divmod(rest, 1<<64) + rest, bytes_to_feed = divmod(rest, 1<<64) + need_eof, chars_to_skip = divmod(rest, 1<<64) + return position, dec_flags, bytes_to_feed, need_eof, chars_to_skip + + def tell(self): + if not self._seekable: + raise IOError("underlying stream is not seekable") + if not self._telling: + raise IOError("telling position disabled by next() call") + self.flush() + position = self.buffer.tell() + decoder = self._decoder + if decoder is None or self._snapshot is None: + if self._decoded_chars: + # This should never happen. + raise AssertionError("pending decoded text") + return position + + # Skip backward to the snapshot point (see _read_chunk). + dec_flags, next_input = self._snapshot + position -= len(next_input) + + # How many decoded characters have been used up since the snapshot? + chars_to_skip = self._decoded_chars_used + if chars_to_skip == 0: + # We haven't moved from the snapshot point. + return self._pack_cookie(position, dec_flags) + + # Starting from the snapshot position, we will walk the decoder + # forward until it gives us enough decoded characters. + saved_state = decoder.getstate() + try: + # Note our initial start point. + decoder.setstate((b'', dec_flags)) + start_pos = position + start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 + need_eof = 0 + + # Feed the decoder one byte at a time. As we go, note the + # nearest "safe start point" before the current location + # (a point where the decoder has nothing buffered, so seek() + # can safely start from there and advance to this location). + next_byte = bytearray(1) + for next_byte[0] in next_input: + bytes_fed += 1 + chars_decoded += len(decoder.decode(next_byte)) + dec_buffer, dec_flags = decoder.getstate() + if not dec_buffer and chars_decoded <= chars_to_skip: + # Decoder buffer is empty, so this is a safe start point. + start_pos += bytes_fed + chars_to_skip -= chars_decoded + start_flags, bytes_fed, chars_decoded = dec_flags, 0, 0 + if chars_decoded >= chars_to_skip: + break + else: + # We didn't get enough decoded data; signal EOF to get more. + chars_decoded += len(decoder.decode(b'', final=True)) + need_eof = 1 + if chars_decoded < chars_to_skip: + raise IOError("can't reconstruct logical file position") + + # The returned cookie corresponds to the last safe start point. + return self._pack_cookie( + start_pos, start_flags, bytes_fed, need_eof, chars_to_skip) + finally: + decoder.setstate(saved_state) + + def seek(self, cookie, whence=0): + if not self._seekable: + raise IOError("underlying stream is not seekable") + if whence == 1: # seek relative to current position + if cookie != 0: + raise IOError("can't do nonzero cur-relative seeks") + # Seeking to the current position should attempt to + # sync the underlying buffer with the current position. + whence = 0 + cookie = self.tell() + if whence == 2: # seek relative to end of file + if cookie != 0: + raise IOError("can't do nonzero end-relative seeks") + self.flush() + position = self.buffer.seek(0, 2) + self._set_decoded_chars('') + self._snapshot = None + if self._decoder: + self._decoder.reset() + return position + if whence != 0: + raise ValueError("invalid whence (%r, should be 0, 1 or 2)" % + (whence,)) + if cookie < 0: + raise ValueError("negative seek position %r" % (cookie,)) + self.flush() + + # The strategy of seek() is to go back to the safe start point + # and replay the effect of read(chars_to_skip) from there. + start_pos, dec_flags, bytes_to_feed, need_eof, chars_to_skip = \ + self._unpack_cookie(cookie) + + # Seek back to the safe start point. + self.buffer.seek(start_pos) + self._set_decoded_chars('') + self._snapshot = None + + # Restore the decoder to its state from the safe start point. + if self._decoder or dec_flags or chars_to_skip: + self._decoder = self._decoder or self._get_decoder() + self._decoder.setstate((b'', dec_flags)) + self._snapshot = (dec_flags, b'') + + if chars_to_skip: + # Just like _read_chunk, feed the decoder and save a snapshot. + input_chunk = self.buffer.read(bytes_to_feed) + self._set_decoded_chars( + self._decoder.decode(input_chunk, need_eof)) + self._snapshot = (dec_flags, input_chunk) + + # Skip chars_to_skip of the decoded characters. + if len(self._decoded_chars) < chars_to_skip: + raise IOError("can't restore logical file position") + self._decoded_chars_used = chars_to_skip + + return cookie + + def read(self, n=None): + if n is None: + n = -1 + decoder = self._decoder or self._get_decoder() + if n < 0: + # Read everything. + result = (self._get_decoded_chars() + + decoder.decode(self.buffer.read(), final=True)) + self._set_decoded_chars('') + self._snapshot = None + return result + else: + # Keep reading chunks until we have n characters to return. + eof = False + result = self._get_decoded_chars(n) + while len(result) < n and not eof: + eof = not self._read_chunk() + result += self._get_decoded_chars(n - len(result)) + return result + + def next(self): + self._telling = False + line = self.readline() + if not line: + self._snapshot = None + self._telling = self._seekable + raise StopIteration + return line + + def readline(self, limit=None): + if limit is None: + limit = -1 + + # Grab all the decoded text (we will rewind any extra bits later). + line = self._get_decoded_chars() + + start = 0 + decoder = self._decoder or self._get_decoder() + + pos = endpos = None + while True: + if self._readtranslate: + # Newlines are already translated, only search for \n + pos = line.find('\n', start) + if pos >= 0: + endpos = pos + 1 + break + else: + start = len(line) + + elif self._readuniversal: + # Universal newline search. Find any of \r, \r\n, \n + # The decoder ensures that \r\n are not split in two pieces + + # In C we'd look for these in parallel of course. + nlpos = line.find("\n", start) + crpos = line.find("\r", start) + if crpos == -1: + if nlpos == -1: + # Nothing found + start = len(line) + else: + # Found \n + endpos = nlpos + 1 + break + elif nlpos == -1: + # Found lone \r + endpos = crpos + 1 + break + elif nlpos < crpos: + # Found \n + endpos = nlpos + 1 + break + elif nlpos == crpos + 1: + # Found \r\n + endpos = crpos + 2 + break + else: + # Found \r + endpos = crpos + 1 + break + else: + # non-universal + pos = line.find(self._readnl) + if pos >= 0: + endpos = pos + len(self._readnl) + break + + if limit >= 0 and len(line) >= limit: + endpos = limit # reached length limit + break + + # No line ending seen yet - get more data + more_line = '' + while self._read_chunk(): + if self._decoded_chars: + break + if self._decoded_chars: + line += self._get_decoded_chars() + else: + # end of file + self._set_decoded_chars('') + self._snapshot = None + return line + + if limit >= 0 and endpos > limit: + endpos = limit # don't exceed limit + + # Rewind _decoded_chars to just after the line ending we found. + self._rewind_decoded_chars(len(line) - endpos) + return line[:endpos] + + @property + def newlines(self): + return self._decoder.newlines if self._decoder else None + +class StringIO(TextIOWrapper): + + # XXX This is really slow, but fully functional + + def __init__(self, initial_value="", encoding="utf-8", + errors="strict", newline="\n"): + super(StringIO, self).__init__(BytesIO(), + encoding=encoding, + errors=errors, + newline=newline) + if initial_value: + if not isinstance(initial_value, unicode): + initial_value = unicode(initial_value) + self.write(initial_value) + self.seek(0) + + def getvalue(self): + self.flush() + return self.buffer.getvalue().decode(self._encoding, self._errors) diff --git a/Lib/test/buffer_tests.py b/Lib/test/buffer_tests.py new file mode 100644 index 0000000..db27759 --- /dev/null +++ b/Lib/test/buffer_tests.py @@ -0,0 +1,206 @@ +# Tests that work for both bytes and buffer objects. +# See PEP 3137. + +import struct +import sys + +class MixinBytesBufferCommonTests(object): + """Tests that work for both bytes and buffer objects. + See PEP 3137. + """ + + def marshal(self, x): + """Convert x into the appropriate type for these tests.""" + raise RuntimeError('test class must provide a marshal method') + + def test_islower(self): + self.assertFalse(self.marshal(b'').islower()) + self.assert_(self.marshal(b'a').islower()) + self.assertFalse(self.marshal(b'A').islower()) + self.assertFalse(self.marshal(b'\n').islower()) + self.assert_(self.marshal(b'abc').islower()) + self.assertFalse(self.marshal(b'aBc').islower()) + self.assert_(self.marshal(b'abc\n').islower()) + self.assertRaises(TypeError, self.marshal(b'abc').islower, 42) + + def test_isupper(self): + self.assertFalse(self.marshal(b'').isupper()) + self.assertFalse(self.marshal(b'a').isupper()) + self.assert_(self.marshal(b'A').isupper()) + self.assertFalse(self.marshal(b'\n').isupper()) + self.assert_(self.marshal(b'ABC').isupper()) + self.assertFalse(self.marshal(b'AbC').isupper()) + self.assert_(self.marshal(b'ABC\n').isupper()) + self.assertRaises(TypeError, self.marshal(b'abc').isupper, 42) + + def test_istitle(self): + self.assertFalse(self.marshal(b'').istitle()) + self.assertFalse(self.marshal(b'a').istitle()) + self.assert_(self.marshal(b'A').istitle()) + self.assertFalse(self.marshal(b'\n').istitle()) + self.assert_(self.marshal(b'A Titlecased Line').istitle()) + self.assert_(self.marshal(b'A\nTitlecased Line').istitle()) + self.assert_(self.marshal(b'A Titlecased, Line').istitle()) + self.assertFalse(self.marshal(b'Not a capitalized String').istitle()) + self.assertFalse(self.marshal(b'Not\ta Titlecase String').istitle()) + self.assertFalse(self.marshal(b'Not--a Titlecase String').istitle()) + self.assertFalse(self.marshal(b'NOT').istitle()) + self.assertRaises(TypeError, self.marshal(b'abc').istitle, 42) + + def test_isspace(self): + self.assertFalse(self.marshal(b'').isspace()) + self.assertFalse(self.marshal(b'a').isspace()) + self.assert_(self.marshal(b' ').isspace()) + self.assert_(self.marshal(b'\t').isspace()) + self.assert_(self.marshal(b'\r').isspace()) + self.assert_(self.marshal(b'\n').isspace()) + self.assert_(self.marshal(b' \t\r\n').isspace()) + self.assertFalse(self.marshal(b' \t\r\na').isspace()) + self.assertRaises(TypeError, self.marshal(b'abc').isspace, 42) + + def test_isalpha(self): + self.assertFalse(self.marshal(b'').isalpha()) + self.assert_(self.marshal(b'a').isalpha()) + self.assert_(self.marshal(b'A').isalpha()) + self.assertFalse(self.marshal(b'\n').isalpha()) + self.assert_(self.marshal(b'abc').isalpha()) + self.assertFalse(self.marshal(b'aBc123').isalpha()) + self.assertFalse(self.marshal(b'abc\n').isalpha()) + self.assertRaises(TypeError, self.marshal(b'abc').isalpha, 42) + + def test_isalnum(self): + self.assertFalse(self.marshal(b'').isalnum()) + self.assert_(self.marshal(b'a').isalnum()) + self.assert_(self.marshal(b'A').isalnum()) + self.assertFalse(self.marshal(b'\n').isalnum()) + self.assert_(self.marshal(b'123abc456').isalnum()) + self.assert_(self.marshal(b'a1b3c').isalnum()) + self.assertFalse(self.marshal(b'aBc000 ').isalnum()) + self.assertFalse(self.marshal(b'abc\n').isalnum()) + self.assertRaises(TypeError, self.marshal(b'abc').isalnum, 42) + + def test_isdigit(self): + self.assertFalse(self.marshal(b'').isdigit()) + self.assertFalse(self.marshal(b'a').isdigit()) + self.assert_(self.marshal(b'0').isdigit()) + self.assert_(self.marshal(b'0123456789').isdigit()) + self.assertFalse(self.marshal(b'0123456789a').isdigit()) + + self.assertRaises(TypeError, self.marshal(b'abc').isdigit, 42) + + def test_lower(self): + self.assertEqual(b'hello', self.marshal(b'HeLLo').lower()) + self.assertEqual(b'hello', self.marshal(b'hello').lower()) + self.assertRaises(TypeError, self.marshal(b'hello').lower, 42) + + def test_upper(self): + self.assertEqual(b'HELLO', self.marshal(b'HeLLo').upper()) + self.assertEqual(b'HELLO', self.marshal(b'HELLO').upper()) + self.assertRaises(TypeError, self.marshal(b'hello').upper, 42) + + def test_capitalize(self): + self.assertEqual(b' hello ', self.marshal(b' hello ').capitalize()) + self.assertEqual(b'Hello ', self.marshal(b'Hello ').capitalize()) + self.assertEqual(b'Hello ', self.marshal(b'hello ').capitalize()) + self.assertEqual(b'Aaaa', self.marshal(b'aaaa').capitalize()) + self.assertEqual(b'Aaaa', self.marshal(b'AaAa').capitalize()) + + self.assertRaises(TypeError, self.marshal(b'hello').capitalize, 42) + + def test_ljust(self): + self.assertEqual(b'abc ', self.marshal(b'abc').ljust(10)) + self.assertEqual(b'abc ', self.marshal(b'abc').ljust(6)) + self.assertEqual(b'abc', self.marshal(b'abc').ljust(3)) + self.assertEqual(b'abc', self.marshal(b'abc').ljust(2)) + self.assertEqual(b'abc*******', self.marshal(b'abc').ljust(10, '*')) + self.assertRaises(TypeError, self.marshal(b'abc').ljust) + + def test_rjust(self): + self.assertEqual(b' abc', self.marshal(b'abc').rjust(10)) + self.assertEqual(b' abc', self.marshal(b'abc').rjust(6)) + self.assertEqual(b'abc', self.marshal(b'abc').rjust(3)) + self.assertEqual(b'abc', self.marshal(b'abc').rjust(2)) + self.assertEqual(b'*******abc', self.marshal(b'abc').rjust(10, '*')) + self.assertRaises(TypeError, self.marshal(b'abc').rjust) + + def test_center(self): + self.assertEqual(b' abc ', self.marshal(b'abc').center(10)) + self.assertEqual(b' abc ', self.marshal(b'abc').center(6)) + self.assertEqual(b'abc', self.marshal(b'abc').center(3)) + self.assertEqual(b'abc', self.marshal(b'abc').center(2)) + self.assertEqual(b'***abc****', self.marshal(b'abc').center(10, '*')) + self.assertRaises(TypeError, self.marshal(b'abc').center) + + def test_swapcase(self): + self.assertEqual(b'hEllO CoMPuTErS', + self.marshal(b'HeLLo cOmpUteRs').swapcase()) + + self.assertRaises(TypeError, self.marshal(b'hello').swapcase, 42) + + def test_zfill(self): + self.assertEqual(b'123', self.marshal(b'123').zfill(2)) + self.assertEqual(b'123', self.marshal(b'123').zfill(3)) + self.assertEqual(b'0123', self.marshal(b'123').zfill(4)) + self.assertEqual(b'+123', self.marshal(b'+123').zfill(3)) + self.assertEqual(b'+123', self.marshal(b'+123').zfill(4)) + self.assertEqual(b'+0123', self.marshal(b'+123').zfill(5)) + self.assertEqual(b'-123', self.marshal(b'-123').zfill(3)) + self.assertEqual(b'-123', self.marshal(b'-123').zfill(4)) + self.assertEqual(b'-0123', self.marshal(b'-123').zfill(5)) + self.assertEqual(b'000', self.marshal(b'').zfill(3)) + self.assertEqual(b'34', self.marshal(b'34').zfill(1)) + self.assertEqual(b'0034', self.marshal(b'34').zfill(4)) + + self.assertRaises(TypeError, self.marshal(b'123').zfill) + + def test_expandtabs(self): + self.assertEqual(b'abc\rab def\ng hi', + self.marshal(b'abc\rab\tdef\ng\thi').expandtabs()) + self.assertEqual(b'abc\rab def\ng hi', + self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8)) + self.assertEqual(b'abc\rab def\ng hi', + self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(4)) + self.assertEqual(b'abc\r\nab def\ng hi', + self.marshal(b'abc\r\nab\tdef\ng\thi').expandtabs(4)) + self.assertEqual(b'abc\rab def\ng hi', + self.marshal(b'abc\rab\tdef\ng\thi').expandtabs()) + self.assertEqual(b'abc\rab def\ng hi', + self.marshal(b'abc\rab\tdef\ng\thi').expandtabs(8)) + self.assertEqual(b'abc\r\nab\r\ndef\ng\r\nhi', + self.marshal(b'abc\r\nab\r\ndef\ng\r\nhi').expandtabs(4)) + self.assertEqual(b' a\n b', self.marshal(b' \ta\n\tb').expandtabs(1)) + + self.assertRaises(TypeError, self.marshal(b'hello').expandtabs, 42, 42) + # This test is only valid when sizeof(int) == sizeof(void*) == 4. + if sys.maxint < (1 << 32) and struct.calcsize('P') == 4: + self.assertRaises(OverflowError, + self.marshal(b'\ta\n\tb').expandtabs, sys.maxint) + + def test_title(self): + self.assertEqual(b' Hello ', self.marshal(b' hello ').title()) + self.assertEqual(b'Hello ', self.marshal(b'hello ').title()) + self.assertEqual(b'Hello ', self.marshal(b'Hello ').title()) + self.assertEqual(b'Format This As Title String', + self.marshal(b'fOrMaT thIs aS titLe String').title()) + self.assertEqual(b'Format,This-As*Title;String', + self.marshal(b'fOrMaT,thIs-aS*titLe;String').title()) + self.assertEqual(b'Getint', self.marshal(b'getInt').title()) + self.assertRaises(TypeError, self.marshal(b'hello').title, 42) + + def test_splitlines(self): + self.assertEqual([b'abc', b'def', b'', b'ghi'], + self.marshal(b'abc\ndef\n\rghi').splitlines()) + self.assertEqual([b'abc', b'def', b'', b'ghi'], + self.marshal(b'abc\ndef\n\r\nghi').splitlines()) + self.assertEqual([b'abc', b'def', b'ghi'], + self.marshal(b'abc\ndef\r\nghi').splitlines()) + self.assertEqual([b'abc', b'def', b'ghi'], + self.marshal(b'abc\ndef\r\nghi\n').splitlines()) + self.assertEqual([b'abc', b'def', b'ghi', b''], + self.marshal(b'abc\ndef\r\nghi\n\r').splitlines()) + self.assertEqual([b'', b'abc', b'def', b'ghi', b''], + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines()) + self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'], + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(1)) + + self.assertRaises(TypeError, self.marshal(b'abc').splitlines, 42, 42) diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 1be5ce0..064e8ca 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -46,3 +46,4 @@ BaseException +-- FutureWarning +-- ImportWarning +-- UnicodeWarning + +-- BytesWarning diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 4e2b37e..3590b8e 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -486,8 +486,9 @@ class CommonTest(unittest.TestCase): 'lstrip', unicode('xyz', 'ascii')) self.checkequal(unicode('xyzzyhello', 'ascii'), 'xyzzyhelloxyzzy', 'rstrip', unicode('xyz', 'ascii')) - self.checkequal(unicode('hello', 'ascii'), 'hello', - 'strip', unicode('xyz', 'ascii')) + # XXX + #self.checkequal(unicode('hello', 'ascii'), 'hello', + # 'strip', unicode('xyz', 'ascii')) self.checkraises(TypeError, 'hello', 'strip', 42, 42) self.checkraises(TypeError, 'hello', 'lstrip', 42, 42) @@ -727,6 +728,9 @@ class CommonTest(unittest.TestCase): self.checkraises(TypeError, '123', 'zfill') +# XXX alias for py3k forward compatibility +BaseTest = CommonTest + class MixinStrUnicodeUserStringTest: # additional tests that only work for # stringlike objects, i.e. str, unicode, UserString diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py new file mode 100644 index 0000000..d2d5b27 --- /dev/null +++ b/Lib/test/test_bytes.py @@ -0,0 +1,982 @@ +"""Unit tests for the bytes and bytearray types. + +XXX This is a mess. Common tests should be moved to buffer_tests.py, +which itself ought to be unified with string_tests.py (and the latter +should be modernized). +""" + +import os +import re +import sys +import copy +import pickle +import tempfile +import unittest +import warnings +import test.test_support +import test.string_tests +import test.buffer_tests + + +class BaseBytesTest(unittest.TestCase): + + def setUp(self): + self.warning_filters = warnings.filters[:] + + def tearDown(self): + warnings.filters = self.warning_filters + + def test_basics(self): + b = self.type2test() + self.assertEqual(type(b), self.type2test) + self.assertEqual(b.__class__, self.type2test) + + def test_empty_sequence(self): + b = self.type2test() + self.assertEqual(len(b), 0) + self.assertRaises(IndexError, lambda: b[0]) + self.assertRaises(IndexError, lambda: b[1]) + self.assertRaises(IndexError, lambda: b[sys.maxint]) + self.assertRaises(IndexError, lambda: b[sys.maxint+1]) + self.assertRaises(IndexError, lambda: b[10**100]) + self.assertRaises(IndexError, lambda: b[-1]) + self.assertRaises(IndexError, lambda: b[-2]) + self.assertRaises(IndexError, lambda: b[-sys.maxint]) + self.assertRaises(IndexError, lambda: b[-sys.maxint-1]) + self.assertRaises(IndexError, lambda: b[-sys.maxint-2]) + self.assertRaises(IndexError, lambda: b[-10**100]) + + def test_from_list(self): + ints = list(range(256)) + b = self.type2test(i for i in ints) + self.assertEqual(len(b), 256) + self.assertEqual(list(b), ints) + + def test_from_index(self): + class C: + def __init__(self, i=0): + self.i = i + def __index__(self): + return self.i + b = self.type2test([C(), C(1), C(254), C(255)]) + self.assertEqual(list(b), [0, 1, 254, 255]) + self.assertRaises(ValueError, bytearray, [C(-1)]) + self.assertRaises(ValueError, bytearray, [C(256)]) + + def test_from_ssize(self): + self.assertEqual(bytearray(0), b'') + self.assertEqual(bytearray(1), b'\x00') + self.assertEqual(bytearray(5), b'\x00\x00\x00\x00\x00') + self.assertRaises(ValueError, bytearray, -1) + + self.assertEqual(bytearray('0', 'ascii'), b'0') + self.assertEqual(bytearray(b'0'), b'0') + + def test_constructor_type_errors(self): + self.assertRaises(TypeError, self.type2test, 0.0) + class C: + pass + self.assertRaises(TypeError, self.type2test, ["0"]) + self.assertRaises(TypeError, self.type2test, [0.0]) + self.assertRaises(TypeError, self.type2test, [None]) + self.assertRaises(TypeError, self.type2test, [C()]) + + def test_constructor_value_errors(self): + self.assertRaises(ValueError, self.type2test, [-1]) + self.assertRaises(ValueError, self.type2test, [-sys.maxint]) + self.assertRaises(ValueError, self.type2test, [-sys.maxint-1]) + self.assertRaises(ValueError, self.type2test, [-sys.maxint-2]) + self.assertRaises(ValueError, self.type2test, [-10**100]) + self.assertRaises(ValueError, self.type2test, [256]) + self.assertRaises(ValueError, self.type2test, [257]) + self.assertRaises(ValueError, self.type2test, [sys.maxint]) + self.assertRaises(ValueError, self.type2test, [sys.maxint+1]) + self.assertRaises(ValueError, self.type2test, [10**100]) + + def test_compare(self): + b1 = self.type2test([1, 2, 3]) + b2 = self.type2test([1, 2, 3]) + b3 = self.type2test([1, 3]) + + self.assertEqual(b1, b2) + self.failUnless(b2 != b3) + self.failUnless(b1 <= b2) + self.failUnless(b1 <= b3) + self.failUnless(b1 < b3) + self.failUnless(b1 >= b2) + self.failUnless(b3 >= b2) + self.failUnless(b3 > b2) + + self.failIf(b1 != b2) + self.failIf(b2 == b3) + self.failIf(b1 > b2) + self.failIf(b1 > b3) + self.failIf(b1 >= b3) + self.failIf(b1 < b2) + self.failIf(b3 < b2) + self.failIf(b3 <= b2) + + def test_compare_to_str(self): + warnings.simplefilter('ignore', BytesWarning) + # Byte comparisons with unicode should always fail! + # Test this for all expected byte orders and Unicode character sizes + self.assertEqual(self.type2test(b"\0a\0b\0c") == u"abc", False) + self.assertEqual(self.type2test(b"\0\0\0a\0\0\0b\0\0\0c") == u"abc", False) + self.assertEqual(self.type2test(b"a\0b\0c\0") == u"abc", False) + self.assertEqual(self.type2test(b"a\0\0\0b\0\0\0c\0\0\0") == u"abc", False) + self.assertEqual(self.type2test() == unicode(), False) + self.assertEqual(self.type2test() != unicode(), True) + + def test_reversed(self): + input = list(map(ord, "Hello")) + b = self.type2test(input) + output = list(reversed(b)) + input.reverse() + self.assertEqual(output, input) + + def test_getslice(self): + def by(s): + return self.type2test(map(ord, s)) + b = by("Hello, world") + + self.assertEqual(b[:5], by("Hello")) + self.assertEqual(b[1:5], by("ello")) + self.assertEqual(b[5:7], by(", ")) + self.assertEqual(b[7:], by("world")) + self.assertEqual(b[7:12], by("world")) + self.assertEqual(b[7:100], by("world")) + + self.assertEqual(b[:-7], by("Hello")) + self.assertEqual(b[-11:-7], by("ello")) + self.assertEqual(b[-7:-5], by(", ")) + self.assertEqual(b[-5:], by("world")) + self.assertEqual(b[-5:12], by("world")) + self.assertEqual(b[-5:100], by("world")) + self.assertEqual(b[-100:5], by("Hello")) + + def test_extended_getslice(self): + # Test extended slicing by comparing with list slicing. + L = list(range(255)) + b = self.type2test(L) + indices = (0, None, 1, 3, 19, 100, -1, -2, -31, -100) + for start in indices: + for stop in indices: + # Skip step 0 (invalid) + for step in indices[1:]: + self.assertEqual(b[start:stop:step], self.type2test(L[start:stop:step])) + + def test_encoding(self): + sample = u"Hello world\n\u1234\u5678\u9abc\udef0" + for enc in ("utf8", "utf16"): + b = self.type2test(sample, enc) + self.assertEqual(b, self.type2test(sample.encode(enc))) + self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin1") + b = self.type2test(sample, "latin1", "ignore") + self.assertEqual(b, self.type2test(sample[:-4], "utf-8")) + + def test_decode(self): + sample = u"Hello world\n\u1234\u5678\u9abc\def0\def0" + for enc in ("utf8", "utf16"): + b = self.type2test(sample, enc) + self.assertEqual(b.decode(enc), sample) + sample = u"Hello world\n\x80\x81\xfe\xff" + b = self.type2test(sample, "latin1") + self.assertRaises(UnicodeDecodeError, b.decode, "utf8") + self.assertEqual(b.decode("utf8", "ignore"), "Hello world\n") + + def test_from_int(self): + b = self.type2test(0) + self.assertEqual(b, self.type2test()) + b = self.type2test(10) + self.assertEqual(b, self.type2test([0]*10)) + b = self.type2test(10000) + self.assertEqual(b, self.type2test([0]*10000)) + + def test_concat(self): + b1 = self.type2test(b"abc") + b2 = self.type2test(b"def") + self.assertEqual(b1 + b2, b"abcdef") + self.assertEqual(b1 + bytes(b"def"), b"abcdef") + self.assertEqual(bytes(b"def") + b1, b"defabc") + self.assertRaises(TypeError, lambda: b1 + u"def") + self.assertRaises(TypeError, lambda: u"abc" + b2) + + def test_repeat(self): + for b in b"abc", self.type2test(b"abc"): + self.assertEqual(b * 3, b"abcabcabc") + self.assertEqual(b * 0, b"") + self.assertEqual(b * -1, b"") + self.assertRaises(TypeError, lambda: b * 3.14) + self.assertRaises(TypeError, lambda: 3.14 * b) + # XXX Shouldn't bytes and bytearray agree on what to raise? + self.assertRaises((OverflowError, MemoryError), + lambda: b * sys.maxint) + + def test_repeat_1char(self): + self.assertEqual(self.type2test(b'x')*100, self.type2test([ord('x')]*100)) + + def test_contains(self): + b = self.type2test(b"abc") + self.failUnless(ord('a') in b) + self.failUnless(int(ord('a')) in b) + self.failIf(200 in b) + self.failIf(200 in b) + self.assertRaises(ValueError, lambda: 300 in b) + self.assertRaises(ValueError, lambda: -1 in b) + self.assertRaises(TypeError, lambda: None in b) + self.assertRaises(TypeError, lambda: float(ord('a')) in b) + self.assertRaises(TypeError, lambda: u"a" in b) + for f in bytes, bytearray: + self.failUnless(f(b"") in b) + self.failUnless(f(b"a") in b) + self.failUnless(f(b"b") in b) + self.failUnless(f(b"c") in b) + self.failUnless(f(b"ab") in b) + self.failUnless(f(b"bc") in b) + self.failUnless(f(b"abc") in b) + self.failIf(f(b"ac") in b) + self.failIf(f(b"d") in b) + self.failIf(f(b"dab") in b) + self.failIf(f(b"abd") in b) + + def test_fromhex(self): + self.assertRaises(TypeError, self.type2test.fromhex) + self.assertRaises(TypeError, self.type2test.fromhex, 1) + self.assertEquals(self.type2test.fromhex(u''), self.type2test()) + b = bytearray([0x1a, 0x2b, 0x30]) + self.assertEquals(self.type2test.fromhex(u'1a2B30'), b) + self.assertEquals(self.type2test.fromhex(u' 1A 2B 30 '), b) + self.assertEquals(self.type2test.fromhex(u'0000'), b'\0\0') + self.assertRaises(TypeError, self.type2test.fromhex, b'1B') + self.assertRaises(ValueError, self.type2test.fromhex, u'a') + self.assertRaises(ValueError, self.type2test.fromhex, u'rt') + self.assertRaises(ValueError, self.type2test.fromhex, u'1a b cd') + self.assertRaises(ValueError, self.type2test.fromhex, u'\x00') + self.assertRaises(ValueError, self.type2test.fromhex, u'12 \x00 34') + + def test_join(self): + self.assertEqual(self.type2test(b"").join([]), b"") + self.assertEqual(self.type2test(b"").join([b""]), b"") + for lst in [[b"abc"], [b"a", b"bc"], [b"ab", b"c"], [b"a", b"b", b"c"]]: + lst = list(map(self.type2test, lst)) + self.assertEqual(self.type2test(b"").join(lst), b"abc") + self.assertEqual(self.type2test(b"").join(tuple(lst)), b"abc") + self.assertEqual(self.type2test(b"").join(iter(lst)), b"abc") + self.assertEqual(self.type2test(b".").join([b"ab", b"cd"]), b"ab.cd") + # XXX more... + + def test_index(self): + b = self.type2test(b'parrot') + self.assertEqual(b.index('p'), 0) + self.assertEqual(b.index('rr'), 2) + self.assertEqual(b.index('t'), 5) + self.assertRaises(ValueError, lambda: b.index('w')) + + def test_count(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.count(b'i'), 4) + self.assertEqual(b.count(b'ss'), 2) + self.assertEqual(b.count(b'w'), 0) + + def test_startswith(self): + b = self.type2test(b'hello') + self.assertFalse(self.type2test().startswith(b"anything")) + self.assertTrue(b.startswith(b"hello")) + self.assertTrue(b.startswith(b"hel")) + self.assertTrue(b.startswith(b"h")) + self.assertFalse(b.startswith(b"hellow")) + self.assertFalse(b.startswith(b"ha")) + + def test_endswith(self): + b = self.type2test(b'hello') + self.assertFalse(bytearray().endswith(b"anything")) + self.assertTrue(b.endswith(b"hello")) + self.assertTrue(b.endswith(b"llo")) + self.assertTrue(b.endswith(b"o")) + self.assertFalse(b.endswith(b"whello")) + self.assertFalse(b.endswith(b"no")) + + def test_find(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.find(b'ss'), 2) + self.assertEqual(b.find(b'ss', 3), 5) + self.assertEqual(b.find(b'ss', 1, 7), 2) + self.assertEqual(b.find(b'ss', 1, 3), -1) + self.assertEqual(b.find(b'w'), -1) + self.assertEqual(b.find(b'mississippian'), -1) + + def test_rfind(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.rfind(b'ss'), 5) + self.assertEqual(b.rfind(b'ss', 3), 5) + self.assertEqual(b.rfind(b'ss', 0, 6), 2) + self.assertEqual(b.rfind(b'w'), -1) + self.assertEqual(b.rfind(b'mississippian'), -1) + + def test_index(self): + b = self.type2test(b'world') + self.assertEqual(b.index(b'w'), 0) + self.assertEqual(b.index(b'orl'), 1) + self.assertRaises(ValueError, b.index, b'worm') + self.assertRaises(ValueError, b.index, b'ldo') + + def test_rindex(self): + # XXX could be more rigorous + b = self.type2test(b'world') + self.assertEqual(b.rindex(b'w'), 0) + self.assertEqual(b.rindex(b'orl'), 1) + self.assertRaises(ValueError, b.rindex, b'worm') + self.assertRaises(ValueError, b.rindex, b'ldo') + + def test_replace(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.replace(b'i', b'a'), b'massassappa') + self.assertEqual(b.replace(b'ss', b'x'), b'mixixippi') + + def test_split(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.split(b'i'), [b'm', b'ss', b'ss', b'pp', b'']) + self.assertEqual(b.split(b'ss'), [b'mi', b'i', b'ippi']) + self.assertEqual(b.split(b'w'), [b]) + + def test_split_whitespace(self): + for b in (b' arf barf ', b'arf\tbarf', b'arf\nbarf', b'arf\rbarf', + b'arf\fbarf', b'arf\vbarf'): + b = self.type2test(b) + self.assertEqual(b.split(), [b'arf', b'barf']) + self.assertEqual(b.split(None), [b'arf', b'barf']) + self.assertEqual(b.split(None, 2), [b'arf', b'barf']) + for b in (b'a\x1Cb', b'a\x1Db', b'a\x1Eb', b'a\x1Fb'): + b = self.type2test(b) + self.assertEqual(b.split(), [b]) + self.assertEqual(self.type2test(b' a bb c ').split(None, 0), [b'a bb c ']) + self.assertEqual(self.type2test(b' a bb c ').split(None, 1), [b'a', b'bb c ']) + self.assertEqual(self.type2test(b' a bb c ').split(None, 2), [b'a', b'bb', b'c ']) + self.assertEqual(self.type2test(b' a bb c ').split(None, 3), [b'a', b'bb', b'c']) + + def test_split_string_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').split, u' ') + + def test_rsplit(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.rsplit(b'i'), [b'm', b'ss', b'ss', b'pp', b'']) + self.assertEqual(b.rsplit(b'ss'), [b'mi', b'i', b'ippi']) + self.assertEqual(b.rsplit(b'w'), [b]) + + def test_rsplit_whitespace(self): + for b in (b' arf barf ', b'arf\tbarf', b'arf\nbarf', b'arf\rbarf', + b'arf\fbarf', b'arf\vbarf'): + b = self.type2test(b) + self.assertEqual(b.rsplit(), [b'arf', b'barf']) + self.assertEqual(b.rsplit(None), [b'arf', b'barf']) + self.assertEqual(b.rsplit(None, 2), [b'arf', b'barf']) + self.assertEqual(self.type2test(b' a bb c ').rsplit(None, 0), [b' a bb c']) + self.assertEqual(self.type2test(b' a bb c ').rsplit(None, 1), [b' a bb', b'c']) + self.assertEqual(self.type2test(b' a bb c ').rsplit(None, 2), [b' a', b'bb', b'c']) + self.assertEqual(self.type2test(b' a bb c ').rsplit(None, 3), [b'a', b'bb', b'c']) + + def test_rsplit_string_error(self): + self.assertRaises(TypeError, self.type2test(b'a b').rsplit, u' ') + + def test_rsplit_unicodewhitespace(self): + b = self.type2test(b"\x09\x0A\x0B\x0C\x0D\x1C\x1D\x1E\x1F") + self.assertEqual(b.split(), [b'\x1c\x1d\x1e\x1f']) + self.assertEqual(b.rsplit(), [b'\x1c\x1d\x1e\x1f']) + + def test_partition(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.partition(b'ss'), (b'mi', b'ss', b'issippi')) + self.assertEqual(b.rpartition(b'w'), (b'', b'', b'mississippi')) + + def test_rpartition(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.rpartition(b'ss'), (b'missi', b'ss', b'ippi')) + self.assertEqual(b.rpartition(b'i'), (b'mississipp', b'i', b'')) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL): + for b in b"", b"a", b"abc", b"\xffab\x80", b"\0\0\377\0\0": + b = self.type2test(b) + ps = pickle.dumps(b, proto) + q = pickle.loads(ps) + self.assertEqual(b, q) + + def test_strip(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.strip(b'i'), b'mississipp') + self.assertEqual(b.strip(b'm'), b'ississippi') + self.assertEqual(b.strip(b'pi'), b'mississ') + self.assertEqual(b.strip(b'im'), b'ssissipp') + self.assertEqual(b.strip(b'pim'), b'ssiss') + self.assertEqual(b.strip(b), b'') + + def test_lstrip(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.lstrip(b'i'), b'mississippi') + self.assertEqual(b.lstrip(b'm'), b'ississippi') + self.assertEqual(b.lstrip(b'pi'), b'mississippi') + self.assertEqual(b.lstrip(b'im'), b'ssissippi') + self.assertEqual(b.lstrip(b'pim'), b'ssissippi') + + def test_rstrip(self): + b = self.type2test(b'mississippi') + self.assertEqual(b.rstrip(b'i'), b'mississipp') + self.assertEqual(b.rstrip(b'm'), b'mississippi') + self.assertEqual(b.rstrip(b'pi'), b'mississ') + self.assertEqual(b.rstrip(b'im'), b'mississipp') + self.assertEqual(b.rstrip(b'pim'), b'mississ') + + def test_strip_whitespace(self): + b = self.type2test(b' \t\n\r\f\vabc \t\n\r\f\v') + self.assertEqual(b.strip(), b'abc') + self.assertEqual(b.lstrip(), b'abc \t\n\r\f\v') + self.assertEqual(b.rstrip(), b' \t\n\r\f\vabc') + + def XXXtest_strip_bytearray(self): + # XXX memoryview not available + self.assertEqual(self.type2test(b'abc').strip(memoryview(b'ac')), b'b') + self.assertEqual(self.type2test(b'abc').lstrip(memoryview(b'ac')), b'bc') + self.assertEqual(self.type2test(b'abc').rstrip(memoryview(b'ac')), b'ab') + + def test_strip_string_error(self): + self.assertRaises(TypeError, self.type2test(b'abc').strip, u'b') + self.assertRaises(TypeError, self.type2test(b'abc').lstrip, u'b') + self.assertRaises(TypeError, self.type2test(b'abc').rstrip, u'b') + + def test_ord(self): + b = self.type2test(b'\0A\x7f\x80\xff') + self.assertEqual([ord(b[i:i+1]) for i in range(len(b))], + [0, 65, 127, 128, 255]) + + +class ByteArrayTest(BaseBytesTest): + type2test = bytearray + + def test_nohash(self): + self.assertRaises(TypeError, hash, bytearray()) + + def test_bytearray_api(self): + short_sample = b"Hello world\n" + sample = short_sample + b"\0"*(20 - len(short_sample)) + tfn = tempfile.mktemp() + try: + # Prepare + with open(tfn, "wb") as f: + f.write(short_sample) + # Test readinto + with open(tfn, "rb") as f: + b = bytearray(20) + n = f.readinto(b) + self.assertEqual(n, len(short_sample)) + # Python 2.x + b_sample = (ord(s) for s in sample) + self.assertEqual(list(b), list(b_sample)) + # Test writing in binary mode + with open(tfn, "wb") as f: + f.write(b) + with open(tfn, "rb") as f: + self.assertEqual(f.read(), sample) + # Text mode is ambiguous; don't test + finally: + try: + os.remove(tfn) + except os.error: + pass + + def test_reverse(self): + b = bytearray(b'hello') + self.assertEqual(b.reverse(), None) + self.assertEqual(b, b'olleh') + b = bytearray(b'hello1') # test even number of items + b.reverse() + self.assertEqual(b, b'1olleh') + b = bytearray() + b.reverse() + self.assertFalse(b) + + def test_regexps(self): + def by(s): + return bytearray(map(ord, s)) + b = by("Hello, world") + self.assertEqual(re.findall(r"\w+", b), [by("Hello"), by("world")]) + + def test_setitem(self): + b = bytearray([1, 2, 3]) + b[1] = 100 + self.assertEqual(b, bytearray([1, 100, 3])) + b[-1] = 200 + self.assertEqual(b, bytearray([1, 100, 200])) + class C: + def __init__(self, i=0): + self.i = i + def __index__(self): + return self.i + b[0] = C(10) + self.assertEqual(b, bytearray([10, 100, 200])) + try: + b[3] = 0 + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + b[-10] = 0 + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + b[0] = 256 + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + b[0] = C(-1) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + b[0] = None + self.fail("Didn't raise TypeError") + except TypeError: + pass + + def test_delitem(self): + b = bytearray(range(10)) + del b[0] + self.assertEqual(b, bytearray(range(1, 10))) + del b[-1] + self.assertEqual(b, bytearray(range(1, 9))) + del b[4] + self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + + def test_setslice(self): + b = bytearray(range(10)) + self.assertEqual(list(b), list(range(10))) + + b[0:5] = bytearray([1, 1, 1, 1, 1]) + self.assertEqual(b, bytearray([1, 1, 1, 1, 1, 5, 6, 7, 8, 9])) + + del b[0:-5] + self.assertEqual(b, bytearray([5, 6, 7, 8, 9])) + + b[0:0] = bytearray([0, 1, 2, 3, 4]) + self.assertEqual(b, bytearray(range(10))) + + b[-7:-3] = bytearray([100, 101]) + self.assertEqual(b, bytearray([0, 1, 2, 100, 101, 7, 8, 9])) + + b[3:5] = [3, 4, 5, 6] + self.assertEqual(b, bytearray(range(10))) + + b[3:0] = [42, 42, 42] + self.assertEqual(b, bytearray([0, 1, 2, 42, 42, 42, 3, 4, 5, 6, 7, 8, 9])) + + def test_extended_set_del_slice(self): + indices = (0, None, 1, 3, 19, 300, -1, -2, -31, -300) + for start in indices: + for stop in indices: + # Skip invalid step 0 + for step in indices[1:]: + L = list(range(255)) + b = bytearray(L) + # Make sure we have a slice of exactly the right length, + # but with different data. + data = L[start:stop:step] + data.reverse() + L[start:stop:step] = data + b[start:stop:step] = data + self.assertEquals(b, bytearray(L)) + + del L[start:stop:step] + del b[start:stop:step] + self.assertEquals(b, bytearray(L)) + + def test_setslice_trap(self): + # This test verifies that we correctly handle assigning self + # to a slice of self (the old Lambert Meertens trap). + b = bytearray(range(256)) + b[8:] = b + self.assertEqual(b, bytearray(list(range(8)) + list(range(256)))) + + def test_iconcat(self): + b = bytearray(b"abc") + b1 = b + b += b"def" + self.assertEqual(b, b"abcdef") + self.assertEqual(b, b1) + self.failUnless(b is b1) + b += b"xyz" + self.assertEqual(b, b"abcdefxyz") + try: + b += u"" + except TypeError: + pass + else: + self.fail("bytes += unicode didn't raise TypeError") + + def test_irepeat(self): + b = bytearray(b"abc") + b1 = b + b *= 3 + self.assertEqual(b, b"abcabcabc") + self.assertEqual(b, b1) + self.failUnless(b is b1) + + def test_irepeat_1char(self): + b = bytearray(b"x") + b1 = b + b *= 100 + self.assertEqual(b, b"x"*100) + self.assertEqual(b, b1) + self.failUnless(b is b1) + + def test_alloc(self): + b = bytearray() + alloc = b.__alloc__() + self.assert_(alloc >= 0) + seq = [alloc] + for i in range(100): + b += b"x" + alloc = b.__alloc__() + self.assert_(alloc >= len(b)) + if alloc not in seq: + seq.append(alloc) + + def test_extend(self): + orig = b'hello' + a = bytearray(orig) + a.extend(a) + self.assertEqual(a, orig + orig) + self.assertEqual(a[5:], orig) + a = bytearray(b'') + # Test iterators that don't have a __length_hint__ + a.extend(map(ord, orig * 25)) + a.extend(ord(x) for x in orig * 25) + self.assertEqual(a, orig * 50) + self.assertEqual(a[-5:], orig) + a = bytearray(b'') + a.extend(iter(map(ord, orig * 50))) + self.assertEqual(a, orig * 50) + self.assertEqual(a[-5:], orig) + a = bytearray(b'') + a.extend(list(map(ord, orig * 50))) + self.assertEqual(a, orig * 50) + self.assertEqual(a[-5:], orig) + a = bytearray(b'') + self.assertRaises(ValueError, a.extend, [0, 1, 2, 256]) + self.assertRaises(ValueError, a.extend, [0, 1, 2, -1]) + self.assertEqual(len(a), 0) + + def test_remove(self): + b = bytearray(b'hello') + b.remove(ord('l')) + self.assertEqual(b, b'helo') + b.remove(ord('l')) + self.assertEqual(b, b'heo') + self.assertRaises(ValueError, lambda: b.remove(ord('l'))) + self.assertRaises(ValueError, lambda: b.remove(400)) + self.assertRaises(TypeError, lambda: b.remove(u'e')) + # remove first and last + b.remove(ord('o')) + b.remove(ord('h')) + self.assertEqual(b, b'e') + self.assertRaises(TypeError, lambda: b.remove(u'e')) + + def test_pop(self): + b = bytearray(b'world') + self.assertEqual(b.pop(), ord('d')) + self.assertEqual(b.pop(0), ord('w')) + self.assertEqual(b.pop(-2), ord('r')) + self.assertRaises(IndexError, lambda: b.pop(10)) + self.assertRaises(OverflowError, lambda: bytearray().pop()) + + def test_nosort(self): + self.assertRaises(AttributeError, lambda: bytearray().sort()) + + def test_append(self): + b = bytearray(b'hell') + b.append(ord('o')) + self.assertEqual(b, b'hello') + self.assertEqual(b.append(100), None) + b = bytearray() + b.append(ord('A')) + self.assertEqual(len(b), 1) + self.assertRaises(TypeError, lambda: b.append(u'o')) + + def test_insert(self): + b = bytearray(b'msssspp') + b.insert(1, ord('i')) + b.insert(4, ord('i')) + b.insert(-2, ord('i')) + b.insert(1000, ord('i')) + self.assertEqual(b, b'mississippi') + self.assertRaises(TypeError, lambda: b.insert(0, b'1')) + + def test_partition_bytearray_doesnt_share_nullstring(self): + a, b, c = bytearray(b"x").partition(b"y") + self.assertEqual(b, b"") + self.assertEqual(c, b"") + self.assert_(b is not c) + b += b"!" + self.assertEqual(c, b"") + a, b, c = bytearray(b"x").partition(b"y") + self.assertEqual(b, b"") + self.assertEqual(c, b"") + # Same for rpartition + b, c, a = bytearray(b"x").rpartition(b"y") + self.assertEqual(b, b"") + self.assertEqual(c, b"") + self.assert_(b is not c) + b += b"!" + self.assertEqual(c, b"") + c, b, a = bytearray(b"x").rpartition(b"y") + self.assertEqual(b, b"") + self.assertEqual(c, b"") + + +class AssortedBytesTest(unittest.TestCase): + # + # Test various combinations of bytes and bytearray + # + + def setUp(self): + self.warning_filters = warnings.filters[:] + + def tearDown(self): + warnings.filters = self.warning_filters + + def test_repr_str(self): + warnings.simplefilter('ignore', BytesWarning) + for f in str, repr: + self.assertEqual(f(bytearray()), "bytearray(b'')") + self.assertEqual(f(bytearray([0])), "bytearray(b'\\x00')") + self.assertEqual(f(bytearray([0, 1, 254, 255])), + "bytearray(b'\\x00\\x01\\xfe\\xff')") + self.assertEqual(f(b"abc"), "b'abc'") + self.assertEqual(f(b"'"), '''b"'"''') # ''' + self.assertEqual(f(b"'\""), r"""b'\'"'""") # ' + + def test_compare_bytes_to_bytearray(self): + self.assertEqual(b"abc" == bytes(b"abc"), True) + self.assertEqual(b"ab" != bytes(b"abc"), True) + self.assertEqual(b"ab" <= bytes(b"abc"), True) + self.assertEqual(b"ab" < bytes(b"abc"), True) + self.assertEqual(b"abc" >= bytes(b"ab"), True) + self.assertEqual(b"abc" > bytes(b"ab"), True) + + self.assertEqual(b"abc" != bytes(b"abc"), False) + self.assertEqual(b"ab" == bytes(b"abc"), False) + self.assertEqual(b"ab" > bytes(b"abc"), False) + self.assertEqual(b"ab" >= bytes(b"abc"), False) + self.assertEqual(b"abc" < bytes(b"ab"), False) + self.assertEqual(b"abc" <= bytes(b"ab"), False) + + self.assertEqual(bytes(b"abc") == b"abc", True) + self.assertEqual(bytes(b"ab") != b"abc", True) + self.assertEqual(bytes(b"ab") <= b"abc", True) + self.assertEqual(bytes(b"ab") < b"abc", True) + self.assertEqual(bytes(b"abc") >= b"ab", True) + self.assertEqual(bytes(b"abc") > b"ab", True) + + self.assertEqual(bytes(b"abc") != b"abc", False) + self.assertEqual(bytes(b"ab") == b"abc", False) + self.assertEqual(bytes(b"ab") > b"abc", False) + self.assertEqual(bytes(b"ab") >= b"abc", False) + self.assertEqual(bytes(b"abc") < b"ab", False) + self.assertEqual(bytes(b"abc") <= b"ab", False) + + def test_doc(self): + self.failUnless(bytearray.__doc__ != None) + self.failUnless(bytearray.__doc__.startswith("bytearray("), bytearray.__doc__) + self.failUnless(bytes.__doc__ != None) + self.failUnless(bytes.__doc__.startswith("bytes("), bytes.__doc__) + + def test_from_bytearray(self): + sample = bytes(b"Hello world\n\x80\x81\xfe\xff") + buf = memoryview(sample) + b = bytearray(buf) + self.assertEqual(b, bytearray(sample)) + + def test_to_str(self): + warnings.simplefilter('ignore', BytesWarning) + self.assertEqual(str(b''), "b''") + self.assertEqual(str(b'x'), "b'x'") + self.assertEqual(str(b'\x80'), "b'\\x80'") + self.assertEqual(str(bytearray(b'')), "bytearray(b'')") + self.assertEqual(str(bytearray(b'x')), "bytearray(b'x')") + self.assertEqual(str(bytearray(b'\x80')), "bytearray(b'\\x80')") + + def test_literal(self): + tests = [ + (b"Wonderful spam", "Wonderful spam"), + (br"Wonderful spam too", "Wonderful spam too"), + (b"\xaa\x00\000\200", "\xaa\x00\000\200"), + (br"\xaa\x00\000\200", r"\xaa\x00\000\200"), + ] + for b, s in tests: + self.assertEqual(b, bytearray(s, 'latin-1')) + for c in range(128, 256): + self.assertRaises(SyntaxError, eval, + 'b"%s"' % chr(c)) + + def test_translate(self): + b = b'hello' + rosetta = bytearray(range(0, 256)) + rosetta[ord('o')] = ord('e') + c = b.translate(rosetta, b'l') + self.assertEqual(b, b'hello') + self.assertEqual(c, b'hee') + + def test_split_bytearray(self): + self.assertEqual(b'a b'.split(memoryview(b' ')), [b'a', b'b']) + + def test_rsplit_bytearray(self): + self.assertEqual(b'a b'.rsplit(memoryview(b' ')), [b'a', b'b']) + + # Optimizations: + # __iter__? (optimization) + # __reversed__? (optimization) + + # XXX More string methods? (Those that don't use character properties) + + # There are tests in string_tests.py that are more + # comprehensive for things like split, partition, etc. + # Unfortunately they are all bundled with tests that + # are not appropriate for bytes + + # I've started porting some of those into bytearray_tests.py, we should port + # the rest that make sense (the code can be cleaned up to use modern + # unittest methods at the same time). + +class BytearrayPEP3137Test(unittest.TestCase, + test.buffer_tests.MixinBytesBufferCommonTests): + def marshal(self, x): + return bytearray(x) + + def test_returns_new_copy(self): + val = self.marshal(b'1234') + # On immutable types these MAY return a reference to themselves + # but on mutable types like bytearray they MUST return a new copy. + for methname in ('zfill', 'rjust', 'ljust', 'center'): + method = getattr(val, methname) + newval = method(3) + self.assertEqual(val, newval) + self.assertTrue(val is not newval, + methname+' returned self on a mutable object') + + +class FixedStringTest(test.string_tests.BaseTest): + + def fixtype(self, obj): + if isinstance(obj, str): + return obj.encode("utf-8") + return super(FixedStringTest, self).fixtype(obj) + + # Currently the bytes containment testing uses a single integer + # value. This may not be the final design, but until then the + # bytes section with in a bytes containment not valid + def test_contains(self): + pass + def test_expandtabs(self): + pass + def test_upper(self): + pass + def test_lower(self): + pass + def test_hash(self): + # XXX check this out + pass + + +class ByteArrayAsStringTest(FixedStringTest): + type2test = bytearray + + +class ByteArraySubclass(bytearray): + pass + +class ByteArraySubclassTest(unittest.TestCase): + + def test_basic(self): + self.assert_(issubclass(ByteArraySubclass, bytearray)) + self.assert_(isinstance(ByteArraySubclass(), bytearray)) + + a, b = b"abcd", b"efgh" + _a, _b = ByteArraySubclass(a), ByteArraySubclass(b) + + # test comparison operators with subclass instances + self.assert_(_a == _a) + self.assert_(_a != _b) + self.assert_(_a < _b) + self.assert_(_a <= _b) + self.assert_(_b >= _a) + self.assert_(_b > _a) + self.assert_(_a is not a) + + # test concat of subclass instances + self.assertEqual(a + b, _a + _b) + self.assertEqual(a + b, a + _b) + self.assertEqual(a + b, _a + b) + + # test repeat + self.assert_(a*5 == _a*5) + + def test_join(self): + # Make sure join returns a NEW object for single item sequences + # involving a subclass. + # Make sure that it is of the appropriate type. + s1 = ByteArraySubclass(b"abcd") + s2 = bytearray().join([s1]) + self.assert_(s1 is not s2) + self.assert_(type(s2) is bytearray, type(s2)) + + # Test reverse, calling join on subclass + s3 = s1.join([b"abcd"]) + self.assert_(type(s3) is bytearray) + + def test_pickle(self): + a = ByteArraySubclass(b"abcd") + a.x = 10 + a.y = ByteArraySubclass(b"efgh") + for proto in range(pickle.HIGHEST_PROTOCOL): + b = pickle.loads(pickle.dumps(a, proto)) + self.assertNotEqual(id(a), id(b)) + self.assertEqual(a, b) + self.assertEqual(a.x, b.x) + self.assertEqual(a.y, b.y) + self.assertEqual(type(a), type(b)) + self.assertEqual(type(a.y), type(b.y)) + + def test_copy(self): + a = ByteArraySubclass(b"abcd") + a.x = 10 + a.y = ByteArraySubclass(b"efgh") + for copy_method in (copy.copy, copy.deepcopy): + b = copy_method(a) + self.assertNotEqual(id(a), id(b)) + self.assertEqual(a, b) + self.assertEqual(a.x, b.x) + self.assertEqual(a.y, b.y) + self.assertEqual(type(a), type(b)) + self.assertEqual(type(a.y), type(b.y)) + + def test_init_override(self): + class subclass(bytearray): + def __init__(self, newarg=1, *args, **kwargs): + bytearray.__init__(self, *args, **kwargs) + x = subclass(4, source=b"abcd") + self.assertEqual(x, b"abcd") + x = subclass(newarg=4, source=b"abcd") + self.assertEqual(x, b"abcd") + +def test_main(): + #test.test_support.run_unittest(BytesTest) + #test.test_support.run_unittest(AssortedBytesTest) + #test.test_support.run_unittest(BytesAsStringTest) + test.test_support.run_unittest( + ByteArrayTest, + ByteArrayAsStringTest, + ByteArraySubclassTest, + BytearrayPEP3137Test) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py new file mode 100644 index 0000000..1791705 --- /dev/null +++ b/Lib/test/test_io.py @@ -0,0 +1,1162 @@ +"""Unit tests for io.py.""" +from __future__ import print_function + +import os +import sys +import time +import array +import unittest +from itertools import chain +from test import test_support + +import codecs +import io # The module under test + + +class MockRawIO(io.RawIOBase): + + def __init__(self, read_stack=()): + self._read_stack = list(read_stack) + self._write_stack = [] + + def read(self, n=None): + try: + return self._read_stack.pop(0) + except: + return b"" + + def write(self, b): + self._write_stack.append(b[:]) + return len(b) + + def writable(self): + return True + + def fileno(self): + return 42 + + def readable(self): + return True + + def seekable(self): + return True + + def seek(self, pos, whence): + pass + + def tell(self): + return 42 + + +class MockFileIO(io.BytesIO): + + def __init__(self, data): + self.read_history = [] + io.BytesIO.__init__(self, data) + + def read(self, n=None): + res = io.BytesIO.read(self, n) + self.read_history.append(None if res is None else len(res)) + return res + + +class MockNonBlockWriterIO(io.RawIOBase): + + def __init__(self, blocking_script): + self._blocking_script = list(blocking_script) + self._write_stack = [] + + def write(self, b): + self._write_stack.append(b[:]) + n = self._blocking_script.pop(0) + if (n < 0): + raise io.BlockingIOError(0, "test blocking", -n) + else: + return n + + def writable(self): + return True + + +class IOTest(unittest.TestCase): + + def tearDown(self): + test_support.unlink(test_support.TESTFN) + + def write_ops(self, f): + self.assertEqual(f.write(b"blah."), 5) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"Hello."), 6) + self.assertEqual(f.tell(), 6) + self.assertEqual(f.seek(-1, 1), 5) + self.assertEqual(f.tell(), 5) + self.assertEqual(f.write(bytearray(b" world\n\n\n")), 9) + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.write(b"h"), 1) + self.assertEqual(f.seek(-1, 2), 13) + self.assertEqual(f.tell(), 13) + self.assertEqual(f.truncate(12), 12) + self.assertEqual(f.tell(), 13) + self.assertRaises(TypeError, f.seek, 0.0) + + def read_ops(self, f, buffered=False): + data = f.read(5) + self.assertEqual(data, b"hello") + data = bytearray(data) + self.assertEqual(f.readinto(data), 5) + self.assertEqual(data, b" worl") + self.assertEqual(f.readinto(data), 2) + self.assertEqual(len(data), 5) + self.assertEqual(data[:2], b"d\n") + self.assertEqual(f.seek(0), 0) + self.assertEqual(f.read(20), b"hello world\n") + self.assertEqual(f.read(1), b"") + self.assertEqual(f.readinto(bytearray(b"x")), 0) + self.assertEqual(f.seek(-6, 2), 6) + self.assertEqual(f.read(5), b"world") + self.assertEqual(f.read(0), b"") + self.assertEqual(f.readinto(bytearray()), 0) + self.assertEqual(f.seek(-6, 1), 5) + self.assertEqual(f.read(5), b" worl") + self.assertEqual(f.tell(), 10) + self.assertRaises(TypeError, f.seek, 0.0) + if buffered: + f.seek(0) + self.assertEqual(f.read(), b"hello world\n") + f.seek(6) + self.assertEqual(f.read(), b"world\n") + self.assertEqual(f.read(), b"") + + LARGE = 2**31 + + def large_file_ops(self, f): + assert f.readable() + assert f.writable() + self.assertEqual(f.seek(self.LARGE), self.LARGE) + self.assertEqual(f.tell(), self.LARGE) + self.assertEqual(f.write(b"xxx"), 3) + self.assertEqual(f.tell(), self.LARGE + 3) + self.assertEqual(f.seek(-1, 1), self.LARGE + 2) + self.assertEqual(f.truncate(), self.LARGE + 2) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 2) + self.assertEqual(f.truncate(self.LARGE + 1), self.LARGE + 1) + self.assertEqual(f.tell(), self.LARGE + 2) + self.assertEqual(f.seek(0, 2), self.LARGE + 1) + self.assertEqual(f.seek(-1, 2), self.LARGE) + self.assertEqual(f.read(2), b"x") + + def test_raw_file_io(self): + f = io.open(test_support.TESTFN, "wb", buffering=0) + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + f.close() + f = io.open(test_support.TESTFN, "rb", buffering=0) + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f) + f.close() + + def test_buffered_file_io(self): + f = io.open(test_support.TESTFN, "wb") + self.assertEqual(f.readable(), False) + self.assertEqual(f.writable(), True) + self.assertEqual(f.seekable(), True) + self.write_ops(f) + f.close() + f = io.open(test_support.TESTFN, "rb") + self.assertEqual(f.readable(), True) + self.assertEqual(f.writable(), False) + self.assertEqual(f.seekable(), True) + self.read_ops(f, True) + f.close() + + def test_readline(self): + f = io.open(test_support.TESTFN, "wb") + f.write(b"abc\ndef\nxyzzy\nfoo") + f.close() + f = io.open(test_support.TESTFN, "rb") + self.assertEqual(f.readline(), b"abc\n") + self.assertEqual(f.readline(10), b"def\n") + self.assertEqual(f.readline(2), b"xy") + self.assertEqual(f.readline(4), b"zzy\n") + self.assertEqual(f.readline(), b"foo") + f.close() + + def test_raw_bytes_io(self): + f = io.BytesIO() + self.write_ops(f) + data = f.getvalue() + self.assertEqual(data, b"hello world\n") + f = io.BytesIO(data) + self.read_ops(f, True) + + def test_large_file_ops(self): + # On Windows and Mac OSX this test comsumes large resources; It takes + # a long time to build the >2GB file and takes >2GB of disk space + # therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or sys.platform == 'darwin': + if not test_support.is_resource_enabled("largefile"): + print("\nTesting large file ops skipped on %s." % sys.platform, + file=sys.stderr) + print("It requires %d bytes and a long time." % self.LARGE, + file=sys.stderr) + print("Use 'regrtest.py -u largefile test_io' to run it.", + file=sys.stderr) + return + f = io.open(test_support.TESTFN, "w+b", 0) + self.large_file_ops(f) + f.close() + f = io.open(test_support.TESTFN, "w+b") + self.large_file_ops(f) + f.close() + + def test_with_open(self): + for bufsize in (0, 1, 100): + f = None + with open(test_support.TESTFN, "wb", bufsize) as f: + f.write(b"xxx") + self.assertEqual(f.closed, True) + f = None + try: + with open(test_support.TESTFN, "wb", bufsize) as f: + 1/0 + except ZeroDivisionError: + self.assertEqual(f.closed, True) + else: + self.fail("1/0 didn't raise an exception") + + def test_destructor(self): + record = [] + class MyFileIO(io.FileIO): + def __del__(self): + record.append(1) + io.FileIO.__del__(self) + def close(self): + record.append(2) + io.FileIO.close(self) + def flush(self): + record.append(3) + io.FileIO.flush(self) + f = MyFileIO(test_support.TESTFN, "w") + f.write("xxx") + del f + self.assertEqual(record, [1, 2, 3]) + + def test_close_flushes(self): + f = io.open(test_support.TESTFN, "wb") + f.write(b"xxx") + f.close() + f = io.open(test_support.TESTFN, "rb") + self.assertEqual(f.read(), b"xxx") + f.close() + + def XXXtest_array_writes(self): + # XXX memory view not available yet + a = array.array('i', range(10)) + n = len(memoryview(a)) + f = io.open(test_support.TESTFN, "wb", 0) + self.assertEqual(f.write(a), n) + f.close() + f = io.open(test_support.TESTFN, "wb") + self.assertEqual(f.write(a), n) + f.close() + + def test_closefd(self): + self.assertRaises(ValueError, io.open, test_support.TESTFN, 'w', + closefd=False) + +class MemorySeekTestMixin: + + def testInit(self): + buf = self.buftype("1234567890") + bytesIo = self.ioclass(buf) + + def testRead(self): + buf = self.buftype("1234567890") + bytesIo = self.ioclass(buf) + + self.assertEquals(buf[:1], bytesIo.read(1)) + self.assertEquals(buf[1:5], bytesIo.read(4)) + self.assertEquals(buf[5:], bytesIo.read(900)) + self.assertEquals(self.EOF, bytesIo.read()) + + def testReadNoArgs(self): + buf = self.buftype("1234567890") + bytesIo = self.ioclass(buf) + + self.assertEquals(buf, bytesIo.read()) + self.assertEquals(self.EOF, bytesIo.read()) + + def testSeek(self): + buf = self.buftype("1234567890") + bytesIo = self.ioclass(buf) + + bytesIo.read(5) + bytesIo.seek(0) + self.assertEquals(buf, bytesIo.read()) + + bytesIo.seek(3) + self.assertEquals(buf[3:], bytesIo.read()) + self.assertRaises(TypeError, bytesIo.seek, 0.0) + + def testTell(self): + buf = self.buftype("1234567890") + bytesIo = self.ioclass(buf) + + self.assertEquals(0, bytesIo.tell()) + bytesIo.seek(5) + self.assertEquals(5, bytesIo.tell()) + bytesIo.seek(10000) + self.assertEquals(10000, bytesIo.tell()) + + +class BytesIOTest(MemorySeekTestMixin, unittest.TestCase): + @staticmethod + def buftype(s): + return s.encode("utf-8") + ioclass = io.BytesIO + EOF = b"" + + +class StringIOTest(MemorySeekTestMixin, unittest.TestCase): + buftype = str + ioclass = io.StringIO + EOF = "" + + +class BufferedReaderTest(unittest.TestCase): + + def testRead(self): + rawio = MockRawIO((b"abc", b"d", b"efg")) + bufio = io.BufferedReader(rawio) + + self.assertEquals(b"abcdef", bufio.read(6)) + + def testBuffering(self): + data = b"abcdefghi" + dlen = len(data) + + tests = [ + [ 100, [ 3, 1, 4, 8 ], [ dlen, 0 ] ], + [ 100, [ 3, 3, 3], [ dlen ] ], + [ 4, [ 1, 2, 4, 2 ], [ 4, 4, 1 ] ], + ] + + for bufsize, buf_read_sizes, raw_read_sizes in tests: + rawio = MockFileIO(data) + bufio = io.BufferedReader(rawio, buffer_size=bufsize) + pos = 0 + for nbytes in buf_read_sizes: + self.assertEquals(bufio.read(nbytes), data[pos:pos+nbytes]) + pos += nbytes + self.assertEquals(rawio.read_history, raw_read_sizes) + + def testReadNonBlocking(self): + # Inject some None's in there to simulate EWOULDBLOCK + rawio = MockRawIO((b"abc", b"d", None, b"efg", None, None)) + bufio = io.BufferedReader(rawio) + + self.assertEquals(b"abcd", bufio.read(6)) + self.assertEquals(b"e", bufio.read(1)) + self.assertEquals(b"fg", bufio.read()) + self.assert_(None is bufio.read()) + self.assertEquals(b"", bufio.read()) + + def testReadToEof(self): + rawio = MockRawIO((b"abc", b"d", b"efg")) + bufio = io.BufferedReader(rawio) + + self.assertEquals(b"abcdefg", bufio.read(9000)) + + def testReadNoArgs(self): + rawio = MockRawIO((b"abc", b"d", b"efg")) + bufio = io.BufferedReader(rawio) + + self.assertEquals(b"abcdefg", bufio.read()) + + def testFileno(self): + rawio = MockRawIO((b"abc", b"d", b"efg")) + bufio = io.BufferedReader(rawio) + + self.assertEquals(42, bufio.fileno()) + + def testFilenoNoFileno(self): + # XXX will we always have fileno() function? If so, kill + # this test. Else, write it. + pass + + +class BufferedWriterTest(unittest.TestCase): + + def testWrite(self): + # Write to the buffered IO but don't overflow the buffer. + writer = MockRawIO() + bufio = io.BufferedWriter(writer, 8) + + bufio.write(b"abc") + + self.assertFalse(writer._write_stack) + + def testWriteOverflow(self): + writer = MockRawIO() + bufio = io.BufferedWriter(writer, 8) + + bufio.write(b"abc") + bufio.write(b"defghijkl") + + self.assertEquals(b"abcdefghijkl", writer._write_stack[0]) + + def testWriteNonBlocking(self): + raw = MockNonBlockWriterIO((9, 2, 22, -6, 10, 12, 12)) + bufio = io.BufferedWriter(raw, 8, 16) + + bufio.write(b"asdf") + bufio.write(b"asdfa") + self.assertEquals(b"asdfasdfa", raw._write_stack[0]) + + bufio.write(b"asdfasdfasdf") + self.assertEquals(b"asdfasdfasdf", raw._write_stack[1]) + bufio.write(b"asdfasdfasdf") + self.assertEquals(b"dfasdfasdf", raw._write_stack[2]) + self.assertEquals(b"asdfasdfasdf", raw._write_stack[3]) + + bufio.write(b"asdfasdfasdf") + + # XXX I don't like this test. It relies too heavily on how the + # algorithm actually works, which we might change. Refactor + # later. + + def testFileno(self): + rawio = MockRawIO((b"abc", b"d", b"efg")) + bufio = io.BufferedWriter(rawio) + + self.assertEquals(42, bufio.fileno()) + + def testFlush(self): + writer = MockRawIO() + bufio = io.BufferedWriter(writer, 8) + + bufio.write(b"abc") + bufio.flush() + + self.assertEquals(b"abc", writer._write_stack[0]) + + +class BufferedRWPairTest(unittest.TestCase): + + def testRWPair(self): + r = MockRawIO(()) + w = MockRawIO() + pair = io.BufferedRWPair(r, w) + + # XXX need implementation + + +class BufferedRandomTest(unittest.TestCase): + + def testReadAndWrite(self): + raw = MockRawIO((b"asdf", b"ghjk")) + rw = io.BufferedRandom(raw, 8, 12) + + self.assertEqual(b"as", rw.read(2)) + rw.write(b"ddd") + rw.write(b"eee") + self.assertFalse(raw._write_stack) # Buffer writes + self.assertEqual(b"ghjk", rw.read()) # This read forces write flush + self.assertEquals(b"dddeee", raw._write_stack[0]) + + def testSeekAndTell(self): + raw = io.BytesIO(b"asdfghjkl") + rw = io.BufferedRandom(raw) + + self.assertEquals(b"as", rw.read(2)) + self.assertEquals(2, rw.tell()) + rw.seek(0, 0) + self.assertEquals(b"asdf", rw.read(4)) + + rw.write(b"asdf") + rw.seek(0, 0) + self.assertEquals(b"asdfasdfl", rw.read()) + self.assertEquals(9, rw.tell()) + rw.seek(-4, 2) + self.assertEquals(5, rw.tell()) + rw.seek(2, 1) + self.assertEquals(7, rw.tell()) + self.assertEquals(b"fl", rw.read(11)) + self.assertRaises(TypeError, rw.seek, 0.0) + +# To fully exercise seek/tell, the StatefulIncrementalDecoder has these +# properties: +# - A single output character can correspond to many bytes of input. +# - The number of input bytes to complete the character can be +# undetermined until the last input byte is received. +# - The number of input bytes can vary depending on previous input. +# - A single input byte can correspond to many characters of output. +# - The number of output characters can be undetermined until the +# last input byte is received. +# - The number of output characters can vary depending on previous input. + +class StatefulIncrementalDecoder(codecs.IncrementalDecoder): + """ + For testing seek/tell behavior with a stateful, buffering decoder. + + Input is a sequence of words. Words may be fixed-length (length set + by input) or variable-length (period-terminated). In variable-length + mode, extra periods are ignored. Possible words are: + - 'i' followed by a number sets the input length, I (maximum 99). + When I is set to 0, words are space-terminated. + - 'o' followed by a number sets the output length, O (maximum 99). + - Any other word is converted into a word followed by a period on + the output. The output word consists of the input word truncated + or padded out with hyphens to make its length equal to O. If O + is 0, the word is output verbatim without truncating or padding. + I and O are initially set to 1. When I changes, any buffered input is + re-scanned according to the new I. EOF also terminates the last word. + """ + + def __init__(self, errors='strict'): + codecs.IncrementalDecoder.__init__(self, errors) + self.reset() + + def __repr__(self): + return '<SID %x>' % id(self) + + def reset(self): + self.i = 1 + self.o = 1 + self.buffer = bytearray() + + def getstate(self): + i, o = self.i ^ 1, self.o ^ 1 # so that flags = 0 after reset() + return bytes(self.buffer), i*100 + o + + def setstate(self, state): + buffer, io = state + self.buffer = bytearray(buffer) + i, o = divmod(io, 100) + self.i, self.o = i ^ 1, o ^ 1 + + def decode(self, input, final=False): + output = '' + for b in input: + if self.i == 0: # variable-length, terminated with period + if b == ord('.'): + if self.buffer: + output += self.process_word() + else: + self.buffer.append(b) + else: # fixed-length, terminate after self.i bytes + self.buffer.append(b) + if len(self.buffer) == self.i: + output += self.process_word() + if final and self.buffer: # EOF terminates the last word + output += self.process_word() + return output + + def process_word(self): + output = '' + if self.buffer[0] == ord('i'): + self.i = min(99, int(self.buffer[1:] or 0)) # set input length + elif self.buffer[0] == ord('o'): + self.o = min(99, int(self.buffer[1:] or 0)) # set output length + else: + output = self.buffer.decode('ascii') + if len(output) < self.o: + output += '-'*self.o # pad out with hyphens + if self.o: + output = output[:self.o] # truncate to output length + output += '.' + self.buffer = bytearray() + return output + +class StatefulIncrementalDecoderTest(unittest.TestCase): + """ + Make sure the StatefulIncrementalDecoder actually works. + """ + + test_cases = [ + # I=1, O=1 (fixed-length input == fixed-length output) + (b'abcd', False, 'a.b.c.d.'), + # I=0, O=0 (variable-length input, variable-length output) + (b'oiabcd', True, 'abcd.'), + # I=0, O=0 (should ignore extra periods) + (b'oi...abcd...', True, 'abcd.'), + # I=0, O=6 (variable-length input, fixed-length output) + (b'i.o6.x.xyz.toolongtofit.', False, 'x-----.xyz---.toolon.'), + # I=2, O=6 (fixed-length input < fixed-length output) + (b'i.i2.o6xyz', True, 'xy----.z-----.'), + # I=6, O=3 (fixed-length input > fixed-length output) + (b'i.o3.i6.abcdefghijklmnop', True, 'abc.ghi.mno.'), + # I=0, then 3; O=29, then 15 (with longer output) + (b'i.o29.a.b.cde.o15.abcdefghijabcdefghij.i3.a.b.c.d.ei00k.l.m', True, + 'a----------------------------.' + + 'b----------------------------.' + + 'cde--------------------------.' + + 'abcdefghijabcde.' + + 'a.b------------.' + + '.c.------------.' + + 'd.e------------.' + + 'k--------------.' + + 'l--------------.' + + 'm--------------.') + ] + + def testDecoder(self): + # Try a few one-shot test cases. + for input, eof, output in self.test_cases: + d = StatefulIncrementalDecoder() + self.assertEquals(d.decode(input, eof), output) + + # Also test an unfinished decode, followed by forcing EOF. + d = StatefulIncrementalDecoder() + self.assertEquals(d.decode(b'oiabcd'), '') + self.assertEquals(d.decode(b'', 1), 'abcd.') + +class TextIOWrapperTest(unittest.TestCase): + + def setUp(self): + self.testdata = b"AAA\r\nBBB\rCCC\r\nDDD\nEEE\r\n" + self.normalized = b"AAA\nBBB\nCCC\nDDD\nEEE\n".decode("ascii") + + def tearDown(self): + test_support.unlink(test_support.TESTFN) + + def testLineBuffering(self): + r = io.BytesIO() + b = io.BufferedWriter(r, 1000) + t = io.TextIOWrapper(b, newline="\n", line_buffering=True) + t.write(u"X") + self.assertEquals(r.getvalue(), b"") # No flush happened + t.write(u"Y\nZ") + self.assertEquals(r.getvalue(), b"XY\nZ") # All got flushed + t.write(u"A\rB") + self.assertEquals(r.getvalue(), b"XY\nZA\rB") + + def testEncodingErrorsReading(self): + # (1) default + b = io.BytesIO(b"abc\n\xff\n") + t = io.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.read) + # (2) explicit strict + b = io.BytesIO(b"abc\n\xff\n") + t = io.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.read) + # (3) ignore + b = io.BytesIO(b"abc\n\xff\n") + t = io.TextIOWrapper(b, encoding="ascii", errors="ignore") + self.assertEquals(t.read(), "abc\n\n") + # (4) replace + b = io.BytesIO(b"abc\n\xff\n") + t = io.TextIOWrapper(b, encoding="ascii", errors="replace") + self.assertEquals(t.read(), u"abc\n\ufffd\n") + + def testEncodingErrorsWriting(self): + # (1) default + b = io.BytesIO() + t = io.TextIOWrapper(b, encoding="ascii") + self.assertRaises(UnicodeError, t.write, u"\xff") + # (2) explicit strict + b = io.BytesIO() + t = io.TextIOWrapper(b, encoding="ascii", errors="strict") + self.assertRaises(UnicodeError, t.write, u"\xff") + # (3) ignore + b = io.BytesIO() + t = io.TextIOWrapper(b, encoding="ascii", errors="ignore", + newline="\n") + t.write(u"abc\xffdef\n") + t.flush() + self.assertEquals(b.getvalue(), b"abcdef\n") + # (4) replace + b = io.BytesIO() + t = io.TextIOWrapper(b, encoding="ascii", errors="replace", + newline="\n") + t.write(u"abc\xffdef\n") + t.flush() + self.assertEquals(b.getvalue(), b"abc?def\n") + + def testNewlinesInput(self): + testdata = b"AAA\nBBB\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" + normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + for newline, expected in [ + (None, normalized.decode("ascii").splitlines(True)), + ("", testdata.decode("ascii").splitlines(True)), + ("\n", ["AAA\n", "BBB\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r\n", ["AAA\nBBB\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r", ["AAA\nBBB\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), + ]: + buf = io.BytesIO(testdata) + txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) + self.assertEquals(txt.readlines(), expected) + txt.seek(0) + self.assertEquals(txt.read(), "".join(expected)) + + def testNewlinesOutput(self): + testdict = { + "": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\n": b"AAA\nBBB\nCCC\nX\rY\r\nZ", + "\r": b"AAA\rBBB\rCCC\rX\rY\r\rZ", + "\r\n": b"AAA\r\nBBB\r\nCCC\r\nX\rY\r\r\nZ", + } + tests = [(None, testdict[os.linesep])] + sorted(testdict.items()) + for newline, expected in tests: + buf = io.BytesIO() + txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) + txt.write("AAA\nB") + txt.write("BB\nCCC\n") + txt.write("X\rY\r\nZ") + txt.flush() + self.assertEquals(buf.getvalue(), expected) + + def testNewlines(self): + input_lines = [ "unix\n", "windows\r\n", "os9\r", "last\n", "nonl" ] + + tests = [ + [ None, [ 'unix\n', 'windows\n', 'os9\n', 'last\n', 'nonl' ] ], + [ '', input_lines ], + [ '\n', [ "unix\n", "windows\r\n", "os9\rlast\n", "nonl" ] ], + [ '\r\n', [ "unix\nwindows\r\n", "os9\rlast\nnonl" ] ], + [ '\r', [ "unix\nwindows\r", "\nos9\r", "last\nnonl" ] ], + ] + + encodings = ('utf-8', 'latin-1') + + # Try a range of buffer sizes to test the case where \r is the last + # character in TextIOWrapper._pending_line. + for encoding in encodings: + # XXX: str.encode() should return bytes + data = bytes(''.join(input_lines).encode(encoding)) + for do_reads in (False, True): + for bufsize in range(1, 10): + for newline, exp_lines in tests: + bufio = io.BufferedReader(io.BytesIO(data), bufsize) + textio = io.TextIOWrapper(bufio, newline=newline, + encoding=encoding) + if do_reads: + got_lines = [] + while True: + c2 = textio.read(2) + if c2 == '': + break + self.assertEquals(len(c2), 2) + got_lines.append(c2 + textio.readline()) + else: + got_lines = list(textio) + + for got_line, exp_line in zip(got_lines, exp_lines): + self.assertEquals(got_line, exp_line) + self.assertEquals(len(got_lines), len(exp_lines)) + + def testNewlinesInput(self): + testdata = b"AAA\nBBB\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" + normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + for newline, expected in [ + (None, normalized.decode("ascii").splitlines(True)), + ("", testdata.decode("ascii").splitlines(True)), + ("\n", ["AAA\n", "BBB\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r\n", ["AAA\nBBB\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), + ("\r", ["AAA\nBBB\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), + ]: + buf = io.BytesIO(testdata) + txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) + self.assertEquals(txt.readlines(), expected) + txt.seek(0) + self.assertEquals(txt.read(), "".join(expected)) + + def testNewlinesOutput(self): + data = u"AAA\nBBB\rCCC\n" + data_lf = b"AAA\nBBB\rCCC\n" + data_cr = b"AAA\rBBB\rCCC\r" + data_crlf = b"AAA\r\nBBB\rCCC\r\n" + save_linesep = os.linesep + try: + for os.linesep, newline, expected in [ + ("\n", None, data_lf), + ("\r\n", None, data_crlf), + ("\n", "", data_lf), + ("\r\n", "", data_lf), + ("\n", "\n", data_lf), + ("\r\n", "\n", data_lf), + ("\n", "\r", data_cr), + ("\r\n", "\r", data_cr), + ("\n", "\r\n", data_crlf), + ("\r\n", "\r\n", data_crlf), + ]: + buf = io.BytesIO() + txt = io.TextIOWrapper(buf, encoding="ascii", newline=newline) + txt.write(data) + txt.close() + self.assertEquals(buf.getvalue(), expected) + finally: + os.linesep = save_linesep + + # Systematic tests of the text I/O API + + def testBasicIO(self): + for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): + for enc in "ascii", "latin1", "utf8" :# , "utf-16-be", "utf-16-le": + f = io.open(test_support.TESTFN, "w+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEquals(f.write(u"abc"), 3) + f.close() + f = io.open(test_support.TESTFN, "r+", encoding=enc) + f._CHUNK_SIZE = chunksize + self.assertEquals(f.tell(), 0) + self.assertEquals(f.read(), u"abc") + cookie = f.tell() + self.assertEquals(f.seek(0), 0) + self.assertEquals(f.read(2), u"ab") + self.assertEquals(f.read(1), u"c") + self.assertEquals(f.read(1), u"") + self.assertEquals(f.read(), u"") + self.assertEquals(f.tell(), cookie) + self.assertEquals(f.seek(0), 0) + self.assertEquals(f.seek(0, 2), cookie) + self.assertEquals(f.write(u"def"), 3) + self.assertEquals(f.seek(cookie), cookie) + self.assertEquals(f.read(), u"def") + if enc.startswith("utf"): + self.multi_line_test(f, enc) + f.close() + + def multi_line_test(self, f, enc): + f.seek(0) + f.truncate() + sample = u"s\xff\u0fff\uffff" + wlines = [] + for size in (0, 1, 2, 3, 4, 5, 30, 31, 32, 33, 62, 63, 64, 65, 1000): + chars = [] + for i in range(size): + chars.append(sample[i % len(sample)]) + line = u"".join(chars) + u"\n" + wlines.append((f.tell(), line)) + f.write(line) + f.seek(0) + rlines = [] + while True: + pos = f.tell() + line = f.readline() + if not line: + break + rlines.append((pos, line)) + self.assertEquals(rlines, wlines) + + def testTelling(self): + f = io.open(test_support.TESTFN, "w+", encoding="utf8") + p0 = f.tell() + f.write(u"\xff\n") + p1 = f.tell() + f.write(u"\xff\n") + p2 = f.tell() + f.seek(0) + self.assertEquals(f.tell(), p0) + self.assertEquals(f.readline(), u"\xff\n") + self.assertEquals(f.tell(), p1) + self.assertEquals(f.readline(), u"\xff\n") + self.assertEquals(f.tell(), p2) + f.seek(0) + for line in f: + self.assertEquals(line, u"\xff\n") + self.assertRaises(IOError, f.tell) + self.assertEquals(f.tell(), p2) + f.close() + + def testSeeking(self): + chunk_size = io.TextIOWrapper._CHUNK_SIZE + prefix_size = chunk_size - 2 + u_prefix = "a" * prefix_size + prefix = bytes(u_prefix.encode("utf-8")) + self.assertEquals(len(u_prefix), len(prefix)) + u_suffix = "\u8888\n" + suffix = bytes(u_suffix.encode("utf-8")) + line = prefix + suffix + f = io.open(test_support.TESTFN, "wb") + f.write(line*2) + f.close() + f = io.open(test_support.TESTFN, "r", encoding="utf-8") + s = f.read(prefix_size) + self.assertEquals(s, unicode(prefix, "ascii")) + self.assertEquals(f.tell(), prefix_size) + self.assertEquals(f.readline(), u_suffix) + + def testSeekingToo(self): + # Regression test for a specific bug + data = b'\xe0\xbf\xbf\n' + f = io.open(test_support.TESTFN, "wb") + f.write(data) + f.close() + f = io.open(test_support.TESTFN, "r", encoding="utf-8") + f._CHUNK_SIZE # Just test that it exists + f._CHUNK_SIZE = 2 + f.readline() + f.tell() + + # FIXME: figure out why the test fails with Python 2.6 + def XXXtestSeekAndTell(self): + """Test seek/tell using the StatefulIncrementalDecoder.""" + + def lookupTestDecoder(name): + if self.codecEnabled and name == 'test_decoder': + return codecs.CodecInfo( + name='test_decoder', encode=None, decode=None, + incrementalencoder=None, + streamreader=None, streamwriter=None, + incrementaldecoder=StatefulIncrementalDecoder) + + def testSeekAndTellWithData(data, min_pos=0): + """Tell/seek to various points within a data stream and ensure + that the decoded data returned by read() is consistent.""" + f = io.open(test_support.TESTFN, 'wb') + f.write(data) + f.close() + f = io.open(test_support.TESTFN, encoding='test_decoder') + decoded = f.read() + f.close() + + for i in range(min_pos, len(decoded) + 1): # seek positions + for j in [1, 5, len(decoded) - i]: # read lengths + f = io.open(test_support.TESTFN, encoding='test_decoder') + self.assertEquals(f.read(i), decoded[:i]) + cookie = f.tell() + self.assertEquals(f.read(j), decoded[i:i + j]) + f.seek(cookie) + self.assertEquals(f.read(), decoded[i:]) + f.close() + + # Register a special incremental decoder for testing. + codecs.register(lookupTestDecoder) + self.codecEnabled = 1 + + # Run the tests. + try: + # Try each test case. + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + testSeekAndTellWithData(input) + + # Position each test case so that it crosses a chunk boundary. + CHUNK_SIZE = io.TextIOWrapper._CHUNK_SIZE + for input, _, _ in StatefulIncrementalDecoderTest.test_cases: + offset = CHUNK_SIZE - len(input)//2 + prefix = b'.'*offset + # Don't bother seeking into the prefix (takes too long). + min_pos = offset*2 + testSeekAndTellWithData(prefix + input, min_pos) + + # Ensure our test decoder won't interfere with subsequent tests. + finally: + self.codecEnabled = 0 + + def testEncodedWrites(self): + data = u"1234567890" + tests = ("utf-16", + "utf-16-le", + "utf-16-be", + "utf-32", + "utf-32-le", + "utf-32-be") + for encoding in tests: + buf = io.BytesIO() + f = io.TextIOWrapper(buf, encoding=encoding) + # Check if the BOM is written only once (see issue1753). + f.write(data) + f.write(data) + f.seek(0) + self.assertEquals(f.read(), data * 2) + self.assertEquals(buf.getvalue(), (data * 2).encode(encoding)) + + def timingTest(self): + timer = time.time + enc = "utf8" + line = "\0\x0f\xff\u0fff\uffff\U000fffff\U0010ffff"*3 + "\n" + nlines = 10000 + nchars = len(line) + nbytes = len(line.encode(enc)) + for chunk_size in (32, 64, 128, 256): + f = io.open(test_support.TESTFN, "w+", encoding=enc) + f._CHUNK_SIZE = chunk_size + t0 = timer() + for i in range(nlines): + f.write(line) + f.flush() + t1 = timer() + f.seek(0) + for line in f: + pass + t2 = timer() + f.seek(0) + while f.readline(): + pass + t3 = timer() + f.seek(0) + while f.readline(): + f.tell() + t4 = timer() + f.close() + if test_support.verbose: + print("\nTiming test: %d lines of %d characters (%d bytes)" % + (nlines, nchars, nbytes)) + print("File chunk size: %6s" % f._CHUNK_SIZE) + print("Writing: %6.3f seconds" % (t1-t0)) + print("Reading using iteration: %6.3f seconds" % (t2-t1)) + print("Reading using readline(): %6.3f seconds" % (t3-t2)) + print("Using readline()+tell(): %6.3f seconds" % (t4-t3)) + + def testReadOneByOne(self): + txt = io.TextIOWrapper(io.BytesIO(b"AA\r\nBB")) + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEquals(reads, "AA\nBB") + + # read in amounts equal to TextIOWrapper._CHUNK_SIZE which is 128. + def testReadByChunk(self): + # make sure "\r\n" straddles 128 char boundary. + txt = io.TextIOWrapper(io.BytesIO(b"A" * 127 + b"\r\nB")) + reads = "" + while True: + c = txt.read(128) + if not c: + break + reads += c + self.assertEquals(reads, "A"*127+"\nB") + + def test_issue1395_1(self): + txt = io.TextIOWrapper(io.BytesIO(self.testdata), encoding="ascii") + + # read one char at a time + reads = "" + while True: + c = txt.read(1) + if not c: + break + reads += c + self.assertEquals(reads, self.normalized) + + def test_issue1395_2(self): + txt = io.TextIOWrapper(io.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = "" + while True: + c = txt.read(4) + if not c: + break + reads += c + self.assertEquals(reads, self.normalized) + + def test_issue1395_3(self): + txt = io.TextIOWrapper(io.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read(4) + reads += txt.readline() + reads += txt.readline() + reads += txt.readline() + self.assertEquals(reads, self.normalized) + + def test_issue1395_4(self): + txt = io.TextIOWrapper(io.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + reads += txt.read() + self.assertEquals(reads, self.normalized) + + def test_issue1395_5(self): + txt = io.TextIOWrapper(io.BytesIO(self.testdata), encoding="ascii") + txt._CHUNK_SIZE = 4 + + reads = txt.read(4) + pos = txt.tell() + txt.seek(0) + txt.seek(pos) + self.assertEquals(txt.read(4), "BBB\n") + + def test_issue2282(self): + buffer = io.BytesIO(self.testdata) + txt = io.TextIOWrapper(buffer, encoding="ascii") + + self.assertEqual(buffer.seekable(), txt.seekable()) + + def test_newline_decoder(self): + import codecs + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = io.IncrementalNewlineDecoder(decoder, translate=True) + + self.assertEquals(decoder.decode(b'\xe8\xa2\x88'), u"\u8888") + + self.assertEquals(decoder.decode(b'\xe8'), u"") + self.assertEquals(decoder.decode(b'\xa2'), u"") + self.assertEquals(decoder.decode(b'\x88'), u"\u8888") + + self.assertEquals(decoder.decode(b'\xe8'), u"") + self.assertRaises(UnicodeDecodeError, decoder.decode, b'', final=True) + + decoder.setstate((b'', 0)) + self.assertEquals(decoder.decode(b'\n'), u"\n") + self.assertEquals(decoder.decode(b'\r'), u"") + self.assertEquals(decoder.decode(b'', final=True), u"\n") + self.assertEquals(decoder.decode(b'\r', final=True), u"\n") + + self.assertEquals(decoder.decode(b'\r'), u"") + self.assertEquals(decoder.decode(b'a'), u"\na") + + self.assertEquals(decoder.decode(b'\r\r\n'), u"\n\n") + self.assertEquals(decoder.decode(b'\r'), u"") + self.assertEquals(decoder.decode(b'\r'), u"\n") + self.assertEquals(decoder.decode(b'\na'), u"\na") + + self.assertEquals(decoder.decode(b'\xe8\xa2\x88\r\n'), u"\u8888\n") + self.assertEquals(decoder.decode(b'\xe8\xa2\x88'), u"\u8888") + self.assertEquals(decoder.decode(b'\n'), u"\n") + self.assertEquals(decoder.decode(b'\xe8\xa2\x88\r'), u"\u8888") + self.assertEquals(decoder.decode(b'\n'), u"\n") + + decoder = codecs.getincrementaldecoder("utf-8")() + decoder = io.IncrementalNewlineDecoder(decoder, translate=True) + self.assertEquals(decoder.newlines, None) + decoder.decode(b"abc\n\r") + self.assertEquals(decoder.newlines, u'\n') + decoder.decode(b"\nabc") + self.assertEquals(decoder.newlines, ('\n', '\r\n')) + decoder.decode(b"abc\r") + self.assertEquals(decoder.newlines, ('\n', '\r\n')) + decoder.decode(b"abc") + self.assertEquals(decoder.newlines, ('\r', '\n', '\r\n')) + decoder.decode(b"abc\r") + decoder.reset() + self.assertEquals(decoder.decode(b"abc"), "abc") + self.assertEquals(decoder.newlines, None) + +# XXX Tests for open() + +class MiscIOTest(unittest.TestCase): + + def testImport__all__(self): + for name in io.__all__: + obj = getattr(io, name, None) + self.assert_(obj is not None, name) + if name == "open": + continue + elif "error" in name.lower(): + self.assert_(issubclass(obj, Exception), name) + else: + self.assert_(issubclass(obj, io.IOBase)) + + +def test_main(): + test_support.run_unittest(IOTest, BytesIOTest, StringIOTest, + BufferedReaderTest, + BufferedWriterTest, BufferedRWPairTest, + BufferedRandomTest, TextIOWrapperTest, + MiscIOTest) + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 0c46f9b..5ed2cc0 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -9,10 +9,10 @@ import unittest from test import test_support import sys -try: +if sys.version_info[0] == 3: # 3.x from io import StringIO -except ImportError: +else: # 2.x from StringIO import StringIO diff --git a/Makefile.pre.in b/Makefile.pre.in index f6934a9..acf060f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -295,6 +295,8 @@ OBJECT_OBJS= \ Objects/abstract.o \ Objects/boolobject.o \ Objects/bufferobject.o \ + Objects/bytes_methods.o \ + Objects/bytesobject.o \ Objects/cellobject.o \ Objects/classobject.o \ Objects/cobject.o \ @@ -518,13 +520,16 @@ Objects/unicodectype.o: $(srcdir)/Objects/unicodectype.c \ $(srcdir)/Objects/unicodetype_db.h STRINGLIB_HEADERS= \ + $(srcdir)/Include/bytes_methods.h \ $(srcdir)/Objects/stringlib/count.h \ + $(srcdir)/Objects/stringlib/ctype.h \ $(srcdir)/Objects/stringlib/fastsearch.h \ $(srcdir)/Objects/stringlib/find.h \ $(srcdir)/Objects/stringlib/formatter.h \ $(srcdir)/Objects/stringlib/partition.h \ $(srcdir)/Objects/stringlib/stringdefs.h \ $(srcdir)/Objects/stringlib/string_format.h \ + $(srcdir)/Objects/stringlib/transmogrify.h \ $(srcdir)/Objects/stringlib/unicodedefs.h Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c \ @@ -532,6 +537,8 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c \ Objects/stringobject.o: $(srcdir)/Objects/stringobject.c \ $(STRINGLIB_HEADERS) +Objects/bytesobject.o: $(srcdir)/Objects/bytesobject.c \ + $(STRINGLIB_HEADERS) Python/formatter_unicode.o: $(srcdir)/Python/formatter_unicode.c \ $(STRINGLIB_HEADERS) @@ -550,6 +557,8 @@ PYTHON_HEADERS= \ Include/ast.h \ Include/bitset.h \ Include/boolobject.h \ + Include/bytes_methods.h \ + Include/bytesobject.h \ Include/bufferobject.h \ Include/cellobject.h \ Include/ceval.h \ diff --git a/Modules/main.c b/Modules/main.c index 0cd879d..21cb487 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -40,7 +40,7 @@ static char **orig_argv; static int orig_argc; /* command line options */ -#define BASE_OPTS "3Bc:dEhim:OQ:StuUvVW:xX?" +#define BASE_OPTS "3bBc:dEhim:OQ:StuUvVW:xX?" #ifndef RISCOS #define PROGRAM_OPTS BASE_OPTS @@ -296,6 +296,9 @@ Py_Main(int argc, char **argv) } switch (c) { + case 'b': + Py_BytesWarningFlag++; + break; case 'd': Py_DebugFlag++; diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c new file mode 100644 index 0000000..de87905 --- /dev/null +++ b/Objects/bytes_methods.c @@ -0,0 +1,610 @@ +#include "Python.h" +#include "bytes_methods.h" + +/* Our own locale-independent ctype.h-like macros */ + +const unsigned int _Py_ctype_table[256] = { + 0, /* 0x0 '\x00' */ + 0, /* 0x1 '\x01' */ + 0, /* 0x2 '\x02' */ + 0, /* 0x3 '\x03' */ + 0, /* 0x4 '\x04' */ + 0, /* 0x5 '\x05' */ + 0, /* 0x6 '\x06' */ + 0, /* 0x7 '\x07' */ + 0, /* 0x8 '\x08' */ + FLAG_SPACE, /* 0x9 '\t' */ + FLAG_SPACE, /* 0xa '\n' */ + FLAG_SPACE, /* 0xb '\v' */ + FLAG_SPACE, /* 0xc '\f' */ + FLAG_SPACE, /* 0xd '\r' */ + 0, /* 0xe '\x0e' */ + 0, /* 0xf '\x0f' */ + 0, /* 0x10 '\x10' */ + 0, /* 0x11 '\x11' */ + 0, /* 0x12 '\x12' */ + 0, /* 0x13 '\x13' */ + 0, /* 0x14 '\x14' */ + 0, /* 0x15 '\x15' */ + 0, /* 0x16 '\x16' */ + 0, /* 0x17 '\x17' */ + 0, /* 0x18 '\x18' */ + 0, /* 0x19 '\x19' */ + 0, /* 0x1a '\x1a' */ + 0, /* 0x1b '\x1b' */ + 0, /* 0x1c '\x1c' */ + 0, /* 0x1d '\x1d' */ + 0, /* 0x1e '\x1e' */ + 0, /* 0x1f '\x1f' */ + FLAG_SPACE, /* 0x20 ' ' */ + 0, /* 0x21 '!' */ + 0, /* 0x22 '"' */ + 0, /* 0x23 '#' */ + 0, /* 0x24 '$' */ + 0, /* 0x25 '%' */ + 0, /* 0x26 '&' */ + 0, /* 0x27 "'" */ + 0, /* 0x28 '(' */ + 0, /* 0x29 ')' */ + 0, /* 0x2a '*' */ + 0, /* 0x2b '+' */ + 0, /* 0x2c ',' */ + 0, /* 0x2d '-' */ + 0, /* 0x2e '.' */ + 0, /* 0x2f '/' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x30 '0' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x31 '1' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x32 '2' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x33 '3' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x34 '4' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x35 '5' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x36 '6' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x37 '7' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x38 '8' */ + FLAG_DIGIT|FLAG_XDIGIT, /* 0x39 '9' */ + 0, /* 0x3a ':' */ + 0, /* 0x3b ';' */ + 0, /* 0x3c '<' */ + 0, /* 0x3d '=' */ + 0, /* 0x3e '>' */ + 0, /* 0x3f '?' */ + 0, /* 0x40 '@' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x41 'A' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x42 'B' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x43 'C' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x44 'D' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x45 'E' */ + FLAG_UPPER|FLAG_XDIGIT, /* 0x46 'F' */ + FLAG_UPPER, /* 0x47 'G' */ + FLAG_UPPER, /* 0x48 'H' */ + FLAG_UPPER, /* 0x49 'I' */ + FLAG_UPPER, /* 0x4a 'J' */ + FLAG_UPPER, /* 0x4b 'K' */ + FLAG_UPPER, /* 0x4c 'L' */ + FLAG_UPPER, /* 0x4d 'M' */ + FLAG_UPPER, /* 0x4e 'N' */ + FLAG_UPPER, /* 0x4f 'O' */ + FLAG_UPPER, /* 0x50 'P' */ + FLAG_UPPER, /* 0x51 'Q' */ + FLAG_UPPER, /* 0x52 'R' */ + FLAG_UPPER, /* 0x53 'S' */ + FLAG_UPPER, /* 0x54 'T' */ + FLAG_UPPER, /* 0x55 'U' */ + FLAG_UPPER, /* 0x56 'V' */ + FLAG_UPPER, /* 0x57 'W' */ + FLAG_UPPER, /* 0x58 'X' */ + FLAG_UPPER, /* 0x59 'Y' */ + FLAG_UPPER, /* 0x5a 'Z' */ + 0, /* 0x5b '[' */ + 0, /* 0x5c '\\' */ + 0, /* 0x5d ']' */ + 0, /* 0x5e '^' */ + 0, /* 0x5f '_' */ + 0, /* 0x60 '`' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x61 'a' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x62 'b' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x63 'c' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x64 'd' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x65 'e' */ + FLAG_LOWER|FLAG_XDIGIT, /* 0x66 'f' */ + FLAG_LOWER, /* 0x67 'g' */ + FLAG_LOWER, /* 0x68 'h' */ + FLAG_LOWER, /* 0x69 'i' */ + FLAG_LOWER, /* 0x6a 'j' */ + FLAG_LOWER, /* 0x6b 'k' */ + FLAG_LOWER, /* 0x6c 'l' */ + FLAG_LOWER, /* 0x6d 'm' */ + FLAG_LOWER, /* 0x6e 'n' */ + FLAG_LOWER, /* 0x6f 'o' */ + FLAG_LOWER, /* 0x70 'p' */ + FLAG_LOWER, /* 0x71 'q' */ + FLAG_LOWER, /* 0x72 'r' */ + FLAG_LOWER, /* 0x73 's' */ + FLAG_LOWER, /* 0x74 't' */ + FLAG_LOWER, /* 0x75 'u' */ + FLAG_LOWER, /* 0x76 'v' */ + FLAG_LOWER, /* 0x77 'w' */ + FLAG_LOWER, /* 0x78 'x' */ + FLAG_LOWER, /* 0x79 'y' */ + FLAG_LOWER, /* 0x7a 'z' */ + 0, /* 0x7b '{' */ + 0, /* 0x7c '|' */ + 0, /* 0x7d '}' */ + 0, /* 0x7e '~' */ + 0, /* 0x7f '\x7f' */ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + + +const unsigned char _Py_ctype_tolower[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, + 0x68, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, + 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, + 0x78, 0x79, 0x7a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + +const unsigned char _Py_ctype_toupper[256] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, + 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, + 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x5b, 0x5c, 0x5d, 0x5e, 0x5f, + 0x60, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, + 0x58, 0x59, 0x5a, 0x7b, 0x7c, 0x7d, 0x7e, 0x7f, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, + 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, + 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, + 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, + 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, + 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, + 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, + 0xc0, 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, + 0xc8, 0xc9, 0xca, 0xcb, 0xcc, 0xcd, 0xce, 0xcf, + 0xd0, 0xd1, 0xd2, 0xd3, 0xd4, 0xd5, 0xd6, 0xd7, + 0xd8, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, + 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, + 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +}; + + +PyDoc_STRVAR_shared(_Py_isspace__doc__, +"B.isspace() -> bool\n\ +\n\ +Return True if all characters in B are whitespace\n\ +and there is at least one character in B, False otherwise."); + +PyObject* +_Py_bytes_isspace(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + + /* Shortcut for single character strings */ + if (len == 1 && ISSPACE(*p)) + Py_RETURN_TRUE; + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + for (; p < e; p++) { + if (!ISSPACE(*p)) + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; +} + + +PyDoc_STRVAR_shared(_Py_isalpha__doc__, +"B.isalpha() -> bool\n\ +\n\ +Return True if all characters in B are alphabetic\n\ +and there is at least one character in B, False otherwise."); + +PyObject* +_Py_bytes_isalpha(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + + /* Shortcut for single character strings */ + if (len == 1 && ISALPHA(*p)) + Py_RETURN_TRUE; + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + for (; p < e; p++) { + if (!ISALPHA(*p)) + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; +} + + +PyDoc_STRVAR_shared(_Py_isalnum__doc__, +"B.isalnum() -> bool\n\ +\n\ +Return True if all characters in B are alphanumeric\n\ +and there is at least one character in B, False otherwise."); + +PyObject* +_Py_bytes_isalnum(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + + /* Shortcut for single character strings */ + if (len == 1 && ISALNUM(*p)) + Py_RETURN_TRUE; + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + for (; p < e; p++) { + if (!ISALNUM(*p)) + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; +} + + +PyDoc_STRVAR_shared(_Py_isdigit__doc__, +"B.isdigit() -> bool\n\ +\n\ +Return True if all characters in B are digits\n\ +and there is at least one character in B, False otherwise."); + +PyObject* +_Py_bytes_isdigit(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + + /* Shortcut for single character strings */ + if (len == 1 && ISDIGIT(*p)) + Py_RETURN_TRUE; + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + for (; p < e; p++) { + if (!ISDIGIT(*p)) + Py_RETURN_FALSE; + } + Py_RETURN_TRUE; +} + + +PyDoc_STRVAR_shared(_Py_islower__doc__, +"B.islower() -> bool\n\ +\n\ +Return True if all cased characters in B are lowercase and there is\n\ +at least one cased character in B, False otherwise."); + +PyObject* +_Py_bytes_islower(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + int cased; + + /* Shortcut for single character strings */ + if (len == 1) + return PyBool_FromLong(ISLOWER(*p)); + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + cased = 0; + for (; p < e; p++) { + if (ISUPPER(*p)) + Py_RETURN_FALSE; + else if (!cased && ISLOWER(*p)) + cased = 1; + } + return PyBool_FromLong(cased); +} + + +PyDoc_STRVAR_shared(_Py_isupper__doc__, +"B.isupper() -> bool\n\ +\n\ +Return True if all cased characters in B are uppercase and there is\n\ +at least one cased character in B, False otherwise."); + +PyObject* +_Py_bytes_isupper(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + int cased; + + /* Shortcut for single character strings */ + if (len == 1) + return PyBool_FromLong(ISUPPER(*p)); + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + cased = 0; + for (; p < e; p++) { + if (ISLOWER(*p)) + Py_RETURN_FALSE; + else if (!cased && ISUPPER(*p)) + cased = 1; + } + return PyBool_FromLong(cased); +} + + +PyDoc_STRVAR_shared(_Py_istitle__doc__, +"B.istitle() -> bool\n\ +\n\ +Return True if B is a titlecased string and there is at least one\n\ +character in B, i.e. uppercase characters may only follow uncased\n\ +characters and lowercase characters only cased ones. Return False\n\ +otherwise."); + +PyObject* +_Py_bytes_istitle(const char *cptr, Py_ssize_t len) +{ + register const unsigned char *p + = (unsigned char *) cptr; + register const unsigned char *e; + int cased, previous_is_cased; + + /* Shortcut for single character strings */ + if (len == 1) + return PyBool_FromLong(ISUPPER(*p)); + + /* Special case for empty strings */ + if (len == 0) + Py_RETURN_FALSE; + + e = p + len; + cased = 0; + previous_is_cased = 0; + for (; p < e; p++) { + register const unsigned char ch = *p; + + if (ISUPPER(ch)) { + if (previous_is_cased) + Py_RETURN_FALSE; + previous_is_cased = 1; + cased = 1; + } + else if (ISLOWER(ch)) { + if (!previous_is_cased) + Py_RETURN_FALSE; + previous_is_cased = 1; + cased = 1; + } + else + previous_is_cased = 0; + } + return PyBool_FromLong(cased); +} + + +PyDoc_STRVAR_shared(_Py_lower__doc__, +"B.lower() -> copy of B\n\ +\n\ +Return a copy of B with all ASCII characters converted to lowercase."); + +void +_Py_bytes_lower(char *result, const char *cptr, Py_ssize_t len) +{ + Py_ssize_t i; + + /* + newobj = PyString_FromStringAndSize(NULL, len); + if (!newobj) + return NULL; + + s = PyString_AS_STRING(newobj); + */ + + Py_MEMCPY(result, cptr, len); + + for (i = 0; i < len; i++) { + int c = Py_CHARMASK(result[i]); + if (ISUPPER(c)) + result[i] = TOLOWER(c); + } +} + + +PyDoc_STRVAR_shared(_Py_upper__doc__, +"B.upper() -> copy of B\n\ +\n\ +Return a copy of B with all ASCII characters converted to uppercase."); + +void +_Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) +{ + Py_ssize_t i; + + /* + newobj = PyString_FromStringAndSize(NULL, len); + if (!newobj) + return NULL; + + s = PyString_AS_STRING(newobj); + */ + + Py_MEMCPY(result, cptr, len); + + for (i = 0; i < len; i++) { + int c = Py_CHARMASK(result[i]); + if (ISLOWER(c)) + result[i] = TOUPPER(c); + } +} + + +PyDoc_STRVAR_shared(_Py_title__doc__, +"B.title() -> copy of B\n\ +\n\ +Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ +characters, all remaining cased characters have lowercase."); + +void +_Py_bytes_title(char *result, char *s, Py_ssize_t len) +{ + Py_ssize_t i; + int previous_is_cased = 0; + + /* + newobj = PyString_FromStringAndSize(NULL, len); + if (newobj == NULL) + return NULL; + s_new = PyString_AsString(newobj); + */ + for (i = 0; i < len; i++) { + int c = Py_CHARMASK(*s++); + if (ISLOWER(c)) { + if (!previous_is_cased) + c = TOUPPER(c); + previous_is_cased = 1; + } else if (ISUPPER(c)) { + if (previous_is_cased) + c = TOLOWER(c); + previous_is_cased = 1; + } else + previous_is_cased = 0; + *result++ = c; + } +} + + +PyDoc_STRVAR_shared(_Py_capitalize__doc__, +"B.capitalize() -> copy of B\n\ +\n\ +Return a copy of B with only its first character capitalized (ASCII)."); + +void +_Py_bytes_capitalize(char *result, char *s, Py_ssize_t len) +{ + Py_ssize_t i; + + /* + newobj = PyString_FromStringAndSize(NULL, len); + if (newobj == NULL) + return NULL; + s_new = PyString_AsString(newobj); + */ + if (0 < len) { + int c = Py_CHARMASK(*s++); + if (ISLOWER(c)) + *result = TOUPPER(c); + else + *result = c; + result++; + } + for (i = 1; i < len; i++) { + int c = Py_CHARMASK(*s++); + if (ISUPPER(c)) + *result = TOLOWER(c); + else + *result = c; + result++; + } +} + + +PyDoc_STRVAR_shared(_Py_swapcase__doc__, +"B.swapcase() -> copy of B\n\ +\n\ +Return a copy of B with uppercase ASCII characters converted\n\ +to lowercase ASCII and vice versa."); + +void +_Py_bytes_swapcase(char *result, char *s, Py_ssize_t len) +{ + Py_ssize_t i; + + /* + newobj = PyString_FromStringAndSize(NULL, len); + if (newobj == NULL) + return NULL; + s_new = PyString_AsString(newobj); + */ + for (i = 0; i < len; i++) { + int c = Py_CHARMASK(*s++); + if (ISLOWER(c)) { + *result = TOUPPER(c); + } + else if (ISUPPER(c)) { + *result = TOLOWER(c); + } + else + *result = c; + result++; + } +} + diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c new file mode 100644 index 0000000..86d58e1 --- /dev/null +++ b/Objects/bytesobject.c @@ -0,0 +1,3374 @@ +/* PyBytes (bytearray) implementation */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "bytes_methods.h" + +static PyBytesObject *nullbytes = NULL; + +void +PyBytes_Fini(void) +{ + Py_CLEAR(nullbytes); +} + +int +PyBytes_Init(void) +{ + nullbytes = PyObject_New(PyBytesObject, &PyBytes_Type); + if (nullbytes == NULL) + return 0; + nullbytes->ob_bytes = NULL; + Py_SIZE(nullbytes) = nullbytes->ob_alloc = 0; + nullbytes->ob_exports = 0; + return 1; +} + +/* end nullbytes support */ + +/* Helpers */ + +static int +_getbytevalue(PyObject* arg, int *value) +{ + long face_value; + + if (PyInt_Check(arg)) { + face_value = PyInt_AsLong(arg); + if (face_value < 0 || face_value >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return 0; + } + } + else if (PyString_CheckExact(arg)) { + if (Py_SIZE(arg) != 1) { + PyErr_SetString(PyExc_ValueError, "string must be of size 1"); + return 0; + } + face_value = Py_CHARMASK(((PyStringObject*)arg)->ob_sval[0]); + } + else { + PyErr_Format(PyExc_TypeError, "an integer or string of size 1 is required"); + return 0; + } + + *value = face_value; + return 1; +} + +static Py_ssize_t +bytes_buffer_getreadbuf(PyBytesObject *self, Py_ssize_t index, const void **ptr) +{ + if ( index != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent bytes segment"); + return -1; + } + *ptr = (void *)self->ob_bytes; + return Py_SIZE(self); +} + +static Py_ssize_t +bytes_buffer_getwritebuf(PyBytesObject *self, Py_ssize_t index, const void **ptr) +{ + if ( index != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent bytes segment"); + return -1; + } + *ptr = (void *)self->ob_bytes; + return Py_SIZE(self); +} + +static Py_ssize_t +bytes_buffer_getsegcount(PyBytesObject *self, Py_ssize_t *lenp) +{ + if ( lenp ) + *lenp = Py_SIZE(self); + return 1; +} + +static Py_ssize_t +bytes_buffer_getcharbuf(PyBytesObject *self, Py_ssize_t index, const char **ptr) +{ + if ( index != 0 ) { + PyErr_SetString(PyExc_SystemError, + "accessing non-existent bytes segment"); + return -1; + } + *ptr = self->ob_bytes; + return Py_SIZE(self); +} + +static int +bytes_getbuffer(PyBytesObject *obj, Py_buffer *view, int flags) +{ + int ret; + void *ptr; + if (view == NULL) { + obj->ob_exports++; + return 0; + } + if (obj->ob_bytes == NULL) + ptr = ""; + else + ptr = obj->ob_bytes; + ret = PyBuffer_FillInfo(view, ptr, Py_SIZE(obj), 0, flags); + if (ret >= 0) { + obj->ob_exports++; + } + return ret; +} + +static void +bytes_releasebuffer(PyBytesObject *obj, Py_buffer *view) +{ + obj->ob_exports--; +} + +static Py_ssize_t +_getbuffer(PyObject *obj, Py_buffer *view) +{ + PyBufferProcs *buffer = Py_TYPE(obj)->tp_as_buffer; + + if (buffer == NULL || buffer->bf_getbuffer == NULL) + { + PyErr_Format(PyExc_TypeError, + "Type %.100s doesn't support the buffer API", + Py_TYPE(obj)->tp_name); + return -1; + } + + if (buffer->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0) + return -1; + return view->len; +} + +/* Direct API functions */ + +PyObject * +PyBytes_FromObject(PyObject *input) +{ + return PyObject_CallFunctionObjArgs((PyObject *)&PyBytes_Type, + input, NULL); +} + +PyObject * +PyBytes_FromStringAndSize(const char *bytes, Py_ssize_t size) +{ + PyBytesObject *new; + Py_ssize_t alloc; + + assert(size >= 0); + + new = PyObject_New(PyBytesObject, &PyBytes_Type); + if (new == NULL) + return NULL; + + if (size == 0) { + new->ob_bytes = NULL; + alloc = 0; + } + else { + alloc = size + 1; + new->ob_bytes = PyMem_Malloc(alloc); + if (new->ob_bytes == NULL) { + Py_DECREF(new); + return PyErr_NoMemory(); + } + if (bytes != NULL) + memcpy(new->ob_bytes, bytes, size); + new->ob_bytes[size] = '\0'; /* Trailing null byte */ + } + Py_SIZE(new) = size; + new->ob_alloc = alloc; + new->ob_exports = 0; + + return (PyObject *)new; +} + +Py_ssize_t +PyBytes_Size(PyObject *self) +{ + assert(self != NULL); + assert(PyBytes_Check(self)); + + return PyBytes_GET_SIZE(self); +} + +char * +PyBytes_AsString(PyObject *self) +{ + assert(self != NULL); + assert(PyBytes_Check(self)); + + return PyBytes_AS_STRING(self); +} + +int +PyBytes_Resize(PyObject *self, Py_ssize_t size) +{ + void *sval; + Py_ssize_t alloc = ((PyBytesObject *)self)->ob_alloc; + + assert(self != NULL); + assert(PyBytes_Check(self)); + assert(size >= 0); + + if (size < alloc / 2) { + /* Major downsize; resize down to exact size */ + alloc = size + 1; + } + else if (size < alloc) { + /* Within allocated size; quick exit */ + Py_SIZE(self) = size; + ((PyBytesObject *)self)->ob_bytes[size] = '\0'; /* Trailing null */ + return 0; + } + else if (size <= alloc * 1.125) { + /* Moderate upsize; overallocate similar to list_resize() */ + alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + } + else { + /* Major upsize; resize up to exact size */ + alloc = size + 1; + } + + if (((PyBytesObject *)self)->ob_exports > 0) { + /* + fprintf(stderr, "%d: %s", ((PyBytesObject *)self)->ob_exports, + ((PyBytesObject *)self)->ob_bytes); + */ + PyErr_SetString(PyExc_BufferError, + "Existing exports of data: object cannot be re-sized"); + return -1; + } + + sval = PyMem_Realloc(((PyBytesObject *)self)->ob_bytes, alloc); + if (sval == NULL) { + PyErr_NoMemory(); + return -1; + } + + ((PyBytesObject *)self)->ob_bytes = sval; + Py_SIZE(self) = size; + ((PyBytesObject *)self)->ob_alloc = alloc; + ((PyBytesObject *)self)->ob_bytes[size] = '\0'; /* Trailing null byte */ + + return 0; +} + +PyObject * +PyBytes_Concat(PyObject *a, PyObject *b) +{ + Py_ssize_t size; + Py_buffer va, vb; + PyBytesObject *result = NULL; + + va.len = -1; + vb.len = -1; + if (_getbuffer(a, &va) < 0 || + _getbuffer(b, &vb) < 0) { + PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", + Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + goto done; + } + + size = va.len + vb.len; + if (size < 0) { + return PyErr_NoMemory(); + goto done; + } + + result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, size); + if (result != NULL) { + memcpy(result->ob_bytes, va.buf, va.len); + memcpy(result->ob_bytes + va.len, vb.buf, vb.len); + } + + done: + if (va.len != -1) + PyObject_ReleaseBuffer(a, &va); + if (vb.len != -1) + PyObject_ReleaseBuffer(b, &vb); + return (PyObject *)result; +} + +/* Functions stuffed into the type object */ + +static Py_ssize_t +bytes_length(PyBytesObject *self) +{ + return Py_SIZE(self); +} + +static PyObject * +bytes_iconcat(PyBytesObject *self, PyObject *other) +{ + Py_ssize_t mysize; + Py_ssize_t size; + Py_buffer vo; + + if (_getbuffer(other, &vo) < 0) { + PyErr_Format(PyExc_TypeError, "can't concat bytes to %.100s", + Py_TYPE(self)->tp_name); + return NULL; + } + + mysize = Py_SIZE(self); + size = mysize + vo.len; + if (size < 0) { + PyObject_ReleaseBuffer(other, &vo); + return PyErr_NoMemory(); + } + if (size < self->ob_alloc) { + Py_SIZE(self) = size; + self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ + } + else if (PyBytes_Resize((PyObject *)self, size) < 0) { + PyObject_ReleaseBuffer(other, &vo); + return NULL; + } + memcpy(self->ob_bytes + mysize, vo.buf, vo.len); + PyObject_ReleaseBuffer(other, &vo); + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +bytes_repeat(PyBytesObject *self, Py_ssize_t count) +{ + PyBytesObject *result; + Py_ssize_t mysize; + Py_ssize_t size; + + if (count < 0) + count = 0; + mysize = Py_SIZE(self); + size = mysize * count; + if (count != 0 && size / count != mysize) + return PyErr_NoMemory(); + result = (PyBytesObject *)PyBytes_FromStringAndSize(NULL, size); + if (result != NULL && size != 0) { + if (mysize == 1) + memset(result->ob_bytes, self->ob_bytes[0], size); + else { + Py_ssize_t i; + for (i = 0; i < count; i++) + memcpy(result->ob_bytes + i*mysize, self->ob_bytes, mysize); + } + } + return (PyObject *)result; +} + +static PyObject * +bytes_irepeat(PyBytesObject *self, Py_ssize_t count) +{ + Py_ssize_t mysize; + Py_ssize_t size; + + if (count < 0) + count = 0; + mysize = Py_SIZE(self); + size = mysize * count; + if (count != 0 && size / count != mysize) + return PyErr_NoMemory(); + if (size < self->ob_alloc) { + Py_SIZE(self) = size; + self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ + } + else if (PyBytes_Resize((PyObject *)self, size) < 0) + return NULL; + + if (mysize == 1) + memset(self->ob_bytes, self->ob_bytes[0], size); + else { + Py_ssize_t i; + for (i = 1; i < count; i++) + memcpy(self->ob_bytes + i*mysize, self->ob_bytes, mysize); + } + + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +bytes_getitem(PyBytesObject *self, Py_ssize_t i) +{ + if (i < 0) + i += Py_SIZE(self); + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return NULL; + } + return PyInt_FromLong((unsigned char)(self->ob_bytes[i])); +} + +static PyObject * +bytes_subscript(PyBytesObject *self, PyObject *item) +{ + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if (i == -1 && PyErr_Occurred()) + return NULL; + + if (i < 0) + i += PyBytes_GET_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return NULL; + } + return PyInt_FromLong((unsigned char)(self->ob_bytes[i])); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength, cur, i; + if (PySlice_GetIndicesEx((PySliceObject *)item, + PyBytes_GET_SIZE(self), + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (slicelength <= 0) + return PyBytes_FromStringAndSize("", 0); + else if (step == 1) { + return PyBytes_FromStringAndSize(self->ob_bytes + start, + slicelength); + } + else { + char *source_buf = PyBytes_AS_STRING(self); + char *result_buf = (char *)PyMem_Malloc(slicelength); + PyObject *result; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + result_buf[i] = source_buf[cur]; + } + result = PyBytes_FromStringAndSize(result_buf, slicelength); + PyMem_Free(result_buf); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, "bytearray indices must be integers"); + return NULL; + } +} + +static int +bytes_setslice(PyBytesObject *self, Py_ssize_t lo, Py_ssize_t hi, + PyObject *values) +{ + Py_ssize_t avail, needed; + void *bytes; + Py_buffer vbytes; + int res = 0; + + vbytes.len = -1; + if (values == (PyObject *)self) { + /* Make a copy and call this function recursively */ + int err; + values = PyBytes_FromObject(values); + if (values == NULL) + return -1; + err = bytes_setslice(self, lo, hi, values); + Py_DECREF(values); + return err; + } + if (values == NULL) { + /* del b[lo:hi] */ + bytes = NULL; + needed = 0; + } + else { + if (_getbuffer(values, &vbytes) < 0) { + PyErr_Format(PyExc_TypeError, + "can't set bytes slice from %.100s", + Py_TYPE(values)->tp_name); + return -1; + } + needed = vbytes.len; + bytes = vbytes.buf; + } + + if (lo < 0) + lo = 0; + if (hi < lo) + hi = lo; + if (hi > Py_SIZE(self)) + hi = Py_SIZE(self); + + avail = hi - lo; + if (avail < 0) + lo = hi = avail = 0; + + if (avail != needed) { + if (avail > needed) { + /* + 0 lo hi old_size + | |<----avail----->|<-----tomove------>| + | |<-needed->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, + Py_SIZE(self) - hi); + } + /* XXX(nnorwitz): need to verify this can't overflow! */ + if (PyBytes_Resize((PyObject *)self, + Py_SIZE(self) + needed - avail) < 0) { + res = -1; + goto finish; + } + if (avail < needed) { + /* + 0 lo hi old_size + | |<-avail->|<-----tomove------>| + | |<----needed---->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, + Py_SIZE(self) - lo - needed); + } + } + + if (needed > 0) + memcpy(self->ob_bytes + lo, bytes, needed); + + + finish: + if (vbytes.len != -1) + PyObject_ReleaseBuffer(values, &vbytes); + return res; +} + +static int +bytes_setitem(PyBytesObject *self, Py_ssize_t i, PyObject *value) +{ + Py_ssize_t ival; + + if (i < 0) + i += Py_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return -1; + } + + if (value == NULL) + return bytes_setslice(self, i, i+1, NULL); + + if (!_getbytevalue(value, &ival)) + return -1; +#if 0 + ival = PyNumber_AsSsize_t(value, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) + return -1; + + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; + } +#endif + + self->ob_bytes[i] = ival; + return 0; +} + +static int +bytes_ass_subscript(PyBytesObject *self, PyObject *item, PyObject *values) +{ + Py_ssize_t start, stop, step, slicelen, needed; + char *bytes; + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if (i == -1 && PyErr_Occurred()) + return -1; + + if (i < 0) + i += PyBytes_GET_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return -1; + } + + if (values == NULL) { + /* Fall through to slice assignment */ + start = i; + stop = i + 1; + step = 1; + slicelen = 1; + } + else { + Py_ssize_t ival = PyNumber_AsSsize_t(values, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) { + /* Also accept str of size 1 in 2.x */ + PyErr_Clear(); + if (!_getbytevalue(values, &ival)) + return -1; + } + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, + "byte must be in range(0, 256)"); + return -1; + } + self->ob_bytes[i] = (char)ival; + return 0; + } + } + else if (PySlice_Check(item)) { + if (PySlice_GetIndicesEx((PySliceObject *)item, + PyBytes_GET_SIZE(self), + &start, &stop, &step, &slicelen) < 0) { + return -1; + } + } + else { + PyErr_SetString(PyExc_TypeError, "bytearray indices must be integer"); + return -1; + } + + if (values == NULL) { + bytes = NULL; + needed = 0; + } + else if (values == (PyObject *)self || !PyBytes_Check(values)) { + /* Make a copy an call this function recursively */ + int err; + values = PyBytes_FromObject(values); + if (values == NULL) + return -1; + err = bytes_ass_subscript(self, item, values); + Py_DECREF(values); + return err; + } + else { + assert(PyBytes_Check(values)); + bytes = ((PyBytesObject *)values)->ob_bytes; + needed = Py_SIZE(values); + } + /* Make sure b[5:2] = ... inserts before 5, not before 2. */ + if ((step < 0 && start < stop) || + (step > 0 && start > stop)) + stop = start; + if (step == 1) { + if (slicelen != needed) { + if (slicelen > needed) { + /* + 0 start stop old_size + | |<---slicelen--->|<-----tomove------>| + | |<-needed->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, + Py_SIZE(self) - stop); + } + if (PyBytes_Resize((PyObject *)self, + Py_SIZE(self) + needed - slicelen) < 0) + return -1; + if (slicelen < needed) { + /* + 0 lo hi old_size + | |<-avail->|<-----tomove------>| + | |<----needed---->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, + Py_SIZE(self) - start - needed); + } + } + + if (needed > 0) + memcpy(self->ob_bytes + start, bytes, needed); + + return 0; + } + else { + if (needed == 0) { + /* Delete slice */ + Py_ssize_t cur, i; + + if (step < 0) { + stop = start + 1; + start = stop + step * (slicelen - 1) - 1; + step = -step; + } + for (cur = start, i = 0; + i < slicelen; cur += step, i++) { + Py_ssize_t lim = step - 1; + + if (cur + step >= PyBytes_GET_SIZE(self)) + lim = PyBytes_GET_SIZE(self) - cur - 1; + + memmove(self->ob_bytes + cur - i, + self->ob_bytes + cur + 1, lim); + } + /* Move the tail of the bytes, in one chunk */ + cur = start + slicelen*step; + if (cur < PyBytes_GET_SIZE(self)) { + memmove(self->ob_bytes + cur - slicelen, + self->ob_bytes + cur, + PyBytes_GET_SIZE(self) - cur); + } + if (PyBytes_Resize((PyObject *)self, + PyBytes_GET_SIZE(self) - slicelen) < 0) + return -1; + + return 0; + } + else { + /* Assign slice */ + Py_ssize_t cur, i; + + if (needed != slicelen) { + PyErr_Format(PyExc_ValueError, + "attempt to assign bytes of size %zd " + "to extended slice of size %zd", + needed, slicelen); + return -1; + } + for (cur = start, i = 0; i < slicelen; cur += step, i++) + self->ob_bytes[cur] = bytes[i]; + return 0; + } + } +} + +static int +bytes_init(PyBytesObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"source", "encoding", "errors", 0}; + PyObject *arg = NULL; + const char *encoding = NULL; + const char *errors = NULL; + Py_ssize_t count; + PyObject *it; + PyObject *(*iternext)(PyObject *); + + if (Py_SIZE(self) != 0) { + /* Empty previous contents (yes, do this first of all!) */ + if (PyBytes_Resize((PyObject *)self, 0) < 0) + return -1; + } + + /* Parse arguments */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:bytes", kwlist, + &arg, &encoding, &errors)) + return -1; + + /* Make a quick exit if no first argument */ + if (arg == NULL) { + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without sequence argument"); + return -1; + } + return 0; + } + + if (PyString_Check(arg)) { + PyObject *new, *encoded; + if (encoding != NULL) { + encoded = PyCodec_Encode(arg, encoding, errors); + if (encoded == NULL) + return -1; + assert(PyString_Check(encoded)); + } + else { + encoded = arg; + Py_INCREF(arg); + } + new = bytes_iconcat(self, arg); + Py_DECREF(encoded); + if (new == NULL) + return -1; + Py_DECREF(new); + return 0; + } + + if (PyUnicode_Check(arg)) { + /* Encode via the codec registry */ + PyObject *encoded, *new; + if (encoding == NULL) { + PyErr_SetString(PyExc_TypeError, + "unicode argument without an encoding"); + return -1; + } + encoded = PyCodec_Encode(arg, encoding, errors); + if (encoded == NULL) + return -1; + assert(PyString_Check(encoded)); + new = bytes_iconcat(self, encoded); + Py_DECREF(encoded); + if (new == NULL) + return -1; + Py_DECREF(new); + return 0; + } + + /* If it's not unicode, there can't be encoding or errors */ + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without a string argument"); + return -1; + } + + /* Is it an int? */ + count = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (count == -1 && PyErr_Occurred()) + PyErr_Clear(); + else { + if (count < 0) { + PyErr_SetString(PyExc_ValueError, "negative count"); + return -1; + } + if (count > 0) { + if (PyBytes_Resize((PyObject *)self, count)) + return -1; + memset(self->ob_bytes, 0, count); + } + return 0; + } + + /* Use the buffer API */ + if (PyObject_CheckBuffer(arg)) { + Py_ssize_t size; + Py_buffer view; + if (PyObject_GetBuffer(arg, &view, PyBUF_FULL_RO) < 0) + return -1; + size = view.len; + if (PyBytes_Resize((PyObject *)self, size) < 0) goto fail; + if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) + goto fail; + PyObject_ReleaseBuffer(arg, &view); + return 0; + fail: + PyObject_ReleaseBuffer(arg, &view); + return -1; + } + + /* XXX Optimize this if the arguments is a list, tuple */ + + /* Get the iterator */ + it = PyObject_GetIter(arg); + if (it == NULL) + return -1; + iternext = *Py_TYPE(it)->tp_iternext; + + /* Run the iterator to exhaustion */ + for (;;) { + PyObject *item; + Py_ssize_t value; + + /* Get the next item */ + item = iternext(it); + if (item == NULL) { + if (PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_StopIteration)) + goto error; + PyErr_Clear(); + } + break; + } + + /* Interpret it as an int (__index__) */ + value = PyNumber_AsSsize_t(item, PyExc_ValueError); + Py_DECREF(item); + if (value == -1 && PyErr_Occurred()) + goto error; + + /* Range check */ + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; + } + + /* Append the byte */ + if (Py_SIZE(self) < self->ob_alloc) + Py_SIZE(self)++; + else if (PyBytes_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) + goto error; + self->ob_bytes[Py_SIZE(self)-1] = value; + } + + /* Clean up and return success */ + Py_DECREF(it); + return 0; + + error: + /* Error handling when it != NULL */ + Py_DECREF(it); + return -1; +} + +/* Mostly copied from string_repr, but without the + "smart quote" functionality. */ +static PyObject * +bytes_repr(PyBytesObject *self) +{ + static const char *hexdigits = "0123456789abcdef"; + const char *quote_prefix = "bytearray(b"; + const char *quote_postfix = ")"; + Py_ssize_t length = Py_SIZE(self); + /* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */ + size_t newsize = 14 + 4 * length; + PyObject *v; + if (newsize > PY_SSIZE_T_MAX || newsize / 4 - 3 != length) { + PyErr_SetString(PyExc_OverflowError, + "bytearray object is too large to make repr"); + return NULL; + } + v = PyUnicode_FromUnicode(NULL, newsize); + if (v == NULL) { + return NULL; + } + else { + register Py_ssize_t i; + register Py_UNICODE c; + register Py_UNICODE *p; + int quote; + + /* Figure out which quote to use; single is preferred */ + quote = '\''; + { + char *test, *start; + start = PyBytes_AS_STRING(self); + for (test = start; test < start+length; ++test) { + if (*test == '"') { + quote = '\''; /* back to single */ + goto decided; + } + else if (*test == '\'') + quote = '"'; + } + decided: + ; + } + + p = PyUnicode_AS_UNICODE(v); + while (*quote_prefix) + *p++ = *quote_prefix++; + *p++ = quote; + + for (i = 0; i < length; i++) { + /* There's at least enough room for a hex escape + and a closing quote. */ + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); + c = self->ob_bytes[i]; + if (c == '\'' || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c == 0) + *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigits[(c & 0xf0) >> 4]; + *p++ = hexdigits[c & 0xf]; + } + else + *p++ = c; + } + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); + *p++ = quote; + while (*quote_postfix) { + *p++ = *quote_postfix++; + } + *p = '\0'; + if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { + Py_DECREF(v); + return NULL; + } + return v; + } +} + +static PyObject * +bytes_str(PyObject *op) +{ +#if 0 + if (Py_BytesWarningFlag) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "str() on a bytearray instance", 1)) + return NULL; + } + return bytes_repr((PyBytesObject*)op); +#endif + return PyString_FromStringAndSize(((PyBytesObject*)op)->ob_bytes, Py_SIZE(op)); +} + +static PyObject * +bytes_richcompare(PyObject *self, PyObject *other, int op) +{ + Py_ssize_t self_size, other_size; + Py_buffer self_bytes, other_bytes; + PyObject *res; + Py_ssize_t minsize; + int cmp; + + /* Bytes can be compared to anything that supports the (binary) + buffer API. Except that a comparison with Unicode is always an + error, even if the comparison is for equality. */ + if (PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type) || + PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type)) { + if (Py_BytesWarningFlag && op == Py_EQ) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "Comparsion between bytearray and string", 1)) + return NULL; + } + + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + self_size = _getbuffer(self, &self_bytes); + if (self_size < 0) { + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + other_size = _getbuffer(other, &other_bytes); + if (other_size < 0) { + PyErr_Clear(); + PyObject_ReleaseBuffer(self, &self_bytes); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + if (self_size != other_size && (op == Py_EQ || op == Py_NE)) { + /* Shortcut: if the lengths differ, the objects differ */ + cmp = (op == Py_NE); + } + else { + minsize = self_size; + if (other_size < minsize) + minsize = other_size; + + cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize); + /* In ISO C, memcmp() guarantees to use unsigned bytes! */ + + if (cmp == 0) { + if (self_size < other_size) + cmp = -1; + else if (self_size > other_size) + cmp = 1; + } + + switch (op) { + case Py_LT: cmp = cmp < 0; break; + case Py_LE: cmp = cmp <= 0; break; + case Py_EQ: cmp = cmp == 0; break; + case Py_NE: cmp = cmp != 0; break; + case Py_GT: cmp = cmp > 0; break; + case Py_GE: cmp = cmp >= 0; break; + } + } + + res = cmp ? Py_True : Py_False; + PyObject_ReleaseBuffer(self, &self_bytes); + PyObject_ReleaseBuffer(other, &other_bytes); + Py_INCREF(res); + return res; +} + +static void +bytes_dealloc(PyBytesObject *self) +{ + if (self->ob_bytes != 0) { + PyMem_Free(self->ob_bytes); + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + + +/* -------------------------------------------------------------------- */ +/* Methods */ + +#define STRINGLIB_CHAR char +#define STRINGLIB_CMP memcmp +#define STRINGLIB_LEN PyBytes_GET_SIZE +#define STRINGLIB_STR PyBytes_AS_STRING +#define STRINGLIB_NEW PyBytes_FromStringAndSize +#define STRINGLIB_EMPTY nullbytes +#define STRINGLIB_CHECK_EXACT PyBytes_CheckExact +#define STRINGLIB_MUTABLE 1 + +#include "stringlib/fastsearch.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/partition.h" +#include "stringlib/ctype.h" +#include "stringlib/transmogrify.h" + + +/* The following Py_LOCAL_INLINE and Py_LOCAL functions +were copied from the old char* style string object. */ + +Py_LOCAL_INLINE(void) +_adjust_indices(Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t len) +{ + if (*end > len) + *end = len; + else if (*end < 0) + *end += len; + if (*end < 0) + *end = 0; + if (*start < 0) + *start += len; + if (*start < 0) + *start = 0; +} + + +Py_LOCAL_INLINE(Py_ssize_t) +bytes_find_internal(PyBytesObject *self, PyObject *args, int dir) +{ + PyObject *subobj; + Py_buffer subbuf; + Py_ssize_t start=0, end=PY_SSIZE_T_MAX; + Py_ssize_t res; + + if (!PyArg_ParseTuple(args, "O|O&O&:find/rfind/index/rindex", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return -2; + if (_getbuffer(subobj, &subbuf) < 0) + return -2; + if (dir > 0) + res = stringlib_find_slice( + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subbuf.buf, subbuf.len, start, end); + else + res = stringlib_rfind_slice( + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + subbuf.buf, subbuf.len, start, end); + PyObject_ReleaseBuffer(subobj, &subbuf); + return res; +} + +PyDoc_STRVAR(find__doc__, +"B.find(sub [,start [,end]]) -> int\n\ +\n\ +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within s[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +static PyObject * +bytes_find(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, +1); + if (result == -2) + return NULL; + return PyInt_FromSsize_t(result); +} + +PyDoc_STRVAR(count__doc__, +"B.count(sub [,start [,end]]) -> int\n\ +\n\ +Return the number of non-overlapping occurrences of subsection sub in\n\ +bytes B[start:end]. Optional arguments start and end are interpreted\n\ +as in slice notation."); + +static PyObject * +bytes_count(PyBytesObject *self, PyObject *args) +{ + PyObject *sub_obj; + const char *str = PyBytes_AS_STRING(self); + Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + Py_buffer vsub; + PyObject *count_obj; + + if (!PyArg_ParseTuple(args, "O|O&O&:count", &sub_obj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + + if (_getbuffer(sub_obj, &vsub) < 0) + return NULL; + + _adjust_indices(&start, &end, PyBytes_GET_SIZE(self)); + + count_obj = PyInt_FromSsize_t( + stringlib_count(str + start, end - start, vsub.buf, vsub.len) + ); + PyObject_ReleaseBuffer(sub_obj, &vsub); + return count_obj; +} + + +PyDoc_STRVAR(index__doc__, +"B.index(sub [,start [,end]]) -> int\n\ +\n\ +Like B.find() but raise ValueError when the subsection is not found."); + +static PyObject * +bytes_index(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, +1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyInt_FromSsize_t(result); +} + + +PyDoc_STRVAR(rfind__doc__, +"B.rfind(sub [,start [,end]]) -> int\n\ +\n\ +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within s[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +static PyObject * +bytes_rfind(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, -1); + if (result == -2) + return NULL; + return PyInt_FromSsize_t(result); +} + + +PyDoc_STRVAR(rindex__doc__, +"B.rindex(sub [,start [,end]]) -> int\n\ +\n\ +Like B.rfind() but raise ValueError when the subsection is not found."); + +static PyObject * +bytes_rindex(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, -1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyInt_FromSsize_t(result); +} + + +static int +bytes_contains(PyObject *self, PyObject *arg) +{ + Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) { + Py_buffer varg; + int pos; + PyErr_Clear(); + if (_getbuffer(arg, &varg) < 0) + return -1; + pos = stringlib_find(PyBytes_AS_STRING(self), Py_SIZE(self), + varg.buf, varg.len, 0); + PyObject_ReleaseBuffer(arg, &varg); + return pos >= 0; + } + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; + } + + return memchr(PyBytes_AS_STRING(self), ival, Py_SIZE(self)) != NULL; +} + + +/* Matches the end (direction >= 0) or start (direction < 0) of self + * against substr, using the start and end arguments. Returns + * -1 on error, 0 if not found and 1 if found. + */ +Py_LOCAL(int) +_bytes_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start, + Py_ssize_t end, int direction) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self); + const char* str; + Py_buffer vsubstr; + int rv = 0; + + str = PyBytes_AS_STRING(self); + + if (_getbuffer(substr, &vsubstr) < 0) + return -1; + + _adjust_indices(&start, &end, len); + + if (direction < 0) { + /* startswith */ + if (start+vsubstr.len > len) { + goto done; + } + } else { + /* endswith */ + if (end-start < vsubstr.len || start > len) { + goto done; + } + + if (end-vsubstr.len > start) + start = end - vsubstr.len; + } + if (end-start >= vsubstr.len) + rv = ! memcmp(str+start, vsubstr.buf, vsubstr.len); + +done: + PyObject_ReleaseBuffer(substr, &vsubstr); + return rv; +} + + +PyDoc_STRVAR(startswith__doc__, +"B.startswith(prefix [,start [,end]]) -> bool\n\ +\n\ +Return True if B starts with the specified prefix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +prefix can also be a tuple of strings to try."); + +static PyObject * +bytes_startswith(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _bytes_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, -1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _bytes_tailmatch(self, subobj, start, end, -1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); +} + +PyDoc_STRVAR(endswith__doc__, +"B.endswith(suffix [,start [,end]]) -> bool\n\ +\n\ +Return True if B ends with the specified suffix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +suffix can also be a tuple of strings to try."); + +static PyObject * +bytes_endswith(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _bytes_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, +1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _bytes_tailmatch(self, subobj, start, end, +1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); +} + + +PyDoc_STRVAR(translate__doc__, +"B.translate(table[, deletechars]) -> bytearray\n\ +\n\ +Return a copy of B, where all characters occurring in the\n\ +optional argument deletechars are removed, and the remaining\n\ +characters have been mapped through the given translation\n\ +table, which must be a bytes object of length 256."); + +static PyObject * +bytes_translate(PyBytesObject *self, PyObject *args) +{ + register char *input, *output; + register const char *table; + register Py_ssize_t i, c, changed = 0; + PyObject *input_obj = (PyObject*)self; + const char *output_start; + Py_ssize_t inlen; + PyObject *result; + int trans_table[256]; + PyObject *tableobj, *delobj = NULL; + Py_buffer vtable, vdel; + + if (!PyArg_UnpackTuple(args, "translate", 1, 2, + &tableobj, &delobj)) + return NULL; + + if (_getbuffer(tableobj, &vtable) < 0) + return NULL; + + if (vtable.len != 256) { + PyErr_SetString(PyExc_ValueError, + "translation table must be 256 characters long"); + result = NULL; + goto done; + } + + if (delobj != NULL) { + if (_getbuffer(delobj, &vdel) < 0) { + result = NULL; + goto done; + } + } + else { + vdel.buf = NULL; + vdel.len = 0; + } + + table = (const char *)vtable.buf; + inlen = PyBytes_GET_SIZE(input_obj); + result = PyBytes_FromStringAndSize((char *)NULL, inlen); + if (result == NULL) + goto done; + output_start = output = PyBytes_AsString(result); + input = PyBytes_AS_STRING(input_obj); + + if (vdel.len == 0) { + /* If no deletions are required, use faster code */ + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (Py_CHARMASK((*output++ = table[c])) != c) + changed = 1; + } + if (changed || !PyBytes_CheckExact(input_obj)) + goto done; + Py_DECREF(result); + Py_INCREF(input_obj); + result = input_obj; + goto done; + } + + for (i = 0; i < 256; i++) + trans_table[i] = Py_CHARMASK(table[i]); + + for (i = 0; i < vdel.len; i++) + trans_table[(int) Py_CHARMASK( ((unsigned char*)vdel.buf)[i] )] = -1; + + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (trans_table[c] != -1) + if (Py_CHARMASK(*output++ = (char)trans_table[c]) == c) + continue; + changed = 1; + } + if (!changed && PyBytes_CheckExact(input_obj)) { + Py_DECREF(result); + Py_INCREF(input_obj); + result = input_obj; + goto done; + } + /* Fix the size of the resulting string */ + if (inlen > 0) + PyBytes_Resize(result, output - output_start); + +done: + PyObject_ReleaseBuffer(tableobj, &vtable); + if (delobj != NULL) + PyObject_ReleaseBuffer(delobj, &vdel); + return result; +} + + +#define FORWARD 1 +#define REVERSE -1 + +/* find and count characters and substrings */ + +#define findchar(target, target_len, c) \ + ((char *)memchr((const void *)(target), c, target_len)) + +/* Don't call if length < 2 */ +#define Py_STRING_MATCH(target, offset, pattern, length) \ + (target[offset] == pattern[0] && \ + target[offset+length-1] == pattern[length-1] && \ + !memcmp(target+offset+1, pattern+1, length-2) ) + + +/* Bytes ops must return a string. */ +/* If the object is subclass of bytes, create a copy */ +Py_LOCAL(PyBytesObject *) +return_self(PyBytesObject *self) +{ + if (PyBytes_CheckExact(self)) { + Py_INCREF(self); + return (PyBytesObject *)self; + } + return (PyBytesObject *)PyBytes_FromStringAndSize( + PyBytes_AS_STRING(self), + PyBytes_GET_SIZE(self)); +} + +Py_LOCAL_INLINE(Py_ssize_t) +countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) +{ + Py_ssize_t count=0; + const char *start=target; + const char *end=target+target_len; + + while ( (start=findchar(start, end-start, c)) != NULL ) { + count++; + if (count >= maxcount) + break; + start += 1; + } + return count; +} + +Py_LOCAL(Py_ssize_t) +findstring(const char *target, Py_ssize_t target_len, + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } + + /* zero-length substrings always match at the first attempt */ + if (pattern_len == 0) + return (direction > 0) ? start : end; + + end -= pattern_len; + + if (direction < 0) { + for (; end >= start; end--) + if (Py_STRING_MATCH(target, end, pattern, pattern_len)) + return end; + } else { + for (; start <= end; start++) + if (Py_STRING_MATCH(target, start, pattern, pattern_len)) + return start; + } + return -1; +} + +Py_LOCAL_INLINE(Py_ssize_t) +countstring(const char *target, Py_ssize_t target_len, + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction, Py_ssize_t maxcount) +{ + Py_ssize_t count=0; + + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } + + /* zero-length substrings match everywhere */ + if (pattern_len == 0 || maxcount == 0) { + if (target_len+1 < maxcount) + return target_len+1; + return maxcount; + } + + end -= pattern_len; + if (direction < 0) { + for (; (end >= start); end--) + if (Py_STRING_MATCH(target, end, pattern, pattern_len)) { + count++; + if (--maxcount <= 0) break; + end -= pattern_len-1; + } + } else { + for (; (start <= end); start++) + if (Py_STRING_MATCH(target, start, pattern, pattern_len)) { + count++; + if (--maxcount <= 0) + break; + start += pattern_len-1; + } + } + return count; +} + + +/* Algorithms for different cases of string replacement */ + +/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_interleave(PyBytesObject *self, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, i, product; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + + /* 1 at the end plus 1 after every character */ + count = self_len+1; + if (maxcount < count) + count = maxcount; + + /* Check for overflow */ + /* result_len = count * to_len + self_len; */ + product = count * to_len; + if (product / to_len != count) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + result_len = product + self_len; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + + if (! (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) ) + return NULL; + + self_s = PyBytes_AS_STRING(self); + result_s = PyBytes_AS_STRING(result); + + /* TODO: special case single character, which doesn't need memcpy */ + + /* Lay the first one down (guaranteed this will occur) */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + count -= 1; + + for (i=0; i<count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + + /* Copy the rest of the original string */ + Py_MEMCPY(result_s, self_s, self_len-i); + + return result; +} + +/* Special case for deleting a single character */ +/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_delete_single_character(PyBytesObject *self, + char from_c, Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + self_s = PyBytes_AS_STRING(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + return return_self(self); + } + + result_len = self_len - count; /* from_len == 1 */ + assert(result_len>=0); + + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + start = next+1; + } + Py_MEMCPY(result_s, start, end-start); + + return result; +} + +/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ + +Py_LOCAL(PyBytesObject *) +replace_delete_substring(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + self_s = PyBytes_AS_STRING(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, 1, + maxcount); + + if (count == 0) { + /* no matches */ + return return_self(self); + } + + result_len = self_len - (count * from_len); + assert (result_len>=0); + + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL ) + return NULL; + + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start + offset; + + Py_MEMCPY(result_s, start, next-start); + + result_s += (next-start); + start = next+from_len; + } + Py_MEMCPY(result_s, start, end-start); + return result; +} + +/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_single_character_in_place(PyBytesObject *self, + char from_c, char to_c, + Py_ssize_t maxcount) +{ + char *self_s, *result_s, *start, *end, *next; + Py_ssize_t self_len; + PyBytesObject *result; + + /* The result string will be the same size */ + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + next = findchar(self_s, self_len, from_c); + + if (next == NULL) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + (next-self_s); + *start = to_c; + start++; + end = result_s + self_len; + + while (--maxcount > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + *next = to_c; + start = next+1; + } + + return result; +} + +/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_substring_in_place(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *result_s, *start, *end; + char *self_s; + Py_ssize_t self_len, offset; + PyBytesObject *result; + + /* The result bytes will be the same size */ + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + offset = findstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD); + if (offset == -1) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + offset; + Py_MEMCPY(start, to_s, from_len); + start += from_len; + end = result_s + self_len; + + while ( --maxcount > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset==-1) + break; + Py_MEMCPY(start+offset, to_s, from_len); + start += offset+from_len; + } + + return result; +} + +/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_single_character(PyBytesObject *self, + char from_c, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, product; + PyBytesObject *result; + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* use the difference between current and new, hence the "-1" */ + /* result_len = self_len + count * (to_len-1) */ + product = count * (to_len-1); + if (product / (to_len-1) != count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += 1; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+1; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end-start); + + return result; +} + +/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyBytesObject *) +replace_substring(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset, product; + PyBytesObject *result; + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* Check for overflow */ + /* result_len = self_len + count * (to_len-from_len) */ + product = count * (to_len-from_len); + if (product / (to_len-from_len) != count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start+offset; + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += from_len; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+from_len; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end-start); + + return result; +} + + +Py_LOCAL(PyBytesObject *) +replace(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + if (maxcount < 0) { + maxcount = PY_SSIZE_T_MAX; + } else if (maxcount == 0 || PyBytes_GET_SIZE(self) == 0) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + + if (maxcount == 0 || + (from_len == 0 && to_len == 0)) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + + /* Handle zero-length special cases */ + + if (from_len == 0) { + /* insert the 'to' bytes everywhere. */ + /* >>> "Python".replace("", ".") */ + /* '.P.y.t.h.o.n.' */ + return replace_interleave(self, to_s, to_len, maxcount); + } + + /* Except for "".replace("", "A") == "A" there is no way beyond this */ + /* point for an empty self bytes to generate a non-empty bytes */ + /* Special case so the remaining code always gets a non-empty bytes */ + if (PyBytes_GET_SIZE(self) == 0) { + return return_self(self); + } + + if (to_len == 0) { + /* delete all occurances of 'from' bytes */ + if (from_len == 1) { + return replace_delete_single_character( + self, from_s[0], maxcount); + } else { + return replace_delete_substring(self, from_s, from_len, maxcount); + } + } + + /* Handle special case where both bytes have the same length */ + + if (from_len == to_len) { + if (from_len == 1) { + return replace_single_character_in_place( + self, + from_s[0], + to_s[0], + maxcount); + } else { + return replace_substring_in_place( + self, from_s, from_len, to_s, to_len, maxcount); + } + } + + /* Otherwise use the more generic algorithms */ + if (from_len == 1) { + return replace_single_character(self, from_s[0], + to_s, to_len, maxcount); + } else { + /* len('from')>=2, len('to')>=1 */ + return replace_substring(self, from_s, from_len, to_s, to_len, maxcount); + } +} + + +PyDoc_STRVAR(replace__doc__, +"B.replace(old, new[, count]) -> bytes\n\ +\n\ +Return a copy of B with all occurrences of subsection\n\ +old replaced by new. If the optional argument count is\n\ +given, only the first count occurrences are replaced."); + +static PyObject * +bytes_replace(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t count = -1; + PyObject *from, *to, *res; + Py_buffer vfrom, vto; + + if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) + return NULL; + + if (_getbuffer(from, &vfrom) < 0) + return NULL; + if (_getbuffer(to, &vto) < 0) { + PyObject_ReleaseBuffer(from, &vfrom); + return NULL; + } + + res = (PyObject *)replace((PyBytesObject *) self, + vfrom.buf, vfrom.len, + vto.buf, vto.len, count); + + PyObject_ReleaseBuffer(from, &vfrom); + PyObject_ReleaseBuffer(to, &vto); + return res; +} + + +/* Overallocate the initial list to reduce the number of reallocs for small + split sizes. Eg, "A A A A A A A A A A".split() (10 elements) has three + resizes, to sizes 4, 8, then 16. Most observed string splits are for human + text (roughly 11 words per line) and field delimited data (usually 1-10 + fields). For large strings the split algorithms are bandwidth limited + so increasing the preallocation likely will not improve things.*/ + +#define MAX_PREALLOC 12 + +/* 5 splits gives 6 elements */ +#define PREALLOC_SIZE(maxsplit) \ + (maxsplit >= MAX_PREALLOC ? MAX_PREALLOC : maxsplit+1) + +#define SPLIT_APPEND(data, left, right) \ + str = PyBytes_FromStringAndSize((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); + +#define SPLIT_ADD(data, left, right) { \ + str = PyBytes_FromStringAndSize((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (count < MAX_PREALLOC) { \ + PyList_SET_ITEM(list, count, str); \ + } else { \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); \ + } \ + count++; } + +/* Always force the list to the expected size. */ +#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count + + +Py_LOCAL_INLINE(PyObject *) +split_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = 0; + while ((j < len) && (maxcount-- > 0)) { + for(; j < len; j++) { + /* I found that using memchr makes no difference */ + if (s[j] == ch) { + SPLIT_ADD(s, i, j); + i = j = j + 1; + break; + } + } + } + if (i <= len) { + SPLIT_ADD(s, i, len); + } + FIX_PREALLOC_SIZE(list); + return list; + + onError: + Py_DECREF(list); + return NULL; +} + + +Py_LOCAL_INLINE(PyObject *) +split_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + for (i = j = 0; i < len; ) { + /* find a token */ + while (i < len && ISSPACE(s[i])) + i++; + j = i; + while (i < len && !ISSPACE(s[i])) + i++; + if (j < i) { + if (maxcount-- <= 0) + break; + SPLIT_ADD(s, j, i); + while (i < len && ISSPACE(s[i])) + i++; + j = i; + } + } + if (j < len) { + SPLIT_ADD(s, j, len); + } + FIX_PREALLOC_SIZE(list); + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +PyDoc_STRVAR(split__doc__, +"B.split([sep[, maxsplit]]) -> list of bytearray\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter.\n\ +If sep is not given, B is split on ASCII whitespace characters\n\ +(space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); + +static PyObject * +bytes_split(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count = 0; + const char *s = PyBytes_AS_STRING(self), *sub; + PyObject *list, *str, *subobj = Py_None; + Py_buffer vsub; +#ifdef USE_FAST + Py_ssize_t pos; +#endif + + if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + + if (subobj == Py_None) + return split_whitespace(s, len, maxsplit); + + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + if (n == 1) + return split_char(s, len, sub[0], maxsplit); + + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + +#ifdef USE_FAST + i = j = 0; + while (maxsplit-- > 0) { + pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH); + if (pos < 0) + break; + j = i+pos; + SPLIT_ADD(s, i, j); + i = j + n; + } +#else + i = j = 0; + while ((j+n <= len) && (maxsplit-- > 0)) { + for (; j+n <= len; j++) { + if (Py_STRING_MATCH(s, j, sub, n)) { + SPLIT_ADD(s, i, j); + i = j = j + n; + break; + } + } + } +#endif + SPLIT_ADD(s, i, len); + FIX_PREALLOC_SIZE(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return list; + + onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; +} + +/* stringlib's partition shares nullbytes in some cases. + undo this, we don't want the nullbytes to be shared. */ +static PyObject * +make_nullbytes_unique(PyObject *result) +{ + if (result != NULL) { + int i; + assert(PyTuple_Check(result)); + assert(PyTuple_GET_SIZE(result) == 3); + for (i = 0; i < 3; i++) { + if (PyTuple_GET_ITEM(result, i) == (PyObject *)nullbytes) { + PyObject *new = PyBytes_FromStringAndSize(NULL, 0); + if (new == NULL) { + Py_DECREF(result); + result = NULL; + break; + } + Py_DECREF(nullbytes); + PyTuple_SET_ITEM(result, i, new); + } + } + } + return result; +} + +PyDoc_STRVAR(partition__doc__, +"B.partition(sep) -> (head, sep, tail)\n\ +\n\ +Searches for the separator sep in B, and returns the part before it,\n\ +the separator itself, and the part after it. If the separator is not\n\ +found, returns B and two empty bytearray objects."); + +static PyObject * +bytes_partition(PyBytesObject *self, PyObject *sep_obj) +{ + PyObject *bytesep, *result; + + bytesep = PyBytes_FromObject(sep_obj); + if (! bytesep) + return NULL; + + result = stringlib_partition( + (PyObject*) self, + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + bytesep, + PyBytes_AS_STRING(bytesep), PyBytes_GET_SIZE(bytesep) + ); + + Py_DECREF(bytesep); + return make_nullbytes_unique(result); +} + +PyDoc_STRVAR(rpartition__doc__, +"B.rpartition(sep) -> (tail, sep, head)\n\ +\n\ +Searches for the separator sep in B, starting at the end of B,\n\ +and returns the part before it, the separator itself, and the\n\ +part after it. If the separator is not found, returns two empty\n\ +bytearray objects and B."); + +static PyObject * +bytes_rpartition(PyBytesObject *self, PyObject *sep_obj) +{ + PyObject *bytesep, *result; + + bytesep = PyBytes_FromObject(sep_obj); + if (! bytesep) + return NULL; + + result = stringlib_rpartition( + (PyObject*) self, + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + bytesep, + PyBytes_AS_STRING(bytesep), PyBytes_GET_SIZE(bytesep) + ); + + Py_DECREF(bytesep); + return make_nullbytes_unique(result); +} + +Py_LOCAL_INLINE(PyObject *) +rsplit_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = len - 1; + while ((i >= 0) && (maxcount-- > 0)) { + for (; i >= 0; i--) { + if (s[i] == ch) { + SPLIT_ADD(s, i + 1, j + 1); + j = i = i - 1; + break; + } + } + } + if (j >= -1) { + SPLIT_ADD(s, 0, j + 1); + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +Py_LOCAL_INLINE(PyObject *) +rsplit_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + for (i = j = len - 1; i >= 0; ) { + /* find a token */ + while (i >= 0 && ISSPACE(s[i])) + i--; + j = i; + while (i >= 0 && !ISSPACE(s[i])) + i--; + if (j > i) { + if (maxcount-- <= 0) + break; + SPLIT_ADD(s, i + 1, j + 1); + while (i >= 0 && ISSPACE(s[i])) + i--; + j = i; + } + } + if (j >= 0) { + SPLIT_ADD(s, 0, j + 1); + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +PyDoc_STRVAR(rsplit__doc__, +"B.rsplit(sep[, maxsplit]) -> list of bytearray\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter,\n\ +starting at the end of B and working to the front.\n\ +If sep is not given, B is split on ASCII whitespace characters\n\ +(space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); + +static PyObject * +bytes_rsplit(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count = 0; + const char *s = PyBytes_AS_STRING(self), *sub; + PyObject *list, *str, *subobj = Py_None; + Py_buffer vsub; + + if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + + if (subobj == Py_None) + return rsplit_whitespace(s, len, maxsplit); + + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + else if (n == 1) + return rsplit_char(s, len, sub[0], maxsplit); + + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + + j = len; + i = j - n; + + while ( (i >= 0) && (maxsplit-- > 0) ) { + for (; i>=0; i--) { + if (Py_STRING_MATCH(s, i, sub, n)) { + SPLIT_ADD(s, i + n, j); + j = i; + i -= n; + break; + } + } + } + SPLIT_ADD(s, 0, j); + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + PyObject_ReleaseBuffer(subobj, &vsub); + return list; + +onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; +} + +PyDoc_STRVAR(reverse__doc__, +"B.reverse() -> None\n\ +\n\ +Reverse the order of the values in B in place."); +static PyObject * +bytes_reverse(PyBytesObject *self, PyObject *unused) +{ + char swap, *head, *tail; + Py_ssize_t i, j, n = Py_SIZE(self); + + j = n / 2; + head = self->ob_bytes; + tail = head + n - 1; + for (i = 0; i < j; i++) { + swap = *head; + *head++ = *tail; + *tail-- = swap; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(insert__doc__, +"B.insert(index, int) -> None\n\ +\n\ +Insert a single item into the bytearray before the given index."); +static PyObject * +bytes_insert(PyBytesObject *self, PyObject *args) +{ + int value; + Py_ssize_t where, n = Py_SIZE(self); + + if (!PyArg_ParseTuple(args, "ni:insert", &where, &value)) + return NULL; + + if (n == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "cannot add more objects to bytes"); + return NULL; + } + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "byte must be in range(0, 256)"); + return NULL; + } + if (PyBytes_Resize((PyObject *)self, n + 1) < 0) + return NULL; + + if (where < 0) { + where += n; + if (where < 0) + where = 0; + } + if (where > n) + where = n; + memmove(self->ob_bytes + where + 1, self->ob_bytes + where, n - where); + self->ob_bytes[where] = value; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(append__doc__, +"B.append(int) -> None\n\ +\n\ +Append a single item to the end of B."); +static PyObject * +bytes_append(PyBytesObject *self, PyObject *arg) +{ + int value; + Py_ssize_t n = Py_SIZE(self); + + if (! _getbytevalue(arg, &value)) + return NULL; + if (n == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "cannot add more objects to bytes"); + return NULL; + } + if (PyBytes_Resize((PyObject *)self, n + 1) < 0) + return NULL; + + self->ob_bytes[n] = value; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(extend__doc__, +"B.extend(iterable int) -> None\n\ +\n\ +Append all the elements from the iterator or sequence to the\n\ +end of B."); +static PyObject * +bytes_extend(PyBytesObject *self, PyObject *arg) +{ + PyObject *it, *item, *tmp, *res; + Py_ssize_t buf_size = 0, len = 0; + int value; + char *buf; + + /* bytes_setslice code only accepts something supporting PEP 3118. */ + if (PyObject_CheckBuffer(arg)) { + if (bytes_setslice(self, Py_SIZE(self), Py_SIZE(self), arg) == -1) + return NULL; + + Py_RETURN_NONE; + } + + it = PyObject_GetIter(arg); + if (it == NULL) + return NULL; + + /* Try to determine the length of the argument. 32 is abitrary. */ + buf_size = _PyObject_LengthHint(arg, 32); + + buf = (char *)PyMem_Malloc(buf_size * sizeof(char)); + if (buf == NULL) + return PyErr_NoMemory(); + + while ((item = PyIter_Next(it)) != NULL) { + if (! _getbytevalue(item, &value)) { + Py_DECREF(item); + Py_DECREF(it); + return NULL; + } + buf[len++] = value; + Py_DECREF(item); + if (len >= buf_size) { + buf_size = len + (len >> 1) + 1; + buf = (char *)PyMem_Realloc(buf, buf_size * sizeof(char)); + if (buf == NULL) { + Py_DECREF(it); + return PyErr_NoMemory(); + } + } + } + Py_DECREF(it); + + /* XXX: Is possible to avoid a full copy of the buffer? */ + tmp = PyBytes_FromStringAndSize(buf, len); + res = bytes_extend(self, tmp); + Py_DECREF(tmp); + PyMem_Free(buf); + + return res; +} + +PyDoc_STRVAR(pop__doc__, +"B.pop([index]) -> int\n\ +\n\ +Remove and return a single item from B. If no index\n\ +argument is give, will pop the last value."); +static PyObject * +bytes_pop(PyBytesObject *self, PyObject *args) +{ + int value; + Py_ssize_t where = -1, n = Py_SIZE(self); + + if (!PyArg_ParseTuple(args, "|n:pop", &where)) + return NULL; + + if (n == 0) { + PyErr_SetString(PyExc_OverflowError, + "cannot pop an empty bytes"); + return NULL; + } + if (where < 0) + where += Py_SIZE(self); + if (where < 0 || where >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "pop index out of range"); + return NULL; + } + + value = self->ob_bytes[where]; + memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + if (PyBytes_Resize((PyObject *)self, n - 1) < 0) + return NULL; + + return PyInt_FromLong(value); +} + +PyDoc_STRVAR(remove__doc__, +"B.remove(int) -> None\n\ +\n\ +Remove the first occurance of a value in B."); +static PyObject * +bytes_remove(PyBytesObject *self, PyObject *arg) +{ + int value; + Py_ssize_t where, n = Py_SIZE(self); + + if (! _getbytevalue(arg, &value)) + return NULL; + + for (where = 0; where < n; where++) { + if (self->ob_bytes[where] == value) + break; + } + if (where == n) { + PyErr_SetString(PyExc_ValueError, "value not found in bytes"); + return NULL; + } + + memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + if (PyBytes_Resize((PyObject *)self, n - 1) < 0) + return NULL; + + Py_RETURN_NONE; +} + +/* XXX These two helpers could be optimized if argsize == 1 */ + +static Py_ssize_t +lstrip_helper(unsigned char *myptr, Py_ssize_t mysize, + void *argptr, Py_ssize_t argsize) +{ + Py_ssize_t i = 0; + while (i < mysize && memchr(argptr, myptr[i], argsize)) + i++; + return i; +} + +static Py_ssize_t +rstrip_helper(unsigned char *myptr, Py_ssize_t mysize, + void *argptr, Py_ssize_t argsize) +{ + Py_ssize_t i = mysize - 1; + while (i >= 0 && memchr(argptr, myptr[i], argsize)) + i--; + return i + 1; +} + +PyDoc_STRVAR(strip__doc__, +"B.strip([bytes]) -> bytearray\n\ +\n\ +Strip leading and trailing bytes contained in the argument.\n\ +If the argument is omitted, strip ASCII whitespace."); +static PyObject * +bytes_strip(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:strip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = lstrip_helper(myptr, mysize, argptr, argsize); + if (left == mysize) + right = left; + else + right = rstrip_helper(myptr, mysize, argptr, argsize); + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyBytes_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(lstrip__doc__, +"B.lstrip([bytes]) -> bytearray\n\ +\n\ +Strip leading bytes contained in the argument.\n\ +If the argument is omitted, strip leading ASCII whitespace."); +static PyObject * +bytes_lstrip(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:lstrip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = lstrip_helper(myptr, mysize, argptr, argsize); + right = mysize; + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyBytes_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(rstrip__doc__, +"B.rstrip([bytes]) -> bytearray\n\ +\n\ +Strip trailing bytes contained in the argument.\n\ +If the argument is omitted, strip trailing ASCII whitespace."); +static PyObject * +bytes_rstrip(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:rstrip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = 0; + right = rstrip_helper(myptr, mysize, argptr, argsize); + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyBytes_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(decode_doc, +"B.decode([encoding[, errors]]) -> unicode object.\n\ +\n\ +Decodes B using the codec registered for encoding. encoding defaults\n\ +to the default encoding. errors may be given to set a different error\n\ +handling scheme. Default is 'strict' meaning that encoding errors raise\n\ +a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ +as well as any other name registered with codecs.register_error that is\n\ +able to handle UnicodeDecodeErrors."); + +static PyObject * +bytes_decode(PyObject *self, PyObject *args) +{ + const char *encoding = NULL; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors)) + return NULL; + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); + return PyCodec_Decode(self, encoding, errors); +} + +PyDoc_STRVAR(alloc_doc, +"B.__alloc__() -> int\n\ +\n\ +Returns the number of bytes actually allocated."); + +static PyObject * +bytes_alloc(PyBytesObject *self) +{ + return PyInt_FromSsize_t(self->ob_alloc); +} + +PyDoc_STRVAR(join_doc, +"B.join(iterable_of_bytes) -> bytes\n\ +\n\ +Concatenates any number of bytearray objects, with B in between each pair."); + +static PyObject * +bytes_join(PyBytesObject *self, PyObject *it) +{ + PyObject *seq; + Py_ssize_t mysize = Py_SIZE(self); + Py_ssize_t i; + Py_ssize_t n; + PyObject **items; + Py_ssize_t totalsize = 0; + PyObject *result; + char *dest; + + seq = PySequence_Fast(it, "can only join an iterable"); + if (seq == NULL) + return NULL; + n = PySequence_Fast_GET_SIZE(seq); + items = PySequence_Fast_ITEMS(seq); + + /* Compute the total size, and check that they are all bytes */ + /* XXX Shouldn't we use _getbuffer() on these items instead? */ + for (i = 0; i < n; i++) { + PyObject *obj = items[i]; + if (!PyBytes_Check(obj) && !PyString_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "can only join an iterable of bytes " + "(item %ld has type '%.100s')", + /* XXX %ld isn't right on Win64 */ + (long)i, Py_TYPE(obj)->tp_name); + goto error; + } + if (i > 0) + totalsize += mysize; + totalsize += Py_SIZE(obj); + if (totalsize < 0) { + PyErr_NoMemory(); + goto error; + } + } + + /* Allocate the result, and copy the bytes */ + result = PyBytes_FromStringAndSize(NULL, totalsize); + if (result == NULL) + goto error; + dest = PyBytes_AS_STRING(result); + for (i = 0; i < n; i++) { + PyObject *obj = items[i]; + Py_ssize_t size = Py_SIZE(obj); + char *buf; + if (PyBytes_Check(obj)) + buf = PyBytes_AS_STRING(obj); + else + buf = PyString_AS_STRING(obj); + if (i) { + memcpy(dest, self->ob_bytes, mysize); + dest += mysize; + } + memcpy(dest, buf, size); + dest += size; + } + + /* Done */ + Py_DECREF(seq); + return result; + + /* Error handling */ + error: + Py_DECREF(seq); + return NULL; +} + +PyDoc_STRVAR(fromhex_doc, +"bytearray.fromhex(string) -> bytearray\n\ +\n\ +Create a bytearray object from a string of hexadecimal numbers.\n\ +Spaces between two numbers are accepted.\n\ +Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef')."); + +static int +hex_digit_to_int(Py_UNICODE c) +{ + if (c >= 128) + return -1; + if (ISDIGIT(c)) + return c - '0'; + else { + if (ISUPPER(c)) + c = TOLOWER(c); + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + } + return -1; +} + +static PyObject * +bytes_fromhex(PyObject *cls, PyObject *args) +{ + PyObject *newbytes, *hexobj; + char *buf; + Py_UNICODE *hex; + Py_ssize_t hexlen, byteslen, i, j; + int top, bot; + + if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) + return NULL; + assert(PyUnicode_Check(hexobj)); + hexlen = PyUnicode_GET_SIZE(hexobj); + hex = PyUnicode_AS_UNICODE(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ + newbytes = PyBytes_FromStringAndSize(NULL, byteslen); + if (!newbytes) + return NULL; + buf = PyBytes_AS_STRING(newbytes); + for (i = j = 0; i < hexlen; i += 2) { + /* skip over spaces in the input */ + while (hex[i] == ' ') + i++; + if (i >= hexlen) + break; + top = hex_digit_to_int(hex[i]); + bot = hex_digit_to_int(hex[i+1]); + if (top == -1 || bot == -1) { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", i); + goto error; + } + buf[j++] = (top << 4) + bot; + } + if (PyBytes_Resize(newbytes, j) < 0) + goto error; + return newbytes; + + error: + Py_DECREF(newbytes); + return NULL; +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +bytes_reduce(PyBytesObject *self) +{ + PyObject *latin1, *dict; + if (self->ob_bytes) + latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, + Py_SIZE(self), NULL); + else + latin1 = PyUnicode_FromString(""); + + dict = PyObject_GetAttrString((PyObject *)self, "__dict__"); + if (dict == NULL) { + PyErr_Clear(); + dict = Py_None; + Py_INCREF(dict); + } + + return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); +} + +static PySequenceMethods bytes_as_sequence = { + (lenfunc)bytes_length, /* sq_length */ + (binaryfunc)PyBytes_Concat, /* sq_concat */ + (ssizeargfunc)bytes_repeat, /* sq_repeat */ + (ssizeargfunc)bytes_getitem, /* sq_item */ + 0, /* sq_slice */ + (ssizeobjargproc)bytes_setitem, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc)bytes_contains, /* sq_contains */ + (binaryfunc)bytes_iconcat, /* sq_inplace_concat */ + (ssizeargfunc)bytes_irepeat, /* sq_inplace_repeat */ +}; + +static PyMappingMethods bytes_as_mapping = { + (lenfunc)bytes_length, + (binaryfunc)bytes_subscript, + (objobjargproc)bytes_ass_subscript, +}; + +static PyBufferProcs bytes_as_buffer = { + (readbufferproc)bytes_buffer_getreadbuf, + (writebufferproc)bytes_buffer_getwritebuf, + (segcountproc)bytes_buffer_getsegcount, + (charbufferproc)bytes_buffer_getcharbuf, + (getbufferproc)bytes_getbuffer, + (releasebufferproc)bytes_releasebuffer, +}; + +static PyMethodDef +bytes_methods[] = { + {"__alloc__", (PyCFunction)bytes_alloc, METH_NOARGS, alloc_doc}, + {"__reduce__", (PyCFunction)bytes_reduce, METH_NOARGS, reduce_doc}, + {"append", (PyCFunction)bytes_append, METH_O, append__doc__}, + {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, + _Py_capitalize__doc__}, + {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, + {"count", (PyCFunction)bytes_count, METH_VARARGS, count__doc__}, + {"decode", (PyCFunction)bytes_decode, METH_VARARGS, decode_doc}, + {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, endswith__doc__}, + {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS, + expandtabs__doc__}, + {"extend", (PyCFunction)bytes_extend, METH_O, extend__doc__}, + {"find", (PyCFunction)bytes_find, METH_VARARGS, find__doc__}, + {"fromhex", (PyCFunction)bytes_fromhex, METH_VARARGS|METH_CLASS, + fromhex_doc}, + {"index", (PyCFunction)bytes_index, METH_VARARGS, index__doc__}, + {"insert", (PyCFunction)bytes_insert, METH_VARARGS, insert__doc__}, + {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, + _Py_isalnum__doc__}, + {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, + _Py_isalpha__doc__}, + {"isdigit", (PyCFunction)stringlib_isdigit, METH_NOARGS, + _Py_isdigit__doc__}, + {"islower", (PyCFunction)stringlib_islower, METH_NOARGS, + _Py_islower__doc__}, + {"isspace", (PyCFunction)stringlib_isspace, METH_NOARGS, + _Py_isspace__doc__}, + {"istitle", (PyCFunction)stringlib_istitle, METH_NOARGS, + _Py_istitle__doc__}, + {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, + _Py_isupper__doc__}, + {"join", (PyCFunction)bytes_join, METH_O, join_doc}, + {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, + {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, + {"lstrip", (PyCFunction)bytes_lstrip, METH_VARARGS, lstrip__doc__}, + {"partition", (PyCFunction)bytes_partition, METH_O, partition__doc__}, + {"pop", (PyCFunction)bytes_pop, METH_VARARGS, pop__doc__}, + {"remove", (PyCFunction)bytes_remove, METH_O, remove__doc__}, + {"replace", (PyCFunction)bytes_replace, METH_VARARGS, replace__doc__}, + {"reverse", (PyCFunction)bytes_reverse, METH_NOARGS, reverse__doc__}, + {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, rfind__doc__}, + {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, rindex__doc__}, + {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, + {"rpartition", (PyCFunction)bytes_rpartition, METH_O, rpartition__doc__}, + {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS, rsplit__doc__}, + {"rstrip", (PyCFunction)bytes_rstrip, METH_VARARGS, rstrip__doc__}, + {"split", (PyCFunction)bytes_split, METH_VARARGS, split__doc__}, + {"splitlines", (PyCFunction)stringlib_splitlines, METH_VARARGS, + splitlines__doc__}, + {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS , + startswith__doc__}, + {"strip", (PyCFunction)bytes_strip, METH_VARARGS, strip__doc__}, + {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, + _Py_swapcase__doc__}, + {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, + {"translate", (PyCFunction)bytes_translate, METH_VARARGS, + translate__doc__}, + {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, + {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, + {NULL} +}; + +PyDoc_STRVAR(bytes_doc, +"bytearray(iterable_of_ints) -> bytearray.\n\ +bytearray(string, encoding[, errors]) -> bytearray.\n\ +bytearray(bytes_or_bytearray) -> mutable copy of bytes_or_bytearray.\n\ +bytearray(memory_view) -> bytearray.\n\ +\n\ +Construct an mutable bytearray object from:\n\ + - an iterable yielding integers in range(256)\n\ + - a text string encoded using the specified encoding\n\ + - a bytes or a bytearray object\n\ + - any object implementing the buffer API.\n\ +\n\ +bytearray(int) -> bytearray.\n\ +\n\ +Construct a zero-initialized bytearray of the given length."); + + +static PyObject *bytes_iter(PyObject *seq); + +PyTypeObject PyBytes_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytearray", + sizeof(PyBytesObject), + 0, + (destructor)bytes_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)bytes_repr, /* tp_repr */ + 0, /* tp_as_number */ + &bytes_as_sequence, /* tp_as_sequence */ + &bytes_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + bytes_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &bytes_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ + bytes_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)bytes_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + bytes_iter, /* tp_iter */ + 0, /* tp_iternext */ + bytes_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)bytes_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + +/*********************** Bytes Iterator ****************************/ + +typedef struct { + PyObject_HEAD + Py_ssize_t it_index; + PyBytesObject *it_seq; /* Set to NULL when iterator is exhausted */ +} bytesiterobject; + +static void +bytesiter_dealloc(bytesiterobject *it) +{ + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_seq); + PyObject_GC_Del(it); +} + +static int +bytesiter_traverse(bytesiterobject *it, visitproc visit, void *arg) +{ + Py_VISIT(it->it_seq); + return 0; +} + +static PyObject * +bytesiter_next(bytesiterobject *it) +{ + PyBytesObject *seq; + PyObject *item; + + assert(it != NULL); + seq = it->it_seq; + if (seq == NULL) + return NULL; + assert(PyBytes_Check(seq)); + + if (it->it_index < PyBytes_GET_SIZE(seq)) { + item = PyInt_FromLong( + (unsigned char)seq->ob_bytes[it->it_index]); + if (item != NULL) + ++it->it_index; + return item; + } + + Py_DECREF(seq); + it->it_seq = NULL; + return NULL; +} + +static PyObject * +bytesiter_length_hint(bytesiterobject *it) +{ + Py_ssize_t len = 0; + if (it->it_seq) + len = PyBytes_GET_SIZE(it->it_seq) - it->it_index; + return PyInt_FromSsize_t(len); +} + +PyDoc_STRVAR(length_hint_doc, + "Private method returning an estimate of len(list(it))."); + +static PyMethodDef bytesiter_methods[] = { + {"__length_hint__", (PyCFunction)bytesiter_length_hint, METH_NOARGS, + length_hint_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyBytesIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytearray_iterator", /* tp_name */ + sizeof(bytesiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)bytesiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)bytesiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)bytesiter_next, /* tp_iternext */ + bytesiter_methods, /* tp_methods */ + 0, +}; + +static PyObject * +bytes_iter(PyObject *seq) +{ + bytesiterobject *it; + + if (!PyBytes_Check(seq)) { + PyErr_BadInternalCall(); + return NULL; + } + it = PyObject_GC_New(bytesiterobject, &PyBytesIter_Type); + if (it == NULL) + return NULL; + it->it_index = 0; + Py_INCREF(seq); + it->it_seq = (PyBytesObject *)seq; + _PyObject_GC_TRACK(it); + return (PyObject *)it; +} diff --git a/Objects/exceptions.c b/Objects/exceptions.c index ec17bc2..2451a91 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -1923,6 +1923,12 @@ SimpleExtendsException(PyExc_Warning, UnicodeWarning, "Base class for warnings about Unicode related problems, mostly\n" "related to conversion problems."); +/* + * BytesWarning extends Warning + */ +SimpleExtendsException(PyExc_Warning, BytesWarning, + "Base class for warnings about bytes and buffer related problems, mostly\n" + "related to conversion from str or comparing to str."); /* Pre-computed MemoryError instance. Best to create this as early as * possible and not wait until a MemoryError is actually raised! @@ -2031,6 +2037,7 @@ _PyExc_Init(void) PRE_INIT(FutureWarning) PRE_INIT(ImportWarning) PRE_INIT(UnicodeWarning) + PRE_INIT(BytesWarning) m = Py_InitModule4("exceptions", functions, exceptions_doc, (PyObject *)NULL, PYTHON_API_VERSION); @@ -2097,6 +2104,7 @@ _PyExc_Init(void) POST_INIT(FutureWarning) POST_INIT(ImportWarning) POST_INIT(UnicodeWarning) + POST_INIT(BytesWarning) PyExc_MemoryErrorInst = BaseException_new(&_PyExc_MemoryError, NULL, NULL); if (!PyExc_MemoryErrorInst) diff --git a/Objects/object.c b/Objects/object.c index e3377f3..e7d84ad 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1986,6 +1986,9 @@ _Py_ReadyTypes(void) if (PyType_Ready(&PyString_Type) < 0) Py_FatalError("Can't initialize 'str'"); + if (PyType_Ready(&PyBytes_Type) < 0) + Py_FatalError("Can't initialize 'bytes'"); + if (PyType_Ready(&PyList_Type) < 0) Py_FatalError("Can't initialize 'list'"); diff --git a/Objects/stringlib/ctype.h b/Objects/stringlib/ctype.h new file mode 100644 index 0000000..8951276 --- /dev/null +++ b/Objects/stringlib/ctype.h @@ -0,0 +1,110 @@ +/* NOTE: this API is -ONLY- for use with single byte character strings. */ +/* Do not use it with Unicode. */ + +#include "bytes_methods.h" + +static PyObject* +stringlib_isspace(PyObject *self) +{ + return _Py_bytes_isspace(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_isalpha(PyObject *self) +{ + return _Py_bytes_isalpha(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_isalnum(PyObject *self) +{ + return _Py_bytes_isalnum(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_isdigit(PyObject *self) +{ + return _Py_bytes_isdigit(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_islower(PyObject *self) +{ + return _Py_bytes_islower(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_isupper(PyObject *self) +{ + return _Py_bytes_isupper(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + +static PyObject* +stringlib_istitle(PyObject *self) +{ + return _Py_bytes_istitle(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +} + + +/* functions that return a new object partially translated by ctype funcs: */ + +static PyObject* +stringlib_lower(PyObject *self) +{ + PyObject* newobj; + newobj = STRINGLIB_NEW(NULL, STRINGLIB_LEN(self)); + if (!newobj) + return NULL; + _Py_bytes_lower(STRINGLIB_STR(newobj), STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + return newobj; +} + +static PyObject* +stringlib_upper(PyObject *self) +{ + PyObject* newobj; + newobj = STRINGLIB_NEW(NULL, STRINGLIB_LEN(self)); + if (!newobj) + return NULL; + _Py_bytes_upper(STRINGLIB_STR(newobj), STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + return newobj; +} + +static PyObject* +stringlib_title(PyObject *self) +{ + PyObject* newobj; + newobj = STRINGLIB_NEW(NULL, STRINGLIB_LEN(self)); + if (!newobj) + return NULL; + _Py_bytes_title(STRINGLIB_STR(newobj), STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + return newobj; +} + +static PyObject* +stringlib_capitalize(PyObject *self) +{ + PyObject* newobj; + newobj = STRINGLIB_NEW(NULL, STRINGLIB_LEN(self)); + if (!newobj) + return NULL; + _Py_bytes_capitalize(STRINGLIB_STR(newobj), STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + return newobj; +} + +static PyObject* +stringlib_swapcase(PyObject *self) +{ + PyObject* newobj; + newobj = STRINGLIB_NEW(NULL, STRINGLIB_LEN(self)); + if (!newobj) + return NULL; + _Py_bytes_swapcase(STRINGLIB_STR(newobj), STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + return newobj; +} + diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h new file mode 100644 index 0000000..fe478c3 --- /dev/null +++ b/Objects/stringlib/transmogrify.h @@ -0,0 +1,362 @@ +/* NOTE: this API is -ONLY- for use with single byte character strings. */ +/* Do not use it with Unicode. */ + +#include "bytes_methods.h" + +#ifndef STRINGLIB_MUTABLE +#warning "STRINGLIB_MUTABLE not defined before #include, assuming 0" +#define STRINGLIB_MUTABLE 0 +#endif + +/* the more complicated methods. parts of these should be pulled out into the + shared code in bytes_methods.c to cut down on duplicate code bloat. */ + +PyDoc_STRVAR(expandtabs__doc__, +"B.expandtabs([tabsize]) -> copy of B\n\ +\n\ +Return a copy of B where all tab characters are expanded using spaces.\n\ +If tabsize is not given, a tab size of 8 characters is assumed."); + +static PyObject* +stringlib_expandtabs(PyObject *self, PyObject *args) +{ + const char *e, *p; + char *q; + Py_ssize_t i, j, old_j; + PyObject *u; + int tabsize = 8; + + if (!PyArg_ParseTuple(args, "|i:expandtabs", &tabsize)) + return NULL; + + /* First pass: determine size of output string */ + i = j = old_j = 0; + e = STRINGLIB_STR(self) + STRINGLIB_LEN(self); + for (p = STRINGLIB_STR(self); p < e; p++) + if (*p == '\t') { + if (tabsize > 0) { + j += tabsize - (j % tabsize); + /* XXX: this depends on a signed integer overflow to < 0 */ + /* C compilers, including gcc, do -NOT- guarantee this. */ + if (old_j > j) { + PyErr_SetString(PyExc_OverflowError, + "result is too long"); + return NULL; + } + old_j = j; + } + } + else { + j++; + if (*p == '\n' || *p == '\r') { + i += j; + old_j = j = 0; + /* XXX: this depends on a signed integer overflow to < 0 */ + /* C compilers, including gcc, do -NOT- guarantee this. */ + if (i < 0) { + PyErr_SetString(PyExc_OverflowError, + "result is too long"); + return NULL; + } + } + } + + if ((i + j) < 0) { + /* XXX: this depends on a signed integer overflow to < 0 */ + /* C compilers, including gcc, do -NOT- guarantee this. */ + PyErr_SetString(PyExc_OverflowError, "result is too long"); + return NULL; + } + + /* Second pass: create output string and fill it */ + u = STRINGLIB_NEW(NULL, i + j); + if (!u) + return NULL; + + j = 0; + q = STRINGLIB_STR(u); + + for (p = STRINGLIB_STR(self); p < e; p++) + if (*p == '\t') { + if (tabsize > 0) { + i = tabsize - (j % tabsize); + j += i; + while (i--) + *q++ = ' '; + } + } + else { + j++; + *q++ = *p; + if (*p == '\n' || *p == '\r') + j = 0; + } + + return u; +} + +Py_LOCAL_INLINE(PyObject *) +pad(PyObject *self, Py_ssize_t left, Py_ssize_t right, char fill) +{ + PyObject *u; + + if (left < 0) + left = 0; + if (right < 0) + right = 0; + + if (left == 0 && right == 0 && STRINGLIB_CHECK_EXACT(self)) { +#if STRINGLIB_MUTABLE + /* We're defined as returning a copy; If the object is mutable + * that means we must make an identical copy. */ + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +#else + Py_INCREF(self); + return (PyObject *)self; +#endif /* STRINGLIB_MUTABLE */ + } + + u = STRINGLIB_NEW(NULL, + left + STRINGLIB_LEN(self) + right); + if (u) { + if (left) + memset(STRINGLIB_STR(u), fill, left); + Py_MEMCPY(STRINGLIB_STR(u) + left, + STRINGLIB_STR(self), + STRINGLIB_LEN(self)); + if (right) + memset(STRINGLIB_STR(u) + left + STRINGLIB_LEN(self), + fill, right); + } + + return u; +} + +PyDoc_STRVAR(ljust__doc__, +"B.ljust(width[, fillchar]) -> copy of B\n" +"\n" +"Return B left justified in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)."); + +static PyObject * +stringlib_ljust(PyObject *self, PyObject *args) +{ + Py_ssize_t width; + char fillchar = ' '; + + if (!PyArg_ParseTuple(args, "n|c:ljust", &width, &fillchar)) + return NULL; + + if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { +#if STRINGLIB_MUTABLE + /* We're defined as returning a copy; If the object is mutable + * that means we must make an identical copy. */ + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +#else + Py_INCREF(self); + return (PyObject*) self; +#endif + } + + return pad(self, 0, width - STRINGLIB_LEN(self), fillchar); +} + + +PyDoc_STRVAR(rjust__doc__, +"B.rjust(width[, fillchar]) -> copy of B\n" +"\n" +"Return B right justified in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)"); + +static PyObject * +stringlib_rjust(PyObject *self, PyObject *args) +{ + Py_ssize_t width; + char fillchar = ' '; + + if (!PyArg_ParseTuple(args, "n|c:rjust", &width, &fillchar)) + return NULL; + + if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { +#if STRINGLIB_MUTABLE + /* We're defined as returning a copy; If the object is mutable + * that means we must make an identical copy. */ + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +#else + Py_INCREF(self); + return (PyObject*) self; +#endif + } + + return pad(self, width - STRINGLIB_LEN(self), 0, fillchar); +} + + +PyDoc_STRVAR(center__doc__, +"B.center(width[, fillchar]) -> copy of B\n" +"\n" +"Return B centered in a string of length width. Padding is\n" +"done using the specified fill character (default is a space)."); + +static PyObject * +stringlib_center(PyObject *self, PyObject *args) +{ + Py_ssize_t marg, left; + Py_ssize_t width; + char fillchar = ' '; + + if (!PyArg_ParseTuple(args, "n|c:center", &width, &fillchar)) + return NULL; + + if (STRINGLIB_LEN(self) >= width && STRINGLIB_CHECK_EXACT(self)) { +#if STRINGLIB_MUTABLE + /* We're defined as returning a copy; If the object is mutable + * that means we must make an identical copy. */ + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +#else + Py_INCREF(self); + return (PyObject*) self; +#endif + } + + marg = width - STRINGLIB_LEN(self); + left = marg / 2 + (marg & width & 1); + + return pad(self, left, marg - left, fillchar); +} + +PyDoc_STRVAR(zfill__doc__, +"B.zfill(width) -> copy of B\n" +"\n" +"Pad a numeric string B with zeros on the left, to fill a field\n" +"of the specified width. B is never truncated."); + +static PyObject * +stringlib_zfill(PyObject *self, PyObject *args) +{ + Py_ssize_t fill; + PyObject *s; + char *p; + Py_ssize_t width; + + if (!PyArg_ParseTuple(args, "n:zfill", &width)) + return NULL; + + if (STRINGLIB_LEN(self) >= width) { + if (STRINGLIB_CHECK_EXACT(self)) { +#if STRINGLIB_MUTABLE + /* We're defined as returning a copy; If the object is mutable + * that means we must make an identical copy. */ + return STRINGLIB_NEW(STRINGLIB_STR(self), STRINGLIB_LEN(self)); +#else + Py_INCREF(self); + return (PyObject*) self; +#endif + } + else + return STRINGLIB_NEW( + STRINGLIB_STR(self), + STRINGLIB_LEN(self) + ); + } + + fill = width - STRINGLIB_LEN(self); + + s = pad(self, fill, 0, '0'); + + if (s == NULL) + return NULL; + + p = STRINGLIB_STR(s); + if (p[fill] == '+' || p[fill] == '-') { + /* move sign to beginning of string */ + p[0] = p[fill]; + p[fill] = '0'; + } + + return (PyObject*) s; +} + + +#define _STRINGLIB_SPLIT_APPEND(data, left, right) \ + str = STRINGLIB_NEW((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); + +PyDoc_STRVAR(splitlines__doc__, +"B.splitlines([keepends]) -> list of lines\n\ +\n\ +Return a list of the lines in B, breaking at line boundaries.\n\ +Line breaks are not included in the resulting list unless keepends\n\ +is given and true."); + +static PyObject* +stringlib_splitlines(PyObject *self, PyObject *args) +{ + register Py_ssize_t i; + register Py_ssize_t j; + Py_ssize_t len; + int keepends = 0; + PyObject *list; + PyObject *str; + char *data; + + if (!PyArg_ParseTuple(args, "|i:splitlines", &keepends)) + return NULL; + + data = STRINGLIB_STR(self); + len = STRINGLIB_LEN(self); + + /* This does not use the preallocated list because splitlines is + usually run with hundreds of newlines. The overhead of + switching between PyList_SET_ITEM and append causes about a + 2-3% slowdown for that common case. A smarter implementation + could move the if check out, so the SET_ITEMs are done first + and the appends only done when the prealloc buffer is full. + That's too much work for little gain.*/ + + list = PyList_New(0); + if (!list) + goto onError; + + for (i = j = 0; i < len; ) { + Py_ssize_t eol; + + /* Find a line and append it */ + while (i < len && data[i] != '\n' && data[i] != '\r') + i++; + + /* Skip the line break reading CRLF as one line break */ + eol = i; + if (i < len) { + if (data[i] == '\r' && i + 1 < len && + data[i+1] == '\n') + i += 2; + else + i++; + if (keepends) + eol = i; + } + _STRINGLIB_SPLIT_APPEND(data, j, eol); + j = i; + } + if (j < len) { + _STRINGLIB_SPLIT_APPEND(data, j, len); + } + + return list; + + onError: + Py_XDECREF(list); + return NULL; +} + +#undef _STRINGLIB_SPLIT_APPEND + diff --git a/Objects/stringobject.c b/Objects/stringobject.c index ed2ffdd..4c36e4b 100644 --- a/Objects/stringobject.c +++ b/Objects/stringobject.c @@ -953,6 +953,8 @@ string_concat(register PyStringObject *a, register PyObject *bb) if (PyUnicode_Check(bb)) return PyUnicode_Concat((PyObject *)a, bb); #endif + if (PyBytes_Check(bb)) + return PyBytes_Concat((PyObject *)a, bb); PyErr_Format(PyExc_TypeError, "cannot concatenate 'str' and '%.200s' objects", Py_TYPE(bb)->tp_name); @@ -1303,6 +1305,13 @@ string_buffer_getcharbuf(PyStringObject *self, Py_ssize_t index, const char **pt return Py_SIZE(self); } +static int +string_buffer_getbuffer(PyStringObject *self, Py_buffer *view, int flags) +{ + return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_SIZE(self), + 0, flags); +} + static PySequenceMethods string_as_sequence = { (lenfunc)string_length, /*sq_length*/ (binaryfunc)string_concat, /*sq_concat*/ @@ -1325,6 +1334,8 @@ static PyBufferProcs string_as_buffer = { (writebufferproc)string_buffer_getwritebuf, (segcountproc)string_buffer_getsegcount, (charbufferproc)string_buffer_getcharbuf, + (getbufferproc)string_buffer_getbuffer, + 0, /* XXX */ }; @@ -4122,7 +4133,8 @@ PyTypeObject PyString_Type = { 0, /* tp_setattro */ &string_as_buffer, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_CHECKTYPES | - Py_TPFLAGS_BASETYPE | Py_TPFLAGS_STRING_SUBCLASS, /* tp_flags */ + Py_TPFLAGS_BASETYPE | Py_TPFLAGS_STRING_SUBCLASS | + Py_TPFLAGS_HAVE_NEWBUFFER, /* tp_flags */ string_doc, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 214e601..6b732dd 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3763,6 +3763,8 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) COPYBUF(bf_getwritebuffer); COPYBUF(bf_getsegcount); COPYBUF(bf_getcharbuffer); + COPYBUF(bf_getbuffer); + COPYBUF(bf_releasebuffer); } basebase = base->tp_base; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 5dd89c4..c5acd1b 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1076,7 +1076,13 @@ PyObject *PyUnicode_FromEncodedObject(register PyObject *obj, if (PyString_Check(obj)) { s = PyString_AS_STRING(obj); len = PyString_GET_SIZE(obj); - } + } + else if (PyBytes_Check(obj)) { + /* Python 2.x specific */ + PyErr_Format(PyExc_TypeError, + "decoding bytearray is not supported"); + return NULL; + } else if (PyObject_AsCharBuffer(obj, &s, &len)) { /* Overwrite the error message with something more useful in case of a TypeError. */ diff --git a/PCbuild/pythoncore.vcproj b/PCbuild/pythoncore.vcproj index f721ac7..b02703a 100644 --- a/PCbuild/pythoncore.vcproj +++ b/PCbuild/pythoncore.vcproj @@ -655,6 +655,14 @@ > </File> <File + RelativePath="..\Include\bytesobject.h" + > + </File> + <File + RelativePath="..\Include\bytes_methods.h" + > + </File> + <File RelativePath="..\Include\cellobject.h" > </File> @@ -1347,6 +1355,14 @@ > </File> <File + RelativePath="..\Objects\bytesobject.c" + > + </File> + <File + RelativePath="..\Objects\bytes_methods.c" + > + </File> + <File RelativePath="..\Objects\cellobject.c" > </File> diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index c760dcb..1c36fe5 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -1437,6 +1437,13 @@ builtin_ord(PyObject *self, PyObject* obj) ord = (long)((unsigned char)*PyString_AS_STRING(obj)); return PyInt_FromLong(ord); } + } else if (PyBytes_Check(obj)) { + size = PyBytes_GET_SIZE(obj); + if (size == 1) { + ord = (long)((unsigned char)*PyBytes_AS_STRING(obj)); + return PyInt_FromLong(ord); + } + #ifdef Py_USING_UNICODE } else if (PyUnicode_Check(obj)) { size = PyUnicode_GET_SIZE(obj); @@ -2552,6 +2559,7 @@ _PyBuiltin_Init(void) SETBUILTIN("basestring", &PyBaseString_Type); SETBUILTIN("bool", &PyBool_Type); /* SETBUILTIN("memoryview", &PyMemoryView_Type); */ + SETBUILTIN("bytearray", &PyBytes_Type); SETBUILTIN("bytes", &PyString_Type); SETBUILTIN("buffer", &PyBuffer_Type); SETBUILTIN("classmethod", &PyClassMethod_Type); diff --git a/Python/pythonrun.c b/Python/pythonrun.c index b8d516d..226fee3 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -72,6 +72,7 @@ int Py_VerboseFlag; /* Needed by import.c */ int Py_InteractiveFlag; /* Needed by Py_FdIsInteractive() below */ int Py_InspectFlag; /* Needed to determine whether to exit at SystemError */ int Py_NoSiteFlag; /* Suppress 'import site' */ +int Py_BytesWarningFlag; /* Warn on str(bytes) and str(buffer) */ int Py_DontWriteBytecodeFlag; /* Suppress writing bytecode files (*.py[co]) */ int Py_UseClassExceptionsFlag = 1; /* Needed by bltinmodule.c: deprecated */ int Py_FrozenFlag; /* Needed by getpath.c */ @@ -193,6 +194,9 @@ Py_InitializeEx(int install_sigs) if (!_PyInt_Init()) Py_FatalError("Py_Initialize: can't init ints"); + if (!PyBytes_Init()) + Py_FatalError("Py_Initialize: can't init bytearray"); + _PyFloat_Init(); interp->modules = PyDict_New(); @@ -251,8 +255,28 @@ Py_InitializeEx(int install_sigs) #endif /* WITH_THREAD */ warnings_module = PyImport_ImportModule("warnings"); - if (!warnings_module) + if (!warnings_module) { PyErr_Clear(); + } + else { + PyObject *o; + char *action[8]; + + if (Py_BytesWarningFlag > 1) + *action = "error"; + else if (Py_BytesWarningFlag) + *action = "default"; + else + *action = "ignore"; + + o = PyObject_CallMethod(warnings_module, + "simplefilter", "sO", + *action, PyExc_BytesWarning); + if (o == NULL) + Py_FatalError("Py_Initialize: can't initialize" + "warning filter for BytesWarning."); + Py_DECREF(o); + } #if defined(Py_USING_UNICODE) && defined(HAVE_LANGINFO_H) && defined(CODESET) /* On Unix, set the file system encoding according to the @@ -471,6 +495,7 @@ Py_Finalize(void) PyList_Fini(); PySet_Fini(); PyString_Fini(); + PyBytes_Fini(); PyInt_Fini(); PyFloat_Fini(); PyDict_Fini(); |