diff options
| -rw-r--r-- | Include/Python.h | 2 | ||||
| -rw-r--r-- | Include/bytearrayobject.h | 53 | ||||
| -rw-r--r-- | Include/bytesobject.h | 138 | ||||
| -rw-r--r-- | Include/stringobject.h | 111 | ||||
| -rw-r--r-- | Makefile.pre.in | 4 | ||||
| -rw-r--r-- | Objects/bytearrayobject.c | 3300 | ||||
| -rw-r--r-- | Objects/bytesobject.c | 5859 | ||||
| -rw-r--r-- | Objects/stringobject.c | 3429 | ||||
| -rw-r--r-- | PCbuild/pythoncore.vcproj | 16 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 2 |
10 files changed, 6457 insertions, 6457 deletions
diff --git a/Include/Python.h b/Include/Python.h index 53f7f59..539de51 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -65,6 +65,7 @@ #include "pydebug.h" +#include "bytearrayobject.h" #include "bytesobject.h" #include "unicodeobject.h" #include "longobject.h" @@ -75,7 +76,6 @@ #include "complexobject.h" #endif #include "rangeobject.h" -#include "stringobject.h" #include "memoryobject.h" #include "tupleobject.h" #include "listobject.h" diff --git a/Include/bytearrayobject.h b/Include/bytearrayobject.h new file mode 100644 index 0000000..2c0b734 --- /dev/null +++ b/Include/bytearrayobject.h @@ -0,0 +1,53 @@ +/* Bytes object interface */ + +#ifndef Py_BYTESOBJECT_H +#define Py_BYTESOBJECT_H +#ifdef __cplusplus +extern "C" { +#endif + +#include <stdarg.h> + +/* Type PyByteArrayObject represents a mutable array of bytes. + * The Python API is that of a sequence; + * the bytes are mapped to ints in [0, 256). + * Bytes are not characters; they may be used to encode characters. + * The only way to go between bytes and str/unicode is via encoding + * and decoding. + * For the convenience of C programmers, the bytes type is considered + * to contain a char pointer, not an unsigned char pointer. + */ + +/* Object layout */ +typedef struct { + PyObject_VAR_HEAD + /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ + int ob_exports; /* how many buffer exports */ + Py_ssize_t ob_alloc; /* How many bytes allocated */ + char *ob_bytes; +} PyByteArrayObject; + +/* Type object */ +PyAPI_DATA(PyTypeObject) PyByteArray_Type; +PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type; + +/* Type check macros */ +#define PyByteArray_Check(self) PyObject_TypeCheck(self, &PyByteArray_Type) +#define PyByteArray_CheckExact(self) (Py_TYPE(self) == &PyByteArray_Type) + +/* Direct API functions */ +PyAPI_FUNC(PyObject *) PyByteArray_FromObject(PyObject *); +PyAPI_FUNC(PyObject *) PyByteArray_Concat(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) PyByteArray_FromStringAndSize(const char *, Py_ssize_t); +PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *); +PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *); +PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); + +/* Macros, trading safety for speed */ +#define PyByteArray_AS_STRING(self) (assert(PyByteArray_Check(self)),((PyByteArrayObject *)(self))->ob_bytes) +#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)),Py_SIZE(self)) + +#ifdef __cplusplus +} +#endif +#endif /* !Py_BYTESOBJECT_H */ diff --git a/Include/bytesobject.h b/Include/bytesobject.h index 2c0b734..1c73fde 100644 --- a/Include/bytesobject.h +++ b/Include/bytesobject.h @@ -1,53 +1,111 @@ -/* Bytes object interface */ -#ifndef Py_BYTESOBJECT_H -#define Py_BYTESOBJECT_H +/* String object interface */ + +#ifndef Py_STRINGOBJECT_H +#define Py_STRINGOBJECT_H #ifdef __cplusplus extern "C" { #endif #include <stdarg.h> -/* Type PyByteArrayObject represents a mutable array of bytes. - * The Python API is that of a sequence; - * the bytes are mapped to ints in [0, 256). - * Bytes are not characters; they may be used to encode characters. - * The only way to go between bytes and str/unicode is via encoding - * and decoding. - * For the convenience of C programmers, the bytes type is considered - * to contain a char pointer, not an unsigned char pointer. - */ - -/* Object layout */ +/* +Type PyBytesObject represents a character string. An extra zero byte is +reserved at the end to ensure it is zero-terminated, but a size is +present so strings with null bytes in them can be represented. This +is an immutable object type. + +There are functions to create new string objects, to test +an object for string-ness, and to get the +string value. The latter function returns a null pointer +if the object is not of the proper type. +There is a variant that takes an explicit size as well as a +variant that assumes a zero-terminated string. Note that none of the +functions should be applied to nil objects. +*/ + +/* Caching the hash (ob_shash) saves recalculation of a string's hash value. + This significantly speeds up dict lookups. */ + typedef struct { PyObject_VAR_HEAD - /* XXX(nnorwitz): should ob_exports be Py_ssize_t? */ - int ob_exports; /* how many buffer exports */ - Py_ssize_t ob_alloc; /* How many bytes allocated */ - char *ob_bytes; -} PyByteArrayObject; - -/* Type object */ -PyAPI_DATA(PyTypeObject) PyByteArray_Type; -PyAPI_DATA(PyTypeObject) PyByteArrayIter_Type; - -/* Type check macros */ -#define PyByteArray_Check(self) PyObject_TypeCheck(self, &PyByteArray_Type) -#define PyByteArray_CheckExact(self) (Py_TYPE(self) == &PyByteArray_Type) - -/* Direct API functions */ -PyAPI_FUNC(PyObject *) PyByteArray_FromObject(PyObject *); -PyAPI_FUNC(PyObject *) PyByteArray_Concat(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) PyByteArray_FromStringAndSize(const char *, Py_ssize_t); -PyAPI_FUNC(Py_ssize_t) PyByteArray_Size(PyObject *); -PyAPI_FUNC(char *) PyByteArray_AsString(PyObject *); -PyAPI_FUNC(int) PyByteArray_Resize(PyObject *, Py_ssize_t); - -/* Macros, trading safety for speed */ -#define PyByteArray_AS_STRING(self) (assert(PyByteArray_Check(self)),((PyByteArrayObject *)(self))->ob_bytes) -#define PyByteArray_GET_SIZE(self) (assert(PyByteArray_Check(self)),Py_SIZE(self)) + long ob_shash; + char ob_sval[1]; + + /* Invariants: + * ob_sval contains space for 'ob_size+1' elements. + * ob_sval[ob_size] == 0. + * ob_shash is the hash of the string or -1 if not computed yet. + */ +} PyBytesObject; + +PyAPI_DATA(PyTypeObject) PyBytes_Type; +PyAPI_DATA(PyTypeObject) PyBytesIter_Type; + +#define PyBytes_Check(op) \ + PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_BYTES_SUBCLASS) +#define PyBytes_CheckExact(op) (Py_TYPE(op) == &PyBytes_Type) + +PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); +PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) + Py_GCC_ATTRIBUTE((format(printf, 1, 0))); +PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) + Py_GCC_ATTRIBUTE((format(printf, 1, 2))); +PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); +PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); +PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); +PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); +PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); +PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); +PyAPI_FUNC(PyObject *) PyBytes_Format(PyObject *, PyObject *); +PyAPI_FUNC(PyObject *) _PyBytes_FormatLong(PyObject*, int, int, + int, char**, int*); +PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, + const char *, Py_ssize_t, + const char *); + +/* Macro, trading safety for speed */ +#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \ + (((PyBytesObject *)(op))->ob_sval)) +#define PyBytes_GET_SIZE(op) (assert(PyBytes_Check(op)),Py_SIZE(op)) + +/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, + x must be an iterable object. */ +PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); + +/* Provides access to the internal data buffer and size of a string + object or the default encoded version of an Unicode object. Passing + NULL as *len parameter will force the string buffer to be + 0-terminated (passing a string with embedded NULL characters will + cause an exception). */ +PyAPI_FUNC(int) PyBytes_AsStringAndSize( + register PyObject *obj, /* string or Unicode object */ + register char **s, /* pointer to buffer variable */ + register Py_ssize_t *len /* pointer to length variable or NULL + (only possible for 0-terminated + strings) */ + ); + +/* Using the current locale, insert the thousands grouping + into the string pointed to by buffer. For the argument descriptions, + see Objects/stringlib/localeutil.h */ + +PyAPI_FUNC(int) _PyBytes_InsertThousandsGrouping(char *buffer, + Py_ssize_t len, + char *plast, + Py_ssize_t buf_size, + Py_ssize_t *count, + int append_zero_char); + +/* Flags used by string formatting */ +#define F_LJUST (1<<0) +#define F_SIGN (1<<1) +#define F_BLANK (1<<2) +#define F_ALT (1<<3) +#define F_ZERO (1<<4) #ifdef __cplusplus } #endif -#endif /* !Py_BYTESOBJECT_H */ +#endif /* !Py_STRINGOBJECT_H */ diff --git a/Include/stringobject.h b/Include/stringobject.h deleted file mode 100644 index 1c73fde..0000000 --- a/Include/stringobject.h +++ /dev/null @@ -1,111 +0,0 @@ - -/* String object interface */ - -#ifndef Py_STRINGOBJECT_H -#define Py_STRINGOBJECT_H -#ifdef __cplusplus -extern "C" { -#endif - -#include <stdarg.h> - -/* -Type PyBytesObject represents a character string. An extra zero byte is -reserved at the end to ensure it is zero-terminated, but a size is -present so strings with null bytes in them can be represented. This -is an immutable object type. - -There are functions to create new string objects, to test -an object for string-ness, and to get the -string value. The latter function returns a null pointer -if the object is not of the proper type. -There is a variant that takes an explicit size as well as a -variant that assumes a zero-terminated string. Note that none of the -functions should be applied to nil objects. -*/ - -/* Caching the hash (ob_shash) saves recalculation of a string's hash value. - This significantly speeds up dict lookups. */ - -typedef struct { - PyObject_VAR_HEAD - long ob_shash; - char ob_sval[1]; - - /* Invariants: - * ob_sval contains space for 'ob_size+1' elements. - * ob_sval[ob_size] == 0. - * ob_shash is the hash of the string or -1 if not computed yet. - */ -} PyBytesObject; - -PyAPI_DATA(PyTypeObject) PyBytes_Type; -PyAPI_DATA(PyTypeObject) PyBytesIter_Type; - -#define PyBytes_Check(op) \ - PyType_FastSubclass(Py_TYPE(op), Py_TPFLAGS_BYTES_SUBCLASS) -#define PyBytes_CheckExact(op) (Py_TYPE(op) == &PyBytes_Type) - -PyAPI_FUNC(PyObject *) PyBytes_FromStringAndSize(const char *, Py_ssize_t); -PyAPI_FUNC(PyObject *) PyBytes_FromString(const char *); -PyAPI_FUNC(PyObject *) PyBytes_FromFormatV(const char*, va_list) - Py_GCC_ATTRIBUTE((format(printf, 1, 0))); -PyAPI_FUNC(PyObject *) PyBytes_FromFormat(const char*, ...) - Py_GCC_ATTRIBUTE((format(printf, 1, 2))); -PyAPI_FUNC(Py_ssize_t) PyBytes_Size(PyObject *); -PyAPI_FUNC(char *) PyBytes_AsString(PyObject *); -PyAPI_FUNC(PyObject *) PyBytes_Repr(PyObject *, int); -PyAPI_FUNC(void) PyBytes_Concat(PyObject **, PyObject *); -PyAPI_FUNC(void) PyBytes_ConcatAndDel(PyObject **, PyObject *); -PyAPI_FUNC(int) _PyBytes_Resize(PyObject **, Py_ssize_t); -PyAPI_FUNC(PyObject *) PyBytes_Format(PyObject *, PyObject *); -PyAPI_FUNC(PyObject *) _PyBytes_FormatLong(PyObject*, int, int, - int, char**, int*); -PyAPI_FUNC(PyObject *) PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, Py_ssize_t, - const char *); - -/* Macro, trading safety for speed */ -#define PyBytes_AS_STRING(op) (assert(PyBytes_Check(op)), \ - (((PyBytesObject *)(op))->ob_sval)) -#define PyBytes_GET_SIZE(op) (assert(PyBytes_Check(op)),Py_SIZE(op)) - -/* _PyBytes_Join(sep, x) is like sep.join(x). sep must be PyBytesObject*, - x must be an iterable object. */ -PyAPI_FUNC(PyObject *) _PyBytes_Join(PyObject *sep, PyObject *x); - -/* Provides access to the internal data buffer and size of a string - object or the default encoded version of an Unicode object. Passing - NULL as *len parameter will force the string buffer to be - 0-terminated (passing a string with embedded NULL characters will - cause an exception). */ -PyAPI_FUNC(int) PyBytes_AsStringAndSize( - register PyObject *obj, /* string or Unicode object */ - register char **s, /* pointer to buffer variable */ - register Py_ssize_t *len /* pointer to length variable or NULL - (only possible for 0-terminated - strings) */ - ); - -/* Using the current locale, insert the thousands grouping - into the string pointed to by buffer. For the argument descriptions, - see Objects/stringlib/localeutil.h */ - -PyAPI_FUNC(int) _PyBytes_InsertThousandsGrouping(char *buffer, - Py_ssize_t len, - char *plast, - Py_ssize_t buf_size, - Py_ssize_t *count, - int append_zero_char); - -/* Flags used by string formatting */ -#define F_LJUST (1<<0) -#define F_SIGN (1<<1) -#define F_BLANK (1<<2) -#define F_ALT (1<<3) -#define F_ZERO (1<<4) - -#ifdef __cplusplus -} -#endif -#endif /* !Py_STRINGOBJECT_H */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 6f70bd5..16cd57a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -299,6 +299,7 @@ OBJECT_OBJS= \ Objects/abstract.o \ Objects/boolobject.o \ Objects/bytes_methods.o \ + Objects/bytearrayobject.o \ Objects/bytesobject.o \ Objects/cellobject.o \ Objects/classobject.o \ @@ -325,7 +326,6 @@ OBJECT_OBJS= \ Objects/rangeobject.o \ Objects/setobject.o \ Objects/sliceobject.o \ - Objects/stringobject.o \ Objects/structseq.o \ Objects/tupleobject.o \ Objects/typeobject.o \ @@ -579,6 +579,7 @@ PYTHON_HEADERS= \ Include/bitset.h \ Include/boolobject.h \ Include/bytes_methods.h \ + Include/bytearrayobject.h \ Include/bytesobject.h \ Include/cellobject.h \ Include/ceval.h \ @@ -636,7 +637,6 @@ PYTHON_HEADERS= \ Include/rangeobject.h \ Include/setobject.h \ Include/sliceobject.h \ - Include/stringobject.h \ Include/structmember.h \ Include/structseq.h \ Include/symtable.h \ diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c new file mode 100644 index 0000000..0a83725 --- /dev/null +++ b/Objects/bytearrayobject.c @@ -0,0 +1,3300 @@ +/* PyByteArray (bytearray) implementation */ + +#define PY_SSIZE_T_CLEAN +#include "Python.h" +#include "structmember.h" +#include "bytes_methods.h" + +static PyByteArrayObject *nullbytes = NULL; + +void +PyByteArray_Fini(void) +{ + Py_CLEAR(nullbytes); +} + +int +PyByteArray_Init(void) +{ + nullbytes = PyObject_New(PyByteArrayObject, &PyByteArray_Type); + if (nullbytes == NULL) + return 0; + nullbytes->ob_bytes = NULL; + Py_SIZE(nullbytes) = nullbytes->ob_alloc = 0; + nullbytes->ob_exports = 0; + return 1; +} + +/* end nullbytes support */ + +/* Helpers */ + +static int +_getbytevalue(PyObject* arg, int *value) +{ + long face_value; + + if (PyLong_Check(arg)) { + face_value = PyLong_AsLong(arg); + if (face_value < 0 || face_value >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return 0; + } + } else { + PyErr_Format(PyExc_TypeError, "an integer is required"); + return 0; + } + + *value = face_value; + return 1; +} + +static int +bytes_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) +{ + int ret; + void *ptr; + if (view == NULL) { + obj->ob_exports++; + return 0; + } + if (obj->ob_bytes == NULL) + ptr = ""; + else + ptr = obj->ob_bytes; + ret = PyBuffer_FillInfo(view, ptr, Py_SIZE(obj), 0, flags); + if (ret >= 0) { + obj->ob_exports++; + } + return ret; +} + +static void +bytes_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) +{ + obj->ob_exports--; +} + +static Py_ssize_t +_getbuffer(PyObject *obj, Py_buffer *view) +{ + PyBufferProcs *buffer = Py_TYPE(obj)->tp_as_buffer; + + if (buffer == NULL || buffer->bf_getbuffer == NULL) + { + PyErr_Format(PyExc_TypeError, + "Type %.100s doesn't support the buffer API", + Py_TYPE(obj)->tp_name); + return -1; + } + + if (buffer->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0) + return -1; + return view->len; +} + +/* Direct API functions */ + +PyObject * +PyByteArray_FromObject(PyObject *input) +{ + return PyObject_CallFunctionObjArgs((PyObject *)&PyByteArray_Type, + input, NULL); +} + +PyObject * +PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) +{ + PyByteArrayObject *new; + Py_ssize_t alloc; + + if (size < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to PyByteArray_FromStringAndSize"); + return NULL; + } + + new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); + if (new == NULL) + return NULL; + + if (size == 0) { + new->ob_bytes = NULL; + alloc = 0; + } + else { + alloc = size + 1; + new->ob_bytes = PyMem_Malloc(alloc); + if (new->ob_bytes == NULL) { + Py_DECREF(new); + return PyErr_NoMemory(); + } + if (bytes != NULL) + memcpy(new->ob_bytes, bytes, size); + new->ob_bytes[size] = '\0'; /* Trailing null byte */ + } + Py_SIZE(new) = size; + new->ob_alloc = alloc; + new->ob_exports = 0; + + return (PyObject *)new; +} + +Py_ssize_t +PyByteArray_Size(PyObject *self) +{ + assert(self != NULL); + assert(PyByteArray_Check(self)); + + return PyByteArray_GET_SIZE(self); +} + +char * +PyByteArray_AsString(PyObject *self) +{ + assert(self != NULL); + assert(PyByteArray_Check(self)); + + return PyByteArray_AS_STRING(self); +} + +int +PyByteArray_Resize(PyObject *self, Py_ssize_t size) +{ + void *sval; + Py_ssize_t alloc = ((PyByteArrayObject *)self)->ob_alloc; + + assert(self != NULL); + assert(PyByteArray_Check(self)); + assert(size >= 0); + + if (size < alloc / 2) { + /* Major downsize; resize down to exact size */ + alloc = size + 1; + } + else if (size < alloc) { + /* Within allocated size; quick exit */ + Py_SIZE(self) = size; + ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null */ + return 0; + } + else if (size <= alloc * 1.125) { + /* Moderate upsize; overallocate similar to list_resize() */ + alloc = size + (size >> 3) + (size < 9 ? 3 : 6); + } + else { + /* Major upsize; resize up to exact size */ + alloc = size + 1; + } + + if (((PyByteArrayObject *)self)->ob_exports > 0) { + /* + fprintf(stderr, "%d: %s", ((PyByteArrayObject *)self)->ob_exports, + ((PyByteArrayObject *)self)->ob_bytes); + */ + PyErr_SetString(PyExc_BufferError, + "Existing exports of data: object cannot be re-sized"); + return -1; + } + + sval = PyMem_Realloc(((PyByteArrayObject *)self)->ob_bytes, alloc); + if (sval == NULL) { + PyErr_NoMemory(); + return -1; + } + + ((PyByteArrayObject *)self)->ob_bytes = sval; + Py_SIZE(self) = size; + ((PyByteArrayObject *)self)->ob_alloc = alloc; + ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null byte */ + + return 0; +} + +PyObject * +PyByteArray_Concat(PyObject *a, PyObject *b) +{ + Py_ssize_t size; + Py_buffer va, vb; + PyByteArrayObject *result = NULL; + + va.len = -1; + vb.len = -1; + if (_getbuffer(a, &va) < 0 || + _getbuffer(b, &vb) < 0) { + PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", + Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + goto done; + } + + size = va.len + vb.len; + if (size < 0) { + return PyErr_NoMemory(); + goto done; + } + + result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, size); + if (result != NULL) { + memcpy(result->ob_bytes, va.buf, va.len); + memcpy(result->ob_bytes + va.len, vb.buf, vb.len); + } + + done: + if (va.len != -1) + PyObject_ReleaseBuffer(a, &va); + if (vb.len != -1) + PyObject_ReleaseBuffer(b, &vb); + return (PyObject *)result; +} + +/* Functions stuffed into the type object */ + +static Py_ssize_t +bytes_length(PyByteArrayObject *self) +{ + return Py_SIZE(self); +} + +static PyObject * +bytes_iconcat(PyByteArrayObject *self, PyObject *other) +{ + Py_ssize_t mysize; + Py_ssize_t size; + Py_buffer vo; + + if (_getbuffer(other, &vo) < 0) { + PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", + Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name); + return NULL; + } + + mysize = Py_SIZE(self); + size = mysize + vo.len; + if (size < 0) { + PyObject_ReleaseBuffer(other, &vo); + return PyErr_NoMemory(); + } + if (size < self->ob_alloc) { + Py_SIZE(self) = size; + self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ + } + else if (PyByteArray_Resize((PyObject *)self, size) < 0) { + PyObject_ReleaseBuffer(other, &vo); + return NULL; + } + memcpy(self->ob_bytes + mysize, vo.buf, vo.len); + PyObject_ReleaseBuffer(other, &vo); + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +bytes_repeat(PyByteArrayObject *self, Py_ssize_t count) +{ + PyByteArrayObject *result; + Py_ssize_t mysize; + Py_ssize_t size; + + if (count < 0) + count = 0; + mysize = Py_SIZE(self); + size = mysize * count; + if (count != 0 && size / count != mysize) + return PyErr_NoMemory(); + result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); + if (result != NULL && size != 0) { + if (mysize == 1) + memset(result->ob_bytes, self->ob_bytes[0], size); + else { + Py_ssize_t i; + for (i = 0; i < count; i++) + memcpy(result->ob_bytes + i*mysize, self->ob_bytes, mysize); + } + } + return (PyObject *)result; +} + +static PyObject * +bytes_irepeat(PyByteArrayObject *self, Py_ssize_t count) +{ + Py_ssize_t mysize; + Py_ssize_t size; + + if (count < 0) + count = 0; + mysize = Py_SIZE(self); + size = mysize * count; + if (count != 0 && size / count != mysize) + return PyErr_NoMemory(); + if (size < self->ob_alloc) { + Py_SIZE(self) = size; + self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ + } + else if (PyByteArray_Resize((PyObject *)self, size) < 0) + return NULL; + + if (mysize == 1) + memset(self->ob_bytes, self->ob_bytes[0], size); + else { + Py_ssize_t i; + for (i = 1; i < count; i++) + memcpy(self->ob_bytes + i*mysize, self->ob_bytes, mysize); + } + + Py_INCREF(self); + return (PyObject *)self; +} + +static PyObject * +bytes_getitem(PyByteArrayObject *self, Py_ssize_t i) +{ + if (i < 0) + i += Py_SIZE(self); + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return NULL; + } + return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); +} + +static PyObject * +bytes_subscript(PyByteArrayObject *self, PyObject *item) +{ + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if (i == -1 && PyErr_Occurred()) + return NULL; + + if (i < 0) + i += PyByteArray_GET_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return NULL; + } + return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength, cur, i; + if (PySlice_GetIndicesEx((PySliceObject *)item, + PyByteArray_GET_SIZE(self), + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (slicelength <= 0) + return PyByteArray_FromStringAndSize("", 0); + else if (step == 1) { + return PyByteArray_FromStringAndSize(self->ob_bytes + start, + slicelength); + } + else { + char *source_buf = PyByteArray_AS_STRING(self); + char *result_buf = (char *)PyMem_Malloc(slicelength); + PyObject *result; + + if (result_buf == NULL) + return PyErr_NoMemory(); + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + result_buf[i] = source_buf[cur]; + } + result = PyByteArray_FromStringAndSize(result_buf, slicelength); + PyMem_Free(result_buf); + return result; + } + } + else { + PyErr_SetString(PyExc_TypeError, "bytearray indices must be integers"); + return NULL; + } +} + +static int +bytes_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, + PyObject *values) +{ + Py_ssize_t avail, needed; + void *bytes; + Py_buffer vbytes; + int res = 0; + + vbytes.len = -1; + if (values == (PyObject *)self) { + /* Make a copy and call this function recursively */ + int err; + values = PyByteArray_FromObject(values); + if (values == NULL) + return -1; + err = bytes_setslice(self, lo, hi, values); + Py_DECREF(values); + return err; + } + if (values == NULL) { + /* del b[lo:hi] */ + bytes = NULL; + needed = 0; + } + else { + if (_getbuffer(values, &vbytes) < 0) { + PyErr_Format(PyExc_TypeError, + "can't set bytes slice from %.100s", + Py_TYPE(values)->tp_name); + return -1; + } + needed = vbytes.len; + bytes = vbytes.buf; + } + + if (lo < 0) + lo = 0; + if (hi < lo) + hi = lo; + if (hi > Py_SIZE(self)) + hi = Py_SIZE(self); + + avail = hi - lo; + if (avail < 0) + lo = hi = avail = 0; + + if (avail != needed) { + if (avail > needed) { + /* + 0 lo hi old_size + | |<----avail----->|<-----tomove------>| + | |<-needed->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, + Py_SIZE(self) - hi); + } + /* XXX(nnorwitz): need to verify this can't overflow! */ + if (PyByteArray_Resize((PyObject *)self, + Py_SIZE(self) + needed - avail) < 0) { + res = -1; + goto finish; + } + if (avail < needed) { + /* + 0 lo hi old_size + | |<-avail->|<-----tomove------>| + | |<----needed---->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, + Py_SIZE(self) - lo - needed); + } + } + + if (needed > 0) + memcpy(self->ob_bytes + lo, bytes, needed); + + + finish: + if (vbytes.len != -1) + PyObject_ReleaseBuffer(values, &vbytes); + return res; +} + +static int +bytes_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) +{ + Py_ssize_t ival; + + if (i < 0) + i += Py_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return -1; + } + + if (value == NULL) + return bytes_setslice(self, i, i+1, NULL); + + ival = PyNumber_AsSsize_t(value, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) + return -1; + + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; + } + + self->ob_bytes[i] = ival; + return 0; +} + +static int +bytes_ass_subscript(PyByteArrayObject *self, PyObject *item, PyObject *values) +{ + Py_ssize_t start, stop, step, slicelen, needed; + char *bytes; + + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + + if (i == -1 && PyErr_Occurred()) + return -1; + + if (i < 0) + i += PyByteArray_GET_SIZE(self); + + if (i < 0 || i >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); + return -1; + } + + if (values == NULL) { + /* Fall through to slice assignment */ + start = i; + stop = i + 1; + step = 1; + slicelen = 1; + } + else { + Py_ssize_t ival = PyNumber_AsSsize_t(values, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) + return -1; + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, + "byte must be in range(0, 256)"); + return -1; + } + self->ob_bytes[i] = (char)ival; + return 0; + } + } + else if (PySlice_Check(item)) { + if (PySlice_GetIndicesEx((PySliceObject *)item, + PyByteArray_GET_SIZE(self), + &start, &stop, &step, &slicelen) < 0) { + return -1; + } + } + else { + PyErr_SetString(PyExc_TypeError, "bytearray indices must be integer"); + return -1; + } + + if (values == NULL) { + bytes = NULL; + needed = 0; + } + else if (values == (PyObject *)self || !PyByteArray_Check(values)) { + /* Make a copy an call this function recursively */ + int err; + values = PyByteArray_FromObject(values); + if (values == NULL) + return -1; + err = bytes_ass_subscript(self, item, values); + Py_DECREF(values); + return err; + } + else { + assert(PyByteArray_Check(values)); + bytes = ((PyByteArrayObject *)values)->ob_bytes; + needed = Py_SIZE(values); + } + /* Make sure b[5:2] = ... inserts before 5, not before 2. */ + if ((step < 0 && start < stop) || + (step > 0 && start > stop)) + stop = start; + if (step == 1) { + if (slicelen != needed) { + if (slicelen > needed) { + /* + 0 start stop old_size + | |<---slicelen--->|<-----tomove------>| + | |<-needed->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, + Py_SIZE(self) - stop); + } + if (PyByteArray_Resize((PyObject *)self, + Py_SIZE(self) + needed - slicelen) < 0) + return -1; + if (slicelen < needed) { + /* + 0 lo hi old_size + | |<-avail->|<-----tomove------>| + | |<----needed---->|<-----tomove------>| + 0 lo new_hi new_size + */ + memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, + Py_SIZE(self) - start - needed); + } + } + + if (needed > 0) + memcpy(self->ob_bytes + start, bytes, needed); + + return 0; + } + else { + if (needed == 0) { + /* Delete slice */ + Py_ssize_t cur, i; + + if (step < 0) { + stop = start + 1; + start = stop + step * (slicelen - 1) - 1; + step = -step; + } + for (cur = start, i = 0; + i < slicelen; cur += step, i++) { + Py_ssize_t lim = step - 1; + + if (cur + step >= PyByteArray_GET_SIZE(self)) + lim = PyByteArray_GET_SIZE(self) - cur - 1; + + memmove(self->ob_bytes + cur - i, + self->ob_bytes + cur + 1, lim); + } + /* Move the tail of the bytes, in one chunk */ + cur = start + slicelen*step; + if (cur < PyByteArray_GET_SIZE(self)) { + memmove(self->ob_bytes + cur - slicelen, + self->ob_bytes + cur, + PyByteArray_GET_SIZE(self) - cur); + } + if (PyByteArray_Resize((PyObject *)self, + PyByteArray_GET_SIZE(self) - slicelen) < 0) + return -1; + + return 0; + } + else { + /* Assign slice */ + Py_ssize_t cur, i; + + if (needed != slicelen) { + PyErr_Format(PyExc_ValueError, + "attempt to assign bytes of size %zd " + "to extended slice of size %zd", + needed, slicelen); + return -1; + } + for (cur = start, i = 0; i < slicelen; cur += step, i++) + self->ob_bytes[cur] = bytes[i]; + return 0; + } + } +} + +static int +bytes_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"source", "encoding", "errors", 0}; + PyObject *arg = NULL; + const char *encoding = NULL; + const char *errors = NULL; + Py_ssize_t count; + PyObject *it; + PyObject *(*iternext)(PyObject *); + + if (Py_SIZE(self) != 0) { + /* Empty previous contents (yes, do this first of all!) */ + if (PyByteArray_Resize((PyObject *)self, 0) < 0) + return -1; + } + + /* Parse arguments */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:bytes", kwlist, + &arg, &encoding, &errors)) + return -1; + + /* Make a quick exit if no first argument */ + if (arg == NULL) { + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without sequence argument"); + return -1; + } + return 0; + } + + if (PyUnicode_Check(arg)) { + /* Encode via the codec registry */ + PyObject *encoded, *new; + if (encoding == NULL) { + PyErr_SetString(PyExc_TypeError, + "string argument without an encoding"); + return -1; + } + encoded = PyCodec_Encode(arg, encoding, errors); + if (encoded == NULL) + return -1; + assert(PyBytes_Check(encoded)); + new = bytes_iconcat(self, encoded); + Py_DECREF(encoded); + if (new == NULL) + return -1; + Py_DECREF(new); + return 0; + } + + /* If it's not unicode, there can't be encoding or errors */ + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without a string argument"); + return -1; + } + + /* Is it an int? */ + count = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (count == -1 && PyErr_Occurred()) + PyErr_Clear(); + else { + if (count < 0) { + PyErr_SetString(PyExc_ValueError, "negative count"); + return -1; + } + if (count > 0) { + if (PyByteArray_Resize((PyObject *)self, count)) + return -1; + memset(self->ob_bytes, 0, count); + } + return 0; + } + + /* Use the buffer API */ + if (PyObject_CheckBuffer(arg)) { + Py_ssize_t size; + Py_buffer view; + if (PyObject_GetBuffer(arg, &view, PyBUF_FULL_RO) < 0) + return -1; + size = view.len; + if (PyByteArray_Resize((PyObject *)self, size) < 0) goto fail; + if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) + goto fail; + PyObject_ReleaseBuffer(arg, &view); + return 0; + fail: + PyObject_ReleaseBuffer(arg, &view); + return -1; + } + + /* XXX Optimize this if the arguments is a list, tuple */ + + /* Get the iterator */ + it = PyObject_GetIter(arg); + if (it == NULL) + return -1; + iternext = *Py_TYPE(it)->tp_iternext; + + /* Run the iterator to exhaustion */ + for (;;) { + PyObject *item; + Py_ssize_t value; + + /* Get the next item */ + item = iternext(it); + if (item == NULL) { + if (PyErr_Occurred()) { + if (!PyErr_ExceptionMatches(PyExc_StopIteration)) + goto error; + PyErr_Clear(); + } + break; + } + + /* Interpret it as an int (__index__) */ + value = PyNumber_AsSsize_t(item, PyExc_ValueError); + Py_DECREF(item); + if (value == -1 && PyErr_Occurred()) + goto error; + + /* Range check */ + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; + } + + /* Append the byte */ + if (Py_SIZE(self) < self->ob_alloc) + Py_SIZE(self)++; + else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) + goto error; + self->ob_bytes[Py_SIZE(self)-1] = value; + } + + /* Clean up and return success */ + Py_DECREF(it); + return 0; + + error: + /* Error handling when it != NULL */ + Py_DECREF(it); + return -1; +} + +/* Mostly copied from string_repr, but without the + "smart quote" functionality. */ +static PyObject * +bytes_repr(PyByteArrayObject *self) +{ + static const char *hexdigits = "0123456789abcdef"; + const char *quote_prefix = "bytearray(b"; + const char *quote_postfix = ")"; + Py_ssize_t length = Py_SIZE(self); + /* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */ + size_t newsize = 14 + 4 * length; + PyObject *v; + if (newsize > PY_SSIZE_T_MAX || newsize / 4 - 3 != length) { + PyErr_SetString(PyExc_OverflowError, + "bytearray object is too large to make repr"); + return NULL; + } + v = PyUnicode_FromUnicode(NULL, newsize); + if (v == NULL) { + return NULL; + } + else { + register Py_ssize_t i; + register Py_UNICODE c; + register Py_UNICODE *p; + int quote; + + /* Figure out which quote to use; single is preferred */ + quote = '\''; + { + char *test, *start; + start = PyByteArray_AS_STRING(self); + for (test = start; test < start+length; ++test) { + if (*test == '"') { + quote = '\''; /* back to single */ + goto decided; + } + else if (*test == '\'') + quote = '"'; + } + decided: + ; + } + + p = PyUnicode_AS_UNICODE(v); + while (*quote_prefix) + *p++ = *quote_prefix++; + *p++ = quote; + + for (i = 0; i < length; i++) { + /* There's at least enough room for a hex escape + and a closing quote. */ + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); + c = self->ob_bytes[i]; + if (c == '\'' || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c == 0) + *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigits[(c & 0xf0) >> 4]; + *p++ = hexdigits[c & 0xf]; + } + else + *p++ = c; + } + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); + *p++ = quote; + while (*quote_postfix) { + *p++ = *quote_postfix++; + } + *p = '\0'; + if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { + Py_DECREF(v); + return NULL; + } + return v; + } +} + +static PyObject * +bytes_str(PyObject *op) +{ + if (Py_BytesWarningFlag) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "str() on a bytearray instance", 1)) + return NULL; + } + return bytes_repr((PyByteArrayObject*)op); +} + +static PyObject * +bytes_richcompare(PyObject *self, PyObject *other, int op) +{ + Py_ssize_t self_size, other_size; + Py_buffer self_bytes, other_bytes; + PyObject *res; + Py_ssize_t minsize; + int cmp; + + /* Bytes can be compared to anything that supports the (binary) + buffer API. Except that a comparison with Unicode is always an + error, even if the comparison is for equality. */ + if (PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type) || + PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type)) { + if (Py_BytesWarningFlag && op == Py_EQ) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "Comparsion between bytearray and string", 1)) + return NULL; + } + + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + self_size = _getbuffer(self, &self_bytes); + if (self_size < 0) { + PyErr_Clear(); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + other_size = _getbuffer(other, &other_bytes); + if (other_size < 0) { + PyErr_Clear(); + PyObject_ReleaseBuffer(self, &self_bytes); + Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + + if (self_size != other_size && (op == Py_EQ || op == Py_NE)) { + /* Shortcut: if the lengths differ, the objects differ */ + cmp = (op == Py_NE); + } + else { + minsize = self_size; + if (other_size < minsize) + minsize = other_size; + + cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize); + /* In ISO C, memcmp() guarantees to use unsigned bytes! */ + + if (cmp == 0) { + if (self_size < other_size) + cmp = -1; + else if (self_size > other_size) + cmp = 1; + } + + switch (op) { + case Py_LT: cmp = cmp < 0; break; + case Py_LE: cmp = cmp <= 0; break; + case Py_EQ: cmp = cmp == 0; break; + case Py_NE: cmp = cmp != 0; break; + case Py_GT: cmp = cmp > 0; break; + case Py_GE: cmp = cmp >= 0; break; + } + } + + res = cmp ? Py_True : Py_False; + PyObject_ReleaseBuffer(self, &self_bytes); + PyObject_ReleaseBuffer(other, &other_bytes); + Py_INCREF(res); + return res; +} + +static void +bytes_dealloc(PyByteArrayObject *self) +{ + if (self->ob_bytes != 0) { + PyMem_Free(self->ob_bytes); + } + Py_TYPE(self)->tp_free((PyObject *)self); +} + + +/* -------------------------------------------------------------------- */ +/* Methods */ + +#define STRINGLIB_CHAR char +#define STRINGLIB_CMP memcmp +#define STRINGLIB_LEN PyByteArray_GET_SIZE +#define STRINGLIB_STR PyByteArray_AS_STRING +#define STRINGLIB_NEW PyByteArray_FromStringAndSize +#define STRINGLIB_EMPTY nullbytes +#define STRINGLIB_CHECK_EXACT PyByteArray_CheckExact +#define STRINGLIB_MUTABLE 1 + +#include "stringlib/fastsearch.h" +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/partition.h" +#include "stringlib/ctype.h" +#include "stringlib/transmogrify.h" + + +/* The following Py_LOCAL_INLINE and Py_LOCAL functions +were copied from the old char* style string object. */ + +Py_LOCAL_INLINE(void) +_adjust_indices(Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t len) +{ + if (*end > len) + *end = len; + else if (*end < 0) + *end += len; + if (*end < 0) + *end = 0; + if (*start < 0) + *start += len; + if (*start < 0) + *start = 0; +} + + +Py_LOCAL_INLINE(Py_ssize_t) +bytes_find_internal(PyByteArrayObject *self, PyObject *args, int dir) +{ + PyObject *subobj; + Py_buffer subbuf; + Py_ssize_t start=0, end=PY_SSIZE_T_MAX; + Py_ssize_t res; + + if (!PyArg_ParseTuple(args, "O|O&O&:find/rfind/index/rindex", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return -2; + if (_getbuffer(subobj, &subbuf) < 0) + return -2; + if (dir > 0) + res = stringlib_find_slice( + PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subbuf.buf, subbuf.len, start, end); + else + res = stringlib_rfind_slice( + PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + subbuf.buf, subbuf.len, start, end); + PyObject_ReleaseBuffer(subobj, &subbuf); + return res; +} + +PyDoc_STRVAR(find__doc__, +"B.find(sub [,start [,end]]) -> int\n\ +\n\ +Return the lowest index in B where subsection sub is found,\n\ +such that sub is contained within s[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +static PyObject * +bytes_find(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, +1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); +} + +PyDoc_STRVAR(count__doc__, +"B.count(sub [,start [,end]]) -> int\n\ +\n\ +Return the number of non-overlapping occurrences of subsection sub in\n\ +bytes B[start:end]. Optional arguments start and end are interpreted\n\ +as in slice notation."); + +static PyObject * +bytes_count(PyByteArrayObject *self, PyObject *args) +{ + PyObject *sub_obj; + const char *str = PyByteArray_AS_STRING(self); + Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + Py_buffer vsub; + PyObject *count_obj; + + if (!PyArg_ParseTuple(args, "O|O&O&:count", &sub_obj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + + if (_getbuffer(sub_obj, &vsub) < 0) + return NULL; + + _adjust_indices(&start, &end, PyByteArray_GET_SIZE(self)); + + count_obj = PyLong_FromSsize_t( + stringlib_count(str + start, end - start, vsub.buf, vsub.len) + ); + PyObject_ReleaseBuffer(sub_obj, &vsub); + return count_obj; +} + + +PyDoc_STRVAR(index__doc__, +"B.index(sub [,start [,end]]) -> int\n\ +\n\ +Like B.find() but raise ValueError when the subsection is not found."); + +static PyObject * +bytes_index(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, +1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyLong_FromSsize_t(result); +} + + +PyDoc_STRVAR(rfind__doc__, +"B.rfind(sub [,start [,end]]) -> int\n\ +\n\ +Return the highest index in B where subsection sub is found,\n\ +such that sub is contained within s[start,end]. Optional\n\ +arguments start and end are interpreted as in slice notation.\n\ +\n\ +Return -1 on failure."); + +static PyObject * +bytes_rfind(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, -1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); +} + + +PyDoc_STRVAR(rindex__doc__, +"B.rindex(sub [,start [,end]]) -> int\n\ +\n\ +Like B.rfind() but raise ValueError when the subsection is not found."); + +static PyObject * +bytes_rindex(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t result = bytes_find_internal(self, args, -1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "subsection not found"); + return NULL; + } + return PyLong_FromSsize_t(result); +} + + +static int +bytes_contains(PyObject *self, PyObject *arg) +{ + Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) { + Py_buffer varg; + int pos; + PyErr_Clear(); + if (_getbuffer(arg, &varg) < 0) + return -1; + pos = stringlib_find(PyByteArray_AS_STRING(self), Py_SIZE(self), + varg.buf, varg.len, 0); + PyObject_ReleaseBuffer(arg, &varg); + return pos >= 0; + } + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; + } + + return memchr(PyByteArray_AS_STRING(self), ival, Py_SIZE(self)) != NULL; +} + + +/* Matches the end (direction >= 0) or start (direction < 0) of self + * against substr, using the start and end arguments. Returns + * -1 on error, 0 if not found and 1 if found. + */ +Py_LOCAL(int) +_bytes_tailmatch(PyByteArrayObject *self, PyObject *substr, Py_ssize_t start, + Py_ssize_t end, int direction) +{ + Py_ssize_t len = PyByteArray_GET_SIZE(self); + const char* str; + Py_buffer vsubstr; + int rv = 0; + + str = PyByteArray_AS_STRING(self); + + if (_getbuffer(substr, &vsubstr) < 0) + return -1; + + _adjust_indices(&start, &end, len); + + if (direction < 0) { + /* startswith */ + if (start+vsubstr.len > len) { + goto done; + } + } else { + /* endswith */ + if (end-start < vsubstr.len || start > len) { + goto done; + } + + if (end-vsubstr.len > start) + start = end - vsubstr.len; + } + if (end-start >= vsubstr.len) + rv = ! memcmp(str+start, vsubstr.buf, vsubstr.len); + +done: + PyObject_ReleaseBuffer(substr, &vsubstr); + return rv; +} + + +PyDoc_STRVAR(startswith__doc__, +"B.startswith(prefix [,start [,end]]) -> bool\n\ +\n\ +Return True if B starts with the specified prefix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +prefix can also be a tuple of strings to try."); + +static PyObject * +bytes_startswith(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _bytes_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, -1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _bytes_tailmatch(self, subobj, start, end, -1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); +} + +PyDoc_STRVAR(endswith__doc__, +"B.endswith(suffix [,start [,end]]) -> bool\n\ +\n\ +Return True if B ends with the specified suffix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +suffix can also be a tuple of strings to try."); + +static PyObject * +bytes_endswith(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _bytes_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, +1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _bytes_tailmatch(self, subobj, start, end, +1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); +} + + +PyDoc_STRVAR(translate__doc__, +"B.translate(table[, deletechars]) -> bytearray\n\ +\n\ +Return a copy of B, where all characters occurring in the\n\ +optional argument deletechars are removed, and the remaining\n\ +characters have been mapped through the given translation\n\ +table, which must be a bytes object of length 256."); + +static PyObject * +bytes_translate(PyByteArrayObject *self, PyObject *args) +{ + register char *input, *output; + register const char *table; + register Py_ssize_t i, c, changed = 0; + PyObject *input_obj = (PyObject*)self; + const char *output_start; + Py_ssize_t inlen; + PyObject *result; + int trans_table[256]; + PyObject *tableobj, *delobj = NULL; + Py_buffer vtable, vdel; + + if (!PyArg_UnpackTuple(args, "translate", 1, 2, + &tableobj, &delobj)) + return NULL; + + if (_getbuffer(tableobj, &vtable) < 0) + return NULL; + + if (vtable.len != 256) { + PyErr_SetString(PyExc_ValueError, + "translation table must be 256 characters long"); + result = NULL; + goto done; + } + + if (delobj != NULL) { + if (_getbuffer(delobj, &vdel) < 0) { + result = NULL; + goto done; + } + } + else { + vdel.buf = NULL; + vdel.len = 0; + } + + table = (const char *)vtable.buf; + inlen = PyByteArray_GET_SIZE(input_obj); + result = PyByteArray_FromStringAndSize((char *)NULL, inlen); + if (result == NULL) + goto done; + output_start = output = PyByteArray_AsString(result); + input = PyByteArray_AS_STRING(input_obj); + + if (vdel.len == 0) { + /* If no deletions are required, use faster code */ + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (Py_CHARMASK((*output++ = table[c])) != c) + changed = 1; + } + if (changed || !PyByteArray_CheckExact(input_obj)) + goto done; + Py_DECREF(result); + Py_INCREF(input_obj); + result = input_obj; + goto done; + } + + for (i = 0; i < 256; i++) + trans_table[i] = Py_CHARMASK(table[i]); + + for (i = 0; i < vdel.len; i++) + trans_table[(int) Py_CHARMASK( ((unsigned char*)vdel.buf)[i] )] = -1; + + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (trans_table[c] != -1) + if (Py_CHARMASK(*output++ = (char)trans_table[c]) == c) + continue; + changed = 1; + } + if (!changed && PyByteArray_CheckExact(input_obj)) { + Py_DECREF(result); + Py_INCREF(input_obj); + result = input_obj; + goto done; + } + /* Fix the size of the resulting string */ + if (inlen > 0) + PyByteArray_Resize(result, output - output_start); + +done: + PyObject_ReleaseBuffer(tableobj, &vtable); + if (delobj != NULL) + PyObject_ReleaseBuffer(delobj, &vdel); + return result; +} + + +#define FORWARD 1 +#define REVERSE -1 + +/* find and count characters and substrings */ + +#define findchar(target, target_len, c) \ + ((char *)memchr((const void *)(target), c, target_len)) + +/* Don't call if length < 2 */ +#define Py_STRING_MATCH(target, offset, pattern, length) \ + (target[offset] == pattern[0] && \ + target[offset+length-1] == pattern[length-1] && \ + !memcmp(target+offset+1, pattern+1, length-2) ) + + +/* Bytes ops must return a string. */ +/* If the object is subclass of bytes, create a copy */ +Py_LOCAL(PyByteArrayObject *) +return_self(PyByteArrayObject *self) +{ + if (PyByteArray_CheckExact(self)) { + Py_INCREF(self); + return (PyByteArrayObject *)self; + } + return (PyByteArrayObject *)PyByteArray_FromStringAndSize( + PyByteArray_AS_STRING(self), + PyByteArray_GET_SIZE(self)); +} + +Py_LOCAL_INLINE(Py_ssize_t) +countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) +{ + Py_ssize_t count=0; + const char *start=target; + const char *end=target+target_len; + + while ( (start=findchar(start, end-start, c)) != NULL ) { + count++; + if (count >= maxcount) + break; + start += 1; + } + return count; +} + +Py_LOCAL(Py_ssize_t) +findstring(const char *target, Py_ssize_t target_len, + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } + + /* zero-length substrings always match at the first attempt */ + if (pattern_len == 0) + return (direction > 0) ? start : end; + + end -= pattern_len; + + if (direction < 0) { + for (; end >= start; end--) + if (Py_STRING_MATCH(target, end, pattern, pattern_len)) + return end; + } else { + for (; start <= end; start++) + if (Py_STRING_MATCH(target, start, pattern, pattern_len)) + return start; + } + return -1; +} + +Py_LOCAL_INLINE(Py_ssize_t) +countstring(const char *target, Py_ssize_t target_len, + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction, Py_ssize_t maxcount) +{ + Py_ssize_t count=0; + + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } + + /* zero-length substrings match everywhere */ + if (pattern_len == 0 || maxcount == 0) { + if (target_len+1 < maxcount) + return target_len+1; + return maxcount; + } + + end -= pattern_len; + if (direction < 0) { + for (; (end >= start); end--) + if (Py_STRING_MATCH(target, end, pattern, pattern_len)) { + count++; + if (--maxcount <= 0) break; + end -= pattern_len-1; + } + } else { + for (; (start <= end); start++) + if (Py_STRING_MATCH(target, start, pattern, pattern_len)) { + count++; + if (--maxcount <= 0) + break; + start += pattern_len-1; + } + } + return count; +} + + +/* Algorithms for different cases of string replacement */ + +/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_interleave(PyByteArrayObject *self, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, i, product; + PyByteArrayObject *result; + + self_len = PyByteArray_GET_SIZE(self); + + /* 1 at the end plus 1 after every character */ + count = self_len+1; + if (maxcount < count) + count = maxcount; + + /* Check for overflow */ + /* result_len = count * to_len + self_len; */ + product = count * to_len; + if (product / to_len != count) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + result_len = product + self_len; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + + if (! (result = (PyByteArrayObject *) + PyByteArray_FromStringAndSize(NULL, result_len)) ) + return NULL; + + self_s = PyByteArray_AS_STRING(self); + result_s = PyByteArray_AS_STRING(result); + + /* TODO: special case single character, which doesn't need memcpy */ + + /* Lay the first one down (guaranteed this will occur) */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + count -= 1; + + for (i=0; i<count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } + + /* Copy the rest of the original string */ + Py_MEMCPY(result_s, self_s, self_len-i); + + return result; +} + +/* Special case for deleting a single character */ +/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_delete_single_character(PyByteArrayObject *self, + char from_c, Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count; + PyByteArrayObject *result; + + self_len = PyByteArray_GET_SIZE(self); + self_s = PyByteArray_AS_STRING(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + return return_self(self); + } + + result_len = self_len - count; /* from_len == 1 */ + assert(result_len>=0); + + if ( (result = (PyByteArrayObject *) + PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyByteArray_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + start = next+1; + } + Py_MEMCPY(result_s, start, end-start); + + return result; +} + +/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ + +Py_LOCAL(PyByteArrayObject *) +replace_delete_substring(PyByteArrayObject *self, + const char *from_s, Py_ssize_t from_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset; + PyByteArrayObject *result; + + self_len = PyByteArray_GET_SIZE(self); + self_s = PyByteArray_AS_STRING(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, 1, + maxcount); + + if (count == 0) { + /* no matches */ + return return_self(self); + } + + result_len = self_len - (count * from_len); + assert (result_len>=0); + + if ( (result = (PyByteArrayObject *) + PyByteArray_FromStringAndSize(NULL, result_len)) == NULL ) + return NULL; + + result_s = PyByteArray_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start + offset; + + Py_MEMCPY(result_s, start, next-start); + + result_s += (next-start); + start = next+from_len; + } + Py_MEMCPY(result_s, start, end-start); + return result; +} + +/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_single_character_in_place(PyByteArrayObject *self, + char from_c, char to_c, + Py_ssize_t maxcount) +{ + char *self_s, *result_s, *start, *end, *next; + Py_ssize_t self_len; + PyByteArrayObject *result; + + /* The result string will be the same size */ + self_s = PyByteArray_AS_STRING(self); + self_len = PyByteArray_GET_SIZE(self); + + next = findchar(self_s, self_len, from_c); + + if (next == NULL) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyByteArray_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + (next-self_s); + *start = to_c; + start++; + end = result_s + self_len; + + while (--maxcount > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + *next = to_c; + start = next+1; + } + + return result; +} + +/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_substring_in_place(PyByteArrayObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *result_s, *start, *end; + char *self_s; + Py_ssize_t self_len, offset; + PyByteArrayObject *result; + + /* The result bytes will be the same size */ + + self_s = PyByteArray_AS_STRING(self); + self_len = PyByteArray_GET_SIZE(self); + + offset = findstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD); + if (offset == -1) { + /* No matches; return the original bytes */ + return return_self(self); + } + + /* Need to make a new bytes */ + result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyByteArray_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + offset; + Py_MEMCPY(start, to_s, from_len); + start += from_len; + end = result_s + self_len; + + while ( --maxcount > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset==-1) + break; + Py_MEMCPY(start+offset, to_s, from_len); + start += offset+from_len; + } + + return result; +} + +/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_single_character(PyByteArrayObject *self, + char from_c, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, product; + PyByteArrayObject *result; + + self_s = PyByteArray_AS_STRING(self); + self_len = PyByteArray_GET_SIZE(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* use the difference between current and new, hence the "-1" */ + /* result_len = self_len + count * (to_len-1) */ + product = count * (to_len-1); + if (product / (to_len-1) != count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + + if ( (result = (PyByteArrayObject *) + PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyByteArray_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += 1; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+1; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end-start); + + return result; +} + +/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ +Py_LOCAL(PyByteArrayObject *) +replace_substring(PyByteArrayObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset, product; + PyByteArrayObject *result; + + self_s = PyByteArray_AS_STRING(self); + self_len = PyByteArray_GET_SIZE(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } + + /* Check for overflow */ + /* result_len = self_len + count * (to_len-from_len) */ + product = count * (to_len-from_len); + if (product / (to_len-from_len) != count) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); + return NULL; + } + + if ( (result = (PyByteArrayObject *) + PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyByteArray_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start+offset; + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += from_len; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+from_len; + } + } + /* Copy the remainder of the remaining bytes */ + Py_MEMCPY(result_s, start, end-start); + + return result; +} + + +Py_LOCAL(PyByteArrayObject *) +replace(PyByteArrayObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + if (maxcount < 0) { + maxcount = PY_SSIZE_T_MAX; + } else if (maxcount == 0 || PyByteArray_GET_SIZE(self) == 0) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + + if (maxcount == 0 || + (from_len == 0 && to_len == 0)) { + /* nothing to do; return the original bytes */ + return return_self(self); + } + + /* Handle zero-length special cases */ + + if (from_len == 0) { + /* insert the 'to' bytes everywhere. */ + /* >>> "Python".replace("", ".") */ + /* '.P.y.t.h.o.n.' */ + return replace_interleave(self, to_s, to_len, maxcount); + } + + /* Except for "".replace("", "A") == "A" there is no way beyond this */ + /* point for an empty self bytes to generate a non-empty bytes */ + /* Special case so the remaining code always gets a non-empty bytes */ + if (PyByteArray_GET_SIZE(self) == 0) { + return return_self(self); + } + + if (to_len == 0) { + /* delete all occurances of 'from' bytes */ + if (from_len == 1) { + return replace_delete_single_character( + self, from_s[0], maxcount); + } else { + return replace_delete_substring(self, from_s, from_len, maxcount); + } + } + + /* Handle special case where both bytes have the same length */ + + if (from_len == to_len) { + if (from_len == 1) { + return replace_single_character_in_place( + self, + from_s[0], + to_s[0], + maxcount); + } else { + return replace_substring_in_place( + self, from_s, from_len, to_s, to_len, maxcount); + } + } + + /* Otherwise use the more generic algorithms */ + if (from_len == 1) { + return replace_single_character(self, from_s[0], + to_s, to_len, maxcount); + } else { + /* len('from')>=2, len('to')>=1 */ + return replace_substring(self, from_s, from_len, to_s, to_len, maxcount); + } +} + + +PyDoc_STRVAR(replace__doc__, +"B.replace(old, new[, count]) -> bytes\n\ +\n\ +Return a copy of B with all occurrences of subsection\n\ +old replaced by new. If the optional argument count is\n\ +given, only the first count occurrences are replaced."); + +static PyObject * +bytes_replace(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t count = -1; + PyObject *from, *to, *res; + Py_buffer vfrom, vto; + + if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) + return NULL; + + if (_getbuffer(from, &vfrom) < 0) + return NULL; + if (_getbuffer(to, &vto) < 0) { + PyObject_ReleaseBuffer(from, &vfrom); + return NULL; + } + + res = (PyObject *)replace((PyByteArrayObject *) self, + vfrom.buf, vfrom.len, + vto.buf, vto.len, count); + + PyObject_ReleaseBuffer(from, &vfrom); + PyObject_ReleaseBuffer(to, &vto); + return res; +} + + +/* Overallocate the initial list to reduce the number of reallocs for small + split sizes. Eg, "A A A A A A A A A A".split() (10 elements) has three + resizes, to sizes 4, 8, then 16. Most observed string splits are for human + text (roughly 11 words per line) and field delimited data (usually 1-10 + fields). For large strings the split algorithms are bandwidth limited + so increasing the preallocation likely will not improve things.*/ + +#define MAX_PREALLOC 12 + +/* 5 splits gives 6 elements */ +#define PREALLOC_SIZE(maxsplit) \ + (maxsplit >= MAX_PREALLOC ? MAX_PREALLOC : maxsplit+1) + +#define SPLIT_APPEND(data, left, right) \ + str = PyByteArray_FromStringAndSize((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); + +#define SPLIT_ADD(data, left, right) { \ + str = PyByteArray_FromStringAndSize((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (count < MAX_PREALLOC) { \ + PyList_SET_ITEM(list, count, str); \ + } else { \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); \ + } \ + count++; } + +/* Always force the list to the expected size. */ +#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count + + +Py_LOCAL_INLINE(PyObject *) +split_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = 0; + while ((j < len) && (maxcount-- > 0)) { + for(; j < len; j++) { + /* I found that using memchr makes no difference */ + if (s[j] == ch) { + SPLIT_ADD(s, i, j); + i = j = j + 1; + break; + } + } + } + if (i <= len) { + SPLIT_ADD(s, i, len); + } + FIX_PREALLOC_SIZE(list); + return list; + + onError: + Py_DECREF(list); + return NULL; +} + + +Py_LOCAL_INLINE(PyObject *) +split_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + for (i = j = 0; i < len; ) { + /* find a token */ + while (i < len && ISSPACE(s[i])) + i++; + j = i; + while (i < len && !ISSPACE(s[i])) + i++; + if (j < i) { + if (maxcount-- <= 0) + break; + SPLIT_ADD(s, j, i); + while (i < len && ISSPACE(s[i])) + i++; + j = i; + } + } + if (j < len) { + SPLIT_ADD(s, j, len); + } + FIX_PREALLOC_SIZE(list); + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +PyDoc_STRVAR(split__doc__, +"B.split([sep[, maxsplit]]) -> list of bytearray\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter.\n\ +If sep is not given, B is split on ASCII whitespace characters\n\ +(space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); + +static PyObject * +bytes_split(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count = 0; + const char *s = PyByteArray_AS_STRING(self), *sub; + PyObject *list, *str, *subobj = Py_None; + Py_buffer vsub; +#ifdef USE_FAST + Py_ssize_t pos; +#endif + + if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + + if (subobj == Py_None) + return split_whitespace(s, len, maxsplit); + + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + if (n == 1) + return split_char(s, len, sub[0], maxsplit); + + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + +#ifdef USE_FAST + i = j = 0; + while (maxsplit-- > 0) { + pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH); + if (pos < 0) + break; + j = i+pos; + SPLIT_ADD(s, i, j); + i = j + n; + } +#else + i = j = 0; + while ((j+n <= len) && (maxsplit-- > 0)) { + for (; j+n <= len; j++) { + if (Py_STRING_MATCH(s, j, sub, n)) { + SPLIT_ADD(s, i, j); + i = j = j + n; + break; + } + } + } +#endif + SPLIT_ADD(s, i, len); + FIX_PREALLOC_SIZE(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return list; + + onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; +} + +/* stringlib's partition shares nullbytes in some cases. + undo this, we don't want the nullbytes to be shared. */ +static PyObject * +make_nullbytes_unique(PyObject *result) +{ + if (result != NULL) { + int i; + assert(PyTuple_Check(result)); + assert(PyTuple_GET_SIZE(result) == 3); + for (i = 0; i < 3; i++) { + if (PyTuple_GET_ITEM(result, i) == (PyObject *)nullbytes) { + PyObject *new = PyByteArray_FromStringAndSize(NULL, 0); + if (new == NULL) { + Py_DECREF(result); + result = NULL; + break; + } + Py_DECREF(nullbytes); + PyTuple_SET_ITEM(result, i, new); + } + } + } + return result; +} + +PyDoc_STRVAR(partition__doc__, +"B.partition(sep) -> (head, sep, tail)\n\ +\n\ +Searches for the separator sep in B, and returns the part before it,\n\ +the separator itself, and the part after it. If the separator is not\n\ +found, returns B and two empty bytearray objects."); + +static PyObject * +bytes_partition(PyByteArrayObject *self, PyObject *sep_obj) +{ + PyObject *bytesep, *result; + + bytesep = PyByteArray_FromObject(sep_obj); + if (! bytesep) + return NULL; + + result = stringlib_partition( + (PyObject*) self, + PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + bytesep, + PyByteArray_AS_STRING(bytesep), PyByteArray_GET_SIZE(bytesep) + ); + + Py_DECREF(bytesep); + return make_nullbytes_unique(result); +} + +PyDoc_STRVAR(rpartition__doc__, +"B.rpartition(sep) -> (tail, sep, head)\n\ +\n\ +Searches for the separator sep in B, starting at the end of B,\n\ +and returns the part before it, the separator itself, and the\n\ +part after it. If the separator is not found, returns two empty\n\ +bytearray objects and B."); + +static PyObject * +bytes_rpartition(PyByteArrayObject *self, PyObject *sep_obj) +{ + PyObject *bytesep, *result; + + bytesep = PyByteArray_FromObject(sep_obj); + if (! bytesep) + return NULL; + + result = stringlib_rpartition( + (PyObject*) self, + PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), + bytesep, + PyByteArray_AS_STRING(bytesep), PyByteArray_GET_SIZE(bytesep) + ); + + Py_DECREF(bytesep); + return make_nullbytes_unique(result); +} + +Py_LOCAL_INLINE(PyObject *) +rsplit_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = len - 1; + while ((i >= 0) && (maxcount-- > 0)) { + for (; i >= 0; i--) { + if (s[i] == ch) { + SPLIT_ADD(s, i + 1, j + 1); + j = i = i - 1; + break; + } + } + } + if (j >= -1) { + SPLIT_ADD(s, 0, j + 1); + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +Py_LOCAL_INLINE(PyObject *) +rsplit_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) +{ + register Py_ssize_t i, j, count = 0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + for (i = j = len - 1; i >= 0; ) { + /* find a token */ + while (i >= 0 && ISSPACE(s[i])) + i--; + j = i; + while (i >= 0 && !ISSPACE(s[i])) + i--; + if (j > i) { + if (maxcount-- <= 0) + break; + SPLIT_ADD(s, i + 1, j + 1); + while (i >= 0 && ISSPACE(s[i])) + i--; + j = i; + } + } + if (j >= 0) { + SPLIT_ADD(s, 0, j + 1); + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + + return list; + + onError: + Py_DECREF(list); + return NULL; +} + +PyDoc_STRVAR(rsplit__doc__, +"B.rsplit(sep[, maxsplit]) -> list of bytearray\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter,\n\ +starting at the end of B and working to the front.\n\ +If sep is not given, B is split on ASCII whitespace characters\n\ +(space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); + +static PyObject * +bytes_rsplit(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count = 0; + const char *s = PyByteArray_AS_STRING(self), *sub; + PyObject *list, *str, *subobj = Py_None; + Py_buffer vsub; + + if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + + if (subobj == Py_None) + return rsplit_whitespace(s, len, maxsplit); + + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + else if (n == 1) + return rsplit_char(s, len, sub[0], maxsplit); + + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + + j = len; + i = j - n; + + while ( (i >= 0) && (maxsplit-- > 0) ) { + for (; i>=0; i--) { + if (Py_STRING_MATCH(s, i, sub, n)) { + SPLIT_ADD(s, i + n, j); + j = i; + i -= n; + break; + } + } + } + SPLIT_ADD(s, 0, j); + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + PyObject_ReleaseBuffer(subobj, &vsub); + return list; + +onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; +} + +PyDoc_STRVAR(reverse__doc__, +"B.reverse() -> None\n\ +\n\ +Reverse the order of the values in B in place."); +static PyObject * +bytes_reverse(PyByteArrayObject *self, PyObject *unused) +{ + char swap, *head, *tail; + Py_ssize_t i, j, n = Py_SIZE(self); + + j = n / 2; + head = self->ob_bytes; + tail = head + n - 1; + for (i = 0; i < j; i++) { + swap = *head; + *head++ = *tail; + *tail-- = swap; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(insert__doc__, +"B.insert(index, int) -> None\n\ +\n\ +Insert a single item into the bytearray before the given index."); +static PyObject * +bytes_insert(PyByteArrayObject *self, PyObject *args) +{ + int value; + Py_ssize_t where, n = Py_SIZE(self); + + if (!PyArg_ParseTuple(args, "ni:insert", &where, &value)) + return NULL; + + if (n == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "cannot add more objects to bytes"); + return NULL; + } + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "byte must be in range(0, 256)"); + return NULL; + } + if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) + return NULL; + + if (where < 0) { + where += n; + if (where < 0) + where = 0; + } + if (where > n) + where = n; + memmove(self->ob_bytes + where + 1, self->ob_bytes + where, n - where); + self->ob_bytes[where] = value; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(append__doc__, +"B.append(int) -> None\n\ +\n\ +Append a single item to the end of B."); +static PyObject * +bytes_append(PyByteArrayObject *self, PyObject *arg) +{ + int value; + Py_ssize_t n = Py_SIZE(self); + + if (! _getbytevalue(arg, &value)) + return NULL; + if (n == PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "cannot add more objects to bytes"); + return NULL; + } + if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) + return NULL; + + self->ob_bytes[n] = value; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(extend__doc__, +"B.extend(iterable int) -> None\n\ +\n\ +Append all the elements from the iterator or sequence to the\n\ +end of B."); +static PyObject * +bytes_extend(PyByteArrayObject *self, PyObject *arg) +{ + PyObject *it, *item, *bytes_obj; + Py_ssize_t buf_size = 0, len = 0; + int value; + char *buf; + + /* bytes_setslice code only accepts something supporting PEP 3118. */ + if (PyObject_CheckBuffer(arg)) { + if (bytes_setslice(self, Py_SIZE(self), Py_SIZE(self), arg) == -1) + return NULL; + + Py_RETURN_NONE; + } + + it = PyObject_GetIter(arg); + if (it == NULL) + return NULL; + + /* Try to determine the length of the argument. 32 is abitrary. */ + buf_size = _PyObject_LengthHint(arg, 32); + + bytes_obj = PyByteArray_FromStringAndSize(NULL, buf_size); + if (bytes_obj == NULL) + return NULL; + buf = PyByteArray_AS_STRING(bytes_obj); + + while ((item = PyIter_Next(it)) != NULL) { + if (! _getbytevalue(item, &value)) { + Py_DECREF(item); + Py_DECREF(it); + Py_DECREF(bytes_obj); + return NULL; + } + buf[len++] = value; + Py_DECREF(item); + + if (len >= buf_size) { + buf_size = len + (len >> 1) + 1; + if (PyByteArray_Resize((PyObject *)bytes_obj, buf_size) < 0) { + Py_DECREF(it); + Py_DECREF(bytes_obj); + return NULL; + } + /* Recompute the `buf' pointer, since the resizing operation may + have invalidated it. */ + buf = PyByteArray_AS_STRING(bytes_obj); + } + } + Py_DECREF(it); + + /* Resize down to exact size. */ + if (PyByteArray_Resize((PyObject *)bytes_obj, len) < 0) { + Py_DECREF(bytes_obj); + return NULL; + } + + if (bytes_setslice(self, Py_SIZE(self), Py_SIZE(self), bytes_obj) == -1) + return NULL; + Py_DECREF(bytes_obj); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(pop__doc__, +"B.pop([index]) -> int\n\ +\n\ +Remove and return a single item from B. If no index\n\ +argument is give, will pop the last value."); +static PyObject * +bytes_pop(PyByteArrayObject *self, PyObject *args) +{ + int value; + Py_ssize_t where = -1, n = Py_SIZE(self); + + if (!PyArg_ParseTuple(args, "|n:pop", &where)) + return NULL; + + if (n == 0) { + PyErr_SetString(PyExc_OverflowError, + "cannot pop an empty bytes"); + return NULL; + } + if (where < 0) + where += Py_SIZE(self); + if (where < 0 || where >= Py_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, "pop index out of range"); + return NULL; + } + + value = self->ob_bytes[where]; + memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) + return NULL; + + return PyLong_FromLong(value); +} + +PyDoc_STRVAR(remove__doc__, +"B.remove(int) -> None\n\ +\n\ +Remove the first occurance of a value in B."); +static PyObject * +bytes_remove(PyByteArrayObject *self, PyObject *arg) +{ + int value; + Py_ssize_t where, n = Py_SIZE(self); + + if (! _getbytevalue(arg, &value)) + return NULL; + + for (where = 0; where < n; where++) { + if (self->ob_bytes[where] == value) + break; + } + if (where == n) { + PyErr_SetString(PyExc_ValueError, "value not found in bytes"); + return NULL; + } + + memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); + if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) + return NULL; + + Py_RETURN_NONE; +} + +/* XXX These two helpers could be optimized if argsize == 1 */ + +static Py_ssize_t +lstrip_helper(unsigned char *myptr, Py_ssize_t mysize, + void *argptr, Py_ssize_t argsize) +{ + Py_ssize_t i = 0; + while (i < mysize && memchr(argptr, myptr[i], argsize)) + i++; + return i; +} + +static Py_ssize_t +rstrip_helper(unsigned char *myptr, Py_ssize_t mysize, + void *argptr, Py_ssize_t argsize) +{ + Py_ssize_t i = mysize - 1; + while (i >= 0 && memchr(argptr, myptr[i], argsize)) + i--; + return i + 1; +} + +PyDoc_STRVAR(strip__doc__, +"B.strip([bytes]) -> bytearray\n\ +\n\ +Strip leading and trailing bytes contained in the argument.\n\ +If the argument is omitted, strip ASCII whitespace."); +static PyObject * +bytes_strip(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:strip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = lstrip_helper(myptr, mysize, argptr, argsize); + if (left == mysize) + right = left; + else + right = rstrip_helper(myptr, mysize, argptr, argsize); + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(lstrip__doc__, +"B.lstrip([bytes]) -> bytearray\n\ +\n\ +Strip leading bytes contained in the argument.\n\ +If the argument is omitted, strip leading ASCII whitespace."); +static PyObject * +bytes_lstrip(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:lstrip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = lstrip_helper(myptr, mysize, argptr, argsize); + right = mysize; + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(rstrip__doc__, +"B.rstrip([bytes]) -> bytearray\n\ +\n\ +Strip trailing bytes contained in the argument.\n\ +If the argument is omitted, strip trailing ASCII whitespace."); +static PyObject * +bytes_rstrip(PyByteArrayObject *self, PyObject *args) +{ + Py_ssize_t left, right, mysize, argsize; + void *myptr, *argptr; + PyObject *arg = Py_None; + Py_buffer varg; + if (!PyArg_ParseTuple(args, "|O:rstrip", &arg)) + return NULL; + if (arg == Py_None) { + argptr = "\t\n\r\f\v "; + argsize = 6; + } + else { + if (_getbuffer(arg, &varg) < 0) + return NULL; + argptr = varg.buf; + argsize = varg.len; + } + myptr = self->ob_bytes; + mysize = Py_SIZE(self); + left = 0; + right = rstrip_helper(myptr, mysize, argptr, argsize); + if (arg != Py_None) + PyObject_ReleaseBuffer(arg, &varg); + return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); +} + +PyDoc_STRVAR(decode_doc, +"B.decode([encoding[, errors]]) -> unicode object.\n\ +\n\ +Decodes B using the codec registered for encoding. encoding defaults\n\ +to the default encoding. errors may be given to set a different error\n\ +handling scheme. Default is 'strict' meaning that encoding errors raise\n\ +a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ +as well as any other name registered with codecs.register_error that is\n\ +able to handle UnicodeDecodeErrors."); + +static PyObject * +bytes_decode(PyObject *self, PyObject *args) +{ + const char *encoding = NULL; + const char *errors = NULL; + + if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors)) + return NULL; + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); + return PyCodec_Decode(self, encoding, errors); +} + +PyDoc_STRVAR(alloc_doc, +"B.__alloc__() -> int\n\ +\n\ +Returns the number of bytes actually allocated."); + +static PyObject * +bytes_alloc(PyByteArrayObject *self) +{ + return PyLong_FromSsize_t(self->ob_alloc); +} + +PyDoc_STRVAR(join_doc, +"B.join(iterable_of_bytes) -> bytes\n\ +\n\ +Concatenates any number of bytearray objects, with B in between each pair."); + +static PyObject * +bytes_join(PyByteArrayObject *self, PyObject *it) +{ + PyObject *seq; + Py_ssize_t mysize = Py_SIZE(self); + Py_ssize_t i; + Py_ssize_t n; + PyObject **items; + Py_ssize_t totalsize = 0; + PyObject *result; + char *dest; + + seq = PySequence_Fast(it, "can only join an iterable"); + if (seq == NULL) + return NULL; + n = PySequence_Fast_GET_SIZE(seq); + items = PySequence_Fast_ITEMS(seq); + + /* Compute the total size, and check that they are all bytes */ + /* XXX Shouldn't we use _getbuffer() on these items instead? */ + for (i = 0; i < n; i++) { + PyObject *obj = items[i]; + if (!PyByteArray_Check(obj) && !PyBytes_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "can only join an iterable of bytes " + "(item %ld has type '%.100s')", + /* XXX %ld isn't right on Win64 */ + (long)i, Py_TYPE(obj)->tp_name); + goto error; + } + if (i > 0) + totalsize += mysize; + totalsize += Py_SIZE(obj); + if (totalsize < 0) { + PyErr_NoMemory(); + goto error; + } + } + + /* Allocate the result, and copy the bytes */ + result = PyByteArray_FromStringAndSize(NULL, totalsize); + if (result == NULL) + goto error; + dest = PyByteArray_AS_STRING(result); + for (i = 0; i < n; i++) { + PyObject *obj = items[i]; + Py_ssize_t size = Py_SIZE(obj); + char *buf; + if (PyByteArray_Check(obj)) + buf = PyByteArray_AS_STRING(obj); + else + buf = PyBytes_AS_STRING(obj); + if (i) { + memcpy(dest, self->ob_bytes, mysize); + dest += mysize; + } + memcpy(dest, buf, size); + dest += size; + } + + /* Done */ + Py_DECREF(seq); + return result; + + /* Error handling */ + error: + Py_DECREF(seq); + return NULL; +} + +PyDoc_STRVAR(fromhex_doc, +"bytearray.fromhex(string) -> bytearray\n\ +\n\ +Create a bytearray object from a string of hexadecimal numbers.\n\ +Spaces between two numbers are accepted.\n\ +Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef')."); + +static int +hex_digit_to_int(Py_UNICODE c) +{ + if (c >= 128) + return -1; + if (ISDIGIT(c)) + return c - '0'; + else { + if (ISUPPER(c)) + c = TOLOWER(c); + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + } + return -1; +} + +static PyObject * +bytes_fromhex(PyObject *cls, PyObject *args) +{ + PyObject *newbytes, *hexobj; + char *buf; + Py_UNICODE *hex; + Py_ssize_t hexlen, byteslen, i, j; + int top, bot; + + if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) + return NULL; + assert(PyUnicode_Check(hexobj)); + hexlen = PyUnicode_GET_SIZE(hexobj); + hex = PyUnicode_AS_UNICODE(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ + newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); + if (!newbytes) + return NULL; + buf = PyByteArray_AS_STRING(newbytes); + for (i = j = 0; i < hexlen; i += 2) { + /* skip over spaces in the input */ + while (hex[i] == ' ') + i++; + if (i >= hexlen) + break; + top = hex_digit_to_int(hex[i]); + bot = hex_digit_to_int(hex[i+1]); + if (top == -1 || bot == -1) { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", i); + goto error; + } + buf[j++] = (top << 4) + bot; + } + if (PyByteArray_Resize(newbytes, j) < 0) + goto error; + return newbytes; + + error: + Py_DECREF(newbytes); + return NULL; +} + +PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); + +static PyObject * +bytes_reduce(PyByteArrayObject *self) +{ + PyObject *latin1, *dict; + if (self->ob_bytes) + latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, + Py_SIZE(self), NULL); + else + latin1 = PyUnicode_FromString(""); + + dict = PyObject_GetAttrString((PyObject *)self, "__dict__"); + if (dict == NULL) { + PyErr_Clear(); + dict = Py_None; + Py_INCREF(dict); + } + + return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); +} + +static PySequenceMethods bytes_as_sequence = { + (lenfunc)bytes_length, /* sq_length */ + (binaryfunc)PyByteArray_Concat, /* sq_concat */ + (ssizeargfunc)bytes_repeat, /* sq_repeat */ + (ssizeargfunc)bytes_getitem, /* sq_item */ + 0, /* sq_slice */ + (ssizeobjargproc)bytes_setitem, /* sq_ass_item */ + 0, /* sq_ass_slice */ + (objobjproc)bytes_contains, /* sq_contains */ + (binaryfunc)bytes_iconcat, /* sq_inplace_concat */ + (ssizeargfunc)bytes_irepeat, /* sq_inplace_repeat */ +}; + +static PyMappingMethods bytes_as_mapping = { + (lenfunc)bytes_length, + (binaryfunc)bytes_subscript, + (objobjargproc)bytes_ass_subscript, +}; + +static PyBufferProcs bytes_as_buffer = { + (getbufferproc)bytes_getbuffer, + (releasebufferproc)bytes_releasebuffer, +}; + +static PyMethodDef +bytes_methods[] = { + {"__alloc__", (PyCFunction)bytes_alloc, METH_NOARGS, alloc_doc}, + {"__reduce__", (PyCFunction)bytes_reduce, METH_NOARGS, reduce_doc}, + {"append", (PyCFunction)bytes_append, METH_O, append__doc__}, + {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, + _Py_capitalize__doc__}, + {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, + {"count", (PyCFunction)bytes_count, METH_VARARGS, count__doc__}, + {"decode", (PyCFunction)bytes_decode, METH_VARARGS, decode_doc}, + {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, endswith__doc__}, + {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS, + expandtabs__doc__}, + {"extend", (PyCFunction)bytes_extend, METH_O, extend__doc__}, + {"find", (PyCFunction)bytes_find, METH_VARARGS, find__doc__}, + {"fromhex", (PyCFunction)bytes_fromhex, METH_VARARGS|METH_CLASS, + fromhex_doc}, + {"index", (PyCFunction)bytes_index, METH_VARARGS, index__doc__}, + {"insert", (PyCFunction)bytes_insert, METH_VARARGS, insert__doc__}, + {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, + _Py_isalnum__doc__}, + {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, + _Py_isalpha__doc__}, + {"isdigit", (PyCFunction)stringlib_isdigit, METH_NOARGS, + _Py_isdigit__doc__}, + {"islower", (PyCFunction)stringlib_islower, METH_NOARGS, + _Py_islower__doc__}, + {"isspace", (PyCFunction)stringlib_isspace, METH_NOARGS, + _Py_isspace__doc__}, + {"istitle", (PyCFunction)stringlib_istitle, METH_NOARGS, + _Py_istitle__doc__}, + {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, + _Py_isupper__doc__}, + {"join", (PyCFunction)bytes_join, METH_O, join_doc}, + {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, + {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, + {"lstrip", (PyCFunction)bytes_lstrip, METH_VARARGS, lstrip__doc__}, + {"partition", (PyCFunction)bytes_partition, METH_O, partition__doc__}, + {"pop", (PyCFunction)bytes_pop, METH_VARARGS, pop__doc__}, + {"remove", (PyCFunction)bytes_remove, METH_O, remove__doc__}, + {"replace", (PyCFunction)bytes_replace, METH_VARARGS, replace__doc__}, + {"reverse", (PyCFunction)bytes_reverse, METH_NOARGS, reverse__doc__}, + {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, rfind__doc__}, + {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, rindex__doc__}, + {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, + {"rpartition", (PyCFunction)bytes_rpartition, METH_O, rpartition__doc__}, + {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS, rsplit__doc__}, + {"rstrip", (PyCFunction)bytes_rstrip, METH_VARARGS, rstrip__doc__}, + {"split", (PyCFunction)bytes_split, METH_VARARGS, split__doc__}, + {"splitlines", (PyCFunction)stringlib_splitlines, METH_VARARGS, + splitlines__doc__}, + {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS , + startswith__doc__}, + {"strip", (PyCFunction)bytes_strip, METH_VARARGS, strip__doc__}, + {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, + _Py_swapcase__doc__}, + {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, + {"translate", (PyCFunction)bytes_translate, METH_VARARGS, + translate__doc__}, + {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, + {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, + {NULL} +}; + +PyDoc_STRVAR(bytes_doc, +"bytearray(iterable_of_ints) -> bytearray.\n\ +bytearray(string, encoding[, errors]) -> bytearray.\n\ +bytearray(bytes_or_bytearray) -> mutable copy of bytes_or_bytearray.\n\ +bytearray(memory_view) -> bytearray.\n\ +\n\ +Construct an mutable bytearray object from:\n\ + - an iterable yielding integers in range(256)\n\ + - a text string encoded using the specified encoding\n\ + - a bytes or a bytearray object\n\ + - any object implementing the buffer API.\n\ +\n\ +bytearray(int) -> bytearray.\n\ +\n\ +Construct a zero-initialized bytearray of the given length."); + + +static PyObject *bytes_iter(PyObject *seq); + +PyTypeObject PyByteArray_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytearray", + sizeof(PyByteArrayObject), + 0, + (destructor)bytes_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)bytes_repr, /* tp_repr */ + 0, /* tp_as_number */ + &bytes_as_sequence, /* tp_as_sequence */ + &bytes_as_mapping, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + bytes_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &bytes_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + bytes_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)bytes_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + bytes_iter, /* tp_iter */ + 0, /* tp_iternext */ + bytes_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)bytes_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + +/*********************** Bytes Iterator ****************************/ + +typedef struct { + PyObject_HEAD + Py_ssize_t it_index; + PyByteArrayObject *it_seq; /* Set to NULL when iterator is exhausted */ +} bytesiterobject; + +static void +bytesiter_dealloc(bytesiterobject *it) +{ + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_seq); + PyObject_GC_Del(it); +} + +static int +bytesiter_traverse(bytesiterobject *it, visitproc visit, void *arg) +{ + Py_VISIT(it->it_seq); + return 0; +} + +static PyObject * +bytesiter_next(bytesiterobject *it) +{ + PyByteArrayObject *seq; + PyObject *item; + + assert(it != NULL); + seq = it->it_seq; + if (seq == NULL) + return NULL; + assert(PyByteArray_Check(seq)); + + if (it->it_index < PyByteArray_GET_SIZE(seq)) { + item = PyLong_FromLong( + (unsigned char)seq->ob_bytes[it->it_index]); + if (item != NULL) + ++it->it_index; + return item; + } + + Py_DECREF(seq); + it->it_seq = NULL; + return NULL; +} + +static PyObject * +bytesiter_length_hint(bytesiterobject *it) +{ + Py_ssize_t len = 0; + if (it->it_seq) + len = PyByteArray_GET_SIZE(it->it_seq) - it->it_index; + return PyLong_FromSsize_t(len); +} + +PyDoc_STRVAR(length_hint_doc, + "Private method returning an estimate of len(list(it))."); + +static PyMethodDef bytesiter_methods[] = { + {"__length_hint__", (PyCFunction)bytesiter_length_hint, METH_NOARGS, + length_hint_doc}, + {NULL, NULL} /* sentinel */ +}; + +PyTypeObject PyByteArrayIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytearray_iterator", /* tp_name */ + sizeof(bytesiterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)bytesiter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)bytesiter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)bytesiter_next, /* tp_iternext */ + bytesiter_methods, /* tp_methods */ + 0, +}; + +static PyObject * +bytes_iter(PyObject *seq) +{ + bytesiterobject *it; + + if (!PyByteArray_Check(seq)) { + PyErr_BadInternalCall(); + return NULL; + } + it = PyObject_GC_New(bytesiterobject, &PyByteArrayIter_Type); + if (it == NULL) + return NULL; + it->it_index = 0; + Py_INCREF(seq); + it->it_seq = (PyByteArrayObject *)seq; + _PyObject_GC_TRACK(it); + return (PyObject *)it; +} diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 0a83725..b9ba73f 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -1,79 +1,13 @@ -/* PyByteArray (bytearray) implementation */ +/* String object implementation */ -#define PY_SSIZE_T_CLEAN -#include "Python.h" -#include "structmember.h" -#include "bytes_methods.h" - -static PyByteArrayObject *nullbytes = NULL; - -void -PyByteArray_Fini(void) -{ - Py_CLEAR(nullbytes); -} - -int -PyByteArray_Init(void) -{ - nullbytes = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (nullbytes == NULL) - return 0; - nullbytes->ob_bytes = NULL; - Py_SIZE(nullbytes) = nullbytes->ob_alloc = 0; - nullbytes->ob_exports = 0; - return 1; -} - -/* end nullbytes support */ - -/* Helpers */ +/* XXX This is now called 'bytes' as far as the user is concerned. + Many docstrings and error messages need to be cleaned up. */ -static int -_getbytevalue(PyObject* arg, int *value) -{ - long face_value; - - if (PyLong_Check(arg)) { - face_value = PyLong_AsLong(arg); - if (face_value < 0 || face_value >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return 0; - } - } else { - PyErr_Format(PyExc_TypeError, "an integer is required"); - return 0; - } - - *value = face_value; - return 1; -} +#define PY_SSIZE_T_CLEAN -static int -bytes_getbuffer(PyByteArrayObject *obj, Py_buffer *view, int flags) -{ - int ret; - void *ptr; - if (view == NULL) { - obj->ob_exports++; - return 0; - } - if (obj->ob_bytes == NULL) - ptr = ""; - else - ptr = obj->ob_bytes; - ret = PyBuffer_FillInfo(view, ptr, Py_SIZE(obj), 0, flags); - if (ret >= 0) { - obj->ob_exports++; - } - return ret; -} +#include "Python.h" -static void -bytes_releasebuffer(PyByteArrayObject *obj, Py_buffer *view) -{ - obj->ob_exports--; -} +#include "bytes_methods.h" static Py_ssize_t _getbuffer(PyObject *obj, Py_buffer *view) @@ -93,1253 +27,1809 @@ _getbuffer(PyObject *obj, Py_buffer *view) return view->len; } -/* Direct API functions */ +#ifdef COUNT_ALLOCS +int null_strings, one_strings; +#endif +static PyBytesObject *characters[UCHAR_MAX + 1]; +static PyBytesObject *nullstring; + +/* + For both PyBytes_FromString() and PyBytes_FromStringAndSize(), the + parameter `size' denotes number of characters to allocate, not counting any + null terminating character. + + For PyBytes_FromString(), the parameter `str' points to a null-terminated + string containing exactly `size' bytes. + + For PyBytes_FromStringAndSize(), the parameter the parameter `str' is + either NULL or else points to a string containing at least `size' bytes. + For PyBytes_FromStringAndSize(), the string in the `str' parameter does + not have to be null-terminated. (Therefore it is safe to construct a + substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) + If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' + bytes (setting the last byte to the null terminating character) and you can + fill in the data yourself. If `str' is non-NULL then the resulting + PyString object must be treated as immutable and you must not fill in nor + alter the data yourself, since the strings may be shared. + + The PyObject member `op->ob_size', which denotes the number of "extra + items" in a variable-size object, will contain the number of bytes + allocated for string data, not counting the null terminating character. It + is therefore equal to the equal to the `size' parameter (for + PyBytes_FromStringAndSize()) or the length of the string in the `str' + parameter (for PyBytes_FromString()). +*/ PyObject * -PyByteArray_FromObject(PyObject *input) +PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) { - return PyObject_CallFunctionObjArgs((PyObject *)&PyByteArray_Type, - input, NULL); + register PyBytesObject *op; + if (size < 0) { + PyErr_SetString(PyExc_SystemError, + "Negative size passed to PyBytes_FromStringAndSize"); + return NULL; + } + if (size == 0 && (op = nullstring) != NULL) { +#ifdef COUNT_ALLOCS + null_strings++; +#endif + Py_INCREF(op); + return (PyObject *)op; + } + if (size == 1 && str != NULL && + (op = characters[*str & UCHAR_MAX]) != NULL) + { +#ifdef COUNT_ALLOCS + one_strings++; +#endif + Py_INCREF(op); + return (PyObject *)op; + } + + /* Inline PyObject_NewVar */ + op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size); + if (op == NULL) + return PyErr_NoMemory(); + PyObject_INIT_VAR(op, &PyBytes_Type, size); + op->ob_shash = -1; + if (str != NULL) + Py_MEMCPY(op->ob_sval, str, size); + op->ob_sval[size] = '\0'; + /* share short strings */ + if (size == 0) { + nullstring = op; + Py_INCREF(op); + } else if (size == 1 && str != NULL) { + characters[*str & UCHAR_MAX] = op; + Py_INCREF(op); + } + return (PyObject *) op; } PyObject * -PyByteArray_FromStringAndSize(const char *bytes, Py_ssize_t size) +PyBytes_FromString(const char *str) { - PyByteArrayObject *new; - Py_ssize_t alloc; + register size_t size; + register PyBytesObject *op; - if (size < 0) { - PyErr_SetString(PyExc_SystemError, - "Negative size passed to PyByteArray_FromStringAndSize"); - return NULL; - } + assert(str != NULL); + size = strlen(str); + if (size > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "string is too long for a Python string"); + return NULL; + } + if (size == 0 && (op = nullstring) != NULL) { +#ifdef COUNT_ALLOCS + null_strings++; +#endif + Py_INCREF(op); + return (PyObject *)op; + } + if (size == 1 && (op = characters[*str & UCHAR_MAX]) != NULL) { +#ifdef COUNT_ALLOCS + one_strings++; +#endif + Py_INCREF(op); + return (PyObject *)op; + } - new = PyObject_New(PyByteArrayObject, &PyByteArray_Type); - if (new == NULL) - return NULL; + /* Inline PyObject_NewVar */ + op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size); + if (op == NULL) + return PyErr_NoMemory(); + PyObject_INIT_VAR(op, &PyBytes_Type, size); + op->ob_shash = -1; + Py_MEMCPY(op->ob_sval, str, size+1); + /* share short strings */ + if (size == 0) { + nullstring = op; + Py_INCREF(op); + } else if (size == 1) { + characters[*str & UCHAR_MAX] = op; + Py_INCREF(op); + } + return (PyObject *) op; +} - if (size == 0) { - new->ob_bytes = NULL; - alloc = 0; - } - else { - alloc = size + 1; - new->ob_bytes = PyMem_Malloc(alloc); - if (new->ob_bytes == NULL) { - Py_DECREF(new); - return PyErr_NoMemory(); - } - if (bytes != NULL) - memcpy(new->ob_bytes, bytes, size); - new->ob_bytes[size] = '\0'; /* Trailing null byte */ - } - Py_SIZE(new) = size; - new->ob_alloc = alloc; - new->ob_exports = 0; +PyObject * +PyBytes_FromFormatV(const char *format, va_list vargs) +{ + va_list count; + Py_ssize_t n = 0; + const char* f; + char *s; + PyObject* string; - return (PyObject *)new; +#ifdef VA_LIST_IS_ARRAY + Py_MEMCPY(count, vargs, sizeof(va_list)); +#else +#ifdef __va_copy + __va_copy(count, vargs); +#else + count = vargs; +#endif +#endif + /* step 1: figure out how large a buffer we need */ + for (f = format; *f; f++) { + if (*f == '%') { + const char* p = f; + while (*++f && *f != '%' && !ISALPHA(*f)) + ; + + /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since + * they don't affect the amount of space we reserve. + */ + if ((*f == 'l' || *f == 'z') && + (f[1] == 'd' || f[1] == 'u')) + ++f; + + switch (*f) { + case 'c': + (void)va_arg(count, int); + /* fall through... */ + case '%': + n++; + break; + case 'd': case 'u': case 'i': case 'x': + (void) va_arg(count, int); + /* 20 bytes is enough to hold a 64-bit + integer. Decimal takes the most space. + This isn't enough for octal. */ + n += 20; + break; + case 's': + s = va_arg(count, char*); + n += strlen(s); + break; + case 'p': + (void) va_arg(count, int); + /* maximum 64-bit pointer representation: + * 0xffffffffffffffff + * so 19 characters is enough. + * XXX I count 18 -- what's the extra for? + */ + n += 19; + break; + default: + /* if we stumble upon an unknown + formatting code, copy the rest of + the format string to the output + string. (we cannot just skip the + code, since there's no way to know + what's in the argument list) */ + n += strlen(p); + goto expand; + } + } else + n++; + } + expand: + /* step 2: fill the buffer */ + /* Since we've analyzed how much space we need for the worst case, + use sprintf directly instead of the slower PyOS_snprintf. */ + string = PyBytes_FromStringAndSize(NULL, n); + if (!string) + return NULL; + + s = PyBytes_AsString(string); + + for (f = format; *f; f++) { + if (*f == '%') { + const char* p = f++; + Py_ssize_t i; + int longflag = 0; + int size_tflag = 0; + /* parse the width.precision part (we're only + interested in the precision value, if any) */ + n = 0; + while (ISDIGIT(*f)) + n = (n*10) + *f++ - '0'; + if (*f == '.') { + f++; + n = 0; + while (ISDIGIT(*f)) + n = (n*10) + *f++ - '0'; + } + while (*f && *f != '%' && !ISALPHA(*f)) + f++; + /* handle the long flag, but only for %ld and %lu. + others can be added when necessary. */ + if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { + longflag = 1; + ++f; + } + /* handle the size_t flag. */ + if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { + size_tflag = 1; + ++f; + } + + switch (*f) { + case 'c': + *s++ = va_arg(vargs, int); + break; + case 'd': + if (longflag) + sprintf(s, "%ld", va_arg(vargs, long)); + else if (size_tflag) + sprintf(s, "%" PY_FORMAT_SIZE_T "d", + va_arg(vargs, Py_ssize_t)); + else + sprintf(s, "%d", va_arg(vargs, int)); + s += strlen(s); + break; + case 'u': + if (longflag) + sprintf(s, "%lu", + va_arg(vargs, unsigned long)); + else if (size_tflag) + sprintf(s, "%" PY_FORMAT_SIZE_T "u", + va_arg(vargs, size_t)); + else + sprintf(s, "%u", + va_arg(vargs, unsigned int)); + s += strlen(s); + break; + case 'i': + sprintf(s, "%i", va_arg(vargs, int)); + s += strlen(s); + break; + case 'x': + sprintf(s, "%x", va_arg(vargs, int)); + s += strlen(s); + break; + case 's': + p = va_arg(vargs, char*); + i = strlen(p); + if (n > 0 && i > n) + i = n; + Py_MEMCPY(s, p, i); + s += i; + break; + case 'p': + sprintf(s, "%p", va_arg(vargs, void*)); + /* %p is ill-defined: ensure leading 0x. */ + if (s[1] == 'X') + s[1] = 'x'; + else if (s[1] != 'x') { + memmove(s+2, s, strlen(s)+1); + s[0] = '0'; + s[1] = 'x'; + } + s += strlen(s); + break; + case '%': + *s++ = '%'; + break; + default: + strcpy(s, p); + s += strlen(s); + goto end; + } + } else + *s++ = *f; + } + + end: + _PyBytes_Resize(&string, s - PyBytes_AS_STRING(string)); + return string; } -Py_ssize_t -PyByteArray_Size(PyObject *self) +PyObject * +PyBytes_FromFormat(const char *format, ...) { - assert(self != NULL); - assert(PyByteArray_Check(self)); + PyObject* ret; + va_list vargs; + +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + ret = PyBytes_FromFormatV(format, vargs); + va_end(vargs); + return ret; +} - return PyByteArray_GET_SIZE(self); +static void +string_dealloc(PyObject *op) +{ + Py_TYPE(op)->tp_free(op); +} + +/* Unescape a backslash-escaped string. If unicode is non-zero, + the string is a u-literal. If recode_encoding is non-zero, + the string is UTF-8 encoded and should be re-encoded in the + specified encoding. */ + +PyObject *PyBytes_DecodeEscape(const char *s, + Py_ssize_t len, + const char *errors, + Py_ssize_t unicode, + const char *recode_encoding) +{ + int c; + char *p, *buf; + const char *end; + PyObject *v; + Py_ssize_t newlen = recode_encoding ? 4*len:len; + v = PyBytes_FromStringAndSize((char *)NULL, newlen); + if (v == NULL) + return NULL; + p = buf = PyBytes_AsString(v); + end = s + len; + while (s < end) { + if (*s != '\\') { + non_esc: + if (recode_encoding && (*s & 0x80)) { + PyObject *u, *w; + char *r; + const char* t; + Py_ssize_t rn; + t = s; + /* Decode non-ASCII bytes as UTF-8. */ + while (t < end && (*t & 0x80)) t++; + u = PyUnicode_DecodeUTF8(s, t - s, errors); + if(!u) goto failed; + + /* Recode them in target encoding. */ + w = PyUnicode_AsEncodedString( + u, recode_encoding, errors); + Py_DECREF(u); + if (!w) goto failed; + + /* Append bytes to output buffer. */ + assert(PyBytes_Check(w)); + r = PyBytes_AS_STRING(w); + rn = PyBytes_GET_SIZE(w); + Py_MEMCPY(p, r, rn); + p += rn; + Py_DECREF(w); + s = t; + } else { + *p++ = *s++; + } + continue; + } + s++; + if (s==end) { + PyErr_SetString(PyExc_ValueError, + "Trailing \\ in string"); + goto failed; + } + switch (*s++) { + /* XXX This assumes ASCII! */ + case '\n': break; + case '\\': *p++ = '\\'; break; + case '\'': *p++ = '\''; break; + case '\"': *p++ = '\"'; break; + case 'b': *p++ = '\b'; break; + case 'f': *p++ = '\014'; break; /* FF */ + case 't': *p++ = '\t'; break; + case 'n': *p++ = '\n'; break; + case 'r': *p++ = '\r'; break; + case 'v': *p++ = '\013'; break; /* VT */ + case 'a': *p++ = '\007'; break; /* BEL, not classic C */ + case '0': case '1': case '2': case '3': + case '4': case '5': case '6': case '7': + c = s[-1] - '0'; + if (s < end && '0' <= *s && *s <= '7') { + c = (c<<3) + *s++ - '0'; + if (s < end && '0' <= *s && *s <= '7') + c = (c<<3) + *s++ - '0'; + } + *p++ = c; + break; + case 'x': + if (s+1 < end && ISXDIGIT(s[0]) && ISXDIGIT(s[1])) { + unsigned int x = 0; + c = Py_CHARMASK(*s); + s++; + if (ISDIGIT(c)) + x = c - '0'; + else if (ISLOWER(c)) + x = 10 + c - 'a'; + else + x = 10 + c - 'A'; + x = x << 4; + c = Py_CHARMASK(*s); + s++; + if (ISDIGIT(c)) + x += c - '0'; + else if (ISLOWER(c)) + x += 10 + c - 'a'; + else + x += 10 + c - 'A'; + *p++ = x; + break; + } + if (!errors || strcmp(errors, "strict") == 0) { + PyErr_SetString(PyExc_ValueError, + "invalid \\x escape"); + goto failed; + } + if (strcmp(errors, "replace") == 0) { + *p++ = '?'; + } else if (strcmp(errors, "ignore") == 0) + /* do nothing */; + else { + PyErr_Format(PyExc_ValueError, + "decoding error; unknown " + "error handling code: %.400s", + errors); + goto failed; + } + default: + *p++ = '\\'; + s--; + goto non_esc; /* an arbitry number of unescaped + UTF-8 bytes may follow. */ + } + } + if (p-buf < newlen) + _PyBytes_Resize(&v, p - buf); + return v; + failed: + Py_DECREF(v); + return NULL; } -char * -PyByteArray_AsString(PyObject *self) +/* -------------------------------------------------------------------- */ +/* object api */ + +Py_ssize_t +PyBytes_Size(register PyObject *op) { - assert(self != NULL); - assert(PyByteArray_Check(self)); + if (!PyBytes_Check(op)) { + PyErr_Format(PyExc_TypeError, + "expected bytes, %.200s found", Py_TYPE(op)->tp_name); + return -1; + } + return Py_SIZE(op); +} - return PyByteArray_AS_STRING(self); +char * +PyBytes_AsString(register PyObject *op) +{ + if (!PyBytes_Check(op)) { + PyErr_Format(PyExc_TypeError, + "expected bytes, %.200s found", Py_TYPE(op)->tp_name); + return NULL; + } + return ((PyBytesObject *)op)->ob_sval; } int -PyByteArray_Resize(PyObject *self, Py_ssize_t size) +PyBytes_AsStringAndSize(register PyObject *obj, + register char **s, + register Py_ssize_t *len) { - void *sval; - Py_ssize_t alloc = ((PyByteArrayObject *)self)->ob_alloc; + if (s == NULL) { + PyErr_BadInternalCall(); + return -1; + } - assert(self != NULL); - assert(PyByteArray_Check(self)); - assert(size >= 0); + if (!PyBytes_Check(obj)) { + PyErr_Format(PyExc_TypeError, + "expected bytes, %.200s found", Py_TYPE(obj)->tp_name); + return -1; + } - if (size < alloc / 2) { - /* Major downsize; resize down to exact size */ - alloc = size + 1; - } - else if (size < alloc) { - /* Within allocated size; quick exit */ - Py_SIZE(self) = size; - ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null */ - return 0; - } - else if (size <= alloc * 1.125) { - /* Moderate upsize; overallocate similar to list_resize() */ - alloc = size + (size >> 3) + (size < 9 ? 3 : 6); - } - else { - /* Major upsize; resize up to exact size */ - alloc = size + 1; - } + *s = PyBytes_AS_STRING(obj); + if (len != NULL) + *len = PyBytes_GET_SIZE(obj); + else if (strlen(*s) != (size_t)PyBytes_GET_SIZE(obj)) { + PyErr_SetString(PyExc_TypeError, + "expected bytes with no null"); + return -1; + } + return 0; +} - if (((PyByteArrayObject *)self)->ob_exports > 0) { - /* - fprintf(stderr, "%d: %s", ((PyByteArrayObject *)self)->ob_exports, - ((PyByteArrayObject *)self)->ob_bytes); - */ - PyErr_SetString(PyExc_BufferError, - "Existing exports of data: object cannot be re-sized"); - return -1; - } +/* -------------------------------------------------------------------- */ +/* Methods */ - sval = PyMem_Realloc(((PyByteArrayObject *)self)->ob_bytes, alloc); - if (sval == NULL) { - PyErr_NoMemory(); - return -1; - } +#define STRINGLIB_CHAR char - ((PyByteArrayObject *)self)->ob_bytes = sval; - Py_SIZE(self) = size; - ((PyByteArrayObject *)self)->ob_alloc = alloc; - ((PyByteArrayObject *)self)->ob_bytes[size] = '\0'; /* Trailing null byte */ +#define STRINGLIB_CMP memcmp +#define STRINGLIB_LEN PyBytes_GET_SIZE +#define STRINGLIB_NEW PyBytes_FromStringAndSize +#define STRINGLIB_STR PyBytes_AS_STRING +/* #define STRINGLIB_WANT_CONTAINS_OBJ 1 */ - return 0; -} +#define STRINGLIB_EMPTY nullstring +#define STRINGLIB_CHECK_EXACT PyBytes_CheckExact +#define STRINGLIB_MUTABLE 0 -PyObject * -PyByteArray_Concat(PyObject *a, PyObject *b) -{ - Py_ssize_t size; - Py_buffer va, vb; - PyByteArrayObject *result = NULL; - - va.len = -1; - vb.len = -1; - if (_getbuffer(a, &va) < 0 || - _getbuffer(b, &vb) < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); - goto done; - } +#include "stringlib/fastsearch.h" - size = va.len + vb.len; - if (size < 0) { - return PyErr_NoMemory(); - goto done; - } +#include "stringlib/count.h" +#include "stringlib/find.h" +#include "stringlib/partition.h" +#include "stringlib/ctype.h" +#include "stringlib/transmogrify.h" - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, size); - if (result != NULL) { - memcpy(result->ob_bytes, va.buf, va.len); - memcpy(result->ob_bytes + va.len, vb.buf, vb.len); - } +#define _Py_InsertThousandsGrouping _PyBytes_InsertThousandsGrouping +#include "stringlib/localeutil.h" - done: - if (va.len != -1) - PyObject_ReleaseBuffer(a, &va); - if (vb.len != -1) - PyObject_ReleaseBuffer(b, &vb); - return (PyObject *)result; +PyObject * +PyBytes_Repr(PyObject *obj, int smartquotes) +{ + static const char *hexdigits = "0123456789abcdef"; + register PyBytesObject* op = (PyBytesObject*) obj; + Py_ssize_t length = Py_SIZE(op); + size_t newsize = 3 + 4 * length; + PyObject *v; + if (newsize > PY_SSIZE_T_MAX || (newsize-3) / 4 != length) { + PyErr_SetString(PyExc_OverflowError, + "bytes object is too large to make repr"); + return NULL; + } + v = PyUnicode_FromUnicode(NULL, newsize); + if (v == NULL) { + return NULL; + } + else { + register Py_ssize_t i; + register Py_UNICODE c; + register Py_UNICODE *p = PyUnicode_AS_UNICODE(v); + int quote; + + /* Figure out which quote to use; single is preferred */ + quote = '\''; + if (smartquotes) { + char *test, *start; + start = PyBytes_AS_STRING(op); + for (test = start; test < start+length; ++test) { + if (*test == '"') { + quote = '\''; /* back to single */ + goto decided; + } + else if (*test == '\'') + quote = '"'; + } + decided: + ; + } + + *p++ = 'b', *p++ = quote; + for (i = 0; i < length; i++) { + /* There's at least enough room for a hex escape + and a closing quote. */ + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); + c = op->ob_sval[i]; + if (c == quote || c == '\\') + *p++ = '\\', *p++ = c; + else if (c == '\t') + *p++ = '\\', *p++ = 't'; + else if (c == '\n') + *p++ = '\\', *p++ = 'n'; + else if (c == '\r') + *p++ = '\\', *p++ = 'r'; + else if (c < ' ' || c >= 0x7f) { + *p++ = '\\'; + *p++ = 'x'; + *p++ = hexdigits[(c & 0xf0) >> 4]; + *p++ = hexdigits[c & 0xf]; + } + else + *p++ = c; + } + assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); + *p++ = quote; + *p = '\0'; + if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { + Py_DECREF(v); + return NULL; + } + return v; + } +} + +static PyObject * +string_repr(PyObject *op) +{ + return PyBytes_Repr(op, 1); } -/* Functions stuffed into the type object */ +static PyObject * +string_str(PyObject *op) +{ + if (Py_BytesWarningFlag) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "str() on a bytes instance", 1)) + return NULL; + } + return string_repr(op); +} static Py_ssize_t -bytes_length(PyByteArrayObject *self) +string_length(PyBytesObject *a) { - return Py_SIZE(self); + return Py_SIZE(a); } +/* This is also used by PyBytes_Concat() */ static PyObject * -bytes_iconcat(PyByteArrayObject *self, PyObject *other) -{ - Py_ssize_t mysize; - Py_ssize_t size; - Py_buffer vo; +string_concat(PyObject *a, PyObject *b) +{ + Py_ssize_t size; + Py_buffer va, vb; + PyObject *result = NULL; + + va.len = -1; + vb.len = -1; + if (_getbuffer(a, &va) < 0 || + _getbuffer(b, &vb) < 0) { + PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", + Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); + goto done; + } - if (_getbuffer(other, &vo) < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(other)->tp_name, Py_TYPE(self)->tp_name); - return NULL; - } + /* Optimize end cases */ + if (va.len == 0 && PyBytes_CheckExact(b)) { + result = b; + Py_INCREF(result); + goto done; + } + if (vb.len == 0 && PyBytes_CheckExact(a)) { + result = a; + Py_INCREF(result); + goto done; + } - mysize = Py_SIZE(self); - size = mysize + vo.len; - if (size < 0) { - PyObject_ReleaseBuffer(other, &vo); - return PyErr_NoMemory(); - } - if (size < self->ob_alloc) { - Py_SIZE(self) = size; - self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ - } - else if (PyByteArray_Resize((PyObject *)self, size) < 0) { - PyObject_ReleaseBuffer(other, &vo); - return NULL; - } - memcpy(self->ob_bytes + mysize, vo.buf, vo.len); - PyObject_ReleaseBuffer(other, &vo); - Py_INCREF(self); - return (PyObject *)self; + size = va.len + vb.len; + if (size < 0) { + PyErr_NoMemory(); + goto done; + } + + result = PyBytes_FromStringAndSize(NULL, size); + if (result != NULL) { + memcpy(PyBytes_AS_STRING(result), va.buf, va.len); + memcpy(PyBytes_AS_STRING(result) + va.len, vb.buf, vb.len); + } + + done: + if (va.len != -1) + PyObject_ReleaseBuffer(a, &va); + if (vb.len != -1) + PyObject_ReleaseBuffer(b, &vb); + return result; } static PyObject * -bytes_repeat(PyByteArrayObject *self, Py_ssize_t count) -{ - PyByteArrayObject *result; - Py_ssize_t mysize; - Py_ssize_t size; - - if (count < 0) - count = 0; - mysize = Py_SIZE(self); - size = mysize * count; - if (count != 0 && size / count != mysize) - return PyErr_NoMemory(); - result = (PyByteArrayObject *)PyByteArray_FromStringAndSize(NULL, size); - if (result != NULL && size != 0) { - if (mysize == 1) - memset(result->ob_bytes, self->ob_bytes[0], size); - else { - Py_ssize_t i; - for (i = 0; i < count; i++) - memcpy(result->ob_bytes + i*mysize, self->ob_bytes, mysize); - } - } - return (PyObject *)result; +string_repeat(register PyBytesObject *a, register Py_ssize_t n) +{ + register Py_ssize_t i; + register Py_ssize_t j; + register Py_ssize_t size; + register PyBytesObject *op; + size_t nbytes; + if (n < 0) + n = 0; + /* watch out for overflows: the size can overflow int, + * and the # of bytes needed can overflow size_t + */ + size = Py_SIZE(a) * n; + if (n && size / n != Py_SIZE(a)) { + PyErr_SetString(PyExc_OverflowError, + "repeated string is too long"); + return NULL; + } + if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { + Py_INCREF(a); + return (PyObject *)a; + } + nbytes = (size_t)size; + if (nbytes + sizeof(PyBytesObject) <= nbytes) { + PyErr_SetString(PyExc_OverflowError, + "repeated string is too long"); + return NULL; + } + op = (PyBytesObject *) + PyObject_MALLOC(sizeof(PyBytesObject) + nbytes); + if (op == NULL) + return PyErr_NoMemory(); + PyObject_INIT_VAR(op, &PyBytes_Type, size); + op->ob_shash = -1; + op->ob_sval[size] = '\0'; + if (Py_SIZE(a) == 1 && n > 0) { + memset(op->ob_sval, a->ob_sval[0] , n); + return (PyObject *) op; + } + i = 0; + if (i < size) { + Py_MEMCPY(op->ob_sval, a->ob_sval, Py_SIZE(a)); + i = Py_SIZE(a); + } + while (i < size) { + j = (i <= size-i) ? i : size-i; + Py_MEMCPY(op->ob_sval+i, op->ob_sval, j); + i += j; + } + return (PyObject *) op; } -static PyObject * -bytes_irepeat(PyByteArrayObject *self, Py_ssize_t count) -{ - Py_ssize_t mysize; - Py_ssize_t size; - - if (count < 0) - count = 0; - mysize = Py_SIZE(self); - size = mysize * count; - if (count != 0 && size / count != mysize) - return PyErr_NoMemory(); - if (size < self->ob_alloc) { - Py_SIZE(self) = size; - self->ob_bytes[Py_SIZE(self)] = '\0'; /* Trailing null byte */ +static int +string_contains(PyObject *self, PyObject *arg) +{ + Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); + if (ival == -1 && PyErr_Occurred()) { + Py_buffer varg; + int pos; + PyErr_Clear(); + if (_getbuffer(arg, &varg) < 0) + return -1; + pos = stringlib_find(PyBytes_AS_STRING(self), Py_SIZE(self), + varg.buf, varg.len, 0); + PyObject_ReleaseBuffer(arg, &varg); + return pos >= 0; } - else if (PyByteArray_Resize((PyObject *)self, size) < 0) - return NULL; - - if (mysize == 1) - memset(self->ob_bytes, self->ob_bytes[0], size); - else { - Py_ssize_t i; - for (i = 1; i < count; i++) - memcpy(self->ob_bytes + i*mysize, self->ob_bytes, mysize); + if (ival < 0 || ival >= 256) { + PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); + return -1; } - Py_INCREF(self); - return (PyObject *)self; + return memchr(PyBytes_AS_STRING(self), ival, Py_SIZE(self)) != NULL; } static PyObject * -bytes_getitem(PyByteArrayObject *self, Py_ssize_t i) +string_item(PyBytesObject *a, register Py_ssize_t i) { - if (i < 0) - i += Py_SIZE(self); - if (i < 0 || i >= Py_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); - return NULL; - } - return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); + if (i < 0 || i >= Py_SIZE(a)) { + PyErr_SetString(PyExc_IndexError, "string index out of range"); + return NULL; + } + return PyLong_FromLong((unsigned char)a->ob_sval[i]); +} + +static PyObject* +string_richcompare(PyBytesObject *a, PyBytesObject *b, int op) +{ + int c; + Py_ssize_t len_a, len_b; + Py_ssize_t min_len; + PyObject *result; + + /* Make sure both arguments are strings. */ + if (!(PyBytes_Check(a) && PyBytes_Check(b))) { + if (Py_BytesWarningFlag && (op == Py_EQ) && + (PyObject_IsInstance((PyObject*)a, + (PyObject*)&PyUnicode_Type) || + PyObject_IsInstance((PyObject*)b, + (PyObject*)&PyUnicode_Type))) { + if (PyErr_WarnEx(PyExc_BytesWarning, + "Comparsion between bytes and string", 1)) + return NULL; + } + result = Py_NotImplemented; + goto out; + } + if (a == b) { + switch (op) { + case Py_EQ:case Py_LE:case Py_GE: + result = Py_True; + goto out; + case Py_NE:case Py_LT:case Py_GT: + result = Py_False; + goto out; + } + } + if (op == Py_EQ) { + /* Supporting Py_NE here as well does not save + much time, since Py_NE is rarely used. */ + if (Py_SIZE(a) == Py_SIZE(b) + && (a->ob_sval[0] == b->ob_sval[0] + && memcmp(a->ob_sval, b->ob_sval, Py_SIZE(a)) == 0)) { + result = Py_True; + } else { + result = Py_False; + } + goto out; + } + len_a = Py_SIZE(a); len_b = Py_SIZE(b); + min_len = (len_a < len_b) ? len_a : len_b; + if (min_len > 0) { + c = Py_CHARMASK(*a->ob_sval) - Py_CHARMASK(*b->ob_sval); + if (c==0) + c = memcmp(a->ob_sval, b->ob_sval, min_len); + } else + c = 0; + if (c == 0) + c = (len_a < len_b) ? -1 : (len_a > len_b) ? 1 : 0; + switch (op) { + case Py_LT: c = c < 0; break; + case Py_LE: c = c <= 0; break; + case Py_EQ: assert(0); break; /* unreachable */ + case Py_NE: c = c != 0; break; + case Py_GT: c = c > 0; break; + case Py_GE: c = c >= 0; break; + default: + result = Py_NotImplemented; + goto out; + } + result = c ? Py_True : Py_False; + out: + Py_INCREF(result); + return result; +} + +static long +string_hash(PyBytesObject *a) +{ + register Py_ssize_t len; + register unsigned char *p; + register long x; + + if (a->ob_shash != -1) + return a->ob_shash; + len = Py_SIZE(a); + p = (unsigned char *) a->ob_sval; + x = *p << 7; + while (--len >= 0) + x = (1000003*x) ^ *p++; + x ^= Py_SIZE(a); + if (x == -1) + x = -2; + a->ob_shash = x; + return x; +} + +static PyObject* +string_subscript(PyBytesObject* self, PyObject* item) +{ + if (PyIndex_Check(item)) { + Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) + return NULL; + if (i < 0) + i += PyBytes_GET_SIZE(self); + if (i < 0 || i >= PyBytes_GET_SIZE(self)) { + PyErr_SetString(PyExc_IndexError, + "string index out of range"); + return NULL; + } + return PyLong_FromLong((unsigned char)self->ob_sval[i]); + } + else if (PySlice_Check(item)) { + Py_ssize_t start, stop, step, slicelength, cur, i; + char* source_buf; + char* result_buf; + PyObject* result; + + if (PySlice_GetIndicesEx((PySliceObject*)item, + PyBytes_GET_SIZE(self), + &start, &stop, &step, &slicelength) < 0) { + return NULL; + } + + if (slicelength <= 0) { + return PyBytes_FromStringAndSize("", 0); + } + else if (start == 0 && step == 1 && + slicelength == PyBytes_GET_SIZE(self) && + PyBytes_CheckExact(self)) { + Py_INCREF(self); + return (PyObject *)self; + } + else if (step == 1) { + return PyBytes_FromStringAndSize( + PyBytes_AS_STRING(self) + start, + slicelength); + } + else { + source_buf = PyBytes_AsString((PyObject*)self); + result_buf = (char *)PyMem_Malloc(slicelength); + if (result_buf == NULL) + return PyErr_NoMemory(); + + for (cur = start, i = 0; i < slicelength; + cur += step, i++) { + result_buf[i] = source_buf[cur]; + } + + result = PyBytes_FromStringAndSize(result_buf, + slicelength); + PyMem_Free(result_buf); + return result; + } + } + else { + PyErr_Format(PyExc_TypeError, + "string indices must be integers, not %.200s", + Py_TYPE(item)->tp_name); + return NULL; + } } -static PyObject * -bytes_subscript(PyByteArrayObject *self, PyObject *item) +static int +string_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags) { - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_SIZE(self), + 0, flags); +} - if (i == -1 && PyErr_Occurred()) - return NULL; +static PySequenceMethods string_as_sequence = { + (lenfunc)string_length, /*sq_length*/ + (binaryfunc)string_concat, /*sq_concat*/ + (ssizeargfunc)string_repeat, /*sq_repeat*/ + (ssizeargfunc)string_item, /*sq_item*/ + 0, /*sq_slice*/ + 0, /*sq_ass_item*/ + 0, /*sq_ass_slice*/ + (objobjproc)string_contains /*sq_contains*/ +}; - if (i < 0) - i += PyByteArray_GET_SIZE(self); +static PyMappingMethods string_as_mapping = { + (lenfunc)string_length, + (binaryfunc)string_subscript, + 0, +}; - if (i < 0 || i >= Py_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); - return NULL; - } - return PyLong_FromLong((unsigned char)(self->ob_bytes[i])); - } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, cur, i; - if (PySlice_GetIndicesEx((PySliceObject *)item, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { - return NULL; - } - - if (slicelength <= 0) - return PyByteArray_FromStringAndSize("", 0); - else if (step == 1) { - return PyByteArray_FromStringAndSize(self->ob_bytes + start, - slicelength); - } - else { - char *source_buf = PyByteArray_AS_STRING(self); - char *result_buf = (char *)PyMem_Malloc(slicelength); - PyObject *result; - - if (result_buf == NULL) - return PyErr_NoMemory(); - - for (cur = start, i = 0; i < slicelength; - cur += step, i++) { - result_buf[i] = source_buf[cur]; - } - result = PyByteArray_FromStringAndSize(result_buf, slicelength); - PyMem_Free(result_buf); - return result; - } - } - else { - PyErr_SetString(PyExc_TypeError, "bytearray indices must be integers"); - return NULL; - } -} +static PyBufferProcs string_as_buffer = { + (getbufferproc)string_buffer_getbuffer, + NULL, +}; -static int -bytes_setslice(PyByteArrayObject *self, Py_ssize_t lo, Py_ssize_t hi, - PyObject *values) -{ - Py_ssize_t avail, needed; - void *bytes; - Py_buffer vbytes; - int res = 0; - - vbytes.len = -1; - if (values == (PyObject *)self) { - /* Make a copy and call this function recursively */ - int err; - values = PyByteArray_FromObject(values); - if (values == NULL) - return -1; - err = bytes_setslice(self, lo, hi, values); - Py_DECREF(values); - return err; - } - if (values == NULL) { - /* del b[lo:hi] */ - bytes = NULL; - needed = 0; - } - else { - if (_getbuffer(values, &vbytes) < 0) { - PyErr_Format(PyExc_TypeError, - "can't set bytes slice from %.100s", - Py_TYPE(values)->tp_name); - return -1; - } - needed = vbytes.len; - bytes = vbytes.buf; - } - if (lo < 0) - lo = 0; - if (hi < lo) - hi = lo; - if (hi > Py_SIZE(self)) - hi = Py_SIZE(self); - - avail = hi - lo; - if (avail < 0) - lo = hi = avail = 0; - - if (avail != needed) { - if (avail > needed) { - /* - 0 lo hi old_size - | |<----avail----->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_SIZE(self) - hi); - } - /* XXX(nnorwitz): need to verify this can't overflow! */ - if (PyByteArray_Resize((PyObject *)self, - Py_SIZE(self) + needed - avail) < 0) { - res = -1; - goto finish; - } - if (avail < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + lo + needed, self->ob_bytes + hi, - Py_SIZE(self) - lo - needed); - } - } +#define LEFTSTRIP 0 +#define RIGHTSTRIP 1 +#define BOTHSTRIP 2 - if (needed > 0) - memcpy(self->ob_bytes + lo, bytes, needed); +/* Arrays indexed by above */ +static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; +#define STRIPNAME(i) (stripformat[i]+3) - finish: - if (vbytes.len != -1) - PyObject_ReleaseBuffer(values, &vbytes); - return res; -} -static int -bytes_setitem(PyByteArrayObject *self, Py_ssize_t i, PyObject *value) -{ - Py_ssize_t ival; +/* Don't call if length < 2 */ +#define Py_STRING_MATCH(target, offset, pattern, length) \ + (target[offset] == pattern[0] && \ + target[offset+length-1] == pattern[length-1] && \ + !memcmp(target+offset+1, pattern+1, length-2) ) - if (i < 0) - i += Py_SIZE(self); - if (i < 0 || i >= Py_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); - return -1; - } +/* Overallocate the initial list to reduce the number of reallocs for small + split sizes. Eg, "A A A A A A A A A A".split() (10 elements) has three + resizes, to sizes 4, 8, then 16. Most observed string splits are for human + text (roughly 11 words per line) and field delimited data (usually 1-10 + fields). For large strings the split algorithms are bandwidth limited + so increasing the preallocation likely will not improve things.*/ - if (value == NULL) - return bytes_setslice(self, i, i+1, NULL); +#define MAX_PREALLOC 12 - ival = PyNumber_AsSsize_t(value, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) - return -1; +/* 5 splits gives 6 elements */ +#define PREALLOC_SIZE(maxsplit) \ + (maxsplit >= MAX_PREALLOC ? MAX_PREALLOC : maxsplit+1) + +#define SPLIT_ADD(data, left, right) { \ + str = PyBytes_FromStringAndSize((data) + (left), \ + (right) - (left)); \ + if (str == NULL) \ + goto onError; \ + if (count < MAX_PREALLOC) { \ + PyList_SET_ITEM(list, count, str); \ + } else { \ + if (PyList_Append(list, str)) { \ + Py_DECREF(str); \ + goto onError; \ + } \ + else \ + Py_DECREF(str); \ + } \ + count++; } - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } +/* Always force the list to the expected size. */ +#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count - self->ob_bytes[i] = ival; - return 0; -} +#define SKIP_SPACE(s, i, len) { while (i<len && ISSPACE(s[i])) i++; } +#define SKIP_NONSPACE(s, i, len) { while (i<len && !ISSPACE(s[i])) i++; } +#define RSKIP_SPACE(s, i) { while (i>=0 && ISSPACE(s[i])) i--; } +#define RSKIP_NONSPACE(s, i) { while (i>=0 && !ISSPACE(s[i])) i--; } -static int -bytes_ass_subscript(PyByteArrayObject *self, PyObject *item, PyObject *values) -{ - Py_ssize_t start, stop, step, slicelen, needed; - char *bytes; +Py_LOCAL_INLINE(PyObject *) +split_whitespace(PyBytesObject *self, Py_ssize_t len, Py_ssize_t maxsplit) +{ + const char *s = PyBytes_AS_STRING(self); + Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxsplit)); + + if (list == NULL) + return NULL; + + i = j = 0; + + while (maxsplit-- > 0) { + SKIP_SPACE(s, i, len); + if (i==len) break; + j = i; i++; + SKIP_NONSPACE(s, i, len); + if (j == 0 && i == len && PyBytes_CheckExact(self)) { + /* No whitespace in self, so just use it as list[0] */ + Py_INCREF(self); + PyList_SET_ITEM(list, 0, (PyObject *)self); + count++; + break; + } + SPLIT_ADD(s, j, i); + } - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); + if (i < len) { + /* Only occurs when maxsplit was reached */ + /* Skip any remaining whitespace and copy to end of string */ + SKIP_SPACE(s, i, len); + if (i != len) + SPLIT_ADD(s, i, len); + } + FIX_PREALLOC_SIZE(list); + return list; + onError: + Py_DECREF(list); + return NULL; +} - if (i == -1 && PyErr_Occurred()) - return -1; +Py_LOCAL_INLINE(PyObject *) +split_char(PyBytesObject *self, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + const char *s = PyBytes_AS_STRING(self); + register Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = 0; + while ((j < len) && (maxcount-- > 0)) { + for(; j<len; j++) { + /* I found that using memchr makes no difference */ + if (s[j] == ch) { + SPLIT_ADD(s, i, j); + i = j = j + 1; + break; + } + } + } + if (i == 0 && count == 0 && PyBytes_CheckExact(self)) { + /* ch not in self, so just use self as list[0] */ + Py_INCREF(self); + PyList_SET_ITEM(list, 0, (PyObject *)self); + count++; + } + else if (i <= len) { + SPLIT_ADD(s, i, len); + } + FIX_PREALLOC_SIZE(list); + return list; - if (i < 0) - i += PyByteArray_GET_SIZE(self); + onError: + Py_DECREF(list); + return NULL; +} - if (i < 0 || i >= Py_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, "bytearray index out of range"); - return -1; - } - - if (values == NULL) { - /* Fall through to slice assignment */ - start = i; - stop = i + 1; - step = 1; - slicelen = 1; - } - else { - Py_ssize_t ival = PyNumber_AsSsize_t(values, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) - return -1; - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, - "byte must be in range(0, 256)"); - return -1; - } - self->ob_bytes[i] = (char)ival; - return 0; - } - } - else if (PySlice_Check(item)) { - if (PySlice_GetIndicesEx((PySliceObject *)item, - PyByteArray_GET_SIZE(self), - &start, &stop, &step, &slicelen) < 0) { - return -1; - } - } - else { - PyErr_SetString(PyExc_TypeError, "bytearray indices must be integer"); - return -1; - } +PyDoc_STRVAR(split__doc__, +"B.split([sep[, maxsplit]]) -> list of bytes\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter.\n\ +If sep is not specified or is None, B is split on ASCII whitespace\n\ +characters (space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); - if (values == NULL) { - bytes = NULL; - needed = 0; - } - else if (values == (PyObject *)self || !PyByteArray_Check(values)) { - /* Make a copy an call this function recursively */ - int err; - values = PyByteArray_FromObject(values); - if (values == NULL) - return -1; - err = bytes_ass_subscript(self, item, values); - Py_DECREF(values); - return err; - } - else { - assert(PyByteArray_Check(values)); - bytes = ((PyByteArrayObject *)values)->ob_bytes; - needed = Py_SIZE(values); - } - /* Make sure b[5:2] = ... inserts before 5, not before 2. */ - if ((step < 0 && start < stop) || - (step > 0 && start > stop)) - stop = start; - if (step == 1) { - if (slicelen != needed) { - if (slicelen > needed) { - /* - 0 start stop old_size - | |<---slicelen--->|<-----tomove------>| - | |<-needed->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_SIZE(self) - stop); - } - if (PyByteArray_Resize((PyObject *)self, - Py_SIZE(self) + needed - slicelen) < 0) - return -1; - if (slicelen < needed) { - /* - 0 lo hi old_size - | |<-avail->|<-----tomove------>| - | |<----needed---->|<-----tomove------>| - 0 lo new_hi new_size - */ - memmove(self->ob_bytes + start + needed, self->ob_bytes + stop, - Py_SIZE(self) - start - needed); - } - } - - if (needed > 0) - memcpy(self->ob_bytes + start, bytes, needed); - - return 0; - } - else { - if (needed == 0) { - /* Delete slice */ - Py_ssize_t cur, i; - - if (step < 0) { - stop = start + 1; - start = stop + step * (slicelen - 1) - 1; - step = -step; - } - for (cur = start, i = 0; - i < slicelen; cur += step, i++) { - Py_ssize_t lim = step - 1; - - if (cur + step >= PyByteArray_GET_SIZE(self)) - lim = PyByteArray_GET_SIZE(self) - cur - 1; - - memmove(self->ob_bytes + cur - i, - self->ob_bytes + cur + 1, lim); - } - /* Move the tail of the bytes, in one chunk */ - cur = start + slicelen*step; - if (cur < PyByteArray_GET_SIZE(self)) { - memmove(self->ob_bytes + cur - slicelen, - self->ob_bytes + cur, - PyByteArray_GET_SIZE(self) - cur); - } - if (PyByteArray_Resize((PyObject *)self, - PyByteArray_GET_SIZE(self) - slicelen) < 0) - return -1; - - return 0; - } - else { - /* Assign slice */ - Py_ssize_t cur, i; - - if (needed != slicelen) { - PyErr_Format(PyExc_ValueError, - "attempt to assign bytes of size %zd " - "to extended slice of size %zd", - needed, slicelen); - return -1; - } - for (cur = start, i = 0; i < slicelen; cur += step, i++) - self->ob_bytes[cur] = bytes[i]; - return 0; - } - } -} +static PyObject * +string_split(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count=0; + const char *s = PyBytes_AS_STRING(self), *sub; + Py_buffer vsub; + PyObject *list, *str, *subobj = Py_None; +#ifdef USE_FAST + Py_ssize_t pos; +#endif -static int -bytes_init(PyByteArrayObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"source", "encoding", "errors", 0}; - PyObject *arg = NULL; - const char *encoding = NULL; - const char *errors = NULL; - Py_ssize_t count; - PyObject *it; - PyObject *(*iternext)(PyObject *); - - if (Py_SIZE(self) != 0) { - /* Empty previous contents (yes, do this first of all!) */ - if (PyByteArray_Resize((PyObject *)self, 0) < 0) - return -1; - } + if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + if (subobj == Py_None) + return split_whitespace(self, len, maxsplit); + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + else if (n == 1) + return split_char(self, len, sub[0], maxsplit); - /* Parse arguments */ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:bytes", kwlist, - &arg, &encoding, &errors)) - return -1; + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } - /* Make a quick exit if no first argument */ - if (arg == NULL) { - if (encoding != NULL || errors != NULL) { - PyErr_SetString(PyExc_TypeError, - "encoding or errors without sequence argument"); - return -1; - } - return 0; - } +#ifdef USE_FAST + i = j = 0; + while (maxsplit-- > 0) { + pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH); + if (pos < 0) + break; + j = i+pos; + SPLIT_ADD(s, i, j); + i = j + n; + } +#else + i = j = 0; + while ((j+n <= len) && (maxsplit-- > 0)) { + for (; j+n <= len; j++) { + if (Py_STRING_MATCH(s, j, sub, n)) { + SPLIT_ADD(s, i, j); + i = j = j + n; + break; + } + } + } +#endif + SPLIT_ADD(s, i, len); + FIX_PREALLOC_SIZE(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return list; - if (PyUnicode_Check(arg)) { - /* Encode via the codec registry */ - PyObject *encoded, *new; - if (encoding == NULL) { - PyErr_SetString(PyExc_TypeError, - "string argument without an encoding"); - return -1; - } - encoded = PyCodec_Encode(arg, encoding, errors); - if (encoded == NULL) - return -1; - assert(PyBytes_Check(encoded)); - new = bytes_iconcat(self, encoded); - Py_DECREF(encoded); - if (new == NULL) - return -1; - Py_DECREF(new); - return 0; - } + onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; +} - /* If it's not unicode, there can't be encoding or errors */ - if (encoding != NULL || errors != NULL) { - PyErr_SetString(PyExc_TypeError, - "encoding or errors without a string argument"); - return -1; - } +PyDoc_STRVAR(partition__doc__, +"B.partition(sep) -> (head, sep, tail)\n\ +\n\ +Searches for the separator sep in B, and returns the part before it,\n\ +the separator itself, and the part after it. If the separator is not\n\ +found, returns B and two empty bytes objects."); - /* Is it an int? */ - count = PyNumber_AsSsize_t(arg, PyExc_ValueError); - if (count == -1 && PyErr_Occurred()) - PyErr_Clear(); - else { - if (count < 0) { - PyErr_SetString(PyExc_ValueError, "negative count"); - return -1; - } - if (count > 0) { - if (PyByteArray_Resize((PyObject *)self, count)) - return -1; - memset(self->ob_bytes, 0, count); - } - return 0; - } +static PyObject * +string_partition(PyBytesObject *self, PyObject *sep_obj) +{ + const char *sep; + Py_ssize_t sep_len; - /* Use the buffer API */ - if (PyObject_CheckBuffer(arg)) { - Py_ssize_t size; - Py_buffer view; - if (PyObject_GetBuffer(arg, &view, PyBUF_FULL_RO) < 0) - return -1; - size = view.len; - if (PyByteArray_Resize((PyObject *)self, size) < 0) goto fail; - if (PyBuffer_ToContiguous(self->ob_bytes, &view, size, 'C') < 0) - goto fail; - PyObject_ReleaseBuffer(arg, &view); - return 0; - fail: - PyObject_ReleaseBuffer(arg, &view); - return -1; - } + if (PyBytes_Check(sep_obj)) { + sep = PyBytes_AS_STRING(sep_obj); + sep_len = PyBytes_GET_SIZE(sep_obj); + } + else if (PyObject_AsCharBuffer(sep_obj, &sep, &sep_len)) + return NULL; - /* XXX Optimize this if the arguments is a list, tuple */ + return stringlib_partition( + (PyObject*) self, + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sep_obj, sep, sep_len + ); +} - /* Get the iterator */ - it = PyObject_GetIter(arg); - if (it == NULL) - return -1; - iternext = *Py_TYPE(it)->tp_iternext; - - /* Run the iterator to exhaustion */ - for (;;) { - PyObject *item; - Py_ssize_t value; - - /* Get the next item */ - item = iternext(it); - if (item == NULL) { - if (PyErr_Occurred()) { - if (!PyErr_ExceptionMatches(PyExc_StopIteration)) - goto error; - PyErr_Clear(); - } - break; - } - - /* Interpret it as an int (__index__) */ - value = PyNumber_AsSsize_t(item, PyExc_ValueError); - Py_DECREF(item); - if (value == -1 && PyErr_Occurred()) - goto error; - - /* Range check */ - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - - /* Append the byte */ - if (Py_SIZE(self) < self->ob_alloc) - Py_SIZE(self)++; - else if (PyByteArray_Resize((PyObject *)self, Py_SIZE(self)+1) < 0) - goto error; - self->ob_bytes[Py_SIZE(self)-1] = value; - } +PyDoc_STRVAR(rpartition__doc__, +"B.rpartition(sep) -> (tail, sep, head)\n\ +\n\ +Searches for the separator sep in B, starting at the end of B,\n\ +and returns the part before it, the separator itself, and the\n\ +part after it. If the separator is not found, returns two empty\n\ +bytes objects and B."); - /* Clean up and return success */ - Py_DECREF(it); - return 0; +static PyObject * +string_rpartition(PyBytesObject *self, PyObject *sep_obj) +{ + const char *sep; + Py_ssize_t sep_len; - error: - /* Error handling when it != NULL */ - Py_DECREF(it); - return -1; + if (PyBytes_Check(sep_obj)) { + sep = PyBytes_AS_STRING(sep_obj); + sep_len = PyBytes_GET_SIZE(sep_obj); + } + else if (PyObject_AsCharBuffer(sep_obj, &sep, &sep_len)) + return NULL; + + return stringlib_rpartition( + (PyObject*) self, + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sep_obj, sep, sep_len + ); } -/* Mostly copied from string_repr, but without the - "smart quote" functionality. */ -static PyObject * -bytes_repr(PyByteArrayObject *self) -{ - static const char *hexdigits = "0123456789abcdef"; - const char *quote_prefix = "bytearray(b"; - const char *quote_postfix = ")"; - Py_ssize_t length = Py_SIZE(self); - /* 14 == strlen(quote_prefix) + 2 + strlen(quote_postfix) */ - size_t newsize = 14 + 4 * length; - PyObject *v; - if (newsize > PY_SSIZE_T_MAX || newsize / 4 - 3 != length) { - PyErr_SetString(PyExc_OverflowError, - "bytearray object is too large to make repr"); - return NULL; - } - v = PyUnicode_FromUnicode(NULL, newsize); - if (v == NULL) { - return NULL; - } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p; - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - { - char *test, *start; - start = PyByteArray_AS_STRING(self); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; - } - - p = PyUnicode_AS_UNICODE(v); - while (*quote_prefix) - *p++ = *quote_prefix++; - *p++ = quote; - - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = self->ob_bytes[i]; - if (c == '\'' || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c == 0) - *p++ = '\\', *p++ = 'x', *p++ = '0', *p++ = '0'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - while (*quote_postfix) { - *p++ = *quote_postfix++; - } - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; - } - return v; - } +Py_LOCAL_INLINE(PyObject *) +rsplit_whitespace(PyBytesObject *self, Py_ssize_t len, Py_ssize_t maxsplit) +{ + const char *s = PyBytes_AS_STRING(self); + Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxsplit)); + + if (list == NULL) + return NULL; + + i = j = len-1; + + while (maxsplit-- > 0) { + RSKIP_SPACE(s, i); + if (i<0) break; + j = i; i--; + RSKIP_NONSPACE(s, i); + if (j == len-1 && i < 0 && PyBytes_CheckExact(self)) { + /* No whitespace in self, so just use it as list[0] */ + Py_INCREF(self); + PyList_SET_ITEM(list, 0, (PyObject *)self); + count++; + break; + } + SPLIT_ADD(s, i + 1, j + 1); + } + if (i >= 0) { + /* Only occurs when maxsplit was reached. Skip any remaining + whitespace and copy to beginning of string. */ + RSKIP_SPACE(s, i); + if (i >= 0) + SPLIT_ADD(s, 0, i + 1); + + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + return list; + onError: + Py_DECREF(list); + return NULL; } -static PyObject * -bytes_str(PyObject *op) -{ - if (Py_BytesWarningFlag) { - if (PyErr_WarnEx(PyExc_BytesWarning, - "str() on a bytearray instance", 1)) - return NULL; +Py_LOCAL_INLINE(PyObject *) +rsplit_char(PyBytesObject *self, Py_ssize_t len, char ch, Py_ssize_t maxcount) +{ + const char *s = PyBytes_AS_STRING(self); + register Py_ssize_t i, j, count=0; + PyObject *str; + PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); + + if (list == NULL) + return NULL; + + i = j = len - 1; + while ((i >= 0) && (maxcount-- > 0)) { + for (; i >= 0; i--) { + if (s[i] == ch) { + SPLIT_ADD(s, i + 1, j + 1); + j = i = i - 1; + break; + } + } + } + if (i < 0 && count == 0 && PyBytes_CheckExact(self)) { + /* ch not in self, so just use self as list[0] */ + Py_INCREF(self); + PyList_SET_ITEM(list, 0, (PyObject *)self); + count++; } - return bytes_repr((PyByteArrayObject*)op); + else if (j >= -1) { + SPLIT_ADD(s, 0, j + 1); + } + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + return list; + + onError: + Py_DECREF(list); + return NULL; } -static PyObject * -bytes_richcompare(PyObject *self, PyObject *other, int op) -{ - Py_ssize_t self_size, other_size; - Py_buffer self_bytes, other_bytes; - PyObject *res; - Py_ssize_t minsize; - int cmp; - - /* Bytes can be compared to anything that supports the (binary) - buffer API. Except that a comparison with Unicode is always an - error, even if the comparison is for equality. */ - if (PyObject_IsInstance(self, (PyObject*)&PyUnicode_Type) || - PyObject_IsInstance(other, (PyObject*)&PyUnicode_Type)) { - if (Py_BytesWarningFlag && op == Py_EQ) { - if (PyErr_WarnEx(PyExc_BytesWarning, - "Comparsion between bytearray and string", 1)) - return NULL; - } +PyDoc_STRVAR(rsplit__doc__, +"B.rsplit([sep[, maxsplit]]) -> list of strings\n\ +\n\ +Return a list of the sections in B, using sep as the delimiter,\n\ +starting at the end of B and working to the front.\n\ +If sep is not given, B is split on ASCII whitespace characters\n\ +(space, tab, return, newline, formfeed, vertical tab).\n\ +If maxsplit is given, at most maxsplit splits are done."); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } - self_size = _getbuffer(self, &self_bytes); - if (self_size < 0) { - PyErr_Clear(); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } +static PyObject * +string_rsplit(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; + Py_ssize_t maxsplit = -1, count=0; + const char *s, *sub; + Py_buffer vsub; + PyObject *list, *str, *subobj = Py_None; + + if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) + return NULL; + if (maxsplit < 0) + maxsplit = PY_SSIZE_T_MAX; + if (subobj == Py_None) + return rsplit_whitespace(self, len, maxsplit); + if (_getbuffer(subobj, &vsub) < 0) + return NULL; + sub = vsub.buf; + n = vsub.len; + + if (n == 0) { + PyErr_SetString(PyExc_ValueError, "empty separator"); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } + else if (n == 1) + return rsplit_char(self, len, sub[0], maxsplit); - other_size = _getbuffer(other, &other_bytes); - if (other_size < 0) { - PyErr_Clear(); - PyObject_ReleaseBuffer(self, &self_bytes); - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + list = PyList_New(PREALLOC_SIZE(maxsplit)); + if (list == NULL) { + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; + } - if (self_size != other_size && (op == Py_EQ || op == Py_NE)) { - /* Shortcut: if the lengths differ, the objects differ */ - cmp = (op == Py_NE); - } - else { - minsize = self_size; - if (other_size < minsize) - minsize = other_size; - - cmp = memcmp(self_bytes.buf, other_bytes.buf, minsize); - /* In ISO C, memcmp() guarantees to use unsigned bytes! */ - - if (cmp == 0) { - if (self_size < other_size) - cmp = -1; - else if (self_size > other_size) - cmp = 1; - } - - switch (op) { - case Py_LT: cmp = cmp < 0; break; - case Py_LE: cmp = cmp <= 0; break; - case Py_EQ: cmp = cmp == 0; break; - case Py_NE: cmp = cmp != 0; break; - case Py_GT: cmp = cmp > 0; break; - case Py_GE: cmp = cmp >= 0; break; - } - } + j = len; + i = j - n; + + s = PyBytes_AS_STRING(self); + while ( (i >= 0) && (maxsplit-- > 0) ) { + for (; i>=0; i--) { + if (Py_STRING_MATCH(s, i, sub, n)) { + SPLIT_ADD(s, i + n, j); + j = i; + i -= n; + break; + } + } + } + SPLIT_ADD(s, 0, j); + FIX_PREALLOC_SIZE(list); + if (PyList_Reverse(list) < 0) + goto onError; + PyObject_ReleaseBuffer(subobj, &vsub); + return list; - res = cmp ? Py_True : Py_False; - PyObject_ReleaseBuffer(self, &self_bytes); - PyObject_ReleaseBuffer(other, &other_bytes); - Py_INCREF(res); - return res; +onError: + Py_DECREF(list); + PyObject_ReleaseBuffer(subobj, &vsub); + return NULL; } -static void -bytes_dealloc(PyByteArrayObject *self) -{ - if (self->ob_bytes != 0) { - PyMem_Free(self->ob_bytes); - } - Py_TYPE(self)->tp_free((PyObject *)self); -} +#undef SPLIT_ADD +#undef MAX_PREALLOC +#undef PREALLOC_SIZE -/* -------------------------------------------------------------------- */ -/* Methods */ +PyDoc_STRVAR(join__doc__, +"B.join(iterable_of_bytes) -> bytes\n\ +\n\ +Concatenates any number of bytes objects, with B in between each pair.\n\ +Example: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'."); -#define STRINGLIB_CHAR char -#define STRINGLIB_CMP memcmp -#define STRINGLIB_LEN PyByteArray_GET_SIZE -#define STRINGLIB_STR PyByteArray_AS_STRING -#define STRINGLIB_NEW PyByteArray_FromStringAndSize -#define STRINGLIB_EMPTY nullbytes -#define STRINGLIB_CHECK_EXACT PyByteArray_CheckExact -#define STRINGLIB_MUTABLE 1 +static PyObject * +string_join(PyObject *self, PyObject *orig) +{ + char *sep = PyBytes_AS_STRING(self); + const Py_ssize_t seplen = PyBytes_GET_SIZE(self); + PyObject *res = NULL; + char *p; + Py_ssize_t seqlen = 0; + size_t sz = 0; + Py_ssize_t i; + PyObject *seq, *item; + + seq = PySequence_Fast(orig, ""); + if (seq == NULL) { + return NULL; + } -#include "stringlib/fastsearch.h" -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/partition.h" -#include "stringlib/ctype.h" -#include "stringlib/transmogrify.h" + seqlen = PySequence_Size(seq); + if (seqlen == 0) { + Py_DECREF(seq); + return PyBytes_FromString(""); + } + if (seqlen == 1) { + item = PySequence_Fast_GET_ITEM(seq, 0); + if (PyBytes_CheckExact(item)) { + Py_INCREF(item); + Py_DECREF(seq); + return item; + } + } + /* There are at least two things to join, or else we have a subclass + * of the builtin types in the sequence. + * Do a pre-pass to figure out the total amount of space we'll + * need (sz), and see whether all argument are bytes. + */ + /* XXX Shouldn't we use _getbuffer() on these items instead? */ + for (i = 0; i < seqlen; i++) { + const size_t old_sz = sz; + item = PySequence_Fast_GET_ITEM(seq, i); + if (!PyBytes_Check(item) && !PyByteArray_Check(item)) { + PyErr_Format(PyExc_TypeError, + "sequence item %zd: expected bytes," + " %.80s found", + i, Py_TYPE(item)->tp_name); + Py_DECREF(seq); + return NULL; + } + sz += Py_SIZE(item); + if (i != 0) + sz += seplen; + if (sz < old_sz || sz > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OverflowError, + "join() result is too long for a Python string"); + Py_DECREF(seq); + return NULL; + } + } -/* The following Py_LOCAL_INLINE and Py_LOCAL functions -were copied from the old char* style string object. */ + /* Allocate result space. */ + res = PyBytes_FromStringAndSize((char*)NULL, sz); + if (res == NULL) { + Py_DECREF(seq); + return NULL; + } -Py_LOCAL_INLINE(void) -_adjust_indices(Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t len) + /* Catenate everything. */ + /* I'm not worried about a PyByteArray item growing because there's + nowhere in this function where we release the GIL. */ + p = PyBytes_AS_STRING(res); + for (i = 0; i < seqlen; ++i) { + size_t n; + char *q; + if (i) { + Py_MEMCPY(p, sep, seplen); + p += seplen; + } + item = PySequence_Fast_GET_ITEM(seq, i); + n = Py_SIZE(item); + if (PyBytes_Check(item)) + q = PyBytes_AS_STRING(item); + else + q = PyByteArray_AS_STRING(item); + Py_MEMCPY(p, q, n); + p += n; + } + + Py_DECREF(seq); + return res; +} + +PyObject * +_PyBytes_Join(PyObject *sep, PyObject *x) { - if (*end > len) - *end = len; - else if (*end < 0) - *end += len; - if (*end < 0) - *end = 0; - if (*start < 0) - *start += len; - if (*start < 0) - *start = 0; + assert(sep != NULL && PyBytes_Check(sep)); + assert(x != NULL); + return string_join(sep, x); } +Py_LOCAL_INLINE(void) +string_adjust_indices(Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t len) +{ + if (*end > len) + *end = len; + else if (*end < 0) + *end += len; + if (*end < 0) + *end = 0; + if (*start < 0) + *start += len; + if (*start < 0) + *start = 0; +} Py_LOCAL_INLINE(Py_ssize_t) -bytes_find_internal(PyByteArrayObject *self, PyObject *args, int dir) -{ - PyObject *subobj; - Py_buffer subbuf; - Py_ssize_t start=0, end=PY_SSIZE_T_MAX; - Py_ssize_t res; - - if (!PyArg_ParseTuple(args, "O|O&O&:find/rfind/index/rindex", &subobj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return -2; - if (_getbuffer(subobj, &subbuf) < 0) - return -2; - if (dir > 0) - res = stringlib_find_slice( - PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subbuf.buf, subbuf.len, start, end); - else - res = stringlib_rfind_slice( - PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subbuf.buf, subbuf.len, start, end); - PyObject_ReleaseBuffer(subobj, &subbuf); - return res; +string_find_internal(PyBytesObject *self, PyObject *args, int dir) +{ + PyObject *subobj; + const char *sub; + Py_ssize_t sub_len; + Py_ssize_t start=0, end=PY_SSIZE_T_MAX; + PyObject *obj_start=Py_None, *obj_end=Py_None; + + if (!PyArg_ParseTuple(args, "O|OO:find/rfind/index/rindex", &subobj, + &obj_start, &obj_end)) + return -2; + /* To support None in "start" and "end" arguments, meaning + the same as if they were not passed. + */ + if (obj_start != Py_None) + if (!_PyEval_SliceIndex(obj_start, &start)) + return -2; + if (obj_end != Py_None) + if (!_PyEval_SliceIndex(obj_end, &end)) + return -2; + + if (PyBytes_Check(subobj)) { + sub = PyBytes_AS_STRING(subobj); + sub_len = PyBytes_GET_SIZE(subobj); + } + else if (PyObject_AsCharBuffer(subobj, &sub, &sub_len)) + /* XXX - the "expected a character buffer object" is pretty + confusing for a non-expert. remap to something else ? */ + return -2; + + if (dir > 0) + return stringlib_find_slice( + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, sub_len, start, end); + else + return stringlib_rfind_slice( + PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), + sub, sub_len, start, end); } + PyDoc_STRVAR(find__doc__, "B.find(sub [,start [,end]]) -> int\n\ \n\ -Return the lowest index in B where subsection sub is found,\n\ -such that sub is contained within s[start,end]. Optional\n\ +Return the lowest index in S where substring sub is found,\n\ +such that sub is contained within s[start:end]. Optional\n\ arguments start and end are interpreted as in slice notation.\n\ \n\ Return -1 on failure."); static PyObject * -bytes_find(PyByteArrayObject *self, PyObject *args) +string_find(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, +1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); -} - -PyDoc_STRVAR(count__doc__, -"B.count(sub [,start [,end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of subsection sub in\n\ -bytes B[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - -static PyObject * -bytes_count(PyByteArrayObject *self, PyObject *args) -{ - PyObject *sub_obj; - const char *str = PyByteArray_AS_STRING(self); - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; - Py_buffer vsub; - PyObject *count_obj; - - if (!PyArg_ParseTuple(args, "O|O&O&:count", &sub_obj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - - if (_getbuffer(sub_obj, &vsub) < 0) - return NULL; - - _adjust_indices(&start, &end, PyByteArray_GET_SIZE(self)); - - count_obj = PyLong_FromSsize_t( - stringlib_count(str + start, end - start, vsub.buf, vsub.len) - ); - PyObject_ReleaseBuffer(sub_obj, &vsub); - return count_obj; + Py_ssize_t result = string_find_internal(self, args, +1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } PyDoc_STRVAR(index__doc__, "B.index(sub [,start [,end]]) -> int\n\ \n\ -Like B.find() but raise ValueError when the subsection is not found."); +Like B.find() but raise ValueError when the substring is not found."); static PyObject * -bytes_index(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t result = bytes_find_internal(self, args, +1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "subsection not found"); - return NULL; - } - return PyLong_FromSsize_t(result); +string_index(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = string_find_internal(self, args, +1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "substring not found"); + return NULL; + } + return PyLong_FromSsize_t(result); } PyDoc_STRVAR(rfind__doc__, "B.rfind(sub [,start [,end]]) -> int\n\ \n\ -Return the highest index in B where subsection sub is found,\n\ -such that sub is contained within s[start,end]. Optional\n\ +Return the highest index in B where substring sub is found,\n\ +such that sub is contained within s[start:end]. Optional\n\ arguments start and end are interpreted as in slice notation.\n\ \n\ Return -1 on failure."); static PyObject * -bytes_rfind(PyByteArrayObject *self, PyObject *args) +string_rfind(PyBytesObject *self, PyObject *args) { - Py_ssize_t result = bytes_find_internal(self, args, -1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); + Py_ssize_t result = string_find_internal(self, args, -1); + if (result == -2) + return NULL; + return PyLong_FromSsize_t(result); } PyDoc_STRVAR(rindex__doc__, "B.rindex(sub [,start [,end]]) -> int\n\ \n\ -Like B.rfind() but raise ValueError when the subsection is not found."); +Like B.rfind() but raise ValueError when the substring is not found."); static PyObject * -bytes_rindex(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t result = bytes_find_internal(self, args, -1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "subsection not found"); - return NULL; - } - return PyLong_FromSsize_t(result); +string_rindex(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t result = string_find_internal(self, args, -1); + if (result == -2) + return NULL; + if (result == -1) { + PyErr_SetString(PyExc_ValueError, + "substring not found"); + return NULL; + } + return PyLong_FromSsize_t(result); } -static int -bytes_contains(PyObject *self, PyObject *arg) +Py_LOCAL_INLINE(PyObject *) +do_xstrip(PyBytesObject *self, int striptype, PyObject *sepobj) +{ + Py_buffer vsep; + char *s = PyBytes_AS_STRING(self); + Py_ssize_t len = PyBytes_GET_SIZE(self); + char *sep; + Py_ssize_t seplen; + Py_ssize_t i, j; + + if (_getbuffer(sepobj, &vsep) < 0) + return NULL; + sep = vsep.buf; + seplen = vsep.len; + + i = 0; + if (striptype != RIGHTSTRIP) { + while (i < len && memchr(sep, Py_CHARMASK(s[i]), seplen)) { + i++; + } + } + + j = len; + if (striptype != LEFTSTRIP) { + do { + j--; + } while (j >= i && memchr(sep, Py_CHARMASK(s[j]), seplen)); + j++; + } + + PyObject_ReleaseBuffer(sepobj, &vsep); + + if (i == 0 && j == len && PyBytes_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*)self; + } + else + return PyBytes_FromStringAndSize(s+i, j-i); +} + + +Py_LOCAL_INLINE(PyObject *) +do_strip(PyBytesObject *self, int striptype) { - Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) { - Py_buffer varg; - int pos; - PyErr_Clear(); - if (_getbuffer(arg, &varg) < 0) - return -1; - pos = stringlib_find(PyByteArray_AS_STRING(self), Py_SIZE(self), - varg.buf, varg.len, 0); - PyObject_ReleaseBuffer(arg, &varg); - return pos >= 0; - } - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } + char *s = PyBytes_AS_STRING(self); + Py_ssize_t len = PyBytes_GET_SIZE(self), i, j; + + i = 0; + if (striptype != RIGHTSTRIP) { + while (i < len && ISSPACE(s[i])) { + i++; + } + } + + j = len; + if (striptype != LEFTSTRIP) { + do { + j--; + } while (j >= i && ISSPACE(s[j])); + j++; + } - return memchr(PyByteArray_AS_STRING(self), ival, Py_SIZE(self)) != NULL; + if (i == 0 && j == len && PyBytes_CheckExact(self)) { + Py_INCREF(self); + return (PyObject*)self; + } + else + return PyBytes_FromStringAndSize(s+i, j-i); } -/* Matches the end (direction >= 0) or start (direction < 0) of self - * against substr, using the start and end arguments. Returns - * -1 on error, 0 if not found and 1 if found. - */ -Py_LOCAL(int) -_bytes_tailmatch(PyByteArrayObject *self, PyObject *substr, Py_ssize_t start, - Py_ssize_t end, int direction) +Py_LOCAL_INLINE(PyObject *) +do_argstrip(PyBytesObject *self, int striptype, PyObject *args) { - Py_ssize_t len = PyByteArray_GET_SIZE(self); - const char* str; - Py_buffer vsubstr; - int rv = 0; + PyObject *sep = NULL; - str = PyByteArray_AS_STRING(self); + if (!PyArg_ParseTuple(args, (char *)stripformat[striptype], &sep)) + return NULL; - if (_getbuffer(substr, &vsubstr) < 0) - return -1; + if (sep != NULL && sep != Py_None) { + return do_xstrip(self, striptype, sep); + } + return do_strip(self, striptype); +} - _adjust_indices(&start, &end, len); - - if (direction < 0) { - /* startswith */ - if (start+vsubstr.len > len) { - goto done; - } - } else { - /* endswith */ - if (end-start < vsubstr.len || start > len) { - goto done; - } - - if (end-vsubstr.len > start) - start = end - vsubstr.len; - } - if (end-start >= vsubstr.len) - rv = ! memcmp(str+start, vsubstr.buf, vsubstr.len); -done: - PyObject_ReleaseBuffer(substr, &vsubstr); - return rv; +PyDoc_STRVAR(strip__doc__, +"B.strip([bytes]) -> bytes\n\ +\n\ +Strip leading and trailing bytes contained in the argument.\n\ +If the argument is omitted, strip trailing ASCII whitespace."); +static PyObject * +string_strip(PyBytesObject *self, PyObject *args) +{ + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, BOTHSTRIP); /* Common case */ + else + return do_argstrip(self, BOTHSTRIP, args); } -PyDoc_STRVAR(startswith__doc__, -"B.startswith(prefix [,start [,end]]) -> bool\n\ +PyDoc_STRVAR(lstrip__doc__, +"B.lstrip([bytes]) -> bytes\n\ \n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of strings to try."); +Strip leading bytes contained in the argument.\n\ +If the argument is omitted, strip leading ASCII whitespace."); +static PyObject * +string_lstrip(PyBytesObject *self, PyObject *args) +{ + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, LEFTSTRIP); /* Common case */ + else + return do_argstrip(self, LEFTSTRIP, args); +} + +PyDoc_STRVAR(rstrip__doc__, +"B.rstrip([bytes]) -> bytes\n\ +\n\ +Strip trailing bytes contained in the argument.\n\ +If the argument is omitted, strip trailing ASCII whitespace."); static PyObject * -bytes_startswith(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &subobj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytes_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, -1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytes_tailmatch(self, subobj, start, end, -1); - if (result == -1) - return NULL; - else - return PyBool_FromLong(result); +string_rstrip(PyBytesObject *self, PyObject *args) +{ + if (PyTuple_GET_SIZE(args) == 0) + return do_strip(self, RIGHTSTRIP); /* Common case */ + else + return do_argstrip(self, RIGHTSTRIP, args); } -PyDoc_STRVAR(endswith__doc__, -"B.endswith(suffix [,start [,end]]) -> bool\n\ + +PyDoc_STRVAR(count__doc__, +"B.count(sub [,start [,end]]) -> int\n\ \n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of strings to try."); +Return the number of non-overlapping occurrences of substring sub in\n\ +string S[start:end]. Optional arguments start and end are interpreted\n\ +as in slice notation."); static PyObject * -bytes_endswith(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &subobj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _bytes_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, +1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _bytes_tailmatch(self, subobj, start, end, +1); - if (result == -1) - return NULL; - else - return PyBool_FromLong(result); +string_count(PyBytesObject *self, PyObject *args) +{ + PyObject *sub_obj; + const char *str = PyBytes_AS_STRING(self), *sub; + Py_ssize_t sub_len; + Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; + + if (!PyArg_ParseTuple(args, "O|O&O&:count", &sub_obj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + + if (PyBytes_Check(sub_obj)) { + sub = PyBytes_AS_STRING(sub_obj); + sub_len = PyBytes_GET_SIZE(sub_obj); + } + else if (PyObject_AsCharBuffer(sub_obj, &sub, &sub_len)) + return NULL; + + string_adjust_indices(&start, &end, PyBytes_GET_SIZE(self)); + + return PyLong_FromSsize_t( + stringlib_count(str + start, end - start, sub, sub_len) + ); } PyDoc_STRVAR(translate__doc__, -"B.translate(table[, deletechars]) -> bytearray\n\ +"B.translate(table[, deletechars]) -> bytes\n\ \n\ Return a copy of B, where all characters occurring in the\n\ optional argument deletechars are removed, and the remaining\n\ @@ -1347,95 +1837,105 @@ characters have been mapped through the given translation\n\ table, which must be a bytes object of length 256."); static PyObject * -bytes_translate(PyByteArrayObject *self, PyObject *args) -{ - register char *input, *output; - register const char *table; - register Py_ssize_t i, c, changed = 0; - PyObject *input_obj = (PyObject*)self; - const char *output_start; - Py_ssize_t inlen; - PyObject *result; - int trans_table[256]; - PyObject *tableobj, *delobj = NULL; - Py_buffer vtable, vdel; - - if (!PyArg_UnpackTuple(args, "translate", 1, 2, - &tableobj, &delobj)) - return NULL; - - if (_getbuffer(tableobj, &vtable) < 0) - return NULL; - - if (vtable.len != 256) { - PyErr_SetString(PyExc_ValueError, - "translation table must be 256 characters long"); - result = NULL; - goto done; - } +string_translate(PyBytesObject *self, PyObject *args) +{ + register char *input, *output; + const char *table; + register Py_ssize_t i, c, changed = 0; + PyObject *input_obj = (PyObject*)self; + const char *output_start, *del_table=NULL; + Py_ssize_t inlen, tablen, dellen = 0; + PyObject *result; + int trans_table[256]; + PyObject *tableobj, *delobj = NULL; + + if (!PyArg_UnpackTuple(args, "translate", 1, 2, + &tableobj, &delobj)) + return NULL; + + if (PyBytes_Check(tableobj)) { + table = PyBytes_AS_STRING(tableobj); + tablen = PyBytes_GET_SIZE(tableobj); + } + else if (tableobj == Py_None) { + table = NULL; + tablen = 256; + } + else if (PyObject_AsCharBuffer(tableobj, &table, &tablen)) + return NULL; - if (delobj != NULL) { - if (_getbuffer(delobj, &vdel) < 0) { - result = NULL; - goto done; - } - } - else { - vdel.buf = NULL; - vdel.len = 0; - } + if (tablen != 256) { + PyErr_SetString(PyExc_ValueError, + "translation table must be 256 characters long"); + return NULL; + } - table = (const char *)vtable.buf; - inlen = PyByteArray_GET_SIZE(input_obj); - result = PyByteArray_FromStringAndSize((char *)NULL, inlen); - if (result == NULL) - goto done; - output_start = output = PyByteArray_AsString(result); - input = PyByteArray_AS_STRING(input_obj); - - if (vdel.len == 0) { - /* If no deletions are required, use faster code */ - for (i = inlen; --i >= 0; ) { - c = Py_CHARMASK(*input++); - if (Py_CHARMASK((*output++ = table[c])) != c) - changed = 1; - } - if (changed || !PyByteArray_CheckExact(input_obj)) - goto done; - Py_DECREF(result); - Py_INCREF(input_obj); - result = input_obj; - goto done; - } + if (delobj != NULL) { + if (PyBytes_Check(delobj)) { + del_table = PyBytes_AS_STRING(delobj); + dellen = PyBytes_GET_SIZE(delobj); + } + else if (PyUnicode_Check(delobj)) { + PyErr_SetString(PyExc_TypeError, + "deletions are implemented differently for unicode"); + return NULL; + } + else if (PyObject_AsCharBuffer(delobj, &del_table, &dellen)) + return NULL; + } + else { + del_table = NULL; + dellen = 0; + } - for (i = 0; i < 256; i++) - trans_table[i] = Py_CHARMASK(table[i]); + inlen = PyBytes_GET_SIZE(input_obj); + result = PyBytes_FromStringAndSize((char *)NULL, inlen); + if (result == NULL) + return NULL; + output_start = output = PyBytes_AsString(result); + input = PyBytes_AS_STRING(input_obj); + + if (dellen == 0 && table != NULL) { + /* If no deletions are required, use faster code */ + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (Py_CHARMASK((*output++ = table[c])) != c) + changed = 1; + } + if (changed || !PyBytes_CheckExact(input_obj)) + return result; + Py_DECREF(result); + Py_INCREF(input_obj); + return input_obj; + } - for (i = 0; i < vdel.len; i++) - trans_table[(int) Py_CHARMASK( ((unsigned char*)vdel.buf)[i] )] = -1; + if (table == NULL) { + for (i = 0; i < 256; i++) + trans_table[i] = Py_CHARMASK(i); + } else { + for (i = 0; i < 256; i++) + trans_table[i] = Py_CHARMASK(table[i]); + } - for (i = inlen; --i >= 0; ) { - c = Py_CHARMASK(*input++); - if (trans_table[c] != -1) - if (Py_CHARMASK(*output++ = (char)trans_table[c]) == c) - continue; - changed = 1; - } - if (!changed && PyByteArray_CheckExact(input_obj)) { - Py_DECREF(result); - Py_INCREF(input_obj); - result = input_obj; - goto done; - } - /* Fix the size of the resulting string */ - if (inlen > 0) - PyByteArray_Resize(result, output - output_start); + for (i = 0; i < dellen; i++) + trans_table[(int) Py_CHARMASK(del_table[i])] = -1; -done: - PyObject_ReleaseBuffer(tableobj, &vtable); - if (delobj != NULL) - PyObject_ReleaseBuffer(delobj, &vdel); - return result; + for (i = inlen; --i >= 0; ) { + c = Py_CHARMASK(*input++); + if (trans_table[c] != -1) + if (Py_CHARMASK(*output++ = (char)trans_table[c]) == c) + continue; + changed = 1; + } + if (!changed && PyBytes_CheckExact(input_obj)) { + Py_DECREF(result); + Py_INCREF(input_obj); + return input_obj; + } + /* Fix the size of the resulting string */ + if (inlen > 0) + _PyBytes_Resize(&result, output - output_start); + return result; } @@ -1444,598 +1944,596 @@ done: /* find and count characters and substrings */ -#define findchar(target, target_len, c) \ +#define findchar(target, target_len, c) \ ((char *)memchr((const void *)(target), c, target_len)) -/* Don't call if length < 2 */ -#define Py_STRING_MATCH(target, offset, pattern, length) \ - (target[offset] == pattern[0] && \ - target[offset+length-1] == pattern[length-1] && \ - !memcmp(target+offset+1, pattern+1, length-2) ) - - -/* Bytes ops must return a string. */ -/* If the object is subclass of bytes, create a copy */ -Py_LOCAL(PyByteArrayObject *) -return_self(PyByteArrayObject *self) +/* String ops must return a string. */ +/* If the object is subclass of string, create a copy */ +Py_LOCAL(PyBytesObject *) +return_self(PyBytesObject *self) { - if (PyByteArray_CheckExact(self)) { - Py_INCREF(self); - return (PyByteArrayObject *)self; - } - return (PyByteArrayObject *)PyByteArray_FromStringAndSize( - PyByteArray_AS_STRING(self), - PyByteArray_GET_SIZE(self)); + if (PyBytes_CheckExact(self)) { + Py_INCREF(self); + return self; + } + return (PyBytesObject *)PyBytes_FromStringAndSize( + PyBytes_AS_STRING(self), + PyBytes_GET_SIZE(self)); } Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, Py_ssize_t target_len, char c, Py_ssize_t maxcount) +countchar(const char *target, int target_len, char c, Py_ssize_t maxcount) { - Py_ssize_t count=0; - const char *start=target; - const char *end=target+target_len; + Py_ssize_t count=0; + const char *start=target; + const char *end=target+target_len; - while ( (start=findchar(start, end-start, c)) != NULL ) { - count++; - if (count >= maxcount) - break; - start += 1; - } - return count; + while ( (start=findchar(start, end-start, c)) != NULL ) { + count++; + if (count >= maxcount) + break; + start += 1; + } + return count; } Py_LOCAL(Py_ssize_t) findstring(const char *target, Py_ssize_t target_len, - const char *pattern, Py_ssize_t pattern_len, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - if (start < 0) { - start += target_len; - if (start < 0) - start = 0; - } - if (end > target_len) { - end = target_len; - } else if (end < 0) { - end += target_len; - if (end < 0) - end = 0; - } + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction) +{ + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } - /* zero-length substrings always match at the first attempt */ - if (pattern_len == 0) - return (direction > 0) ? start : end; + /* zero-length substrings always match at the first attempt */ + if (pattern_len == 0) + return (direction > 0) ? start : end; - end -= pattern_len; + end -= pattern_len; - if (direction < 0) { - for (; end >= start; end--) - if (Py_STRING_MATCH(target, end, pattern, pattern_len)) - return end; - } else { - for (; start <= end; start++) - if (Py_STRING_MATCH(target, start, pattern, pattern_len)) - return start; - } - return -1; + if (direction < 0) { + for (; end >= start; end--) + if (Py_STRING_MATCH(target, end, pattern, pattern_len)) + return end; + } else { + for (; start <= end; start++) + if (Py_STRING_MATCH(target, start,pattern,pattern_len)) + return start; + } + return -1; } Py_LOCAL_INLINE(Py_ssize_t) countstring(const char *target, Py_ssize_t target_len, - const char *pattern, Py_ssize_t pattern_len, - Py_ssize_t start, - Py_ssize_t end, - int direction, Py_ssize_t maxcount) + const char *pattern, Py_ssize_t pattern_len, + Py_ssize_t start, + Py_ssize_t end, + int direction, Py_ssize_t maxcount) { - Py_ssize_t count=0; + Py_ssize_t count=0; - if (start < 0) { - start += target_len; - if (start < 0) - start = 0; - } - if (end > target_len) { - end = target_len; - } else if (end < 0) { - end += target_len; - if (end < 0) - end = 0; - } + if (start < 0) { + start += target_len; + if (start < 0) + start = 0; + } + if (end > target_len) { + end = target_len; + } else if (end < 0) { + end += target_len; + if (end < 0) + end = 0; + } - /* zero-length substrings match everywhere */ - if (pattern_len == 0 || maxcount == 0) { - if (target_len+1 < maxcount) - return target_len+1; - return maxcount; - } + /* zero-length substrings match everywhere */ + if (pattern_len == 0 || maxcount == 0) { + if (target_len+1 < maxcount) + return target_len+1; + return maxcount; + } - end -= pattern_len; - if (direction < 0) { - for (; (end >= start); end--) - if (Py_STRING_MATCH(target, end, pattern, pattern_len)) { - count++; - if (--maxcount <= 0) break; - end -= pattern_len-1; - } - } else { - for (; (start <= end); start++) - if (Py_STRING_MATCH(target, start, pattern, pattern_len)) { - count++; - if (--maxcount <= 0) - break; - start += pattern_len-1; - } - } - return count; + end -= pattern_len; + if (direction < 0) { + for (; (end >= start); end--) + if (Py_STRING_MATCH(target, end,pattern,pattern_len)) { + count++; + if (--maxcount <= 0) break; + end -= pattern_len-1; + } + } else { + for (; (start <= end); start++) + if (Py_STRING_MATCH(target, start, + pattern, pattern_len)) { + count++; + if (--maxcount <= 0) + break; + start += pattern_len-1; + } + } + return count; } /* Algorithms for different cases of string replacement */ /* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_interleave(PyByteArrayObject *self, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - Py_ssize_t self_len, result_len; - Py_ssize_t count, i, product; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - - /* 1 at the end plus 1 after every character */ - count = self_len+1; - if (maxcount < count) - count = maxcount; - - /* Check for overflow */ - /* result_len = count * to_len + self_len; */ - product = count * to_len; - if (product / to_len != count) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - result_len = product + self_len; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } +Py_LOCAL(PyBytesObject *) +replace_interleave(PyBytesObject *self, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + Py_ssize_t self_len, result_len; + Py_ssize_t count, i, product; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + + /* 1 at the end plus 1 after every character */ + count = self_len+1; + if (maxcount < count) + count = maxcount; + + /* Check for overflow */ + /* result_len = count * to_len + self_len; */ + product = count * to_len; + if (product / to_len != count) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + result_len = product + self_len; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } - if (! (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) ) - return NULL; + if (! (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) ) + return NULL; - self_s = PyByteArray_AS_STRING(self); - result_s = PyByteArray_AS_STRING(result); + self_s = PyBytes_AS_STRING(self); + result_s = PyBytes_AS_STRING(result); - /* TODO: special case single character, which doesn't need memcpy */ + /* TODO: special case single character, which doesn't need memcpy */ - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; + /* Lay the first one down (guaranteed this will occur) */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + count -= 1; - for (i=0; i<count; i++) { - *result_s++ = *self_s++; - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - } + for (i=0; i<count; i++) { + *result_s++ = *self_s++; + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + } - /* Copy the rest of the original string */ - Py_MEMCPY(result_s, self_s, self_len-i); + /* Copy the rest of the original string */ + Py_MEMCPY(result_s, self_s, self_len-i); - return result; + return result; } /* Special case for deleting a single character */ /* len(self)>=1, len(from)==1, to="", maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_delete_single_character(PyByteArrayObject *self, - char from_c, Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - self_s = PyByteArray_AS_STRING(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - return return_self(self); - } +Py_LOCAL(PyBytesObject *) +replace_delete_single_character(PyBytesObject *self, + char from_c, Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + self_s = PyBytes_AS_STRING(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + return return_self(self); + } - result_len = self_len - count; /* from_len == 1 */ - assert(result_len>=0); - - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - start = next+1; - } - Py_MEMCPY(result_s, start, end-start); + result_len = self_len - count; /* from_len == 1 */ + assert(result_len>=0); + + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + start = next+1; + } + Py_MEMCPY(result_s, start, end-start); - return result; + return result; } /* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_delete_substring(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyByteArrayObject *result; - - self_len = PyByteArray_GET_SIZE(self); - self_s = PyByteArray_AS_STRING(self); - - count = countstring(self_s, self_len, - from_s, from_len, - 0, self_len, 1, - maxcount); - - if (count == 0) { - /* no matches */ - return return_self(self); - } +Py_LOCAL(PyBytesObject *) +replace_delete_substring(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + Py_ssize_t maxcount) { + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset; + PyBytesObject *result; + + self_len = PyBytes_GET_SIZE(self); + self_s = PyBytes_AS_STRING(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, 1, + maxcount); + + if (count == 0) { + /* no matches */ + return return_self(self); + } - result_len = self_len - (count * from_len); - assert (result_len>=0); + result_len = self_len - (count * from_len); + assert (result_len>=0); - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL ) - return NULL; + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL ) + return NULL; - result_s = PyByteArray_AS_STRING(result); + result_s = PyBytes_AS_STRING(result); - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset == -1) - break; - next = start + offset; + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start + offset; - Py_MEMCPY(result_s, start, next-start); + Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - start = next+from_len; - } - Py_MEMCPY(result_s, start, end-start); - return result; + result_s += (next-start); + start = next+from_len; + } + Py_MEMCPY(result_s, start, end-start); + return result; } /* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_single_character_in_place(PyByteArrayObject *self, - char from_c, char to_c, - Py_ssize_t maxcount) +Py_LOCAL(PyBytesObject *) +replace_single_character_in_place(PyBytesObject *self, + char from_c, char to_c, + Py_ssize_t maxcount) { - char *self_s, *result_s, *start, *end, *next; - Py_ssize_t self_len; - PyByteArrayObject *result; + char *self_s, *result_s, *start, *end, *next; + Py_ssize_t self_len; + PyBytesObject *result; - /* The result string will be the same size */ - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); + /* The result string will be the same size */ + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); - next = findchar(self_s, self_len, from_c); - - if (next == NULL) { - /* No matches; return the original bytes */ - return return_self(self); - } - - /* Need to make a new bytes */ - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); + next = findchar(self_s, self_len, from_c); - /* change everything in-place, starting with this one */ - start = result_s + (next-self_s); - *start = to_c; - start++; - end = result_s + self_len; + if (next == NULL) { + /* No matches; return the original string */ + return return_self(self); + } - while (--maxcount > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - *next = to_c; - start = next+1; - } + /* Need to make a new string */ + result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + (next-self_s); + *start = to_c; + start++; + end = result_s + self_len; + + while (--maxcount > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + *next = to_c; + start = next+1; + } - return result; + return result; } /* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_substring_in_place(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *result_s, *start, *end; - char *self_s; - Py_ssize_t self_len, offset; - PyByteArrayObject *result; - - /* The result bytes will be the same size */ - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - offset = findstring(self_s, self_len, - from_s, from_len, - 0, self_len, FORWARD); - if (offset == -1) { - /* No matches; return the original bytes */ - return return_self(self); - } +Py_LOCAL(PyBytesObject *) +replace_substring_in_place(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *result_s, *start, *end; + char *self_s; + Py_ssize_t self_len, offset; + PyBytesObject *result; + + /* The result string will be the same size */ + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + offset = findstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD); + if (offset == -1) { + /* No matches; return the original string */ + return return_self(self); + } - /* Need to make a new bytes */ - result = (PyByteArrayObject *) PyByteArray_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + offset; - Py_MEMCPY(start, to_s, from_len); - start += from_len; - end = result_s + self_len; - - while ( --maxcount > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset==-1) - break; - Py_MEMCPY(start+offset, to_s, from_len); - start += offset+from_len; - } + /* Need to make a new string */ + result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); + if (result == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + Py_MEMCPY(result_s, self_s, self_len); + + /* change everything in-place, starting with this one */ + start = result_s + offset; + Py_MEMCPY(start, to_s, from_len); + start += from_len; + end = result_s + self_len; + + while ( --maxcount > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset==-1) + break; + Py_MEMCPY(start+offset, to_s, from_len); + start += offset+from_len; + } - return result; + return result; } /* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_single_character(PyByteArrayObject *self, - char from_c, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, product; - PyByteArrayObject *result; - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } +Py_LOCAL(PyBytesObject *) +replace_single_character(PyBytesObject *self, + char from_c, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) +{ + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, product; + PyBytesObject *result; + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + count = countchar(self_s, self_len, from_c, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } - /* use the difference between current and new, hence the "-1" */ - /* result_len = self_len + count * (to_len-1) */ - product = count * (to_len-1); - if (product / (to_len-1) != count) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } - result_len = self_len + product; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } + /* use the difference between current and new, hence the "-1" */ + /* result_len = self_len + count * (to_len-1) */ + product = count * (to_len-1); + if (product / (to_len-1) != count) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += 1; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+1; - } - } - /* Copy the remainder of the remaining bytes */ - Py_MEMCPY(result_s, start, end-start); + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + next = findchar(start, end-start, from_c); + if (next == NULL) + break; + + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += 1; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+1; + } + } + /* Copy the remainder of the remaining string */ + Py_MEMCPY(result_s, start, end-start); - return result; + return result; } /* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyByteArrayObject *) -replace_substring(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset, product; - PyByteArrayObject *result; - - self_s = PyByteArray_AS_STRING(self); - self_len = PyByteArray_GET_SIZE(self); - - count = countstring(self_s, self_len, - from_s, from_len, - 0, self_len, FORWARD, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } +Py_LOCAL(PyBytesObject *) +replace_substring(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) { + char *self_s, *result_s; + char *start, *next, *end; + Py_ssize_t self_len, result_len; + Py_ssize_t count, offset, product; + PyBytesObject *result; + + self_s = PyBytes_AS_STRING(self); + self_len = PyBytes_GET_SIZE(self); + + count = countstring(self_s, self_len, + from_s, from_len, + 0, self_len, FORWARD, maxcount); + if (count == 0) { + /* no matches, return unchanged */ + return return_self(self); + } - /* Check for overflow */ - /* result_len = self_len + count * (to_len-from_len) */ - product = count * (to_len-from_len); - if (product / (to_len-from_len) != count) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } - result_len = self_len + product; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, "replace bytes is too long"); - return NULL; - } + /* Check for overflow */ + /* result_len = self_len + count * (to_len-from_len) */ + product = count * (to_len-from_len); + if (product / (to_len-from_len) != count) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } + result_len = self_len + product; + if (result_len < 0) { + PyErr_SetString(PyExc_OverflowError, + "replace string is too long"); + return NULL; + } - if ( (result = (PyByteArrayObject *) - PyByteArray_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyByteArray_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset == -1) - break; - next = start+offset; - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += from_len; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+from_len; - } - } - /* Copy the remainder of the remaining bytes */ - Py_MEMCPY(result_s, start, end-start); + if ( (result = (PyBytesObject *) + PyBytes_FromStringAndSize(NULL, result_len)) == NULL) + return NULL; + result_s = PyBytes_AS_STRING(result); + + start = self_s; + end = self_s + self_len; + while (count-- > 0) { + offset = findstring(start, end-start, + from_s, from_len, + 0, end-start, FORWARD); + if (offset == -1) + break; + next = start+offset; + if (next == start) { + /* replace with the 'to' */ + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start += from_len; + } else { + /* copy the unchanged old then the 'to' */ + Py_MEMCPY(result_s, start, next-start); + result_s += (next-start); + Py_MEMCPY(result_s, to_s, to_len); + result_s += to_len; + start = next+from_len; + } + } + /* Copy the remainder of the remaining string */ + Py_MEMCPY(result_s, start, end-start); - return result; + return result; } -Py_LOCAL(PyByteArrayObject *) -replace(PyByteArrayObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) +Py_LOCAL(PyBytesObject *) +replace(PyBytesObject *self, + const char *from_s, Py_ssize_t from_len, + const char *to_s, Py_ssize_t to_len, + Py_ssize_t maxcount) { - if (maxcount < 0) { - maxcount = PY_SSIZE_T_MAX; - } else if (maxcount == 0 || PyByteArray_GET_SIZE(self) == 0) { - /* nothing to do; return the original bytes */ - return return_self(self); - } + if (maxcount < 0) { + maxcount = PY_SSIZE_T_MAX; + } else if (maxcount == 0 || PyBytes_GET_SIZE(self) == 0) { + /* nothing to do; return the original string */ + return return_self(self); + } - if (maxcount == 0 || - (from_len == 0 && to_len == 0)) { - /* nothing to do; return the original bytes */ - return return_self(self); - } + if (maxcount == 0 || + (from_len == 0 && to_len == 0)) { + /* nothing to do; return the original string */ + return return_self(self); + } - /* Handle zero-length special cases */ + /* Handle zero-length special cases */ - if (from_len == 0) { - /* insert the 'to' bytes everywhere. */ - /* >>> "Python".replace("", ".") */ - /* '.P.y.t.h.o.n.' */ - return replace_interleave(self, to_s, to_len, maxcount); - } + if (from_len == 0) { + /* insert the 'to' string everywhere. */ + /* >>> "Python".replace("", ".") */ + /* '.P.y.t.h.o.n.' */ + return replace_interleave(self, to_s, to_len, maxcount); + } - /* Except for "".replace("", "A") == "A" there is no way beyond this */ - /* point for an empty self bytes to generate a non-empty bytes */ - /* Special case so the remaining code always gets a non-empty bytes */ - if (PyByteArray_GET_SIZE(self) == 0) { - return return_self(self); - } + /* Except for "".replace("", "A") == "A" there is no way beyond this */ + /* point for an empty self string to generate a non-empty string */ + /* Special case so the remaining code always gets a non-empty string */ + if (PyBytes_GET_SIZE(self) == 0) { + return return_self(self); + } - if (to_len == 0) { - /* delete all occurances of 'from' bytes */ - if (from_len == 1) { - return replace_delete_single_character( - self, from_s[0], maxcount); - } else { - return replace_delete_substring(self, from_s, from_len, maxcount); - } - } + if (to_len == 0) { + /* delete all occurances of 'from' string */ + if (from_len == 1) { + return replace_delete_single_character( + self, from_s[0], maxcount); + } else { + return replace_delete_substring(self, from_s, + from_len, maxcount); + } + } - /* Handle special case where both bytes have the same length */ - - if (from_len == to_len) { - if (from_len == 1) { - return replace_single_character_in_place( - self, - from_s[0], - to_s[0], - maxcount); - } else { - return replace_substring_in_place( - self, from_s, from_len, to_s, to_len, maxcount); - } - } + /* Handle special case where both strings have the same length */ + + if (from_len == to_len) { + if (from_len == 1) { + return replace_single_character_in_place( + self, + from_s[0], + to_s[0], + maxcount); + } else { + return replace_substring_in_place( + self, from_s, from_len, to_s, to_len, + maxcount); + } + } - /* Otherwise use the more generic algorithms */ - if (from_len == 1) { - return replace_single_character(self, from_s[0], - to_s, to_len, maxcount); - } else { - /* len('from')>=2, len('to')>=1 */ - return replace_substring(self, from_s, from_len, to_s, to_len, maxcount); - } + /* Otherwise use the more generic algorithms */ + if (from_len == 1) { + return replace_single_character(self, from_s[0], + to_s, to_len, maxcount); + } else { + /* len('from')>=2, len('to')>=1 */ + return replace_substring(self, from_s, from_len, to_s, to_len, + maxcount); + } } - PyDoc_STRVAR(replace__doc__, "B.replace(old, new[, count]) -> bytes\n\ \n\ @@ -2044,1257 +2542,888 @@ old replaced by new. If the optional argument count is\n\ given, only the first count occurrences are replaced."); static PyObject * -bytes_replace(PyByteArrayObject *self, PyObject *args) +string_replace(PyBytesObject *self, PyObject *args) { - Py_ssize_t count = -1; - PyObject *from, *to, *res; - Py_buffer vfrom, vto; + Py_ssize_t count = -1; + PyObject *from, *to; + const char *from_s, *to_s; + Py_ssize_t from_len, to_len; - if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) - return NULL; + if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) + return NULL; - if (_getbuffer(from, &vfrom) < 0) - return NULL; - if (_getbuffer(to, &vto) < 0) { - PyObject_ReleaseBuffer(from, &vfrom); - return NULL; - } + if (PyBytes_Check(from)) { + from_s = PyBytes_AS_STRING(from); + from_len = PyBytes_GET_SIZE(from); + } + else if (PyObject_AsCharBuffer(from, &from_s, &from_len)) + return NULL; - res = (PyObject *)replace((PyByteArrayObject *) self, - vfrom.buf, vfrom.len, - vto.buf, vto.len, count); + if (PyBytes_Check(to)) { + to_s = PyBytes_AS_STRING(to); + to_len = PyBytes_GET_SIZE(to); + } + else if (PyObject_AsCharBuffer(to, &to_s, &to_len)) + return NULL; - PyObject_ReleaseBuffer(from, &vfrom); - PyObject_ReleaseBuffer(to, &vto); - return res; + return (PyObject *)replace((PyBytesObject *) self, + from_s, from_len, + to_s, to_len, count); } +/** End DALKE **/ -/* Overallocate the initial list to reduce the number of reallocs for small - split sizes. Eg, "A A A A A A A A A A".split() (10 elements) has three - resizes, to sizes 4, 8, then 16. Most observed string splits are for human - text (roughly 11 words per line) and field delimited data (usually 1-10 - fields). For large strings the split algorithms are bandwidth limited - so increasing the preallocation likely will not improve things.*/ - -#define MAX_PREALLOC 12 - -/* 5 splits gives 6 elements */ -#define PREALLOC_SIZE(maxsplit) \ - (maxsplit >= MAX_PREALLOC ? MAX_PREALLOC : maxsplit+1) - -#define SPLIT_APPEND(data, left, right) \ - str = PyByteArray_FromStringAndSize((data) + (left), \ - (right) - (left)); \ - if (str == NULL) \ - goto onError; \ - if (PyList_Append(list, str)) { \ - Py_DECREF(str); \ - goto onError; \ - } \ - else \ - Py_DECREF(str); - -#define SPLIT_ADD(data, left, right) { \ - str = PyByteArray_FromStringAndSize((data) + (left), \ - (right) - (left)); \ - if (str == NULL) \ - goto onError; \ - if (count < MAX_PREALLOC) { \ - PyList_SET_ITEM(list, count, str); \ - } else { \ - if (PyList_Append(list, str)) { \ - Py_DECREF(str); \ - goto onError; \ - } \ - else \ - Py_DECREF(str); \ - } \ - count++; } - -/* Always force the list to the expected size. */ -#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count - - -Py_LOCAL_INLINE(PyObject *) -split_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) -{ - register Py_ssize_t i, j, count = 0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - i = j = 0; - while ((j < len) && (maxcount-- > 0)) { - for(; j < len; j++) { - /* I found that using memchr makes no difference */ - if (s[j] == ch) { - SPLIT_ADD(s, i, j); - i = j = j + 1; - break; - } - } - } - if (i <= len) { - SPLIT_ADD(s, i, len); - } - FIX_PREALLOC_SIZE(list); - return list; +/* Matches the end (direction >= 0) or start (direction < 0) of self + * against substr, using the start and end arguments. Returns + * -1 on error, 0 if not found and 1 if found. + */ +Py_LOCAL(int) +_string_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start, + Py_ssize_t end, int direction) +{ + Py_ssize_t len = PyBytes_GET_SIZE(self); + Py_ssize_t slen; + const char* sub; + const char* str; - onError: - Py_DECREF(list); - return NULL; + if (PyBytes_Check(substr)) { + sub = PyBytes_AS_STRING(substr); + slen = PyBytes_GET_SIZE(substr); + } + else if (PyObject_AsCharBuffer(substr, &sub, &slen)) + return -1; + str = PyBytes_AS_STRING(self); + + string_adjust_indices(&start, &end, len); + + if (direction < 0) { + /* startswith */ + if (start+slen > len) + return 0; + } else { + /* endswith */ + if (end-start < slen || start > len) + return 0; + + if (end-slen > start) + start = end - slen; + } + if (end-start >= slen) + return ! memcmp(str+start, sub, slen); + return 0; } -Py_LOCAL_INLINE(PyObject *) -split_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) -{ - register Py_ssize_t i, j, count = 0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - for (i = j = 0; i < len; ) { - /* find a token */ - while (i < len && ISSPACE(s[i])) - i++; - j = i; - while (i < len && !ISSPACE(s[i])) - i++; - if (j < i) { - if (maxcount-- <= 0) - break; - SPLIT_ADD(s, j, i); - while (i < len && ISSPACE(s[i])) - i++; - j = i; - } - } - if (j < len) { - SPLIT_ADD(s, j, len); - } - FIX_PREALLOC_SIZE(list); - return list; - - onError: - Py_DECREF(list); - return NULL; -} - -PyDoc_STRVAR(split__doc__, -"B.split([sep[, maxsplit]]) -> list of bytearray\n\ +PyDoc_STRVAR(startswith__doc__, +"B.startswith(prefix [,start [,end]]) -> bool\n\ \n\ -Return a list of the sections in B, using sep as the delimiter.\n\ -If sep is not given, B is split on ASCII whitespace characters\n\ -(space, tab, return, newline, formfeed, vertical tab).\n\ -If maxsplit is given, at most maxsplit splits are done."); +Return True if B starts with the specified prefix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +prefix can also be a tuple of strings to try."); static PyObject * -bytes_split(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j; - Py_ssize_t maxsplit = -1, count = 0; - const char *s = PyByteArray_AS_STRING(self), *sub; - PyObject *list, *str, *subobj = Py_None; - Py_buffer vsub; -#ifdef USE_FAST - Py_ssize_t pos; -#endif - - if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) - return NULL; - if (maxsplit < 0) - maxsplit = PY_SSIZE_T_MAX; - - if (subobj == Py_None) - return split_whitespace(s, len, maxsplit); - - if (_getbuffer(subobj, &vsub) < 0) - return NULL; - sub = vsub.buf; - n = vsub.len; - - if (n == 0) { - PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - if (n == 1) - return split_char(s, len, sub[0], maxsplit); - - list = PyList_New(PREALLOC_SIZE(maxsplit)); - if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - -#ifdef USE_FAST - i = j = 0; - while (maxsplit-- > 0) { - pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH); - if (pos < 0) - break; - j = i+pos; - SPLIT_ADD(s, i, j); - i = j + n; - } -#else - i = j = 0; - while ((j+n <= len) && (maxsplit-- > 0)) { - for (; j+n <= len; j++) { - if (Py_STRING_MATCH(s, j, sub, n)) { - SPLIT_ADD(s, i, j); - i = j = j + n; - break; - } - } - } -#endif - SPLIT_ADD(s, i, len); - FIX_PREALLOC_SIZE(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return list; - - onError: - Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; +string_startswith(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _string_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, -1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _string_tailmatch(self, subobj, start, end, -1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); } -/* stringlib's partition shares nullbytes in some cases. - undo this, we don't want the nullbytes to be shared. */ -static PyObject * -make_nullbytes_unique(PyObject *result) -{ - if (result != NULL) { - int i; - assert(PyTuple_Check(result)); - assert(PyTuple_GET_SIZE(result) == 3); - for (i = 0; i < 3; i++) { - if (PyTuple_GET_ITEM(result, i) == (PyObject *)nullbytes) { - PyObject *new = PyByteArray_FromStringAndSize(NULL, 0); - if (new == NULL) { - Py_DECREF(result); - result = NULL; - break; - } - Py_DECREF(nullbytes); - PyTuple_SET_ITEM(result, i, new); - } - } - } - return result; -} -PyDoc_STRVAR(partition__doc__, -"B.partition(sep) -> (head, sep, tail)\n\ +PyDoc_STRVAR(endswith__doc__, +"B.endswith(suffix [,start [,end]]) -> bool\n\ \n\ -Searches for the separator sep in B, and returns the part before it,\n\ -the separator itself, and the part after it. If the separator is not\n\ -found, returns B and two empty bytearray objects."); +Return True if B ends with the specified suffix, False otherwise.\n\ +With optional start, test B beginning at that position.\n\ +With optional end, stop comparing B at that position.\n\ +suffix can also be a tuple of strings to try."); static PyObject * -bytes_partition(PyByteArrayObject *self, PyObject *sep_obj) -{ - PyObject *bytesep, *result; - - bytesep = PyByteArray_FromObject(sep_obj); - if (! bytesep) - return NULL; - - result = stringlib_partition( - (PyObject*) self, - PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - bytesep, - PyByteArray_AS_STRING(bytesep), PyByteArray_GET_SIZE(bytesep) - ); - - Py_DECREF(bytesep); - return make_nullbytes_unique(result); +string_endswith(PyBytesObject *self, PyObject *args) +{ + Py_ssize_t start = 0; + Py_ssize_t end = PY_SSIZE_T_MAX; + PyObject *subobj; + int result; + + if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &subobj, + _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) + return NULL; + if (PyTuple_Check(subobj)) { + Py_ssize_t i; + for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { + result = _string_tailmatch(self, + PyTuple_GET_ITEM(subobj, i), + start, end, +1); + if (result == -1) + return NULL; + else if (result) { + Py_RETURN_TRUE; + } + } + Py_RETURN_FALSE; + } + result = _string_tailmatch(self, subobj, start, end, +1); + if (result == -1) + return NULL; + else + return PyBool_FromLong(result); } -PyDoc_STRVAR(rpartition__doc__, -"B.rpartition(sep) -> (tail, sep, head)\n\ + +PyDoc_STRVAR(decode__doc__, +"B.decode([encoding[, errors]]) -> object\n\ \n\ -Searches for the separator sep in B, starting at the end of B,\n\ -and returns the part before it, the separator itself, and the\n\ -part after it. If the separator is not found, returns two empty\n\ -bytearray objects and B."); +Decodes S using the codec registered for encoding. encoding defaults\n\ +to the default encoding. errors may be given to set a different error\n\ +handling scheme. Default is 'strict' meaning that encoding errors raise\n\ +a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ +as well as any other name registerd with codecs.register_error that is\n\ +able to handle UnicodeDecodeErrors."); static PyObject * -bytes_rpartition(PyByteArrayObject *self, PyObject *sep_obj) +string_decode(PyObject *self, PyObject *args) { - PyObject *bytesep, *result; - - bytesep = PyByteArray_FromObject(sep_obj); - if (! bytesep) - return NULL; - - result = stringlib_rpartition( - (PyObject*) self, - PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - bytesep, - PyByteArray_AS_STRING(bytesep), PyByteArray_GET_SIZE(bytesep) - ); - - Py_DECREF(bytesep); - return make_nullbytes_unique(result); -} + const char *encoding = NULL; + const char *errors = NULL; -Py_LOCAL_INLINE(PyObject *) -rsplit_char(const char *s, Py_ssize_t len, char ch, Py_ssize_t maxcount) -{ - register Py_ssize_t i, j, count=0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - i = j = len - 1; - while ((i >= 0) && (maxcount-- > 0)) { - for (; i >= 0; i--) { - if (s[i] == ch) { - SPLIT_ADD(s, i + 1, j + 1); - j = i = i - 1; - break; - } - } - } - if (j >= -1) { - SPLIT_ADD(s, 0, j + 1); - } - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - - return list; - - onError: - Py_DECREF(list); - return NULL; + if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors)) + return NULL; + if (encoding == NULL) + encoding = PyUnicode_GetDefaultEncoding(); + return PyCodec_Decode(self, encoding, errors); } -Py_LOCAL_INLINE(PyObject *) -rsplit_whitespace(const char *s, Py_ssize_t len, Py_ssize_t maxcount) -{ - register Py_ssize_t i, j, count = 0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - for (i = j = len - 1; i >= 0; ) { - /* find a token */ - while (i >= 0 && ISSPACE(s[i])) - i--; - j = i; - while (i >= 0 && !ISSPACE(s[i])) - i--; - if (j > i) { - if (maxcount-- <= 0) - break; - SPLIT_ADD(s, i + 1, j + 1); - while (i >= 0 && ISSPACE(s[i])) - i--; - j = i; - } - } - if (j >= 0) { - SPLIT_ADD(s, 0, j + 1); - } - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - - return list; - - onError: - Py_DECREF(list); - return NULL; -} -PyDoc_STRVAR(rsplit__doc__, -"B.rsplit(sep[, maxsplit]) -> list of bytearray\n\ +PyDoc_STRVAR(fromhex_doc, +"bytes.fromhex(string) -> bytes\n\ \n\ -Return a list of the sections in B, using sep as the delimiter,\n\ -starting at the end of B and working to the front.\n\ -If sep is not given, B is split on ASCII whitespace characters\n\ -(space, tab, return, newline, formfeed, vertical tab).\n\ -If maxsplit is given, at most maxsplit splits are done."); - -static PyObject * -bytes_rsplit(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t len = PyByteArray_GET_SIZE(self), n, i, j; - Py_ssize_t maxsplit = -1, count = 0; - const char *s = PyByteArray_AS_STRING(self), *sub; - PyObject *list, *str, *subobj = Py_None; - Py_buffer vsub; - - if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) - return NULL; - if (maxsplit < 0) - maxsplit = PY_SSIZE_T_MAX; - - if (subobj == Py_None) - return rsplit_whitespace(s, len, maxsplit); - - if (_getbuffer(subobj, &vsub) < 0) - return NULL; - sub = vsub.buf; - n = vsub.len; - - if (n == 0) { - PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - else if (n == 1) - return rsplit_char(s, len, sub[0], maxsplit); - - list = PyList_New(PREALLOC_SIZE(maxsplit)); - if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - - j = len; - i = j - n; - - while ( (i >= 0) && (maxsplit-- > 0) ) { - for (; i>=0; i--) { - if (Py_STRING_MATCH(s, i, sub, n)) { - SPLIT_ADD(s, i + n, j); - j = i; - i -= n; - break; - } - } - } - SPLIT_ADD(s, 0, j); - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - PyObject_ReleaseBuffer(subobj, &vsub); - return list; +Create a bytes object from a string of hexadecimal numbers.\n\ +Spaces between two numbers are accepted.\n\ +Example: bytes.fromhex('B9 01EF') -> b'\\xb9\\x01\\xef'."); -onError: - Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; +static int +hex_digit_to_int(Py_UNICODE c) +{ + if (c >= 128) + return -1; + if (ISDIGIT(c)) + return c - '0'; + else { + if (ISUPPER(c)) + c = TOLOWER(c); + if (c >= 'a' && c <= 'f') + return c - 'a' + 10; + } + return -1; } -PyDoc_STRVAR(reverse__doc__, -"B.reverse() -> None\n\ -\n\ -Reverse the order of the values in B in place."); static PyObject * -bytes_reverse(PyByteArrayObject *self, PyObject *unused) -{ - char swap, *head, *tail; - Py_ssize_t i, j, n = Py_SIZE(self); - - j = n / 2; - head = self->ob_bytes; - tail = head + n - 1; - for (i = 0; i < j; i++) { - swap = *head; - *head++ = *tail; - *tail-- = swap; - } +string_fromhex(PyObject *cls, PyObject *args) +{ + PyObject *newstring, *hexobj; + char *buf; + Py_UNICODE *hex; + Py_ssize_t hexlen, byteslen, i, j; + int top, bot; + + if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) + return NULL; + assert(PyUnicode_Check(hexobj)); + hexlen = PyUnicode_GET_SIZE(hexobj); + hex = PyUnicode_AS_UNICODE(hexobj); + byteslen = hexlen/2; /* This overestimates if there are spaces */ + newstring = PyBytes_FromStringAndSize(NULL, byteslen); + if (!newstring) + return NULL; + buf = PyBytes_AS_STRING(newstring); + for (i = j = 0; i < hexlen; i += 2) { + /* skip over spaces in the input */ + while (hex[i] == ' ') + i++; + if (i >= hexlen) + break; + top = hex_digit_to_int(hex[i]); + bot = hex_digit_to_int(hex[i+1]); + if (top == -1 || bot == -1) { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", i); + goto error; + } + buf[j++] = (top << 4) + bot; + } + if (j != byteslen && _PyBytes_Resize(&newstring, j) < 0) + goto error; + return newstring; - Py_RETURN_NONE; + error: + Py_XDECREF(newstring); + return NULL; } -PyDoc_STRVAR(insert__doc__, -"B.insert(index, int) -> None\n\ -\n\ -Insert a single item into the bytearray before the given index."); -static PyObject * -bytes_insert(PyByteArrayObject *self, PyObject *args) -{ - int value; - Py_ssize_t where, n = Py_SIZE(self); - - if (!PyArg_ParseTuple(args, "ni:insert", &where, &value)) - return NULL; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytes"); - return NULL; - } - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "byte must be in range(0, 256)"); - return NULL; - } - if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) - return NULL; - - if (where < 0) { - where += n; - if (where < 0) - where = 0; - } - if (where > n) - where = n; - memmove(self->ob_bytes + where + 1, self->ob_bytes + where, n - where); - self->ob_bytes[where] = value; - - Py_RETURN_NONE; -} - -PyDoc_STRVAR(append__doc__, -"B.append(int) -> None\n\ -\n\ -Append a single item to the end of B."); static PyObject * -bytes_append(PyByteArrayObject *self, PyObject *arg) +string_getnewargs(PyBytesObject *v) { - int value; - Py_ssize_t n = Py_SIZE(self); - - if (! _getbytevalue(arg, &value)) - return NULL; - if (n == PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "cannot add more objects to bytes"); - return NULL; - } - if (PyByteArray_Resize((PyObject *)self, n + 1) < 0) - return NULL; - - self->ob_bytes[n] = value; - - Py_RETURN_NONE; + return Py_BuildValue("(s#)", v->ob_sval, Py_SIZE(v)); } -PyDoc_STRVAR(extend__doc__, -"B.extend(iterable int) -> None\n\ -\n\ -Append all the elements from the iterator or sequence to the\n\ -end of B."); -static PyObject * -bytes_extend(PyByteArrayObject *self, PyObject *arg) -{ - PyObject *it, *item, *bytes_obj; - Py_ssize_t buf_size = 0, len = 0; - int value; - char *buf; - - /* bytes_setslice code only accepts something supporting PEP 3118. */ - if (PyObject_CheckBuffer(arg)) { - if (bytes_setslice(self, Py_SIZE(self), Py_SIZE(self), arg) == -1) - return NULL; - Py_RETURN_NONE; - } - - it = PyObject_GetIter(arg); - if (it == NULL) - return NULL; - - /* Try to determine the length of the argument. 32 is abitrary. */ - buf_size = _PyObject_LengthHint(arg, 32); - - bytes_obj = PyByteArray_FromStringAndSize(NULL, buf_size); - if (bytes_obj == NULL) - return NULL; - buf = PyByteArray_AS_STRING(bytes_obj); - - while ((item = PyIter_Next(it)) != NULL) { - if (! _getbytevalue(item, &value)) { - Py_DECREF(item); - Py_DECREF(it); - Py_DECREF(bytes_obj); - return NULL; - } - buf[len++] = value; - Py_DECREF(item); - - if (len >= buf_size) { - buf_size = len + (len >> 1) + 1; - if (PyByteArray_Resize((PyObject *)bytes_obj, buf_size) < 0) { - Py_DECREF(it); - Py_DECREF(bytes_obj); - return NULL; - } - /* Recompute the `buf' pointer, since the resizing operation may - have invalidated it. */ - buf = PyByteArray_AS_STRING(bytes_obj); - } - } - Py_DECREF(it); - - /* Resize down to exact size. */ - if (PyByteArray_Resize((PyObject *)bytes_obj, len) < 0) { - Py_DECREF(bytes_obj); - return NULL; - } - - if (bytes_setslice(self, Py_SIZE(self), Py_SIZE(self), bytes_obj) == -1) - return NULL; - Py_DECREF(bytes_obj); - - Py_RETURN_NONE; -} +static PyMethodDef +string_methods[] = { + {"__getnewargs__", (PyCFunction)string_getnewargs, METH_NOARGS}, + {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, + _Py_capitalize__doc__}, + {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, + {"count", (PyCFunction)string_count, METH_VARARGS, count__doc__}, + {"decode", (PyCFunction)string_decode, METH_VARARGS, decode__doc__}, + {"endswith", (PyCFunction)string_endswith, METH_VARARGS, + endswith__doc__}, + {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS, + expandtabs__doc__}, + {"find", (PyCFunction)string_find, METH_VARARGS, find__doc__}, + {"fromhex", (PyCFunction)string_fromhex, METH_VARARGS|METH_CLASS, + fromhex_doc}, + {"index", (PyCFunction)string_index, METH_VARARGS, index__doc__}, + {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, + _Py_isalnum__doc__}, + {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, + _Py_isalpha__doc__}, + {"isdigit", (PyCFunction)stringlib_isdigit, METH_NOARGS, + _Py_isdigit__doc__}, + {"islower", (PyCFunction)stringlib_islower, METH_NOARGS, + _Py_islower__doc__}, + {"isspace", (PyCFunction)stringlib_isspace, METH_NOARGS, + _Py_isspace__doc__}, + {"istitle", (PyCFunction)stringlib_istitle, METH_NOARGS, + _Py_istitle__doc__}, + {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, + _Py_isupper__doc__}, + {"join", (PyCFunction)string_join, METH_O, join__doc__}, + {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, + {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, + {"lstrip", (PyCFunction)string_lstrip, METH_VARARGS, lstrip__doc__}, + {"partition", (PyCFunction)string_partition, METH_O, partition__doc__}, + {"replace", (PyCFunction)string_replace, METH_VARARGS, replace__doc__}, + {"rfind", (PyCFunction)string_rfind, METH_VARARGS, rfind__doc__}, + {"rindex", (PyCFunction)string_rindex, METH_VARARGS, rindex__doc__}, + {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, + {"rpartition", (PyCFunction)string_rpartition, METH_O, + rpartition__doc__}, + {"rsplit", (PyCFunction)string_rsplit, METH_VARARGS, rsplit__doc__}, + {"rstrip", (PyCFunction)string_rstrip, METH_VARARGS, rstrip__doc__}, + {"split", (PyCFunction)string_split, METH_VARARGS, split__doc__}, + {"splitlines", (PyCFunction)stringlib_splitlines, METH_VARARGS, + splitlines__doc__}, + {"startswith", (PyCFunction)string_startswith, METH_VARARGS, + startswith__doc__}, + {"strip", (PyCFunction)string_strip, METH_VARARGS, strip__doc__}, + {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, + _Py_swapcase__doc__}, + {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, + {"translate", (PyCFunction)string_translate, METH_VARARGS, + translate__doc__}, + {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, + {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, + {NULL, NULL} /* sentinel */ +}; -PyDoc_STRVAR(pop__doc__, -"B.pop([index]) -> int\n\ -\n\ -Remove and return a single item from B. If no index\n\ -argument is give, will pop the last value."); static PyObject * -bytes_pop(PyByteArrayObject *self, PyObject *args) -{ - int value; - Py_ssize_t where = -1, n = Py_SIZE(self); - - if (!PyArg_ParseTuple(args, "|n:pop", &where)) - return NULL; - - if (n == 0) { - PyErr_SetString(PyExc_OverflowError, - "cannot pop an empty bytes"); - return NULL; - } - if (where < 0) - where += Py_SIZE(self); - if (where < 0 || where >= Py_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, "pop index out of range"); - return NULL; - } +str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); - value = self->ob_bytes[where]; - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); - if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) - return NULL; - - return PyLong_FromLong(value); -} - -PyDoc_STRVAR(remove__doc__, -"B.remove(int) -> None\n\ -\n\ -Remove the first occurance of a value in B."); static PyObject * -bytes_remove(PyByteArrayObject *self, PyObject *arg) -{ - int value; - Py_ssize_t where, n = Py_SIZE(self); - - if (! _getbytevalue(arg, &value)) - return NULL; - - for (where = 0; where < n; where++) { - if (self->ob_bytes[where] == value) - break; - } - if (where == n) { - PyErr_SetString(PyExc_ValueError, "value not found in bytes"); - return NULL; - } +string_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *x = NULL, *it; + const char *encoding = NULL; + const char *errors = NULL; + PyObject *new = NULL; + Py_ssize_t i, size; + static char *kwlist[] = {"source", "encoding", "errors", 0}; + + if (type != &PyBytes_Type) + return str_subtype_new(type, args, kwds); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:bytes", kwlist, &x, + &encoding, &errors)) + return NULL; + if (x == NULL) { + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without sequence " + "argument"); + return NULL; + } + return PyBytes_FromString(""); + } - memmove(self->ob_bytes + where, self->ob_bytes + where + 1, n - where); - if (PyByteArray_Resize((PyObject *)self, n - 1) < 0) - return NULL; + if (PyUnicode_Check(x)) { + /* Encode via the codec registry */ + if (encoding == NULL) { + PyErr_SetString(PyExc_TypeError, + "string argument without an encoding"); + return NULL; + } + new = PyCodec_Encode(x, encoding, errors); + if (new == NULL) + return NULL; + assert(PyBytes_Check(new)); + return new; + } - Py_RETURN_NONE; -} + /* If it's not unicode, there can't be encoding or errors */ + if (encoding != NULL || errors != NULL) { + PyErr_SetString(PyExc_TypeError, + "encoding or errors without a string argument"); + return NULL; + } -/* XXX These two helpers could be optimized if argsize == 1 */ + /* Is it an int? */ + size = PyNumber_AsSsize_t(x, PyExc_ValueError); + if (size == -1 && PyErr_Occurred()) { + PyErr_Clear(); + } + else { + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative count"); + return NULL; + } + new = PyBytes_FromStringAndSize(NULL, size); + if (new == NULL) { + return NULL; + } + if (size > 0) { + memset(((PyBytesObject*)new)->ob_sval, 0, size); + } + return new; + } -static Py_ssize_t -lstrip_helper(unsigned char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) -{ - Py_ssize_t i = 0; - while (i < mysize && memchr(argptr, myptr[i], argsize)) - i++; - return i; -} + /* Use the modern buffer interface */ + if (PyObject_CheckBuffer(x)) { + Py_buffer view; + if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) + return NULL; + new = PyBytes_FromStringAndSize(NULL, view.len); + if (!new) + goto fail; + // XXX(brett.cannon): Better way to get to internal buffer? + if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, + &view, view.len, 'C') < 0) + goto fail; + PyObject_ReleaseBuffer(x, &view); + return new; + fail: + Py_XDECREF(new); + PyObject_ReleaseBuffer(x, &view); + return NULL; + } -static Py_ssize_t -rstrip_helper(unsigned char *myptr, Py_ssize_t mysize, - void *argptr, Py_ssize_t argsize) -{ - Py_ssize_t i = mysize - 1; - while (i >= 0 && memchr(argptr, myptr[i], argsize)) - i--; - return i + 1; -} + /* For iterator version, create a string object and resize as needed */ + /* XXX(gb): is 64 a good value? also, optimize if length is known */ + /* XXX(guido): perhaps use Pysequence_Fast() -- I can't imagine the + input being a truly long iterator. */ + size = 64; + new = PyBytes_FromStringAndSize(NULL, size); + if (new == NULL) + return NULL; + + /* XXX Optimize this if the arguments is a list, tuple */ + + /* Get the iterator */ + it = PyObject_GetIter(x); + if (it == NULL) + goto error; + + /* Run the iterator to exhaustion */ + for (i = 0; ; i++) { + PyObject *item; + Py_ssize_t value; + + /* Get the next item */ + item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) + goto error; + break; + } + + /* Interpret it as an int (__index__) */ + value = PyNumber_AsSsize_t(item, PyExc_ValueError); + Py_DECREF(item); + if (value == -1 && PyErr_Occurred()) + goto error; + + /* Range check */ + if (value < 0 || value >= 256) { + PyErr_SetString(PyExc_ValueError, + "bytes must be in range(0, 256)"); + goto error; + } + + /* Append the byte */ + if (i >= size) { + size *= 2; + if (_PyBytes_Resize(&new, size) < 0) + goto error; + } + ((PyBytesObject *)new)->ob_sval[i] = value; + } + _PyBytes_Resize(&new, i); -PyDoc_STRVAR(strip__doc__, -"B.strip([bytes]) -> bytearray\n\ -\n\ -Strip leading and trailing bytes contained in the argument.\n\ -If the argument is omitted, strip ASCII whitespace."); -static PyObject * -bytes_strip(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t left, right, mysize, argsize; - void *myptr, *argptr; - PyObject *arg = Py_None; - Py_buffer varg; - if (!PyArg_ParseTuple(args, "|O:strip", &arg)) - return NULL; - if (arg == Py_None) { - argptr = "\t\n\r\f\v "; - argsize = 6; - } - else { - if (_getbuffer(arg, &varg) < 0) - return NULL; - argptr = varg.buf; - argsize = varg.len; - } - myptr = self->ob_bytes; - mysize = Py_SIZE(self); - left = lstrip_helper(myptr, mysize, argptr, argsize); - if (left == mysize) - right = left; - else - right = rstrip_helper(myptr, mysize, argptr, argsize); - if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); -} + /* Clean up and return success */ + Py_DECREF(it); + return new; -PyDoc_STRVAR(lstrip__doc__, -"B.lstrip([bytes]) -> bytearray\n\ -\n\ -Strip leading bytes contained in the argument.\n\ -If the argument is omitted, strip leading ASCII whitespace."); -static PyObject * -bytes_lstrip(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t left, right, mysize, argsize; - void *myptr, *argptr; - PyObject *arg = Py_None; - Py_buffer varg; - if (!PyArg_ParseTuple(args, "|O:lstrip", &arg)) - return NULL; - if (arg == Py_None) { - argptr = "\t\n\r\f\v "; - argsize = 6; - } - else { - if (_getbuffer(arg, &varg) < 0) - return NULL; - argptr = varg.buf; - argsize = varg.len; - } - myptr = self->ob_bytes; - mysize = Py_SIZE(self); - left = lstrip_helper(myptr, mysize, argptr, argsize); - right = mysize; - if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); + error: + /* Error handling when new != NULL */ + Py_XDECREF(it); + Py_DECREF(new); + return NULL; } -PyDoc_STRVAR(rstrip__doc__, -"B.rstrip([bytes]) -> bytearray\n\ -\n\ -Strip trailing bytes contained in the argument.\n\ -If the argument is omitted, strip trailing ASCII whitespace."); static PyObject * -bytes_rstrip(PyByteArrayObject *self, PyObject *args) -{ - Py_ssize_t left, right, mysize, argsize; - void *myptr, *argptr; - PyObject *arg = Py_None; - Py_buffer varg; - if (!PyArg_ParseTuple(args, "|O:rstrip", &arg)) - return NULL; - if (arg == Py_None) { - argptr = "\t\n\r\f\v "; - argsize = 6; - } - else { - if (_getbuffer(arg, &varg) < 0) - return NULL; - argptr = varg.buf; - argsize = varg.len; - } - myptr = self->ob_bytes; - mysize = Py_SIZE(self); - left = 0; - right = rstrip_helper(myptr, mysize, argptr, argsize); - if (arg != Py_None) - PyObject_ReleaseBuffer(arg, &varg); - return PyByteArray_FromStringAndSize(self->ob_bytes + left, right - left); +str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + PyObject *tmp, *pnew; + Py_ssize_t n; + + assert(PyType_IsSubtype(type, &PyBytes_Type)); + tmp = string_new(&PyBytes_Type, args, kwds); + if (tmp == NULL) + return NULL; + assert(PyBytes_CheckExact(tmp)); + n = PyBytes_GET_SIZE(tmp); + pnew = type->tp_alloc(type, n); + if (pnew != NULL) { + Py_MEMCPY(PyBytes_AS_STRING(pnew), + PyBytes_AS_STRING(tmp), n+1); + ((PyBytesObject *)pnew)->ob_shash = + ((PyBytesObject *)tmp)->ob_shash; + } + Py_DECREF(tmp); + return pnew; } -PyDoc_STRVAR(decode_doc, -"B.decode([encoding[, errors]]) -> unicode object.\n\ +PyDoc_STRVAR(string_doc, +"bytes(iterable_of_ints) -> bytes.\n\ +bytes(string, encoding[, errors]) -> bytes\n\ +bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer.\n\ +bytes(memory_view) -> bytes.\n\ \n\ -Decodes B using the codec registered for encoding. encoding defaults\n\ -to the default encoding. errors may be given to set a different error\n\ -handling scheme. Default is 'strict' meaning that encoding errors raise\n\ -a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ -as well as any other name registered with codecs.register_error that is\n\ -able to handle UnicodeDecodeErrors."); - -static PyObject * -bytes_decode(PyObject *self, PyObject *args) -{ - const char *encoding = NULL; - const char *errors = NULL; +Construct an immutable array of bytes from:\n\ + - an iterable yielding integers in range(256)\n\ + - a text string encoded using the specified encoding\n\ + - a bytes or a buffer object\n\ + - any object implementing the buffer API."); + +static PyObject *str_iter(PyObject *seq); + +PyTypeObject PyBytes_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytes", + sizeof(PyBytesObject), + sizeof(char), + string_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)string_repr, /* tp_repr */ + 0, /* tp_as_number */ + &string_as_sequence, /* tp_as_sequence */ + &string_as_mapping, /* tp_as_mapping */ + (hashfunc)string_hash, /* tp_hash */ + 0, /* tp_call */ + string_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + &string_as_buffer, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_BYTES_SUBCLASS, /* tp_flags */ + string_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)string_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + str_iter, /* tp_iter */ + 0, /* tp_iternext */ + string_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + &PyBaseObject_Type, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + string_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; - if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors)) - return NULL; - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - return PyCodec_Decode(self, encoding, errors); +void +PyBytes_Concat(register PyObject **pv, register PyObject *w) +{ + register PyObject *v; + assert(pv != NULL); + if (*pv == NULL) + return; + if (w == NULL) { + Py_DECREF(*pv); + *pv = NULL; + return; + } + v = string_concat(*pv, w); + Py_DECREF(*pv); + *pv = v; } -PyDoc_STRVAR(alloc_doc, -"B.__alloc__() -> int\n\ -\n\ -Returns the number of bytes actually allocated."); - -static PyObject * -bytes_alloc(PyByteArrayObject *self) +void +PyBytes_ConcatAndDel(register PyObject **pv, register PyObject *w) { - return PyLong_FromSsize_t(self->ob_alloc); + PyBytes_Concat(pv, w); + Py_XDECREF(w); } -PyDoc_STRVAR(join_doc, -"B.join(iterable_of_bytes) -> bytes\n\ -\n\ -Concatenates any number of bytearray objects, with B in between each pair."); -static PyObject * -bytes_join(PyByteArrayObject *self, PyObject *it) -{ - PyObject *seq; - Py_ssize_t mysize = Py_SIZE(self); - Py_ssize_t i; - Py_ssize_t n; - PyObject **items; - Py_ssize_t totalsize = 0; - PyObject *result; - char *dest; - - seq = PySequence_Fast(it, "can only join an iterable"); - if (seq == NULL) - return NULL; - n = PySequence_Fast_GET_SIZE(seq); - items = PySequence_Fast_ITEMS(seq); - - /* Compute the total size, and check that they are all bytes */ - /* XXX Shouldn't we use _getbuffer() on these items instead? */ - for (i = 0; i < n; i++) { - PyObject *obj = items[i]; - if (!PyByteArray_Check(obj) && !PyBytes_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "can only join an iterable of bytes " - "(item %ld has type '%.100s')", - /* XXX %ld isn't right on Win64 */ - (long)i, Py_TYPE(obj)->tp_name); - goto error; - } - if (i > 0) - totalsize += mysize; - totalsize += Py_SIZE(obj); - if (totalsize < 0) { - PyErr_NoMemory(); - goto error; - } - } - - /* Allocate the result, and copy the bytes */ - result = PyByteArray_FromStringAndSize(NULL, totalsize); - if (result == NULL) - goto error; - dest = PyByteArray_AS_STRING(result); - for (i = 0; i < n; i++) { - PyObject *obj = items[i]; - Py_ssize_t size = Py_SIZE(obj); - char *buf; - if (PyByteArray_Check(obj)) - buf = PyByteArray_AS_STRING(obj); - else - buf = PyBytes_AS_STRING(obj); - if (i) { - memcpy(dest, self->ob_bytes, mysize); - dest += mysize; - } - memcpy(dest, buf, size); - dest += size; - } +/* The following function breaks the notion that strings are immutable: + it changes the size of a string. We get away with this only if there + is only one module referencing the object. You can also think of it + as creating a new string object and destroying the old one, only + more efficiently. In any case, don't use this if the string may + already be known to some other part of the code... + Note that if there's not enough memory to resize the string, the original + string object at *pv is deallocated, *pv is set to NULL, an "out of + memory" exception is set, and -1 is returned. Else (on success) 0 is + returned, and the value in *pv may or may not be the same as on input. + As always, an extra byte is allocated for a trailing \0 byte (newsize + does *not* include that), and a trailing \0 byte is stored. +*/ - /* Done */ - Py_DECREF(seq); - return result; +int +_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) +{ + register PyObject *v; + register PyBytesObject *sv; + v = *pv; + if (!PyBytes_Check(v) || Py_REFCNT(v) != 1 || newsize < 0) { + *pv = 0; + Py_DECREF(v); + PyErr_BadInternalCall(); + return -1; + } + /* XXX UNREF/NEWREF interface should be more symmetrical */ + _Py_DEC_REFTOTAL; + _Py_ForgetReference(v); + *pv = (PyObject *) + PyObject_REALLOC((char *)v, sizeof(PyBytesObject) + newsize); + if (*pv == NULL) { + PyObject_Del(v); + PyErr_NoMemory(); + return -1; + } + _Py_NewReference(*pv); + sv = (PyBytesObject *) *pv; + Py_SIZE(sv) = newsize; + sv->ob_sval[newsize] = '\0'; + sv->ob_shash = -1; /* invalidate cached hash value */ + return 0; +} + +/* _PyBytes_FormatLong emulates the format codes d, u, o, x and X, and + * the F_ALT flag, for Python's long (unbounded) ints. It's not used for + * Python's regular ints. + * Return value: a new PyString*, or NULL if error. + * . *pbuf is set to point into it, + * *plen set to the # of chars following that. + * Caller must decref it when done using pbuf. + * The string starting at *pbuf is of the form + * "-"? ("0x" | "0X")? digit+ + * "0x"/"0X" are present only for x and X conversions, with F_ALT + * set in flags. The case of hex digits will be correct, + * There will be at least prec digits, zero-filled on the left if + * necessary to get that many. + * val object to be converted + * flags bitmask of format flags; only F_ALT is looked at + * prec minimum number of digits; 0-fill on left if needed + * type a character in [duoxX]; u acts the same as d + * + * CAUTION: o, x and X conversions on regular ints can never + * produce a '-' sign, but can for Python's unbounded ints. + */ +PyObject* +_PyBytes_FormatLong(PyObject *val, int flags, int prec, int type, + char **pbuf, int *plen) +{ + PyObject *result = NULL; + char *buf; + Py_ssize_t i; + int sign; /* 1 if '-', else 0 */ + int len; /* number of characters */ + Py_ssize_t llen; + int numdigits; /* len == numnondigits + numdigits */ + int numnondigits = 0; + + /* Avoid exceeding SSIZE_T_MAX */ + if (prec > PY_SSIZE_T_MAX-3) { + PyErr_SetString(PyExc_OverflowError, + "precision too large"); + return NULL; + } - /* Error handling */ - error: - Py_DECREF(seq); - return NULL; -} + switch (type) { + case 'd': + case 'u': + /* Special-case boolean: we want 0/1 */ + if (PyBool_Check(val)) + result = PyNumber_ToBase(val, 10); + else + result = Py_TYPE(val)->tp_str(val); + break; + case 'o': + numnondigits = 2; + result = PyNumber_ToBase(val, 8); + break; + case 'x': + case 'X': + numnondigits = 2; + result = PyNumber_ToBase(val, 16); + break; + default: + assert(!"'type' not in [duoxX]"); + } + if (!result) + return NULL; -PyDoc_STRVAR(fromhex_doc, -"bytearray.fromhex(string) -> bytearray\n\ -\n\ -Create a bytearray object from a string of hexadecimal numbers.\n\ -Spaces between two numbers are accepted.\n\ -Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef')."); + buf = PyUnicode_AsString(result); + if (!buf) { + Py_DECREF(result); + return NULL; + } -static int -hex_digit_to_int(Py_UNICODE c) -{ - if (c >= 128) - return -1; - if (ISDIGIT(c)) - return c - '0'; - else { - if (ISUPPER(c)) - c = TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} + /* To modify the string in-place, there can only be one reference. */ + if (Py_REFCNT(result) != 1) { + PyErr_BadInternalCall(); + return NULL; + } + llen = PyUnicode_GetSize(result); + if (llen > INT_MAX) { + PyErr_SetString(PyExc_ValueError, + "string too large in _PyBytes_FormatLong"); + return NULL; + } + len = (int)llen; + if (buf[len-1] == 'L') { + --len; + buf[len] = '\0'; + } + sign = buf[0] == '-'; + numnondigits += sign; + numdigits = len - numnondigits; + assert(numdigits > 0); + + /* Get rid of base marker unless F_ALT */ + if (((flags & F_ALT) == 0 && + (type == 'o' || type == 'x' || type == 'X'))) { + assert(buf[sign] == '0'); + assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || + buf[sign+1] == 'o'); + numnondigits -= 2; + buf += 2; + len -= 2; + if (sign) + buf[0] = '-'; + assert(len == numnondigits + numdigits); + assert(numdigits > 0); + } -static PyObject * -bytes_fromhex(PyObject *cls, PyObject *args) -{ - PyObject *newbytes, *hexobj; - char *buf; - Py_UNICODE *hex; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - - if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) - return NULL; - assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newbytes = PyByteArray_FromStringAndSize(NULL, byteslen); - if (!newbytes) - return NULL; - buf = PyByteArray_AS_STRING(newbytes); - for (i = j = 0; i < hexlen; i += 2) { - /* skip over spaces in the input */ - while (hex[i] == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); - goto error; - } - buf[j++] = (top << 4) + bot; - } - if (PyByteArray_Resize(newbytes, j) < 0) - goto error; - return newbytes; + /* Fill with leading zeroes to meet minimum width. */ + if (prec > numdigits) { + PyObject *r1 = PyBytes_FromStringAndSize(NULL, + numnondigits + prec); + char *b1; + if (!r1) { + Py_DECREF(result); + return NULL; + } + b1 = PyBytes_AS_STRING(r1); + for (i = 0; i < numnondigits; ++i) + *b1++ = *buf++; + for (i = 0; i < prec - numdigits; i++) + *b1++ = '0'; + for (i = 0; i < numdigits; i++) + *b1++ = *buf++; + *b1 = '\0'; + Py_DECREF(result); + result = r1; + buf = PyBytes_AS_STRING(result); + len = numnondigits + prec; + } - error: - Py_DECREF(newbytes); - return NULL; + /* Fix up case for hex conversions. */ + if (type == 'X') { + /* Need to convert all lower case letters to upper case. + and need to convert 0x to 0X (and -0x to -0X). */ + for (i = 0; i < len; i++) + if (buf[i] >= 'a' && buf[i] <= 'x') + buf[i] -= 'a'-'A'; + } + *pbuf = buf; + *plen = len; + return result; } -PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); - -static PyObject * -bytes_reduce(PyByteArrayObject *self) +void +PyBytes_Fini(void) { - PyObject *latin1, *dict; - if (self->ob_bytes) - latin1 = PyUnicode_DecodeLatin1(self->ob_bytes, - Py_SIZE(self), NULL); - else - latin1 = PyUnicode_FromString(""); - - dict = PyObject_GetAttrString((PyObject *)self, "__dict__"); - if (dict == NULL) { - PyErr_Clear(); - dict = Py_None; - Py_INCREF(dict); - } - - return Py_BuildValue("(O(Ns)N)", Py_TYPE(self), latin1, "latin-1", dict); + int i; + for (i = 0; i < UCHAR_MAX + 1; i++) { + Py_XDECREF(characters[i]); + characters[i] = NULL; + } + Py_XDECREF(nullstring); + nullstring = NULL; } -static PySequenceMethods bytes_as_sequence = { - (lenfunc)bytes_length, /* sq_length */ - (binaryfunc)PyByteArray_Concat, /* sq_concat */ - (ssizeargfunc)bytes_repeat, /* sq_repeat */ - (ssizeargfunc)bytes_getitem, /* sq_item */ - 0, /* sq_slice */ - (ssizeobjargproc)bytes_setitem, /* sq_ass_item */ - 0, /* sq_ass_slice */ - (objobjproc)bytes_contains, /* sq_contains */ - (binaryfunc)bytes_iconcat, /* sq_inplace_concat */ - (ssizeargfunc)bytes_irepeat, /* sq_inplace_repeat */ -}; - -static PyMappingMethods bytes_as_mapping = { - (lenfunc)bytes_length, - (binaryfunc)bytes_subscript, - (objobjargproc)bytes_ass_subscript, -}; - -static PyBufferProcs bytes_as_buffer = { - (getbufferproc)bytes_getbuffer, - (releasebufferproc)bytes_releasebuffer, -}; - -static PyMethodDef -bytes_methods[] = { - {"__alloc__", (PyCFunction)bytes_alloc, METH_NOARGS, alloc_doc}, - {"__reduce__", (PyCFunction)bytes_reduce, METH_NOARGS, reduce_doc}, - {"append", (PyCFunction)bytes_append, METH_O, append__doc__}, - {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, - _Py_capitalize__doc__}, - {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, - {"count", (PyCFunction)bytes_count, METH_VARARGS, count__doc__}, - {"decode", (PyCFunction)bytes_decode, METH_VARARGS, decode_doc}, - {"endswith", (PyCFunction)bytes_endswith, METH_VARARGS, endswith__doc__}, - {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS, - expandtabs__doc__}, - {"extend", (PyCFunction)bytes_extend, METH_O, extend__doc__}, - {"find", (PyCFunction)bytes_find, METH_VARARGS, find__doc__}, - {"fromhex", (PyCFunction)bytes_fromhex, METH_VARARGS|METH_CLASS, - fromhex_doc}, - {"index", (PyCFunction)bytes_index, METH_VARARGS, index__doc__}, - {"insert", (PyCFunction)bytes_insert, METH_VARARGS, insert__doc__}, - {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, - _Py_isalnum__doc__}, - {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, - _Py_isalpha__doc__}, - {"isdigit", (PyCFunction)stringlib_isdigit, METH_NOARGS, - _Py_isdigit__doc__}, - {"islower", (PyCFunction)stringlib_islower, METH_NOARGS, - _Py_islower__doc__}, - {"isspace", (PyCFunction)stringlib_isspace, METH_NOARGS, - _Py_isspace__doc__}, - {"istitle", (PyCFunction)stringlib_istitle, METH_NOARGS, - _Py_istitle__doc__}, - {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, - _Py_isupper__doc__}, - {"join", (PyCFunction)bytes_join, METH_O, join_doc}, - {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, - {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, - {"lstrip", (PyCFunction)bytes_lstrip, METH_VARARGS, lstrip__doc__}, - {"partition", (PyCFunction)bytes_partition, METH_O, partition__doc__}, - {"pop", (PyCFunction)bytes_pop, METH_VARARGS, pop__doc__}, - {"remove", (PyCFunction)bytes_remove, METH_O, remove__doc__}, - {"replace", (PyCFunction)bytes_replace, METH_VARARGS, replace__doc__}, - {"reverse", (PyCFunction)bytes_reverse, METH_NOARGS, reverse__doc__}, - {"rfind", (PyCFunction)bytes_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction)bytes_rindex, METH_VARARGS, rindex__doc__}, - {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, - {"rpartition", (PyCFunction)bytes_rpartition, METH_O, rpartition__doc__}, - {"rsplit", (PyCFunction)bytes_rsplit, METH_VARARGS, rsplit__doc__}, - {"rstrip", (PyCFunction)bytes_rstrip, METH_VARARGS, rstrip__doc__}, - {"split", (PyCFunction)bytes_split, METH_VARARGS, split__doc__}, - {"splitlines", (PyCFunction)stringlib_splitlines, METH_VARARGS, - splitlines__doc__}, - {"startswith", (PyCFunction)bytes_startswith, METH_VARARGS , - startswith__doc__}, - {"strip", (PyCFunction)bytes_strip, METH_VARARGS, strip__doc__}, - {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, - _Py_swapcase__doc__}, - {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, - {"translate", (PyCFunction)bytes_translate, METH_VARARGS, - translate__doc__}, - {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, - {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, - {NULL} -}; - -PyDoc_STRVAR(bytes_doc, -"bytearray(iterable_of_ints) -> bytearray.\n\ -bytearray(string, encoding[, errors]) -> bytearray.\n\ -bytearray(bytes_or_bytearray) -> mutable copy of bytes_or_bytearray.\n\ -bytearray(memory_view) -> bytearray.\n\ -\n\ -Construct an mutable bytearray object from:\n\ - - an iterable yielding integers in range(256)\n\ - - a text string encoded using the specified encoding\n\ - - a bytes or a bytearray object\n\ - - any object implementing the buffer API.\n\ -\n\ -bytearray(int) -> bytearray.\n\ -\n\ -Construct a zero-initialized bytearray of the given length."); - - -static PyObject *bytes_iter(PyObject *seq); - -PyTypeObject PyByteArray_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "bytearray", - sizeof(PyByteArrayObject), - 0, - (destructor)bytes_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - (reprfunc)bytes_repr, /* tp_repr */ - 0, /* tp_as_number */ - &bytes_as_sequence, /* tp_as_sequence */ - &bytes_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - bytes_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - &bytes_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - bytes_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - (richcmpfunc)bytes_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - bytes_iter, /* tp_iter */ - 0, /* tp_iternext */ - bytes_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)bytes_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ - PyObject_Del, /* tp_free */ -}; - -/*********************** Bytes Iterator ****************************/ +/*********************** Str Iterator ****************************/ typedef struct { - PyObject_HEAD - Py_ssize_t it_index; - PyByteArrayObject *it_seq; /* Set to NULL when iterator is exhausted */ -} bytesiterobject; + PyObject_HEAD + Py_ssize_t it_index; + PyBytesObject *it_seq; /* Set to NULL when iterator is exhausted */ +} striterobject; static void -bytesiter_dealloc(bytesiterobject *it) +striter_dealloc(striterobject *it) { - _PyObject_GC_UNTRACK(it); - Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); + _PyObject_GC_UNTRACK(it); + Py_XDECREF(it->it_seq); + PyObject_GC_Del(it); } static int -bytesiter_traverse(bytesiterobject *it, visitproc visit, void *arg) +striter_traverse(striterobject *it, visitproc visit, void *arg) { - Py_VISIT(it->it_seq); - return 0; + Py_VISIT(it->it_seq); + return 0; } static PyObject * -bytesiter_next(bytesiterobject *it) -{ - PyByteArrayObject *seq; - PyObject *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) - return NULL; - assert(PyByteArray_Check(seq)); - - if (it->it_index < PyByteArray_GET_SIZE(seq)) { - item = PyLong_FromLong( - (unsigned char)seq->ob_bytes[it->it_index]); - if (item != NULL) - ++it->it_index; - return item; - } +striter_next(striterobject *it) +{ + PyBytesObject *seq; + PyObject *item; + + assert(it != NULL); + seq = it->it_seq; + if (seq == NULL) + return NULL; + assert(PyBytes_Check(seq)); + + if (it->it_index < PyBytes_GET_SIZE(seq)) { + item = PyLong_FromLong( + (unsigned char)seq->ob_sval[it->it_index]); + if (item != NULL) + ++it->it_index; + return item; + } - Py_DECREF(seq); - it->it_seq = NULL; - return NULL; + Py_DECREF(seq); + it->it_seq = NULL; + return NULL; } static PyObject * -bytesiter_length_hint(bytesiterobject *it) +striter_len(striterobject *it) { - Py_ssize_t len = 0; - if (it->it_seq) - len = PyByteArray_GET_SIZE(it->it_seq) - it->it_index; - return PyLong_FromSsize_t(len); + Py_ssize_t len = 0; + if (it->it_seq) + len = PyBytes_GET_SIZE(it->it_seq) - it->it_index; + return PyLong_FromSsize_t(len); } PyDoc_STRVAR(length_hint_doc, - "Private method returning an estimate of len(list(it))."); + "Private method returning an estimate of len(list(it))."); -static PyMethodDef bytesiter_methods[] = { - {"__length_hint__", (PyCFunction)bytesiter_length_hint, METH_NOARGS, - length_hint_doc}, - {NULL, NULL} /* sentinel */ +static PyMethodDef striter_methods[] = { + {"__length_hint__", (PyCFunction)striter_len, METH_NOARGS, + length_hint_doc}, + {NULL, NULL} /* sentinel */ }; -PyTypeObject PyByteArrayIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "bytearray_iterator", /* tp_name */ - sizeof(bytesiterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)bytesiter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - 0, /* tp_doc */ - (traverseproc)bytesiter_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)bytesiter_next, /* tp_iternext */ - bytesiter_methods, /* tp_methods */ - 0, +PyTypeObject PyBytesIter_Type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "bytes_iterator", /* tp_name */ + sizeof(striterobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)striter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ + 0, /* tp_doc */ + (traverseproc)striter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + PyObject_SelfIter, /* tp_iter */ + (iternextfunc)striter_next, /* tp_iternext */ + striter_methods, /* tp_methods */ + 0, }; static PyObject * -bytes_iter(PyObject *seq) +str_iter(PyObject *seq) { - bytesiterobject *it; + striterobject *it; - if (!PyByteArray_Check(seq)) { - PyErr_BadInternalCall(); - return NULL; - } - it = PyObject_GC_New(bytesiterobject, &PyByteArrayIter_Type); - if (it == NULL) - return NULL; - it->it_index = 0; - Py_INCREF(seq); - it->it_seq = (PyByteArrayObject *)seq; - _PyObject_GC_TRACK(it); - return (PyObject *)it; + if (!PyBytes_Check(seq)) { + PyErr_BadInternalCall(); + return NULL; + } + it = PyObject_GC_New(striterobject, &PyBytesIter_Type); + if (it == NULL) + return NULL; + it->it_index = 0; + Py_INCREF(seq); + it->it_seq = (PyBytesObject *)seq; + _PyObject_GC_TRACK(it); + return (PyObject *)it; } diff --git a/Objects/stringobject.c b/Objects/stringobject.c deleted file mode 100644 index b9ba73f..0000000 --- a/Objects/stringobject.c +++ /dev/null @@ -1,3429 +0,0 @@ -/* String object implementation */ - -/* XXX This is now called 'bytes' as far as the user is concerned. - Many docstrings and error messages need to be cleaned up. */ - -#define PY_SSIZE_T_CLEAN - -#include "Python.h" - -#include "bytes_methods.h" - -static Py_ssize_t -_getbuffer(PyObject *obj, Py_buffer *view) -{ - PyBufferProcs *buffer = Py_TYPE(obj)->tp_as_buffer; - - if (buffer == NULL || buffer->bf_getbuffer == NULL) - { - PyErr_Format(PyExc_TypeError, - "Type %.100s doesn't support the buffer API", - Py_TYPE(obj)->tp_name); - return -1; - } - - if (buffer->bf_getbuffer(obj, view, PyBUF_SIMPLE) < 0) - return -1; - return view->len; -} - -#ifdef COUNT_ALLOCS -int null_strings, one_strings; -#endif - -static PyBytesObject *characters[UCHAR_MAX + 1]; -static PyBytesObject *nullstring; - -/* - For both PyBytes_FromString() and PyBytes_FromStringAndSize(), the - parameter `size' denotes number of characters to allocate, not counting any - null terminating character. - - For PyBytes_FromString(), the parameter `str' points to a null-terminated - string containing exactly `size' bytes. - - For PyBytes_FromStringAndSize(), the parameter the parameter `str' is - either NULL or else points to a string containing at least `size' bytes. - For PyBytes_FromStringAndSize(), the string in the `str' parameter does - not have to be null-terminated. (Therefore it is safe to construct a - substring by calling `PyBytes_FromStringAndSize(origstring, substrlen)'.) - If `str' is NULL then PyBytes_FromStringAndSize() will allocate `size+1' - bytes (setting the last byte to the null terminating character) and you can - fill in the data yourself. If `str' is non-NULL then the resulting - PyString object must be treated as immutable and you must not fill in nor - alter the data yourself, since the strings may be shared. - - The PyObject member `op->ob_size', which denotes the number of "extra - items" in a variable-size object, will contain the number of bytes - allocated for string data, not counting the null terminating character. It - is therefore equal to the equal to the `size' parameter (for - PyBytes_FromStringAndSize()) or the length of the string in the `str' - parameter (for PyBytes_FromString()). -*/ -PyObject * -PyBytes_FromStringAndSize(const char *str, Py_ssize_t size) -{ - register PyBytesObject *op; - if (size < 0) { - PyErr_SetString(PyExc_SystemError, - "Negative size passed to PyBytes_FromStringAndSize"); - return NULL; - } - if (size == 0 && (op = nullstring) != NULL) { -#ifdef COUNT_ALLOCS - null_strings++; -#endif - Py_INCREF(op); - return (PyObject *)op; - } - if (size == 1 && str != NULL && - (op = characters[*str & UCHAR_MAX]) != NULL) - { -#ifdef COUNT_ALLOCS - one_strings++; -#endif - Py_INCREF(op); - return (PyObject *)op; - } - - /* Inline PyObject_NewVar */ - op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size); - if (op == NULL) - return PyErr_NoMemory(); - PyObject_INIT_VAR(op, &PyBytes_Type, size); - op->ob_shash = -1; - if (str != NULL) - Py_MEMCPY(op->ob_sval, str, size); - op->ob_sval[size] = '\0'; - /* share short strings */ - if (size == 0) { - nullstring = op; - Py_INCREF(op); - } else if (size == 1 && str != NULL) { - characters[*str & UCHAR_MAX] = op; - Py_INCREF(op); - } - return (PyObject *) op; -} - -PyObject * -PyBytes_FromString(const char *str) -{ - register size_t size; - register PyBytesObject *op; - - assert(str != NULL); - size = strlen(str); - if (size > PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "string is too long for a Python string"); - return NULL; - } - if (size == 0 && (op = nullstring) != NULL) { -#ifdef COUNT_ALLOCS - null_strings++; -#endif - Py_INCREF(op); - return (PyObject *)op; - } - if (size == 1 && (op = characters[*str & UCHAR_MAX]) != NULL) { -#ifdef COUNT_ALLOCS - one_strings++; -#endif - Py_INCREF(op); - return (PyObject *)op; - } - - /* Inline PyObject_NewVar */ - op = (PyBytesObject *)PyObject_MALLOC(sizeof(PyBytesObject) + size); - if (op == NULL) - return PyErr_NoMemory(); - PyObject_INIT_VAR(op, &PyBytes_Type, size); - op->ob_shash = -1; - Py_MEMCPY(op->ob_sval, str, size+1); - /* share short strings */ - if (size == 0) { - nullstring = op; - Py_INCREF(op); - } else if (size == 1) { - characters[*str & UCHAR_MAX] = op; - Py_INCREF(op); - } - return (PyObject *) op; -} - -PyObject * -PyBytes_FromFormatV(const char *format, va_list vargs) -{ - va_list count; - Py_ssize_t n = 0; - const char* f; - char *s; - PyObject* string; - -#ifdef VA_LIST_IS_ARRAY - Py_MEMCPY(count, vargs, sizeof(va_list)); -#else -#ifdef __va_copy - __va_copy(count, vargs); -#else - count = vargs; -#endif -#endif - /* step 1: figure out how large a buffer we need */ - for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f; - while (*++f && *f != '%' && !ISALPHA(*f)) - ; - - /* skip the 'l' or 'z' in {%ld, %zd, %lu, %zu} since - * they don't affect the amount of space we reserve. - */ - if ((*f == 'l' || *f == 'z') && - (f[1] == 'd' || f[1] == 'u')) - ++f; - - switch (*f) { - case 'c': - (void)va_arg(count, int); - /* fall through... */ - case '%': - n++; - break; - case 'd': case 'u': case 'i': case 'x': - (void) va_arg(count, int); - /* 20 bytes is enough to hold a 64-bit - integer. Decimal takes the most space. - This isn't enough for octal. */ - n += 20; - break; - case 's': - s = va_arg(count, char*); - n += strlen(s); - break; - case 'p': - (void) va_arg(count, int); - /* maximum 64-bit pointer representation: - * 0xffffffffffffffff - * so 19 characters is enough. - * XXX I count 18 -- what's the extra for? - */ - n += 19; - break; - default: - /* if we stumble upon an unknown - formatting code, copy the rest of - the format string to the output - string. (we cannot just skip the - code, since there's no way to know - what's in the argument list) */ - n += strlen(p); - goto expand; - } - } else - n++; - } - expand: - /* step 2: fill the buffer */ - /* Since we've analyzed how much space we need for the worst case, - use sprintf directly instead of the slower PyOS_snprintf. */ - string = PyBytes_FromStringAndSize(NULL, n); - if (!string) - return NULL; - - s = PyBytes_AsString(string); - - for (f = format; *f; f++) { - if (*f == '%') { - const char* p = f++; - Py_ssize_t i; - int longflag = 0; - int size_tflag = 0; - /* parse the width.precision part (we're only - interested in the precision value, if any) */ - n = 0; - while (ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - if (*f == '.') { - f++; - n = 0; - while (ISDIGIT(*f)) - n = (n*10) + *f++ - '0'; - } - while (*f && *f != '%' && !ISALPHA(*f)) - f++; - /* handle the long flag, but only for %ld and %lu. - others can be added when necessary. */ - if (*f == 'l' && (f[1] == 'd' || f[1] == 'u')) { - longflag = 1; - ++f; - } - /* handle the size_t flag. */ - if (*f == 'z' && (f[1] == 'd' || f[1] == 'u')) { - size_tflag = 1; - ++f; - } - - switch (*f) { - case 'c': - *s++ = va_arg(vargs, int); - break; - case 'd': - if (longflag) - sprintf(s, "%ld", va_arg(vargs, long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "d", - va_arg(vargs, Py_ssize_t)); - else - sprintf(s, "%d", va_arg(vargs, int)); - s += strlen(s); - break; - case 'u': - if (longflag) - sprintf(s, "%lu", - va_arg(vargs, unsigned long)); - else if (size_tflag) - sprintf(s, "%" PY_FORMAT_SIZE_T "u", - va_arg(vargs, size_t)); - else - sprintf(s, "%u", - va_arg(vargs, unsigned int)); - s += strlen(s); - break; - case 'i': - sprintf(s, "%i", va_arg(vargs, int)); - s += strlen(s); - break; - case 'x': - sprintf(s, "%x", va_arg(vargs, int)); - s += strlen(s); - break; - case 's': - p = va_arg(vargs, char*); - i = strlen(p); - if (n > 0 && i > n) - i = n; - Py_MEMCPY(s, p, i); - s += i; - break; - case 'p': - sprintf(s, "%p", va_arg(vargs, void*)); - /* %p is ill-defined: ensure leading 0x. */ - if (s[1] == 'X') - s[1] = 'x'; - else if (s[1] != 'x') { - memmove(s+2, s, strlen(s)+1); - s[0] = '0'; - s[1] = 'x'; - } - s += strlen(s); - break; - case '%': - *s++ = '%'; - break; - default: - strcpy(s, p); - s += strlen(s); - goto end; - } - } else - *s++ = *f; - } - - end: - _PyBytes_Resize(&string, s - PyBytes_AS_STRING(string)); - return string; -} - -PyObject * -PyBytes_FromFormat(const char *format, ...) -{ - PyObject* ret; - va_list vargs; - -#ifdef HAVE_STDARG_PROTOTYPES - va_start(vargs, format); -#else - va_start(vargs); -#endif - ret = PyBytes_FromFormatV(format, vargs); - va_end(vargs); - return ret; -} - -static void -string_dealloc(PyObject *op) -{ - Py_TYPE(op)->tp_free(op); -} - -/* Unescape a backslash-escaped string. If unicode is non-zero, - the string is a u-literal. If recode_encoding is non-zero, - the string is UTF-8 encoded and should be re-encoded in the - specified encoding. */ - -PyObject *PyBytes_DecodeEscape(const char *s, - Py_ssize_t len, - const char *errors, - Py_ssize_t unicode, - const char *recode_encoding) -{ - int c; - char *p, *buf; - const char *end; - PyObject *v; - Py_ssize_t newlen = recode_encoding ? 4*len:len; - v = PyBytes_FromStringAndSize((char *)NULL, newlen); - if (v == NULL) - return NULL; - p = buf = PyBytes_AsString(v); - end = s + len; - while (s < end) { - if (*s != '\\') { - non_esc: - if (recode_encoding && (*s & 0x80)) { - PyObject *u, *w; - char *r; - const char* t; - Py_ssize_t rn; - t = s; - /* Decode non-ASCII bytes as UTF-8. */ - while (t < end && (*t & 0x80)) t++; - u = PyUnicode_DecodeUTF8(s, t - s, errors); - if(!u) goto failed; - - /* Recode them in target encoding. */ - w = PyUnicode_AsEncodedString( - u, recode_encoding, errors); - Py_DECREF(u); - if (!w) goto failed; - - /* Append bytes to output buffer. */ - assert(PyBytes_Check(w)); - r = PyBytes_AS_STRING(w); - rn = PyBytes_GET_SIZE(w); - Py_MEMCPY(p, r, rn); - p += rn; - Py_DECREF(w); - s = t; - } else { - *p++ = *s++; - } - continue; - } - s++; - if (s==end) { - PyErr_SetString(PyExc_ValueError, - "Trailing \\ in string"); - goto failed; - } - switch (*s++) { - /* XXX This assumes ASCII! */ - case '\n': break; - case '\\': *p++ = '\\'; break; - case '\'': *p++ = '\''; break; - case '\"': *p++ = '\"'; break; - case 'b': *p++ = '\b'; break; - case 'f': *p++ = '\014'; break; /* FF */ - case 't': *p++ = '\t'; break; - case 'n': *p++ = '\n'; break; - case 'r': *p++ = '\r'; break; - case 'v': *p++ = '\013'; break; /* VT */ - case 'a': *p++ = '\007'; break; /* BEL, not classic C */ - case '0': case '1': case '2': case '3': - case '4': case '5': case '6': case '7': - c = s[-1] - '0'; - if (s < end && '0' <= *s && *s <= '7') { - c = (c<<3) + *s++ - '0'; - if (s < end && '0' <= *s && *s <= '7') - c = (c<<3) + *s++ - '0'; - } - *p++ = c; - break; - case 'x': - if (s+1 < end && ISXDIGIT(s[0]) && ISXDIGIT(s[1])) { - unsigned int x = 0; - c = Py_CHARMASK(*s); - s++; - if (ISDIGIT(c)) - x = c - '0'; - else if (ISLOWER(c)) - x = 10 + c - 'a'; - else - x = 10 + c - 'A'; - x = x << 4; - c = Py_CHARMASK(*s); - s++; - if (ISDIGIT(c)) - x += c - '0'; - else if (ISLOWER(c)) - x += 10 + c - 'a'; - else - x += 10 + c - 'A'; - *p++ = x; - break; - } - if (!errors || strcmp(errors, "strict") == 0) { - PyErr_SetString(PyExc_ValueError, - "invalid \\x escape"); - goto failed; - } - if (strcmp(errors, "replace") == 0) { - *p++ = '?'; - } else if (strcmp(errors, "ignore") == 0) - /* do nothing */; - else { - PyErr_Format(PyExc_ValueError, - "decoding error; unknown " - "error handling code: %.400s", - errors); - goto failed; - } - default: - *p++ = '\\'; - s--; - goto non_esc; /* an arbitry number of unescaped - UTF-8 bytes may follow. */ - } - } - if (p-buf < newlen) - _PyBytes_Resize(&v, p - buf); - return v; - failed: - Py_DECREF(v); - return NULL; -} - -/* -------------------------------------------------------------------- */ -/* object api */ - -Py_ssize_t -PyBytes_Size(register PyObject *op) -{ - if (!PyBytes_Check(op)) { - PyErr_Format(PyExc_TypeError, - "expected bytes, %.200s found", Py_TYPE(op)->tp_name); - return -1; - } - return Py_SIZE(op); -} - -char * -PyBytes_AsString(register PyObject *op) -{ - if (!PyBytes_Check(op)) { - PyErr_Format(PyExc_TypeError, - "expected bytes, %.200s found", Py_TYPE(op)->tp_name); - return NULL; - } - return ((PyBytesObject *)op)->ob_sval; -} - -int -PyBytes_AsStringAndSize(register PyObject *obj, - register char **s, - register Py_ssize_t *len) -{ - if (s == NULL) { - PyErr_BadInternalCall(); - return -1; - } - - if (!PyBytes_Check(obj)) { - PyErr_Format(PyExc_TypeError, - "expected bytes, %.200s found", Py_TYPE(obj)->tp_name); - return -1; - } - - *s = PyBytes_AS_STRING(obj); - if (len != NULL) - *len = PyBytes_GET_SIZE(obj); - else if (strlen(*s) != (size_t)PyBytes_GET_SIZE(obj)) { - PyErr_SetString(PyExc_TypeError, - "expected bytes with no null"); - return -1; - } - return 0; -} - -/* -------------------------------------------------------------------- */ -/* Methods */ - -#define STRINGLIB_CHAR char - -#define STRINGLIB_CMP memcmp -#define STRINGLIB_LEN PyBytes_GET_SIZE -#define STRINGLIB_NEW PyBytes_FromStringAndSize -#define STRINGLIB_STR PyBytes_AS_STRING -/* #define STRINGLIB_WANT_CONTAINS_OBJ 1 */ - -#define STRINGLIB_EMPTY nullstring -#define STRINGLIB_CHECK_EXACT PyBytes_CheckExact -#define STRINGLIB_MUTABLE 0 - -#include "stringlib/fastsearch.h" - -#include "stringlib/count.h" -#include "stringlib/find.h" -#include "stringlib/partition.h" -#include "stringlib/ctype.h" -#include "stringlib/transmogrify.h" - -#define _Py_InsertThousandsGrouping _PyBytes_InsertThousandsGrouping -#include "stringlib/localeutil.h" - -PyObject * -PyBytes_Repr(PyObject *obj, int smartquotes) -{ - static const char *hexdigits = "0123456789abcdef"; - register PyBytesObject* op = (PyBytesObject*) obj; - Py_ssize_t length = Py_SIZE(op); - size_t newsize = 3 + 4 * length; - PyObject *v; - if (newsize > PY_SSIZE_T_MAX || (newsize-3) / 4 != length) { - PyErr_SetString(PyExc_OverflowError, - "bytes object is too large to make repr"); - return NULL; - } - v = PyUnicode_FromUnicode(NULL, newsize); - if (v == NULL) { - return NULL; - } - else { - register Py_ssize_t i; - register Py_UNICODE c; - register Py_UNICODE *p = PyUnicode_AS_UNICODE(v); - int quote; - - /* Figure out which quote to use; single is preferred */ - quote = '\''; - if (smartquotes) { - char *test, *start; - start = PyBytes_AS_STRING(op); - for (test = start; test < start+length; ++test) { - if (*test == '"') { - quote = '\''; /* back to single */ - goto decided; - } - else if (*test == '\'') - quote = '"'; - } - decided: - ; - } - - *p++ = 'b', *p++ = quote; - for (i = 0; i < length; i++) { - /* There's at least enough room for a hex escape - and a closing quote. */ - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 5); - c = op->ob_sval[i]; - if (c == quote || c == '\\') - *p++ = '\\', *p++ = c; - else if (c == '\t') - *p++ = '\\', *p++ = 't'; - else if (c == '\n') - *p++ = '\\', *p++ = 'n'; - else if (c == '\r') - *p++ = '\\', *p++ = 'r'; - else if (c < ' ' || c >= 0x7f) { - *p++ = '\\'; - *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; - } - else - *p++ = c; - } - assert(newsize - (p - PyUnicode_AS_UNICODE(v)) >= 1); - *p++ = quote; - *p = '\0'; - if (PyUnicode_Resize(&v, (p - PyUnicode_AS_UNICODE(v)))) { - Py_DECREF(v); - return NULL; - } - return v; - } -} - -static PyObject * -string_repr(PyObject *op) -{ - return PyBytes_Repr(op, 1); -} - -static PyObject * -string_str(PyObject *op) -{ - if (Py_BytesWarningFlag) { - if (PyErr_WarnEx(PyExc_BytesWarning, - "str() on a bytes instance", 1)) - return NULL; - } - return string_repr(op); -} - -static Py_ssize_t -string_length(PyBytesObject *a) -{ - return Py_SIZE(a); -} - -/* This is also used by PyBytes_Concat() */ -static PyObject * -string_concat(PyObject *a, PyObject *b) -{ - Py_ssize_t size; - Py_buffer va, vb; - PyObject *result = NULL; - - va.len = -1; - vb.len = -1; - if (_getbuffer(a, &va) < 0 || - _getbuffer(b, &vb) < 0) { - PyErr_Format(PyExc_TypeError, "can't concat %.100s to %.100s", - Py_TYPE(a)->tp_name, Py_TYPE(b)->tp_name); - goto done; - } - - /* Optimize end cases */ - if (va.len == 0 && PyBytes_CheckExact(b)) { - result = b; - Py_INCREF(result); - goto done; - } - if (vb.len == 0 && PyBytes_CheckExact(a)) { - result = a; - Py_INCREF(result); - goto done; - } - - size = va.len + vb.len; - if (size < 0) { - PyErr_NoMemory(); - goto done; - } - - result = PyBytes_FromStringAndSize(NULL, size); - if (result != NULL) { - memcpy(PyBytes_AS_STRING(result), va.buf, va.len); - memcpy(PyBytes_AS_STRING(result) + va.len, vb.buf, vb.len); - } - - done: - if (va.len != -1) - PyObject_ReleaseBuffer(a, &va); - if (vb.len != -1) - PyObject_ReleaseBuffer(b, &vb); - return result; -} - -static PyObject * -string_repeat(register PyBytesObject *a, register Py_ssize_t n) -{ - register Py_ssize_t i; - register Py_ssize_t j; - register Py_ssize_t size; - register PyBytesObject *op; - size_t nbytes; - if (n < 0) - n = 0; - /* watch out for overflows: the size can overflow int, - * and the # of bytes needed can overflow size_t - */ - size = Py_SIZE(a) * n; - if (n && size / n != Py_SIZE(a)) { - PyErr_SetString(PyExc_OverflowError, - "repeated string is too long"); - return NULL; - } - if (size == Py_SIZE(a) && PyBytes_CheckExact(a)) { - Py_INCREF(a); - return (PyObject *)a; - } - nbytes = (size_t)size; - if (nbytes + sizeof(PyBytesObject) <= nbytes) { - PyErr_SetString(PyExc_OverflowError, - "repeated string is too long"); - return NULL; - } - op = (PyBytesObject *) - PyObject_MALLOC(sizeof(PyBytesObject) + nbytes); - if (op == NULL) - return PyErr_NoMemory(); - PyObject_INIT_VAR(op, &PyBytes_Type, size); - op->ob_shash = -1; - op->ob_sval[size] = '\0'; - if (Py_SIZE(a) == 1 && n > 0) { - memset(op->ob_sval, a->ob_sval[0] , n); - return (PyObject *) op; - } - i = 0; - if (i < size) { - Py_MEMCPY(op->ob_sval, a->ob_sval, Py_SIZE(a)); - i = Py_SIZE(a); - } - while (i < size) { - j = (i <= size-i) ? i : size-i; - Py_MEMCPY(op->ob_sval+i, op->ob_sval, j); - i += j; - } - return (PyObject *) op; -} - -static int -string_contains(PyObject *self, PyObject *arg) -{ - Py_ssize_t ival = PyNumber_AsSsize_t(arg, PyExc_ValueError); - if (ival == -1 && PyErr_Occurred()) { - Py_buffer varg; - int pos; - PyErr_Clear(); - if (_getbuffer(arg, &varg) < 0) - return -1; - pos = stringlib_find(PyBytes_AS_STRING(self), Py_SIZE(self), - varg.buf, varg.len, 0); - PyObject_ReleaseBuffer(arg, &varg); - return pos >= 0; - } - if (ival < 0 || ival >= 256) { - PyErr_SetString(PyExc_ValueError, "byte must be in range(0, 256)"); - return -1; - } - - return memchr(PyBytes_AS_STRING(self), ival, Py_SIZE(self)) != NULL; -} - -static PyObject * -string_item(PyBytesObject *a, register Py_ssize_t i) -{ - if (i < 0 || i >= Py_SIZE(a)) { - PyErr_SetString(PyExc_IndexError, "string index out of range"); - return NULL; - } - return PyLong_FromLong((unsigned char)a->ob_sval[i]); -} - -static PyObject* -string_richcompare(PyBytesObject *a, PyBytesObject *b, int op) -{ - int c; - Py_ssize_t len_a, len_b; - Py_ssize_t min_len; - PyObject *result; - - /* Make sure both arguments are strings. */ - if (!(PyBytes_Check(a) && PyBytes_Check(b))) { - if (Py_BytesWarningFlag && (op == Py_EQ) && - (PyObject_IsInstance((PyObject*)a, - (PyObject*)&PyUnicode_Type) || - PyObject_IsInstance((PyObject*)b, - (PyObject*)&PyUnicode_Type))) { - if (PyErr_WarnEx(PyExc_BytesWarning, - "Comparsion between bytes and string", 1)) - return NULL; - } - result = Py_NotImplemented; - goto out; - } - if (a == b) { - switch (op) { - case Py_EQ:case Py_LE:case Py_GE: - result = Py_True; - goto out; - case Py_NE:case Py_LT:case Py_GT: - result = Py_False; - goto out; - } - } - if (op == Py_EQ) { - /* Supporting Py_NE here as well does not save - much time, since Py_NE is rarely used. */ - if (Py_SIZE(a) == Py_SIZE(b) - && (a->ob_sval[0] == b->ob_sval[0] - && memcmp(a->ob_sval, b->ob_sval, Py_SIZE(a)) == 0)) { - result = Py_True; - } else { - result = Py_False; - } - goto out; - } - len_a = Py_SIZE(a); len_b = Py_SIZE(b); - min_len = (len_a < len_b) ? len_a : len_b; - if (min_len > 0) { - c = Py_CHARMASK(*a->ob_sval) - Py_CHARMASK(*b->ob_sval); - if (c==0) - c = memcmp(a->ob_sval, b->ob_sval, min_len); - } else - c = 0; - if (c == 0) - c = (len_a < len_b) ? -1 : (len_a > len_b) ? 1 : 0; - switch (op) { - case Py_LT: c = c < 0; break; - case Py_LE: c = c <= 0; break; - case Py_EQ: assert(0); break; /* unreachable */ - case Py_NE: c = c != 0; break; - case Py_GT: c = c > 0; break; - case Py_GE: c = c >= 0; break; - default: - result = Py_NotImplemented; - goto out; - } - result = c ? Py_True : Py_False; - out: - Py_INCREF(result); - return result; -} - -static long -string_hash(PyBytesObject *a) -{ - register Py_ssize_t len; - register unsigned char *p; - register long x; - - if (a->ob_shash != -1) - return a->ob_shash; - len = Py_SIZE(a); - p = (unsigned char *) a->ob_sval; - x = *p << 7; - while (--len >= 0) - x = (1000003*x) ^ *p++; - x ^= Py_SIZE(a); - if (x == -1) - x = -2; - a->ob_shash = x; - return x; -} - -static PyObject* -string_subscript(PyBytesObject* self, PyObject* item) -{ - if (PyIndex_Check(item)) { - Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); - if (i == -1 && PyErr_Occurred()) - return NULL; - if (i < 0) - i += PyBytes_GET_SIZE(self); - if (i < 0 || i >= PyBytes_GET_SIZE(self)) { - PyErr_SetString(PyExc_IndexError, - "string index out of range"); - return NULL; - } - return PyLong_FromLong((unsigned char)self->ob_sval[i]); - } - else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, cur, i; - char* source_buf; - char* result_buf; - PyObject* result; - - if (PySlice_GetIndicesEx((PySliceObject*)item, - PyBytes_GET_SIZE(self), - &start, &stop, &step, &slicelength) < 0) { - return NULL; - } - - if (slicelength <= 0) { - return PyBytes_FromStringAndSize("", 0); - } - else if (start == 0 && step == 1 && - slicelength == PyBytes_GET_SIZE(self) && - PyBytes_CheckExact(self)) { - Py_INCREF(self); - return (PyObject *)self; - } - else if (step == 1) { - return PyBytes_FromStringAndSize( - PyBytes_AS_STRING(self) + start, - slicelength); - } - else { - source_buf = PyBytes_AsString((PyObject*)self); - result_buf = (char *)PyMem_Malloc(slicelength); - if (result_buf == NULL) - return PyErr_NoMemory(); - - for (cur = start, i = 0; i < slicelength; - cur += step, i++) { - result_buf[i] = source_buf[cur]; - } - - result = PyBytes_FromStringAndSize(result_buf, - slicelength); - PyMem_Free(result_buf); - return result; - } - } - else { - PyErr_Format(PyExc_TypeError, - "string indices must be integers, not %.200s", - Py_TYPE(item)->tp_name); - return NULL; - } -} - -static int -string_buffer_getbuffer(PyBytesObject *self, Py_buffer *view, int flags) -{ - return PyBuffer_FillInfo(view, (void *)self->ob_sval, Py_SIZE(self), - 0, flags); -} - -static PySequenceMethods string_as_sequence = { - (lenfunc)string_length, /*sq_length*/ - (binaryfunc)string_concat, /*sq_concat*/ - (ssizeargfunc)string_repeat, /*sq_repeat*/ - (ssizeargfunc)string_item, /*sq_item*/ - 0, /*sq_slice*/ - 0, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ - (objobjproc)string_contains /*sq_contains*/ -}; - -static PyMappingMethods string_as_mapping = { - (lenfunc)string_length, - (binaryfunc)string_subscript, - 0, -}; - -static PyBufferProcs string_as_buffer = { - (getbufferproc)string_buffer_getbuffer, - NULL, -}; - - -#define LEFTSTRIP 0 -#define RIGHTSTRIP 1 -#define BOTHSTRIP 2 - -/* Arrays indexed by above */ -static const char *stripformat[] = {"|O:lstrip", "|O:rstrip", "|O:strip"}; - -#define STRIPNAME(i) (stripformat[i]+3) - - -/* Don't call if length < 2 */ -#define Py_STRING_MATCH(target, offset, pattern, length) \ - (target[offset] == pattern[0] && \ - target[offset+length-1] == pattern[length-1] && \ - !memcmp(target+offset+1, pattern+1, length-2) ) - - -/* Overallocate the initial list to reduce the number of reallocs for small - split sizes. Eg, "A A A A A A A A A A".split() (10 elements) has three - resizes, to sizes 4, 8, then 16. Most observed string splits are for human - text (roughly 11 words per line) and field delimited data (usually 1-10 - fields). For large strings the split algorithms are bandwidth limited - so increasing the preallocation likely will not improve things.*/ - -#define MAX_PREALLOC 12 - -/* 5 splits gives 6 elements */ -#define PREALLOC_SIZE(maxsplit) \ - (maxsplit >= MAX_PREALLOC ? MAX_PREALLOC : maxsplit+1) - -#define SPLIT_ADD(data, left, right) { \ - str = PyBytes_FromStringAndSize((data) + (left), \ - (right) - (left)); \ - if (str == NULL) \ - goto onError; \ - if (count < MAX_PREALLOC) { \ - PyList_SET_ITEM(list, count, str); \ - } else { \ - if (PyList_Append(list, str)) { \ - Py_DECREF(str); \ - goto onError; \ - } \ - else \ - Py_DECREF(str); \ - } \ - count++; } - -/* Always force the list to the expected size. */ -#define FIX_PREALLOC_SIZE(list) Py_SIZE(list) = count - -#define SKIP_SPACE(s, i, len) { while (i<len && ISSPACE(s[i])) i++; } -#define SKIP_NONSPACE(s, i, len) { while (i<len && !ISSPACE(s[i])) i++; } -#define RSKIP_SPACE(s, i) { while (i>=0 && ISSPACE(s[i])) i--; } -#define RSKIP_NONSPACE(s, i) { while (i>=0 && !ISSPACE(s[i])) i--; } - -Py_LOCAL_INLINE(PyObject *) -split_whitespace(PyBytesObject *self, Py_ssize_t len, Py_ssize_t maxsplit) -{ - const char *s = PyBytes_AS_STRING(self); - Py_ssize_t i, j, count=0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxsplit)); - - if (list == NULL) - return NULL; - - i = j = 0; - - while (maxsplit-- > 0) { - SKIP_SPACE(s, i, len); - if (i==len) break; - j = i; i++; - SKIP_NONSPACE(s, i, len); - if (j == 0 && i == len && PyBytes_CheckExact(self)) { - /* No whitespace in self, so just use it as list[0] */ - Py_INCREF(self); - PyList_SET_ITEM(list, 0, (PyObject *)self); - count++; - break; - } - SPLIT_ADD(s, j, i); - } - - if (i < len) { - /* Only occurs when maxsplit was reached */ - /* Skip any remaining whitespace and copy to end of string */ - SKIP_SPACE(s, i, len); - if (i != len) - SPLIT_ADD(s, i, len); - } - FIX_PREALLOC_SIZE(list); - return list; - onError: - Py_DECREF(list); - return NULL; -} - -Py_LOCAL_INLINE(PyObject *) -split_char(PyBytesObject *self, Py_ssize_t len, char ch, Py_ssize_t maxcount) -{ - const char *s = PyBytes_AS_STRING(self); - register Py_ssize_t i, j, count=0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - i = j = 0; - while ((j < len) && (maxcount-- > 0)) { - for(; j<len; j++) { - /* I found that using memchr makes no difference */ - if (s[j] == ch) { - SPLIT_ADD(s, i, j); - i = j = j + 1; - break; - } - } - } - if (i == 0 && count == 0 && PyBytes_CheckExact(self)) { - /* ch not in self, so just use self as list[0] */ - Py_INCREF(self); - PyList_SET_ITEM(list, 0, (PyObject *)self); - count++; - } - else if (i <= len) { - SPLIT_ADD(s, i, len); - } - FIX_PREALLOC_SIZE(list); - return list; - - onError: - Py_DECREF(list); - return NULL; -} - -PyDoc_STRVAR(split__doc__, -"B.split([sep[, maxsplit]]) -> list of bytes\n\ -\n\ -Return a list of the sections in B, using sep as the delimiter.\n\ -If sep is not specified or is None, B is split on ASCII whitespace\n\ -characters (space, tab, return, newline, formfeed, vertical tab).\n\ -If maxsplit is given, at most maxsplit splits are done."); - -static PyObject * -string_split(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; - Py_ssize_t maxsplit = -1, count=0; - const char *s = PyBytes_AS_STRING(self), *sub; - Py_buffer vsub; - PyObject *list, *str, *subobj = Py_None; -#ifdef USE_FAST - Py_ssize_t pos; -#endif - - if (!PyArg_ParseTuple(args, "|On:split", &subobj, &maxsplit)) - return NULL; - if (maxsplit < 0) - maxsplit = PY_SSIZE_T_MAX; - if (subobj == Py_None) - return split_whitespace(self, len, maxsplit); - if (_getbuffer(subobj, &vsub) < 0) - return NULL; - sub = vsub.buf; - n = vsub.len; - - if (n == 0) { - PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - else if (n == 1) - return split_char(self, len, sub[0], maxsplit); - - list = PyList_New(PREALLOC_SIZE(maxsplit)); - if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - -#ifdef USE_FAST - i = j = 0; - while (maxsplit-- > 0) { - pos = fastsearch(s+i, len-i, sub, n, FAST_SEARCH); - if (pos < 0) - break; - j = i+pos; - SPLIT_ADD(s, i, j); - i = j + n; - } -#else - i = j = 0; - while ((j+n <= len) && (maxsplit-- > 0)) { - for (; j+n <= len; j++) { - if (Py_STRING_MATCH(s, j, sub, n)) { - SPLIT_ADD(s, i, j); - i = j = j + n; - break; - } - } - } -#endif - SPLIT_ADD(s, i, len); - FIX_PREALLOC_SIZE(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return list; - - onError: - Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; -} - -PyDoc_STRVAR(partition__doc__, -"B.partition(sep) -> (head, sep, tail)\n\ -\n\ -Searches for the separator sep in B, and returns the part before it,\n\ -the separator itself, and the part after it. If the separator is not\n\ -found, returns B and two empty bytes objects."); - -static PyObject * -string_partition(PyBytesObject *self, PyObject *sep_obj) -{ - const char *sep; - Py_ssize_t sep_len; - - if (PyBytes_Check(sep_obj)) { - sep = PyBytes_AS_STRING(sep_obj); - sep_len = PyBytes_GET_SIZE(sep_obj); - } - else if (PyObject_AsCharBuffer(sep_obj, &sep, &sep_len)) - return NULL; - - return stringlib_partition( - (PyObject*) self, - PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), - sep_obj, sep, sep_len - ); -} - -PyDoc_STRVAR(rpartition__doc__, -"B.rpartition(sep) -> (tail, sep, head)\n\ -\n\ -Searches for the separator sep in B, starting at the end of B,\n\ -and returns the part before it, the separator itself, and the\n\ -part after it. If the separator is not found, returns two empty\n\ -bytes objects and B."); - -static PyObject * -string_rpartition(PyBytesObject *self, PyObject *sep_obj) -{ - const char *sep; - Py_ssize_t sep_len; - - if (PyBytes_Check(sep_obj)) { - sep = PyBytes_AS_STRING(sep_obj); - sep_len = PyBytes_GET_SIZE(sep_obj); - } - else if (PyObject_AsCharBuffer(sep_obj, &sep, &sep_len)) - return NULL; - - return stringlib_rpartition( - (PyObject*) self, - PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), - sep_obj, sep, sep_len - ); -} - -Py_LOCAL_INLINE(PyObject *) -rsplit_whitespace(PyBytesObject *self, Py_ssize_t len, Py_ssize_t maxsplit) -{ - const char *s = PyBytes_AS_STRING(self); - Py_ssize_t i, j, count=0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxsplit)); - - if (list == NULL) - return NULL; - - i = j = len-1; - - while (maxsplit-- > 0) { - RSKIP_SPACE(s, i); - if (i<0) break; - j = i; i--; - RSKIP_NONSPACE(s, i); - if (j == len-1 && i < 0 && PyBytes_CheckExact(self)) { - /* No whitespace in self, so just use it as list[0] */ - Py_INCREF(self); - PyList_SET_ITEM(list, 0, (PyObject *)self); - count++; - break; - } - SPLIT_ADD(s, i + 1, j + 1); - } - if (i >= 0) { - /* Only occurs when maxsplit was reached. Skip any remaining - whitespace and copy to beginning of string. */ - RSKIP_SPACE(s, i); - if (i >= 0) - SPLIT_ADD(s, 0, i + 1); - - } - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - return list; - onError: - Py_DECREF(list); - return NULL; -} - -Py_LOCAL_INLINE(PyObject *) -rsplit_char(PyBytesObject *self, Py_ssize_t len, char ch, Py_ssize_t maxcount) -{ - const char *s = PyBytes_AS_STRING(self); - register Py_ssize_t i, j, count=0; - PyObject *str; - PyObject *list = PyList_New(PREALLOC_SIZE(maxcount)); - - if (list == NULL) - return NULL; - - i = j = len - 1; - while ((i >= 0) && (maxcount-- > 0)) { - for (; i >= 0; i--) { - if (s[i] == ch) { - SPLIT_ADD(s, i + 1, j + 1); - j = i = i - 1; - break; - } - } - } - if (i < 0 && count == 0 && PyBytes_CheckExact(self)) { - /* ch not in self, so just use self as list[0] */ - Py_INCREF(self); - PyList_SET_ITEM(list, 0, (PyObject *)self); - count++; - } - else if (j >= -1) { - SPLIT_ADD(s, 0, j + 1); - } - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - return list; - - onError: - Py_DECREF(list); - return NULL; -} - -PyDoc_STRVAR(rsplit__doc__, -"B.rsplit([sep[, maxsplit]]) -> list of strings\n\ -\n\ -Return a list of the sections in B, using sep as the delimiter,\n\ -starting at the end of B and working to the front.\n\ -If sep is not given, B is split on ASCII whitespace characters\n\ -(space, tab, return, newline, formfeed, vertical tab).\n\ -If maxsplit is given, at most maxsplit splits are done."); - - -static PyObject * -string_rsplit(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t len = PyBytes_GET_SIZE(self), n, i, j; - Py_ssize_t maxsplit = -1, count=0; - const char *s, *sub; - Py_buffer vsub; - PyObject *list, *str, *subobj = Py_None; - - if (!PyArg_ParseTuple(args, "|On:rsplit", &subobj, &maxsplit)) - return NULL; - if (maxsplit < 0) - maxsplit = PY_SSIZE_T_MAX; - if (subobj == Py_None) - return rsplit_whitespace(self, len, maxsplit); - if (_getbuffer(subobj, &vsub) < 0) - return NULL; - sub = vsub.buf; - n = vsub.len; - - if (n == 0) { - PyErr_SetString(PyExc_ValueError, "empty separator"); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - else if (n == 1) - return rsplit_char(self, len, sub[0], maxsplit); - - list = PyList_New(PREALLOC_SIZE(maxsplit)); - if (list == NULL) { - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; - } - - j = len; - i = j - n; - - s = PyBytes_AS_STRING(self); - while ( (i >= 0) && (maxsplit-- > 0) ) { - for (; i>=0; i--) { - if (Py_STRING_MATCH(s, i, sub, n)) { - SPLIT_ADD(s, i + n, j); - j = i; - i -= n; - break; - } - } - } - SPLIT_ADD(s, 0, j); - FIX_PREALLOC_SIZE(list); - if (PyList_Reverse(list) < 0) - goto onError; - PyObject_ReleaseBuffer(subobj, &vsub); - return list; - -onError: - Py_DECREF(list); - PyObject_ReleaseBuffer(subobj, &vsub); - return NULL; -} - -#undef SPLIT_ADD -#undef MAX_PREALLOC -#undef PREALLOC_SIZE - - -PyDoc_STRVAR(join__doc__, -"B.join(iterable_of_bytes) -> bytes\n\ -\n\ -Concatenates any number of bytes objects, with B in between each pair.\n\ -Example: b'.'.join([b'ab', b'pq', b'rs']) -> b'ab.pq.rs'."); - -static PyObject * -string_join(PyObject *self, PyObject *orig) -{ - char *sep = PyBytes_AS_STRING(self); - const Py_ssize_t seplen = PyBytes_GET_SIZE(self); - PyObject *res = NULL; - char *p; - Py_ssize_t seqlen = 0; - size_t sz = 0; - Py_ssize_t i; - PyObject *seq, *item; - - seq = PySequence_Fast(orig, ""); - if (seq == NULL) { - return NULL; - } - - seqlen = PySequence_Size(seq); - if (seqlen == 0) { - Py_DECREF(seq); - return PyBytes_FromString(""); - } - if (seqlen == 1) { - item = PySequence_Fast_GET_ITEM(seq, 0); - if (PyBytes_CheckExact(item)) { - Py_INCREF(item); - Py_DECREF(seq); - return item; - } - } - - /* There are at least two things to join, or else we have a subclass - * of the builtin types in the sequence. - * Do a pre-pass to figure out the total amount of space we'll - * need (sz), and see whether all argument are bytes. - */ - /* XXX Shouldn't we use _getbuffer() on these items instead? */ - for (i = 0; i < seqlen; i++) { - const size_t old_sz = sz; - item = PySequence_Fast_GET_ITEM(seq, i); - if (!PyBytes_Check(item) && !PyByteArray_Check(item)) { - PyErr_Format(PyExc_TypeError, - "sequence item %zd: expected bytes," - " %.80s found", - i, Py_TYPE(item)->tp_name); - Py_DECREF(seq); - return NULL; - } - sz += Py_SIZE(item); - if (i != 0) - sz += seplen; - if (sz < old_sz || sz > PY_SSIZE_T_MAX) { - PyErr_SetString(PyExc_OverflowError, - "join() result is too long for a Python string"); - Py_DECREF(seq); - return NULL; - } - } - - /* Allocate result space. */ - res = PyBytes_FromStringAndSize((char*)NULL, sz); - if (res == NULL) { - Py_DECREF(seq); - return NULL; - } - - /* Catenate everything. */ - /* I'm not worried about a PyByteArray item growing because there's - nowhere in this function where we release the GIL. */ - p = PyBytes_AS_STRING(res); - for (i = 0; i < seqlen; ++i) { - size_t n; - char *q; - if (i) { - Py_MEMCPY(p, sep, seplen); - p += seplen; - } - item = PySequence_Fast_GET_ITEM(seq, i); - n = Py_SIZE(item); - if (PyBytes_Check(item)) - q = PyBytes_AS_STRING(item); - else - q = PyByteArray_AS_STRING(item); - Py_MEMCPY(p, q, n); - p += n; - } - - Py_DECREF(seq); - return res; -} - -PyObject * -_PyBytes_Join(PyObject *sep, PyObject *x) -{ - assert(sep != NULL && PyBytes_Check(sep)); - assert(x != NULL); - return string_join(sep, x); -} - -Py_LOCAL_INLINE(void) -string_adjust_indices(Py_ssize_t *start, Py_ssize_t *end, Py_ssize_t len) -{ - if (*end > len) - *end = len; - else if (*end < 0) - *end += len; - if (*end < 0) - *end = 0; - if (*start < 0) - *start += len; - if (*start < 0) - *start = 0; -} - -Py_LOCAL_INLINE(Py_ssize_t) -string_find_internal(PyBytesObject *self, PyObject *args, int dir) -{ - PyObject *subobj; - const char *sub; - Py_ssize_t sub_len; - Py_ssize_t start=0, end=PY_SSIZE_T_MAX; - PyObject *obj_start=Py_None, *obj_end=Py_None; - - if (!PyArg_ParseTuple(args, "O|OO:find/rfind/index/rindex", &subobj, - &obj_start, &obj_end)) - return -2; - /* To support None in "start" and "end" arguments, meaning - the same as if they were not passed. - */ - if (obj_start != Py_None) - if (!_PyEval_SliceIndex(obj_start, &start)) - return -2; - if (obj_end != Py_None) - if (!_PyEval_SliceIndex(obj_end, &end)) - return -2; - - if (PyBytes_Check(subobj)) { - sub = PyBytes_AS_STRING(subobj); - sub_len = PyBytes_GET_SIZE(subobj); - } - else if (PyObject_AsCharBuffer(subobj, &sub, &sub_len)) - /* XXX - the "expected a character buffer object" is pretty - confusing for a non-expert. remap to something else ? */ - return -2; - - if (dir > 0) - return stringlib_find_slice( - PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), - sub, sub_len, start, end); - else - return stringlib_rfind_slice( - PyBytes_AS_STRING(self), PyBytes_GET_SIZE(self), - sub, sub_len, start, end); -} - - -PyDoc_STRVAR(find__doc__, -"B.find(sub [,start [,end]]) -> int\n\ -\n\ -Return the lowest index in S where substring sub is found,\n\ -such that sub is contained within s[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -string_find(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t result = string_find_internal(self, args, +1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); -} - - -PyDoc_STRVAR(index__doc__, -"B.index(sub [,start [,end]]) -> int\n\ -\n\ -Like B.find() but raise ValueError when the substring is not found."); - -static PyObject * -string_index(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t result = string_find_internal(self, args, +1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "substring not found"); - return NULL; - } - return PyLong_FromSsize_t(result); -} - - -PyDoc_STRVAR(rfind__doc__, -"B.rfind(sub [,start [,end]]) -> int\n\ -\n\ -Return the highest index in B where substring sub is found,\n\ -such that sub is contained within s[start:end]. Optional\n\ -arguments start and end are interpreted as in slice notation.\n\ -\n\ -Return -1 on failure."); - -static PyObject * -string_rfind(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t result = string_find_internal(self, args, -1); - if (result == -2) - return NULL; - return PyLong_FromSsize_t(result); -} - - -PyDoc_STRVAR(rindex__doc__, -"B.rindex(sub [,start [,end]]) -> int\n\ -\n\ -Like B.rfind() but raise ValueError when the substring is not found."); - -static PyObject * -string_rindex(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t result = string_find_internal(self, args, -1); - if (result == -2) - return NULL; - if (result == -1) { - PyErr_SetString(PyExc_ValueError, - "substring not found"); - return NULL; - } - return PyLong_FromSsize_t(result); -} - - -Py_LOCAL_INLINE(PyObject *) -do_xstrip(PyBytesObject *self, int striptype, PyObject *sepobj) -{ - Py_buffer vsep; - char *s = PyBytes_AS_STRING(self); - Py_ssize_t len = PyBytes_GET_SIZE(self); - char *sep; - Py_ssize_t seplen; - Py_ssize_t i, j; - - if (_getbuffer(sepobj, &vsep) < 0) - return NULL; - sep = vsep.buf; - seplen = vsep.len; - - i = 0; - if (striptype != RIGHTSTRIP) { - while (i < len && memchr(sep, Py_CHARMASK(s[i]), seplen)) { - i++; - } - } - - j = len; - if (striptype != LEFTSTRIP) { - do { - j--; - } while (j >= i && memchr(sep, Py_CHARMASK(s[j]), seplen)); - j++; - } - - PyObject_ReleaseBuffer(sepobj, &vsep); - - if (i == 0 && j == len && PyBytes_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*)self; - } - else - return PyBytes_FromStringAndSize(s+i, j-i); -} - - -Py_LOCAL_INLINE(PyObject *) -do_strip(PyBytesObject *self, int striptype) -{ - char *s = PyBytes_AS_STRING(self); - Py_ssize_t len = PyBytes_GET_SIZE(self), i, j; - - i = 0; - if (striptype != RIGHTSTRIP) { - while (i < len && ISSPACE(s[i])) { - i++; - } - } - - j = len; - if (striptype != LEFTSTRIP) { - do { - j--; - } while (j >= i && ISSPACE(s[j])); - j++; - } - - if (i == 0 && j == len && PyBytes_CheckExact(self)) { - Py_INCREF(self); - return (PyObject*)self; - } - else - return PyBytes_FromStringAndSize(s+i, j-i); -} - - -Py_LOCAL_INLINE(PyObject *) -do_argstrip(PyBytesObject *self, int striptype, PyObject *args) -{ - PyObject *sep = NULL; - - if (!PyArg_ParseTuple(args, (char *)stripformat[striptype], &sep)) - return NULL; - - if (sep != NULL && sep != Py_None) { - return do_xstrip(self, striptype, sep); - } - return do_strip(self, striptype); -} - - -PyDoc_STRVAR(strip__doc__, -"B.strip([bytes]) -> bytes\n\ -\n\ -Strip leading and trailing bytes contained in the argument.\n\ -If the argument is omitted, strip trailing ASCII whitespace."); -static PyObject * -string_strip(PyBytesObject *self, PyObject *args) -{ - if (PyTuple_GET_SIZE(args) == 0) - return do_strip(self, BOTHSTRIP); /* Common case */ - else - return do_argstrip(self, BOTHSTRIP, args); -} - - -PyDoc_STRVAR(lstrip__doc__, -"B.lstrip([bytes]) -> bytes\n\ -\n\ -Strip leading bytes contained in the argument.\n\ -If the argument is omitted, strip leading ASCII whitespace."); -static PyObject * -string_lstrip(PyBytesObject *self, PyObject *args) -{ - if (PyTuple_GET_SIZE(args) == 0) - return do_strip(self, LEFTSTRIP); /* Common case */ - else - return do_argstrip(self, LEFTSTRIP, args); -} - - -PyDoc_STRVAR(rstrip__doc__, -"B.rstrip([bytes]) -> bytes\n\ -\n\ -Strip trailing bytes contained in the argument.\n\ -If the argument is omitted, strip trailing ASCII whitespace."); -static PyObject * -string_rstrip(PyBytesObject *self, PyObject *args) -{ - if (PyTuple_GET_SIZE(args) == 0) - return do_strip(self, RIGHTSTRIP); /* Common case */ - else - return do_argstrip(self, RIGHTSTRIP, args); -} - - -PyDoc_STRVAR(count__doc__, -"B.count(sub [,start [,end]]) -> int\n\ -\n\ -Return the number of non-overlapping occurrences of substring sub in\n\ -string S[start:end]. Optional arguments start and end are interpreted\n\ -as in slice notation."); - -static PyObject * -string_count(PyBytesObject *self, PyObject *args) -{ - PyObject *sub_obj; - const char *str = PyBytes_AS_STRING(self), *sub; - Py_ssize_t sub_len; - Py_ssize_t start = 0, end = PY_SSIZE_T_MAX; - - if (!PyArg_ParseTuple(args, "O|O&O&:count", &sub_obj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - - if (PyBytes_Check(sub_obj)) { - sub = PyBytes_AS_STRING(sub_obj); - sub_len = PyBytes_GET_SIZE(sub_obj); - } - else if (PyObject_AsCharBuffer(sub_obj, &sub, &sub_len)) - return NULL; - - string_adjust_indices(&start, &end, PyBytes_GET_SIZE(self)); - - return PyLong_FromSsize_t( - stringlib_count(str + start, end - start, sub, sub_len) - ); -} - - -PyDoc_STRVAR(translate__doc__, -"B.translate(table[, deletechars]) -> bytes\n\ -\n\ -Return a copy of B, where all characters occurring in the\n\ -optional argument deletechars are removed, and the remaining\n\ -characters have been mapped through the given translation\n\ -table, which must be a bytes object of length 256."); - -static PyObject * -string_translate(PyBytesObject *self, PyObject *args) -{ - register char *input, *output; - const char *table; - register Py_ssize_t i, c, changed = 0; - PyObject *input_obj = (PyObject*)self; - const char *output_start, *del_table=NULL; - Py_ssize_t inlen, tablen, dellen = 0; - PyObject *result; - int trans_table[256]; - PyObject *tableobj, *delobj = NULL; - - if (!PyArg_UnpackTuple(args, "translate", 1, 2, - &tableobj, &delobj)) - return NULL; - - if (PyBytes_Check(tableobj)) { - table = PyBytes_AS_STRING(tableobj); - tablen = PyBytes_GET_SIZE(tableobj); - } - else if (tableobj == Py_None) { - table = NULL; - tablen = 256; - } - else if (PyObject_AsCharBuffer(tableobj, &table, &tablen)) - return NULL; - - if (tablen != 256) { - PyErr_SetString(PyExc_ValueError, - "translation table must be 256 characters long"); - return NULL; - } - - if (delobj != NULL) { - if (PyBytes_Check(delobj)) { - del_table = PyBytes_AS_STRING(delobj); - dellen = PyBytes_GET_SIZE(delobj); - } - else if (PyUnicode_Check(delobj)) { - PyErr_SetString(PyExc_TypeError, - "deletions are implemented differently for unicode"); - return NULL; - } - else if (PyObject_AsCharBuffer(delobj, &del_table, &dellen)) - return NULL; - } - else { - del_table = NULL; - dellen = 0; - } - - inlen = PyBytes_GET_SIZE(input_obj); - result = PyBytes_FromStringAndSize((char *)NULL, inlen); - if (result == NULL) - return NULL; - output_start = output = PyBytes_AsString(result); - input = PyBytes_AS_STRING(input_obj); - - if (dellen == 0 && table != NULL) { - /* If no deletions are required, use faster code */ - for (i = inlen; --i >= 0; ) { - c = Py_CHARMASK(*input++); - if (Py_CHARMASK((*output++ = table[c])) != c) - changed = 1; - } - if (changed || !PyBytes_CheckExact(input_obj)) - return result; - Py_DECREF(result); - Py_INCREF(input_obj); - return input_obj; - } - - if (table == NULL) { - for (i = 0; i < 256; i++) - trans_table[i] = Py_CHARMASK(i); - } else { - for (i = 0; i < 256; i++) - trans_table[i] = Py_CHARMASK(table[i]); - } - - for (i = 0; i < dellen; i++) - trans_table[(int) Py_CHARMASK(del_table[i])] = -1; - - for (i = inlen; --i >= 0; ) { - c = Py_CHARMASK(*input++); - if (trans_table[c] != -1) - if (Py_CHARMASK(*output++ = (char)trans_table[c]) == c) - continue; - changed = 1; - } - if (!changed && PyBytes_CheckExact(input_obj)) { - Py_DECREF(result); - Py_INCREF(input_obj); - return input_obj; - } - /* Fix the size of the resulting string */ - if (inlen > 0) - _PyBytes_Resize(&result, output - output_start); - return result; -} - - -#define FORWARD 1 -#define REVERSE -1 - -/* find and count characters and substrings */ - -#define findchar(target, target_len, c) \ - ((char *)memchr((const void *)(target), c, target_len)) - -/* String ops must return a string. */ -/* If the object is subclass of string, create a copy */ -Py_LOCAL(PyBytesObject *) -return_self(PyBytesObject *self) -{ - if (PyBytes_CheckExact(self)) { - Py_INCREF(self); - return self; - } - return (PyBytesObject *)PyBytes_FromStringAndSize( - PyBytes_AS_STRING(self), - PyBytes_GET_SIZE(self)); -} - -Py_LOCAL_INLINE(Py_ssize_t) -countchar(const char *target, int target_len, char c, Py_ssize_t maxcount) -{ - Py_ssize_t count=0; - const char *start=target; - const char *end=target+target_len; - - while ( (start=findchar(start, end-start, c)) != NULL ) { - count++; - if (count >= maxcount) - break; - start += 1; - } - return count; -} - -Py_LOCAL(Py_ssize_t) -findstring(const char *target, Py_ssize_t target_len, - const char *pattern, Py_ssize_t pattern_len, - Py_ssize_t start, - Py_ssize_t end, - int direction) -{ - if (start < 0) { - start += target_len; - if (start < 0) - start = 0; - } - if (end > target_len) { - end = target_len; - } else if (end < 0) { - end += target_len; - if (end < 0) - end = 0; - } - - /* zero-length substrings always match at the first attempt */ - if (pattern_len == 0) - return (direction > 0) ? start : end; - - end -= pattern_len; - - if (direction < 0) { - for (; end >= start; end--) - if (Py_STRING_MATCH(target, end, pattern, pattern_len)) - return end; - } else { - for (; start <= end; start++) - if (Py_STRING_MATCH(target, start,pattern,pattern_len)) - return start; - } - return -1; -} - -Py_LOCAL_INLINE(Py_ssize_t) -countstring(const char *target, Py_ssize_t target_len, - const char *pattern, Py_ssize_t pattern_len, - Py_ssize_t start, - Py_ssize_t end, - int direction, Py_ssize_t maxcount) -{ - Py_ssize_t count=0; - - if (start < 0) { - start += target_len; - if (start < 0) - start = 0; - } - if (end > target_len) { - end = target_len; - } else if (end < 0) { - end += target_len; - if (end < 0) - end = 0; - } - - /* zero-length substrings match everywhere */ - if (pattern_len == 0 || maxcount == 0) { - if (target_len+1 < maxcount) - return target_len+1; - return maxcount; - } - - end -= pattern_len; - if (direction < 0) { - for (; (end >= start); end--) - if (Py_STRING_MATCH(target, end,pattern,pattern_len)) { - count++; - if (--maxcount <= 0) break; - end -= pattern_len-1; - } - } else { - for (; (start <= end); start++) - if (Py_STRING_MATCH(target, start, - pattern, pattern_len)) { - count++; - if (--maxcount <= 0) - break; - start += pattern_len-1; - } - } - return count; -} - - -/* Algorithms for different cases of string replacement */ - -/* len(self)>=1, from="", len(to)>=1, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_interleave(PyBytesObject *self, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - Py_ssize_t self_len, result_len; - Py_ssize_t count, i, product; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - - /* 1 at the end plus 1 after every character */ - count = self_len+1; - if (maxcount < count) - count = maxcount; - - /* Check for overflow */ - /* result_len = count * to_len + self_len; */ - product = count * to_len; - if (product / to_len != count) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - result_len = product + self_len; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - - if (! (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) ) - return NULL; - - self_s = PyBytes_AS_STRING(self); - result_s = PyBytes_AS_STRING(result); - - /* TODO: special case single character, which doesn't need memcpy */ - - /* Lay the first one down (guaranteed this will occur) */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - count -= 1; - - for (i=0; i<count; i++) { - *result_s++ = *self_s++; - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - } - - /* Copy the rest of the original string */ - Py_MEMCPY(result_s, self_s, self_len-i); - - return result; -} - -/* Special case for deleting a single character */ -/* len(self)>=1, len(from)==1, to="", maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_delete_single_character(PyBytesObject *self, - char from_c, Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - self_s = PyBytes_AS_STRING(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - return return_self(self); - } - - result_len = self_len - count; /* from_len == 1 */ - assert(result_len>=0); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - start = next+1; - } - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, to="", maxcount>=1 */ - -Py_LOCAL(PyBytesObject *) -replace_delete_substring(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - Py_ssize_t maxcount) { - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset; - PyBytesObject *result; - - self_len = PyBytes_GET_SIZE(self); - self_s = PyBytes_AS_STRING(self); - - count = countstring(self_s, self_len, - from_s, from_len, - 0, self_len, 1, - maxcount); - - if (count == 0) { - /* no matches */ - return return_self(self); - } - - result_len = self_len - (count * from_len); - assert (result_len>=0); - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL ) - return NULL; - - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset == -1) - break; - next = start + offset; - - Py_MEMCPY(result_s, start, next-start); - - result_s += (next-start); - start = next+from_len; - } - Py_MEMCPY(result_s, start, end-start); - return result; -} - -/* len(self)>=1, len(from)==len(to)==1, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_single_character_in_place(PyBytesObject *self, - char from_c, char to_c, - Py_ssize_t maxcount) -{ - char *self_s, *result_s, *start, *end, *next; - Py_ssize_t self_len; - PyBytesObject *result; - - /* The result string will be the same size */ - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - next = findchar(self_s, self_len, from_c); - - if (next == NULL) { - /* No matches; return the original string */ - return return_self(self); - } - - /* Need to make a new string */ - result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + (next-self_s); - *start = to_c; - start++; - end = result_s + self_len; - - while (--maxcount > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - *next = to_c; - start = next+1; - } - - return result; -} - -/* len(self)>=1, len(from)==len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_substring_in_place(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *result_s, *start, *end; - char *self_s; - Py_ssize_t self_len, offset; - PyBytesObject *result; - - /* The result string will be the same size */ - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - offset = findstring(self_s, self_len, - from_s, from_len, - 0, self_len, FORWARD); - if (offset == -1) { - /* No matches; return the original string */ - return return_self(self); - } - - /* Need to make a new string */ - result = (PyBytesObject *) PyBytes_FromStringAndSize(NULL, self_len); - if (result == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - Py_MEMCPY(result_s, self_s, self_len); - - /* change everything in-place, starting with this one */ - start = result_s + offset; - Py_MEMCPY(start, to_s, from_len); - start += from_len; - end = result_s + self_len; - - while ( --maxcount > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset==-1) - break; - Py_MEMCPY(start+offset, to_s, from_len); - start += offset+from_len; - } - - return result; -} - -/* len(self)>=1, len(from)==1, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_single_character(PyBytesObject *self, - char from_c, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, product; - PyBytesObject *result; - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - count = countchar(self_s, self_len, from_c, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* use the difference between current and new, hence the "-1" */ - /* result_len = self_len + count * (to_len-1) */ - product = count * (to_len-1); - if (product / (to_len-1) != count) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - result_len = self_len + product; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - next = findchar(start, end-start, from_c); - if (next == NULL) - break; - - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += 1; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+1; - } - } - /* Copy the remainder of the remaining string */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - -/* len(self)>=1, len(from)>=2, len(to)>=2, maxcount>=1 */ -Py_LOCAL(PyBytesObject *) -replace_substring(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) { - char *self_s, *result_s; - char *start, *next, *end; - Py_ssize_t self_len, result_len; - Py_ssize_t count, offset, product; - PyBytesObject *result; - - self_s = PyBytes_AS_STRING(self); - self_len = PyBytes_GET_SIZE(self); - - count = countstring(self_s, self_len, - from_s, from_len, - 0, self_len, FORWARD, maxcount); - if (count == 0) { - /* no matches, return unchanged */ - return return_self(self); - } - - /* Check for overflow */ - /* result_len = self_len + count * (to_len-from_len) */ - product = count * (to_len-from_len); - if (product / (to_len-from_len) != count) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - result_len = self_len + product; - if (result_len < 0) { - PyErr_SetString(PyExc_OverflowError, - "replace string is too long"); - return NULL; - } - - if ( (result = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, result_len)) == NULL) - return NULL; - result_s = PyBytes_AS_STRING(result); - - start = self_s; - end = self_s + self_len; - while (count-- > 0) { - offset = findstring(start, end-start, - from_s, from_len, - 0, end-start, FORWARD); - if (offset == -1) - break; - next = start+offset; - if (next == start) { - /* replace with the 'to' */ - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start += from_len; - } else { - /* copy the unchanged old then the 'to' */ - Py_MEMCPY(result_s, start, next-start); - result_s += (next-start); - Py_MEMCPY(result_s, to_s, to_len); - result_s += to_len; - start = next+from_len; - } - } - /* Copy the remainder of the remaining string */ - Py_MEMCPY(result_s, start, end-start); - - return result; -} - - -Py_LOCAL(PyBytesObject *) -replace(PyBytesObject *self, - const char *from_s, Py_ssize_t from_len, - const char *to_s, Py_ssize_t to_len, - Py_ssize_t maxcount) -{ - if (maxcount < 0) { - maxcount = PY_SSIZE_T_MAX; - } else if (maxcount == 0 || PyBytes_GET_SIZE(self) == 0) { - /* nothing to do; return the original string */ - return return_self(self); - } - - if (maxcount == 0 || - (from_len == 0 && to_len == 0)) { - /* nothing to do; return the original string */ - return return_self(self); - } - - /* Handle zero-length special cases */ - - if (from_len == 0) { - /* insert the 'to' string everywhere. */ - /* >>> "Python".replace("", ".") */ - /* '.P.y.t.h.o.n.' */ - return replace_interleave(self, to_s, to_len, maxcount); - } - - /* Except for "".replace("", "A") == "A" there is no way beyond this */ - /* point for an empty self string to generate a non-empty string */ - /* Special case so the remaining code always gets a non-empty string */ - if (PyBytes_GET_SIZE(self) == 0) { - return return_self(self); - } - - if (to_len == 0) { - /* delete all occurances of 'from' string */ - if (from_len == 1) { - return replace_delete_single_character( - self, from_s[0], maxcount); - } else { - return replace_delete_substring(self, from_s, - from_len, maxcount); - } - } - - /* Handle special case where both strings have the same length */ - - if (from_len == to_len) { - if (from_len == 1) { - return replace_single_character_in_place( - self, - from_s[0], - to_s[0], - maxcount); - } else { - return replace_substring_in_place( - self, from_s, from_len, to_s, to_len, - maxcount); - } - } - - /* Otherwise use the more generic algorithms */ - if (from_len == 1) { - return replace_single_character(self, from_s[0], - to_s, to_len, maxcount); - } else { - /* len('from')>=2, len('to')>=1 */ - return replace_substring(self, from_s, from_len, to_s, to_len, - maxcount); - } -} - -PyDoc_STRVAR(replace__doc__, -"B.replace(old, new[, count]) -> bytes\n\ -\n\ -Return a copy of B with all occurrences of subsection\n\ -old replaced by new. If the optional argument count is\n\ -given, only the first count occurrences are replaced."); - -static PyObject * -string_replace(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t count = -1; - PyObject *from, *to; - const char *from_s, *to_s; - Py_ssize_t from_len, to_len; - - if (!PyArg_ParseTuple(args, "OO|n:replace", &from, &to, &count)) - return NULL; - - if (PyBytes_Check(from)) { - from_s = PyBytes_AS_STRING(from); - from_len = PyBytes_GET_SIZE(from); - } - else if (PyObject_AsCharBuffer(from, &from_s, &from_len)) - return NULL; - - if (PyBytes_Check(to)) { - to_s = PyBytes_AS_STRING(to); - to_len = PyBytes_GET_SIZE(to); - } - else if (PyObject_AsCharBuffer(to, &to_s, &to_len)) - return NULL; - - return (PyObject *)replace((PyBytesObject *) self, - from_s, from_len, - to_s, to_len, count); -} - -/** End DALKE **/ - -/* Matches the end (direction >= 0) or start (direction < 0) of self - * against substr, using the start and end arguments. Returns - * -1 on error, 0 if not found and 1 if found. - */ -Py_LOCAL(int) -_string_tailmatch(PyBytesObject *self, PyObject *substr, Py_ssize_t start, - Py_ssize_t end, int direction) -{ - Py_ssize_t len = PyBytes_GET_SIZE(self); - Py_ssize_t slen; - const char* sub; - const char* str; - - if (PyBytes_Check(substr)) { - sub = PyBytes_AS_STRING(substr); - slen = PyBytes_GET_SIZE(substr); - } - else if (PyObject_AsCharBuffer(substr, &sub, &slen)) - return -1; - str = PyBytes_AS_STRING(self); - - string_adjust_indices(&start, &end, len); - - if (direction < 0) { - /* startswith */ - if (start+slen > len) - return 0; - } else { - /* endswith */ - if (end-start < slen || start > len) - return 0; - - if (end-slen > start) - start = end - slen; - } - if (end-start >= slen) - return ! memcmp(str+start, sub, slen); - return 0; -} - - -PyDoc_STRVAR(startswith__doc__, -"B.startswith(prefix [,start [,end]]) -> bool\n\ -\n\ -Return True if B starts with the specified prefix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -prefix can also be a tuple of strings to try."); - -static PyObject * -string_startswith(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!PyArg_ParseTuple(args, "O|O&O&:startswith", &subobj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _string_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, -1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _string_tailmatch(self, subobj, start, end, -1); - if (result == -1) - return NULL; - else - return PyBool_FromLong(result); -} - - -PyDoc_STRVAR(endswith__doc__, -"B.endswith(suffix [,start [,end]]) -> bool\n\ -\n\ -Return True if B ends with the specified suffix, False otherwise.\n\ -With optional start, test B beginning at that position.\n\ -With optional end, stop comparing B at that position.\n\ -suffix can also be a tuple of strings to try."); - -static PyObject * -string_endswith(PyBytesObject *self, PyObject *args) -{ - Py_ssize_t start = 0; - Py_ssize_t end = PY_SSIZE_T_MAX; - PyObject *subobj; - int result; - - if (!PyArg_ParseTuple(args, "O|O&O&:endswith", &subobj, - _PyEval_SliceIndex, &start, _PyEval_SliceIndex, &end)) - return NULL; - if (PyTuple_Check(subobj)) { - Py_ssize_t i; - for (i = 0; i < PyTuple_GET_SIZE(subobj); i++) { - result = _string_tailmatch(self, - PyTuple_GET_ITEM(subobj, i), - start, end, +1); - if (result == -1) - return NULL; - else if (result) { - Py_RETURN_TRUE; - } - } - Py_RETURN_FALSE; - } - result = _string_tailmatch(self, subobj, start, end, +1); - if (result == -1) - return NULL; - else - return PyBool_FromLong(result); -} - - -PyDoc_STRVAR(decode__doc__, -"B.decode([encoding[, errors]]) -> object\n\ -\n\ -Decodes S using the codec registered for encoding. encoding defaults\n\ -to the default encoding. errors may be given to set a different error\n\ -handling scheme. Default is 'strict' meaning that encoding errors raise\n\ -a UnicodeDecodeError. Other possible values are 'ignore' and 'replace'\n\ -as well as any other name registerd with codecs.register_error that is\n\ -able to handle UnicodeDecodeErrors."); - -static PyObject * -string_decode(PyObject *self, PyObject *args) -{ - const char *encoding = NULL; - const char *errors = NULL; - - if (!PyArg_ParseTuple(args, "|ss:decode", &encoding, &errors)) - return NULL; - if (encoding == NULL) - encoding = PyUnicode_GetDefaultEncoding(); - return PyCodec_Decode(self, encoding, errors); -} - - -PyDoc_STRVAR(fromhex_doc, -"bytes.fromhex(string) -> bytes\n\ -\n\ -Create a bytes object from a string of hexadecimal numbers.\n\ -Spaces between two numbers are accepted.\n\ -Example: bytes.fromhex('B9 01EF') -> b'\\xb9\\x01\\xef'."); - -static int -hex_digit_to_int(Py_UNICODE c) -{ - if (c >= 128) - return -1; - if (ISDIGIT(c)) - return c - '0'; - else { - if (ISUPPER(c)) - c = TOLOWER(c); - if (c >= 'a' && c <= 'f') - return c - 'a' + 10; - } - return -1; -} - -static PyObject * -string_fromhex(PyObject *cls, PyObject *args) -{ - PyObject *newstring, *hexobj; - char *buf; - Py_UNICODE *hex; - Py_ssize_t hexlen, byteslen, i, j; - int top, bot; - - if (!PyArg_ParseTuple(args, "U:fromhex", &hexobj)) - return NULL; - assert(PyUnicode_Check(hexobj)); - hexlen = PyUnicode_GET_SIZE(hexobj); - hex = PyUnicode_AS_UNICODE(hexobj); - byteslen = hexlen/2; /* This overestimates if there are spaces */ - newstring = PyBytes_FromStringAndSize(NULL, byteslen); - if (!newstring) - return NULL; - buf = PyBytes_AS_STRING(newstring); - for (i = j = 0; i < hexlen; i += 2) { - /* skip over spaces in the input */ - while (hex[i] == ' ') - i++; - if (i >= hexlen) - break; - top = hex_digit_to_int(hex[i]); - bot = hex_digit_to_int(hex[i+1]); - if (top == -1 || bot == -1) { - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", i); - goto error; - } - buf[j++] = (top << 4) + bot; - } - if (j != byteslen && _PyBytes_Resize(&newstring, j) < 0) - goto error; - return newstring; - - error: - Py_XDECREF(newstring); - return NULL; -} - - -static PyObject * -string_getnewargs(PyBytesObject *v) -{ - return Py_BuildValue("(s#)", v->ob_sval, Py_SIZE(v)); -} - - -static PyMethodDef -string_methods[] = { - {"__getnewargs__", (PyCFunction)string_getnewargs, METH_NOARGS}, - {"capitalize", (PyCFunction)stringlib_capitalize, METH_NOARGS, - _Py_capitalize__doc__}, - {"center", (PyCFunction)stringlib_center, METH_VARARGS, center__doc__}, - {"count", (PyCFunction)string_count, METH_VARARGS, count__doc__}, - {"decode", (PyCFunction)string_decode, METH_VARARGS, decode__doc__}, - {"endswith", (PyCFunction)string_endswith, METH_VARARGS, - endswith__doc__}, - {"expandtabs", (PyCFunction)stringlib_expandtabs, METH_VARARGS, - expandtabs__doc__}, - {"find", (PyCFunction)string_find, METH_VARARGS, find__doc__}, - {"fromhex", (PyCFunction)string_fromhex, METH_VARARGS|METH_CLASS, - fromhex_doc}, - {"index", (PyCFunction)string_index, METH_VARARGS, index__doc__}, - {"isalnum", (PyCFunction)stringlib_isalnum, METH_NOARGS, - _Py_isalnum__doc__}, - {"isalpha", (PyCFunction)stringlib_isalpha, METH_NOARGS, - _Py_isalpha__doc__}, - {"isdigit", (PyCFunction)stringlib_isdigit, METH_NOARGS, - _Py_isdigit__doc__}, - {"islower", (PyCFunction)stringlib_islower, METH_NOARGS, - _Py_islower__doc__}, - {"isspace", (PyCFunction)stringlib_isspace, METH_NOARGS, - _Py_isspace__doc__}, - {"istitle", (PyCFunction)stringlib_istitle, METH_NOARGS, - _Py_istitle__doc__}, - {"isupper", (PyCFunction)stringlib_isupper, METH_NOARGS, - _Py_isupper__doc__}, - {"join", (PyCFunction)string_join, METH_O, join__doc__}, - {"ljust", (PyCFunction)stringlib_ljust, METH_VARARGS, ljust__doc__}, - {"lower", (PyCFunction)stringlib_lower, METH_NOARGS, _Py_lower__doc__}, - {"lstrip", (PyCFunction)string_lstrip, METH_VARARGS, lstrip__doc__}, - {"partition", (PyCFunction)string_partition, METH_O, partition__doc__}, - {"replace", (PyCFunction)string_replace, METH_VARARGS, replace__doc__}, - {"rfind", (PyCFunction)string_rfind, METH_VARARGS, rfind__doc__}, - {"rindex", (PyCFunction)string_rindex, METH_VARARGS, rindex__doc__}, - {"rjust", (PyCFunction)stringlib_rjust, METH_VARARGS, rjust__doc__}, - {"rpartition", (PyCFunction)string_rpartition, METH_O, - rpartition__doc__}, - {"rsplit", (PyCFunction)string_rsplit, METH_VARARGS, rsplit__doc__}, - {"rstrip", (PyCFunction)string_rstrip, METH_VARARGS, rstrip__doc__}, - {"split", (PyCFunction)string_split, METH_VARARGS, split__doc__}, - {"splitlines", (PyCFunction)stringlib_splitlines, METH_VARARGS, - splitlines__doc__}, - {"startswith", (PyCFunction)string_startswith, METH_VARARGS, - startswith__doc__}, - {"strip", (PyCFunction)string_strip, METH_VARARGS, strip__doc__}, - {"swapcase", (PyCFunction)stringlib_swapcase, METH_NOARGS, - _Py_swapcase__doc__}, - {"title", (PyCFunction)stringlib_title, METH_NOARGS, _Py_title__doc__}, - {"translate", (PyCFunction)string_translate, METH_VARARGS, - translate__doc__}, - {"upper", (PyCFunction)stringlib_upper, METH_NOARGS, _Py_upper__doc__}, - {"zfill", (PyCFunction)stringlib_zfill, METH_VARARGS, zfill__doc__}, - {NULL, NULL} /* sentinel */ -}; - -static PyObject * -str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds); - -static PyObject * -string_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyObject *x = NULL, *it; - const char *encoding = NULL; - const char *errors = NULL; - PyObject *new = NULL; - Py_ssize_t i, size; - static char *kwlist[] = {"source", "encoding", "errors", 0}; - - if (type != &PyBytes_Type) - return str_subtype_new(type, args, kwds); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|Oss:bytes", kwlist, &x, - &encoding, &errors)) - return NULL; - if (x == NULL) { - if (encoding != NULL || errors != NULL) { - PyErr_SetString(PyExc_TypeError, - "encoding or errors without sequence " - "argument"); - return NULL; - } - return PyBytes_FromString(""); - } - - if (PyUnicode_Check(x)) { - /* Encode via the codec registry */ - if (encoding == NULL) { - PyErr_SetString(PyExc_TypeError, - "string argument without an encoding"); - return NULL; - } - new = PyCodec_Encode(x, encoding, errors); - if (new == NULL) - return NULL; - assert(PyBytes_Check(new)); - return new; - } - - /* If it's not unicode, there can't be encoding or errors */ - if (encoding != NULL || errors != NULL) { - PyErr_SetString(PyExc_TypeError, - "encoding or errors without a string argument"); - return NULL; - } - - /* Is it an int? */ - size = PyNumber_AsSsize_t(x, PyExc_ValueError); - if (size == -1 && PyErr_Occurred()) { - PyErr_Clear(); - } - else { - if (size < 0) { - PyErr_SetString(PyExc_ValueError, "negative count"); - return NULL; - } - new = PyBytes_FromStringAndSize(NULL, size); - if (new == NULL) { - return NULL; - } - if (size > 0) { - memset(((PyBytesObject*)new)->ob_sval, 0, size); - } - return new; - } - - /* Use the modern buffer interface */ - if (PyObject_CheckBuffer(x)) { - Py_buffer view; - if (PyObject_GetBuffer(x, &view, PyBUF_FULL_RO) < 0) - return NULL; - new = PyBytes_FromStringAndSize(NULL, view.len); - if (!new) - goto fail; - // XXX(brett.cannon): Better way to get to internal buffer? - if (PyBuffer_ToContiguous(((PyBytesObject *)new)->ob_sval, - &view, view.len, 'C') < 0) - goto fail; - PyObject_ReleaseBuffer(x, &view); - return new; - fail: - Py_XDECREF(new); - PyObject_ReleaseBuffer(x, &view); - return NULL; - } - - /* For iterator version, create a string object and resize as needed */ - /* XXX(gb): is 64 a good value? also, optimize if length is known */ - /* XXX(guido): perhaps use Pysequence_Fast() -- I can't imagine the - input being a truly long iterator. */ - size = 64; - new = PyBytes_FromStringAndSize(NULL, size); - if (new == NULL) - return NULL; - - /* XXX Optimize this if the arguments is a list, tuple */ - - /* Get the iterator */ - it = PyObject_GetIter(x); - if (it == NULL) - goto error; - - /* Run the iterator to exhaustion */ - for (i = 0; ; i++) { - PyObject *item; - Py_ssize_t value; - - /* Get the next item */ - item = PyIter_Next(it); - if (item == NULL) { - if (PyErr_Occurred()) - goto error; - break; - } - - /* Interpret it as an int (__index__) */ - value = PyNumber_AsSsize_t(item, PyExc_ValueError); - Py_DECREF(item); - if (value == -1 && PyErr_Occurred()) - goto error; - - /* Range check */ - if (value < 0 || value >= 256) { - PyErr_SetString(PyExc_ValueError, - "bytes must be in range(0, 256)"); - goto error; - } - - /* Append the byte */ - if (i >= size) { - size *= 2; - if (_PyBytes_Resize(&new, size) < 0) - goto error; - } - ((PyBytesObject *)new)->ob_sval[i] = value; - } - _PyBytes_Resize(&new, i); - - /* Clean up and return success */ - Py_DECREF(it); - return new; - - error: - /* Error handling when new != NULL */ - Py_XDECREF(it); - Py_DECREF(new); - return NULL; -} - -static PyObject * -str_subtype_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - PyObject *tmp, *pnew; - Py_ssize_t n; - - assert(PyType_IsSubtype(type, &PyBytes_Type)); - tmp = string_new(&PyBytes_Type, args, kwds); - if (tmp == NULL) - return NULL; - assert(PyBytes_CheckExact(tmp)); - n = PyBytes_GET_SIZE(tmp); - pnew = type->tp_alloc(type, n); - if (pnew != NULL) { - Py_MEMCPY(PyBytes_AS_STRING(pnew), - PyBytes_AS_STRING(tmp), n+1); - ((PyBytesObject *)pnew)->ob_shash = - ((PyBytesObject *)tmp)->ob_shash; - } - Py_DECREF(tmp); - return pnew; -} - -PyDoc_STRVAR(string_doc, -"bytes(iterable_of_ints) -> bytes.\n\ -bytes(string, encoding[, errors]) -> bytes\n\ -bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer.\n\ -bytes(memory_view) -> bytes.\n\ -\n\ -Construct an immutable array of bytes from:\n\ - - an iterable yielding integers in range(256)\n\ - - a text string encoded using the specified encoding\n\ - - a bytes or a buffer object\n\ - - any object implementing the buffer API."); - -static PyObject *str_iter(PyObject *seq); - -PyTypeObject PyBytes_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "bytes", - sizeof(PyBytesObject), - sizeof(char), - string_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - (reprfunc)string_repr, /* tp_repr */ - 0, /* tp_as_number */ - &string_as_sequence, /* tp_as_sequence */ - &string_as_mapping, /* tp_as_mapping */ - (hashfunc)string_hash, /* tp_hash */ - 0, /* tp_call */ - string_str, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - &string_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_BYTES_SUBCLASS, /* tp_flags */ - string_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - (richcmpfunc)string_richcompare, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - str_iter, /* tp_iter */ - 0, /* tp_iternext */ - string_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - &PyBaseObject_Type, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - string_new, /* tp_new */ - PyObject_Del, /* tp_free */ -}; - -void -PyBytes_Concat(register PyObject **pv, register PyObject *w) -{ - register PyObject *v; - assert(pv != NULL); - if (*pv == NULL) - return; - if (w == NULL) { - Py_DECREF(*pv); - *pv = NULL; - return; - } - v = string_concat(*pv, w); - Py_DECREF(*pv); - *pv = v; -} - -void -PyBytes_ConcatAndDel(register PyObject **pv, register PyObject *w) -{ - PyBytes_Concat(pv, w); - Py_XDECREF(w); -} - - -/* The following function breaks the notion that strings are immutable: - it changes the size of a string. We get away with this only if there - is only one module referencing the object. You can also think of it - as creating a new string object and destroying the old one, only - more efficiently. In any case, don't use this if the string may - already be known to some other part of the code... - Note that if there's not enough memory to resize the string, the original - string object at *pv is deallocated, *pv is set to NULL, an "out of - memory" exception is set, and -1 is returned. Else (on success) 0 is - returned, and the value in *pv may or may not be the same as on input. - As always, an extra byte is allocated for a trailing \0 byte (newsize - does *not* include that), and a trailing \0 byte is stored. -*/ - -int -_PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) -{ - register PyObject *v; - register PyBytesObject *sv; - v = *pv; - if (!PyBytes_Check(v) || Py_REFCNT(v) != 1 || newsize < 0) { - *pv = 0; - Py_DECREF(v); - PyErr_BadInternalCall(); - return -1; - } - /* XXX UNREF/NEWREF interface should be more symmetrical */ - _Py_DEC_REFTOTAL; - _Py_ForgetReference(v); - *pv = (PyObject *) - PyObject_REALLOC((char *)v, sizeof(PyBytesObject) + newsize); - if (*pv == NULL) { - PyObject_Del(v); - PyErr_NoMemory(); - return -1; - } - _Py_NewReference(*pv); - sv = (PyBytesObject *) *pv; - Py_SIZE(sv) = newsize; - sv->ob_sval[newsize] = '\0'; - sv->ob_shash = -1; /* invalidate cached hash value */ - return 0; -} - -/* _PyBytes_FormatLong emulates the format codes d, u, o, x and X, and - * the F_ALT flag, for Python's long (unbounded) ints. It's not used for - * Python's regular ints. - * Return value: a new PyString*, or NULL if error. - * . *pbuf is set to point into it, - * *plen set to the # of chars following that. - * Caller must decref it when done using pbuf. - * The string starting at *pbuf is of the form - * "-"? ("0x" | "0X")? digit+ - * "0x"/"0X" are present only for x and X conversions, with F_ALT - * set in flags. The case of hex digits will be correct, - * There will be at least prec digits, zero-filled on the left if - * necessary to get that many. - * val object to be converted - * flags bitmask of format flags; only F_ALT is looked at - * prec minimum number of digits; 0-fill on left if needed - * type a character in [duoxX]; u acts the same as d - * - * CAUTION: o, x and X conversions on regular ints can never - * produce a '-' sign, but can for Python's unbounded ints. - */ -PyObject* -_PyBytes_FormatLong(PyObject *val, int flags, int prec, int type, - char **pbuf, int *plen) -{ - PyObject *result = NULL; - char *buf; - Py_ssize_t i; - int sign; /* 1 if '-', else 0 */ - int len; /* number of characters */ - Py_ssize_t llen; - int numdigits; /* len == numnondigits + numdigits */ - int numnondigits = 0; - - /* Avoid exceeding SSIZE_T_MAX */ - if (prec > PY_SSIZE_T_MAX-3) { - PyErr_SetString(PyExc_OverflowError, - "precision too large"); - return NULL; - } - - switch (type) { - case 'd': - case 'u': - /* Special-case boolean: we want 0/1 */ - if (PyBool_Check(val)) - result = PyNumber_ToBase(val, 10); - else - result = Py_TYPE(val)->tp_str(val); - break; - case 'o': - numnondigits = 2; - result = PyNumber_ToBase(val, 8); - break; - case 'x': - case 'X': - numnondigits = 2; - result = PyNumber_ToBase(val, 16); - break; - default: - assert(!"'type' not in [duoxX]"); - } - if (!result) - return NULL; - - buf = PyUnicode_AsString(result); - if (!buf) { - Py_DECREF(result); - return NULL; - } - - /* To modify the string in-place, there can only be one reference. */ - if (Py_REFCNT(result) != 1) { - PyErr_BadInternalCall(); - return NULL; - } - llen = PyUnicode_GetSize(result); - if (llen > INT_MAX) { - PyErr_SetString(PyExc_ValueError, - "string too large in _PyBytes_FormatLong"); - return NULL; - } - len = (int)llen; - if (buf[len-1] == 'L') { - --len; - buf[len] = '\0'; - } - sign = buf[0] == '-'; - numnondigits += sign; - numdigits = len - numnondigits; - assert(numdigits > 0); - - /* Get rid of base marker unless F_ALT */ - if (((flags & F_ALT) == 0 && - (type == 'o' || type == 'x' || type == 'X'))) { - assert(buf[sign] == '0'); - assert(buf[sign+1] == 'x' || buf[sign+1] == 'X' || - buf[sign+1] == 'o'); - numnondigits -= 2; - buf += 2; - len -= 2; - if (sign) - buf[0] = '-'; - assert(len == numnondigits + numdigits); - assert(numdigits > 0); - } - - /* Fill with leading zeroes to meet minimum width. */ - if (prec > numdigits) { - PyObject *r1 = PyBytes_FromStringAndSize(NULL, - numnondigits + prec); - char *b1; - if (!r1) { - Py_DECREF(result); - return NULL; - } - b1 = PyBytes_AS_STRING(r1); - for (i = 0; i < numnondigits; ++i) - *b1++ = *buf++; - for (i = 0; i < prec - numdigits; i++) - *b1++ = '0'; - for (i = 0; i < numdigits; i++) - *b1++ = *buf++; - *b1 = '\0'; - Py_DECREF(result); - result = r1; - buf = PyBytes_AS_STRING(result); - len = numnondigits + prec; - } - - /* Fix up case for hex conversions. */ - if (type == 'X') { - /* Need to convert all lower case letters to upper case. - and need to convert 0x to 0X (and -0x to -0X). */ - for (i = 0; i < len; i++) - if (buf[i] >= 'a' && buf[i] <= 'x') - buf[i] -= 'a'-'A'; - } - *pbuf = buf; - *plen = len; - return result; -} - -void -PyBytes_Fini(void) -{ - int i; - for (i = 0; i < UCHAR_MAX + 1; i++) { - Py_XDECREF(characters[i]); - characters[i] = NULL; - } - Py_XDECREF(nullstring); - nullstring = NULL; -} - -/*********************** Str Iterator ****************************/ - -typedef struct { - PyObject_HEAD - Py_ssize_t it_index; - PyBytesObject *it_seq; /* Set to NULL when iterator is exhausted */ -} striterobject; - -static void -striter_dealloc(striterobject *it) -{ - _PyObject_GC_UNTRACK(it); - Py_XDECREF(it->it_seq); - PyObject_GC_Del(it); -} - -static int -striter_traverse(striterobject *it, visitproc visit, void *arg) -{ - Py_VISIT(it->it_seq); - return 0; -} - -static PyObject * -striter_next(striterobject *it) -{ - PyBytesObject *seq; - PyObject *item; - - assert(it != NULL); - seq = it->it_seq; - if (seq == NULL) - return NULL; - assert(PyBytes_Check(seq)); - - if (it->it_index < PyBytes_GET_SIZE(seq)) { - item = PyLong_FromLong( - (unsigned char)seq->ob_sval[it->it_index]); - if (item != NULL) - ++it->it_index; - return item; - } - - Py_DECREF(seq); - it->it_seq = NULL; - return NULL; -} - -static PyObject * -striter_len(striterobject *it) -{ - Py_ssize_t len = 0; - if (it->it_seq) - len = PyBytes_GET_SIZE(it->it_seq) - it->it_index; - return PyLong_FromSsize_t(len); -} - -PyDoc_STRVAR(length_hint_doc, - "Private method returning an estimate of len(list(it))."); - -static PyMethodDef striter_methods[] = { - {"__length_hint__", (PyCFunction)striter_len, METH_NOARGS, - length_hint_doc}, - {NULL, NULL} /* sentinel */ -}; - -PyTypeObject PyBytesIter_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - "bytes_iterator", /* tp_name */ - sizeof(striterobject), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)striter_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_compare */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC,/* tp_flags */ - 0, /* tp_doc */ - (traverseproc)striter_traverse, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)striter_next, /* tp_iternext */ - striter_methods, /* tp_methods */ - 0, -}; - -static PyObject * -str_iter(PyObject *seq) -{ - striterobject *it; - - if (!PyBytes_Check(seq)) { - PyErr_BadInternalCall(); - return NULL; - } - it = PyObject_GC_New(striterobject, &PyBytesIter_Type); - if (it == NULL) - return NULL; - it->it_index = 0; - Py_INCREF(seq); - it->it_seq = (PyBytesObject *)seq; - _PyObject_GC_TRACK(it); - return (PyObject *)it; -} diff --git a/PCbuild/pythoncore.vcproj b/PCbuild/pythoncore.vcproj index 50242b8..5cb0e35 100644 --- a/PCbuild/pythoncore.vcproj +++ b/PCbuild/pythoncore.vcproj @@ -655,6 +655,10 @@ > </File> <File + RelativePath="..\Include\bytearrayobject.h" + > + </File> + <File RelativePath="..\Include\bytesobject.h" > </File> @@ -923,10 +927,6 @@ > </File> <File - RelativePath="..\Include\stringobject.h" - > - </File> - <File RelativePath="..\Include\structmember.h" > </File> @@ -1343,6 +1343,10 @@ > </File> <File + RelativePath="..\Objects\bytearrayobject.c" + > + </File> + <File RelativePath="..\Objects\bytesobject.c" > </File> @@ -1463,10 +1467,6 @@ > </File> <File - RelativePath="..\Objects\stringobject.c" - > - </File> - <File RelativePath="..\Objects\structseq.c" > </File> diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 18fb122..487405f 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -12,7 +12,7 @@ #ifndef PGEN #include "unicodeobject.h" -#include "stringobject.h" +#include "bytesobject.h" #include "fileobject.h" #include "codecs.h" #include "abstract.h" |