diff options
Diffstat (limited to 'Modules')
95 files changed, 10836 insertions, 5955 deletions
diff --git a/Modules/Setup.dist b/Modules/Setup.dist index e1c8fe4..2859fa5 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -127,6 +127,9 @@ _io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesi # builtin module avoids some bootstrapping problems and reduces overhead. zipimport zipimport.c +# faulthandler module +faulthandler faulthandler.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If @@ -207,7 +210,7 @@ _symtable symtablemodule.c # # First, look at Setup.config; configure may have set this for you. -#crypt cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems +#_crypt _cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems # Some more UNIX dependent modules -- off by default, since these diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 7fecfc6..9da255e 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -86,7 +86,9 @@ insort_right(PyObject *self, PyObject *args, PyObject *kw) if (PyList_Insert(list, index, item) < 0) return NULL; } else { - result = PyObject_CallMethod(list, "insert", "nO", index, item); + _Py_identifier(insert); + + result = _PyObject_CallMethodId(list, &PyId_insert, "nO", index, item); if (result == NULL) return NULL; Py_DECREF(result); @@ -186,7 +188,9 @@ insort_left(PyObject *self, PyObject *args, PyObject *kw) if (PyList_Insert(list, index, item) < 0) return NULL; } else { - result = PyObject_CallMethod(list, "insert", "iO", index, item); + _Py_identifier(insert); + + result = _PyObject_CallMethodId(list, &PyId_insert, "iO", index, item); if (result == NULL) return NULL; Py_DECREF(result); diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c new file mode 100644 index 0000000..b407df9 --- /dev/null +++ b/Modules/_bz2module.c @@ -0,0 +1,590 @@ +/* _bz2 - Low-level Python interface to libbzip2. */ + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "structmember.h" + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +#include <bzlib.h> +#include <stdio.h> + + +#ifndef BZ_CONFIG_ERROR +#define BZ2_bzCompress bzCompress +#define BZ2_bzCompressInit bzCompressInit +#define BZ2_bzCompressEnd bzCompressEnd +#define BZ2_bzDecompress bzDecompress +#define BZ2_bzDecompressInit bzDecompressInit +#define BZ2_bzDecompressEnd bzDecompressEnd +#endif /* ! BZ_CONFIG_ERROR */ + + +#ifdef WITH_THREAD +#define ACQUIRE_LOCK(obj) do { \ + if (!PyThread_acquire_lock((obj)->lock, 0)) { \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((obj)->lock, 1); \ + Py_END_ALLOW_THREADS \ + } } while (0) +#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) +#else +#define ACQUIRE_LOCK(obj) +#define RELEASE_LOCK(obj) +#endif + +#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) + + +typedef struct { + PyObject_HEAD + bz_stream bzs; + int flushed; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2Compressor; + +typedef struct { + PyObject_HEAD + bz_stream bzs; + char eof; /* T_BOOL expects a char */ + PyObject *unused_data; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2Decompressor; + + +/* Helper functions. */ + +static int +catch_bz2_error(int bzerror) +{ + switch(bzerror) { + case BZ_OK: + case BZ_RUN_OK: + case BZ_FLUSH_OK: + case BZ_FINISH_OK: + case BZ_STREAM_END: + return 0; + +#ifdef BZ_CONFIG_ERROR + case BZ_CONFIG_ERROR: + PyErr_SetString(PyExc_SystemError, + "libbzip2 was not compiled correctly"); + return 1; +#endif + case BZ_PARAM_ERROR: + PyErr_SetString(PyExc_ValueError, + "Internal error - " + "invalid parameters passed to libbzip2"); + return 1; + case BZ_MEM_ERROR: + PyErr_NoMemory(); + return 1; + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + PyErr_SetString(PyExc_IOError, "Invalid data stream"); + return 1; + case BZ_IO_ERROR: + PyErr_SetString(PyExc_IOError, "Unknown I/O error"); + return 1; + case BZ_UNEXPECTED_EOF: + PyErr_SetString(PyExc_EOFError, + "Compressed file ended before the logical " + "end-of-stream was detected"); + return 1; + case BZ_SEQUENCE_ERROR: + PyErr_SetString(PyExc_RuntimeError, + "Internal error - " + "Invalid sequence of commands sent to libbzip2"); + return 1; + default: + PyErr_Format(PyExc_IOError, + "Unrecognized error from libbzip2: %d", bzerror); + return 1; + } +} + +#if BUFSIZ < 8192 +#define SMALLCHUNK 8192 +#else +#define SMALLCHUNK BUFSIZ +#endif + +static int +grow_buffer(PyObject **buf) +{ + /* Expand the buffer by an amount proportional to the current size, + giving us amortized linear-time behavior. Use a less-than-double + growth factor to avoid excessive allocation. */ + size_t size = PyBytes_GET_SIZE(*buf); + return _PyBytes_Resize(buf, size + (size >> 3) + 6); +} + + +/* BZ2Compressor class. */ + +static PyObject * +compress(BZ2Compressor *c, char *data, size_t len, int action) +{ + size_t data_size = 0; + PyObject *result; + + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); + if (result == NULL) + return NULL; + c->bzs.next_in = data; + /* On a 64-bit system, len might not fit in avail_in (an unsigned int). + Do compression in chunks of no more than UINT_MAX bytes each. */ + c->bzs.avail_in = MIN(len, UINT_MAX); + len -= c->bzs.avail_in; + c->bzs.next_out = PyBytes_AS_STRING(result); + c->bzs.avail_out = PyBytes_GET_SIZE(result); + for (;;) { + char *this_out; + int bzerror; + + Py_BEGIN_ALLOW_THREADS + this_out = c->bzs.next_out; + bzerror = BZ2_bzCompress(&c->bzs, action); + data_size += c->bzs.next_out - this_out; + Py_END_ALLOW_THREADS + if (catch_bz2_error(bzerror)) + goto error; + + if (c->bzs.avail_in == 0 && len > 0) { + c->bzs.avail_in = MIN(len, UINT_MAX); + len -= c->bzs.avail_in; + } + + /* In regular compression mode, stop when input data is exhausted. + In flushing mode, stop when all buffered data has been flushed. */ + if ((action == BZ_RUN && c->bzs.avail_in == 0) || + (action == BZ_FINISH && bzerror == BZ_STREAM_END)) + break; + + if (c->bzs.avail_out == 0) { + if (grow_buffer(&result) < 0) + goto error; + c->bzs.next_out = PyBytes_AS_STRING(result) + data_size; + c->bzs.avail_out = PyBytes_GET_SIZE(result) - data_size; + } + } + if (data_size != PyBytes_GET_SIZE(result)) + if (_PyBytes_Resize(&result, data_size) < 0) + goto error; + return result; + +error: + Py_XDECREF(result); + return NULL; +} + +PyDoc_STRVAR(BZ2Compressor_compress__doc__, +"compress(data) -> bytes\n" +"\n" +"Provide data to the compressor object. Returns a chunk of\n" +"compressed data if possible, or b'' otherwise.\n" +"\n" +"When you have finished providing data to the compressor, call the\n" +"flush() method to finish the compression process.\n"); + +static PyObject * +BZ2Compressor_compress(BZ2Compressor *self, PyObject *args) +{ + Py_buffer buffer; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(args, "y*:compress", &buffer)) + return NULL; + + ACQUIRE_LOCK(self); + if (self->flushed) + PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); + else + result = compress(self, buffer.buf, buffer.len, BZ_RUN); + RELEASE_LOCK(self); + PyBuffer_Release(&buffer); + return result; +} + +PyDoc_STRVAR(BZ2Compressor_flush__doc__, +"flush() -> bytes\n" +"\n" +"Finish the compression process. Returns the compressed data left\n" +"in internal buffers.\n" +"\n" +"The compressor object may not be used after this method is called.\n"); + +static PyObject * +BZ2Compressor_flush(BZ2Compressor *self, PyObject *noargs) +{ + PyObject *result = NULL; + + ACQUIRE_LOCK(self); + if (self->flushed) + PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); + else { + self->flushed = 1; + result = compress(self, NULL, 0, BZ_FINISH); + } + RELEASE_LOCK(self); + return result; +} + +static int +BZ2Compressor_init(BZ2Compressor *self, PyObject *args, PyObject *kwargs) +{ + int compresslevel = 9; + int bzerror; + + if (!PyArg_ParseTuple(args, "|i:BZ2Compressor", &compresslevel)) + return -1; + if (!(1 <= compresslevel && compresslevel <= 9)) { + PyErr_SetString(PyExc_ValueError, + "compresslevel must be between 1 and 9"); + return -1; + } + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return -1; + } +#endif + + bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0); + if (catch_bz2_error(bzerror)) + goto error; + + return 0; + +error: +#ifdef WITH_THREAD + PyThread_free_lock(self->lock); + self->lock = NULL; +#endif + return -1; +} + +static void +BZ2Compressor_dealloc(BZ2Compressor *self) +{ + BZ2_bzCompressEnd(&self->bzs); +#ifdef WITH_THREAD + if (self->lock != NULL) + PyThread_free_lock(self->lock); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef BZ2Compressor_methods[] = { + {"compress", (PyCFunction)BZ2Compressor_compress, METH_VARARGS, + BZ2Compressor_compress__doc__}, + {"flush", (PyCFunction)BZ2Compressor_flush, METH_NOARGS, + BZ2Compressor_flush__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Compressor__doc__, +"BZ2Compressor(compresslevel=9)\n" +"\n" +"Create a compressor object for compressing data incrementally.\n" +"\n" +"compresslevel, if given, must be a number between 1 and 9.\n" +"\n" +"For one-shot compression, use the compress() function instead.\n"); + +static PyTypeObject BZ2Compressor_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_bz2.BZ2Compressor", /* tp_name */ + sizeof(BZ2Compressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)BZ2Compressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + BZ2Compressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BZ2Compressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BZ2Compressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* BZ2Decompressor class. */ + +static PyObject * +decompress(BZ2Decompressor *d, char *data, size_t len) +{ + size_t data_size = 0; + PyObject *result; + + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); + if (result == NULL) + return result; + d->bzs.next_in = data; + /* On a 64-bit system, len might not fit in avail_in (an unsigned int). + Do decompression in chunks of no more than UINT_MAX bytes each. */ + d->bzs.avail_in = MIN(len, UINT_MAX); + len -= d->bzs.avail_in; + d->bzs.next_out = PyBytes_AS_STRING(result); + d->bzs.avail_out = PyBytes_GET_SIZE(result); + for (;;) { + char *this_out; + int bzerror; + + Py_BEGIN_ALLOW_THREADS + this_out = d->bzs.next_out; + bzerror = BZ2_bzDecompress(&d->bzs); + data_size += d->bzs.next_out - this_out; + Py_END_ALLOW_THREADS + if (catch_bz2_error(bzerror)) + goto error; + if (bzerror == BZ_STREAM_END) { + d->eof = 1; + len += d->bzs.avail_in; + if (len > 0) { /* Save leftover input to unused_data */ + Py_CLEAR(d->unused_data); + d->unused_data = PyBytes_FromStringAndSize(d->bzs.next_in, len); + if (d->unused_data == NULL) + goto error; + } + break; + } + if (d->bzs.avail_in == 0) { + if (len == 0) + break; + d->bzs.avail_in = MIN(len, UINT_MAX); + len -= d->bzs.avail_in; + } + if (d->bzs.avail_out == 0) { + if (grow_buffer(&result) < 0) + goto error; + d->bzs.next_out = PyBytes_AS_STRING(result) + data_size; + d->bzs.avail_out = PyBytes_GET_SIZE(result) - data_size; + } + } + if (data_size != PyBytes_GET_SIZE(result)) + if (_PyBytes_Resize(&result, data_size) < 0) + goto error; + return result; + +error: + Py_XDECREF(result); + return NULL; +} + +PyDoc_STRVAR(BZ2Decompressor_decompress__doc__, +"decompress(data) -> bytes\n" +"\n" +"Provide data to the decompressor object. Returns a chunk of\n" +"decompressed data if possible, or b'' otherwise.\n" +"\n" +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream\n" +"is ignored and saved in the unused_data attribute.\n"); + +static PyObject * +BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *args) +{ + Py_buffer buffer; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(args, "y*:decompress", &buffer)) + return NULL; + + ACQUIRE_LOCK(self); + if (self->eof) + PyErr_SetString(PyExc_EOFError, "End of stream already reached"); + else + result = decompress(self, buffer.buf, buffer.len); + RELEASE_LOCK(self); + PyBuffer_Release(&buffer); + return result; +} + +static int +BZ2Decompressor_init(BZ2Decompressor *self, PyObject *args, PyObject *kwargs) +{ + int bzerror; + + if (!PyArg_ParseTuple(args, ":BZ2Decompressor")) + return -1; + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return -1; + } +#endif + + self->unused_data = PyBytes_FromStringAndSize("", 0); + if (self->unused_data == NULL) + goto error; + + bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); + if (catch_bz2_error(bzerror)) + goto error; + + return 0; + +error: + Py_CLEAR(self->unused_data); +#ifdef WITH_THREAD + PyThread_free_lock(self->lock); + self->lock = NULL; +#endif + return -1; +} + +static void +BZ2Decompressor_dealloc(BZ2Decompressor *self) +{ + BZ2_bzDecompressEnd(&self->bzs); + Py_CLEAR(self->unused_data); +#ifdef WITH_THREAD + if (self->lock != NULL) + PyThread_free_lock(self->lock); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef BZ2Decompressor_methods[] = { + {"decompress", (PyCFunction)BZ2Decompressor_decompress, METH_VARARGS, + BZ2Decompressor_decompress__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Decompressor_eof__doc__, +"True if the end-of-stream marker has been reached."); + +PyDoc_STRVAR(BZ2Decompressor_unused_data__doc__, +"Data found after the end of the compressed stream."); + +static PyMemberDef BZ2Decompressor_members[] = { + {"eof", T_BOOL, offsetof(BZ2Decompressor, eof), + READONLY, BZ2Decompressor_eof__doc__}, + {"unused_data", T_OBJECT_EX, offsetof(BZ2Decompressor, unused_data), + READONLY, BZ2Decompressor_unused_data__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Decompressor__doc__, +"BZ2Decompressor()\n" +"\n" +"Create a decompressor object for decompressing data incrementally.\n" +"\n" +"For one-shot decompression, use the decompress() function instead.\n"); + +static PyTypeObject BZ2Decompressor_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_bz2.BZ2Decompressor", /* tp_name */ + sizeof(BZ2Decompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)BZ2Decompressor_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + BZ2Decompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BZ2Decompressor_methods, /* tp_methods */ + BZ2Decompressor_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BZ2Decompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* Module initialization. */ + +static struct PyModuleDef _bz2module = { + PyModuleDef_HEAD_INIT, + "_bz2", + NULL, + -1, + NULL, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit__bz2(void) +{ + PyObject *m; + + if (PyType_Ready(&BZ2Compressor_Type) < 0) + return NULL; + if (PyType_Ready(&BZ2Decompressor_Type) < 0) + return NULL; + + m = PyModule_Create(&_bz2module); + if (m == NULL) + return NULL; + + Py_INCREF(&BZ2Compressor_Type); + PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Compressor_Type); + + Py_INCREF(&BZ2Decompressor_Type); + PyModule_AddObject(m, "BZ2Decompressor", + (PyObject *)&BZ2Decompressor_Type); + + return m; +} diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index eb740f9..26c8788 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -162,7 +162,6 @@ static PyObject * escape_encode(PyObject *self, PyObject *args) { - static const char *hexdigits = "0123456789abcdef"; PyObject *str; Py_ssize_t size; Py_ssize_t newsize; @@ -205,8 +204,8 @@ escape_encode(PyObject *self, else if (c < ' ' || c >= 0x7f) { *p++ = '\\'; *p++ = 'x'; - *p++ = hexdigits[(c & 0xf0) >> 4]; - *p++ = hexdigits[c & 0xf]; + *p++ = Py_hexdigits[(c & 0xf0) >> 4]; + *p++ = Py_hexdigits[c & 0xf]; } else *p++ = c; @@ -588,7 +587,7 @@ charmap_decode(PyObject *self, return codec_tuple(unicode, pbuf.len); } -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS static PyObject * mbcs_decode(PyObject *self, @@ -613,7 +612,7 @@ mbcs_decode(PyObject *self, return codec_tuple(decoded, consumed); } -#endif /* MS_WINDOWS */ +#endif /* HAVE_MBCS */ /* --- Encoder ------------------------------------------------------------ */ @@ -700,12 +699,10 @@ utf_8_encode(PyObject *self, return NULL; str = PyUnicode_FromObject(str); - if (str == NULL) + if (str == NULL || PyUnicode_READY(str) == -1) return NULL; - v = codec_tuple(PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(str), - PyUnicode_GET_SIZE(str), - errors), - PyUnicode_GET_SIZE(str)); + v = codec_tuple(PyUnicode_AsEncodedString(str, "utf-8", errors), + PyUnicode_GET_LENGTH(str)); Py_DECREF(str); return v; } @@ -989,7 +986,7 @@ charmap_build(PyObject *self, PyObject *args) return PyUnicode_BuildEncodingMap(map); } -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS static PyObject * mbcs_encode(PyObject *self, @@ -1014,7 +1011,7 @@ mbcs_encode(PyObject *self, return v; } -#endif /* MS_WINDOWS */ +#endif /* HAVE_MBCS */ /* --- Error handler registry --------------------------------------------- */ @@ -1101,7 +1098,7 @@ static PyMethodDef _codecs_functions[] = { {"charmap_decode", charmap_decode, METH_VARARGS}, {"charmap_build", charmap_build, METH_VARARGS}, {"readbuffer_encode", readbuffer_encode, METH_VARARGS}, -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS {"mbcs_encode", mbcs_encode, METH_VARARGS}, {"mbcs_decode", mbcs_decode, METH_VARARGS}, #endif diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 5545d1e..0c68aa8 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -767,8 +767,9 @@ static PyObject * deque_reduce(dequeobject *deque) { PyObject *dict, *result, *aslist; + _Py_identifier(__dict__); - dict = PyObject_GetAttrString((PyObject *)deque, "__dict__"); + dict = _PyObject_GetAttrId((PyObject *)deque, &PyId___dict__); if (dict == NULL) PyErr_Clear(); aslist = PySequence_List((PyObject *)deque); @@ -832,8 +833,7 @@ deque_richcompare(PyObject *v, PyObject *w, int op) if (!PyObject_TypeCheck(v, &deque_type) || !PyObject_TypeCheck(w, &deque_type)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } /* Shortcuts */ @@ -1335,13 +1335,15 @@ defdict_reduce(defdictobject *dd) PyObject *items; PyObject *iter; PyObject *result; + _Py_identifier(items); + if (dd->default_factory == NULL || dd->default_factory == Py_None) args = PyTuple_New(0); else args = PyTuple_Pack(1, dd->default_factory); if (args == NULL) return NULL; - items = PyObject_CallMethod((PyObject *)dd, "items", "()"); + items = _PyObject_CallMethodId((PyObject *)dd, &PyId_items, "()"); if (items == NULL) { Py_DECREF(args); return NULL; @@ -1552,12 +1554,8 @@ _count_elements(PyObject *self, PyObject *args) if (PyDict_CheckExact(mapping)) { while (1) { key = PyIter_Next(it); - if (key == NULL) { - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - break; - } + if (key == NULL) + break; oldval = PyDict_GetItem(mapping, key); if (oldval == NULL) { if (PyDict_SetItem(mapping, key, one) == -1) @@ -1575,12 +1573,8 @@ _count_elements(PyObject *self, PyObject *args) } else { while (1) { key = PyIter_Next(it); - if (key == NULL) { - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - break; - } + if (key == NULL) + break; oldval = PyObject_GetItem(mapping, key); if (oldval == NULL) { if (!PyErr_Occurred() || !PyErr_ExceptionMatches(PyExc_KeyError)) diff --git a/Modules/cryptmodule.c b/Modules/_cryptmodule.c index d5a42ff..5100788 100644 --- a/Modules/cryptmodule.c +++ b/Modules/_cryptmodule.c @@ -45,7 +45,7 @@ static PyMethodDef crypt_methods[] = { static struct PyModuleDef cryptmodule = { PyModuleDef_HEAD_INIT, - "crypt", + "_crypt", NULL, -1, crypt_methods, @@ -56,7 +56,7 @@ static struct PyModuleDef cryptmodule = { }; PyMODINIT_FUNC -PyInit_crypt(void) +PyInit__crypt(void) { return PyModule_Create(&cryptmodule); } diff --git a/Modules/_csv.c b/Modules/_csv.c index a749cec..59c74e7 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -47,9 +47,9 @@ typedef struct { PyObject_HEAD int doublequote; /* is " represented by ""? */ - Py_UNICODE delimiter; /* field separator */ - Py_UNICODE quotechar; /* quote character */ - Py_UNICODE escapechar; /* escape character */ + Py_UCS4 delimiter; /* field separator */ + Py_UCS4 quotechar; /* quote character */ + Py_UCS4 escapechar; /* escape character */ int skipinitialspace; /* ignore spaces following delimiter? */ PyObject *lineterminator; /* string to write between records */ int quoting; /* style of quoting to write */ @@ -68,7 +68,7 @@ typedef struct { PyObject *fields; /* field list for current record */ ParserState state; /* current CSV parse state */ - Py_UNICODE *field; /* build current field in here */ + Py_UCS4 *field; /* temporary buffer */ Py_ssize_t field_size; /* size of allocated buffer */ Py_ssize_t field_len; /* length of current field */ int numeric_field; /* treat field as numeric */ @@ -86,7 +86,7 @@ typedef struct { DialectObj *dialect; /* parsing dialect */ - Py_UNICODE *rec; /* buffer for parser.join */ + Py_UCS4 *rec; /* buffer for parser.join */ Py_ssize_t rec_size; /* size of allocated record */ Py_ssize_t rec_len; /* length of record */ int num_fields; /* number of fields in record */ @@ -121,14 +121,14 @@ get_string(PyObject *str) } static PyObject * -get_nullchar_as_None(Py_UNICODE c) +get_nullchar_as_None(Py_UCS4 c) { if (c == '\0') { Py_INCREF(Py_None); return Py_None; } else - return PyUnicode_FromUnicode((Py_UNICODE *)&c, 1); + return PyUnicode_FromOrdinal(c); } static PyObject * @@ -199,25 +199,23 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) } static int -_set_char(const char *name, Py_UNICODE *target, PyObject *src, Py_UNICODE dflt) +_set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) { if (src == NULL) *target = dflt; else { *target = '\0'; if (src != Py_None) { - Py_UNICODE *buf; Py_ssize_t len; - buf = PyUnicode_AsUnicode(src); len = PyUnicode_GetSize(src); - if (buf == NULL || len > 1) { + if (len > 1) { PyErr_Format(PyExc_TypeError, "\"%s\" must be an 1-character string", name); return -1; } if (len > 0) - *target = buf[0]; + *target = PyUnicode_READ_CHAR(src, 0); } } return 0; @@ -498,7 +496,8 @@ parse_save_field(ReaderObj *self) { PyObject *field; - field = PyUnicode_FromUnicode(self->field, self->field_len); + field = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->field, self->field_len); if (field == NULL) return -1; self->field_len = 0; @@ -507,11 +506,9 @@ parse_save_field(ReaderObj *self) self->numeric_field = 0; tmp = PyNumber_Float(field); - if (tmp == NULL) { - Py_DECREF(field); - return -1; - } Py_DECREF(field); + if (tmp == NULL) + return -1; field = tmp; } PyList_Append(self->fields, field); @@ -526,16 +523,16 @@ parse_grow_buff(ReaderObj *self) self->field_size = 4096; if (self->field != NULL) PyMem_Free(self->field); - self->field = PyMem_New(Py_UNICODE, self->field_size); + self->field = PyMem_New(Py_UCS4, self->field_size); } else { + Py_UCS4 *field = self->field; if (self->field_size > PY_SSIZE_T_MAX / 2) { PyErr_NoMemory(); return 0; } self->field_size *= 2; - self->field = PyMem_Resize(self->field, Py_UNICODE, - self->field_size); + self->field = PyMem_Resize(field, Py_UCS4, self->field_size); } if (self->field == NULL) { PyErr_NoMemory(); @@ -545,7 +542,7 @@ parse_grow_buff(ReaderObj *self) } static int -parse_add_char(ReaderObj *self, Py_UNICODE c) +parse_add_char(ReaderObj *self, Py_UCS4 c) { if (self->field_len >= field_limit) { PyErr_Format(error_obj, "field larger than field limit (%ld)", @@ -559,7 +556,7 @@ parse_add_char(ReaderObj *self, Py_UNICODE c) } static int -parse_process_char(ReaderObj *self, Py_UNICODE c) +parse_process_char(ReaderObj *self, Py_UCS4 c) { DialectObj *dialect = self->dialect; @@ -744,10 +741,12 @@ parse_reset(ReaderObj *self) static PyObject * Reader_iternext(ReaderObj *self) { - PyObject *lineobj; PyObject *fields = NULL; - Py_UNICODE *line, c; - Py_ssize_t linelen; + Py_UCS4 c; + Py_ssize_t pos, linelen; + unsigned int kind; + void *data; + PyObject *lineobj; if (parse_reset(self) < 0) return NULL; @@ -771,14 +770,12 @@ Reader_iternext(ReaderObj *self) return NULL; } ++self->line_num; - line = PyUnicode_AsUnicode(lineobj); - linelen = PyUnicode_GetSize(lineobj); - if (line == NULL || linelen < 0) { - Py_DECREF(lineobj); - return NULL; - } + kind = PyUnicode_KIND(lineobj); + data = PyUnicode_DATA(lineobj); + pos = 0; + linelen = PyUnicode_GET_LENGTH(lineobj); while (linelen--) { - c = *line++; + c = PyUnicode_READ(kind, data, pos); if (c == '\0') { Py_DECREF(lineobj); PyErr_Format(error_obj, @@ -789,6 +786,7 @@ Reader_iternext(ReaderObj *self) Py_DECREF(lineobj); goto err; } + pos++; } Py_DECREF(lineobj); if (parse_process_char(self, 0) < 0) @@ -945,8 +943,9 @@ join_reset(WriterObj *self) * record length. */ static Py_ssize_t -join_append_data(WriterObj *self, Py_UNICODE *field, int quote_empty, - int *quoted, int copy_phase) +join_append_data(WriterObj *self, unsigned int field_kind, void *field_data, + Py_ssize_t field_len, int quote_empty, int *quoted, + int copy_phase) { DialectObj *dialect = self->dialect; int i; @@ -976,13 +975,10 @@ join_append_data(WriterObj *self, Py_UNICODE *field, int quote_empty, /* Copy/count field data */ /* If field is null just pass over */ - for (i = 0; field; i++) { - Py_UNICODE c = field[i]; + for (i = 0; field_data && (i < field_len); i++) { + Py_UCS4 c = PyUnicode_READ(field_kind, field_data, i); int want_escape = 0; - if (c == '\0') - break; - if (c == dialect->delimiter || c == dialect->escapechar || c == dialect->quotechar || @@ -1049,14 +1045,13 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR; if (self->rec != NULL) PyMem_Free(self->rec); - self->rec = PyMem_New(Py_UNICODE, self->rec_size); + self->rec = PyMem_New(Py_UCS4, self->rec_size); } else { - Py_UNICODE* old_rec = self->rec; + Py_UCS4* old_rec = self->rec; self->rec_size = (rec_len / MEM_INCR + 1) * MEM_INCR; - self->rec = PyMem_Resize(self->rec, Py_UNICODE, - self->rec_size); + self->rec = PyMem_Resize(old_rec, Py_UCS4, self->rec_size); if (self->rec == NULL) PyMem_Free(old_rec); } @@ -1069,11 +1064,20 @@ join_check_rec_size(WriterObj *self, Py_ssize_t rec_len) } static int -join_append(WriterObj *self, Py_UNICODE *field, int *quoted, int quote_empty) +join_append(WriterObj *self, PyObject *field, int *quoted, int quote_empty) { + unsigned int field_kind = -1; + void *field_data = NULL; + Py_ssize_t field_len = 0; Py_ssize_t rec_len; - rec_len = join_append_data(self, field, quote_empty, quoted, 0); + if (field != NULL) { + field_kind = PyUnicode_KIND(field); + field_data = PyUnicode_DATA(field); + field_len = PyUnicode_GET_LENGTH(field); + } + rec_len = join_append_data(self, field_kind, field_data, field_len, + quote_empty, quoted, 0); if (rec_len < 0) return 0; @@ -1081,7 +1085,8 @@ join_append(WriterObj *self, Py_UNICODE *field, int *quoted, int quote_empty) if (!join_check_rec_size(self, rec_len)) return 0; - self->rec_len = join_append_data(self, field, quote_empty, quoted, 1); + self->rec_len = join_append_data(self, field_kind, field_data, field_len, + quote_empty, quoted, 1); self->num_fields++; return 1; @@ -1090,10 +1095,11 @@ join_append(WriterObj *self, Py_UNICODE *field, int *quoted, int quote_empty) static int join_append_lineterminator(WriterObj *self) { - Py_ssize_t terminator_len; - Py_UNICODE *terminator; + Py_ssize_t terminator_len, i; + unsigned int term_kind; + void *term_data; - terminator_len = PyUnicode_GetSize(self->dialect->lineterminator); + terminator_len = PyUnicode_GET_LENGTH(self->dialect->lineterminator); if (terminator_len == -1) return 0; @@ -1101,11 +1107,10 @@ join_append_lineterminator(WriterObj *self) if (!join_check_rec_size(self, self->rec_len + terminator_len)) return 0; - terminator = PyUnicode_AsUnicode(self->dialect->lineterminator); - if (terminator == NULL) - return 0; - memmove(self->rec + self->rec_len, terminator, - sizeof(Py_UNICODE)*terminator_len); + term_kind = PyUnicode_KIND(self->dialect->lineterminator); + term_data = PyUnicode_DATA(self->dialect->lineterminator); + for (i = 0; i < terminator_len; i++) + self->rec[self->rec_len + i] = PyUnicode_READ(term_kind, term_data, i); self->rec_len += terminator_len; return 1; @@ -1122,6 +1127,7 @@ csv_writerow(WriterObj *self, PyObject *seq) { DialectObj *dialect = self->dialect; Py_ssize_t len, i; + PyObject *line, *result; if (!PySequence_Check(seq)) return PyErr_Format(error_obj, "sequence expected"); @@ -1155,14 +1161,11 @@ csv_writerow(WriterObj *self, PyObject *seq) } if (PyUnicode_Check(field)) { - append_ok = join_append(self, - PyUnicode_AS_UNICODE(field), - "ed, len == 1); + append_ok = join_append(self, field, "ed, len == 1); Py_DECREF(field); } else if (field == Py_None) { - append_ok = join_append(self, NULL, - "ed, len == 1); + append_ok = join_append(self, NULL, "ed, len == 1); Py_DECREF(field); } else { @@ -1172,9 +1175,7 @@ csv_writerow(WriterObj *self, PyObject *seq) Py_DECREF(field); if (str == NULL) return NULL; - append_ok = join_append(self, - PyUnicode_AS_UNICODE(str), - "ed, len == 1); + append_ok = join_append(self, str, "ed, len == 1); Py_DECREF(str); } if (!append_ok) @@ -1186,9 +1187,13 @@ csv_writerow(WriterObj *self, PyObject *seq) if (!join_append_lineterminator(self)) return 0; - return PyObject_CallFunction(self->writeline, - "(u#)", self->rec, - self->rec_len); + line = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (void *) self->rec, self->rec_len); + if (line == NULL) + return NULL; + result = PyObject_CallFunctionObjArgs(self->writeline, line, NULL); + Py_DECREF(line); + return result; } PyDoc_STRVAR(csv_writerows_doc, @@ -1312,6 +1317,7 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) { PyObject * output_file, * dialect = NULL; WriterObj * self = PyObject_GC_New(WriterObj, &Writer_Type); + _Py_identifier(write); if (!self) return NULL; @@ -1328,7 +1334,7 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) Py_DECREF(self); return NULL; } - self->writeline = PyObject_GetAttrString(output_file, "write"); + self->writeline = _PyObject_GetAttrId(output_file, &PyId_write); if (self->writeline == NULL || !PyCallable_Check(self->writeline)) { PyErr_SetString(PyExc_TypeError, "argument 1 must have a \"write\" method"); diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8f378e2..1d7cf94 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1843,11 +1843,9 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } if (PyUnicode_Check(proto)) { - PyObject *v = _PyUnicode_AsDefaultEncodedString(proto, NULL); - if (!v) + proto_str = PyUnicode_AsUTF8AndSize(proto, &proto_len); + if (!proto_str) goto error; - proto_str = PyBytes_AS_STRING(v); - proto_len = PyBytes_GET_SIZE(v); } else { PyErr_SetString(PyExc_TypeError, "class must define a '_type_' string attribute"); @@ -3681,8 +3679,10 @@ _build_result(PyObject *result, PyObject *callargs, PyTuple_SET_ITEM(tup, index, v); index++; } else if (bit & outmask) { + _Py_identifier(__ctypes_from_outparam__); + v = PyTuple_GET_ITEM(callargs, i); - v = PyObject_CallMethod(v, "__ctypes_from_outparam__", NULL); + v = _PyObject_CallMethodId(v, &PyId___ctypes_from_outparam__, NULL); if (v == NULL || numretvals == 1) { Py_DECREF(callargs); return v; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 1f3c1c0..adc2870 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -658,13 +658,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) #ifdef CTYPES_UNICODE if (PyUnicode_Check(obj)) { -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - pa->ffi_type = &ffi_type_pointer; - pa->value.p = PyUnicode_AS_UNICODE(obj); - Py_INCREF(obj); - pa->keep = obj; - return 0; -#else pa->ffi_type = &ffi_type_pointer; pa->value.p = PyUnicode_AsWideCharString(obj, NULL); if (pa->value.p == NULL) @@ -675,7 +668,6 @@ static int ConvParam(PyObject *obj, Py_ssize_t index, struct argument *pa) return -1; } return 0; -#endif } #endif @@ -952,9 +944,9 @@ void _ctypes_extend_error(PyObject *exc_class, char *fmt, ...) else { PyErr_Clear(); PyUnicode_AppendAndDel(&s, PyUnicode_FromString("???")); - if (s == NULL) - goto error; } + if (s == NULL) + goto error; PyErr_SetObject(exc_class, s); error: Py_XDECREF(tp); @@ -1695,13 +1687,15 @@ unpickle(PyObject *self, PyObject *args) PyObject *state; PyObject *result; PyObject *tmp; + _Py_identifier(__new__); + _Py_identifier(__setstate__); if (!PyArg_ParseTuple(args, "OO", &typ, &state)) return NULL; - result = PyObject_CallMethod(typ, "__new__", "O", typ); + result = _PyObject_CallMethodId(typ, &PyId___new__, "O", typ); if (result == NULL) return NULL; - tmp = PyObject_CallMethod(result, "__setstate__", "O", state); + tmp = _PyObject_CallMethodId(result, &PyId___setstate__, "O", state); if (tmp == NULL) { Py_DECREF(result); return NULL; diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index ccaa3c9..d324ca9 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -9,7 +9,6 @@ #define CTYPES_CFIELD_CAPSULE_NAME_PYMEM "_ctypes/cfield.c pymem" -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T static void pymem_destructor(PyObject *ptr) { void *p = PyCapsule_GetPointer(ptr, CTYPES_CFIELD_CAPSULE_NAME_PYMEM); @@ -17,7 +16,6 @@ static void pymem_destructor(PyObject *ptr) PyMem_Free(p); } } -#endif /******************************************************************/ @@ -52,7 +50,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, { CFieldObject *self; PyObject *proto; - Py_ssize_t size, align, length; + Py_ssize_t size, align; SETFUNC setfunc = NULL; GETFUNC getfunc = NULL; StgDictObject *dict; @@ -106,7 +104,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, } size = dict->size; - length = dict->length; proto = desc; /* Field descriptors for 'c_char * n' are be scpecial cased to @@ -1239,32 +1236,24 @@ u_get(void *ptr, Py_ssize_t size) static PyObject * U_get(void *ptr, Py_ssize_t size) { - PyObject *result; Py_ssize_t len; - Py_UNICODE *p; + wchar_t *p; size /= sizeof(wchar_t); /* we count character units here, not bytes */ - result = PyUnicode_FromWideChar((wchar_t *)ptr, size); - if (!result) - return NULL; /* We need 'result' to be able to count the characters with wcslen, since ptr may not be NUL terminated. If the length is smaller (if it was actually NUL terminated, we construct a new one and throw away the result. */ /* chop off at the first NUL character, if any. */ - p = PyUnicode_AS_UNICODE(result); - for (len = 0; len < size; ++len) + p = (wchar_t*)ptr; + for (len = 0; len < size; ++len) { if (!p[len]) break; - - if (len < size) { - PyObject *ob = PyUnicode_FromWideChar((wchar_t *)ptr, len); - Py_DECREF(result); - return ob; } - return result; + + return PyUnicode_FromWideChar((wchar_t *)ptr, len); } static PyObject * @@ -1402,6 +1391,9 @@ z_get(void *ptr, Py_ssize_t size) static PyObject * Z_set(void *ptr, PyObject *value, Py_ssize_t size) { + PyObject *keep; + wchar_t *buffer; + if (value == Py_None) { *(wchar_t **)ptr = NULL; Py_INCREF(value); @@ -1421,37 +1413,20 @@ Z_set(void *ptr, PyObject *value, Py_ssize_t size) "unicode string or integer address expected instead of %s instance", value->ob_type->tp_name); return NULL; - } else - Py_INCREF(value); -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - /* We can copy directly. Hm, are unicode objects always NUL - terminated in Python, internally? - */ - *(wchar_t **)ptr = (wchar_t *) PyUnicode_AS_UNICODE(value); - return value; -#else - { - /* We must create a wchar_t* buffer from the unicode object, - and keep it alive */ - PyObject *keep; - wchar_t *buffer; - - buffer = PyUnicode_AsWideCharString(value, NULL); - if (!buffer) { - Py_DECREF(value); - return NULL; - } - keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); - if (!keep) { - Py_DECREF(value); - PyMem_Free(buffer); - return NULL; - } - *(wchar_t **)ptr = (wchar_t *)buffer; - Py_DECREF(value); - return keep; } -#endif + + /* We must create a wchar_t* buffer from the unicode object, + and keep it alive */ + buffer = PyUnicode_AsWideCharString(value, NULL); + if (!buffer) + return NULL; + keep = PyCapsule_New(buffer, CTYPES_CFIELD_CAPSULE_NAME_PYMEM, pymem_destructor); + if (!keep) { + PyMem_Free(buffer); + return NULL; + } + *(wchar_t **)ptr = buffer; + return keep; } static PyObject * diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 262d0b4..e5b0e4c 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -426,9 +426,9 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct StgDictObject *dict; int bitsize = 0; - if (!pair || !PyArg_ParseTuple(pair, "OO|i", &name, &desc, &bitsize)) { - PyErr_SetString(PyExc_AttributeError, - "'_fields_' must be a sequence of pairs"); + if (!pair || !PyArg_ParseTuple(pair, "UO|i", &name, &desc, &bitsize)) { + PyErr_SetString(PyExc_TypeError, + "'_fields_' must be a sequence of (name, C type) pairs"); Py_XDECREF(pair); return -1; } @@ -478,6 +478,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct } } else bitsize = 0; + if (isStruct && !isPacked) { char *fieldfmt = dict->format ? dict->format : "B"; char *fieldname = _PyUnicode_AsString(name); @@ -487,10 +488,6 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct if (fieldname == NULL) { - PyErr_Format(PyExc_TypeError, - "structure field name must be string not %s", - name->ob_type->tp_name); - Py_DECREF(pair); return -1; } @@ -515,6 +512,7 @@ PyCStructUnionType_update_stgdict(PyObject *type, PyObject *fields, int isStruct return -1; } } + if (isStruct) { prop = PyCField_FromDesc(desc, i, &field_size, bitsize, &bitofs, diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 092fb69..7c89acc 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -105,10 +105,6 @@ char *PyCursesVersion = "2.2"; #include "Python.h" -#ifdef __osf__ -#define STRICT_SYSV_CURSES /* Don't use ncurses extensions */ -#endif - #ifdef __hpux #define STRICT_SYSV_CURSES #endif @@ -207,8 +203,11 @@ PyCurses_ConvertToChtype(PyObject *obj, chtype *ch) } else if(PyBytes_Check(obj) && (PyBytes_Size(obj) == 1)) { *ch = (chtype) *PyBytes_AsString(obj); - } else if (PyUnicode_Check(obj) && PyUnicode_GetSize(obj) == 1) { - *ch = (chtype) *PyUnicode_AS_UNICODE(obj); + } else if (PyUnicode_Check(obj) && PyUnicode_GET_LENGTH(obj) == 1) { + Py_UCS4 ucs = PyUnicode_READ(PyUnicode_KIND(obj), + PyUnicode_DATA(obj), + 0); + *ch = (chtype)ucs; } else { return 0; } @@ -910,6 +909,40 @@ PyCursesWindow_GetKey(PyCursesWindowObject *self, PyObject *args) } } +#ifdef HAVE_NCURSESW +static PyObject * +PyCursesWindow_Get_WCh(PyCursesWindowObject *self, PyObject *args) +{ + int x, y; + int ct; + wint_t rtn; + + switch (PyTuple_Size(args)) { + case 0: + Py_BEGIN_ALLOW_THREADS + ct = wget_wch(self->win,&rtn); + Py_END_ALLOW_THREADS + break; + case 2: + if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) + return NULL; + Py_BEGIN_ALLOW_THREADS + ct = mvwget_wch(self->win,y,x,&rtn); + Py_END_ALLOW_THREADS + break; + default: + PyErr_SetString(PyExc_TypeError, "get_wch requires 0 or 2 arguments"); + return NULL; + } + if (ct == ERR) { + /* get_wch() returns ERR in nodelay mode */ + PyErr_SetString(PyCursesError, "no input"); + return NULL; + } + return PyLong_FromLong(rtn); +} +#endif + static PyObject * PyCursesWindow_GetStr(PyCursesWindowObject *self, PyObject *args) { @@ -1385,10 +1418,12 @@ PyCursesWindow_PutWin(PyCursesWindowObject *self, PyObject *stream) while (1) { char buf[BUFSIZ]; Py_ssize_t n = fread(buf, 1, BUFSIZ, fp); + _Py_identifier(write); + if (n <= 0) break; Py_DECREF(res); - res = PyObject_CallMethod(stream, "write", "y#", buf, n); + res = _PyObject_CallMethodId(stream, &PyId_write, "y#", buf, n); if (res == NULL) break; } @@ -1608,6 +1643,9 @@ static PyMethodDef PyCursesWindow_Methods[] = { {"getbkgd", (PyCFunction)PyCursesWindow_GetBkgd, METH_NOARGS}, {"getch", (PyCFunction)PyCursesWindow_GetCh, METH_VARARGS}, {"getkey", (PyCFunction)PyCursesWindow_GetKey, METH_VARARGS}, +#ifdef HAVE_NCURSESW + {"get_wch", (PyCFunction)PyCursesWindow_Get_WCh, METH_VARARGS}, +#endif {"getmaxyx", (PyCFunction)PyCursesWindow_getmaxyx, METH_NOARGS}, {"getparyx", (PyCFunction)PyCursesWindow_getparyx, METH_NOARGS}, {"getstr", (PyCFunction)PyCursesWindow_GetStr, METH_VARARGS}, @@ -1875,6 +1913,7 @@ PyCurses_GetWin(PyCursesWindowObject *self, PyObject *stream) WINDOW *win; PyCursesInitialised; + _Py_identifier(read); strcpy(fn, "/tmp/py.curses.getwin.XXXXXX"); fd = mkstemp(fn); @@ -1886,7 +1925,7 @@ PyCurses_GetWin(PyCursesWindowObject *self, PyObject *stream) remove(fn); return PyErr_SetFromErrnoWithFilename(PyExc_IOError, fn); } - data = PyObject_CallMethod(stream, "read", ""); + data = _PyObject_CallMethodId(stream, &PyId_read, ""); if (data == NULL) { fclose(fp); remove(fn); @@ -2663,6 +2702,71 @@ PyCurses_UngetCh(PyObject *self, PyObject *args) return PyCursesCheckERR(ungetch(ch), "ungetch"); } +#ifdef HAVE_NCURSESW +/* Convert an object to a character (wchar_t): + + - int + - str of length 1 + + Return 1 on success, 0 on error. */ +static int +PyCurses_ConvertToWchar_t(PyObject *obj, + wchar_t *wch) +{ + if (PyUnicode_Check(obj)) { + wchar_t buffer[2]; + if (PyUnicode_AsWideChar(obj, buffer, 2) != 1) { + PyErr_Format(PyExc_TypeError, + "expect bytes or str of length 1, or int, " + "got a str of length %zi", + PyUnicode_GET_LENGTH(obj)); + return 0; + } + *wch = buffer[0]; + return 2; + } + else if (PyLong_CheckExact(obj)) { + long value; + int overflow; + value = PyLong_AsLongAndOverflow(obj, &overflow); + if (overflow) { + PyErr_SetString(PyExc_OverflowError, + "int doesn't fit in long"); + return 0; + } + *wch = (wchar_t)value; + if ((long)*wch != value) { + PyErr_Format(PyExc_OverflowError, + "character doesn't fit in wchar_t"); + return 0; + } + return 1; + } + else { + PyErr_Format(PyExc_TypeError, + "expect bytes or str of length 1, or int, got %s", + Py_TYPE(obj)->tp_name); + return 0; + } +} + +static PyObject * +PyCurses_Unget_Wch(PyObject *self, PyObject *args) +{ + PyObject *obj; + wchar_t wch; + + PyCursesInitialised; + + if (!PyArg_ParseTuple(args,"O", &obj)) + return NULL; + + if (!PyCurses_ConvertToWchar_t(obj, &wch)) + return NULL; + return PyCursesCheckERR(unget_wch(wch), "unget_wch"); +} +#endif + static PyObject * PyCurses_Use_Env(PyObject *self, PyObject *args) { @@ -2790,6 +2894,9 @@ static PyMethodDef PyCurses_methods[] = { {"typeahead", (PyCFunction)PyCurses_TypeAhead, METH_VARARGS}, {"unctrl", (PyCFunction)PyCurses_UnCtrl, METH_VARARGS}, {"ungetch", (PyCFunction)PyCurses_UngetCh, METH_VARARGS}, +#ifdef HAVE_NCURSESW + {"unget_wch", (PyCFunction)PyCurses_Unget_Wch, METH_VARARGS}, +#endif {"use_env", (PyCFunction)PyCurses_Use_Env, METH_VARARGS}, #ifndef STRICT_SYSV_CURSES {"use_default_colors", (PyCFunction)PyCurses_Use_Default_Colors, METH_NOARGS}, diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 6ee5317..4dceaa9 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -766,7 +766,7 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; -/* The interned UTC timezone instance */ +/* The interned UTC timezone instance */ static PyObject *PyDateTime_TimeZone_UTC; /* Create new timezone instance checking offset range. This @@ -946,6 +946,7 @@ static PyObject * call_tzname(PyObject *tzinfo, PyObject *tzinfoarg) { PyObject *result; + _Py_identifier(tzname); assert(tzinfo != NULL); assert(check_tzinfo_subclass(tzinfo) >= 0); @@ -954,7 +955,7 @@ call_tzname(PyObject *tzinfo, PyObject *tzinfoarg) if (tzinfo == Py_None) Py_RETURN_NONE; - result = PyObject_CallMethod(tzinfo, "tzname", "O", tzinfoarg); + result = _PyObject_CallMethodId(tzinfo, &PyId_tzname, "O", tzinfoarg); if (result == NULL || result == Py_None) return result; @@ -985,9 +986,8 @@ append_keyword_tzinfo(PyObject *repr, PyObject *tzinfo) if (tzinfo == Py_None) return repr; /* Get rid of the trailing ')'. */ - assert(PyUnicode_AS_UNICODE(repr)[PyUnicode_GET_SIZE(repr)-1] == ')'); - temp = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(repr), - PyUnicode_GET_SIZE(repr) - 1); + assert(PyUnicode_READ_CHAR(repr, PyUnicode_GET_LENGTH(repr)-1) == ')'); + temp = PyUnicode_Substring(repr, 0, PyUnicode_GET_LENGTH(repr) - 1); Py_DECREF(repr); if (temp == NULL) return NULL; @@ -1079,6 +1079,8 @@ make_Zreplacement(PyObject *object, PyObject *tzinfoarg) PyObject *temp; PyObject *tzinfo = get_tzinfo_member(object); PyObject *Zreplacement = PyUnicode_FromStringAndSize(NULL, 0); + _Py_identifier(replace); + if (Zreplacement == NULL) return NULL; if (tzinfo == Py_None || tzinfo == NULL) @@ -1099,7 +1101,7 @@ make_Zreplacement(PyObject *object, PyObject *tzinfoarg) * strftime doesn't treat them as format codes. */ Py_DECREF(Zreplacement); - Zreplacement = PyObject_CallMethod(temp, "replace", "ss", "%", "%%"); + Zreplacement = _PyObject_CallMethodId(temp, &PyId_replace, "ss", "%", "%%"); Py_DECREF(temp); if (Zreplacement == NULL) return NULL; @@ -1166,31 +1168,6 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, if (!pin) return NULL; - /* Give up if the year is before 1000. - * Python strftime() plays games with the year, and different - * games depending on whether envar PYTHON2K is set. This makes - * years before 1000 a nightmare, even if the platform strftime - * supports them (and not all do). - * We could get a lot farther here by avoiding Python's strftime - * wrapper and calling the C strftime() directly, but that isn't - * an option in the Python implementation of this module. - */ - { - long year; - PyObject *pyyear = PySequence_GetItem(timetuple, 0); - if (pyyear == NULL) return NULL; - assert(PyLong_Check(pyyear)); - year = PyLong_AsLong(pyyear); - Py_DECREF(pyyear); - if (year < 1000) { - PyErr_Format(PyExc_ValueError, "year=%ld is before " - "1000; the datetime strftime() " - "methods require year >= 1000", - year); - return NULL; - } - } - /* Scan the input format, looking for %z/%Z/%f escapes, building * a new format. Since computing the replacements for those codes * is expensive, don't unless they're actually used. @@ -1307,12 +1284,15 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, { PyObject *format; PyObject *time = PyImport_ImportModuleNoBlock("time"); + if (time == NULL) goto Done; format = PyUnicode_FromString(PyBytes_AS_STRING(newfmt)); if (format != NULL) { - result = PyObject_CallMethod(time, "strftime", "OO", - format, timetuple, NULL); + _Py_identifier(strftime); + + result = _PyObject_CallMethodId(time, &PyId_strftime, "OO", + format, timetuple, NULL); Py_DECREF(format); } Py_DECREF(time); @@ -1338,7 +1318,9 @@ time_time(void) PyObject *time = PyImport_ImportModuleNoBlock("time"); if (time != NULL) { - result = PyObject_CallMethod(time, "time", "()"); + _Py_identifier(time); + + result = _PyObject_CallMethodId(time, &PyId_time, "()"); Py_DECREF(time); } return result; @@ -1355,13 +1337,15 @@ build_struct_time(int y, int m, int d, int hh, int mm, int ss, int dstflag) time = PyImport_ImportModuleNoBlock("time"); if (time != NULL) { - result = PyObject_CallMethod(time, "struct_time", - "((iiiiiiiii))", - y, m, d, - hh, mm, ss, - weekday(y, m, d), - days_before_month(y, m) + d, - dstflag); + _Py_identifier(struct_time); + + result = _PyObject_CallMethodId(time, &PyId_struct_time, + "((iiiiiiiii))", + y, m, d, + hh, mm, ss, + weekday(y, m, d), + days_before_month(y, m) + d, + dstflag); Py_DECREF(time); } return result; @@ -1461,7 +1445,7 @@ delta_to_microseconds(PyDateTime_Delta *self) goto Done; Py_DECREF(x1); Py_DECREF(x2); - x1 = x2 = NULL; + /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ x1 = PyNumber_Multiply(x3, us_per_second); /* us */ @@ -1594,11 +1578,12 @@ multiply_float_timedelta(PyObject *floatobj, PyDateTime_Delta *delta) PyObject *result = NULL; PyObject *pyus_in = NULL, *temp, *pyus_out; PyObject *ratio = NULL; + _Py_identifier(as_integer_ratio); pyus_in = delta_to_microseconds(delta); if (pyus_in == NULL) return NULL; - ratio = PyObject_CallMethod(floatobj, "as_integer_ratio", NULL); + ratio = _PyObject_CallMethodId(floatobj, &PyId_as_integer_ratio, NULL); if (ratio == NULL) goto error; temp = PyNumber_Multiply(pyus_in, PyTuple_GET_ITEM(ratio, 0)); @@ -1692,11 +1677,12 @@ truedivide_timedelta_float(PyDateTime_Delta *delta, PyObject *f) PyObject *result = NULL; PyObject *pyus_in = NULL, *temp, *pyus_out; PyObject *ratio = NULL; + _Py_identifier(as_integer_ratio); pyus_in = delta_to_microseconds(delta); if (pyus_in == NULL) return NULL; - ratio = PyObject_CallMethod(f, "as_integer_ratio", NULL); + ratio = _PyObject_CallMethodId(f, &PyId_as_integer_ratio, NULL); if (ratio == NULL) goto error; temp = PyNumber_Multiply(pyus_in, PyTuple_GET_ITEM(ratio, 1)); @@ -1837,8 +1823,7 @@ delta_richcompare(PyObject *self, PyObject *other, int op) return diff_to_bool(diff, op); } else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } } @@ -1936,10 +1921,8 @@ delta_remainder(PyObject *left, PyObject *right) PyObject *pyus_remainder; PyObject *remainder; - if (!PyDelta_Check(left) || !PyDelta_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyDelta_Check(left) || !PyDelta_Check(right)) + Py_RETURN_NOTIMPLEMENTED; pyus_left = delta_to_microseconds((PyDateTime_Delta *)left); if (pyus_left == NULL) @@ -1974,10 +1957,8 @@ delta_divmod(PyObject *left, PyObject *right) PyObject *delta; PyObject *result; - if (!PyDelta_Check(left) || !PyDelta_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyDelta_Check(left) || !PyDelta_Check(right)) + Py_RETURN_NOTIMPLEMENTED; pyus_left = delta_to_microseconds((PyDateTime_Delta *)left); if (pyus_left == NULL) @@ -2492,6 +2473,7 @@ date_today(PyObject *cls, PyObject *dummy) { PyObject *time; PyObject *result; + _Py_identifier(fromtimestamp); time = time_time(); if (time == NULL) @@ -2503,7 +2485,7 @@ date_today(PyObject *cls, PyObject *dummy) * time.time() delivers; if someone were gonzo about optimization, * date.today() could get away with plain C time(). */ - result = PyObject_CallMethod(cls, "fromtimestamp", "O", time); + result = _PyObject_CallMethodId(cls, &PyId_fromtimestamp, "O", time); Py_DECREF(time); return result; } @@ -2571,10 +2553,9 @@ add_date_timedelta(PyDateTime_Date *date, PyDateTime_Delta *delta, int negate) static PyObject * date_add(PyObject *left, PyObject *right) { - if (PyDateTime_Check(left) || PyDateTime_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (PyDateTime_Check(left) || PyDateTime_Check(right)) + Py_RETURN_NOTIMPLEMENTED; + if (PyDate_Check(left)) { /* date + ??? */ if (PyDelta_Check(right)) @@ -2593,17 +2574,15 @@ date_add(PyObject *left, PyObject *right) (PyDateTime_Delta *) left, 0); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * date_subtract(PyObject *left, PyObject *right) { - if (PyDateTime_Check(left) || PyDateTime_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (PyDateTime_Check(left) || PyDateTime_Check(right)) + Py_RETURN_NOTIMPLEMENTED; + if (PyDate_Check(left)) { if (PyDate_Check(right)) { /* date - date */ @@ -2622,8 +2601,7 @@ date_subtract(PyObject *left, PyObject *right) 1); } } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } @@ -2648,7 +2626,9 @@ date_isoformat(PyDateTime_Date *self) static PyObject * date_str(PyDateTime_Date *self) { - return PyObject_CallMethod((PyObject *)self, "isoformat", "()"); + _Py_identifier(isoformat); + + return _PyObject_CallMethodId((PyObject *)self, &PyId_isoformat, "()"); } @@ -2667,13 +2647,14 @@ date_strftime(PyDateTime_Date *self, PyObject *args, PyObject *kw) PyObject *result; PyObject *tuple; PyObject *format; + _Py_identifier(timetuple); static char *keywords[] = {"format", NULL}; if (! PyArg_ParseTupleAndKeywords(args, kw, "U:strftime", keywords, &format)) return NULL; - tuple = PyObject_CallMethod((PyObject *)self, "timetuple", "()"); + tuple = _PyObject_CallMethodId((PyObject *)self, &PyId_timetuple, "()"); if (tuple == NULL) return NULL; result = wrap_strftime((PyObject *)self, format, tuple, @@ -2686,6 +2667,7 @@ static PyObject * date_format(PyDateTime_Date *self, PyObject *args) { PyObject *format; + _Py_identifier(strftime); if (!PyArg_ParseTuple(args, "U:__format__", &format)) return NULL; @@ -2694,7 +2676,7 @@ date_format(PyDateTime_Date *self, PyObject *args) if (PyUnicode_GetSize(format) == 0) return PyObject_Str((PyObject *)self); - return PyObject_CallMethod((PyObject *)self, "strftime", "O", format); + return _PyObject_CallMethodId((PyObject *)self, &PyId_strftime, "O", format); } /* ISO methods. */ @@ -2740,10 +2722,8 @@ date_richcompare(PyObject *self, PyObject *other, int op) _PyDateTime_DATE_DATASIZE); return diff_to_bool(diff, op); } - else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + else + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -2782,13 +2762,13 @@ static Py_hash_t generic_hash(unsigned char *data, int len) { register unsigned char *p; - register Py_hash_t x; + register Py_uhash_t x; p = (unsigned char *) data; - x = *p << 7; + x = (Py_uhash_t)*p << 7; while (--len >= 0) - x = (1000003*x) ^ *p++; - x ^= len; + x = (1000003U*x) ^ (Py_uhash_t)*p++; + x ^= (Py_uhash_t)len; if (x == -1) x = -2; @@ -3097,12 +3077,14 @@ tzinfo_reduce(PyObject *self) { PyObject *args, *state, *tmp; PyObject *getinitargs, *getstate; + _Py_identifier(__getinitargs__); + _Py_identifier(__getstate__); tmp = PyTuple_New(0); if (tmp == NULL) return NULL; - getinitargs = PyObject_GetAttrString(self, "__getinitargs__"); + getinitargs = _PyObject_GetAttrId(self, &PyId___getinitargs__); if (getinitargs != NULL) { args = PyObject_CallObject(getinitargs, tmp); Py_DECREF(getinitargs); @@ -3117,7 +3099,7 @@ tzinfo_reduce(PyObject *self) Py_INCREF(args); } - getstate = PyObject_GetAttrString(self, "__getstate__"); + getstate = _PyObject_GetAttrId(self, &PyId___getstate__); if (getstate != NULL) { state = PyObject_CallObject(getstate, tmp); Py_DECREF(getstate); @@ -3240,10 +3222,8 @@ static PyObject * timezone_richcompare(PyDateTime_TimeZone *self, PyDateTime_TimeZone *other, int op) { - if (op != Py_EQ && op != Py_NE) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (op != Py_EQ && op != Py_NE) + Py_RETURN_NOTIMPLEMENTED; return delta_richcompare(self->offset, other->offset, op); } @@ -3288,7 +3268,6 @@ timezone_repr(PyDateTime_TimeZone *self) static PyObject * timezone_str(PyDateTime_TimeZone *self) { - char buf[10]; int hours, minutes, seconds; PyObject *offset; char sign; @@ -3314,11 +3293,9 @@ timezone_str(PyDateTime_TimeZone *self) Py_DECREF(offset); minutes = divmod(seconds, 60, &seconds); hours = divmod(minutes, 60, &minutes); - assert(seconds == 0); /* XXX ignore sub-minute data, curently not allowed. */ - PyOS_snprintf(buf, sizeof(buf), "UTC%c%02d:%02d", sign, hours, minutes); - - return PyUnicode_FromString(buf); + assert(seconds == 0); + return PyUnicode_FromFormat("UTC%c%02d:%02d", sign, hours, minutes); } static PyObject * @@ -3615,7 +3592,9 @@ time_repr(PyDateTime_Time *self) static PyObject * time_str(PyDateTime_Time *self) { - return PyObject_CallMethod((PyObject *)self, "isoformat", "()"); + _Py_identifier(isoformat); + + return _PyObject_CallMethodId((PyObject *)self, &PyId_isoformat, "()"); } static PyObject * @@ -3692,10 +3671,8 @@ time_richcompare(PyObject *self, PyObject *other, int op) PyObject *offset1, *offset2; int diff; - if (! PyTime_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (! PyTime_Check(other)) + Py_RETURN_NOTIMPLEMENTED; if (GET_TIME_TZINFO(self) == GET_TIME_TZINFO(other)) { diff = memcmp(((PyDateTime_Time *)self)->data, @@ -4196,7 +4173,9 @@ datetime_now(PyObject *cls, PyObject *args, PyObject *kw) if (self != NULL && tzinfo != Py_None) { /* Convert UTC to tzinfo's zone. */ PyObject *temp = self; - self = PyObject_CallMethod(tzinfo, "fromutc", "O", self); + _Py_identifier(fromutc); + + self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "O", self); Py_DECREF(temp); } return self; @@ -4233,7 +4212,9 @@ datetime_fromtimestamp(PyObject *cls, PyObject *args, PyObject *kw) if (self != NULL && tzinfo != Py_None) { /* Convert UTC to tzinfo's zone. */ PyObject *temp = self; - self = PyObject_CallMethod(tzinfo, "fromutc", "O", self); + _Py_identifier(fromutc); + + self = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "O", self); Py_DECREF(temp); } return self; @@ -4257,9 +4238,10 @@ static PyObject * datetime_strptime(PyObject *cls, PyObject *args) { static PyObject *module = NULL; - const Py_UNICODE *string, *format; + PyObject *string, *format; + _Py_identifier(_strptime_datetime); - if (!PyArg_ParseTuple(args, "uu:strptime", &string, &format)) + if (!PyArg_ParseTuple(args, "UU:strptime", &string, &format)) return NULL; if (module == NULL) { @@ -4267,8 +4249,8 @@ datetime_strptime(PyObject *cls, PyObject *args) if (module == NULL) return NULL; } - return PyObject_CallMethod(module, "_strptime_datetime", "Ouu", - cls, string, format); + return _PyObject_CallMethodId(module, &PyId__strptime_datetime, "OOO", + cls, string, format); } /* Return new datetime from date/datetime and time arguments. */ @@ -4384,8 +4366,7 @@ datetime_add(PyObject *left, PyObject *right) (PyDateTime_Delta *) left, 1); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -4514,7 +4495,9 @@ datetime_repr(PyDateTime_DateTime *self) static PyObject * datetime_str(PyDateTime_DateTime *self) { - return PyObject_CallMethod((PyObject *)self, "isoformat", "(s)", " "); + _Py_identifier(isoformat); + + return _PyObject_CallMethodId((PyObject *)self, &PyId_isoformat, "(s)", " "); } static PyObject * @@ -4587,8 +4570,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_TRUE; return cmperror(self, other); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } if (GET_DT_TZINFO(self) == GET_DT_TZINFO(other)) { @@ -4722,6 +4704,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) PyObject *offset; PyObject *temp; PyObject *tzinfo; + _Py_identifier(fromutc); static char *keywords[] = {"tz", NULL}; if (! PyArg_ParseTupleAndKeywords(args, kw, "O!:astimezone", keywords, @@ -4763,7 +4746,7 @@ datetime_astimezone(PyDateTime_DateTime *self, PyObject *args, PyObject *kw) Py_DECREF(temp); temp = result; - result = PyObject_CallMethod(tzinfo, "fromutc", "O", temp); + result = _PyObject_CallMethodId(tzinfo, &PyId_fromutc, "O", temp); Py_DECREF(temp); return result; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 827acce..327b873 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -212,6 +212,7 @@ dbm_contains(PyObject *self, PyObject *arg) { dbmobject *dp = (dbmobject *)self; datum key, val; + Py_ssize_t size; if ((dp)->di_dbm == NULL) { PyErr_SetString(DbmError, @@ -219,8 +220,9 @@ dbm_contains(PyObject *self, PyObject *arg) return -1; } if (PyUnicode_Check(arg)) { - arg = _PyUnicode_AsDefaultEncodedString(arg, NULL); - if (arg == NULL) + key.dptr = PyUnicode_AsUTF8AndSize(arg, &size); + key.dsize = size; + if (key.dptr == NULL) return -1; } if (!PyBytes_Check(arg)) { @@ -229,8 +231,10 @@ dbm_contains(PyObject *self, PyObject *arg) arg->ob_type->tp_name); return -1; } - key.dptr = PyBytes_AS_STRING(arg); - key.dsize = PyBytes_GET_SIZE(arg); + else { + key.dptr = PyBytes_AS_STRING(arg); + key.dsize = PyBytes_GET_SIZE(arg); + } val = dbm_fetch(dp->di_dbm, key); return val.dptr != NULL; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 3061d8e..fcd1271 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -226,7 +226,7 @@ typedef struct { PyObject* *children; PyObject* _children[STATIC_CHILDREN]; - + } ElementObjectExtra; typedef struct { @@ -354,14 +354,14 @@ element_resize(ElementObject* self, int extra) /* use Python 2.4's list growth strategy */ size = (size >> 3) + (size < 9 ? 3 : 6) + size; /* Coverity CID #182 size_error: Allocating 1 bytes to pointer "children" - * which needs at least 4 bytes. - * Although it's a false alarm always assume at least one child to + * which needs at least 4 bytes. + * Although it's a false alarm always assume at least one child to * be safe. */ size = size ? size : 1; if (self->extra->children != self->extra->_children) { /* Coverity CID #182 size_error: Allocating 1 bytes to pointer - * "children", which needs at least 4 bytes. Although it's a + * "children", which needs at least 4 bytes. Although it's a * false alarm always assume at least one child to be safe. */ children = PyObject_Realloc(self->extra->children, @@ -606,7 +606,7 @@ element_copy(ElementObject* self, PyObject* args) Py_INCREF(JOIN_OBJ(element->tail)); if (self->extra) { - + if (element_resize(element, self->extra->length) < 0) { Py_DECREF(element); return NULL; @@ -618,7 +618,7 @@ element_copy(ElementObject* self, PyObject* args) } element->extra->length = self->extra->length; - + } return (PyObject*) element; @@ -661,7 +661,7 @@ element_deepcopy(ElementObject* self, PyObject* args) if (!element) return NULL; - + text = deepcopy(JOIN_OBJ(self->text), memo); if (!text) goto error; @@ -675,7 +675,7 @@ element_deepcopy(ElementObject* self, PyObject* args) element->tail = JOIN_SET(tail, JOIN_GET(self->tail)); if (self->extra) { - + if (element_resize(element, self->extra->length) < 0) goto error; @@ -689,7 +689,7 @@ element_deepcopy(ElementObject* self, PyObject* args) } element->extra->length = self->extra->length; - + } /* add object to memo dictionary (so deepcopy won't visit it again) */ @@ -723,13 +723,16 @@ checkpath(PyObject* tag) (ch == '/' || ch == '*' || ch == '[' || ch == '@' || ch == '.') if (PyUnicode_Check(tag)) { - Py_UNICODE *p = PyUnicode_AS_UNICODE(tag); - for (i = 0; i < PyUnicode_GET_SIZE(tag); i++) { - if (p[i] == '{') + const Py_ssize_t len = PyUnicode_GET_LENGTH(tag); + void *data = PyUnicode_DATA(tag); + unsigned int kind = PyUnicode_KIND(tag); + for (i = 0; i < len; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch == '{') check = 0; - else if (p[i] == '}') + else if (ch == '}') check = 1; - else if (check && PATHCHAR(p[i])) + else if (check && PATHCHAR(ch)) return 1; } return 0; @@ -787,20 +790,22 @@ static PyObject* element_find(ElementObject* self, PyObject* args) { int i; - PyObject* tag; PyObject* namespaces = Py_None; + if (!PyArg_ParseTuple(args, "O|O:find", &tag, &namespaces)) return NULL; - if (checkpath(tag) || namespaces != Py_None) - return PyObject_CallMethod( - elementpath_obj, "find", "OOO", self, tag, namespaces + if (checkpath(tag) || namespaces != Py_None) { + _Py_identifier(find); + return _PyObject_CallMethodId( + elementpath_obj, &PyId_find, "OOO", self, tag, namespaces ); + } if (!self->extra) Py_RETURN_NONE; - + for (i = 0; i < self->extra->length; i++) { PyObject* item = self->extra->children[i]; if (Element_CheckExact(item) && @@ -817,16 +822,17 @@ static PyObject* element_findtext(ElementObject* self, PyObject* args) { int i; - PyObject* tag; PyObject* default_value = Py_None; PyObject* namespaces = Py_None; + _Py_identifier(findtext); + if (!PyArg_ParseTuple(args, "O|OO:findtext", &tag, &default_value, &namespaces)) return NULL; if (checkpath(tag) || namespaces != Py_None) - return PyObject_CallMethod( - elementpath_obj, "findtext", "OOOO", self, tag, default_value, namespaces + return _PyObject_CallMethodId( + elementpath_obj, &PyId_findtext, "OOOO", self, tag, default_value, namespaces ); if (!self->extra) { @@ -855,16 +861,18 @@ element_findall(ElementObject* self, PyObject* args) { int i; PyObject* out; - PyObject* tag; PyObject* namespaces = Py_None; + if (!PyArg_ParseTuple(args, "O|O:findall", &tag, &namespaces)) return NULL; - if (checkpath(tag) || namespaces != Py_None) - return PyObject_CallMethod( - elementpath_obj, "findall", "OOO", self, tag, namespaces + if (checkpath(tag) || namespaces != Py_None) { + _Py_identifier(findall); + return _PyObject_CallMethodId( + elementpath_obj, &PyId_findall, "OOO", self, tag, namespaces ); + } out = PyList_New(0); if (!out) @@ -892,11 +900,13 @@ element_iterfind(ElementObject* self, PyObject* args) { PyObject* tag; PyObject* namespaces = Py_None; + _Py_identifier(iterfind); + if (!PyArg_ParseTuple(args, "O|O:iterfind", &tag, &namespaces)) return NULL; - return PyObject_CallMethod( - elementpath_obj, "iterfind", "OOO", self, tag, namespaces + return _PyObject_CallMethodId( + elementpath_obj, &PyId_iterfind, "OOO", self, tag, namespaces ); } @@ -953,7 +963,7 @@ static PyObject* element_iter(ElementObject* self, PyObject* args) { PyObject* result; - + PyObject* tag = Py_None; if (!PyArg_ParseTuple(args, "|O:iter", &tag)) return NULL; @@ -985,7 +995,7 @@ static PyObject* element_itertext(ElementObject* self, PyObject* args) { PyObject* result; - + if (!PyArg_ParseTuple(args, ":itertext")) return NULL; @@ -1483,7 +1493,7 @@ element_getattro(ElementObject* self, PyObject* nameobj) if (PyUnicode_Check(nameobj)) name = _PyUnicode_AsString(nameobj); - + if (name == NULL) return NULL; @@ -2113,7 +2123,7 @@ makeuniversal(XMLParserObject* self, const char* string) Py_INCREF(key); tag = key; } - + /* decode universal name */ p = PyBytes_AS_STRING(tag); value = PyUnicode_DecodeUTF8(p, size, "strict"); @@ -2138,13 +2148,15 @@ makeuniversal(XMLParserObject* self, const char* string) static void expat_set_error(const char* message, int line, int column) { - PyObject *error; - PyObject *position; - char buffer[256]; + PyObject *errmsg, *error, *position; - sprintf(buffer, "%.100s: line %d, column %d", message, line, column); + errmsg = PyUnicode_FromFormat("%s: line %d, column %d", + message, line, column); + if (errmsg == NULL) + return; - error = PyObject_CallFunction(elementtree_parseerror_obj, "s", buffer); + error = PyObject_CallFunction(elementtree_parseerror_obj, "O", errmsg); + Py_DECREF(errmsg); if (!error) return; @@ -2399,29 +2411,33 @@ expat_unknown_encoding_handler(XMLParserObject *self, const XML_Char *name, XML_Encoding *info) { PyObject* u; - Py_UNICODE* p; unsigned char s[256]; int i; + void *data; + unsigned int kind; memset(info, 0, sizeof(XML_Encoding)); for (i = 0; i < 256; i++) s[i] = i; - + u = PyUnicode_Decode((char*) s, 256, name, "replace"); if (!u) return XML_STATUS_ERROR; + if (PyUnicode_READY(u)) + return XML_STATUS_ERROR; - if (PyUnicode_GET_SIZE(u) != 256) { + if (PyUnicode_GET_LENGTH(u) != 256) { Py_DECREF(u); return XML_STATUS_ERROR; } - p = PyUnicode_AS_UNICODE(u); - + kind = PyUnicode_KIND(u); + data = PyUnicode_DATA(u); for (i = 0; i < 256; i++) { - if (p[i] != Py_UNICODE_REPLACEMENT_CHARACTER) - info->map[i] = p[i]; + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if (ch != Py_UNICODE_REPLACEMENT_CHARACTER) + info->map[i] = ch; else info->map[i] = -1; } @@ -2466,7 +2482,7 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) PyObject_Del(self); return NULL; } - + self->names = PyDict_New(); if (!self->names) { PyObject_Del(self->entity); @@ -2645,7 +2661,7 @@ xmlparser_parse(XMLParserObject* self, PyObject* args) reader = PyObject_GetAttrString(fileobj, "read"); if (!reader) return NULL; - + /* read from open file object */ for (;;) { @@ -2796,7 +2812,7 @@ static PyMethodDef xmlparser_methods[] = { {NULL, NULL} }; -static PyObject* +static PyObject* xmlparser_getattro(XMLParserObject* self, PyObject* nameobj) { if (PyUnicode_Check(nameobj)) { @@ -2957,7 +2973,7 @@ PyInit__elementtree(void) " break\n" " parser.feed(data)\n" " self._root = parser.close()\n" - " else:\n" + " else:\n" " parser = cElementTree.XMLParser()\n" " self._root = parser._parse(source)\n" " return self._root\n" diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d8a283b..0882d36 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -330,6 +330,176 @@ static PyTypeObject partial_type = { }; +/* cmp_to_key ***************************************************************/ + +typedef struct { + PyObject_HEAD + PyObject *cmp; + PyObject *object; +} keyobject; + +static void +keyobject_dealloc(keyobject *ko) +{ + Py_DECREF(ko->cmp); + Py_XDECREF(ko->object); + PyObject_FREE(ko); +} + +static int +keyobject_traverse(keyobject *ko, visitproc visit, void *arg) +{ + Py_VISIT(ko->cmp); + if (ko->object) + Py_VISIT(ko->object); + return 0; +} + +static int +keyobject_clear(keyobject *ko) +{ + Py_CLEAR(ko->cmp); + if (ko->object) + Py_CLEAR(ko->object); + return 0; +} + +static PyMemberDef keyobject_members[] = { + {"obj", T_OBJECT, + offsetof(keyobject, object), 0, + PyDoc_STR("Value wrapped by a key function.")}, + {NULL} +}; + +static PyObject * +keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); + +static PyObject * +keyobject_richcompare(PyObject *ko, PyObject *other, int op); + +static PyTypeObject keyobject_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "functools.KeyWrapper", /* tp_name */ + sizeof(keyobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)keyobject_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + (ternaryfunc)keyobject_call, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)keyobject_traverse, /* tp_traverse */ + (inquiry)keyobject_clear, /* tp_clear */ + keyobject_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + keyobject_members, /* tp_members */ + 0, /* tp_getset */ +}; + +static PyObject * +keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds) +{ + PyObject *object; + keyobject *result; + static char *kwargs[] = {"obj", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:K", kwargs, &object)) + return NULL; + result = PyObject_New(keyobject, &keyobject_type); + if (!result) + return NULL; + Py_INCREF(ko->cmp); + result->cmp = ko->cmp; + Py_INCREF(object); + result->object = object; + return (PyObject *)result; +} + +static PyObject * +keyobject_richcompare(PyObject *ko, PyObject *other, int op) +{ + PyObject *res; + PyObject *args; + PyObject *x; + PyObject *y; + PyObject *compare; + PyObject *answer; + static PyObject *zero; + + if (zero == NULL) { + zero = PyLong_FromLong(0); + if (!zero) + return NULL; + } + + if (Py_TYPE(other) != &keyobject_type){ + PyErr_Format(PyExc_TypeError, "other argument must be K instance"); + return NULL; + } + compare = ((keyobject *) ko)->cmp; + assert(compare != NULL); + x = ((keyobject *) ko)->object; + y = ((keyobject *) other)->object; + if (!x || !y){ + PyErr_Format(PyExc_AttributeError, "object"); + return NULL; + } + + /* Call the user's comparison function and translate the 3-way + * result into true or false (or error). + */ + args = PyTuple_New(2); + if (args == NULL) + return NULL; + Py_INCREF(x); + Py_INCREF(y); + PyTuple_SET_ITEM(args, 0, x); + PyTuple_SET_ITEM(args, 1, y); + res = PyObject_Call(compare, args, NULL); + Py_DECREF(args); + if (res == NULL) + return NULL; + answer = PyObject_RichCompare(res, zero, op); + Py_DECREF(res); + return answer; +} + +static PyObject * +functools_cmp_to_key(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *cmp; + static char *kwargs[] = {"mycmp", NULL}; + keyobject *object; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:cmp_to_key", kwargs, &cmp)) + return NULL; + object = PyObject_New(keyobject, &keyobject_type); + if (!object) + return NULL; + Py_INCREF(cmp); + object->cmp = cmp; + object->object = NULL; + return (PyObject *)object; +} + +PyDoc_STRVAR(functools_cmp_to_key_doc, +"Convert a cmp= function into a key= function."); + /* reduce (used to be a builtin) ********************************************/ static PyObject * @@ -413,6 +583,8 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + {"cmp_to_key", (PyCFunction)functools_cmp_to_key, + METH_VARARGS | METH_KEYWORDS, functools_cmp_to_key_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index dd4317f..d37689e 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -201,13 +201,11 @@ EVP_hexdigest(EVPobject *self, PyObject *unused) /* Make hex version of the digest */ for(i=j=0; i<digest_size; i++) { - char c; + unsigned char c; c = (digest[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; c = (digest[i] & 0xf); - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; } retval = PyUnicode_FromStringAndSize(hex_digest, digest_size * 2); PyMem_Free(hex_digest); diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 44bdac6..9061a41 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -36,6 +36,7 @@ PyObject *_PyIO_str_nl; PyObject *_PyIO_str_read; PyObject *_PyIO_str_read1; PyObject *_PyIO_str_readable; +PyObject *_PyIO_str_readall; PyObject *_PyIO_str_readinto; PyObject *_PyIO_str_readline; PyObject *_PyIO_str_reset; @@ -90,89 +91,6 @@ PyDoc_STRVAR(module_doc, /* - * BlockingIOError extends IOError - */ - -static int -blockingioerror_init(PyBlockingIOErrorObject *self, PyObject *args, - PyObject *kwds) -{ - PyObject *myerrno = NULL, *strerror = NULL; - PyObject *baseargs = NULL; - Py_ssize_t written = 0; - - assert(PyTuple_Check(args)); - - self->written = 0; - if (!PyArg_ParseTuple(args, "OO|n:BlockingIOError", - &myerrno, &strerror, &written)) - return -1; - - baseargs = PyTuple_Pack(2, myerrno, strerror); - if (baseargs == NULL) - return -1; - /* This will take care of initializing of myerrno and strerror members */ - if (((PyTypeObject *)PyExc_IOError)->tp_init( - (PyObject *)self, baseargs, kwds) == -1) { - Py_DECREF(baseargs); - return -1; - } - Py_DECREF(baseargs); - - self->written = written; - return 0; -} - -static PyMemberDef blockingioerror_members[] = { - {"characters_written", T_PYSSIZET, offsetof(PyBlockingIOErrorObject, written), 0}, - {NULL} /* Sentinel */ -}; - -static PyTypeObject _PyExc_BlockingIOError = { - PyVarObject_HEAD_INIT(NULL, 0) - "BlockingIOError", /*tp_name*/ - sizeof(PyBlockingIOErrorObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_compare */ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - PyDoc_STR("Exception raised when I/O would block " - "on a non-blocking I/O stream"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - blockingioerror_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)blockingioerror_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; -PyObject *PyExc_BlockingIOError = (PyObject *)&_PyExc_BlockingIOError; - - -/* * The main open() function */ PyDoc_STRVAR(open_doc, @@ -307,6 +225,9 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds) PyObject *raw, *modeobj = NULL, *buffer = NULL, *wrapper = NULL; + _Py_identifier(isatty); + _Py_identifier(fileno); + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|sizzzi:open", kwlist, &file, &mode, &buffering, &encoding, &errors, &newline, @@ -420,7 +341,7 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds) /* buffering */ { - PyObject *res = PyObject_CallMethod(raw, "isatty", NULL); + PyObject *res = _PyObject_CallMethodId(raw, &PyId_isatty, NULL); if (res == NULL) goto error; isatty = PyLong_AsLong(res); @@ -442,7 +363,7 @@ io_open(PyObject *self, PyObject *args, PyObject *kwds) { struct stat st; long fileno; - PyObject *res = PyObject_CallMethod(raw, "fileno", NULL); + PyObject *res = _PyObject_CallMethodId(raw, &PyId_fileno, NULL); if (res == NULL) goto error; @@ -690,9 +611,11 @@ PyInit__io(void) state->unsupported_operation) < 0) goto fail; - /* BlockingIOError */ - _PyExc_BlockingIOError.tp_base = (PyTypeObject *) PyExc_IOError; - ADD_TYPE(&_PyExc_BlockingIOError, "BlockingIOError"); + /* BlockingIOError, for compatibility */ + Py_INCREF(PyExc_BlockingIOError); + if (PyModule_AddObject(m, "BlockingIOError", + (PyObject *) PyExc_BlockingIOError) < 0) + goto fail; /* Concrete base types of the IO ABCs. (the ABCs themselves are declared through inheritance in io.py) @@ -767,6 +690,8 @@ PyInit__io(void) goto fail; if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable"))) goto fail; + if (!(_PyIO_str_readall = PyUnicode_InternFromString("readall"))) + goto fail; if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto"))) goto fail; if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline"))) diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 925e4f2..b3a8471 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -55,20 +55,11 @@ extern PyObject *_PyIncrementalNewlineDecoder_decode( Otherwise, the function will scan further and return garbage. */ extern Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed); + int kind, char *start, char *end, Py_ssize_t *consumed); #define DEFAULT_BUFFER_SIZE (8 * 1024) /* bytes */ -typedef struct { - PyException_HEAD - PyObject *myerrno; - PyObject *strerror; - PyObject *filename; /* Not used, but part of the IOError object */ - Py_ssize_t written; -} PyBlockingIOErrorObject; -PyAPI_DATA(PyObject *) PyExc_BlockingIOError; - /* * Offset type for positioning. */ @@ -155,6 +146,7 @@ extern PyObject *_PyIO_str_nl; extern PyObject *_PyIO_str_read; extern PyObject *_PyIO_str_read1; extern PyObject *_PyIO_str_readable; +extern PyObject *_PyIO_str_readall; extern PyObject *_PyIO_str_readinto; extern PyObject *_PyIO_str_readline; extern PyObject *_PyIO_str_reset; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index c979ac2..c72bfb9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1,9 +1,9 @@ /* An implementation of Buffered I/O as defined by PEP 3116 - "New I/O" - + Classes defined here: BufferedIOBase, BufferedReader, BufferedWriter, BufferedRandom. - + Written by Amaury Forgeot d'Arc and Antoine Pitrou */ @@ -13,6 +13,18 @@ #include "pythread.h" #include "_iomodule.h" +_Py_identifier(close); +_Py_identifier(_dealloc_warn); +_Py_identifier(flush); +_Py_identifier(isatty); +_Py_identifier(peek); +_Py_identifier(read); +_Py_identifier(read1); +_Py_identifier(readable); +_Py_identifier(readinto); +_Py_identifier(writable); +_Py_identifier(write); + /* * BufferedIOBase class, inherits from IOBase. */ @@ -38,12 +50,13 @@ bufferediobase_readinto(PyObject *self, PyObject *args) Py_buffer buf; Py_ssize_t len; PyObject *data; + _Py_identifier(read); if (!PyArg_ParseTuple(args, "w*:readinto", &buf)) { return NULL; } - data = PyObject_CallMethod(self, "read", "n", buf.len); + data = _PyObject_CallMethodId(self, &PyId_read, "n", buf.len); if (data == NULL) goto error; @@ -198,7 +211,7 @@ typedef struct { int readable; int writable; int deallocating; - + /* True if this is a vanilla Buffered object (rather than a user derived class) *and* the raw stream is a vanilla FileIO object. */ int fast_closed_checks; @@ -237,7 +250,7 @@ typedef struct { /* Implementation notes: - + * BufferedReader, BufferedWriter and BufferedRandom try to share most methods (this is helped by the members `readable` and `writable`, which are initialized in the respective constructors) @@ -255,7 +268,7 @@ typedef struct { NOTE: we should try to maintain block alignment of reads and writes to the raw stream (according to the buffer size), but for now it is only done in read() and friends. - + */ /* These macros protect the buffered object against concurrent operations. */ @@ -410,7 +423,7 @@ buffered_dealloc_warn(buffered *self, PyObject *source) { if (self->ok && self->raw) { PyObject *r; - r = PyObject_CallMethod(self->raw, "_dealloc_warn", "O", source); + r = _PyObject_CallMethodId(self->raw, &PyId__dealloc_warn, "O", source); if (r) Py_DECREF(r); else @@ -589,14 +602,15 @@ _bufferedreader_reset_buf(buffered *self); static void _bufferedwriter_reset_buf(buffered *self); static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t); +_bufferedreader_peek_unlocked(buffered *self); static PyObject * _bufferedreader_read_all(buffered *self); static PyObject * _bufferedreader_read_fast(buffered *self, Py_ssize_t); static PyObject * _bufferedreader_read_generic(buffered *self, Py_ssize_t); - +static Py_ssize_t +_bufferedreader_raw_read(buffered *self, char *start, Py_ssize_t len); /* * Helpers @@ -608,14 +622,14 @@ static Py_ssize_t * _buffered_check_blocking_error(void) { PyObject *t, *v, *tb; - PyBlockingIOErrorObject *err; + PyOSErrorObject *err; PyErr_Fetch(&t, &v, &tb); if (v == NULL || !PyErr_GivenExceptionMatches(v, PyExc_BlockingIOError)) { PyErr_Restore(t, v, tb); return NULL; } - err = (PyBlockingIOErrorObject *) v; + err = (PyOSErrorObject *) v; /* TODO: sanity check (err->written >= 0) */ PyErr_Restore(t, v, tb); return &err->written; @@ -635,7 +649,7 @@ _buffered_raw_tell(buffered *self) if (!PyErr_Occurred()) PyErr_Format(PyExc_IOError, "Raw stream returned invalid position %" PY_PRIdOFF, - (PY_OFF_T_COMPAT)n); + (PY_OFF_T_COMPAT)n); return -1; } self->abs_pos = n; @@ -668,7 +682,7 @@ _buffered_raw_seek(buffered *self, Py_off_t target, int whence) if (!PyErr_Occurred()) PyErr_Format(PyExc_IOError, "Raw stream returned invalid position %" PY_PRIdOFF, - (PY_OFF_T_COMPAT)n); + (PY_OFF_T_COMPAT)n); return -1; } self->abs_pos = n; @@ -809,7 +823,7 @@ buffered_peek(buffered *self, PyObject *args) goto end; Py_CLEAR(res); } - res = _bufferedreader_peek_unlocked(self, n); + res = _bufferedreader_peek_unlocked(self); end: LEAVE_BUFFERED(self) @@ -875,7 +889,7 @@ buffered_read1(buffered *self, PyObject *args) if (!ENTER_BUFFERED(self)) return NULL; - + /* Return up to n bytes. If at least one byte is buffered, we only return buffered bytes. Otherwise, we do one raw read. */ @@ -924,10 +938,78 @@ end: static PyObject * buffered_readinto(buffered *self, PyObject *args) { + Py_buffer buf; + Py_ssize_t n, written = 0, remaining; + PyObject *res = NULL; + CHECK_INITIALIZED(self) - - /* TODO: use raw.readinto() (or a direct copy from our buffer) instead! */ - return bufferediobase_readinto((PyObject *)self, args); + + if (!PyArg_ParseTuple(args, "w*:readinto", &buf)) + return NULL; + + n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (n > 0) { + if (n >= buf.len) { + memcpy(buf.buf, self->buffer + self->pos, buf.len); + self->pos += buf.len; + res = PyLong_FromSsize_t(buf.len); + goto end_unlocked; + } + memcpy(buf.buf, self->buffer + self->pos, n); + self->pos += n; + written = n; + } + + if (!ENTER_BUFFERED(self)) + goto end_unlocked; + + if (self->writable) { + res = buffered_flush_and_rewind_unlocked(self); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + + _bufferedreader_reset_buf(self); + self->pos = 0; + + for (remaining = buf.len - written; + remaining > 0; + written += n, remaining -= n) { + /* If remaining bytes is larger than internal buffer size, copy + * directly into caller's buffer. */ + if (remaining > self->buffer_size) { + n = _bufferedreader_raw_read(self, (char *) buf.buf + written, + remaining); + } + else { + n = _bufferedreader_fill_buffer(self); + if (n > 0) { + if (n > remaining) + n = remaining; + memcpy((char *) buf.buf + written, + self->buffer + self->pos, n); + self->pos += n; + continue; /* short circuit */ + } + } + if (n == 0 || (n == -2 && written > 0)) + break; + if (n < 0) { + if (n == -2) { + Py_INCREF(Py_None); + res = Py_None; + } + goto end; + } + } + res = PyLong_FromSsize_t(written); + +end: + LEAVE_BUFFERED(self); +end_unlocked: + PyBuffer_Release(&buf); + return res; } static PyObject * @@ -1345,33 +1427,58 @@ static PyObject * _bufferedreader_read_all(buffered *self) { Py_ssize_t current_size; - PyObject *res, *data = NULL; - PyObject *chunks = PyList_New(0); - - if (chunks == NULL) - return NULL; + PyObject *res, *data = NULL, *chunk, *chunks; /* First copy what we have in the current buffer. */ current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (current_size) { data = PyBytes_FromStringAndSize( self->buffer + self->pos, current_size); - if (data == NULL) { - Py_DECREF(chunks); + if (data == NULL) return NULL; - } self->pos += current_size; } /* We're going past the buffer's bounds, flush it */ if (self->writable) { res = buffered_flush_and_rewind_unlocked(self); - if (res == NULL) { - Py_DECREF(chunks); + if (res == NULL) return NULL; - } Py_CLEAR(res); } _bufferedreader_reset_buf(self); + + if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) { + chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL); + if (chunk == NULL) + return NULL; + if (chunk != Py_None && !PyBytes_Check(chunk)) { + Py_XDECREF(data); + Py_DECREF(chunk); + PyErr_SetString(PyExc_TypeError, "readall() should return bytes"); + return NULL; + } + if (chunk == Py_None) { + if (current_size == 0) + return chunk; + else { + Py_DECREF(chunk); + return data; + } + } + else if (current_size) { + PyBytes_Concat(&data, chunk); + Py_DECREF(chunk); + if (data == NULL) + return NULL; + return data; + } else + return chunk; + } + + chunks = PyList_New(0); + if (chunks == NULL) + return NULL; + while (1) { if (data) { if (PyList_Append(chunks, data) < 0) { @@ -1533,7 +1640,7 @@ error: } static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t n) +_bufferedreader_peek_unlocked(buffered *self) { Py_ssize_t have, r; @@ -1575,6 +1682,7 @@ static PyMethodDef bufferedreader_methods[] = { {"read", (PyCFunction)buffered_read, METH_VARARGS}, {"peek", (PyCFunction)buffered_peek, METH_VARARGS}, {"read1", (PyCFunction)buffered_read1, METH_VARARGS}, + {"readinto", (PyCFunction)buffered_readinto, METH_VARARGS}, {"readline", (PyCFunction)buffered_readline, METH_VARARGS}, {"seek", (PyCFunction)buffered_seek, METH_VARARGS}, {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, @@ -2121,13 +2229,13 @@ bufferedrwpair_dealloc(rwpair *self) } static PyObject * -_forward_call(buffered *self, const char *name, PyObject *args) +_forward_call(buffered *self, _Py_Identifier *name, PyObject *args) { - PyObject *func = PyObject_GetAttrString((PyObject *)self, name); + PyObject *func = _PyObject_GetAttrId((PyObject *)self, name); PyObject *ret; if (func == NULL) { - PyErr_SetString(PyExc_AttributeError, name); + PyErr_SetString(PyExc_AttributeError, name->string); return NULL; } @@ -2139,66 +2247,66 @@ _forward_call(buffered *self, const char *name, PyObject *args) static PyObject * bufferedrwpair_read(rwpair *self, PyObject *args) { - return _forward_call(self->reader, "read", args); + return _forward_call(self->reader, &PyId_read, args); } static PyObject * bufferedrwpair_peek(rwpair *self, PyObject *args) { - return _forward_call(self->reader, "peek", args); + return _forward_call(self->reader, &PyId_peek, args); } static PyObject * bufferedrwpair_read1(rwpair *self, PyObject *args) { - return _forward_call(self->reader, "read1", args); + return _forward_call(self->reader, &PyId_read1, args); } static PyObject * bufferedrwpair_readinto(rwpair *self, PyObject *args) { - return _forward_call(self->reader, "readinto", args); + return _forward_call(self->reader, &PyId_readinto, args); } static PyObject * bufferedrwpair_write(rwpair *self, PyObject *args) { - return _forward_call(self->writer, "write", args); + return _forward_call(self->writer, &PyId_write, args); } static PyObject * bufferedrwpair_flush(rwpair *self, PyObject *args) { - return _forward_call(self->writer, "flush", args); + return _forward_call(self->writer, &PyId_flush, args); } static PyObject * bufferedrwpair_readable(rwpair *self, PyObject *args) { - return _forward_call(self->reader, "readable", args); + return _forward_call(self->reader, &PyId_readable, args); } static PyObject * bufferedrwpair_writable(rwpair *self, PyObject *args) { - return _forward_call(self->writer, "writable", args); + return _forward_call(self->writer, &PyId_writable, args); } static PyObject * bufferedrwpair_close(rwpair *self, PyObject *args) { - PyObject *ret = _forward_call(self->writer, "close", args); + PyObject *ret = _forward_call(self->writer, &PyId_close, args); if (ret == NULL) return NULL; Py_DECREF(ret); - return _forward_call(self->reader, "close", args); + return _forward_call(self->reader, &PyId_close, args); } static PyObject * bufferedrwpair_isatty(rwpair *self, PyObject *args) { - PyObject *ret = _forward_call(self->writer, "isatty", args); + PyObject *ret = _forward_call(self->writer, &PyId_isatty, args); if (ret != Py_False) { /* either True or exception */ @@ -2206,7 +2314,7 @@ bufferedrwpair_isatty(rwpair *self, PyObject *args) } Py_DECREF(ret); - return _forward_call(self->reader, "isatty", args); + return _forward_call(self->reader, &PyId_isatty, args); } static PyObject * diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index b40513f..65ec931 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -938,13 +938,11 @@ static int bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) { int ret; - void *ptr; bytesio *b = (bytesio *) obj->source; if (view == NULL) { b->exports++; return 0; } - ptr = (void *) obj; ret = PyBuffer_FillInfo(view, (PyObject*)obj, b->buf, b->string_size, 0, flags); if (ret >= 0) { diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index be5c9f8..c6b89f3 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -122,6 +122,7 @@ internal_close(fileio *self) static PyObject * fileio_close(fileio *self) { + _Py_identifier(close); if (!self->closefd) { self->fd = -1; Py_RETURN_NONE; @@ -137,8 +138,8 @@ fileio_close(fileio *self) if (errno < 0) return NULL; - return PyObject_CallMethod((PyObject*)&PyRawIOBase_Type, - "close", "O", self); + return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type, + &PyId_close, "O", self); } static PyObject * @@ -253,9 +254,11 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) } #ifdef MS_WINDOWS - if (PyUnicode_Check(nameobj)) - widename = PyUnicode_AS_UNICODE(nameobj); - if (widename == NULL) + if (PyUnicode_Check(nameobj)) { + widename = PyUnicode_AsUnicode(nameobj); + if (widename == NULL) + return -1; + } else #endif if (fd < 0) { @@ -372,7 +375,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds) if (self->fd < 0) { #ifdef MS_WINDOWS if (widename != NULL) - PyErr_SetFromErrnoWithUnicodeFilename(PyExc_IOError, widename); + PyErr_SetFromErrnoWithFilenameObject(PyExc_IOError, nameobj); else #endif PyErr_SetFromErrnoWithFilename(PyExc_IOError, name); @@ -541,21 +544,25 @@ fileio_readinto(fileio *self, PyObject *args) } static size_t -new_buffersize(fileio *self, size_t currentsize) +new_buffersize(fileio *self, size_t currentsize +#ifdef HAVE_FSTAT + , Py_off_t pos, Py_off_t end +#endif + ) { #ifdef HAVE_FSTAT - off_t pos, end; - struct stat st; - if (fstat(self->fd, &st) == 0) { - end = st.st_size; - pos = lseek(self->fd, 0L, SEEK_CUR); + if (end != (Py_off_t)-1) { /* Files claiming a size smaller than SMALLCHUNK may actually be streaming pseudo-files. In this case, we apply the more aggressive algorithm below. */ if (end >= SMALLCHUNK && end >= pos && pos >= 0) { /* Add 1 so if the file were to grow we'd notice. */ - return currentsize + end - pos + 1; + Py_off_t bufsize = currentsize + end - pos + 1; + if (bufsize < PY_SSIZE_T_MAX) + return (size_t)bufsize; + else + return PY_SSIZE_T_MAX; } } #endif @@ -568,9 +575,14 @@ new_buffersize(fileio *self, size_t currentsize) static PyObject * fileio_readall(fileio *self) { +#ifdef HAVE_FSTAT + struct stat st; + Py_off_t pos, end; +#endif PyObject *result; Py_ssize_t total = 0; int n; + size_t newsize; if (self->fd < 0) return err_closed(); @@ -581,8 +593,23 @@ fileio_readall(fileio *self) if (result == NULL) return NULL; +#ifdef HAVE_FSTAT +#if defined(MS_WIN64) || defined(MS_WINDOWS) + pos = _lseeki64(self->fd, 0L, SEEK_CUR); +#else + pos = lseek(self->fd, 0L, SEEK_CUR); +#endif + if (fstat(self->fd, &st) == 0) + end = st.st_size; + else + end = (Py_off_t)-1; +#endif while (1) { - size_t newsize = new_buffersize(self, total); +#ifdef HAVE_FSTAT + newsize = new_buffersize(self, total, pos, end); +#else + newsize = new_buffersize(self, total); +#endif if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { PyErr_SetString(PyExc_OverflowError, "unbounded read returned more bytes " @@ -621,6 +648,9 @@ fileio_readall(fileio *self) return NULL; } total += n; +#ifdef HAVE_FSTAT + pos += n; +#endif } if (PyBytes_GET_SIZE(result) > total) { diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index 35c7cdd..6a94a04 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -97,7 +97,9 @@ PyDoc_STRVAR(iobase_tell_doc, static PyObject * iobase_tell(PyObject *self, PyObject *args) { - return PyObject_CallMethod(self, "seek", "ii", 0, 1); + _Py_identifier(seek); + + return _PyObject_CallMethodId(self, &PyId_seek, "ii", 0, 1); } PyDoc_STRVAR(iobase_truncate_doc, @@ -464,6 +466,7 @@ iobase_readline(PyObject *self, PyObject *args) int has_peek = 0; PyObject *buffer, *result; Py_ssize_t old_size = -1; + _Py_identifier(read); if (!PyArg_ParseTuple(args, "|O&:readline", &_PyIO_ConvertSsize_t, &limit)) { return NULL; @@ -481,7 +484,9 @@ iobase_readline(PyObject *self, PyObject *args) PyObject *b; if (has_peek) { - PyObject *readahead = PyObject_CallMethod(self, "peek", "i", 1); + _Py_identifier(peek); + PyObject *readahead = _PyObject_CallMethodId(self, &PyId_peek, "i", 1); + if (readahead == NULL) goto fail; if (!PyBytes_Check(readahead)) { @@ -515,7 +520,7 @@ iobase_readline(PyObject *self, PyObject *args) Py_DECREF(readahead); } - b = PyObject_CallMethod(self, "read", "n", nreadahead); + b = _PyObject_CallMethodId(self, &PyId_read, "n", nreadahead); if (b == NULL) goto fail; if (!PyBytes_Check(b)) { @@ -601,7 +606,9 @@ iobase_readlines(PyObject *self, PyObject *args) /* XXX special-casing this made sense in the Python version in order to remove the bytecode interpretation overhead, but it could probably be removed here. */ - PyObject *ret = PyObject_CallMethod(result, "extend", "O", self); + _Py_identifier(extend); + PyObject *ret = _PyObject_CallMethodId(result, &PyId_extend, "O", self); + if (ret == NULL) { Py_DECREF(result); return NULL; @@ -781,8 +788,11 @@ rawiobase_read(PyObject *self, PyObject *args) return NULL; } - if (n < 0) - return PyObject_CallMethod(self, "readall", NULL); + if (n < 0) { + _Py_identifier(readall); + + return _PyObject_CallMethodId(self, &PyId_readall, NULL); + } /* TODO: allocate a bytes object directly instead and manually construct a writable memoryview pointing to it. */ @@ -823,8 +833,9 @@ rawiobase_readall(PyObject *self, PyObject *args) return NULL; while (1) { - PyObject *data = PyObject_CallMethod(self, "read", - "i", DEFAULT_BUFFER_SIZE); + _Py_identifier(read); + PyObject *data = _PyObject_CallMethodId(self, &PyId_read, + "i", DEFAULT_BUFFER_SIZE); if (!data) { Py_DECREF(chunks); return NULL; diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index c9d14b1..83a2465 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -9,7 +9,7 @@ typedef struct { PyObject_HEAD - Py_UNICODE *buf; + Py_UCS4 *buf; Py_ssize_t pos; Py_ssize_t string_size; size_t buf_size; @@ -21,7 +21,7 @@ typedef struct { PyObject *decoder; PyObject *readnl; PyObject *writenl; - + PyObject *dict; PyObject *weakreflist; } stringio; @@ -56,7 +56,7 @@ resize_buffer(stringio *self, size_t size) /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ size_t alloc = self->buf_size; - Py_UNICODE *new_buf = NULL; + Py_UCS4 *new_buf = NULL; assert(self->buf != NULL); @@ -84,10 +84,9 @@ resize_buffer(stringio *self, size_t size) alloc = size + 1; } - if (alloc > ((size_t)-1) / sizeof(Py_UNICODE)) + if (alloc > PY_SIZE_MAX / sizeof(Py_UCS4)) goto overflow; - new_buf = (Py_UNICODE *)PyMem_Realloc(self->buf, - alloc * sizeof(Py_UNICODE)); + new_buf = (Py_UCS4 *)PyMem_Realloc(self->buf, alloc * sizeof(Py_UCS4)); if (new_buf == NULL) { PyErr_NoMemory(); return -1; @@ -108,9 +107,9 @@ resize_buffer(stringio *self, size_t size) static Py_ssize_t write_str(stringio *self, PyObject *obj) { - Py_UNICODE *str; Py_ssize_t len; PyObject *decoded = NULL; + assert(self->buf != NULL); assert(self->pos >= 0); @@ -132,8 +131,11 @@ write_str(stringio *self, PyObject *obj) return -1; assert(PyUnicode_Check(decoded)); - str = PyUnicode_AS_UNICODE(decoded); - len = PyUnicode_GET_SIZE(decoded); + if (PyUnicode_READY(decoded)) { + Py_DECREF(decoded); + return -1; + } + len = PyUnicode_GET_LENGTH(decoded); assert(len >= 0); @@ -161,18 +163,21 @@ write_str(stringio *self, PyObject *obj) */ memset(self->buf + self->string_size, '\0', - (self->pos - self->string_size) * sizeof(Py_UNICODE)); + (self->pos - self->string_size) * sizeof(Py_UCS4)); } /* Copy the data to the internal buffer, overwriting some of the existing data if self->pos < self->string_size. */ - memcpy(self->buf + self->pos, str, len * sizeof(Py_UNICODE)); - self->pos += len; + if (!PyUnicode_AsUCS4(decoded, + self->buf + self->pos, + self->buf_size - self->pos, + 0)) + goto fail; /* Set the new length of the internal string if it has changed. */ - if (self->string_size < self->pos) { + self->pos += len; + if (self->string_size < self->pos) self->string_size = self->pos; - } Py_DECREF(decoded); return 0; @@ -190,7 +195,8 @@ stringio_getvalue(stringio *self) { CHECK_INITIALIZED(self); CHECK_CLOSED(self); - return PyUnicode_FromUnicode(self->buf, self->string_size); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, self->buf, + self->string_size); } PyDoc_STRVAR(stringio_tell_doc, @@ -214,7 +220,7 @@ static PyObject * stringio_read(stringio *self, PyObject *args) { Py_ssize_t size, n; - Py_UNICODE *output; + Py_UCS4 *output; PyObject *arg = Py_None; CHECK_INITIALIZED(self); @@ -247,19 +253,19 @@ stringio_read(stringio *self, PyObject *args) output = self->buf + self->pos; self->pos += size; - return PyUnicode_FromUnicode(output, size); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, output, size); } /* Internal helper, used by stringio_readline and stringio_iternext */ static PyObject * _stringio_readline(stringio *self, Py_ssize_t limit) { - Py_UNICODE *start, *end, old_char; + Py_UCS4 *start, *end, old_char; Py_ssize_t len, consumed; /* In case of overseek, return the empty string */ if (self->pos >= self->string_size) - return PyUnicode_FromString(""); + return PyUnicode_New(0, 0); start = self->buf + self->pos; if (limit < 0 || limit > self->string_size - self->pos) @@ -270,14 +276,14 @@ _stringio_readline(stringio *self, Py_ssize_t limit) *end = '\0'; len = _PyIO_find_line_ending( self->readtranslate, self->readuniversal, self->readnl, - start, end, &consumed); + PyUnicode_4BYTE_KIND, (char*)start, (char*)end, &consumed); *end = old_char; /* If we haven't found any line ending, we just return everything (`consumed` is ignored). */ if (len < 0) len = limit; self->pos += len; - return PyUnicode_FromUnicode(start, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, start, len); } PyDoc_STRVAR(stringio_readline_doc, @@ -337,7 +343,7 @@ stringio_iternext(stringio *self) if (line == NULL) return NULL; - if (PyUnicode_GET_SIZE(line) == 0) { + if (PyUnicode_GET_LENGTH(line) == 0) { /* Reached EOF */ Py_DECREF(line); return NULL; @@ -462,8 +468,10 @@ stringio_write(stringio *self, PyObject *obj) Py_TYPE(obj)->tp_name); return NULL; } + if (PyUnicode_READY(obj)) + return NULL; CHECK_CLOSED(self); - size = PyUnicode_GET_SIZE(obj); + size = PyUnicode_GET_LENGTH(obj); if (size > 0 && write_str(self, obj) < 0) return NULL; @@ -535,7 +543,7 @@ stringio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) /* tp_alloc initializes all the fields to zero. So we don't have to initialize them here. */ - self->buf = (Py_UNICODE *)PyMem_Malloc(0); + self->buf = (Py_UCS4 *)PyMem_Malloc(0); if (self->buf == NULL) { Py_DECREF(self); return PyErr_NoMemory(); @@ -747,11 +755,22 @@ stringio_setstate(stringio *self, PyObject *state) once by __init__. So we do not take any chance and replace object's buffer completely. */ { - Py_UNICODE *buf = PyUnicode_AS_UNICODE(PyTuple_GET_ITEM(state, 0)); - Py_ssize_t bufsize = PyUnicode_GET_SIZE(PyTuple_GET_ITEM(state, 0)); - if (resize_buffer(self, bufsize) < 0) + PyObject *item; + Py_UCS4 *buf; + Py_ssize_t bufsize; + + item = PyTuple_GET_ITEM(state, 0); + buf = PyUnicode_AsUCS4Copy(item); + if (buf == NULL) return NULL; - memcpy(self->buf, buf, bufsize * sizeof(Py_UNICODE)); + bufsize = PyUnicode_GET_LENGTH(item); + + if (resize_buffer(self, bufsize) < 0) { + PyMem_Free(buf); + return NULL; + } + memcpy(self->buf, buf, bufsize * sizeof(Py_UCS4)); + PyMem_Free(buf); self->string_size = bufsize; } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 9c5f441..2ded719 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -11,6 +11,24 @@ #include "structmember.h" #include "_iomodule.h" +_Py_identifier(close); +_Py_identifier(_dealloc_warn); +_Py_identifier(decode); +_Py_identifier(device_encoding); +_Py_identifier(fileno); +_Py_identifier(flush); +_Py_identifier(getpreferredencoding); +_Py_identifier(isatty); +_Py_identifier(read); +_Py_identifier(readable); +_Py_identifier(replace); +_Py_identifier(reset); +_Py_identifier(seek); +_Py_identifier(seekable); +_Py_identifier(setstate); +_Py_identifier(tell); +_Py_identifier(writable); + /* TextIOBase */ PyDoc_STRVAR(textiobase_doc, @@ -274,18 +292,26 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, goto error; } - output_len = PyUnicode_GET_SIZE(output); + if (PyUnicode_READY(output) == -1) + goto error; + + output_len = PyUnicode_GET_LENGTH(output); if (self->pendingcr && (final || output_len > 0)) { - Py_UNICODE *out; - PyObject *modified = PyUnicode_FromUnicode(NULL, output_len + 1); + /* Prefix output with CR */ + int kind; + PyObject *modified; + char *out; + + modified = PyUnicode_New(output_len + 1, + PyUnicode_MAX_CHAR_VALUE(output)); if (modified == NULL) goto error; - out = PyUnicode_AS_UNICODE(modified); - out[0] = '\r'; - memcpy(out + 1, PyUnicode_AS_UNICODE(output), - output_len * sizeof(Py_UNICODE)); + kind = PyUnicode_KIND(modified); + out = PyUnicode_DATA(modified); + PyUnicode_WRITE(kind, PyUnicode_DATA(modified), 0, '\r'); + memcpy(out + kind, PyUnicode_DATA(output), kind * output_len); Py_DECREF(output); - output = modified; + output = modified; /* output remains ready */ self->pendingcr = 0; output_len++; } @@ -295,21 +321,13 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, */ if (!final) { if (output_len > 0 - && PyUnicode_AS_UNICODE(output)[output_len - 1] == '\r') { - - if (Py_REFCNT(output) == 1) { - if (PyUnicode_Resize(&output, output_len - 1) < 0) - goto error; - } - else { - PyObject *modified = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(output), - output_len - 1); - if (modified == NULL) - goto error; - Py_DECREF(output); - output = modified; - } + && PyUnicode_READ_CHAR(output, output_len - 1) == '\r') + { + PyObject *modified = PyUnicode_Substring(output, 0, output_len -1); + if (modified == NULL) + goto error; + Py_DECREF(output); + output = modified; self->pendingcr = 1; } } @@ -317,13 +335,15 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, /* Record which newlines are read and do newline translation if desired, all in one pass. */ { - Py_UNICODE *in_str; + void *in_str; Py_ssize_t len; int seennl = self->seennl; int only_lf = 0; + int kind; - in_str = PyUnicode_AS_UNICODE(output); - len = PyUnicode_GET_SIZE(output); + in_str = PyUnicode_DATA(output); + len = PyUnicode_GET_LENGTH(output); + kind = PyUnicode_KIND(output); if (len == 0) return output; @@ -332,7 +352,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, for the \r *byte* with the libc's optimized memchr. */ if (seennl == SEEN_LF || seennl == 0) { - only_lf = (memchr(in_str, '\r', len * sizeof(Py_UNICODE)) == NULL); + only_lf = (memchr(in_str, '\r', kind * len) == NULL); } if (only_lf) { @@ -340,21 +360,19 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, (there's nothing else to be done, even when in translation mode) */ if (seennl == 0 && - memchr(in_str, '\n', len * sizeof(Py_UNICODE)) != NULL) { - Py_UNICODE *s, *end; - s = in_str; - end = in_str + len; + memchr(in_str, '\n', kind * len) != NULL) { + Py_ssize_t i = 0; for (;;) { - Py_UNICODE c; + Py_UCS4 c; /* Fast loop for non-control characters */ - while (*s > '\n') - s++; - c = *s++; + while (PyUnicode_READ(kind, in_str, i) > '\n') + i++; + c = PyUnicode_READ(kind, in_str, i++); if (c == '\n') { seennl |= SEEN_LF; break; } - if (s > end) + if (i >= len) break; } } @@ -362,29 +380,27 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, need translating */ } else if (!self->translate) { - Py_UNICODE *s, *end; + Py_ssize_t i = 0; /* We have already seen all newline types, no need to scan again */ if (seennl == SEEN_ALL) goto endscan; - s = in_str; - end = in_str + len; for (;;) { - Py_UNICODE c; + Py_UCS4 c; /* Fast loop for non-control characters */ - while (*s > '\r') - s++; - c = *s++; + while (PyUnicode_READ(kind, in_str, i) > '\r') + i++; + c = PyUnicode_READ(kind, in_str, i++); if (c == '\n') seennl |= SEEN_LF; else if (c == '\r') { - if (*s == '\n') { + if (PyUnicode_READ(kind, in_str, i) == '\n') { seennl |= SEEN_CRLF; - s++; + i++; } else seennl |= SEEN_CR; } - if (s > end) + if (i >= len) break; if (seennl == SEEN_ALL) break; @@ -393,61 +409,50 @@ _PyIncrementalNewlineDecoder_decode(PyObject *_self, ; } else { - PyObject *translated = NULL; - Py_UNICODE *out_str; - Py_UNICODE *in, *out, *end; - if (Py_REFCNT(output) != 1) { - /* We could try to optimize this so that we only do a copy - when there is something to translate. On the other hand, - most decoders should only output non-shared strings, i.e. - translation is done in place. */ - translated = PyUnicode_FromUnicode(NULL, len); - if (translated == NULL) - goto error; - assert(Py_REFCNT(translated) == 1); - memcpy(PyUnicode_AS_UNICODE(translated), - PyUnicode_AS_UNICODE(output), - len * sizeof(Py_UNICODE)); - } - else { - translated = output; + void *translated; + int kind = PyUnicode_KIND(output); + void *in_str = PyUnicode_DATA(output); + Py_ssize_t in, out; + /* XXX: Previous in-place translation here is disabled as + resizing is not possible anymore */ + /* We could try to optimize this so that we only do a copy + when there is something to translate. On the other hand, + we already know there is a \r byte, so chances are high + that something needs to be done. */ + translated = PyMem_Malloc(kind * len); + if (translated == NULL) { + PyErr_NoMemory(); + goto error; } - out_str = PyUnicode_AS_UNICODE(translated); - in = in_str; - out = out_str; - end = in_str + len; + in = out = 0; for (;;) { - Py_UNICODE c; + Py_UCS4 c; /* Fast loop for non-control characters */ - while ((c = *in++) > '\r') - *out++ = c; + while ((c = PyUnicode_READ(kind, in_str, in++)) > '\r') + PyUnicode_WRITE(kind, translated, out++, c); if (c == '\n') { - *out++ = c; + PyUnicode_WRITE(kind, translated, out++, c); seennl |= SEEN_LF; continue; } if (c == '\r') { - if (*in == '\n') { + if (PyUnicode_READ(kind, in_str, in) == '\n') { in++; seennl |= SEEN_CRLF; } else seennl |= SEEN_CR; - *out++ = '\n'; + PyUnicode_WRITE(kind, translated, out++, '\n'); continue; } - if (in > end) + if (in > len) break; - *out++ = c; - } - if (translated != output) { - Py_DECREF(output); - output = translated; - } - if (out - out_str != len) { - if (PyUnicode_Resize(&output, out - out_str) < 0) - goto error; + PyUnicode_WRITE(kind, translated, out++, c); } + Py_DECREF(output); + output = PyUnicode_FromKindAndData(kind, translated, out); + if (!output) + goto error; } self->seennl |= seennl; } @@ -514,8 +519,8 @@ incrementalnewlinedecoder_setstate(nldecoder_object *self, PyObject *state) flag >>= 1; if (self->decoder != Py_None) - return PyObject_CallMethod(self->decoder, - "setstate", "((OK))", buffer, flag); + return _PyObject_CallMethodId(self->decoder, + &PyId_setstate, "((OK))", buffer, flag); else Py_RETURN_NONE; } @@ -680,12 +685,16 @@ typedef struct PyObject *pending_bytes; /* list of bytes objects waiting to be written, or NULL */ Py_ssize_t pending_bytes_count; - PyObject *snapshot; + /* snapshot is either None, or a tuple (dec_flags, next_input) where * dec_flags is the second (integer) item of the decoder state and * next_input is the chunk of input bytes that comes next after the * snapshot point. We use this to reconstruct decoder states in tell(). */ + PyObject *snapshot; + /* Bytes-to-characters ratio for the current chunk. Serves as input for + the heuristic in tell(). */ + double b2cratio; /* Cache raw object if it's a FileIO object */ PyObject *raw; @@ -701,9 +710,7 @@ typedef struct static PyObject * ascii_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeASCII(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsASCIIString(text, PyBytes_AS_STRING(self->errors)); } static PyObject * @@ -773,17 +780,13 @@ utf32_encode(textio *self, PyObject *text) static PyObject * utf8_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsUTF8String(text, PyBytes_AS_STRING(self->errors)); } static PyObject * latin1_encode(textio *self, PyObject *text) { - return PyUnicode_EncodeLatin1(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), - PyBytes_AS_STRING(self->errors)); + return _PyUnicode_AsLatin1String(text, PyBytes_AS_STRING(self->errors)); } /* Map normalized encoding names onto the specialized encoding funcs */ @@ -852,11 +855,12 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) self->decoded_chars_used = 0; self->pending_bytes_count = 0; self->encodefunc = NULL; + self->b2cratio = 0.0; if (encoding == NULL) { /* Try os.device_encoding(fileno) */ PyObject *fileno; - fileno = PyObject_CallMethod(buffer, "fileno", NULL); + fileno = _PyObject_CallMethodId(buffer, &PyId_fileno, NULL); /* Ignore only AttributeError and UnsupportedOperation */ if (fileno == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError) || @@ -868,9 +872,9 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) } } else { - self->encoding = PyObject_CallMethod(state->os_module, - "device_encoding", - "N", fileno); + self->encoding = _PyObject_CallMethodId(state->os_module, + &PyId_device_encoding, + "N", fileno); if (self->encoding == NULL) goto error; else if (!PyUnicode_Check(self->encoding)) @@ -887,8 +891,8 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) } else { use_locale: - self->encoding = PyObject_CallMethod( - state->locale_module, "getpreferredencoding", NULL); + self->encoding = _PyObject_CallMethodId( + state->locale_module, &PyId_getpreferredencoding, NULL); if (self->encoding == NULL) { catch_ImportError: /* @@ -953,7 +957,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) #endif /* Build the decoder object */ - res = PyObject_CallMethod(buffer, "readable", NULL); + res = _PyObject_CallMethodId(buffer, &PyId_readable, NULL); if (res == NULL) goto error; r = PyObject_IsTrue(res); @@ -978,7 +982,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) } /* Build the encoder object */ - res = PyObject_CallMethod(buffer, "writable", NULL); + res = _PyObject_CallMethodId(buffer, &PyId_writable, NULL); if (res == NULL) goto error; r = PyObject_IsTrue(res); @@ -1036,7 +1040,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) Py_DECREF(raw); } - res = PyObject_CallMethod(buffer, "seekable", NULL); + res = _PyObject_CallMethodId(buffer, &PyId_seekable, NULL); if (res == NULL) goto error; self->seekable = self->telling = PyObject_IsTrue(res); @@ -1208,18 +1212,6 @@ textiowrapper_detach(textio *self) return buffer; } -Py_LOCAL_INLINE(const Py_UNICODE *) -findchar(const Py_UNICODE *s, Py_ssize_t size, Py_UNICODE ch) -{ - /* like wcschr, but doesn't stop at NULL characters */ - while (size-- > 0) { - if (*s == ch) - return s; - s++; - } - return NULL; -} - /* Flush the internal write buffer. This doesn't explicitly flush the underlying buffered object, though. */ static int @@ -1264,6 +1256,9 @@ textiowrapper_write(textio *self, PyObject *args) return NULL; } + if (PyUnicode_READY(text) == -1) + return NULL; + CHECK_CLOSED(self); if (self->encoder == NULL) @@ -1271,16 +1266,15 @@ textiowrapper_write(textio *self, PyObject *args) Py_INCREF(text); - textlen = PyUnicode_GetSize(text); + textlen = PyUnicode_GET_LENGTH(text); if ((self->writetranslate && self->writenl != NULL) || self->line_buffering) - if (findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\n')) + if (PyUnicode_FindChar(text, '\n', 0, PyUnicode_GET_LENGTH(text), 1) != -1) haslf = 1; if (haslf && self->writetranslate && self->writenl != NULL) { - PyObject *newtext = PyObject_CallMethod( - text, "replace", "ss", "\n", self->writenl); + PyObject *newtext = _PyObject_CallMethodId( + text, &PyId_replace, "ss", "\n", self->writenl); Py_DECREF(text); if (newtext == NULL) return NULL; @@ -1291,8 +1285,7 @@ textiowrapper_write(textio *self, PyObject *args) needflush = 1; else if (self->line_buffering && (haslf || - findchar(PyUnicode_AS_UNICODE(text), - PyUnicode_GET_SIZE(text), '\r'))) + PyUnicode_FindChar(text, '\r', 0, PyUnicode_GET_LENGTH(text), 1) != -1)) needflush = 1; /* XXX What if we were just reading? */ @@ -1336,7 +1329,7 @@ textiowrapper_write(textio *self, PyObject *args) Py_CLEAR(self->snapshot); if (self->decoder) { - ret = PyObject_CallMethod(self->decoder, "reset", NULL); + ret = _PyObject_CallMethodId(self->decoder, &PyId_reset, NULL); if (ret == NULL) return NULL; Py_DECREF(ret); @@ -1364,7 +1357,8 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n) if (self->decoded_chars == NULL) return PyUnicode_FromStringAndSize(NULL, 0); - avail = (PyUnicode_GET_SIZE(self->decoded_chars) + /* decoded_chars is guaranteed to be "ready". */ + avail = (PyUnicode_GET_LENGTH(self->decoded_chars) - self->decoded_chars_used); assert(avail >= 0); @@ -1373,9 +1367,9 @@ textiowrapper_get_decoded_chars(textio *self, Py_ssize_t n) n = avail; if (self->decoded_chars_used > 0 || n < avail) { - chars = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(self->decoded_chars) - + self->decoded_chars_used, n); + chars = PyUnicode_Substring(self->decoded_chars, + self->decoded_chars_used, + self->decoded_chars_used + n); if (chars == NULL) return NULL; } @@ -1397,6 +1391,7 @@ textiowrapper_read_chunk(textio *self) PyObject *dec_flags = NULL; PyObject *input_chunk = NULL; PyObject *decoded_chars, *chunk_size; + Py_ssize_t nbytes, nchars; int eof; /* The return value is True unless EOF was reached. The decoded string is @@ -1443,7 +1438,8 @@ textiowrapper_read_chunk(textio *self) goto fail; assert(PyBytes_Check(input_chunk)); - eof = (PyBytes_Size(input_chunk) == 0); + nbytes = PyBytes_Size(input_chunk); + eof = (nbytes == 0); if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) { decoded_chars = _PyIncrementalNewlineDecoder_decode( @@ -1457,8 +1453,15 @@ textiowrapper_read_chunk(textio *self) /* TODO sanity check: isinstance(decoded_chars, unicode) */ if (decoded_chars == NULL) goto fail; + if (PyUnicode_READY(decoded_chars) == -1) + goto fail; textiowrapper_set_decoded_chars(self, decoded_chars); - if (PyUnicode_GET_SIZE(decoded_chars) > 0) + nchars = PyUnicode_GET_LENGTH(decoded_chars); + if (nchars > 0) + self->b2cratio = (double) nbytes / nchars; + else + self->b2cratio = 0.0; + if (nchars > 0) eof = 0; if (self->telling) { @@ -1505,12 +1508,17 @@ textiowrapper_read(textio *self, PyObject *args) if (n < 0) { /* Read everything */ - PyObject *bytes = PyObject_CallMethod(self->buffer, "read", NULL); + PyObject *bytes = _PyObject_CallMethodId(self->buffer, &PyId_read, NULL); PyObject *decoded; if (bytes == NULL) goto fail; - decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode, - bytes, Py_True, NULL); + + if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) + decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, + bytes, 1); + else + decoded = PyObject_CallMethodObjArgs( + self->decoder, _PyIO_str_decode, bytes, Py_True, NULL); Py_DECREF(bytes); if (decoded == NULL) goto fail; @@ -1536,7 +1544,9 @@ textiowrapper_read(textio *self, PyObject *args) result = textiowrapper_get_decoded_chars(self, n); if (result == NULL) goto fail; - remaining -= PyUnicode_GET_SIZE(result); + if (PyUnicode_READY(result) == -1) + goto fail; + remaining -= PyUnicode_GET_LENGTH(result); /* Keep reading chunks until we have n characters to return */ while (remaining > 0) { @@ -1556,7 +1566,7 @@ textiowrapper_read(textio *self, PyObject *args) result = textiowrapper_get_decoded_chars(self, remaining); if (result == NULL) goto fail; - remaining -= PyUnicode_GET_SIZE(result); + remaining -= PyUnicode_GET_LENGTH(result); } if (chunks != NULL) { if (result != NULL && PyList_Append(chunks, result) < 0) @@ -1576,36 +1586,35 @@ textiowrapper_read(textio *self, PyObject *args) } -/* NOTE: `end` must point to the real end of the Py_UNICODE storage, +/* NOTE: `end` must point to the real end of the Py_UCS4 storage, that is to the NUL character. Otherwise the function will produce incorrect results. */ -static Py_UNICODE * -find_control_char(Py_UNICODE *start, Py_UNICODE *end, Py_UNICODE ch) +static char * +find_control_char(int kind, char *s, char *end, Py_UCS4 ch) { - Py_UNICODE *s = start; for (;;) { - while (*s > ch) - s++; - if (*s == ch) + while (PyUnicode_READ(kind, s, 0) > ch) + s += kind; + if (PyUnicode_READ(kind, s, 0) == ch) return s; if (s == end) return NULL; - s++; + s += kind; } } Py_ssize_t _PyIO_find_line_ending( int translated, int universal, PyObject *readnl, - Py_UNICODE *start, Py_UNICODE *end, Py_ssize_t *consumed) + int kind, char *start, char *end, Py_ssize_t *consumed) { - Py_ssize_t len = end - start; + Py_ssize_t len = ((char*)end - (char*)start)/kind; if (translated) { /* Newlines are already translated, only search for \n */ - Py_UNICODE *pos = find_control_char(start, end, '\n'); + char *pos = find_control_char(kind, start, end, '\n'); if (pos != NULL) - return pos - start + 1; + return (pos - start)/kind + 1; else { *consumed = len; return -1; @@ -1615,63 +1624,66 @@ _PyIO_find_line_ending( /* Universal newline search. Find any of \r, \r\n, \n * The decoder ensures that \r\n are not split in two pieces */ - Py_UNICODE *s = start; + char *s = start; for (;;) { - Py_UNICODE ch; + Py_UCS4 ch; /* Fast path for non-control chars. The loop always ends - since the Py_UNICODE storage is NUL-terminated. */ - while (*s > '\r') - s++; + since the Unicode string is NUL-terminated. */ + while (PyUnicode_READ(kind, s, 0) > '\r') + s += kind; if (s >= end) { *consumed = len; return -1; } - ch = *s++; + ch = PyUnicode_READ(kind, s, 0); + s += kind; if (ch == '\n') - return s - start; + return (s - start)/kind; if (ch == '\r') { - if (*s == '\n') - return s - start + 1; + if (PyUnicode_READ(kind, s, 0) == '\n') + return (s - start)/kind + 1; else - return s - start; + return (s - start)/kind; } } } else { /* Non-universal mode. */ - Py_ssize_t readnl_len = PyUnicode_GET_SIZE(readnl); - Py_UNICODE *nl = PyUnicode_AS_UNICODE(readnl); + Py_ssize_t readnl_len = PyUnicode_GET_LENGTH(readnl); + char *nl = PyUnicode_DATA(readnl); + /* Assume that readnl is an ASCII character. */ + assert(PyUnicode_KIND(readnl) == PyUnicode_1BYTE_KIND); if (readnl_len == 1) { - Py_UNICODE *pos = find_control_char(start, end, nl[0]); + char *pos = find_control_char(kind, start, end, nl[0]); if (pos != NULL) - return pos - start + 1; + return (pos - start)/kind + 1; *consumed = len; return -1; } else { - Py_UNICODE *s = start; - Py_UNICODE *e = end - readnl_len + 1; - Py_UNICODE *pos; + char *s = start; + char *e = end - (readnl_len - 1)*kind; + char *pos; if (e < s) e = s; while (s < e) { Py_ssize_t i; - Py_UNICODE *pos = find_control_char(s, end, nl[0]); + char *pos = find_control_char(kind, s, end, nl[0]); if (pos == NULL || pos >= e) break; for (i = 1; i < readnl_len; i++) { - if (pos[i] != nl[i]) + if (PyUnicode_READ(kind, pos, i) != nl[i]) break; } if (i == readnl_len) - return pos - start + readnl_len; - s = pos + 1; + return (pos - start)/kind + readnl_len; + s = pos + kind; } - pos = find_control_char(e, end, nl[0]); + pos = find_control_char(kind, e, end, nl[0]); if (pos == NULL) *consumed = len; else - *consumed = pos - start; + *consumed = (pos - start)/kind; return -1; } } @@ -1692,14 +1704,15 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) chunked = 0; while (1) { - Py_UNICODE *ptr; + char *ptr; Py_ssize_t line_len; + int kind; Py_ssize_t consumed = 0; /* First, get some data if necessary */ res = 1; while (!self->decoded_chars || - !PyUnicode_GET_SIZE(self->decoded_chars)) { + !PyUnicode_GET_LENGTH(self->decoded_chars)) { res = textiowrapper_read_chunk(self); if (res < 0) goto error; @@ -1724,18 +1737,24 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) assert(self->decoded_chars_used == 0); line = PyUnicode_Concat(remaining, self->decoded_chars); start = 0; - offset_to_buffer = PyUnicode_GET_SIZE(remaining); + offset_to_buffer = PyUnicode_GET_LENGTH(remaining); Py_CLEAR(remaining); if (line == NULL) goto error; + if (PyUnicode_READY(line) == -1) + goto error; } - ptr = PyUnicode_AS_UNICODE(line); - line_len = PyUnicode_GET_SIZE(line); + ptr = PyUnicode_DATA(line); + line_len = PyUnicode_GET_LENGTH(line); + kind = PyUnicode_KIND(line); endpos = _PyIO_find_line_ending( self->readtranslate, self->readuniversal, self->readnl, - ptr + start, ptr + line_len, &consumed); + kind, + ptr + kind * start, + ptr + kind * line_len, + &consumed); if (endpos >= 0) { endpos += start; if (limit >= 0 && (endpos - start) + chunked >= limit) @@ -1759,21 +1778,20 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) if (chunks == NULL) goto error; } - s = PyUnicode_FromUnicode(ptr + start, endpos - start); + s = PyUnicode_Substring(line, start, endpos); if (s == NULL) goto error; if (PyList_Append(chunks, s) < 0) { Py_DECREF(s); goto error; } - chunked += PyUnicode_GET_SIZE(s); + chunked += PyUnicode_GET_LENGTH(s); Py_DECREF(s); } /* There may be some remaining bytes we'll have to prepend to the next chunk of data */ if (endpos < line_len) { - remaining = PyUnicode_FromUnicode( - ptr + endpos, line_len - endpos); + remaining = PyUnicode_Substring(line, endpos, line_len); if (remaining == NULL) goto error; } @@ -1785,19 +1803,12 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) if (line != NULL) { /* Our line ends in the current buffer */ self->decoded_chars_used = endpos - offset_to_buffer; - if (start > 0 || endpos < PyUnicode_GET_SIZE(line)) { - if (start == 0 && Py_REFCNT(line) == 1) { - if (PyUnicode_Resize(&line, endpos) < 0) - goto error; - } - else { - PyObject *s = PyUnicode_FromUnicode( - PyUnicode_AS_UNICODE(line) + start, endpos - start); - Py_CLEAR(line); - if (s == NULL) - goto error; - line = s; - } + if (start > 0 || endpos < PyUnicode_GET_LENGTH(line)) { + PyObject *s = PyUnicode_Substring(line, start, endpos); + Py_CLEAR(line); + if (s == NULL) + goto error; + line = s; } } if (remaining != NULL) { @@ -1811,16 +1822,20 @@ _textiowrapper_readline(textio *self, Py_ssize_t limit) Py_CLEAR(remaining); } if (chunks != NULL) { - if (line != NULL && PyList_Append(chunks, line) < 0) - goto error; - Py_CLEAR(line); + if (line != NULL) { + if (PyList_Append(chunks, line) < 0) + goto error; + Py_DECREF(line); + } line = PyUnicode_Join(_PyIO_empty_str, chunks); if (line == NULL) goto error; - Py_DECREF(chunks); + Py_CLEAR(chunks); + } + if (line == NULL) { + Py_INCREF(_PyIO_empty_str); + line = _PyIO_empty_str; } - if (line == NULL) - line = PyUnicode_FromStringAndSize(NULL, 0); return line; @@ -1943,8 +1958,8 @@ _textiowrapper_decoder_setstate(textio *self, cookie_type *cookie) if (cookie->start_pos == 0 && cookie->dec_flags == 0) res = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_reset, NULL); else - res = PyObject_CallMethod(self->decoder, "setstate", - "((yi))", "", cookie->dec_flags); + res = _PyObject_CallMethodId(self->decoder, &PyId_setstate, + "((yi))", "", cookie->dec_flags); if (res == NULL) return -1; Py_DECREF(res); @@ -2008,13 +2023,12 @@ textiowrapper_seek(textio *self, PyObject *args) * sync the underlying buffer with the current position. */ Py_DECREF(cookieObj); - cookieObj = PyObject_CallMethod((PyObject *)self, "tell", NULL); + cookieObj = _PyObject_CallMethodId((PyObject *)self, &PyId_tell, NULL); if (cookieObj == NULL) goto fail; } else if (whence == 2) { /* seek relative to end of file */ - cmp = PyObject_RichCompareBool(cookieObj, _PyIO_zero, Py_EQ); if (cmp < 0) goto fail; @@ -2024,7 +2038,7 @@ textiowrapper_seek(textio *self, PyObject *args) goto fail; } - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + res = _PyObject_CallMethodId((PyObject *)self, &PyId_flush, NULL); if (res == NULL) goto fail; Py_DECREF(res); @@ -2032,13 +2046,13 @@ textiowrapper_seek(textio *self, PyObject *args) textiowrapper_set_decoded_chars(self, NULL); Py_CLEAR(self->snapshot); if (self->decoder) { - res = PyObject_CallMethod(self->decoder, "reset", NULL); + res = _PyObject_CallMethodId(self->decoder, &PyId_reset, NULL); if (res == NULL) goto fail; Py_DECREF(res); } - res = PyObject_CallMethod(self->buffer, "seek", "ii", 0, 2); + res = _PyObject_CallMethodId(self->buffer, &PyId_seek, "ii", 0, 2); Py_XDECREF(cookieObj); return res; } @@ -2091,8 +2105,8 @@ textiowrapper_seek(textio *self, PyObject *args) if (cookie.chars_to_skip) { /* Just like _read_chunk, feed the decoder and save a snapshot. */ - PyObject *input_chunk = PyObject_CallMethod( - self->buffer, "read", "i", cookie.bytes_to_feed); + PyObject *input_chunk = _PyObject_CallMethodId( + self->buffer, &PyId_read, "i", cookie.bytes_to_feed); PyObject *decoded; if (input_chunk == NULL) @@ -2106,11 +2120,15 @@ textiowrapper_seek(textio *self, PyObject *args) goto fail; } - decoded = PyObject_CallMethod(self->decoder, "decode", - "Oi", input_chunk, (int)cookie.need_eof); + decoded = _PyObject_CallMethodId(self->decoder, &PyId_decode, + "Oi", input_chunk, (int)cookie.need_eof); if (decoded == NULL) goto fail; + if (PyUnicode_READY(decoded) == -1) { + Py_DECREF(decoded); + goto fail; + } textiowrapper_set_decoded_chars(self, decoded); @@ -2147,8 +2165,12 @@ textiowrapper_tell(textio *self, PyObject *args) cookie_type cookie = {0,0,0,0,0}; PyObject *next_input; Py_ssize_t chars_to_skip, chars_decoded; + Py_ssize_t skip_bytes, skip_back; PyObject *saved_state = NULL; char *input, *input_end; + char *dec_buffer; + Py_ssize_t dec_buffer_len; + int dec_flags; CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -2165,12 +2187,12 @@ textiowrapper_tell(textio *self, PyObject *args) if (_textiowrapper_writeflush(self) < 0) return NULL; - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + res = _PyObject_CallMethodId((PyObject *)self, &PyId_flush, NULL); if (res == NULL) goto fail; Py_DECREF(res); - posobj = PyObject_CallMethod(self->buffer, "tell", NULL); + posobj = _PyObject_CallMethodId(self->buffer, &PyId_tell, NULL); if (posobj == NULL) goto fail; @@ -2184,6 +2206,7 @@ textiowrapper_tell(textio *self, PyObject *args) #else cookie.start_pos = PyLong_AsLong(posobj); #endif + Py_DECREF(posobj); if (PyErr_Occurred()) goto fail; @@ -2198,57 +2221,99 @@ textiowrapper_tell(textio *self, PyObject *args) /* How many decoded characters have been used up since the snapshot? */ if (self->decoded_chars_used == 0) { /* We haven't moved from the snapshot point. */ - Py_DECREF(posobj); return textiowrapper_build_cookie(&cookie); } chars_to_skip = self->decoded_chars_used; - /* Starting from the snapshot position, we will walk the decoder - * forward until it gives us enough decoded characters. - */ + /* Decoder state will be restored at the end */ saved_state = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_getstate, NULL); if (saved_state == NULL) goto fail; - /* Note our initial start point. */ - if (_textiowrapper_decoder_setstate(self, &cookie) < 0) - goto fail; +#define DECODER_GETSTATE() do { \ + PyObject *_state = PyObject_CallMethodObjArgs(self->decoder, \ + _PyIO_str_getstate, NULL); \ + if (_state == NULL) \ + goto fail; \ + if (!PyArg_Parse(_state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { \ + Py_DECREF(_state); \ + goto fail; \ + } \ + Py_DECREF(_state); \ + } while (0) + + /* TODO: replace assert with exception */ +#define DECODER_DECODE(start, len, res) do { \ + PyObject *_decoded = _PyObject_CallMethodId( \ + self->decoder, &PyId_decode, "y#", start, len); \ + if (_decoded == NULL) \ + goto fail; \ + assert (PyUnicode_Check(_decoded)); \ + res = PyUnicode_GET_LENGTH(_decoded); \ + Py_DECREF(_decoded); \ + } while (0) - /* Feed the decoder one byte at a time. As we go, note the - * nearest "safe start point" before the current location - * (a point where the decoder has nothing buffered, so seek() + /* Fast search for an acceptable start point, close to our + current pos */ + skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); + skip_back = 1; + assert(skip_back <= PyBytes_GET_SIZE(next_input)); + input = PyBytes_AS_STRING(next_input); + while (skip_bytes > 0) { + /* Decode up to temptative start point */ + if (_textiowrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + DECODER_DECODE(input, skip_bytes, chars_decoded); + if (chars_decoded <= chars_to_skip) { + DECODER_GETSTATE(); + if (dec_buffer_len == 0) { + /* Before pos and no bytes buffered in decoder => OK */ + cookie.dec_flags = dec_flags; + chars_to_skip -= chars_decoded; + break; + } + /* Skip back by buffered amount and reset heuristic */ + skip_bytes -= dec_buffer_len; + skip_back = 1; + } + else { + /* We're too far ahead, skip back a bit */ + skip_bytes -= skip_back; + skip_back *= 2; + } + } + if (skip_bytes <= 0) { + skip_bytes = 0; + if (_textiowrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + } + + /* Note our initial start point. */ + cookie.start_pos += skip_bytes; + cookie.chars_to_skip = chars_to_skip; + if (chars_to_skip == 0) + goto finally; + + /* We should be close to the desired position. Now feed the decoder one + * byte at a time until we reach the `chars_to_skip` target. + * As we go, note the nearest "safe start point" before the current + * location (a point where the decoder has nothing buffered, so seek() * can safely start from there and advance to this location). */ chars_decoded = 0; input = PyBytes_AS_STRING(next_input); input_end = input + PyBytes_GET_SIZE(next_input); + input += skip_bytes; while (input < input_end) { - PyObject *state; - char *dec_buffer; - Py_ssize_t dec_buffer_len; - int dec_flags; - - PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "y#", input, 1); - if (decoded == NULL) - goto fail; - assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); - Py_DECREF(decoded); + Py_ssize_t n; + DECODER_DECODE(input, 1, n); + /* We got n chars for 1 byte */ + chars_decoded += n; cookie.bytes_to_feed += 1; - - state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (state == NULL) - goto fail; - if (!PyArg_Parse(state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { - Py_DECREF(state); - goto fail; - } - Py_DECREF(state); + DECODER_GETSTATE(); if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) { /* Decoder buffer is empty, so this is a safe start point. */ @@ -2264,12 +2329,12 @@ textiowrapper_tell(textio *self, PyObject *args) } if (input == input_end) { /* We didn't get enough decoded data; signal EOF to get more. */ - PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "yi", "", /* final = */ 1); + PyObject *decoded = _PyObject_CallMethodId( + self->decoder, &PyId_decode, "yi", "", /* final = */ 1); if (decoded == NULL) goto fail; assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); + chars_decoded += PyUnicode_GET_LENGTH(decoded); Py_DECREF(decoded); cookie.need_eof = 1; @@ -2280,9 +2345,8 @@ textiowrapper_tell(textio *self, PyObject *args) } } - /* finally */ - Py_XDECREF(posobj); - res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); +finally: + res = _PyObject_CallMethodId(self->decoder, &PyId_setstate, "(O)", saved_state); Py_DECREF(saved_state); if (res == NULL) return NULL; @@ -2292,13 +2356,12 @@ textiowrapper_tell(textio *self, PyObject *args) cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int); return textiowrapper_build_cookie(&cookie); - fail: - Py_XDECREF(posobj); +fail: if (saved_state) { PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); - res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); + res = _PyObject_CallMethodId(self->decoder, &PyId_setstate, "(O)", saved_state); Py_DECREF(saved_state); if (res == NULL) return NULL; @@ -2386,35 +2449,35 @@ static PyObject * textiowrapper_fileno(textio *self, PyObject *args) { CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "fileno", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_fileno, NULL); } static PyObject * textiowrapper_seekable(textio *self, PyObject *args) { CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "seekable", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_seekable, NULL); } static PyObject * textiowrapper_readable(textio *self, PyObject *args) { CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "readable", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_readable, NULL); } static PyObject * textiowrapper_writable(textio *self, PyObject *args) { CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "writable", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_writable, NULL); } static PyObject * textiowrapper_isatty(textio *self, PyObject *args) { CHECK_INITIALIZED(self); - return PyObject_CallMethod(self->buffer, "isatty", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_isatty, NULL); } static PyObject * @@ -2433,7 +2496,7 @@ textiowrapper_flush(textio *self, PyObject *args) self->telling = self->seekable; if (_textiowrapper_writeflush(self) < 0) return NULL; - return PyObject_CallMethod(self->buffer, "flush", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_flush, NULL); } static PyObject * @@ -2456,20 +2519,20 @@ textiowrapper_close(textio *self, PyObject *args) } else { if (self->deallocating) { - res = PyObject_CallMethod(self->buffer, "_dealloc_warn", "O", self); + res = _PyObject_CallMethodId(self->buffer, &PyId__dealloc_warn, "O", self); if (res) Py_DECREF(res); else PyErr_Clear(); } - res = PyObject_CallMethod((PyObject *)self, "flush", NULL); + res = _PyObject_CallMethodId((PyObject *)self, &PyId_flush, NULL); if (res == NULL) { return NULL; } else Py_DECREF(res); - return PyObject_CallMethod(self->buffer, "close", NULL); + return _PyObject_CallMethodId(self->buffer, &PyId_close, NULL); } } @@ -2497,10 +2560,10 @@ textiowrapper_iternext(textio *self) } } - if (line == NULL) + if (line == NULL || PyUnicode_READY(line) == -1) return NULL; - if (PyUnicode_GET_SIZE(line) == 0) { + if (PyUnicode_GET_LENGTH(line) == 0) { /* Reached EOF or would have blocked */ Py_DECREF(line); Py_CLEAR(self->snapshot); diff --git a/Modules/_json.c b/Modules/_json.c index 0924873..cafd5a9 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -76,6 +76,21 @@ static PyMemberDef encoder_members[] = { }; static PyObject * +join_list_unicode(PyObject *lst) +{ + /* return u''.join(lst) */ + static PyObject *sep = NULL; + if (sep == NULL) { + sep = PyUnicode_FromStringAndSize("", 0); + if (sep == NULL) + return NULL; + } + return PyUnicode_Join(sep, lst); +} + +/* Forward decls */ + +static PyObject * ascii_escape_unicode(PyObject *pystr); static PyObject * py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); @@ -101,11 +116,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyObject *self); static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, _PyAccu *acc, PyObject *seq, Py_ssize_t indent_level); static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, _PyAccu *acc, PyObject *obj, Py_ssize_t indent_level); static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, PyObject *dct, Py_ssize_t indent_level); static PyObject * _encoded_const(PyObject *obj); static void @@ -122,13 +137,6 @@ encoder_encode_float(PyEncoderObject *s, PyObject *obj); #define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"') #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\n') || ((c) == '\r')) -#define MIN_EXPANSION 6 -#ifdef Py_UNICODE_WIDE -#define MAX_EXPANSION (2 * MIN_EXPANSION) -#else -#define MAX_EXPANSION MIN_EXPANSION -#endif - static int _convertPyInt_AsSsize_t(PyObject *o, Py_ssize_t *size_ptr) { @@ -147,7 +155,7 @@ _convertPyInt_FromSsize_t(Py_ssize_t *size_ptr) } static Py_ssize_t -ascii_escape_unichar(Py_UNICODE c, Py_UNICODE *output, Py_ssize_t chars) +ascii_escape_unichar(Py_UCS4 c, unsigned char *output, Py_ssize_t chars) { /* Escape unicode code point c to ASCII escape sequences in char *output. output must have at least 12 bytes unused to @@ -162,25 +170,23 @@ ascii_escape_unichar(Py_UNICODE c, Py_UNICODE *output, Py_ssize_t chars) case '\r': output[chars++] = 'r'; break; case '\t': output[chars++] = 't'; break; default: -#ifdef Py_UNICODE_WIDE if (c >= 0x10000) { /* UTF-16 surrogate pair */ - Py_UNICODE v = c - 0x10000; + Py_UCS4 v = c - 0x10000; c = 0xd800 | ((v >> 10) & 0x3ff); output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 12) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 8) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 4) & 0xf]; + output[chars++] = Py_hexdigits[(c ) & 0xf]; c = 0xdc00 | (v & 0x3ff); output[chars++] = '\\'; } -#endif output[chars++] = 'u'; - output[chars++] = "0123456789abcdef"[(c >> 12) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 8) & 0xf]; - output[chars++] = "0123456789abcdef"[(c >> 4) & 0xf]; - output[chars++] = "0123456789abcdef"[(c ) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 12) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 8) & 0xf]; + output[chars++] = Py_hexdigits[(c >> 4) & 0xf]; + output[chars++] = Py_hexdigits[(c ) & 0xf]; } return chars; } @@ -192,54 +198,52 @@ ascii_escape_unicode(PyObject *pystr) Py_ssize_t i; Py_ssize_t input_chars; Py_ssize_t output_size; - Py_ssize_t max_output_size; Py_ssize_t chars; PyObject *rval; - Py_UNICODE *output; - Py_UNICODE *input_unicode; + void *input; + unsigned char *output; + int kind; - input_chars = PyUnicode_GET_SIZE(pystr); - input_unicode = PyUnicode_AS_UNICODE(pystr); + if (PyUnicode_READY(pystr) == -1) + return NULL; + + input_chars = PyUnicode_GET_LENGTH(pystr); + input = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); - /* One char input can be up to 6 chars output, estimate 4 of these */ - output_size = 2 + (MIN_EXPANSION * 4) + input_chars; - max_output_size = 2 + (input_chars * MAX_EXPANSION); - rval = PyUnicode_FromStringAndSize(NULL, output_size); + /* Compute the output size */ + for (i = 0, output_size = 2; i < input_chars; i++) { + Py_UCS4 c = PyUnicode_READ(kind, input, i); + if (S_CHAR(c)) + output_size++; + else { + switch(c) { + case '\\': case '"': case '\b': case '\f': + case '\n': case '\r': case '\t': + output_size += 2; break; + default: + output_size += c >= 0x10000 ? 12 : 6; + } + } + } + + rval = PyUnicode_New(output_size, 127); if (rval == NULL) { return NULL; } - output = PyUnicode_AS_UNICODE(rval); + output = PyUnicode_1BYTE_DATA(rval); chars = 0; output[chars++] = '"'; for (i = 0; i < input_chars; i++) { - Py_UNICODE c = input_unicode[i]; + Py_UCS4 c = PyUnicode_READ(kind, input, i); if (S_CHAR(c)) { output[chars++] = c; } else { chars = ascii_escape_unichar(c, output, chars); } - if (output_size - chars < (1 + MAX_EXPANSION)) { - /* There's more than four, so let's resize by a lot */ - Py_ssize_t new_output_size = output_size * 2; - /* This is an upper bound */ - if (new_output_size > max_output_size) { - new_output_size = max_output_size; - } - /* Make sure that the output size changed before resizing */ - if (new_output_size != output_size) { - output_size = new_output_size; - if (PyUnicode_Resize(&rval, output_size) == -1) { - return NULL; - } - output = PyUnicode_AS_UNICODE(rval); - } - } } output[chars++] = '"'; - if (PyUnicode_Resize(&rval, chars) == -1) { - return NULL; - } return rval; } @@ -267,19 +271,6 @@ raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) } static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *sep = NULL; - if (sep == NULL) { - sep = PyUnicode_FromStringAndSize("", 0); - if (sep == NULL) - return NULL; - } - return PyUnicode_Join(sep, lst); -} - -static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { /* return (rval, idx) tuple, stealing reference to rval */ PyObject *tpl; @@ -333,22 +324,30 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next Return value is a new PyUnicode */ PyObject *rval = NULL; - Py_ssize_t len = PyUnicode_GET_SIZE(pystr); + Py_ssize_t len; Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; - const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); + Py_ssize_t next /* = begin */; + const void *buf; + int kind; PyObject *chunks = NULL; PyObject *chunk = NULL; + if (PyUnicode_READY(pystr) == -1) + return 0; + + len = PyUnicode_GET_LENGTH(pystr); + buf = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + if (end < 0 || len <= end) { PyErr_SetString(PyExc_ValueError, "end is out of bounds"); goto bail; } while (1) { /* Find the end of the string or the next escape */ - Py_UNICODE c = 0; + Py_UCS4 c = 0; for (next = end; next < len; next++) { - c = buf[next]; + c = PyUnicode_READ(kind, buf, next); if (c == '"' || c == '\\') { break; } @@ -364,7 +363,10 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next /* Pick up this chunk if it's not zero length */ if (next != end) { APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&buf[end], next - end); + chunk = PyUnicode_FromKindAndData( + kind, + (char*)buf + kind * end, + next - end); if (chunk == NULL) { goto bail; } @@ -378,7 +380,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next raise_errmsg("Unterminated string starting at", pystr, begin); goto bail; } - c = buf[next]; + c = PyUnicode_READ(kind, buf, next); if (c != 'u') { /* Non-unicode backslash escapes */ end = next + 1; @@ -408,7 +410,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next } /* Decode 4 hex digits */ for (; next < end; next++) { - Py_UNICODE digit = buf[next]; + Py_UCS4 digit = PyUnicode_READ(kind, buf, next); c <<= 4; switch (digit) { case '0': case '1': case '2': case '3': case '4': @@ -425,22 +427,22 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next goto bail; } } -#ifdef Py_UNICODE_WIDE /* Surrogate pair */ if ((c & 0xfc00) == 0xd800) { - Py_UNICODE c2 = 0; + Py_UCS4 c2 = 0; if (end + 6 >= len) { raise_errmsg("Unpaired high surrogate", pystr, end - 5); goto bail; } - if (buf[next++] != '\\' || buf[next++] != 'u') { + if (PyUnicode_READ(kind, buf, next++) != '\\' || + PyUnicode_READ(kind, buf, next++) != 'u') { raise_errmsg("Unpaired high surrogate", pystr, end - 5); goto bail; } end += 6; /* Decode 4 hex digits */ for (; next < end; next++) { - Py_UNICODE digit = buf[next]; + Py_UCS4 digit = PyUnicode_READ(kind, buf, next); c2 <<= 4; switch (digit) { case '0': case '1': case '2': case '3': case '4': @@ -467,10 +469,9 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next raise_errmsg("Unpaired low surrogate", pystr, end - 5); goto bail; } -#endif } APPEND_OLD_CHUNK - chunk = PyUnicode_FromUnicode(&c, 1); + chunk = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, &c, 1); if (chunk == NULL) { goto bail; } @@ -608,8 +609,9 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss Returns a new PyObject (usually a dict, but object_hook can change that) */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; PyObject *val = NULL; PyObject *rval = NULL; PyObject *key = NULL; @@ -617,6 +619,13 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss int has_pairs_hook = (s->object_pairs_hook != Py_None); Py_ssize_t next_idx; + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; + if (has_pairs_hook) rval = PyList_New(0); else @@ -625,15 +634,15 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss return NULL; /* skip whitespace after { */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind,str, idx))) idx++; /* only loop if the object is non-empty */ - if (idx <= end_idx && str[idx] != '}') { + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != '}') { while (idx <= end_idx) { PyObject *memokey; /* read key */ - if (str[idx] != '"') { + if (PyUnicode_READ(kind, str, idx) != '"') { raise_errmsg("Expecting property name", pystr, idx); goto bail; } @@ -653,13 +662,13 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss idx = next_idx; /* skip whitespace between key and : delimiter, read :, skip whitespace */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; - if (idx > end_idx || str[idx] != ':') { + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ':') { raise_errmsg("Expecting : delimiter", pystr, idx); goto bail; } idx++; - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* read any JSON term */ val = scan_once_unicode(s, pystr, idx, &next_idx); @@ -687,26 +696,26 @@ _parse_object_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ss idx = next_idx; /* skip whitespace before } or , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* bail if the object is closed or we didn't get the , delimiter */ if (idx > end_idx) break; - if (str[idx] == '}') { + if (PyUnicode_READ(kind, str, idx) == '}') { break; } - else if (str[idx] != ',') { + else if (PyUnicode_READ(kind, str, idx) != ',') { raise_errmsg("Expecting , delimiter", pystr, idx); goto bail; } idx++; /* skip whitespace after , delimiter */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; } } /* verify that idx < end_idx, str[idx] should be '}' */ - if (idx > end_idx || str[idx] != '}') { + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != '}') { raise_errmsg("Expecting object", pystr, end_idx); goto bail; } @@ -742,19 +751,27 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi Returns a new PyList */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; PyObject *val = NULL; PyObject *rval = PyList_New(0); Py_ssize_t next_idx; if (rval == NULL) return NULL; + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; + /* skip whitespace after [ */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* only loop if the array is non-empty */ - if (idx <= end_idx && str[idx] != ']') { + if (idx <= end_idx && PyUnicode_READ(kind, str, idx) != ']') { while (idx <= end_idx) { /* read any JSON term */ @@ -769,26 +786,26 @@ _parse_array_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssi idx = next_idx; /* skip whitespace between term and , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; /* bail if the array is closed or we didn't get the , delimiter */ if (idx > end_idx) break; - if (str[idx] == ']') { + if (PyUnicode_READ(kind, str, idx) == ']') { break; } - else if (str[idx] != ',') { + else if (PyUnicode_READ(kind, str, idx) != ',') { raise_errmsg("Expecting , delimiter", pystr, idx); goto bail; } idx++; /* skip whitespace after , */ - while (idx <= end_idx && IS_WHITESPACE(str[idx])) idx++; + while (idx <= end_idx && IS_WHITESPACE(PyUnicode_READ(kind, str, idx))) idx++; } } - /* verify that idx < end_idx, str[idx] should be ']' */ - if (idx > end_idx || str[idx] != ']') { + /* verify that idx < end_idx, PyUnicode_READ(kind, str, idx) should be ']' */ + if (idx > end_idx || PyUnicode_READ(kind, str, idx) != ']') { raise_errmsg("Expecting object", pystr, end_idx); goto bail; } @@ -820,7 +837,7 @@ _parse_constant(PyScannerObject *s, char *constant, Py_ssize_t idx, Py_ssize_t * /* rval = parse_constant(constant) */ rval = PyObject_CallFunctionObjArgs(s->parse_constant, cstr, NULL); - idx += PyUnicode_GET_SIZE(cstr); + idx += PyUnicode_GET_LENGTH(cstr); Py_DECREF(cstr); *next_idx_ptr = idx; return rval; @@ -837,15 +854,24 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ PyInt, PyLong, or PyFloat. May return other types if parse_int or parse_float are set */ - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t end_idx = PyUnicode_GET_SIZE(pystr) - 1; + void *str; + int kind; + Py_ssize_t end_idx; Py_ssize_t idx = start; int is_float = 0; PyObject *rval; - PyObject *numstr; + PyObject *numstr = NULL; + PyObject *custom_func; + + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + end_idx = PyUnicode_GET_LENGTH(pystr) - 1; /* read a sign if it's there, make sure it's not the end of the string */ - if (str[idx] == '-') { + if (PyUnicode_READ(kind, str, idx) == '-') { idx++; if (idx > end_idx) { PyErr_SetNone(PyExc_StopIteration); @@ -854,12 +880,12 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } /* read as many integer digits as we find as long as it doesn't start with 0 */ - if (str[idx] >= '1' && str[idx] <= '9') { + if (PyUnicode_READ(kind, str, idx) >= '1' && PyUnicode_READ(kind, str, idx) <= '9') { idx++; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; } /* if it starts with 0 we only expect one integer digit */ - else if (str[idx] == '0') { + else if (PyUnicode_READ(kind, str, idx) == '0') { idx++; } /* no integer digits, error */ @@ -869,25 +895,25 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } /* if the next char is '.' followed by a digit then read all float digits */ - if (idx < end_idx && str[idx] == '.' && str[idx + 1] >= '0' && str[idx + 1] <= '9') { + if (idx < end_idx && PyUnicode_READ(kind, str, idx) == '.' && PyUnicode_READ(kind, str, idx + 1) >= '0' && PyUnicode_READ(kind, str, idx + 1) <= '9') { is_float = 1; idx += 2; - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; } /* if the next char is 'e' or 'E' then maybe read the exponent (or backtrack) */ - if (idx < end_idx && (str[idx] == 'e' || str[idx] == 'E')) { + if (idx < end_idx && (PyUnicode_READ(kind, str, idx) == 'e' || PyUnicode_READ(kind, str, idx) == 'E')) { Py_ssize_t e_start = idx; idx++; /* read an exponent sign if present */ - if (idx < end_idx && (str[idx] == '-' || str[idx] == '+')) idx++; + if (idx < end_idx && (PyUnicode_READ(kind, str, idx) == '-' || PyUnicode_READ(kind, str, idx) == '+')) idx++; /* read all digits */ - while (idx <= end_idx && str[idx] >= '0' && str[idx] <= '9') idx++; + while (idx <= end_idx && PyUnicode_READ(kind, str, idx) >= '0' && PyUnicode_READ(kind, str, idx) <= '9') idx++; /* if we got a digit, then parse as float. if not, backtrack */ - if (str[idx - 1] >= '0' && str[idx - 1] <= '9') { + if (PyUnicode_READ(kind, str, idx - 1) >= '0' && PyUnicode_READ(kind, str, idx - 1) <= '9') { is_float = 1; } else { @@ -895,22 +921,39 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } } - /* copy the section we determined to be a number */ - numstr = PyUnicode_FromUnicode(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - rval = PyFloat_FromString(numstr); - } + if (is_float && s->parse_float != (PyObject *)&PyFloat_Type) + custom_func = s->parse_float; + else if (!is_float && s->parse_int != (PyObject *) &PyLong_Type) + custom_func = s->parse_int; + else + custom_func = NULL; + + if (custom_func) { + /* copy the section we determined to be a number */ + numstr = PyUnicode_FromKindAndData(kind, + (char*)str + kind * start, + idx - start); + if (numstr == NULL) + return NULL; + rval = PyObject_CallFunctionObjArgs(custom_func, numstr, NULL); } else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + Py_ssize_t i, n; + char *buf; + /* Straight conversion to ASCII, to avoid costly conversion of + decimal unicode digits (which cannot appear here) */ + n = idx - start; + numstr = PyBytes_FromStringAndSize(NULL, n); + if (numstr == NULL) + return NULL; + buf = PyBytes_AS_STRING(numstr); + for (i = 0; i < n; i++) { + buf[i] = (char) PyUnicode_READ(kind, str, i + start); + } + if (is_float) + rval = PyFloat_FromString(numstr); + else + rval = PyLong_FromString(buf, NULL, 10); } Py_DECREF(numstr); *next_idx_ptr = idx; @@ -928,13 +971,23 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ Returns a new PyObject representation of the term. */ PyObject *res; - Py_UNICODE *str = PyUnicode_AS_UNICODE(pystr); - Py_ssize_t length = PyUnicode_GET_SIZE(pystr); + void *str; + int kind; + Py_ssize_t length; + + if (PyUnicode_READY(pystr) == -1) + return NULL; + + str = PyUnicode_DATA(pystr); + kind = PyUnicode_KIND(pystr); + length = PyUnicode_GET_LENGTH(pystr); + if (idx >= length) { PyErr_SetNone(PyExc_StopIteration); return NULL; } - switch (str[idx]) { + + switch (PyUnicode_READ(kind, str, idx)) { case '"': /* string */ return scanstring_unicode(pystr, idx + 1, @@ -958,7 +1011,7 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ return res; case 'n': /* null */ - if ((idx + 3 < length) && str[idx + 1] == 'u' && str[idx + 2] == 'l' && str[idx + 3] == 'l') { + if ((idx + 3 < length) && PyUnicode_READ(kind, str, idx + 1) == 'u' && PyUnicode_READ(kind, str, idx + 2) == 'l' && PyUnicode_READ(kind, str, idx + 3) == 'l') { Py_INCREF(Py_None); *next_idx_ptr = idx + 4; return Py_None; @@ -966,7 +1019,7 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ break; case 't': /* true */ - if ((idx + 3 < length) && str[idx + 1] == 'r' && str[idx + 2] == 'u' && str[idx + 3] == 'e') { + if ((idx + 3 < length) && PyUnicode_READ(kind, str, idx + 1) == 'r' && PyUnicode_READ(kind, str, idx + 2) == 'u' && PyUnicode_READ(kind, str, idx + 3) == 'e') { Py_INCREF(Py_True); *next_idx_ptr = idx + 4; return Py_True; @@ -974,7 +1027,10 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ break; case 'f': /* false */ - if ((idx + 4 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'l' && str[idx + 3] == 's' && str[idx + 4] == 'e') { + if ((idx + 4 < length) && PyUnicode_READ(kind, str, idx + 1) == 'a' && + PyUnicode_READ(kind, str, idx + 2) == 'l' && + PyUnicode_READ(kind, str, idx + 3) == 's' && + PyUnicode_READ(kind, str, idx + 4) == 'e') { Py_INCREF(Py_False); *next_idx_ptr = idx + 5; return Py_False; @@ -982,19 +1038,33 @@ scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_ break; case 'N': /* NaN */ - if ((idx + 2 < length) && str[idx + 1] == 'a' && str[idx + 2] == 'N') { + if ((idx + 2 < length) && PyUnicode_READ(kind, str, idx + 1) == 'a' && + PyUnicode_READ(kind, str, idx + 2) == 'N') { return _parse_constant(s, "NaN", idx, next_idx_ptr); } break; case 'I': /* Infinity */ - if ((idx + 7 < length) && str[idx + 1] == 'n' && str[idx + 2] == 'f' && str[idx + 3] == 'i' && str[idx + 4] == 'n' && str[idx + 5] == 'i' && str[idx + 6] == 't' && str[idx + 7] == 'y') { + if ((idx + 7 < length) && PyUnicode_READ(kind, str, idx + 1) == 'n' && + PyUnicode_READ(kind, str, idx + 2) == 'f' && + PyUnicode_READ(kind, str, idx + 3) == 'i' && + PyUnicode_READ(kind, str, idx + 4) == 'n' && + PyUnicode_READ(kind, str, idx + 5) == 'i' && + PyUnicode_READ(kind, str, idx + 6) == 't' && + PyUnicode_READ(kind, str, idx + 7) == 'y') { return _parse_constant(s, "Infinity", idx, next_idx_ptr); } break; case '-': /* -Infinity */ - if ((idx + 8 < length) && str[idx + 1] == 'I' && str[idx + 2] == 'n' && str[idx + 3] == 'f' && str[idx + 4] == 'i' && str[idx + 5] == 'n' && str[idx + 6] == 'i' && str[idx + 7] == 't' && str[idx + 8] == 'y') { + if ((idx + 8 < length) && PyUnicode_READ(kind, str, idx + 1) == 'I' && + PyUnicode_READ(kind, str, idx + 2) == 'n' && + PyUnicode_READ(kind, str, idx + 3) == 'f' && + PyUnicode_READ(kind, str, idx + 4) == 'i' && + PyUnicode_READ(kind, str, idx + 5) == 'n' && + PyUnicode_READ(kind, str, idx + 6) == 'i' && + PyUnicode_READ(kind, str, idx + 7) == 't' && + PyUnicode_READ(kind, str, idx + 8) == 'y') { return _parse_constant(s, "-Infinity", idx, next_idx_ptr); } break; @@ -1210,22 +1280,22 @@ encoder_call(PyObject *self, PyObject *args, PyObject *kwds) /* Python callable interface to encode_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; PyObject *obj; - PyObject *rval; Py_ssize_t indent_level; PyEncoderObject *s; + _PyAccu acc; + assert(PyEncoder_Check(self)); s = (PyEncoderObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, &obj, _convertPyInt_AsSsize_t, &indent_level)) return NULL; - rval = PyList_New(0); - if (rval == NULL) + if (_PyAccu_Init(&acc)) return NULL; - if (encoder_listencode_obj(s, rval, obj, indent_level)) { - Py_DECREF(rval); + if (encoder_listencode_obj(s, &acc, obj, indent_level)) { + _PyAccu_Destroy(&acc); return NULL; } - return rval; + return _PyAccu_FinishAsList(&acc); } static PyObject * @@ -1297,18 +1367,19 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) } static int -_steal_list_append(PyObject *lst, PyObject *stolen) +_steal_accumulate(_PyAccu *acc, PyObject *stolen) { /* Append stolen and then decrement its reference count */ - int rval = PyList_Append(lst, stolen); + int rval = _PyAccu_Accumulate(acc, stolen); Py_DECREF(stolen); return rval; } static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) +encoder_listencode_obj(PyEncoderObject *s, _PyAccu *acc, + PyObject *obj, Py_ssize_t indent_level) { - /* Encode Python object obj to a JSON term, rval is a PyList */ + /* Encode Python object obj to a JSON term */ PyObject *newobj; int rv; @@ -1316,38 +1387,38 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi PyObject *cstr = _encoded_const(obj); if (cstr == NULL) return -1; - return _steal_list_append(rval, cstr); + return _steal_accumulate(acc, cstr); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyLong_Check(obj)) { PyObject *encoded = PyObject_Str(obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyFloat_Check(obj)) { PyObject *encoded = encoder_encode_float(s, obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, rval, obj, indent_level); + rv = encoder_listencode_list(s, acc, obj, indent_level); Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, rval, obj, indent_level); + rv = encoder_listencode_dict(s, acc, obj, indent_level); Py_LeaveRecursiveCall(); return rv; } @@ -1378,7 +1449,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_obj(s, rval, newobj, indent_level); + rv = encoder_listencode_obj(s, acc, newobj, indent_level); Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1398,9 +1469,10 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi } static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) +encoder_listencode_dict(PyEncoderObject *s, _PyAccu *acc, + PyObject *dct, Py_ssize_t indent_level) { - /* Encode Python dict dct a JSON term, rval is a PyList */ + /* Encode Python dict dct a JSON term */ static PyObject *open_dict = NULL; static PyObject *close_dict = NULL; static PyObject *empty_dict = NULL; @@ -1420,7 +1492,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss return -1; } if (Py_SIZE(dct) == 0) - return PyList_Append(rval, empty_dict); + return _PyAccu_Accumulate(acc, empty_dict); if (s->markers != Py_None) { int has_key; @@ -1438,7 +1510,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss } } - if (PyList_Append(rval, open_dict)) + if (_PyAccu_Accumulate(acc, open_dict)) goto bail; if (s->indent != Py_None) { @@ -1525,7 +1597,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss } if (idx) { - if (PyList_Append(rval, s->item_separator)) + if (_PyAccu_Accumulate(acc, s->item_separator)) goto bail; } @@ -1533,16 +1605,16 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss Py_CLEAR(kstr); if (encoded == NULL) goto bail; - if (PyList_Append(rval, encoded)) { + if (_PyAccu_Accumulate(acc, encoded)) { Py_DECREF(encoded); goto bail; } Py_DECREF(encoded); - if (PyList_Append(rval, s->key_separator)) + if (_PyAccu_Accumulate(acc, s->key_separator)) goto bail; value = PyTuple_GET_ITEM(item, 1); - if (encoder_listencode_obj(s, rval, value, indent_level)) + if (encoder_listencode_obj(s, acc, value, indent_level)) goto bail; idx += 1; Py_DECREF(item); @@ -1556,14 +1628,13 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss goto bail; Py_CLEAR(ident); } + /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ indent_level -= 1; - /* - yield '\n' + (' ' * (_indent * _current_indent_level)) - */ - } - if (PyList_Append(rval, close_dict)) + + yield '\n' + (' ' * (_indent * _current_indent_level)) + }*/ + if (_PyAccu_Accumulate(acc, close_dict)) goto bail; return 0; @@ -1577,9 +1648,10 @@ bail: static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) +encoder_listencode_list(PyEncoderObject *s, _PyAccu *acc, + PyObject *seq, Py_ssize_t indent_level) { - /* Encode Python list seq to a JSON term, rval is a PyList */ + /* Encode Python list seq to a JSON term */ static PyObject *open_array = NULL; static PyObject *close_array = NULL; static PyObject *empty_array = NULL; @@ -1603,7 +1675,7 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss num_items = PySequence_Fast_GET_SIZE(s_fast); if (num_items == 0) { Py_DECREF(s_fast); - return PyList_Append(rval, empty_array); + return _PyAccu_Accumulate(acc, empty_array); } if (s->markers != Py_None) { @@ -1623,7 +1695,7 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss } seq_items = PySequence_Fast_ITEMS(s_fast); - if (PyList_Append(rval, open_array)) + if (_PyAccu_Accumulate(acc, open_array)) goto bail; if (s->indent != Py_None) { /* TODO: DOES NOT RUN */ @@ -1637,10 +1709,10 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss for (i = 0; i < num_items; i++) { PyObject *obj = seq_items[i]; if (i) { - if (PyList_Append(rval, s->item_separator)) + if (_PyAccu_Accumulate(acc, s->item_separator)) goto bail; } - if (encoder_listencode_obj(s, rval, obj, indent_level)) + if (encoder_listencode_obj(s, acc, obj, indent_level)) goto bail; } if (ident != NULL) { @@ -1648,14 +1720,14 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss goto bail; Py_CLEAR(ident); } + + /* TODO: DOES NOT RUN if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ indent_level -= 1; - /* - yield '\n' + (' ' * (_indent * _current_indent_level)) - */ - } - if (PyList_Append(rval, close_array)) + + yield '\n' + (' ' * (_indent * _current_indent_level)) + }*/ + if (_PyAccu_Accumulate(acc, close_array)) goto bail; Py_DECREF(s_fast); return 0; diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index 1001dd2..9bba1b3 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -271,37 +271,28 @@ Return a string that can be used as a key for locale-aware comparisons."); static PyObject* PyLocale_strxfrm(PyObject* self, PyObject* args) { - Py_UNICODE *s0; - Py_ssize_t n0; - wchar_t *s, *buf = NULL; - size_t n1, n2; + PyObject *str; + Py_ssize_t n1; + wchar_t *s = NULL, *buf = NULL; + size_t n2; PyObject *result = NULL; -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T - Py_ssize_t i; -#endif - if (!PyArg_ParseTuple(args, "u#:strxfrm", &s0, &n0)) + if (!PyArg_ParseTuple(args, "U:strxfrm", &str)) return NULL; -#if Py_UNICODE_SIZE == SIZEOF_WCHAR_T - s = (wchar_t *) s0; -#else - s = PyMem_Malloc((n0+1)*sizeof(wchar_t)); - if (!s) - return PyErr_NoMemory(); - for (i=0; i<=n0; i++) - s[i] = s0[i]; -#endif + s = PyUnicode_AsWideCharString(str, &n1); + if (s == NULL) + goto exit; /* assume no change in size, first */ - n1 = wcslen(s) + 1; - buf = PyMem_Malloc(n1*sizeof(wchar_t)); + n1 = n1 + 1; + buf = PyMem_Malloc(n1 * sizeof(wchar_t)); if (!buf) { PyErr_NoMemory(); goto exit; } n2 = wcsxfrm(buf, s, n1); - if (n2 >= n1) { + if (n2 >= (size_t)n1) { /* more space needed */ buf = PyMem_Realloc(buf, (n2+1)*sizeof(wchar_t)); if (!buf) { @@ -311,11 +302,11 @@ PyLocale_strxfrm(PyObject* self, PyObject* args) n2 = wcsxfrm(buf, s, n2+1); } result = PyUnicode_FromWideChar(buf, n2); - exit: - if (buf) PyMem_Free(buf); -#if Py_UNICODE_SIZE != SIZEOF_WCHAR_T - PyMem_Free(s); -#endif +exit: + if (buf) + PyMem_Free(buf); + if (s) + PyMem_Free(s); return result; } #endif diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index cc412bf..b0a226b 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -176,36 +176,29 @@ normalizeUserObj(PyObject *obj) if (fn->m_self == NULL) { /* built-in function: look up the module name */ PyObject *mod = fn->m_module; - const char *modname; - if (mod && PyUnicode_Check(mod)) { - /* XXX: The following will truncate module names with embedded - * null-characters. It is unlikely that this can happen in - * practice and the concequences are not serious enough to - * introduce extra checks here. - */ - modname = _PyUnicode_AsString(mod); - if (modname == NULL) { - modname = "<encoding error>"; - PyErr_Clear(); + PyObject *modname = NULL; + if (mod != NULL) { + if (PyUnicode_Check(mod)) { + modname = mod; + Py_INCREF(modname); } - } - else if (mod && PyModule_Check(mod)) { - modname = PyModule_GetName(mod); - if (modname == NULL) { - PyErr_Clear(); - modname = "builtins"; + else if (PyModule_Check(mod)) { + modname = PyModule_GetNameObject(mod); + if (modname == NULL) + PyErr_Clear(); } } - else { - modname = "builtins"; + if (modname != NULL) { + if (PyUnicode_CompareWithASCIIString(modname, "builtins") != 0) { + PyObject *result; + result = PyUnicode_FromFormat("<%U.%s>", modname, + fn->m_ml->ml_name); + Py_DECREF(modname); + return result; + } + Py_DECREF(modname); } - if (strcmp(modname, "builtins") != 0) - return PyUnicode_FromFormat("<%s.%s>", - modname, - fn->m_ml->ml_name); - else - return PyUnicode_FromFormat("<%s>", - fn->m_ml->ml_name); + return PyUnicode_FromFormat("<%s>", fn->m_ml->ml_name); } else { /* built-in method: try to return diff --git a/Modules/_multiprocessing/connection.h b/Modules/_multiprocessing/connection.h deleted file mode 100644 index 002d5aa..0000000 --- a/Modules/_multiprocessing/connection.h +++ /dev/null @@ -1,527 +0,0 @@ -/* - * Definition of a `Connection` type. - * Used by `socket_connection.c` and `pipe_connection.c`. - * - * connection.h - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#ifndef CONNECTION_H -#define CONNECTION_H - -/* - * Read/write flags - */ - -#define READABLE 1 -#define WRITABLE 2 - -#define CHECK_READABLE(self) \ - if (!(self->flags & READABLE)) { \ - PyErr_SetString(PyExc_IOError, "connection is write-only"); \ - return NULL; \ - } - -#define CHECK_WRITABLE(self) \ - if (!(self->flags & WRITABLE)) { \ - PyErr_SetString(PyExc_IOError, "connection is read-only"); \ - return NULL; \ - } - -/* - * Allocation and deallocation - */ - -static PyObject * -connection_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - ConnectionObject *self; - HANDLE handle; - BOOL readable = TRUE, writable = TRUE; - - static char *kwlist[] = {"handle", "readable", "writable", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, F_HANDLE "|ii", kwlist, - &handle, &readable, &writable)) - return NULL; - - if (handle == INVALID_HANDLE_VALUE || (Py_ssize_t)handle < 0) { - PyErr_Format(PyExc_IOError, "invalid handle %zd", - (Py_ssize_t)handle); - return NULL; - } - - if (!readable && !writable) { - PyErr_SetString(PyExc_ValueError, - "either readable or writable must be true"); - return NULL; - } - - self = PyObject_New(ConnectionObject, type); - if (self == NULL) - return NULL; - - self->weakreflist = NULL; - self->handle = handle; - self->flags = 0; - - if (readable) - self->flags |= READABLE; - if (writable) - self->flags |= WRITABLE; - assert(self->flags >= 1 && self->flags <= 3); - - return (PyObject*)self; -} - -static void -connection_dealloc(ConnectionObject* self) -{ - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject*)self); - - if (self->handle != INVALID_HANDLE_VALUE) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - } - PyObject_Del(self); -} - -/* - * Functions for transferring buffers - */ - -static PyObject * -connection_sendbytes(ConnectionObject *self, PyObject *args) -{ - Py_buffer pbuffer; - char *buffer; - Py_ssize_t length, offset=0, size=PY_SSIZE_T_MIN; - int res; - - if (!PyArg_ParseTuple(args, F_RBUFFER "*|" F_PY_SSIZE_T F_PY_SSIZE_T, - &pbuffer, &offset, &size)) - return NULL; - buffer = pbuffer.buf; - length = pbuffer.len; - - CHECK_WRITABLE(self); /* XXX release buffer in case of failure */ - - if (offset < 0) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "offset is negative"); - return NULL; - } - if (length < offset) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "buffer length < offset"); - return NULL; - } - - if (size == PY_SSIZE_T_MIN) { - size = length - offset; - } else { - if (size < 0) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "size is negative"); - return NULL; - } - if (offset + size > length) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, - "buffer length < offset + size"); - return NULL; - } - } - - res = conn_send_string(self, buffer + offset, size); - - PyBuffer_Release(&pbuffer); - if (res < 0) { - if (PyErr_Occurred()) - return NULL; - else - return mp_SetError(PyExc_IOError, res); - } - - Py_RETURN_NONE; -} - -static PyObject * -connection_recvbytes(ConnectionObject *self, PyObject *args) -{ - char *freeme = NULL; - Py_ssize_t res, maxlength = PY_SSIZE_T_MAX; - PyObject *result = NULL; - - if (!PyArg_ParseTuple(args, "|" F_PY_SSIZE_T, &maxlength)) - return NULL; - - CHECK_READABLE(self); - - if (maxlength < 0) { - PyErr_SetString(PyExc_ValueError, "maxlength < 0"); - return NULL; - } - - res = conn_recv_string(self, self->buffer, CONNECTION_BUFFER_SIZE, - &freeme, maxlength); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - result = PyBytes_FromStringAndSize(self->buffer, res); - } else { - result = PyBytes_FromStringAndSize(freeme, res); - PyMem_Free(freeme); - } - } - - return result; -} - -static PyObject * -connection_recvbytes_into(ConnectionObject *self, PyObject *args) -{ - char *freeme = NULL, *buffer = NULL; - Py_ssize_t res, length, offset = 0; - PyObject *result = NULL; - Py_buffer pbuf; - - CHECK_READABLE(self); - - if (!PyArg_ParseTuple(args, "w*|" F_PY_SSIZE_T, - &pbuf, &offset)) - return NULL; - - buffer = pbuf.buf; - length = pbuf.len; - - if (offset < 0) { - PyErr_SetString(PyExc_ValueError, "negative offset"); - goto _error; - } - - if (offset > length) { - PyErr_SetString(PyExc_ValueError, "offset too large"); - goto _error; - } - - res = conn_recv_string(self, buffer+offset, length-offset, - &freeme, PY_SSIZE_T_MAX); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - result = PyInt_FromSsize_t(res); - } else { - result = PyObject_CallFunction(BufferTooShort, - F_RBUFFER "#", - freeme, res); - PyMem_Free(freeme); - if (result) { - PyErr_SetObject(BufferTooShort, result); - Py_DECREF(result); - } - goto _error; - } - } - -_cleanup: - PyBuffer_Release(&pbuf); - return result; - -_error: - result = NULL; - goto _cleanup; -} - -/* - * Functions for transferring objects - */ - -static PyObject * -connection_send_obj(ConnectionObject *self, PyObject *obj) -{ - char *buffer; - int res; - Py_ssize_t length; - PyObject *pickled_string = NULL; - - CHECK_WRITABLE(self); - - pickled_string = PyObject_CallFunctionObjArgs(pickle_dumps, obj, - pickle_protocol, NULL); - if (!pickled_string) - goto failure; - - if (PyBytes_AsStringAndSize(pickled_string, &buffer, &length) < 0) - goto failure; - - res = conn_send_string(self, buffer, (int)length); - - if (res < 0) { - mp_SetError(PyExc_IOError, res); - goto failure; - } - - Py_XDECREF(pickled_string); - Py_RETURN_NONE; - - failure: - Py_XDECREF(pickled_string); - return NULL; -} - -static PyObject * -connection_recv_obj(ConnectionObject *self) -{ - char *freeme = NULL; - Py_ssize_t res; - PyObject *temp = NULL, *result = NULL; - - CHECK_READABLE(self); - - res = conn_recv_string(self, self->buffer, CONNECTION_BUFFER_SIZE, - &freeme, PY_SSIZE_T_MAX); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - temp = PyBytes_FromStringAndSize(self->buffer, res); - } else { - temp = PyBytes_FromStringAndSize(freeme, res); - PyMem_Free(freeme); - } - } - - if (temp) - result = PyObject_CallFunctionObjArgs(pickle_loads, - temp, NULL); - Py_XDECREF(temp); - return result; -} - -/* - * Other functions - */ - -static PyObject * -connection_poll(ConnectionObject *self, PyObject *args) -{ - PyObject *timeout_obj = NULL; - double timeout = 0.0; - int res; - - CHECK_READABLE(self); - - if (!PyArg_ParseTuple(args, "|O", &timeout_obj)) - return NULL; - - if (timeout_obj == NULL) { - timeout = 0.0; - } else if (timeout_obj == Py_None) { - timeout = -1.0; /* block forever */ - } else { - timeout = PyFloat_AsDouble(timeout_obj); - if (PyErr_Occurred()) - return NULL; - if (timeout < 0.0) - timeout = 0.0; - } - - Py_BEGIN_ALLOW_THREADS - res = conn_poll(self, timeout, _save); - Py_END_ALLOW_THREADS - - switch (res) { - case TRUE: - Py_RETURN_TRUE; - case FALSE: - Py_RETURN_FALSE; - default: - return mp_SetError(PyExc_IOError, res); - } -} - -static PyObject * -connection_fileno(ConnectionObject* self) -{ - if (self->handle == INVALID_HANDLE_VALUE) { - PyErr_SetString(PyExc_IOError, "handle is invalid"); - return NULL; - } - return PyInt_FromLong((long)self->handle); -} - -static PyObject * -connection_close(ConnectionObject *self) -{ - if (self->handle != INVALID_HANDLE_VALUE) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } - - Py_RETURN_NONE; -} - -static PyObject * -connection_repr(ConnectionObject *self) -{ - static char *conn_type[] = {"read-only", "write-only", "read-write"}; - - assert(self->flags >= 1 && self->flags <= 3); - return FROM_FORMAT("<%s %s, handle %zd>", - conn_type[self->flags - 1], - CONNECTION_NAME, (Py_ssize_t)self->handle); -} - -/* - * Getters and setters - */ - -static PyObject * -connection_closed(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE)); -} - -static PyObject * -connection_readable(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->flags & READABLE)); -} - -static PyObject * -connection_writable(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->flags & WRITABLE)); -} - -/* - * Tables - */ - -static PyMethodDef connection_methods[] = { - {"send_bytes", (PyCFunction)connection_sendbytes, METH_VARARGS, - "send the byte data from a readable buffer-like object"}, - {"recv_bytes", (PyCFunction)connection_recvbytes, METH_VARARGS, - "receive byte data as a string"}, - {"recv_bytes_into",(PyCFunction)connection_recvbytes_into,METH_VARARGS, - "receive byte data into a writeable buffer-like object\n" - "returns the number of bytes read"}, - - {"send", (PyCFunction)connection_send_obj, METH_O, - "send a (picklable) object"}, - {"recv", (PyCFunction)connection_recv_obj, METH_NOARGS, - "receive a (picklable) object"}, - - {"poll", (PyCFunction)connection_poll, METH_VARARGS, - "whether there is any input available to be read"}, - {"fileno", (PyCFunction)connection_fileno, METH_NOARGS, - "file descriptor or handle of the connection"}, - {"close", (PyCFunction)connection_close, METH_NOARGS, - "close the connection"}, - - {NULL} /* Sentinel */ -}; - -static PyGetSetDef connection_getset[] = { - {"closed", (getter)connection_closed, NULL, - "True if the connection is closed", NULL}, - {"readable", (getter)connection_readable, NULL, - "True if the connection is readable", NULL}, - {"writable", (getter)connection_writable, NULL, - "True if the connection is writable", NULL}, - {NULL} -}; - -/* - * Connection type - */ - -PyDoc_STRVAR(connection_doc, - "Connection type whose constructor signature is\n\n" - " Connection(handle, readable=True, writable=True).\n\n" - "The constructor does *not* duplicate the handle."); - -PyTypeObject CONNECTION_TYPE = { - PyVarObject_HEAD_INIT(NULL, 0) - /* tp_name */ "_multiprocessing." CONNECTION_NAME, - /* tp_basicsize */ sizeof(ConnectionObject), - /* tp_itemsize */ 0, - /* tp_dealloc */ (destructor)connection_dealloc, - /* tp_print */ 0, - /* tp_getattr */ 0, - /* tp_setattr */ 0, - /* tp_reserved */ 0, - /* tp_repr */ (reprfunc)connection_repr, - /* tp_as_number */ 0, - /* tp_as_sequence */ 0, - /* tp_as_mapping */ 0, - /* tp_hash */ 0, - /* tp_call */ 0, - /* tp_str */ 0, - /* tp_getattro */ 0, - /* tp_setattro */ 0, - /* tp_as_buffer */ 0, - /* tp_flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_WEAKREFS, - /* tp_doc */ connection_doc, - /* tp_traverse */ 0, - /* tp_clear */ 0, - /* tp_richcompare */ 0, - /* tp_weaklistoffset */ offsetof(ConnectionObject, weakreflist), - /* tp_iter */ 0, - /* tp_iternext */ 0, - /* tp_methods */ connection_methods, - /* tp_members */ 0, - /* tp_getset */ connection_getset, - /* tp_base */ 0, - /* tp_dict */ 0, - /* tp_descr_get */ 0, - /* tp_descr_set */ 0, - /* tp_dictoffset */ 0, - /* tp_init */ 0, - /* tp_alloc */ 0, - /* tp_new */ connection_new, -}; - -#endif /* CONNECTION_H */ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 83df1eb..890b96d 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -8,15 +8,9 @@ #include "multiprocessing.h" -#ifdef SCM_RIGHTS - #define HAVE_FD_TRANSFER 1 -#else - #define HAVE_FD_TRANSFER 0 -#endif PyObject *create_win32_namespace(void); -PyObject *pickle_dumps, *pickle_loads, *pickle_protocol; PyObject *ProcessError, *BufferTooShort; /* @@ -49,16 +43,6 @@ mp_SetError(PyObject *Type, int num) case MP_MEMORY_ERROR: PyErr_NoMemory(); break; - case MP_END_OF_FILE: - PyErr_SetNone(PyExc_EOFError); - break; - case MP_EARLY_END_OF_FILE: - PyErr_SetString(PyExc_IOError, - "got end of file during message"); - break; - case MP_BAD_MESSAGE_LENGTH: - PyErr_SetString(PyExc_IOError, "bad message length"); - break; case MP_EXCEPTION_HAS_BEEN_SET: break; default: @@ -86,115 +70,7 @@ ProcessingCtrlHandler(DWORD dwCtrlType) return FALSE; } -/* - * Unix only - */ - -#else /* !MS_WINDOWS */ - -#if HAVE_FD_TRANSFER - -/* Functions for transferring file descriptors between processes. - Reimplements some of the functionality of the fdcred - module at http://www.mca-ltd.com/resources/fdcred_1.tgz. */ -/* Based in http://resin.csoft.net/cgi-bin/man.cgi?section=3&topic=CMSG_DATA */ - -static PyObject * -multiprocessing_sendfd(PyObject *self, PyObject *args) -{ - int conn, fd, res; - struct iovec dummy_iov; - char dummy_char; - struct msghdr msg; - struct cmsghdr *cmsg; - union { - struct cmsghdr hdr; - unsigned char buf[CMSG_SPACE(sizeof(int))]; - } cmsgbuf; - - if (!PyArg_ParseTuple(args, "ii", &conn, &fd)) - return NULL; - - dummy_iov.iov_base = &dummy_char; - dummy_iov.iov_len = 1; - - memset(&msg, 0, sizeof(msg)); - msg.msg_control = &cmsgbuf.buf; - msg.msg_controllen = sizeof(cmsgbuf.buf); - msg.msg_iov = &dummy_iov; - msg.msg_iovlen = 1; - - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_len = CMSG_LEN(sizeof(int)); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - * (int *) CMSG_DATA(cmsg) = fd; - - Py_BEGIN_ALLOW_THREADS - res = sendmsg(conn, &msg, 0); - Py_END_ALLOW_THREADS - - if (res < 0) - return PyErr_SetFromErrno(PyExc_OSError); - Py_RETURN_NONE; -} - -static PyObject * -multiprocessing_recvfd(PyObject *self, PyObject *args) -{ - int conn, fd, res; - char dummy_char; - struct iovec dummy_iov; - struct msghdr msg = {0}; - struct cmsghdr *cmsg; - union { - struct cmsghdr hdr; - unsigned char buf[CMSG_SPACE(sizeof(int))]; - } cmsgbuf; - - if (!PyArg_ParseTuple(args, "i", &conn)) - return NULL; - - dummy_iov.iov_base = &dummy_char; - dummy_iov.iov_len = 1; - - memset(&msg, 0, sizeof(msg)); - msg.msg_control = &cmsgbuf.buf; - msg.msg_controllen = sizeof(cmsgbuf.buf); - msg.msg_iov = &dummy_iov; - msg.msg_iovlen = 1; - - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(int)); - msg.msg_controllen = cmsg->cmsg_len; - - Py_BEGIN_ALLOW_THREADS - res = recvmsg(conn, &msg, 0); - Py_END_ALLOW_THREADS - - if (res < 0) - return PyErr_SetFromErrno(PyExc_OSError); - - if (msg.msg_controllen < CMSG_LEN(sizeof(int)) || - (cmsg = CMSG_FIRSTHDR(&msg)) == NULL || - cmsg->cmsg_level != SOL_SOCKET || - cmsg->cmsg_type != SCM_RIGHTS || - cmsg->cmsg_len < CMSG_LEN(sizeof(int))) { - /* If at least one control message is present, there should be - no room for any further data in the buffer. */ - PyErr_SetString(PyExc_RuntimeError, "No file descriptor received"); - return NULL; - } - - fd = * (int *) CMSG_DATA(cmsg); - return Py_BuildValue("i", fd); -} - -#endif /* HAVE_FD_TRANSFER */ - -#endif /* !MS_WINDOWS */ +#endif /* MS_WINDOWS */ /* @@ -210,7 +86,7 @@ multiprocessing_address_of_buffer(PyObject *self, PyObject *obj) if (PyObject_AsWriteBuffer(obj, &buffer, &buffer_len) < 0) return NULL; - return Py_BuildValue("N" F_PY_SSIZE_T, + return Py_BuildValue("Nn", PyLong_FromVoidPtr(buffer), buffer_len); } @@ -223,16 +99,6 @@ static PyMethodDef module_methods[] = { {"address_of_buffer", multiprocessing_address_of_buffer, METH_O, "address_of_buffer(obj) -> int\n" "Return address of obj assuming obj supports buffer inteface"}, -#if HAVE_FD_TRANSFER - {"sendfd", multiprocessing_sendfd, METH_VARARGS, - "sendfd(sockfd, fd) -> None\n" - "Send file descriptor given by fd over the unix domain socket\n" - "whose file decriptor is sockfd"}, - {"recvfd", multiprocessing_recvfd, METH_VARARGS, - "recvfd(sockfd) -> fd\n" - "Receive a file descriptor over a unix domain socket\n" - "whose file decriptor is sockfd"}, -#endif {NULL} }; @@ -264,15 +130,6 @@ PyInit__multiprocessing(void) if (!module) return NULL; - /* Get copy of objects from pickle */ - temp = PyImport_ImportModule(PICKLE_MODULE); - if (!temp) - return NULL; - pickle_dumps = PyObject_GetAttrString(temp, "dumps"); - pickle_loads = PyObject_GetAttrString(temp, "loads"); - pickle_protocol = PyObject_GetAttrString(temp, "HIGHEST_PROTOCOL"); - Py_XDECREF(temp); - /* Get copy of BufferTooShort */ temp = PyImport_ImportModule("multiprocessing"); if (!temp) @@ -280,12 +137,6 @@ PyInit__multiprocessing(void) BufferTooShort = PyObject_GetAttrString(temp, "BufferTooShort"); Py_XDECREF(temp); - /* Add connection type to module */ - if (PyType_Ready(&ConnectionType) < 0) - return NULL; - Py_INCREF(&ConnectionType); - PyModule_AddObject(module, "Connection", (PyObject*)&ConnectionType); - #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) /* Add SemLock type to module */ @@ -309,13 +160,6 @@ PyInit__multiprocessing(void) #endif #ifdef MS_WINDOWS - /* Add PipeConnection to module */ - if (PyType_Ready(&PipeConnectionType) < 0) - return NULL; - Py_INCREF(&PipeConnectionType); - PyModule_AddObject(module, "PipeConnection", - (PyObject*)&PipeConnectionType); - /* Initialize win32 class and add to multiprocessing */ temp = create_win32_namespace(); if (!temp) @@ -352,9 +196,6 @@ PyInit__multiprocessing(void) #ifdef HAVE_SEM_TIMEDWAIT ADD_FLAG(HAVE_SEM_TIMEDWAIT); #endif -#ifdef HAVE_FD_TRANSFER - ADD_FLAG(HAVE_FD_TRANSFER); -#endif #ifdef HAVE_BROKEN_SEM_GETVALUE ADD_FLAG(HAVE_BROKEN_SEM_GETVALUE); #endif diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index 14425de..ac0dfd7 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -3,12 +3,6 @@ #define PY_SSIZE_T_CLEAN -#ifdef __sun -/* The control message API is only available on Solaris - if XPG 4.2 or later is requested. */ -#define _XOPEN_SOURCE 500 -#endif - #include "Python.h" #include "structmember.h" #include "pythread.h" @@ -29,10 +23,6 @@ # define SEM_VALUE_MAX LONG_MAX #else # include <fcntl.h> /* O_CREAT and O_EXCL */ -# include <netinet/in.h> -# include <sys/socket.h> -# include <sys/uio.h> -# include <arpa/inet.h> /* htonl() and ntohl() */ # if defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED) # include <semaphore.h> typedef sem_t *SEM_HANDLE; @@ -64,20 +54,6 @@ /* - * Make sure Py_ssize_t available - */ - -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) - typedef int Py_ssize_t; -# define PY_SSIZE_T_MAX INT_MAX -# define PY_SSIZE_T_MIN INT_MIN -# define F_PY_SSIZE_T "i" -# define PyInt_FromSsize_t(n) PyInt_FromLong((long)n) -#else -# define F_PY_SSIZE_T "n" -#endif - -/* * Format codes */ @@ -105,12 +81,6 @@ # define T_SEM_HANDLE T_POINTER #endif -#if PY_VERSION_HEX >= 0x03000000 -# define F_RBUFFER "y" -#else -# define F_RBUFFER "s" -#endif - /* * Error codes which can be returned by functions called without GIL */ @@ -118,11 +88,8 @@ #define MP_SUCCESS (0) #define MP_STANDARD_ERROR (-1) #define MP_MEMORY_ERROR (-1001) -#define MP_END_OF_FILE (-1002) -#define MP_EARLY_END_OF_FILE (-1003) -#define MP_BAD_MESSAGE_LENGTH (-1004) -#define MP_SOCKET_ERROR (-1005) -#define MP_EXCEPTION_HAS_BEEN_SET (-1006) +#define MP_SOCKET_ERROR (-1002) +#define MP_EXCEPTION_HAS_BEEN_SET (-1003) PyObject *mp_SetError(PyObject *Type, int num); @@ -130,57 +97,15 @@ PyObject *mp_SetError(PyObject *Type, int num); * Externs - not all will really exist on all platforms */ -extern PyObject *pickle_dumps; -extern PyObject *pickle_loads; -extern PyObject *pickle_protocol; extern PyObject *BufferTooShort; extern PyTypeObject SemLockType; -extern PyTypeObject ConnectionType; extern PyTypeObject PipeConnectionType; extern HANDLE sigint_event; /* - * Py3k compatibility - */ - -#if PY_VERSION_HEX >= 0x03000000 -# define PICKLE_MODULE "pickle" -# define FROM_FORMAT PyUnicode_FromFormat -# define PyInt_FromLong PyLong_FromLong -# define PyInt_FromSsize_t PyLong_FromSsize_t -#else -# define PICKLE_MODULE "cPickle" -# define FROM_FORMAT PyString_FromFormat -#endif - -#ifndef PyVarObject_HEAD_INIT -# define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif - -#ifndef Py_TPFLAGS_HAVE_WEAKREFS -# define Py_TPFLAGS_HAVE_WEAKREFS 0 -#endif - -/* - * Connection definition - */ - -#define CONNECTION_BUFFER_SIZE 1024 - -typedef struct { - PyObject_HEAD - HANDLE handle; - int flags; - PyObject *weakreflist; - char buffer[CONNECTION_BUFFER_SIZE]; -} ConnectionObject; - -/* * Miscellaneous */ -#define MAX_MESSAGE_LENGTH 0x7fffffff - #ifndef MIN # define MIN(x, y) ((x) < (y) ? x : y) # define MAX(x, y) ((x) > (y) ? x : y) diff --git a/Modules/_multiprocessing/pipe_connection.c b/Modules/_multiprocessing/pipe_connection.c deleted file mode 100644 index 05dde0c..0000000 --- a/Modules/_multiprocessing/pipe_connection.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * A type which wraps a pipe handle in message oriented mode - * - * pipe_connection.c - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#include "multiprocessing.h" - -#define CLOSE(h) CloseHandle(h) - -/* - * Send string to the pipe; assumes in message oriented mode - */ - -static Py_ssize_t -conn_send_string(ConnectionObject *conn, char *string, size_t length) -{ - DWORD amount_written; - BOOL ret; - - Py_BEGIN_ALLOW_THREADS - ret = WriteFile(conn->handle, string, length, &amount_written, NULL); - Py_END_ALLOW_THREADS - - if (ret == 0 && GetLastError() == ERROR_NO_SYSTEM_RESOURCES) { - PyErr_Format(PyExc_ValueError, "Cannnot send %" PY_FORMAT_SIZE_T "d bytes over connection", length); - return MP_STANDARD_ERROR; - } - - return ret ? MP_SUCCESS : MP_STANDARD_ERROR; -} - -/* - * Attempts to read into buffer, or if buffer too small into *newbuffer. - * - * Returns number of bytes read. Assumes in message oriented mode. - */ - -static Py_ssize_t -conn_recv_string(ConnectionObject *conn, char *buffer, - size_t buflength, char **newbuffer, size_t maxlength) -{ - DWORD left, length, full_length, err; - BOOL ret; - *newbuffer = NULL; - - Py_BEGIN_ALLOW_THREADS - ret = ReadFile(conn->handle, buffer, MIN(buflength, maxlength), - &length, NULL); - Py_END_ALLOW_THREADS - if (ret) - return length; - - err = GetLastError(); - if (err != ERROR_MORE_DATA) { - if (err == ERROR_BROKEN_PIPE) - return MP_END_OF_FILE; - return MP_STANDARD_ERROR; - } - - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, NULL, &left)) - return MP_STANDARD_ERROR; - - full_length = length + left; - if (full_length > maxlength) - return MP_BAD_MESSAGE_LENGTH; - - *newbuffer = PyMem_Malloc(full_length); - if (*newbuffer == NULL) - return MP_MEMORY_ERROR; - - memcpy(*newbuffer, buffer, length); - - Py_BEGIN_ALLOW_THREADS - ret = ReadFile(conn->handle, *newbuffer+length, left, &length, NULL); - Py_END_ALLOW_THREADS - if (ret) { - assert(length == left); - return full_length; - } else { - PyMem_Free(*newbuffer); - return MP_STANDARD_ERROR; - } -} - -/* - * Check whether any data is available for reading - */ - -static int -conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) -{ - DWORD bytes, deadline, delay; - int difference, res; - BOOL block = FALSE; - - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, &bytes, NULL)) - return MP_STANDARD_ERROR; - - if (timeout == 0.0) - return bytes > 0; - - if (timeout < 0.0) - block = TRUE; - else - /* XXX does not check for overflow */ - deadline = GetTickCount() + (DWORD)(1000 * timeout + 0.5); - - Sleep(0); - - for (delay = 1 ; ; delay += 1) { - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, &bytes, NULL)) - return MP_STANDARD_ERROR; - else if (bytes > 0) - return TRUE; - - if (!block) { - difference = deadline - GetTickCount(); - if (difference < 0) - return FALSE; - if ((int)delay > difference) - delay = difference; - } - - if (delay > 20) - delay = 20; - - Sleep(delay); - - /* check for signals */ - Py_BLOCK_THREADS - res = PyErr_CheckSignals(); - Py_UNBLOCK_THREADS - - if (res) - return MP_EXCEPTION_HAS_BEEN_SET; - } -} - -/* - * "connection.h" defines the PipeConnection type using the definitions above - */ - -#define CONNECTION_NAME "PipeConnection" -#define CONNECTION_TYPE PipeConnectionType - -#include "connection.h" diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index c2cd914..6749f23 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -481,7 +481,7 @@ semlock_dealloc(SemLockObject* self) static PyObject * semlock_count(SemLockObject *self) { - return PyInt_FromLong((long)self->count); + return PyLong_FromLong((long)self->count); } static PyObject * @@ -505,7 +505,7 @@ semlock_getvalue(SemLockObject *self) the number of waiting threads */ if (sval < 0) sval = 0; - return PyInt_FromLong((long)sval); + return PyLong_FromLong((long)sval); #endif } diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c deleted file mode 100644 index 7ebf338..0000000 --- a/Modules/_multiprocessing/socket_connection.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * A type which wraps a socket - * - * socket_connection.c - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#include "multiprocessing.h" - -#ifdef MS_WINDOWS -# define WRITE(h, buffer, length) send((SOCKET)h, buffer, length, 0) -# define READ(h, buffer, length) recv((SOCKET)h, buffer, length, 0) -# define CLOSE(h) closesocket((SOCKET)h) -#else -# define WRITE(h, buffer, length) write(h, buffer, length) -# define READ(h, buffer, length) read(h, buffer, length) -# define CLOSE(h) close(h) -#endif - -/* - * Send string to file descriptor - */ - -static Py_ssize_t -_conn_sendall(HANDLE h, char *string, size_t length) -{ - char *p = string; - Py_ssize_t res; - - while (length > 0) { - res = WRITE(h, p, length); - if (res < 0) - return MP_SOCKET_ERROR; - length -= res; - p += res; - } - - return MP_SUCCESS; -} - -/* - * Receive string of exact length from file descriptor - */ - -static Py_ssize_t -_conn_recvall(HANDLE h, char *buffer, size_t length) -{ - size_t remaining = length; - Py_ssize_t temp; - char *p = buffer; - - while (remaining > 0) { - temp = READ(h, p, remaining); - if (temp <= 0) { - if (temp == 0) - return remaining == length ? - MP_END_OF_FILE : MP_EARLY_END_OF_FILE; - else - return temp; - } - remaining -= temp; - p += temp; - } - - return MP_SUCCESS; -} - -/* - * Send a string prepended by the string length in network byte order - */ - -static Py_ssize_t -conn_send_string(ConnectionObject *conn, char *string, size_t length) -{ - Py_ssize_t res; - /* The "header" of the message is a 32 bit unsigned number (in - network order) which specifies the length of the "body". If - the message is shorter than about 16kb then it is quicker to - combine the "header" and the "body" of the message and send - them at once. */ - if (length < (16*1024)) { - char *message; - - message = PyMem_Malloc(length+4); - if (message == NULL) - return MP_MEMORY_ERROR; - - *(UINT32*)message = htonl((UINT32)length); - memcpy(message+4, string, length); - Py_BEGIN_ALLOW_THREADS - res = _conn_sendall(conn->handle, message, length+4); - Py_END_ALLOW_THREADS - PyMem_Free(message); - } else { - UINT32 lenbuff; - - if (length > MAX_MESSAGE_LENGTH) - return MP_BAD_MESSAGE_LENGTH; - - lenbuff = htonl((UINT32)length); - Py_BEGIN_ALLOW_THREADS - res = _conn_sendall(conn->handle, (char*)&lenbuff, 4) || - _conn_sendall(conn->handle, string, length); - Py_END_ALLOW_THREADS - } - return res; -} - -/* - * Attempts to read into buffer, or failing that into *newbuffer - * - * Returns number of bytes read. - */ - -static Py_ssize_t -conn_recv_string(ConnectionObject *conn, char *buffer, - size_t buflength, char **newbuffer, size_t maxlength) -{ - int res; - UINT32 ulength; - - *newbuffer = NULL; - - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, (char*)&ulength, 4); - Py_END_ALLOW_THREADS - if (res < 0) - return res; - - ulength = ntohl(ulength); - if (ulength > maxlength) - return MP_BAD_MESSAGE_LENGTH; - - if (ulength <= buflength) { - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, buffer, (size_t)ulength); - Py_END_ALLOW_THREADS - return res < 0 ? res : ulength; - } else { - *newbuffer = PyMem_Malloc((size_t)ulength); - if (*newbuffer == NULL) - return MP_MEMORY_ERROR; - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, *newbuffer, (size_t)ulength); - Py_END_ALLOW_THREADS - return res < 0 ? (Py_ssize_t)res : (Py_ssize_t)ulength; - } -} - -/* - * Check whether any data is available for reading -- neg timeout blocks - */ - -static int -conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) -{ - int res; - fd_set rfds; - - /* - * Verify the handle, issue 3321. Not required for windows. - */ - #ifndef MS_WINDOWS - if (((int)conn->handle) < 0 || ((int)conn->handle) >= FD_SETSIZE) { - Py_BLOCK_THREADS - PyErr_SetString(PyExc_IOError, "handle out of range in select()"); - Py_UNBLOCK_THREADS - return MP_EXCEPTION_HAS_BEEN_SET; - } - #endif - - FD_ZERO(&rfds); - FD_SET((SOCKET)conn->handle, &rfds); - - if (timeout < 0.0) { - res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); - } else { - struct timeval tv; - tv.tv_sec = (long)timeout; - tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5); - res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv); - } - - if (res < 0) { - return MP_SOCKET_ERROR; - } else if (FD_ISSET(conn->handle, &rfds)) { - return TRUE; - } else { - assert(res == 0); - return FALSE; - } -} - -/* - * "connection.h" defines the Connection type using defs above - */ - -#define CONNECTION_NAME "Connection" -#define CONNECTION_TYPE ConnectionType - -#include "connection.h" diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index 452d608..c017b2a 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -12,10 +12,223 @@ #define WIN32_FUNCTION(func) \ {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_STATIC, ""} +#define WIN32_KWARGS_FUNCTION(func) \ + {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_KEYWORDS | METH_STATIC, ""} + #define WIN32_CONSTANT(fmt, con) \ PyDict_SetItemString(Win32Type.tp_dict, #con, Py_BuildValue(fmt, con)) +/* Grab CancelIoEx dynamically from kernel32 */ +static int has_CancelIoEx = -1; +static BOOL (CALLBACK *Py_CancelIoEx)(HANDLE, LPOVERLAPPED); + +static int +check_CancelIoEx() +{ + if (has_CancelIoEx == -1) + { + HINSTANCE hKernel32 = GetModuleHandle("KERNEL32"); + * (FARPROC *) &Py_CancelIoEx = GetProcAddress(hKernel32, + "CancelIoEx"); + has_CancelIoEx = (Py_CancelIoEx != NULL); + } + return has_CancelIoEx; +} + + +/* + * A Python object wrapping an OVERLAPPED structure and other useful data + * for overlapped I/O + */ + +typedef struct { + PyObject_HEAD + OVERLAPPED overlapped; + /* For convenience, we store the file handle too */ + HANDLE handle; + /* Whether there's I/O in flight */ + int pending; + /* Whether I/O completed successfully */ + int completed; + /* Buffer used for reading (optional) */ + PyObject *read_buffer; + /* Buffer used for writing (optional) */ + Py_buffer write_buffer; +} OverlappedObject; + +static void +overlapped_dealloc(OverlappedObject *self) +{ + int err = GetLastError(); + if (self->pending) { + if (check_CancelIoEx()) + Py_CancelIoEx(self->handle, &self->overlapped); + else { + PyErr_SetString(PyExc_RuntimeError, + "I/O operations still in flight while destroying " + "Overlapped object, the process may crash"); + PyErr_WriteUnraisable(NULL); + } + } + CloseHandle(self->overlapped.hEvent); + SetLastError(err); + if (self->write_buffer.obj) + PyBuffer_Release(&self->write_buffer); + Py_CLEAR(self->read_buffer); + PyObject_Del(self); +} + +static PyObject * +overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *waitobj) +{ + int wait; + BOOL res; + DWORD transferred = 0; + + wait = PyObject_IsTrue(waitobj); + if (wait < 0) + return NULL; + Py_BEGIN_ALLOW_THREADS + res = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait != 0); + Py_END_ALLOW_THREADS + + if (!res) { + int err = GetLastError(); + if (err == ERROR_IO_INCOMPLETE) + Py_RETURN_NONE; + if (err != ERROR_MORE_DATA) { + self->pending = 0; + return PyErr_SetExcFromWindowsErr(PyExc_IOError, err); + } + } + self->pending = 0; + self->completed = 1; + if (self->read_buffer) { + assert(PyBytes_CheckExact(self->read_buffer)); + if (_PyBytes_Resize(&self->read_buffer, transferred)) + return NULL; + } + return Py_BuildValue("lN", (long) transferred, PyBool_FromLong(res)); +} + +static PyObject * +overlapped_getbuffer(OverlappedObject *self) +{ + PyObject *res; + if (!self->completed) { + PyErr_SetString(PyExc_ValueError, + "can't get read buffer before GetOverlappedResult() " + "signals the operation completed"); + return NULL; + } + res = self->read_buffer ? self->read_buffer : Py_None; + Py_INCREF(res); + return res; +} + +static PyObject * +overlapped_cancel(OverlappedObject *self) +{ + BOOL res = TRUE; + + if (self->pending) { + Py_BEGIN_ALLOW_THREADS + if (check_CancelIoEx()) + res = Py_CancelIoEx(self->handle, &self->overlapped); + else + res = CancelIo(self->handle); + Py_END_ALLOW_THREADS + } + + /* CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between */ + if (!res && GetLastError() != ERROR_NOT_FOUND) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + self->pending = 0; + Py_RETURN_NONE; +} + +static PyMethodDef overlapped_methods[] = { + {"GetOverlappedResult", (PyCFunction) overlapped_GetOverlappedResult, + METH_O, NULL}, + {"getbuffer", (PyCFunction) overlapped_getbuffer, METH_NOARGS, NULL}, + {"cancel", (PyCFunction) overlapped_cancel, METH_NOARGS, NULL}, + {NULL} +}; + +static PyMemberDef overlapped_members[] = { + {"event", T_HANDLE, + offsetof(OverlappedObject, overlapped) + offsetof(OVERLAPPED, hEvent), + READONLY, "overlapped event handle"}, + {NULL} +}; + +PyTypeObject OverlappedType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* tp_name */ "_multiprocessing.win32.Overlapped", + /* tp_basicsize */ sizeof(OverlappedObject), + /* tp_itemsize */ 0, + /* tp_dealloc */ (destructor) overlapped_dealloc, + /* tp_print */ 0, + /* tp_getattr */ 0, + /* tp_setattr */ 0, + /* tp_reserved */ 0, + /* tp_repr */ 0, + /* tp_as_number */ 0, + /* tp_as_sequence */ 0, + /* tp_as_mapping */ 0, + /* tp_hash */ 0, + /* tp_call */ 0, + /* tp_str */ 0, + /* tp_getattro */ 0, + /* tp_setattro */ 0, + /* tp_as_buffer */ 0, + /* tp_flags */ Py_TPFLAGS_DEFAULT, + /* tp_doc */ "OVERLAPPED structure wrapper", + /* tp_traverse */ 0, + /* tp_clear */ 0, + /* tp_richcompare */ 0, + /* tp_weaklistoffset */ 0, + /* tp_iter */ 0, + /* tp_iternext */ 0, + /* tp_methods */ overlapped_methods, + /* tp_members */ overlapped_members, + /* tp_getset */ 0, + /* tp_base */ 0, + /* tp_dict */ 0, + /* tp_descr_get */ 0, + /* tp_descr_set */ 0, + /* tp_dictoffset */ 0, + /* tp_init */ 0, + /* tp_alloc */ 0, + /* tp_new */ 0, +}; + +static OverlappedObject * +new_overlapped(HANDLE handle) +{ + OverlappedObject *self; + + self = PyObject_New(OverlappedObject, &OverlappedType); + if (!self) + return NULL; + self->handle = handle; + self->read_buffer = NULL; + self->pending = 0; + self->completed = 0; + memset(&self->overlapped, 0, sizeof(OVERLAPPED)); + memset(&self->write_buffer, 0, sizeof(Py_buffer)); + /* Manual reset, initially non-signalled */ + self->overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + return self; +} + + +/* + * Module functions + */ + static PyObject * win32_CloseHandle(PyObject *self, PyObject *args) { @@ -36,20 +249,44 @@ win32_CloseHandle(PyObject *self, PyObject *args) } static PyObject * -win32_ConnectNamedPipe(PyObject *self, PyObject *args) +win32_ConnectNamedPipe(PyObject *self, PyObject *args, PyObject *kwds) { HANDLE hNamedPipe; - LPOVERLAPPED lpOverlapped; + int use_overlapped = 0; BOOL success; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "overlapped", NULL}; - if (!PyArg_ParseTuple(args, F_HANDLE F_POINTER, - &hNamedPipe, &lpOverlapped)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "|i", kwlist, + &hNamedPipe, &use_overlapped)) return NULL; + if (use_overlapped) { + overlapped = new_overlapped(hNamedPipe); + if (!overlapped) + return NULL; + } + Py_BEGIN_ALLOW_THREADS - success = ConnectNamedPipe(hNamedPipe, lpOverlapped); + success = ConnectNamedPipe(hNamedPipe, + overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + if (overlapped) { + int err = GetLastError(); + /* Overlapped ConnectNamedPipe never returns a success code */ + assert(success == 0); + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err == ERROR_PIPE_CONNECTED) + SetEvent(overlapped->overlapped.hEvent); + else { + Py_DECREF(overlapped); + return PyErr_SetFromWindowsErr(err); + } + return (PyObject *) overlapped; + } if (!success) return PyErr_SetFromWindowsErr(0); @@ -215,16 +452,295 @@ win32_WaitNamedPipe(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +win32_closesocket(PyObject *self, PyObject *args) +{ + HANDLE handle; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE ":closesocket" , &handle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = closesocket((SOCKET) handle); + Py_END_ALLOW_THREADS + + if (ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + Py_RETURN_NONE; +} + +static PyObject * +win32_recv(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size, nread; + PyObject *buf; + + if (!PyArg_ParseTuple(args, F_HANDLE "i:recv" , &handle, &size)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + + Py_BEGIN_ALLOW_THREADS + nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0); + Py_END_ALLOW_THREADS + + if (nread < 0) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + } + _PyBytes_Resize(&buf, nread); + return buf; +} + +static PyObject * +win32_send(PyObject *self, PyObject *args) +{ + HANDLE handle; + Py_buffer buf; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = send((SOCKET) handle, buf.buf, buf.len, 0); + Py_END_ALLOW_THREADS + + PyBuffer_Release(&buf); + if (ret < 0) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + return PyLong_FromLong(ret); +} + +static PyObject * +win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) +{ + HANDLE handle; + Py_buffer _buf, *buf; + PyObject *bufobj; + int written; + BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "buffer", "overlapped", NULL}; + + /* First get handle and use_overlapped to know which Py_buffer to use */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "O|i:WriteFile", kwlist, + &handle, &bufobj, &use_overlapped)) + return NULL; + + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) + return NULL; + buf = &overlapped->write_buffer; + } + else + buf = &_buf; + + if (!PyArg_Parse(bufobj, "y*", buf)) { + Py_XDECREF(overlapped); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = WriteFile(handle, buf->buf, buf->len, &written, + overlapped ? &overlapped->overlapped : NULL); + Py_END_ALLOW_THREADS + + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + + PyBuffer_Release(buf); + if (!ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + return PyLong_FromLong(written); +} + +static PyObject * +win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) +{ + HANDLE handle; + int size; + DWORD nread; + PyObject *buf; + BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "size", "overlapped", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "i|i:ReadFile", kwlist, + &handle, &size, &use_overlapped)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) { + Py_DECREF(buf); + return NULL; + } + /* Steals reference to buf */ + overlapped->read_buffer = buf; + } + + Py_BEGIN_ALLOW_THREADS + ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, + overlapped ? &overlapped->overlapped : NULL); + Py_END_ALLOW_THREADS + + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err != ERROR_MORE_DATA) { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + + if (!ret && GetLastError() != ERROR_MORE_DATA) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("NN", buf, PyBool_FromLong(ret)); +} + +static PyObject * +win32_PeekNamedPipe(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size = 0; + PyObject *buf = NULL; + DWORD nread, navail, nleft; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "|i:PeekNamedPipe" , &handle, &size)) + return NULL; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative size"); + return NULL; + } + + if (size) { + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, PyBytes_AS_STRING(buf), size, &nread, + &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("Nii", buf, navail, nleft); + } + else { + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, NULL, 0, NULL, &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + return Py_BuildValue("ii", navail, nleft); + } +} + +static PyObject * +win32_WaitForMultipleObjects(PyObject* self, PyObject* args) +{ + DWORD result; + PyObject *handle_seq; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + Py_ssize_t nhandles, i; + int wait_flag; + int milliseconds = INFINITE; + + if (!PyArg_ParseTuple(args, "Oi|i:WaitForMultipleObjects", + &handle_seq, &wait_flag, &milliseconds)) + return NULL; + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_doc); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) + return NULL; + if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + MAXIMUM_WAIT_OBJECTS, nhandles); + return NULL; + } + for (i = 0; i < nhandles; i++) { + HANDLE h; + PyObject *v = PySequence_GetItem(handle_seq, i); + if (v == NULL) + return NULL; + if (!PyArg_Parse(v, F_HANDLE, &h)) + return NULL; + handles[i] = h; + } + + Py_BEGIN_ALLOW_THREADS + result = WaitForMultipleObjects((DWORD) nhandles, handles, + (BOOL) wait_flag, (DWORD) milliseconds); + Py_END_ALLOW_THREADS + + if (result == WAIT_FAILED) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + + return PyLong_FromLong((int) result); +} + + static PyMethodDef win32_methods[] = { WIN32_FUNCTION(CloseHandle), WIN32_FUNCTION(GetLastError), WIN32_FUNCTION(OpenProcess), WIN32_FUNCTION(ExitProcess), - WIN32_FUNCTION(ConnectNamedPipe), + WIN32_KWARGS_FUNCTION(ConnectNamedPipe), WIN32_FUNCTION(CreateFile), WIN32_FUNCTION(CreateNamedPipe), + WIN32_KWARGS_FUNCTION(ReadFile), + WIN32_FUNCTION(PeekNamedPipe), WIN32_FUNCTION(SetNamedPipeHandleState), + WIN32_FUNCTION(WaitForMultipleObjects), WIN32_FUNCTION(WaitNamedPipe), + WIN32_KWARGS_FUNCTION(WriteFile), + WIN32_FUNCTION(closesocket), + WIN32_FUNCTION(recv), + WIN32_FUNCTION(send), {NULL} }; @@ -243,10 +759,18 @@ create_win32_namespace(void) return NULL; Py_INCREF(&Win32Type); + if (PyType_Ready(&OverlappedType) < 0) + return NULL; + PyDict_SetItemString(Win32Type.tp_dict, "Overlapped", + (PyObject *) &OverlappedType); + WIN32_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); + WIN32_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); + WIN32_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WIN32_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); WIN32_CONSTANT(F_DWORD, GENERIC_READ); WIN32_CONSTANT(F_DWORD, GENERIC_WRITE); WIN32_CONSTANT(F_DWORD, INFINITE); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index cb58349..ad185a7 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -605,7 +605,7 @@ PyMemoTable_Set(PyMemoTable *self, PyObject *key, Py_ssize_t value) /*************************************************************************/ /* Helpers for creating the argument tuple passed to functions. This has the - performance advantage of calling PyTuple_New() only once. + performance advantage of calling PyTuple_New() only once. XXX(avassalotti): Inline directly in _Pickler_FastCall() and _Unpickler_FastCall(). */ @@ -813,7 +813,7 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *proto_obj, fix_imports = PyObject_IsTrue(fix_imports_obj); if (fix_imports == -1) return -1; - + self->proto = proto; self->bin = proto > 0; self->fix_imports = fix_imports && proto < 3; @@ -826,8 +826,9 @@ _Pickler_SetProtocol(PicklerObject *self, PyObject *proto_obj, static int _Pickler_SetOutputStream(PicklerObject *self, PyObject *file) { + _Py_identifier(write); assert(file != NULL); - self->write = PyObject_GetAttrString(file, "write"); + self->write = _PyObject_GetAttrId(file, &PyId_write); if (self->write == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_SetString(PyExc_TypeError, @@ -909,7 +910,7 @@ _Unpickler_ReadFromFile(UnpicklerObject *self, Py_ssize_t n) Py_ssize_t read_size, prefetched_size = 0; assert(self->read != NULL); - + if (_Unpickler_SkipConsumed(self) < 0) return -1; @@ -1037,7 +1038,7 @@ _Unpickler_Readline(UnpicklerObject *self, char **result) self->next_read_idx = num_read; return _Unpickler_CopyLine(self, self->input_buffer, num_read, result); } - + /* If we get here, we've run off the end of the input string. Return the remaining string and let the caller figure it out. */ *result = self->input_buffer + self->next_read_idx; @@ -1173,15 +1174,19 @@ _Unpickler_New(void) static int _Unpickler_SetInputStream(UnpicklerObject *self, PyObject *file) { - self->peek = PyObject_GetAttrString(file, "peek"); + _Py_identifier(peek); + _Py_identifier(read); + _Py_identifier(readline); + + self->peek = _PyObject_GetAttrId(file, &PyId_peek); if (self->peek == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); else return -1; } - self->read = PyObject_GetAttrString(file, "read"); - self->readline = PyObject_GetAttrString(file, "readline"); + self->read = _PyObject_GetAttrId(file, &PyId_read); + self->readline = _PyObject_GetAttrId(file, &PyId_readline); if (self->readline == NULL || self->read == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_SetString(PyExc_TypeError, @@ -1543,7 +1548,10 @@ save_long(PicklerObject *self, PyObject *obj) PyErr_Clear(); } else - return save_int(self, val); +#if SIZEOF_LONG > 4 + if (val <= 0x7fffffffL && val >= -0x80000000L) +#endif + return save_int(self, val); if (self->proto >= 2) { /* Linear-time pickling. */ @@ -1598,7 +1606,7 @@ save_long(PicklerObject *self, PyObject *obj) * bits. */ if (sign < 0 && - nbytes > 1 && + nbytes > 1 && pdata[nbytes - 1] == 0xff && (pdata[nbytes - 2] & 0x80) != 0) { nbytes--; @@ -1664,7 +1672,7 @@ save_float(PicklerObject *self, PyObject *obj) return -1; if (_Pickler_Write(self, pdata, 9) < 0) return -1; - } + } else { int result = -1; char *buf = NULL; @@ -1766,90 +1774,65 @@ save_bytes(PicklerObject *self, PyObject *obj) /* A copy of PyUnicode_EncodeRawUnicodeEscape() that also translates backslash and newline characters to \uXXXX escapes. */ static PyObject * -raw_unicode_escape(const Py_UNICODE *s, Py_ssize_t size) +raw_unicode_escape(PyObject *obj) { PyObject *repr, *result; char *p; - char *q; + Py_ssize_t i, size, expandsize; + void *data; + unsigned int kind; - static const char *hexdigits = "0123456789abcdef"; + if (PyUnicode_READY(obj)) + return NULL; + + size = PyUnicode_GET_LENGTH(obj); + data = PyUnicode_DATA(obj); + kind = PyUnicode_KIND(obj); + if (kind == PyUnicode_4BYTE_KIND) + expandsize = 10; + else + expandsize = 6; -#ifdef Py_UNICODE_WIDE - const Py_ssize_t expandsize = 10; -#else - const Py_ssize_t expandsize = 6; -#endif - if (size > PY_SSIZE_T_MAX / expandsize) return PyErr_NoMemory(); - repr = PyByteArray_FromStringAndSize(NULL, expandsize * size); if (repr == NULL) return NULL; if (size == 0) goto done; - p = q = PyByteArray_AS_STRING(repr); - while (size-- > 0) { - Py_UNICODE ch = *s++; -#ifdef Py_UNICODE_WIDE + p = PyByteArray_AS_STRING(repr); + for (i=0; i < size; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); /* Map 32-bit characters to '\Uxxxxxxxx' */ if (ch >= 0x10000) { *p++ = '\\'; *p++ = 'U'; - *p++ = hexdigits[(ch >> 28) & 0xf]; - *p++ = hexdigits[(ch >> 24) & 0xf]; - *p++ = hexdigits[(ch >> 20) & 0xf]; - *p++ = hexdigits[(ch >> 16) & 0xf]; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 28) & 0xf]; + *p++ = Py_hexdigits[(ch >> 24) & 0xf]; + *p++ = Py_hexdigits[(ch >> 20) & 0xf]; + *p++ = Py_hexdigits[(ch >> 16) & 0xf]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } - else -#else - /* Map UTF-16 surrogate pairs to '\U00xxxxxx' */ - if (ch >= 0xD800 && ch < 0xDC00) { - Py_UNICODE ch2; - Py_UCS4 ucs; - - ch2 = *s++; - size--; - if (ch2 >= 0xDC00 && ch2 <= 0xDFFF) { - ucs = (((ch & 0x03FF) << 10) | (ch2 & 0x03FF)) + 0x00010000; - *p++ = '\\'; - *p++ = 'U'; - *p++ = hexdigits[(ucs >> 28) & 0xf]; - *p++ = hexdigits[(ucs >> 24) & 0xf]; - *p++ = hexdigits[(ucs >> 20) & 0xf]; - *p++ = hexdigits[(ucs >> 16) & 0xf]; - *p++ = hexdigits[(ucs >> 12) & 0xf]; - *p++ = hexdigits[(ucs >> 8) & 0xf]; - *p++ = hexdigits[(ucs >> 4) & 0xf]; - *p++ = hexdigits[ucs & 0xf]; - continue; - } - /* Fall through: isolated surrogates are copied as-is */ - s--; - size++; - } -#endif /* Map 16-bit characters to '\uxxxx' */ - if (ch >= 256 || ch == '\\' || ch == '\n') { + else if (ch >= 256 || ch == '\\' || ch == '\n') { *p++ = '\\'; *p++ = 'u'; - *p++ = hexdigits[(ch >> 12) & 0xf]; - *p++ = hexdigits[(ch >> 8) & 0xf]; - *p++ = hexdigits[(ch >> 4) & 0xf]; - *p++ = hexdigits[ch & 15]; + *p++ = Py_hexdigits[(ch >> 12) & 0xf]; + *p++ = Py_hexdigits[(ch >> 8) & 0xf]; + *p++ = Py_hexdigits[(ch >> 4) & 0xf]; + *p++ = Py_hexdigits[ch & 15]; } /* Copy everything else as-is */ else *p++ = (char) ch; } - size = p - q; + size = p - PyByteArray_AS_STRING(repr); - done: +done: result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size); Py_DECREF(repr); return result; @@ -1864,9 +1847,7 @@ save_unicode(PicklerObject *self, PyObject *obj) if (self->bin) { char pdata[5]; - encoded = PyUnicode_EncodeUTF8(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj), - "surrogatepass"); + encoded = PyUnicode_AsEncodedString(obj, "utf-8", "surrogatepass"); if (encoded == NULL) goto error; @@ -1892,8 +1873,7 @@ save_unicode(PicklerObject *self, PyObject *obj) else { const char unicode_op = UNICODE; - encoded = raw_unicode_escape(PyUnicode_AS_UNICODE(obj), - PyUnicode_GET_SIZE(obj)); + encoded = raw_unicode_escape(obj); if (encoded == NULL) goto error; @@ -2512,7 +2492,9 @@ save_dict(PicklerObject *self, PyObject *obj) status = batch_dict_exact(self, obj); Py_LeaveRecursiveCall(); } else { - items = PyObject_CallMethod(obj, "items", "()"); + _Py_identifier(items); + + items = _PyObject_CallMethodId(obj, &PyId_items, "()"); if (items == NULL) goto error; iter = PyObject_GetIter(items); @@ -2849,6 +2831,28 @@ save_pers(PicklerObject *self, PyObject *obj, PyObject *func) return status; } +static PyObject * +get_class(PyObject *obj) +{ + PyObject *cls; + static PyObject *str_class; + + if (str_class == NULL) { + str_class = PyUnicode_InternFromString("__class__"); + if (str_class == NULL) + return NULL; + } + cls = PyObject_GetAttr(obj, str_class); + if (cls == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + cls = (PyObject *) Py_TYPE(obj); + Py_INCREF(cls); + } + } + return cls; +} + /* We're saving obj, and args is the 2-thru-5 tuple returned by the * appropriate __reduce__ method for obj. */ @@ -2914,17 +2918,18 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) /* Protocol 2 special case: if callable's name is __newobj__, use NEWOBJ. */ if (use_newobj) { - static PyObject *newobj_str = NULL; - PyObject *name_str; + static PyObject *newobj_str = NULL, *name_str = NULL; + PyObject *name; if (newobj_str == NULL) { newobj_str = PyUnicode_InternFromString("__newobj__"); - if (newobj_str == NULL) + name_str = PyUnicode_InternFromString("__name__"); + if (newobj_str == NULL || name_str == NULL) return -1; } - name_str = PyObject_GetAttrString(callable, "__name__"); - if (name_str == NULL) { + name = PyObject_GetAttr(callable, name_str); + if (name == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); else @@ -2932,9 +2937,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) use_newobj = 0; } else { - use_newobj = PyUnicode_Check(name_str) && - PyUnicode_Compare(name_str, newobj_str) == 0; - Py_DECREF(name_str); + use_newobj = PyUnicode_Check(name) && + PyUnicode_Compare(name, newobj_str) == 0; + Py_DECREF(name); } } if (use_newobj) { @@ -2950,20 +2955,14 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } cls = PyTuple_GET_ITEM(argtup, 0); - if (!PyObject_HasAttrString(cls, "__new__")) { + if (!PyType_Check(cls)) { PyErr_SetString(PicklingError, "args[0] from " - "__newobj__ args has no __new__"); + "__newobj__ args is not a type"); return -1; } if (obj != NULL) { - obj_class = PyObject_GetAttrString(obj, "__class__"); - if (obj_class == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) - PyErr_Clear(); - else - return -1; - } + obj_class = get_class(obj); p = obj_class != cls; /* true iff a problem */ Py_DECREF(obj_class); if (p) { @@ -3037,7 +3036,7 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) return -1; if (state) { - if (save(self, state, 0) < 0 || + if (save(self, state, 0) < 0 || _Pickler_Write(self, &build_op, 1) < 0) return -1; } @@ -3291,7 +3290,7 @@ Pickler_dump(PicklerObject *self, PyObject *args) Developers often forget to call __init__() in their subclasses, which would trigger a segfault without this check. */ if (self->write == NULL) { - PyErr_Format(PicklingError, + PyErr_Format(PicklingError, "Pickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -3395,6 +3394,7 @@ Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds) PyObject *file; PyObject *proto_obj = NULL; PyObject *fix_imports = Py_True; + _Py_identifier(persistent_id); if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|OO:Pickler", kwlist, &file, &proto_obj, &fix_imports)) @@ -3430,9 +3430,9 @@ Pickler_init(PicklerObject *self, PyObject *args, PyObject *kwds) self->fast_nesting = 0; self->fast_memo = NULL; self->pers_func = NULL; - if (PyObject_HasAttrString((PyObject *)self, "persistent_id")) { - self->pers_func = PyObject_GetAttrString((PyObject *)self, - "persistent_id"); + if (_PyObject_HasAttrId((PyObject *)self, &PyId_persistent_id)) { + self->pers_func = _PyObject_GetAttrId((PyObject *)self, + &PyId_persistent_id); if (self->pers_func == NULL) return -1; } @@ -3771,7 +3771,7 @@ static PyTypeObject Pickler_Type = { 0, /*tp_is_gc*/ }; -/* Temporary helper for calling self.find_class(). +/* Temporary helper for calling self.find_class(). XXX: It would be nice to able to avoid Python function call overhead, by using directly the C version of find_class(), when find_class() is not @@ -3781,8 +3781,10 @@ static PyTypeObject Pickler_Type = { static PyObject * find_class(UnpicklerObject *self, PyObject *module_name, PyObject *global_name) { - return PyObject_CallMethod((PyObject *)self, "find_class", "OO", - module_name, global_name); + _Py_identifier(find_class); + + return _PyObject_CallMethodId((PyObject *)self, &PyId_find_class, "OO", + module_name, global_name); } static Py_ssize_t @@ -3824,7 +3826,7 @@ load_int(UnpicklerObject *self) return bad_readline(); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? + /* XXX: Should the base argument of strtol() be explicitly set to 10? XXX(avassalotti): Should this uses PyOS_strtol()? */ x = strtol(s, &endptr, 0); @@ -4189,7 +4191,7 @@ load_binstring(UnpicklerObject *self) x = calc_binint(s, 4); if (x < 0) { - PyErr_SetString(UnpicklingError, + PyErr_SetString(UnpicklingError, "BINSTRING pickle has negative byte count"); return -1; } @@ -4395,7 +4397,9 @@ instantiate(PyObject *cls, PyObject *args) result = PyObject_CallObject(cls, args); } else { - result = PyObject_CallMethod(cls, "__new__", "O", cls); + _Py_identifier(__new__); + + result = _PyObject_CallMethodId(cls, &PyId___new__, "O", cls); } return result; } @@ -4936,8 +4940,9 @@ do_append(UnpicklerObject *self, Py_ssize_t x) } else { PyObject *append_func; + _Py_identifier(append); - append_func = PyObject_GetAttrString(list, "append"); + append_func = _PyObject_GetAttrId(list, &PyId_append); if (append_func == NULL) return -1; for (i = x; i < len; i++) { @@ -4983,7 +4988,7 @@ do_setitems(UnpicklerObject *self, Py_ssize_t x) return stack_underflow(); if (len == x) /* nothing to do */ return 0; - if ((len - x) % 2 != 0) { + if ((len - x) % 2 != 0) { /* Currupt or hostile pickle -- we never write one like this. */ PyErr_SetString(UnpicklingError, "odd number of items for SETITEMS"); return -1; @@ -5024,6 +5029,7 @@ load_build(UnpicklerObject *self) PyObject *state, *inst, *slotstate; PyObject *setstate; int status = 0; + _Py_identifier(__setstate__); /* Stack is ... instance, state. We want to leave instance at * the stack top, possibly mutated via instance.__setstate__(state). @@ -5037,7 +5043,7 @@ load_build(UnpicklerObject *self) inst = self->stack->data[Py_SIZE(self->stack) - 1]; - setstate = PyObject_GetAttrString(inst, "__setstate__"); + setstate = _PyObject_GetAttrId(inst, &PyId___setstate__); if (setstate == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); @@ -5080,12 +5086,13 @@ load_build(UnpicklerObject *self) PyObject *dict; PyObject *d_key, *d_value; Py_ssize_t i; + _Py_identifier(__dict__); if (!PyDict_Check(state)) { PyErr_SetString(UnpicklingError, "state is not a dictionary"); goto error; } - dict = PyObject_GetAttrString(inst, "__dict__"); + dict = _PyObject_GetAttrId(inst, &PyId___dict__); if (dict == NULL) goto error; @@ -5340,7 +5347,7 @@ Unpickler_load(UnpicklerObject *self) not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ if (self->read == NULL) { - PyErr_Format(UnpicklingError, + PyErr_Format(UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", Py_TYPE(self)->tp_name); return NULL; @@ -5442,7 +5449,7 @@ Unpickler_find_class(UnpicklerObject *self, PyObject *args) global = PyObject_GetAttr(module, global_name); Py_DECREF(module); } - else { + else { global = PyObject_GetAttr(module, global_name); } return global; @@ -5585,8 +5592,9 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) return -1; if (PyObject_HasAttrString((PyObject *)self, "persistent_load")) { - self->pers_func = PyObject_GetAttrString((PyObject *)self, - "persistent_load"); + _Py_identifier(persistent_load); + self->pers_func = _PyObject_GetAttrId((PyObject *)self, + &PyId_persistent_load); if (self->pers_func == NULL) return -1; } @@ -5619,7 +5627,7 @@ Unpickler_init(UnpicklerObject *self, PyObject *args, PyObject *kwds) * intentional, as these should be treated as black-box implementation details. * * We do, however, have to implement pickling/unpickling support because of - * real-world code like cvs2svn. + * real-world code like cvs2svn. */ typedef struct { diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 11b24a0..e6a9283 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1,7 +1,7 @@ /* Authors: Gregory P. Smith & Jeffrey Yasskin */ #include "Python.h" -#ifdef HAVE_PIPE2 -#define _GNU_SOURCE +#if defined(HAVE_PIPE2) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE #endif #include <unistd.h> #include <fcntl.h> @@ -18,7 +18,9 @@ static long max_fd; static int _enable_gc(PyObject *gc_module) { PyObject *result; - result = PyObject_CallMethod(gc_module, "enable", NULL); + _Py_identifier(enable); + + result = _PyObject_CallMethodId(gc_module, &PyId_enable, NULL); if (result == NULL) return 1; Py_DECREF(result); @@ -249,10 +251,13 @@ subprocess_fork_exec(PyObject* self, PyObject *args) /* We need to call gc.disable() when we'll be calling preexec_fn */ if (preexec_fn != Py_None) { PyObject *result; + _Py_identifier(isenabled); + _Py_identifier(disable); + gc_module = PyImport_ImportModule("gc"); if (gc_module == NULL) return NULL; - result = PyObject_CallMethod(gc_module, "isenabled", NULL); + result = _PyObject_CallMethodId(gc_module, &PyId_isenabled, NULL); if (result == NULL) { Py_DECREF(gc_module); return NULL; @@ -263,7 +268,7 @@ subprocess_fork_exec(PyObject* self, PyObject *args) Py_DECREF(gc_module); return NULL; } - result = PyObject_CallMethod(gc_module, "disable", NULL); + result = _PyObject_CallMethodId(gc_module, &PyId_disable, NULL); if (result == NULL) { Py_DECREF(gc_module); return NULL; diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7d12d5e..c2f218f 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -675,7 +675,7 @@ void _pysqlite_final_callback(sqlite3_context* context) { PyObject* function_result = NULL; PyObject** aggregate_instance; - PyObject* aggregate_class; + _Py_identifier(finalize); #ifdef WITH_THREAD PyGILState_STATE threadstate; @@ -683,8 +683,6 @@ void _pysqlite_final_callback(sqlite3_context* context) threadstate = PyGILState_Ensure(); #endif - aggregate_class = (PyObject*)sqlite3_user_data(context); - aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); if (!*aggregate_instance) { /* this branch is executed if there was an exception in the aggregate's @@ -693,7 +691,7 @@ void _pysqlite_final_callback(sqlite3_context* context) goto error; } - function_result = PyObject_CallMethod(*aggregate_instance, "finalize", ""); + function_result = _PyObject_CallMethodId(*aggregate_instance, &PyId_finalize, ""); if (!function_result) { if (_enable_callback_tracebacks) { PyErr_Print(); @@ -909,6 +907,38 @@ static int _progress_handler(void* user_arg) return rc; } +static void _trace_callback(void* user_arg, const char* statement_string) +{ + PyObject *py_statement = NULL; + PyObject *ret = NULL; + +#ifdef WITH_THREAD + PyGILState_STATE gilstate; + + gilstate = PyGILState_Ensure(); +#endif + py_statement = PyUnicode_DecodeUTF8(statement_string, + strlen(statement_string), "replace"); + if (py_statement) { + ret = PyObject_CallFunctionObjArgs((PyObject*)user_arg, py_statement, NULL); + Py_DECREF(py_statement); + } + + if (ret) { + Py_DECREF(ret); + } else { + if (_enable_callback_tracebacks) { + PyErr_Print(); + } else { + PyErr_Clear(); + } + } + +#ifdef WITH_THREAD + PyGILState_Release(gilstate); +#endif +} + static PyObject* pysqlite_connection_set_authorizer(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) { PyObject* authorizer_cb; @@ -968,6 +998,34 @@ static PyObject* pysqlite_connection_set_progress_handler(pysqlite_Connection* s return Py_None; } +static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) +{ + PyObject* trace_callback; + + static char *kwlist[] = { "trace_callback", NULL }; + + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:set_trace_callback", + kwlist, &trace_callback)) { + return NULL; + } + + if (trace_callback == Py_None) { + /* None clears the trace callback previously set */ + sqlite3_trace(self->db, 0, (void*)0); + } else { + if (PyDict_SetItem(self->function_pinboard, trace_callback, Py_None) == -1) + return NULL; + sqlite3_trace(self->db, _trace_callback, trace_callback); + } + + Py_INCREF(Py_None); + return Py_None; +} + #ifdef HAVE_LOAD_EXTENSION static PyObject* pysqlite_enable_load_extension(pysqlite_Connection* self, PyObject* args) { @@ -1173,8 +1231,9 @@ PyObject* pysqlite_connection_execute(pysqlite_Connection* self, PyObject* args, PyObject* cursor = 0; PyObject* result = 0; PyObject* method = 0; + _Py_identifier(cursor); - cursor = PyObject_CallMethod((PyObject*)self, "cursor", ""); + cursor = _PyObject_CallMethodId((PyObject*)self, &PyId_cursor, ""); if (!cursor) { goto error; } @@ -1202,8 +1261,9 @@ PyObject* pysqlite_connection_executemany(pysqlite_Connection* self, PyObject* a PyObject* cursor = 0; PyObject* result = 0; PyObject* method = 0; + _Py_identifier(cursor); - cursor = PyObject_CallMethod((PyObject*)self, "cursor", ""); + cursor = _PyObject_CallMethodId((PyObject*)self, &PyId_cursor, ""); if (!cursor) { goto error; } @@ -1231,8 +1291,9 @@ PyObject* pysqlite_connection_executescript(pysqlite_Connection* self, PyObject* PyObject* cursor = 0; PyObject* result = 0; PyObject* method = 0; + _Py_identifier(cursor); - cursor = PyObject_CallMethod((PyObject*)self, "cursor", ""); + cursor = _PyObject_CallMethodId((PyObject*)self, &PyId_cursor, ""); if (!cursor) { goto error; } @@ -1379,10 +1440,12 @@ pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) PyObject* uppercase_name = 0; PyObject* name; PyObject* retval; - Py_UNICODE* chk; Py_ssize_t i, len; + _Py_identifier(upper); char *uppercase_name_str; int rc; + unsigned int kind; + void *data; if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { goto finally; @@ -1392,17 +1455,21 @@ pysqlite_connection_create_collation(pysqlite_Connection* self, PyObject* args) goto finally; } - uppercase_name = PyObject_CallMethod(name, "upper", ""); + uppercase_name = _PyObject_CallMethodId(name, &PyId_upper, ""); if (!uppercase_name) { goto finally; } - len = PyUnicode_GET_SIZE(uppercase_name); - chk = PyUnicode_AS_UNICODE(uppercase_name); - for (i=0; i<len; i++, chk++) { - if ((*chk >= '0' && *chk <= '9') - || (*chk >= 'A' && *chk <= 'Z') - || (*chk == '_')) + if (PyUnicode_READY(uppercase_name)) + goto finally; + len = PyUnicode_GET_LENGTH(uppercase_name); + kind = PyUnicode_KIND(uppercase_name); + data = PyUnicode_DATA(uppercase_name); + for (i=0; i<len; i++) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i); + if ((ch >= '0' && ch <= '9') + || (ch >= 'A' && ch <= 'Z') + || (ch == '_')) { continue; } else { @@ -1521,6 +1588,8 @@ static PyMethodDef connection_methods[] = { #endif {"set_progress_handler", (PyCFunction)pysqlite_connection_set_progress_handler, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Sets progress handler callback. Non-standard.")}, + {"set_trace_callback", (PyCFunction)pysqlite_connection_set_trace_callback, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("Sets a trace callback called for each SQL statement (passed as unicode). Non-standard.")}, {"execute", (PyCFunction)pysqlite_connection_execute, METH_VARARGS, PyDoc_STR("Executes a SQL statement. Non-standard.")}, {"executemany", (PyCFunction)pysqlite_connection_executemany, METH_VARARGS, diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index b9a4358..b47fed4 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -126,11 +126,9 @@ static int pysqlite_cursor_init(pysqlite_Cursor* self, PyObject* args, PyObject* static void pysqlite_cursor_dealloc(pysqlite_Cursor* self) { - int rc; - /* Reset the statement if the user has not closed the cursor */ if (self->statement) { - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); Py_DECREF(self->statement); } @@ -152,8 +150,9 @@ PyObject* _pysqlite_get_converter(PyObject* key) { PyObject* upcase_key; PyObject* retval; + _Py_identifier(upper); - upcase_key = PyObject_CallMethod(key, "upper", ""); + upcase_key = _PyObject_CallMethodId(key, &PyId_upper, ""); if (!upcase_key) { return NULL; } @@ -432,9 +431,14 @@ static int check_cursor(pysqlite_Cursor* cur) if (cur->closed) { PyErr_SetString(pysqlite_ProgrammingError, "Cannot operate on a closed cursor."); return 0; - } else { - return pysqlite_check_thread(cur->connection) && pysqlite_check_connection(cur->connection); } + + if (cur->locked) { + PyErr_SetString(pysqlite_ProgrammingError, "Recursive use of cursors not allowed."); + return 0; + } + + return pysqlite_check_thread(cur->connection) && pysqlite_check_connection(cur->connection); } PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* args) @@ -457,9 +461,10 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* int allow_8bit_chars; if (!check_cursor(self)) { - return NULL; + goto error; } + self->locked = 1; self->reset = 0; /* Make shooting yourself in the foot with not utf-8 decodable 8-bit-strings harder */ @@ -472,12 +477,12 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* if (multiple) { /* executemany() */ if (!PyArg_ParseTuple(args, "OO", &operation, &second_argument)) { - return NULL; + goto error; } if (!PyUnicode_Check(operation)) { PyErr_SetString(PyExc_ValueError, "operation parameter must be str"); - return NULL; + goto error; } if (PyIter_Check(second_argument)) { @@ -488,23 +493,23 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* /* sequence */ parameters_iter = PyObject_GetIter(second_argument); if (!parameters_iter) { - return NULL; + goto error; } } } else { /* execute() */ if (!PyArg_ParseTuple(args, "O|O", &operation, &second_argument)) { - return NULL; + goto error; } if (!PyUnicode_Check(operation)) { PyErr_SetString(PyExc_ValueError, "operation parameter must be str"); - return NULL; + goto error; } parameters_list = PyList_New(0); if (!parameters_list) { - return NULL; + goto error; } if (second_argument == NULL) { @@ -529,7 +534,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* if (self->statement != NULL) { /* There is an active statement */ - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); } operation_cstr = _PyUnicode_AsStringAndSize(operation, &operation_len); @@ -734,7 +739,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } if (multiple) { - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); } Py_XDECREF(parameters); } @@ -744,7 +749,8 @@ error: * ROLLBACK could have happened */ #ifdef SQLITE_VERSION_NUMBER #if SQLITE_VERSION_NUMBER >= 3002002 - self->connection->inTransaction = !sqlite3_get_autocommit(self->connection->db); + if (self->connection && self->connection->db) + self->connection->inTransaction = !sqlite3_get_autocommit(self->connection->db); #endif #endif @@ -752,6 +758,8 @@ error: Py_XDECREF(parameters_iter); Py_XDECREF(parameters_list); + self->locked = 0; + if (PyErr_Occurred()) { self->rowcount = -1L; return NULL; diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h index 5d8b5c1..118ba38 100644 --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -42,6 +42,7 @@ typedef struct pysqlite_Statement* statement; int closed; int reset; + int locked; int initialized; /* the next row to be returned, NULL if no next row available */ diff --git a/Modules/_sqlite/microprotocols.c b/Modules/_sqlite/microprotocols.c index c730afa..c0f5e22 100644 --- a/Modules/_sqlite/microprotocols.c +++ b/Modules/_sqlite/microprotocols.c @@ -95,7 +95,9 @@ pysqlite_microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) /* try to have the protocol adapt this object*/ if (PyObject_HasAttrString(proto, "__adapt__")) { - PyObject *adapted = PyObject_CallMethod(proto, "__adapt__", "O", obj); + _Py_identifier(__adapt__); + PyObject *adapted = _PyObject_CallMethodId(proto, &PyId___adapt__, "O", obj); + if (adapted) { if (adapted != Py_None) { return adapted; @@ -110,7 +112,9 @@ pysqlite_microprotocols_adapt(PyObject *obj, PyObject *proto, PyObject *alt) /* and finally try to have the object adapt itself */ if (PyObject_HasAttrString(obj, "__conform__")) { - PyObject *adapted = PyObject_CallMethod(obj, "__conform__","O", proto); + _Py_identifier(__conform__); + PyObject *adapted = _PyObject_CallMethodId(obj, &PyId___conform__,"O", proto); + if (adapted) { if (adapted != Py_None) { return adapted; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index cbc3b8e..20e1ec7 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -179,13 +179,14 @@ static PyObject* module_register_converter(PyObject* self, PyObject* args) PyObject* name = NULL; PyObject* callable; PyObject* retval = NULL; + _Py_identifier(upper); if (!PyArg_ParseTuple(args, "UO", &orig_name, &callable)) { return NULL; } /* convert the name to upper case */ - name = PyObject_CallMethod(orig_name, "upper", ""); + name = _PyObject_CallMethodId(orig_name, &PyId_upper, ""); if (!name) { goto error; } diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c index 3d44094..b50658c 100644 --- a/Modules/_sqlite/row.c +++ b/Modules/_sqlite/row.c @@ -173,10 +173,9 @@ static Py_hash_t pysqlite_row_hash(pysqlite_Row *self) static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other, int opid) { - if (opid != Py_EQ && opid != Py_NE) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (opid != Py_EQ && opid != Py_NE) + Py_RETURN_NOTIMPLEMENTED; + if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) { pysqlite_Row *other = (pysqlite_Row *)_other; PyObject *res = PyObject_RichCompare(self->description, other->description, opid); @@ -186,8 +185,7 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other, return PyObject_RichCompare(self->data, other->data, opid); } } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } PyMappingMethods pysqlite_row_as_mapping = { diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index f89fc9a..4e039c1 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -369,11 +369,9 @@ void pysqlite_statement_mark_dirty(pysqlite_Statement* self) void pysqlite_statement_dealloc(pysqlite_Statement* self) { - int rc; - if (self->st) { Py_BEGIN_ALLOW_THREADS - rc = sqlite3_finalize(self->st); + sqlite3_finalize(self->st); Py_END_ALLOW_THREADS } diff --git a/Modules/_sre.c b/Modules/_sre.c index 2b764b1..395a120 100644 --- a/Modules/_sre.c +++ b/Modules/_sre.c @@ -163,21 +163,17 @@ static unsigned int sre_lower_locale(unsigned int ch) /* unicode-specific character predicates */ -#if defined(HAVE_UNICODE) - -#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL((Py_UNICODE)(ch)) -#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE((Py_UNICODE)(ch)) -#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK((Py_UNICODE)(ch)) -#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM((Py_UNICODE)(ch)) -#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM((ch)) || (ch) == '_') +#define SRE_UNI_IS_DIGIT(ch) Py_UNICODE_ISDECIMAL(ch) +#define SRE_UNI_IS_SPACE(ch) Py_UNICODE_ISSPACE(ch) +#define SRE_UNI_IS_LINEBREAK(ch) Py_UNICODE_ISLINEBREAK(ch) +#define SRE_UNI_IS_ALNUM(ch) Py_UNICODE_ISALNUM(ch) +#define SRE_UNI_IS_WORD(ch) (SRE_UNI_IS_ALNUM(ch) || (ch) == '_') static unsigned int sre_lower_unicode(unsigned int ch) { - return (unsigned int) Py_UNICODE_TOLOWER((Py_UNICODE)(ch)); + return (unsigned int) Py_UNICODE_TOLOWER(ch); } -#endif - LOCAL(int) sre_category(SRE_CODE category, unsigned int ch) { @@ -205,7 +201,6 @@ sre_category(SRE_CODE category, unsigned int ch) case SRE_CATEGORY_LOC_NOT_WORD: return !SRE_LOC_IS_WORD(ch); -#if defined(HAVE_UNICODE) case SRE_CATEGORY_UNI_DIGIT: return SRE_UNI_IS_DIGIT(ch); case SRE_CATEGORY_UNI_NOT_DIGIT: @@ -222,24 +217,6 @@ sre_category(SRE_CODE category, unsigned int ch) return SRE_UNI_IS_LINEBREAK(ch); case SRE_CATEGORY_UNI_NOT_LINEBREAK: return !SRE_UNI_IS_LINEBREAK(ch); -#else - case SRE_CATEGORY_UNI_DIGIT: - return SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_NOT_DIGIT: - return !SRE_IS_DIGIT(ch); - case SRE_CATEGORY_UNI_SPACE: - return SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_NOT_SPACE: - return !SRE_IS_SPACE(ch); - case SRE_CATEGORY_UNI_WORD: - return SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_NOT_WORD: - return !SRE_LOC_IS_WORD(ch); - case SRE_CATEGORY_UNI_LINEBREAK: - return SRE_IS_LINEBREAK(ch); - case SRE_CATEGORY_UNI_NOT_LINEBREAK: - return !SRE_IS_LINEBREAK(ch); -#endif } return 0; } @@ -280,6 +257,7 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) /* generate 8-bit version */ #define SRE_CHAR unsigned char +#define SRE_CHARGET(state, buf, index) ((unsigned char*)buf)[index] #define SRE_AT sre_at #define SRE_COUNT sre_count #define SRE_CHARSET sre_charset @@ -287,15 +265,11 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) #define SRE_MATCH sre_match #define SRE_MATCH_CONTEXT sre_match_context #define SRE_SEARCH sre_search -#define SRE_LITERAL_TEMPLATE sre_literal_template - -#if defined(HAVE_UNICODE) #define SRE_RECURSIVE #include "_sre.c" #undef SRE_RECURSIVE -#undef SRE_LITERAL_TEMPLATE #undef SRE_SEARCH #undef SRE_MATCH #undef SRE_MATCH_CONTEXT @@ -304,10 +278,15 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) #undef SRE_COUNT #undef SRE_AT #undef SRE_CHAR +#undef SRE_CHARGET -/* generate 16-bit unicode version */ +/* generate 8/16/32-bit unicode version */ -#define SRE_CHAR Py_UNICODE +#define SRE_CHAR void +#define SRE_CHARGET(state, buf, index) \ + ((state->charsize==1) ? ((Py_UCS1*)buf)[index] : \ + (state->charsize==2) ? ((Py_UCS2*)buf)[index] : \ + ((Py_UCS4*)buf)[index]) #define SRE_AT sre_uat #define SRE_COUNT sre_ucount #define SRE_CHARSET sre_ucharset @@ -315,8 +294,6 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) #define SRE_MATCH sre_umatch #define SRE_MATCH_CONTEXT sre_umatch_context #define SRE_SEARCH sre_usearch -#define SRE_LITERAL_TEMPLATE sre_uliteral_template -#endif #endif /* SRE_RECURSIVE */ @@ -327,7 +304,7 @@ data_stack_grow(SRE_STATE* state, Py_ssize_t size) settings */ LOCAL(int) -SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) +SRE_AT(SRE_STATE* state, char* ptr, SRE_CODE at) { /* check if pointer is at given position */ @@ -341,16 +318,16 @@ SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) case SRE_AT_BEGINNING_LINE: return ((void*) ptr == state->beginning || - SRE_IS_LINEBREAK((int) ptr[-1])); + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, -1))); case SRE_AT_END: - return (((void*) (ptr+1) == state->end && - SRE_IS_LINEBREAK((int) ptr[0])) || + return (((void*) (ptr+state->charsize) == state->end && + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))) || ((void*) ptr == state->end)); case SRE_AT_END_LINE: return ((void*) ptr == state->end || - SRE_IS_LINEBREAK((int) ptr[0])); + SRE_IS_LINEBREAK((int) SRE_CHARGET(state, ptr, 0))); case SRE_AT_END_STRING: return ((void*) ptr == state->end); @@ -359,57 +336,55 @@ SRE_AT(SRE_STATE* state, SRE_CHAR* ptr, SRE_CODE at) if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_IS_WORD((int) ptr[-1]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_IS_WORD((int) ptr[0]) : 0; + SRE_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; case SRE_AT_LOC_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_LOC_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_LOC_IS_WORD((int) ptr[-1]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_LOC_IS_WORD((int) ptr[0]) : 0; + SRE_LOC_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; -#if defined(HAVE_UNICODE) case SRE_AT_UNI_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp != thatp; case SRE_AT_UNI_NON_BOUNDARY: if (state->beginning == state->end) return 0; thatp = ((void*) ptr > state->beginning) ? - SRE_UNI_IS_WORD((int) ptr[-1]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, -1)) : 0; thisp = ((void*) ptr < state->end) ? - SRE_UNI_IS_WORD((int) ptr[0]) : 0; + SRE_UNI_IS_WORD((int) SRE_CHARGET(state, ptr, 0)) : 0; return thisp == thatp; -#endif } @@ -476,7 +451,7 @@ SRE_CHARSET(SRE_CODE* set, SRE_CODE ch) count = *(set++); if (sizeof(SRE_CODE) == 2) { - block = ((unsigned char*)set)[ch >> 8]; + block = ((char*)set)[ch >> 8]; set += 128; if (set[block*16 + ((ch & 255)>>4)] & (1 << (ch & 15))) return ok; @@ -486,7 +461,7 @@ SRE_CHARSET(SRE_CODE* set, SRE_CODE ch) /* !(c & ~N) == (c < N+1) for any unsigned c, this avoids * warnings when c's type supports only numbers < N+1 */ if (!(ch & ~65535)) - block = ((unsigned char*)set)[ch >> 8]; + block = ((char*)set)[ch >> 8]; else block = -1; set += 64; @@ -512,28 +487,29 @@ LOCAL(Py_ssize_t) SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) { SRE_CODE chr; - SRE_CHAR* ptr = (SRE_CHAR *)state->ptr; - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* ptr = (char *)state->ptr; + char* end = (char *)state->end; Py_ssize_t i; /* adjust end */ if (maxcount < end - ptr && maxcount != 65535) - end = ptr + maxcount; + end = ptr + maxcount*state->charsize; switch (pattern[0]) { case SRE_OP_IN: /* repeated set */ TRACE(("|%p|%p|COUNT IN\n", pattern, ptr)); - while (ptr < end && SRE_CHARSET(pattern + 2, *ptr)) - ptr++; + while (ptr < end && + SRE_CHARSET(pattern + 2, SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; break; case SRE_OP_ANY: /* repeated dot wildcard. */ TRACE(("|%p|%p|COUNT ANY\n", pattern, ptr)); - while (ptr < end && !SRE_IS_LINEBREAK(*ptr)) - ptr++; + while (ptr < end && !SRE_IS_LINEBREAK(SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; break; case SRE_OP_ANY_ALL: @@ -547,38 +523,38 @@ SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) /* repeated literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr == chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) == chr) + ptr += state->charsize; break; case SRE_OP_LITERAL_IGNORE: /* repeated literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) == chr) - ptr++; + while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) == chr) + ptr += state->charsize; break; case SRE_OP_NOT_LITERAL: /* repeated non-literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT NOT_LITERAL %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) *ptr != chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr) + ptr += state->charsize; break; case SRE_OP_NOT_LITERAL_IGNORE: /* repeated non-literal */ chr = pattern[1]; TRACE(("|%p|%p|COUNT NOT_LITERAL_IGNORE %d\n", pattern, ptr, chr)); - while (ptr < end && (SRE_CODE) state->lower(*ptr) != chr) - ptr++; + while (ptr < end && (SRE_CODE) state->lower(SRE_CHARGET(state, ptr, 0)) != chr) + ptr += state->charsize; break; default: /* repeated single character pattern */ TRACE(("|%p|%p|COUNT SUBPATTERN\n", pattern, ptr)); - while ((SRE_CHAR*) state->ptr < end) { + while ((char*) state->ptr < end) { i = SRE_MATCH(state, pattern); if (i < 0) return i; @@ -586,12 +562,12 @@ SRE_COUNT(SRE_STATE* state, SRE_CODE* pattern, Py_ssize_t maxcount) break; } TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, - (SRE_CHAR*) state->ptr - ptr)); - return (SRE_CHAR*) state->ptr - ptr; + ((char*)state->ptr - ptr)/state->charsize)); + return ((char*)state->ptr - ptr)/state->charsize; } - TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, ptr - (SRE_CHAR*) state->ptr)); - return ptr - (SRE_CHAR*) state->ptr; + TRACE(("|%p|%p|COUNT %d\n", pattern, ptr, (ptr - (char*) state->ptr)/state->charsize)); + return (ptr - (char*) state->ptr)/state->charsize; } #if 0 /* not used in this release */ @@ -602,8 +578,8 @@ SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) returns the number of SRE_CODE objects to skip if successful, 0 if no match */ - SRE_CHAR* end = state->end; - SRE_CHAR* ptr = state->ptr; + char* end = state->end; + char* ptr = state->ptr; Py_ssize_t i; /* check minimal length */ @@ -614,7 +590,7 @@ SRE_INFO(SRE_STATE* state, SRE_CODE* pattern) if (pattern[2] & SRE_INFO_PREFIX && pattern[5] > 1) { /* <length> <skip> <prefix data> <overlap data> */ for (i = 0; i < pattern[5]; i++) - if ((SRE_CODE) ptr[i] != pattern[7 + i]) + if ((SRE_CODE) SRE_CHARGET(state, ptr, i) != pattern[7 + i]) return 0; return pattern[0] + 2 * pattern[6]; } @@ -783,7 +759,7 @@ do { \ typedef struct { Py_ssize_t last_ctx_pos; Py_ssize_t jump; - SRE_CHAR* ptr; + char* ptr; SRE_CODE* pattern; Py_ssize_t count; Py_ssize_t lastmark; @@ -799,7 +775,7 @@ typedef struct { LOCAL(Py_ssize_t) SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) { - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* end = (char*)state->end; Py_ssize_t alloc_pos, ctx_pos = -1; Py_ssize_t i, ret = 0; Py_ssize_t jump; @@ -818,12 +794,12 @@ SRE_MATCH(SRE_STATE* state, SRE_CODE* pattern) entrance: - ctx->ptr = (SRE_CHAR *)state->ptr; + ctx->ptr = (char *)state->ptr; if (ctx->pattern[0] == SRE_OP_INFO) { /* optimization info block */ /* <INFO> <1=skip> <2=flags> <3=min> ... */ - if (ctx->pattern[3] && (end - ctx->ptr) < ctx->pattern[3]) { + if (ctx->pattern[3] && (end - ctx->ptr)/state->charsize < ctx->pattern[3]) { TRACE(("reject (got %d chars, need %d)\n", (end - ctx->ptr), ctx->pattern[3])); RETURN_FAILURE; @@ -865,10 +841,10 @@ entrance: /* <LITERAL> <code> */ TRACE(("|%p|%p|LITERAL %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] != ctx->pattern[0]) + if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[0]) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_NOT_LITERAL: @@ -876,10 +852,10 @@ entrance: /* <NOT_LITERAL> <code> */ TRACE(("|%p|%p|NOT_LITERAL %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || (SRE_CODE) ctx->ptr[0] == ctx->pattern[0]) + if (ctx->ptr >= end || (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) == ctx->pattern[0]) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_SUCCESS: @@ -902,19 +878,19 @@ entrance: /* <CATEGORY> <code> */ TRACE(("|%p|%p|CATEGORY %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); - if (ctx->ptr >= end || !sre_category(ctx->pattern[0], ctx->ptr[0])) + if (ctx->ptr >= end || !sre_category(ctx->pattern[0], SRE_CHARGET(state, ctx->ptr, 0))) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_ANY: /* match anything (except a newline) */ /* <ANY> */ TRACE(("|%p|%p|ANY\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || SRE_IS_LINEBREAK(ctx->ptr[0])) - RETURN_FAILURE; - ctx->ptr++; + if (ctx->ptr >= end || SRE_IS_LINEBREAK(SRE_CHARGET(state, ctx->ptr, 0))) + RETURN_FAILURE; + ctx->ptr += state->charsize; break; case SRE_OP_ANY_ALL: @@ -923,47 +899,47 @@ entrance: TRACE(("|%p|%p|ANY_ALL\n", ctx->pattern, ctx->ptr)); if (ctx->ptr >= end) RETURN_FAILURE; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_IN: /* match set member (or non_member) */ /* <IN> <skip> <set> */ TRACE(("|%p|%p|IN\n", ctx->pattern, ctx->ptr)); - if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, *ctx->ptr)) - RETURN_FAILURE; + if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern + 1, SRE_CHARGET(state, ctx->ptr, 0))) + RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_LITERAL_IGNORE: TRACE(("|%p|%p|LITERAL_IGNORE %d\n", ctx->pattern, ctx->ptr, ctx->pattern[0])); if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*ctx->pattern)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*ctx->pattern)) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_NOT_LITERAL_IGNORE: TRACE(("|%p|%p|NOT_LITERAL_IGNORE %d\n", ctx->pattern, ctx->ptr, *ctx->pattern)); if (ctx->ptr >= end || - state->lower(*ctx->ptr) == state->lower(*ctx->pattern)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) == state->lower(*ctx->pattern)) RETURN_FAILURE; ctx->pattern++; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_IN_IGNORE: TRACE(("|%p|%p|IN_IGNORE\n", ctx->pattern, ctx->ptr)); if (ctx->ptr >= end || !SRE_CHARSET(ctx->pattern+1, - (SRE_CODE)state->lower(*ctx->ptr))) + (SRE_CODE)state->lower(SRE_CHARGET(state, ctx->ptr, 0)))) RETURN_FAILURE; ctx->pattern += ctx->pattern[0]; - ctx->ptr++; + ctx->ptr += state->charsize; break; case SRE_OP_JUMP: @@ -986,11 +962,11 @@ entrance: for (; ctx->pattern[0]; ctx->pattern += ctx->pattern[0]) { if (ctx->pattern[1] == SRE_OP_LITERAL && (ctx->ptr >= end || - (SRE_CODE) *ctx->ptr != ctx->pattern[2])) + (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0) != ctx->pattern[2])) continue; if (ctx->pattern[1] == SRE_OP_IN && (ctx->ptr >= end || - !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) *ctx->ptr))) + !SRE_CHARSET(ctx->pattern + 3, (SRE_CODE) SRE_CHARGET(state, ctx->ptr, 0)))) continue; state->ptr = ctx->ptr; DO_JUMP(JUMP_BRANCH, jump_branch, ctx->pattern+1); @@ -1021,7 +997,7 @@ entrance: TRACE(("|%p|%p|REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->ptr + state->charsize * ctx->pattern[1] > end) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1030,7 +1006,7 @@ entrance: RETURN_ON_ERROR(ret); DATA_LOOKUP_AT(SRE_MATCH_CONTEXT, ctx, ctx_pos); ctx->count = ret; - ctx->ptr += ctx->count; + ctx->ptr += state->charsize * ctx->count; /* when we arrive here, count contains the number of matches, and ctx->ptr points to the tail of the target @@ -1054,8 +1030,9 @@ entrance: ctx->u.chr = ctx->pattern[ctx->pattern[0]+1]; for (;;) { while (ctx->count >= (Py_ssize_t) ctx->pattern[1] && - (ctx->ptr >= end || *ctx->ptr != ctx->u.chr)) { - ctx->ptr--; + (ctx->ptr >= end || + SRE_CHARGET(state, ctx->ptr, 0) != ctx->u.chr)) { + ctx->ptr -= state->charsize; ctx->count--; } if (ctx->count < (Py_ssize_t) ctx->pattern[1]) @@ -1070,7 +1047,7 @@ entrance: LASTMARK_RESTORE(); - ctx->ptr--; + ctx->ptr -= state->charsize; ctx->count--; } @@ -1084,7 +1061,7 @@ entrance: RETURN_ON_ERROR(ret); RETURN_SUCCESS; } - ctx->ptr--; + ctx->ptr -= state->charsize; ctx->count--; LASTMARK_RESTORE(); } @@ -1104,7 +1081,7 @@ entrance: TRACE(("|%p|%p|MIN_REPEAT_ONE %d %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1], ctx->pattern[2])); - if (ctx->ptr + ctx->pattern[1] > end) + if (ctx->ptr + state->charsize * ctx->pattern[1] > end) RETURN_FAILURE; /* cannot match */ state->ptr = ctx->ptr; @@ -1121,7 +1098,7 @@ entrance: RETURN_FAILURE; /* advance past minimum matches of repeat */ ctx->count = ret; - ctx->ptr += ctx->count; + ctx->ptr += state->charsize * ctx->count; } if (ctx->pattern[ctx->pattern[0]] == SRE_OP_SUCCESS) { @@ -1148,7 +1125,7 @@ entrance: if (ret == 0) break; assert(ret == 1); - ctx->ptr++; + ctx->ptr += state->charsize; ctx->count++; LASTMARK_RESTORE(); } @@ -1320,14 +1297,16 @@ entrance: if (groupref >= state->lastmark) { RETURN_FAILURE; } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + char* p = (char*) state->mark[groupref]; + char* e = (char*) state->mark[groupref+1]; if (!p || !e || e < p) RETURN_FAILURE; while (p < e) { - if (ctx->ptr >= end || *ctx->ptr != *p) + if (ctx->ptr >= end || + SRE_CHARGET(state, ctx->ptr, 0) != SRE_CHARGET(state, p, 0)) RETURN_FAILURE; - p++; ctx->ptr++; + p += state->charsize; + ctx->ptr += state->charsize; } } } @@ -1344,15 +1323,16 @@ entrance: if (groupref >= state->lastmark) { RETURN_FAILURE; } else { - SRE_CHAR* p = (SRE_CHAR*) state->mark[groupref]; - SRE_CHAR* e = (SRE_CHAR*) state->mark[groupref+1]; + char* p = (char*) state->mark[groupref]; + char* e = (char*) state->mark[groupref+1]; if (!p || !e || e < p) RETURN_FAILURE; while (p < e) { if (ctx->ptr >= end || - state->lower(*ctx->ptr) != state->lower(*p)) + state->lower(SRE_CHARGET(state, ctx->ptr, 0)) != state->lower(*p)) RETURN_FAILURE; - p++; ctx->ptr++; + p++; + ctx->ptr += state->charsize; } } } @@ -1386,7 +1366,7 @@ entrance: /* <ASSERT> <skip> <back> <pattern> */ TRACE(("|%p|%p|ASSERT %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; + state->ptr = ctx->ptr - state->charsize * ctx->pattern[1]; if (state->ptr < state->beginning) RETURN_FAILURE; DO_JUMP(JUMP_ASSERT, jump_assert, ctx->pattern+2); @@ -1399,7 +1379,7 @@ entrance: /* <ASSERT_NOT> <skip> <back> <pattern> */ TRACE(("|%p|%p|ASSERT_NOT %d\n", ctx->pattern, ctx->ptr, ctx->pattern[1])); - state->ptr = ctx->ptr - ctx->pattern[1]; + state->ptr = ctx->ptr - state->charsize * ctx->pattern[1]; if (state->ptr >= state->beginning) { DO_JUMP(JUMP_ASSERT_NOT, jump_assert_not, ctx->pattern+2); if (ret) { @@ -1481,8 +1461,8 @@ exit: LOCAL(Py_ssize_t) SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) { - SRE_CHAR* ptr = (SRE_CHAR *)state->start; - SRE_CHAR* end = (SRE_CHAR *)state->end; + char* ptr = (char*)state->start; + char* end = (char*)state->end; Py_ssize_t status = 0; Py_ssize_t prefix_len = 0; Py_ssize_t prefix_skip = 0; @@ -1500,9 +1480,9 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) if (pattern[3] > 1) { /* adjust end point (but make sure we leave at least one character in there, so literal search will work) */ - end -= pattern[3]-1; + end -= (pattern[3]-1) * state->charsize; if (end <= ptr) - end = ptr+1; + end = ptr + state->charsize; } if (flags & SRE_INFO_PREFIX) { @@ -1528,10 +1508,10 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) /* pattern starts with a known prefix. use the overlap table to skip forward as fast as we possibly can */ Py_ssize_t i = 0; - end = (SRE_CHAR *)state->end; + end = (char *)state->end; while (ptr < end) { for (;;) { - if ((SRE_CODE) ptr[0] != prefix[i]) { + if ((SRE_CODE) SRE_CHARGET(state, ptr, 0) != prefix[i]) { if (!i) break; else @@ -1540,8 +1520,8 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) if (++i == prefix_len) { /* found a potential match */ TRACE(("|%p|%p|SEARCH SCAN\n", pattern, ptr)); - state->start = ptr + 1 - prefix_len; - state->ptr = ptr + 1 - prefix_len + prefix_skip; + state->start = ptr - (prefix_len - 1) * state->charsize; + state->ptr = ptr - (prefix_len - prefix_skip - 1) * state->charsize; if (flags & SRE_INFO_LITERAL) return 1; /* we got all of it */ status = SRE_MATCH(state, pattern + 2*prefix_skip); @@ -1553,7 +1533,7 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) break; } } - ptr++; + ptr += state->charsize; } return 0; } @@ -1563,15 +1543,16 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) /* pattern starts with a literal character. this is used for short prefixes, and if fast search is disabled */ SRE_CODE chr = pattern[1]; - end = (SRE_CHAR *)state->end; + end = (char*)state->end; for (;;) { - while (ptr < end && (SRE_CODE) ptr[0] != chr) - ptr++; + while (ptr < end && (SRE_CODE) SRE_CHARGET(state, ptr, 0) != chr) + ptr += state->charsize; if (ptr >= end) return 0; TRACE(("|%p|%p|SEARCH LITERAL\n", pattern, ptr)); state->start = ptr; - state->ptr = ++ptr; + ptr += state->charsize; + state->ptr = ptr; if (flags & SRE_INFO_LITERAL) return 1; /* we got all of it */ status = SRE_MATCH(state, pattern + 2); @@ -1580,10 +1561,10 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) } } else if (charset) { /* pattern starts with a character from a known set */ - end = (SRE_CHAR *)state->end; + end = (char*)state->end; for (;;) { - while (ptr < end && !SRE_CHARSET(charset, ptr[0])) - ptr++; + while (ptr < end && !SRE_CHARSET(charset, SRE_CHARGET(state, ptr, 0))) + ptr += state->charsize; if (ptr >= end) return 0; TRACE(("|%p|%p|SEARCH CHARSET\n", pattern, ptr)); @@ -1592,13 +1573,14 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) status = SRE_MATCH(state, pattern); if (status != 0) break; - ptr++; + ptr += state->charsize; } } else /* general case */ while (ptr <= end) { TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); - state->start = state->ptr = ptr++; + state->start = state->ptr = ptr; + ptr += state->charsize; status = SRE_MATCH(state, pattern); if (status != 0) break; @@ -1607,16 +1589,6 @@ SRE_SEARCH(SRE_STATE* state, SRE_CODE* pattern) return status; } -LOCAL(int) -SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len) -{ - /* check if given string is a literal template (i.e. no escapes) */ - while (len-- > 0) - if (*ptr++ == '\\') - return 0; - return 1; -} - #if !defined(SRE_RECURSIVE) /* -------------------------------------------------------------------- */ @@ -1626,6 +1598,23 @@ SRE_LITERAL_TEMPLATE(SRE_CHAR* ptr, Py_ssize_t len) static PyObject*pattern_new_match(PatternObject*, SRE_STATE*, int); static PyObject*pattern_scanner(PatternObject*, PyObject*); +static int +sre_literal_template(int charsize, char* ptr, Py_ssize_t len) +{ + /* check if given string is a literal template (i.e. no escapes) */ + struct { + int charsize; + } state = { + charsize + }; + while (len-- > 0) { + if (SRE_CHARGET((&state), ptr, 0) == '\\') + return 0; + ptr += charsize; + } + return 1; +} + static PyObject * sre_codesize(PyObject* self, PyObject *unused) { @@ -1641,11 +1630,7 @@ sre_getlower(PyObject* self, PyObject* args) if (flags & SRE_FLAG_LOCALE) return Py_BuildValue("i", sre_lower_locale(character)); if (flags & SRE_FLAG_UNICODE) -#if defined(HAVE_UNICODE) return Py_BuildValue("i", sre_lower_unicode(character)); -#else - return Py_BuildValue("i", sre_lower_locale(character)); -#endif return Py_BuildValue("i", sre_lower(character)); } @@ -1664,7 +1649,8 @@ state_reset(SRE_STATE* state) } static void* -getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) +getstring(PyObject* string, Py_ssize_t* p_length, + int* p_logical_charsize, int* p_charsize) { /* given a python object, return a data pointer, a length (in characters), and a character size. return NULL if the object @@ -1679,13 +1665,16 @@ getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) /* Unicode objects do not support the buffer API. So, get the data directly instead. */ if (PyUnicode_Check(string)) { - ptr = (void *)PyUnicode_AS_DATA(string); - *p_length = PyUnicode_GET_SIZE(string); - *p_charsize = sizeof(Py_UNICODE); + if (PyUnicode_READY(string) == -1) + return NULL; + ptr = PyUnicode_DATA(string); + *p_length = PyUnicode_GET_LENGTH(string); + *p_charsize = PyUnicode_KIND(string); + *p_logical_charsize = 4; return ptr; } - /* get pointer to string buffer */ + /* get pointer to byte string buffer */ view.len = -1; buffer = Py_TYPE(string)->tp_as_buffer; if (!buffer || !buffer->bf_getbuffer || @@ -1713,10 +1702,6 @@ getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) if (PyBytes_Check(string) || bytes == size) charsize = 1; -#if defined(HAVE_UNICODE) - else if (bytes == (Py_ssize_t) (size * sizeof(Py_UNICODE))) - charsize = sizeof(Py_UNICODE); -#endif else { PyErr_SetString(PyExc_TypeError, "buffer size mismatch"); return NULL; @@ -1724,6 +1709,7 @@ getstring(PyObject* string, Py_ssize_t* p_length, int* p_charsize) *p_length = size; *p_charsize = charsize; + *p_logical_charsize = charsize; if (ptr == NULL) { PyErr_SetString(PyExc_ValueError, @@ -1739,7 +1725,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, /* prepare state object */ Py_ssize_t length; - int charsize; + int logical_charsize, charsize; void* ptr; memset(state, 0, sizeof(SRE_STATE)); @@ -1747,20 +1733,20 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, state->lastmark = -1; state->lastindex = -1; - ptr = getstring(string, &length, &charsize); + ptr = getstring(string, &length, &logical_charsize, &charsize); if (!ptr) return NULL; - if (charsize == 1 && pattern->charsize > 1) { - PyErr_SetString(PyExc_TypeError, - "can't use a string pattern on a bytes-like object"); - return NULL; - } - if (charsize > 1 && pattern->charsize == 1) { - PyErr_SetString(PyExc_TypeError, - "can't use a bytes pattern on a string-like object"); - return NULL; - } + if (logical_charsize == 1 && pattern->logical_charsize > 1) { + PyErr_SetString(PyExc_TypeError, + "can't use a string pattern on a bytes-like object"); + return NULL; + } + if (logical_charsize > 1 && pattern->logical_charsize == 1) { + PyErr_SetString(PyExc_TypeError, + "can't use a bytes pattern on a string-like object"); + return NULL; + } /* adjust boundaries */ if (start < 0) @@ -1773,6 +1759,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, else if (end > length) end = length; + state->logical_charsize = logical_charsize; state->charsize = charsize; state->beginning = ptr; @@ -1788,11 +1775,7 @@ state_init(SRE_STATE* state, PatternObject* pattern, PyObject* string, if (pattern->flags & SRE_FLAG_LOCALE) state->lower = sre_lower_locale; else if (pattern->flags & SRE_FLAG_UNICODE) -#if defined(HAVE_UNICODE) state->lower = sre_lower_unicode; -#else - state->lower = sre_lower_locale; -#endif else state->lower = sre_lower; @@ -1891,12 +1874,10 @@ pattern_match(PatternObject* self, PyObject* args, PyObject* kw) TRACE(("|%p|%p|MATCH\n", PatternObject_GetCode(self), state.ptr)); - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_match(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_umatch(&state, PatternObject_GetCode(self)); -#endif } TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); @@ -1928,12 +1909,10 @@ pattern_search(PatternObject* self, PyObject* args, PyObject* kw) TRACE(("|%p|%p|SEARCH\n", PatternObject_GetCode(self), state.ptr)); - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } TRACE(("|%p|%p|END\n", PatternObject_GetCode(self), state.ptr)); @@ -2075,16 +2054,14 @@ pattern_findall(PatternObject* self, PyObject* args, PyObject* kw) state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2205,16 +2182,14 @@ pattern_split(PatternObject* self, PyObject* args, PyObject* kw) state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2295,7 +2270,7 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, int status; Py_ssize_t n; Py_ssize_t i, b, e; - int bint; + int logical_charsize, charsize; int filter_is_callable; if (PyCallable_Check(ptemplate)) { @@ -2306,16 +2281,10 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, } else { /* if not callable, check if it's a literal string */ int literal; - ptr = getstring(ptemplate, &n, &bint); - b = bint; + ptr = getstring(ptemplate, &n, &logical_charsize, &charsize); + b = charsize; if (ptr) { - if (b == 1) { - literal = sre_literal_template((unsigned char *)ptr, n); - } else { -#if defined(HAVE_UNICODE) - literal = sre_uliteral_template((Py_UNICODE *)ptr, n); -#endif - } + literal = sre_literal_template(b, ptr, n); } else { PyErr_Clear(); literal = 0; @@ -2357,16 +2326,14 @@ pattern_subx(PatternObject* self, PyObject* ptemplate, PyObject* string, state.ptr = state.start; - if (state.charsize == 1) { + if (state.logical_charsize == 1) { status = sre_search(&state, PatternObject_GetCode(self)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(&state, PatternObject_GetCode(self)); -#endif } - if (PyErr_Occurred()) - goto error; + if (PyErr_Occurred()) + goto error; if (status <= 0) { if (status == 0) @@ -2584,20 +2551,20 @@ PyDoc_STRVAR(pattern_doc, "Compiled regular expression objects"); static PyMethodDef pattern_methods[] = { {"match", (PyCFunction) pattern_match, METH_VARARGS|METH_KEYWORDS, - pattern_match_doc}, + pattern_match_doc}, {"search", (PyCFunction) pattern_search, METH_VARARGS|METH_KEYWORDS, - pattern_search_doc}, + pattern_search_doc}, {"sub", (PyCFunction) pattern_sub, METH_VARARGS|METH_KEYWORDS, - pattern_sub_doc}, + pattern_sub_doc}, {"subn", (PyCFunction) pattern_subn, METH_VARARGS|METH_KEYWORDS, - pattern_subn_doc}, + pattern_subn_doc}, {"split", (PyCFunction) pattern_split, METH_VARARGS|METH_KEYWORDS, - pattern_split_doc}, + pattern_split_doc}, {"findall", (PyCFunction) pattern_findall, METH_VARARGS|METH_KEYWORDS, - pattern_findall_doc}, + pattern_findall_doc}, #if PY_VERSION_HEX >= 0x02020000 {"finditer", (PyCFunction) pattern_finditer, METH_VARARGS, - pattern_finditer_doc}, + pattern_finditer_doc}, #endif {"scanner", (PyCFunction) pattern_scanner, METH_VARARGS}, {"__copy__", (PyCFunction) pattern_copy, METH_NOARGS}, @@ -2618,31 +2585,31 @@ static PyTypeObject Pattern_Type = { PyVarObject_HEAD_INIT(NULL, 0) "_" SRE_MODULE ".SRE_Pattern", sizeof(PatternObject), sizeof(SRE_CODE), - (destructor)pattern_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - pattern_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(PatternObject, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - pattern_methods, /* tp_methods */ - pattern_members, /* tp_members */ + (destructor)pattern_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + pattern_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + offsetof(PatternObject, weakreflist), /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + pattern_methods, /* tp_methods */ + pattern_members, /* tp_members */ }; static int _validate(PatternObject *self); /* Forward */ @@ -2694,15 +2661,18 @@ _compile(PyObject* self_, PyObject* args) return NULL; } - if (pattern == Py_None) - self->charsize = -1; - else { - Py_ssize_t p_length; - if (!getstring(pattern, &p_length, &self->charsize)) { - Py_DECREF(self); - return NULL; - } - } + if (pattern == Py_None) { + self->logical_charsize = -1; + self->charsize = -1; + } + else { + Py_ssize_t p_length; + if (!getstring(pattern, &p_length, &self->logical_charsize, + &self->charsize)) { + Py_DECREF(self); + return NULL; + } + } Py_INCREF(pattern); self->pattern = pattern; @@ -3238,8 +3208,8 @@ match_getindex(MatchObject* self, PyObject* index) Py_ssize_t i; if (index == NULL) - /* Default value */ - return 0; + /* Default value */ + return 0; if (PyLong_Check(index)) return PyLong_AsSsize_t(index); @@ -3579,7 +3549,7 @@ static PyObject * match_lastindex_get(MatchObject *self) { if (self->lastindex >= 0) - return Py_BuildValue("i", self->lastindex); + return Py_BuildValue("i", self->lastindex); Py_INCREF(Py_None); return Py_None; } @@ -3632,32 +3602,32 @@ static PyTypeObject Match_Type = { PyVarObject_HEAD_INIT(NULL,0) "_" SRE_MODULE ".SRE_Match", sizeof(MatchObject), sizeof(Py_ssize_t), - (destructor)match_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - match_methods, /* tp_methods */ - match_members, /* tp_members */ - match_getset, /* tp_getset */ + (destructor)match_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + match_methods, /* tp_methods */ + match_members, /* tp_members */ + match_getset, /* tp_getset */ }; static PyObject* @@ -3746,12 +3716,10 @@ scanner_match(ScannerObject* self, PyObject *unused) state->ptr = state->start; - if (state->charsize == 1) { + if (state->logical_charsize == 1) { status = sre_match(state, PatternObject_GetCode(self->pattern)); } else { -#if defined(HAVE_UNICODE) status = sre_umatch(state, PatternObject_GetCode(self->pattern)); -#endif } if (PyErr_Occurred()) return NULL; @@ -3779,12 +3747,10 @@ scanner_search(ScannerObject* self, PyObject *unused) state->ptr = state->start; - if (state->charsize == 1) { + if (state->logical_charsize == 1) { status = sre_search(state, PatternObject_GetCode(self->pattern)); } else { -#if defined(HAVE_UNICODE) status = sre_usearch(state, PatternObject_GetCode(self->pattern)); -#endif } if (PyErr_Occurred()) return NULL; @@ -3808,7 +3774,7 @@ static PyMethodDef scanner_methods[] = { #define SCAN_OFF(x) offsetof(ScannerObject, x) static PyMemberDef scanner_members[] = { - {"pattern", T_OBJECT, SCAN_OFF(pattern), READONLY}, + {"pattern", T_OBJECT, SCAN_OFF(pattern), READONLY}, {NULL} /* Sentinel */ }; @@ -3817,31 +3783,31 @@ static PyTypeObject Scanner_Type = { "_" SRE_MODULE ".SRE_Scanner", sizeof(ScannerObject), 0, (destructor)scanner_dealloc,/* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - scanner_methods, /* tp_methods */ - scanner_members, /* tp_members */ - 0, /* tp_getset */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + scanner_methods, /* tp_methods */ + scanner_members, /* tp_members */ + 0, /* tp_getset */ }; static PyObject* @@ -3883,15 +3849,15 @@ static PyMethodDef _functions[] = { }; static struct PyModuleDef sremodule = { - PyModuleDef_HEAD_INIT, - "_" SRE_MODULE, - NULL, - -1, - _functions, - NULL, - NULL, - NULL, - NULL + PyModuleDef_HEAD_INIT, + "_" SRE_MODULE, + NULL, + -1, + _functions, + NULL, + NULL, + NULL, + NULL }; PyMODINIT_FUNC PyInit__sre(void) @@ -3907,7 +3873,7 @@ PyMODINIT_FUNC PyInit__sre(void) m = PyModule_Create(&sremodule); if (m == NULL) - return NULL; + return NULL; d = PyModule_GetDict(m); x = PyLong_FromLong(SRE_MAGIC); diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 84ec477..7fee74c 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -18,16 +18,21 @@ #ifdef WITH_THREAD #include "pythread.h" +#define PySSL_BEGIN_ALLOW_THREADS_S(save) \ + do { if (_ssl_locks_count>0) { (save) = PyEval_SaveThread(); } } while (0) +#define PySSL_END_ALLOW_THREADS_S(save) \ + do { if (_ssl_locks_count>0) { PyEval_RestoreThread(save); } } while (0) #define PySSL_BEGIN_ALLOW_THREADS { \ PyThreadState *_save = NULL; \ - if (_ssl_locks_count>0) {_save = PyEval_SaveThread();} -#define PySSL_BLOCK_THREADS if (_ssl_locks_count>0){PyEval_RestoreThread(_save)}; -#define PySSL_UNBLOCK_THREADS if (_ssl_locks_count>0){_save = PyEval_SaveThread()}; -#define PySSL_END_ALLOW_THREADS if (_ssl_locks_count>0){PyEval_RestoreThread(_save);} \ - } + PySSL_BEGIN_ALLOW_THREADS_S(_save); +#define PySSL_BLOCK_THREADS PySSL_END_ALLOW_THREADS_S(_save); +#define PySSL_UNBLOCK_THREADS PySSL_BEGIN_ALLOW_THREADS_S(_save); +#define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } #else /* no WITH_THREAD */ +#define PySSL_BEGIN_ALLOW_THREADS_S(save) +#define PySSL_END_ALLOW_THREADS_S(save) #define PySSL_BEGIN_ALLOW_THREADS #define PySSL_BLOCK_THREADS #define PySSL_UNBLOCK_THREADS @@ -124,6 +129,17 @@ static unsigned int _ssl_locks_count = 0; # undef HAVE_SSL_CTX_CLEAR_OPTIONS #endif +/* In case of 'tls-unique' it will be 12 bytes for TLS, 36 bytes for + * older SSL, but let's be safe */ +#define PySSL_CB_MAXLEN 128 + +/* SSL_get_finished got added to OpenSSL in 0.9.5 */ +#if OPENSSL_VERSION_NUMBER >= 0x0090500fL +# define HAVE_OPENSSL_FINISHED 1 +#else +# define HAVE_OPENSSL_FINISHED 0 +#endif + typedef struct { PyObject_HEAD SSL_CTX *ctx; @@ -135,6 +151,7 @@ typedef struct { SSL *ssl; X509 *peer_cert; int shutdown_seen_zero; + enum py_ssl_server_or_client socket_type; } PySSLSocket; static PyTypeObject PySSLContext_Type; @@ -328,6 +345,7 @@ newPySSLSocket(SSL_CTX *ctx, PySocketSockObject *sock, SSL_set_accept_state(self->ssl); PySSL_END_ALLOW_THREADS + self->socket_type = socket_type; self->Socket = PyWeakref_NewRef((PyObject *) sock, NULL); return self; } @@ -356,7 +374,6 @@ static PyObject *PySSL_SSLdo_handshake(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ - sockstate = 0; do { PySSL_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); @@ -1090,7 +1107,6 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) goto error; } do { - err = 0; PySSL_BEGIN_ALLOW_THREADS len = SSL_write(self->ssl, buf.buf, buf.len); err = SSL_get_error(self->ssl, len); @@ -1226,7 +1242,6 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) } } do { - err = 0; PySSL_BEGIN_ALLOW_THREADS count = SSL_read(self->ssl, mem, len); err = SSL_get_error(self->ssl, count); @@ -1378,6 +1393,41 @@ PyDoc_STRVAR(PySSL_SSLshutdown_doc, Does the SSL shutdown handshake with the remote end, and returns\n\ the underlying socket object."); +#if HAVE_OPENSSL_FINISHED +static PyObject * +PySSL_tls_unique_cb(PySSLSocket *self) +{ + PyObject *retval = NULL; + char buf[PySSL_CB_MAXLEN]; + int len; + + if (SSL_session_reused(self->ssl) ^ !self->socket_type) { + /* if session is resumed XOR we are the client */ + len = SSL_get_finished(self->ssl, buf, PySSL_CB_MAXLEN); + } + else { + /* if a new session XOR we are the server */ + len = SSL_get_peer_finished(self->ssl, buf, PySSL_CB_MAXLEN); + } + + /* It cannot be negative in current OpenSSL version as of July 2011 */ + assert(len >= 0); + if (len == 0) + Py_RETURN_NONE; + + retval = PyBytes_FromStringAndSize(buf, len); + + return retval; +} + +PyDoc_STRVAR(PySSL_tls_unique_cb_doc, +"tls_unique_cb() -> bytes\n\ +\n\ +Returns the 'tls-unique' channel binding data, as defined by RFC 5929.\n\ +\n\ +If the TLS handshake is not yet complete, None is returned"); + +#endif /* HAVE_OPENSSL_FINISHED */ static PyMethodDef PySSLMethods[] = { {"do_handshake", (PyCFunction)PySSL_SSLdo_handshake, METH_NOARGS}, @@ -1392,6 +1442,10 @@ static PyMethodDef PySSLMethods[] = { {"cipher", (PyCFunction)PySSL_cipher, METH_NOARGS}, {"shutdown", (PyCFunction)PySSL_SSLshutdown, METH_NOARGS, PySSL_SSLshutdown_doc}, +#if HAVE_OPENSSL_FINISHED + {"tls_unique_cb", (PyCFunction)PySSL_tls_unique_cb, METH_NOARGS, + PySSL_tls_unique_cb_doc}, +#endif {NULL, NULL} }; @@ -1584,19 +1638,118 @@ set_options(PySSLContext *self, PyObject *arg, void *c) return 0; } +typedef struct { + PyThreadState *thread_state; + PyObject *callable; + char *password; + Py_ssize_t size; + int error; +} _PySSLPasswordInfo; + +static int +_pwinfo_set(_PySSLPasswordInfo *pw_info, PyObject* password, + const char *bad_type_error) +{ + /* Set the password and size fields of a _PySSLPasswordInfo struct + from a unicode, bytes, or byte array object. + The password field will be dynamically allocated and must be freed + by the caller */ + PyObject *password_bytes = NULL; + const char *data = NULL; + Py_ssize_t size; + + if (PyUnicode_Check(password)) { + password_bytes = PyUnicode_AsEncodedString(password, NULL, NULL); + if (!password_bytes) { + goto error; + } + data = PyBytes_AS_STRING(password_bytes); + size = PyBytes_GET_SIZE(password_bytes); + } else if (PyBytes_Check(password)) { + data = PyBytes_AS_STRING(password); + size = PyBytes_GET_SIZE(password); + } else if (PyByteArray_Check(password)) { + data = PyByteArray_AS_STRING(password); + size = PyByteArray_GET_SIZE(password); + } else { + PyErr_SetString(PyExc_TypeError, bad_type_error); + goto error; + } + + free(pw_info->password); + pw_info->password = malloc(size); + if (!pw_info->password) { + PyErr_SetString(PyExc_MemoryError, + "unable to allocate password buffer"); + goto error; + } + memcpy(pw_info->password, data, size); + pw_info->size = size; + + Py_XDECREF(password_bytes); + return 1; + +error: + Py_XDECREF(password_bytes); + return 0; +} + +static int +_password_callback(char *buf, int size, int rwflag, void *userdata) +{ + _PySSLPasswordInfo *pw_info = (_PySSLPasswordInfo*) userdata; + PyObject *fn_ret = NULL; + + PySSL_END_ALLOW_THREADS_S(pw_info->thread_state); + + if (pw_info->callable) { + fn_ret = PyObject_CallFunctionObjArgs(pw_info->callable, NULL); + if (!fn_ret) { + /* TODO: It would be nice to move _ctypes_add_traceback() into the + core python API, so we could use it to add a frame here */ + goto error; + } + + if (!_pwinfo_set(pw_info, fn_ret, + "password callback must return a string")) { + goto error; + } + Py_CLEAR(fn_ret); + } + + if (pw_info->size > size) { + PyErr_Format(PyExc_ValueError, + "password cannot be longer than %d bytes", size); + goto error; + } + + PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + memcpy(buf, pw_info->password, pw_info->size); + return pw_info->size; + +error: + Py_XDECREF(fn_ret); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->error = 1; + return -1; +} + static PyObject * load_cert_chain(PySSLContext *self, PyObject *args, PyObject *kwds) { - char *kwlist[] = {"certfile", "keyfile", NULL}; - PyObject *certfile, *keyfile = NULL; + char *kwlist[] = {"certfile", "keyfile", "password", NULL}; + PyObject *certfile, *keyfile = NULL, *password = NULL; PyObject *certfile_bytes = NULL, *keyfile_bytes = NULL; + pem_password_cb *orig_passwd_cb = self->ctx->default_passwd_callback; + void *orig_passwd_userdata = self->ctx->default_passwd_callback_userdata; + _PySSLPasswordInfo pw_info = { NULL, NULL, NULL, 0, 0 }; int r; errno = 0; ERR_clear_error(); if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O|O:load_cert_chain", kwlist, - &certfile, &keyfile)) + "O|OO:load_cert_chain", kwlist, + &certfile, &keyfile, &password)) return NULL; if (keyfile == Py_None) keyfile = NULL; @@ -1610,12 +1763,26 @@ load_cert_chain(PySSLContext *self, PyObject *args, PyObject *kwds) "keyfile should be a valid filesystem path"); goto error; } - PySSL_BEGIN_ALLOW_THREADS + if (password && password != Py_None) { + if (PyCallable_Check(password)) { + pw_info.callable = password; + } else if (!_pwinfo_set(&pw_info, password, + "password should be a string or callable")) { + goto error; + } + SSL_CTX_set_default_passwd_cb(self->ctx, _password_callback); + SSL_CTX_set_default_passwd_cb_userdata(self->ctx, &pw_info); + } + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); r = SSL_CTX_use_certificate_chain_file(self->ctx, PyBytes_AS_STRING(certfile_bytes)); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); if (r != 1) { - if (errno != 0) { + if (pw_info.error) { + ERR_clear_error(); + /* the password callback has already set the error information */ + } + else if (errno != 0) { ERR_clear_error(); PyErr_SetFromErrno(PyExc_IOError); } @@ -1624,33 +1791,43 @@ load_cert_chain(PySSLContext *self, PyObject *args, PyObject *kwds) } goto error; } - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); r = SSL_CTX_use_PrivateKey_file(self->ctx, PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), SSL_FILETYPE_PEM); - PySSL_END_ALLOW_THREADS - Py_XDECREF(keyfile_bytes); - Py_XDECREF(certfile_bytes); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + Py_CLEAR(keyfile_bytes); + Py_CLEAR(certfile_bytes); if (r != 1) { - if (errno != 0) { + if (pw_info.error) { + ERR_clear_error(); + /* the password callback has already set the error information */ + } + else if (errno != 0) { ERR_clear_error(); PyErr_SetFromErrno(PyExc_IOError); } else { _setSSLError(NULL, 0, __FILE__, __LINE__); } - return NULL; + goto error; } - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); r = SSL_CTX_check_private_key(self->ctx); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); if (r != 1) { _setSSLError(NULL, 0, __FILE__, __LINE__); - return NULL; + goto error; } + SSL_CTX_set_default_passwd_cb(self->ctx, orig_passwd_cb); + SSL_CTX_set_default_passwd_cb_userdata(self->ctx, orig_passwd_userdata); + free(pw_info.password); Py_RETURN_NONE; error: + SSL_CTX_set_default_passwd_cb(self->ctx, orig_passwd_cb); + SSL_CTX_set_default_passwd_cb_userdata(self->ctx, orig_passwd_userdata); + free(pw_info.password); Py_XDECREF(keyfile_bytes); Py_XDECREF(certfile_bytes); return NULL; @@ -1888,6 +2065,69 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower\n\ bound on the entropy contained in string. See RFC 1750."); static PyObject * +PySSL_RAND(int len, int pseudo) +{ + int ok; + PyObject *bytes; + unsigned long err; + const char *errstr; + PyObject *v; + + bytes = PyBytes_FromStringAndSize(NULL, len); + if (bytes == NULL) + return NULL; + if (pseudo) { + ok = RAND_pseudo_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 0 || ok == 1) + return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); + } + else { + ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 1) + return bytes; + } + Py_DECREF(bytes); + + err = ERR_get_error(); + errstr = ERR_reason_error_string(err); + v = Py_BuildValue("(ks)", err, errstr); + if (v != NULL) { + PyErr_SetObject(PySSLErrorObject, v); + Py_DECREF(v); + } + return NULL; +} + +static PyObject * +PySSL_RAND_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_bytes", &len)) + return NULL; + return PySSL_RAND(len, 0); +} + +PyDoc_STRVAR(PySSL_RAND_bytes_doc, +"RAND_bytes(n) -> bytes\n\ +\n\ +Generate n cryptographically strong pseudo-random bytes."); + +static PyObject * +PySSL_RAND_pseudo_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_pseudo_bytes", &len)) + return NULL; + return PySSL_RAND(len, 1); +} + +PyDoc_STRVAR(PySSL_RAND_pseudo_bytes_doc, +"RAND_pseudo_bytes(n) -> (bytes, is_cryptographic)\n\ +\n\ +Generate n pseudo-random bytes. is_cryptographic is True if the bytes\ +generated are cryptographically strong."); + +static PyObject * PySSL_RAND_status(PyObject *self) { return PyLong_FromLong(RAND_status()); @@ -1940,6 +2180,10 @@ static PyMethodDef PySSL_methods[] = { #ifdef HAVE_OPENSSL_RAND {"RAND_add", PySSL_RAND_add, METH_VARARGS, PySSL_RAND_add_doc}, + {"RAND_bytes", PySSL_RAND_bytes, METH_VARARGS, + PySSL_RAND_bytes_doc}, + {"RAND_pseudo_bytes", PySSL_RAND_pseudo_bytes, METH_VARARGS, + PySSL_RAND_pseudo_bytes_doc}, {"RAND_egd", PySSL_RAND_egd, METH_VARARGS, PySSL_RAND_egd_doc}, {"RAND_status", (PyCFunction)PySSL_RAND_status, METH_NOARGS, @@ -2155,6 +2399,14 @@ PyInit__ssl(void) Py_INCREF(r); PyModule_AddObject(m, "HAS_SNI", r); +#if HAVE_OPENSSL_FINISHED + r = Py_True; +#else + r = Py_False; +#endif + Py_INCREF(r); + PyModule_AddObject(m, "HAS_TLS_UNIQUE", r); + /* OpenSSL version */ /* SSLeay() gives us the version of the library linked against, which could be different from the headers version. diff --git a/Modules/_struct.c b/Modules/_struct.c index 1604b90..dcdc83e 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -58,6 +58,7 @@ typedef struct { char c; long x; } st_long; typedef struct { char c; float x; } st_float; typedef struct { char c; double x; } st_double; typedef struct { char c; void *x; } st_void_p; +typedef struct { char c; size_t x; } st_size_t; #define SHORT_ALIGN (sizeof(st_short) - sizeof(short)) #define INT_ALIGN (sizeof(st_int) - sizeof(int)) @@ -65,6 +66,7 @@ typedef struct { char c; void *x; } st_void_p; #define FLOAT_ALIGN (sizeof(st_float) - sizeof(float)) #define DOUBLE_ALIGN (sizeof(st_double) - sizeof(double)) #define VOID_P_ALIGN (sizeof(st_void_p) - sizeof(void *)) +#define SIZE_T_ALIGN (sizeof(st_size_t) - sizeof(size_t)) /* We can't support q and Q in native mode unless the compiler does; in std mode, they're 8 bytes on all platforms. */ @@ -213,6 +215,52 @@ get_ulonglong(PyObject *v, unsigned PY_LONG_LONG *p) #endif +/* Same, but handling Py_ssize_t */ + +static int +get_ssize_t(PyObject *v, Py_ssize_t *p) +{ + Py_ssize_t x; + + v = get_pylong(v); + if (v == NULL) + return -1; + assert(PyLong_Check(v)); + x = PyLong_AsSsize_t(v); + Py_DECREF(v); + if (x == (Py_ssize_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) + PyErr_SetString(StructError, + "argument out of range"); + return -1; + } + *p = x; + return 0; +} + +/* Same, but handling size_t */ + +static int +get_size_t(PyObject *v, size_t *p) +{ + size_t x; + + v = get_pylong(v); + if (v == NULL) + return -1; + assert(PyLong_Check(v)); + x = PyLong_AsSize_t(v); + Py_DECREF(v); + if (x == (size_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) + PyErr_SetString(StructError, + "argument out of range"); + return -1; + } + *p = x; + return 0; +} + #define RANGE_ERROR(x, f, flag, mask) return _range_error(f, flag) @@ -369,6 +417,23 @@ nu_ulong(const char *p, const formatdef *f) return PyLong_FromUnsignedLong(x); } +static PyObject * +nu_ssize_t(const char *p, const formatdef *f) +{ + Py_ssize_t x; + memcpy((char *)&x, p, sizeof x); + return PyLong_FromSsize_t(x); +} + +static PyObject * +nu_size_t(const char *p, const formatdef *f) +{ + size_t x; + memcpy((char *)&x, p, sizeof x); + return PyLong_FromSize_t(x); +} + + /* Native mode doesn't support q or Q unless the platform C supports long long (or, on Windows, __int64). */ @@ -558,6 +623,26 @@ np_ulong(char *p, PyObject *v, const formatdef *f) return 0; } +static int +np_ssize_t(char *p, PyObject *v, const formatdef *f) +{ + Py_ssize_t x; + if (get_ssize_t(v, &x) < 0) + return -1; + memcpy(p, (char *)&x, sizeof x); + return 0; +} + +static int +np_size_t(char *p, PyObject *v, const formatdef *f) +{ + size_t x; + if (get_size_t(v, &x) < 0) + return -1; + memcpy(p, (char *)&x, sizeof x); + return 0; +} + #ifdef HAVE_LONG_LONG static int @@ -651,6 +736,8 @@ static formatdef native_table[] = { {'I', sizeof(int), INT_ALIGN, nu_uint, np_uint}, {'l', sizeof(long), LONG_ALIGN, nu_long, np_long}, {'L', sizeof(long), LONG_ALIGN, nu_ulong, np_ulong}, + {'n', sizeof(size_t), SIZE_T_ALIGN, nu_ssize_t, np_ssize_t}, + {'N', sizeof(size_t), SIZE_T_ALIGN, nu_size_t, np_size_t}, #ifdef HAVE_LONG_LONG {'q', sizeof(PY_LONG_LONG), LONG_LONG_ALIGN, nu_longlong, np_longlong}, {'Q', sizeof(PY_LONG_LONG), LONG_LONG_ALIGN, nu_ulonglong,np_ulonglong}, @@ -1951,7 +2038,8 @@ these can be preceded by a decimal repeat count:\n\ l:long; L:unsigned long; f:float; d:double.\n\ Special cases (preceding decimal count indicates length):\n\ s:string (array of char); p: pascal string (with count byte).\n\ -Special case (only available in native format):\n\ +Special cases (only available in native format):\n\ + n:ssize_t; N:size_t;\n\ P:an integer type that is wide enough to hold a pointer.\n\ Special case (not in native mode unless 'long long' in platform C):\n\ q:long long; Q:unsigned long long\n\ diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f19d0df..f1eac91 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -22,14 +22,7 @@ static PyObject *TestError; /* set to exception object in init */ static PyObject * raiseTestError(const char* test_name, const char* msg) { - char buf[2048]; - - if (strlen(test_name) + strlen(msg) > sizeof(buf) - 50) - PyErr_SetString(TestError, "internal error msg too large"); - else { - PyOS_snprintf(buf, sizeof(buf), "%s: %s", test_name, msg); - PyErr_SetString(TestError, buf); - } + PyErr_Format(TestError, "%s: %s", test_name, msg); return NULL; } @@ -43,11 +36,9 @@ static PyObject* sizeof_error(const char* fatname, const char* typname, int expected, int got) { - char buf[1024]; - PyOS_snprintf(buf, sizeof(buf), - "%.200s #define == %d but sizeof(%.200s) == %d", + PyErr_Format(TestError, + "%s #define == %d but sizeof(%s) == %d", fatname, expected, typname, got); - PyErr_SetString(TestError, buf); return (PyObject*)NULL; } @@ -778,6 +769,68 @@ test_long_long_and_overflow(PyObject *self) return Py_None; } +/* Test the PyLong_As{Size,Ssize}_t API. At present this just tests that + non-integer arguments are handled correctly. It should be extended to + test overflow handling. + */ + +static PyObject * +test_long_as_size_t(PyObject *self) +{ + size_t out_u; + Py_ssize_t out_s; + + Py_INCREF(Py_None); + + out_u = PyLong_AsSize_t(Py_None); + if (out_u != (size_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + out_s = PyLong_AsSsize_t(Py_None); + if (out_s != (Py_ssize_t)-1 || !PyErr_Occurred()) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_size_t", + "PyLong_AsSsize_t(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + +/* Test the PyLong_AsDouble API. At present this just tests that + non-integer arguments are handled correctly. + */ + +static PyObject * +test_long_as_double(PyObject *self) +{ + double out; + + Py_INCREF(Py_None); + + out = PyLong_AsDouble(Py_None); + if (out != -1.0 || !PyErr_Occurred()) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return raiseTestError("test_long_as_double", + "PyLong_AsDouble(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + /* Py_INCREF(Py_None) omitted - we already have a reference to it. */ + return Py_None; +} + /* Test the L code for PyArg_ParseTuple. This should deliver a PY_LONG_LONG for both long and int arguments. The test may leak a little memory if it fails. @@ -1012,6 +1065,15 @@ test_k_code(PyObject *self) } static PyObject * +getargs_c(PyObject *self, PyObject *args) +{ + char c; + if (!PyArg_ParseTuple(args, "c", &c)) + return NULL; + return PyBytes_FromStringAndSize(&c, 1); +} + +static PyObject * getargs_s(PyObject *self, PyObject *args) { char *str; @@ -1293,7 +1355,7 @@ static PyObject * test_Z_code(PyObject *self) { PyObject *tuple, *obj; - Py_UNICODE *value1, *value2; + const Py_UNICODE *value1, *value2; Py_ssize_t len1, len2; tuple = PyTuple_New(2); @@ -1531,7 +1593,7 @@ test_long_numbits(PyObject *self) {-0xfffffffL, 28, -1}}; int i; - for (i = 0; i < sizeof(testcases) / sizeof(struct triple); ++i) { + for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) { PyObject *plong = PyLong_FromLong(testcases[i].input); size_t nbits = _PyLong_NumBits(plong); int sign = _PyLong_Sign(plong); @@ -2267,6 +2329,8 @@ static PyMethodDef TestMethods[] = { {"test_long_api", (PyCFunction)test_long_api, METH_NOARGS}, {"test_long_and_overflow", (PyCFunction)test_long_and_overflow, METH_NOARGS}, + {"test_long_as_double", (PyCFunction)test_long_as_double,METH_NOARGS}, + {"test_long_as_size_t", (PyCFunction)test_long_as_size_t,METH_NOARGS}, {"test_long_numbits", (PyCFunction)test_long_numbits, METH_NOARGS}, {"test_k_code", (PyCFunction)test_k_code, METH_NOARGS}, {"test_empty_argparse", (PyCFunction)test_empty_argparse,METH_NOARGS}, @@ -2298,6 +2362,7 @@ static PyMethodDef TestMethods[] = { (PyCFunction)test_long_long_and_overflow, METH_NOARGS}, {"test_L_code", (PyCFunction)test_L_code, METH_NOARGS}, #endif + {"getargs_c", getargs_c, METH_VARARGS}, {"getargs_s", getargs_s, METH_VARARGS}, {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_s_hash", getargs_s_hash, METH_VARARGS}, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ea038de..6e39ca0 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -53,6 +53,7 @@ acquire_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds) _PyTime_timeval curtime; _PyTime_timeval endtime; + if (microseconds > 0) { _PyTime_gettimeofday(&endtime); endtime.tv_sec += microseconds / (1000 * 1000); @@ -75,7 +76,7 @@ acquire_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds) /* If we're using a timeout, recompute the timeout after processing * signals, since those can take time. */ - if (microseconds >= 0) { + if (microseconds > 0) { _PyTime_gettimeofday(&curtime); microseconds = ((endtime.tv_sec - curtime.tv_sec) * 1000000 + (endtime.tv_usec - curtime.tv_usec)); @@ -413,6 +414,12 @@ rlock_release_save(rlockobject *self) long owner; unsigned long count; + if (self->rlock_count == 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot release un-acquired lock"); + return NULL; + } + owner = self->rlock_owner; count = self->rlock_count; self->rlock_count = 0; @@ -1222,11 +1229,9 @@ the suggested approach in the absence of more specific information)."); static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, - start_new_doc}, + METH_VARARGS, start_new_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, - start_new_doc}, + METH_VARARGS, start_new_doc}, {"allocate_lock", (PyCFunction)thread_PyThread_allocate_lock, METH_NOARGS, allocate_doc}, {"allocate", (PyCFunction)thread_PyThread_allocate_lock, @@ -1242,8 +1247,7 @@ static PyMethodDef thread_methods[] = { {"_count", (PyCFunction)thread__count, METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, - METH_VARARGS, - stack_size_doc}, + METH_VARARGS, stack_size_doc}, {NULL, NULL} /* sentinel */ }; @@ -1307,7 +1311,9 @@ PyInit__thread(void) /* Add a symbolic constant */ d = PyModule_GetDict(m); - ThreadError = PyErr_NewException("_thread.error", NULL, NULL); + ThreadError = PyExc_RuntimeError; + Py_INCREF(ThreadError); + PyDict_SetItemString(d, "error", ThreadError); Locktype.tp_doc = lock_doc; Py_INCREF(&Locktype); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 672837f..ffb8ab0 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -80,18 +80,6 @@ Copyright (C) 1994 Steen Lumholt. #error "Tk older than 8.3.1 not supported" #endif -/* Unicode conversion assumes that Tcl_UniChar is two bytes. - We cannot test this directly, so we test UTF-8 size instead, - expecting that TCL_UTF_MAX is changed if Tcl ever supports - either UTF-16 or UCS-4. - Redhat 8 sets TCL_UTF_MAX to 6, and uses wchar_t for - Tcl_Unichar. This is also ok as long as Python uses UCS-4, - as well. -*/ -#if TCL_UTF_MAX != 3 && !(defined(Py_UNICODE_WIDE) && TCL_UTF_MAX==6) -#error "unsupported Tcl configuration" -#endif - #if !(defined(MS_WINDOWS) || defined(__CYGWIN__)) #define HAVE_CREATEFILEHANDLER #endif @@ -975,38 +963,44 @@ AsObj(PyObject *value) return result; } else if (PyUnicode_Check(value)) { - Py_UNICODE *inbuf = PyUnicode_AS_UNICODE(value); - Py_ssize_t size = PyUnicode_GET_SIZE(value); - /* This #ifdef assumes that Tcl uses UCS-2. - See TCL_UTF_MAX test above. */ -#if defined(Py_UNICODE_WIDE) && TCL_UTF_MAX == 3 + void *inbuf; + Py_ssize_t size; + int kind; Tcl_UniChar *outbuf = NULL; Py_ssize_t i; - size_t allocsize = ((size_t)size) * sizeof(Tcl_UniChar); - if (allocsize >= size) - outbuf = (Tcl_UniChar*)ckalloc(allocsize); + size_t allocsize; + + if (PyUnicode_READY(value) == -1) + return NULL; + + inbuf = PyUnicode_DATA(value); + size = PyUnicode_GET_LENGTH(value); + kind = PyUnicode_KIND(value); + allocsize = ((size_t)size) * sizeof(Tcl_UniChar); + outbuf = (Tcl_UniChar*)ckalloc(allocsize); /* Else overflow occurred, and we take the next exit */ if (!outbuf) { PyErr_NoMemory(); return NULL; } for (i = 0; i < size; i++) { - if (inbuf[i] >= 0x10000) { + Py_UCS4 ch = PyUnicode_READ(kind, inbuf, i); + /* We cannot test for sizeof(Tcl_UniChar) directly, + so we test for UTF-8 size instead. */ +#if TCL_UTF_MAX == 3 + if (ch >= 0x10000) { /* Tcl doesn't do UTF-16, yet. */ PyErr_SetString(PyExc_ValueError, "unsupported character"); ckfree(FREECAST outbuf); return NULL; +#endif } - outbuf[i] = inbuf[i]; + outbuf[i] = ch; } result = Tcl_NewUnicodeObj(outbuf, size); ckfree(FREECAST outbuf); return result; -#else - return Tcl_NewUnicodeObj(inbuf, size); -#endif - } else if(PyTclObject_Check(value)) { Tcl_Obj *v = ((PyTclObject*)value)->value; @@ -1088,24 +1082,14 @@ FromObj(PyObject* tkapp, Tcl_Obj *value) } if (value->typePtr == app->StringType) { -#if defined(Py_UNICODE_WIDE) && TCL_UTF_MAX==3 - PyObject *result; - int size; - Tcl_UniChar *input; - Py_UNICODE *output; - - size = Tcl_GetCharLength(value); - result = PyUnicode_FromUnicode(NULL, size); - if (!result) - return NULL; - input = Tcl_GetUnicode(value); - output = PyUnicode_AS_UNICODE(result); - while (size--) - *output++ = *input++; - return result; +#if TCL_UTF_MAX==3 + return PyUnicode_FromKindAndData( + PyUnicode_2BYTE_KIND, Tcl_GetUnicode(value), + Tcl_GetCharLength(value)); #else - return PyUnicode_FromUnicode(Tcl_GetUnicode(value), - Tcl_GetCharLength(value)); + return PyUnicode_FromKindAndData( + PyUnicode_4BYTE_KIND, Tcl_GetUnicode(value), + Tcl_GetCharLength(value)); #endif } @@ -2005,7 +1989,7 @@ static int PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, char *argv[]) { PythonCmd_ClientData *data = (PythonCmd_ClientData *)clientData; - PyObject *self, *func, *arg, *res; + PyObject *func, *arg, *res; int i, rv; Tcl_Obj *obj_res; @@ -2014,7 +1998,6 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, char *argv[]) /* TBD: no error checking here since we know, via the * Tkapp_CreateCommand() that the client data is a two-tuple */ - self = data->self; func = data->func; /* Create argument list (argv1, ..., argvN) */ @@ -2413,11 +2396,9 @@ static PyObject * Tktt_Repr(PyObject *self) { TkttObject *v = (TkttObject *)self; - char buf[100]; - - PyOS_snprintf(buf, sizeof(buf), "<tktimertoken at %p%s>", v, - v->func == NULL ? ", handler deleted" : ""); - return PyUnicode_FromString(buf); + return PyUnicode_FromFormat("<tktimertoken at %p%s>", + v, + v->func == NULL ? ", handler deleted" : ""); } static PyTypeObject Tktt_Type = diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index fe6106c..cb86de7 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -22,7 +22,7 @@ struct arrayobject; /* Forward */ * functions aren't visible yet. */ struct arraydescr { - Py_UNICODE typecode; + char typecode; int itemsize; PyObject * (*getitem)(struct arrayobject *, Py_ssize_t); int (*setitem)(struct arrayobject *, Py_ssize_t, PyObject *); @@ -174,24 +174,25 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) static PyObject * u_getitem(arrayobject *ap, Py_ssize_t i) { - return PyUnicode_FromUnicode(&((Py_UNICODE *) ap->ob_item)[i], 1); + return PyUnicode_FromOrdinal(((Py_UCS4 *) ap->ob_item)[i]); } static int u_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) { - Py_UNICODE *p; - Py_ssize_t len; + PyObject *p; - if (!PyArg_Parse(v, "u#;array item must be unicode character", &p, &len)) + if (!PyArg_Parse(v, "U;array item must be unicode character", &p)) + return -1; + if (PyUnicode_READY(p)) return -1; - if (len != 1) { + if (PyUnicode_GET_LENGTH(p) != 1) { PyErr_SetString(PyExc_TypeError, "array item must be unicode character"); return -1; } if (i >= 0) - ((Py_UNICODE *)ap->ob_item)[i] = p[0]; + ((Py_UCS4 *)ap->ob_item)[i] = PyUnicode_READ_CHAR(p, 0); return 0; } @@ -356,6 +357,59 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } +#ifdef HAVE_LONG_LONG + +static PyObject * +q_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyLong_FromLongLong(((PY_LONG_LONG *)ap->ob_item)[i]); +} + +static int +q_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + PY_LONG_LONG x; + if (!PyArg_Parse(v, "L;array item must be integer", &x)) + return -1; + if (i >= 0) + ((PY_LONG_LONG *)ap->ob_item)[i] = x; + return 0; +} + +static PyObject * +QQ_getitem(arrayobject *ap, Py_ssize_t i) +{ + return PyLong_FromUnsignedLongLong( + ((unsigned PY_LONG_LONG *)ap->ob_item)[i]); +} + +static int +QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) +{ + unsigned PY_LONG_LONG x; + if (PyLong_Check(v)) { + x = PyLong_AsUnsignedLongLong(v); + if (x == (unsigned PY_LONG_LONG) -1 && PyErr_Occurred()) + return -1; + } + else { + PY_LONG_LONG y; + if (!PyArg_Parse(v, "L;array item must be integer", &y)) + return -1; + if (y < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned long long is less than minimum"); + return -1; + } + x = (unsigned PY_LONG_LONG)y; + } + + if (i >= 0) + ((unsigned PY_LONG_LONG *)ap->ob_item)[i] = x; + return 0; +} +#endif + static PyObject * f_getitem(arrayobject *ap, Py_ssize_t i) { @@ -390,6 +444,13 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) return 0; } +#if SIZEOF_INT == 4 +# define STRUCT_LONG_FORMAT "I" +#elif SIZEOF_LONG == 4 +# define STRUCT_LONG_FORMAT "L" +#else +# error "Unable to get struct format for Py_UCS4" +#endif /* Description of types. * @@ -399,13 +460,17 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) static struct arraydescr descriptors[] = { {'b', 1, b_getitem, b_setitem, "b", 1, 1}, {'B', 1, BB_getitem, BB_setitem, "B", 1, 0}, - {'u', sizeof(Py_UNICODE), u_getitem, u_setitem, "u", 0, 0}, + {'u', sizeof(Py_UCS4), u_getitem, u_setitem, STRUCT_LONG_FORMAT, 0, 0}, {'h', sizeof(short), h_getitem, h_setitem, "h", 1, 1}, {'H', sizeof(short), HH_getitem, HH_setitem, "H", 1, 0}, {'i', sizeof(int), i_getitem, i_setitem, "i", 1, 1}, {'I', sizeof(int), II_getitem, II_setitem, "I", 1, 0}, {'l', sizeof(long), l_getitem, l_setitem, "l", 1, 1}, {'L', sizeof(long), LL_getitem, LL_setitem, "L", 1, 0}, +#ifdef HAVE_LONG_LONG + {'q', sizeof(PY_LONG_LONG), q_getitem, q_setitem, "q", 1, 1}, + {'Q', sizeof(PY_LONG_LONG), QQ_getitem, QQ_setitem, "Q", 1, 0}, +#endif {'f', sizeof(float), f_getitem, f_setitem, "f", 0, 0}, {'d', sizeof(double), d_getitem, d_setitem, "d", 0, 0}, {'\0', 0, 0, 0, 0, 0, 0} /* Sentinel */ @@ -514,10 +579,8 @@ array_richcompare(PyObject *v, PyObject *w, int op) Py_ssize_t i, k; PyObject *res; - if (!array_Check(v) || !array_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!array_Check(v) || !array_Check(w)) + Py_RETURN_NOTIMPLEMENTED; va = (arrayobject *)v; wa = (arrayobject *)w; @@ -876,7 +939,6 @@ array_inplace_repeat(arrayobject *self, Py_ssize_t n) if (Py_SIZE(self) > 0) { if (n < 0) n = 0; - items = self->ob_item; if ((self->ob_descr->itemsize != 0) && (Py_SIZE(self) > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { return PyErr_NoMemory(); @@ -1191,6 +1253,7 @@ array_fromfile(arrayobject *self, PyObject *args) PyObject *f, *b, *res; Py_ssize_t itemsize = self->ob_descr->itemsize; Py_ssize_t n, nbytes; + _Py_identifier(read); int not_enough_bytes; if (!PyArg_ParseTuple(args, "On:fromfile", &f, &n)) @@ -1202,7 +1265,7 @@ array_fromfile(arrayobject *self, PyObject *args) return NULL; } - b = PyObject_CallMethod(f, "read", "n", nbytes); + b = _PyObject_CallMethodId(f, &PyId_read, "n", nbytes); if (b == NULL) return NULL; @@ -1259,12 +1322,14 @@ array_tofile(arrayobject *self, PyObject *f) char* ptr = self->ob_item + i*BLOCKSIZE; Py_ssize_t size = BLOCKSIZE; PyObject *bytes, *res; + _Py_identifier(write); + if (i*BLOCKSIZE + size > nbytes) size = nbytes - i*BLOCKSIZE; bytes = PyBytes_FromStringAndSize(ptr, size); if (bytes == NULL) return NULL; - res = PyObject_CallMethod(f, "write", "O", bytes); + res = _PyObject_CallMethodId(f, &PyId_write, "O", bytes); Py_DECREF(bytes); if (res == NULL) return NULL; @@ -1454,25 +1519,26 @@ This method is deprecated. Use tobytes instead."); static PyObject * array_fromunicode(arrayobject *self, PyObject *args) { - Py_UNICODE *ustr; + PyObject *ustr; Py_ssize_t n; - Py_UNICODE typecode; - if (!PyArg_ParseTuple(args, "u#:fromunicode", &ustr, &n)) + if (!PyArg_ParseTuple(args, "U:fromunicode", &ustr)) return NULL; - typecode = self->ob_descr->typecode; - if ((typecode != 'u')) { + if (self->ob_descr->typecode != 'u') { PyErr_SetString(PyExc_ValueError, "fromunicode() may only be called on " "unicode type arrays"); return NULL; } + if (PyUnicode_READY(ustr)) + return NULL; + n = PyUnicode_GET_LENGTH(ustr); if (n > 0) { Py_ssize_t old_size = Py_SIZE(self); if (array_resize(self, old_size + n) == -1) return NULL; - memcpy(self->ob_item + old_size * sizeof(Py_UNICODE), - ustr, n * sizeof(Py_UNICODE)); + if (!PyUnicode_AsUCS4(ustr, (Py_UCS4 *)self->ob_item + old_size, n, 0)) + return NULL; } Py_INCREF(Py_None); @@ -1491,14 +1557,14 @@ append Unicode data to an array of some other type."); static PyObject * array_tounicode(arrayobject *self, PyObject *unused) { - Py_UNICODE typecode; - typecode = self->ob_descr->typecode; - if ((typecode != 'u')) { + if (self->ob_descr->typecode != 'u') { PyErr_SetString(PyExc_ValueError, "tounicode() may only be called on unicode type arrays"); return NULL; } - return PyUnicode_FromUnicode((Py_UNICODE *) self->ob_item, Py_SIZE(self)); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + (Py_UCS4 *) self->ob_item, + Py_SIZE(self)); } PyDoc_STRVAR(tounicode_doc, @@ -1588,7 +1654,7 @@ static const struct mformatdescr { * be found. */ static enum machine_format_code -typecode_to_mformat_code(int typecode) +typecode_to_mformat_code(char typecode) { #ifdef WORDS_BIGENDIAN const int is_big_endian = 1; @@ -1605,13 +1671,7 @@ typecode_to_mformat_code(int typecode) return UNSIGNED_INT8; case 'u': - if (sizeof(Py_UNICODE) == 2) { - return UTF16_LE + is_big_endian; - } - if (sizeof(Py_UNICODE) == 4) { - return UTF32_LE + is_big_endian; - } - return UNKNOWN_FORMAT; + return UTF32_LE + is_big_endian; case 'f': if (sizeof(float) == 4) { @@ -1658,6 +1718,16 @@ typecode_to_mformat_code(int typecode) intsize = sizeof(long); is_signed = 0; break; +#if HAVE_LONG_LONG + case 'q': + intsize = sizeof(PY_LONG_LONG); + is_signed = 1; + break; + case 'Q': + intsize = sizeof(PY_LONG_LONG); + is_signed = 0; + break; +#endif default: return UNKNOWN_FORMAT; } @@ -1688,7 +1758,7 @@ static PyObject *array_new(PyTypeObject *type, PyObject *args, PyObject *kwds); * NULL is returned to indicate a failure. */ static PyObject * -make_array(PyTypeObject *arraytype, Py_UNICODE typecode, PyObject *items) +make_array(PyTypeObject *arraytype, char typecode, PyObject *items) { PyObject *new_args; PyObject *array_obj; @@ -1697,7 +1767,7 @@ make_array(PyTypeObject *arraytype, Py_UNICODE typecode, PyObject *items) assert(arraytype != NULL); assert(items != NULL); - typecode_obj = PyUnicode_FromUnicode(&typecode, 1); + typecode_obj = PyUnicode_FromOrdinal(typecode); if (typecode_obj == NULL) return NULL; @@ -1727,17 +1797,14 @@ array_reconstructor(PyObject *self, PyObject *args) PyObject *items; PyObject *converted_items; PyObject *result; - int typecode_int; - Py_UNICODE typecode; + int typecode; enum machine_format_code mformat_code; struct arraydescr *descr; if (!PyArg_ParseTuple(args, "OCiO:array._array_reconstructor", - &arraytype, &typecode_int, &mformat_code, &items)) + &arraytype, &typecode, &mformat_code, &items)) return NULL; - typecode = (Py_UNICODE)typecode_int; - if (!PyType_Check(arraytype)) { PyErr_Format(PyExc_TypeError, "first argument must a type object, not %.200s", @@ -1751,7 +1818,7 @@ array_reconstructor(PyObject *self, PyObject *args) return NULL; } for (descr = descriptors; descr->typecode != '\0'; descr++) { - if (descr->typecode == typecode) + if ((int)descr->typecode == typecode) break; } if (descr->typecode == '\0') { @@ -1773,9 +1840,9 @@ array_reconstructor(PyObject *self, PyObject *args) } /* Fast path: No decoding has to be done. */ - if (mformat_code == typecode_to_mformat_code(typecode) || + if (mformat_code == typecode_to_mformat_code((char)typecode) || mformat_code == UNKNOWN_FORMAT) { - return make_array(arraytype, typecode, items); + return make_array(arraytype, (char)typecode, items); } /* Slow path: Decode the byte string according to the given machine @@ -1921,7 +1988,7 @@ array_reconstructor(PyObject *self, PyObject *args) return NULL; } - result = make_array(arraytype, typecode, converted_items); + result = make_array(arraytype, (char)typecode, converted_items); Py_DECREF(converted_items); return result; } @@ -1936,14 +2003,16 @@ array_reduce_ex(arrayobject *array, PyObject *value) int mformat_code; static PyObject *array_reconstructor = NULL; long protocol; + _Py_identifier(_array_reconstructor); + _Py_identifier(__dict__); if (array_reconstructor == NULL) { PyObject *array_module = PyImport_ImportModule("array"); if (array_module == NULL) return NULL; - array_reconstructor = PyObject_GetAttrString( + array_reconstructor = _PyObject_GetAttrId( array_module, - "_array_reconstructor"); + &PyId__array_reconstructor); Py_DECREF(array_module); if (array_reconstructor == NULL) return NULL; @@ -1958,7 +2027,7 @@ array_reduce_ex(arrayobject *array, PyObject *value) if (protocol == -1 && PyErr_Occurred()) return NULL; - dict = PyObject_GetAttrString((PyObject *)array, "__dict__"); + dict = _PyObject_GetAttrId((PyObject *)array, &PyId___dict__); if (dict == NULL) { if (!PyErr_ExceptionMatches(PyExc_AttributeError)) return NULL; @@ -2010,8 +2079,8 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyObject * array_get_typecode(arrayobject *a, void *closure) { - Py_UNICODE tc = a->ob_descr->typecode; - return PyUnicode_FromUnicode(&tc, 1); + char typecode = a->ob_descr->typecode; + return PyUnicode_FromOrdinal(typecode); } static PyObject * @@ -2083,7 +2152,7 @@ static PyMethodDef array_methods[] = { static PyObject * array_repr(arrayobject *a) { - Py_UNICODE typecode; + char typecode; PyObject *s, *v = NULL; Py_ssize_t len; @@ -2092,7 +2161,7 @@ array_repr(arrayobject *a) if (len == 0) { return PyUnicode_FromFormat("array('%c')", (int)typecode); } - if ((typecode == 'u')) + if (typecode == 'u') v = array_tounicode(a, NULL); else v = array_tolist(a, NULL); @@ -2287,7 +2356,7 @@ array_ass_subscr(arrayobject* self, PyObject* item, PyObject* value) self->ob_item + (cur + 1) * itemsize, lim * itemsize); } - cur = start + slicelength * step; + cur = start + (size_t)slicelength * step; if (cur < (size_t)Py_SIZE(self)) { memmove(self->ob_item + (cur-slicelength) * itemsize, self->ob_item + cur * itemsize, @@ -2350,14 +2419,8 @@ array_buffer_getbuf(arrayobject *self, Py_buffer *view, int flags) view->strides = &(view->itemsize); view->format = NULL; view->internal = NULL; - if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) { + if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) view->format = self->ob_descr->formats; -#ifdef Py_UNICODE_WIDE - if (self->ob_descr->typecode == 'u') { - view->format = "w"; - } -#endif - } finish: self->ob_exports++; @@ -2471,19 +2534,25 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(v); } else if (initial != NULL && PyUnicode_Check(initial)) { - Py_ssize_t n = PyUnicode_GET_DATA_SIZE(initial); + Py_ssize_t n; + if (PyUnicode_READY(initial)) { + Py_DECREF(a); + return NULL; + } + n = PyUnicode_GET_LENGTH(initial); if (n > 0) { arrayobject *self = (arrayobject *)a; - char *item = self->ob_item; - item = (char *)PyMem_Realloc(item, n); + Py_UCS4 *item = (Py_UCS4 *)self->ob_item; + item = (Py_UCS4 *)PyMem_Realloc(item, n * sizeof(Py_UCS4)); if (item == NULL) { PyErr_NoMemory(); Py_DECREF(a); return NULL; } - self->ob_item = item; - Py_SIZE(self) = n / sizeof(Py_UNICODE); - memcpy(item, PyUnicode_AS_DATA(initial), n); + self->ob_item = (char*)item; + Py_SIZE(self) = n; + if (!PyUnicode_AsUCS4(initial, item, n, 0)) + return NULL; self->allocated = Py_SIZE(self); } } @@ -2504,7 +2573,11 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } } PyErr_SetString(PyExc_ValueError, +#ifdef HAVE_LONG_LONG + "bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)"); +#else "bad typecode (must be b, B, u, h, H, i, I, l, L, f or d)"); +#endif return NULL; } @@ -2527,12 +2600,18 @@ is a single character. The following type codes are defined:\n\ 'I' unsigned integer 2 \n\ 'l' signed integer 4 \n\ 'L' unsigned integer 4 \n\ + 'q' signed integer 8 (see note) \n\ + 'Q' unsigned integer 8 (see note) \n\ 'f' floating point 4 \n\ 'd' floating point 8 \n\ \n\ -NOTE: The 'u' typecode corresponds to Python's unicode character. On \n\ +NOTE: The 'u' type code corresponds to Python's unicode character. On \n\ narrow builds this is 2-bytes on wide builds this is 4-bytes.\n\ \n\ +NOTE: The 'q' and 'Q' type codes are only available if the platform \n\ +C compiler used to build Python supports 'long long', or, on Windows, \n\ +'__int64'.\n\ +\n\ The constructor is:\n\ \n\ array(typecode [, initializer]) -- create a new array\n\ @@ -2736,9 +2815,9 @@ PyMODINIT_FUNC PyInit_array(void) { PyObject *m; + char buffer[Py_ARRAY_LENGTH(descriptors)], *p; PyObject *typecodes; Py_ssize_t size = 0; - register Py_UNICODE *p; struct arraydescr *descr; if (PyType_Ready(&Arraytype) < 0) @@ -2757,13 +2836,13 @@ PyInit_array(void) size++; } - typecodes = PyUnicode_FromStringAndSize(NULL, size); - p = PyUnicode_AS_UNICODE(typecodes); + p = buffer; for (descr = descriptors; descr->typecode != '\0'; descr++) { *p++ = (char)descr->typecode; } + typecodes = PyUnicode_DecodeASCII(buffer, p - buffer, NULL); - PyModule_AddObject(m, "typecodes", (PyObject *)typecodes); + PyModule_AddObject(m, "typecodes", typecodes); if (PyErr_Occurred()) { Py_DECREF(m); diff --git a/Modules/audioop.c b/Modules/audioop.c index a031d42..daf70dc 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -513,7 +513,6 @@ audioop_findfit(PyObject *self, PyObject *args) best_result = result; best_j = 0; - j = 0; for ( j=1; j<=len1-len2; j++) { aj_m1 = (double)cp1[j-1]; @@ -599,7 +598,6 @@ audioop_findmax(PyObject *self, PyObject *args) best_result = result; best_j = 0; - j = 0; for ( j=1; j<=len1-len2; j++) { aj_m1 = (double)cp1[j-1]; @@ -1433,7 +1431,6 @@ audioop_lin2adpcm(PyObject *self, PyObject *args) if ( state == Py_None ) { /* First time, it seems. Set defaults */ valpred = 0; - step = 7; index = 0; } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) ) return 0; @@ -1534,7 +1531,6 @@ audioop_adpcm2lin(PyObject *self, PyObject *args) if ( state == Py_None ) { /* First time, it seems. Set defaults */ valpred = 0; - step = 7; index = 0; } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) ) return 0; diff --git a/Modules/binascii.c b/Modules/binascii.c index 19681b4..dc4fef5 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -1078,13 +1078,11 @@ binascii_hexlify(PyObject *self, PyObject *args) /* make hex version of string, taken from shamodule.c */ for (i=j=0; i < arglen; i++) { - char c; + unsigned char c; c = (argbuf[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - retbuf[j++] = c; + retbuf[j++] = Py_hexdigits[c]; c = argbuf[i] & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - retbuf[j++] = c; + retbuf[j++] = Py_hexdigits[c]; } PyBuffer_Release(&parg); return retval; diff --git a/Modules/bz2module.c b/Modules/bz2module.c deleted file mode 100644 index a671e8d..0000000 --- a/Modules/bz2module.c +++ /dev/null @@ -1,2169 +0,0 @@ -/* - -python-bz2 - python bz2 library interface - -Copyright (c) 2002 Gustavo Niemeyer <niemeyer@conectiva.com> -Copyright (c) 2002 Python Software Foundation; All Rights Reserved - -*/ - -#include "Python.h" -#include <stdio.h> -#include <bzlib.h> -#include "structmember.h" - -#ifdef WITH_THREAD -#include "pythread.h" -#endif - -static char __author__[] = -"The bz2 python module was written by:\n\ -\n\ - Gustavo Niemeyer <niemeyer@conectiva.com>\n\ -"; - -/* Our very own off_t-like type, 64-bit if possible */ -/* copied from Objects/fileobject.c */ -#if !defined(HAVE_LARGEFILE_SUPPORT) -typedef off_t Py_off_t; -#elif SIZEOF_OFF_T >= 8 -typedef off_t Py_off_t; -#elif SIZEOF_FPOS_T >= 8 -typedef fpos_t Py_off_t; -#else -#error "Large file support, but neither off_t nor fpos_t is large enough." -#endif - -#define BUF(v) PyBytes_AS_STRING(v) - -#define MODE_CLOSED 0 -#define MODE_READ 1 -#define MODE_READ_EOF 2 -#define MODE_WRITE 3 - -#define BZ2FileObject_Check(v) (Py_TYPE(v) == &BZ2File_Type) - - -#ifdef BZ_CONFIG_ERROR - -#if SIZEOF_LONG >= 8 -#define BZS_TOTAL_OUT(bzs) \ - (((long)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) -#elif SIZEOF_LONG_LONG >= 8 -#define BZS_TOTAL_OUT(bzs) \ - (((PY_LONG_LONG)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) -#else -#define BZS_TOTAL_OUT(bzs) \ - bzs->total_out_lo32 -#endif - -#else /* ! BZ_CONFIG_ERROR */ - -#define BZ2_bzRead bzRead -#define BZ2_bzReadOpen bzReadOpen -#define BZ2_bzReadClose bzReadClose -#define BZ2_bzWrite bzWrite -#define BZ2_bzWriteOpen bzWriteOpen -#define BZ2_bzWriteClose bzWriteClose -#define BZ2_bzCompress bzCompress -#define BZ2_bzCompressInit bzCompressInit -#define BZ2_bzCompressEnd bzCompressEnd -#define BZ2_bzDecompress bzDecompress -#define BZ2_bzDecompressInit bzDecompressInit -#define BZ2_bzDecompressEnd bzDecompressEnd - -#define BZS_TOTAL_OUT(bzs) bzs->total_out - -#endif /* ! BZ_CONFIG_ERROR */ - - -#ifdef WITH_THREAD -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock(obj->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock(obj->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while(0) -#define RELEASE_LOCK(obj) PyThread_release_lock(obj->lock) -#else -#define ACQUIRE_LOCK(obj) -#define RELEASE_LOCK(obj) -#endif - -/* Bits in f_newlinetypes */ -#define NEWLINE_UNKNOWN 0 /* No newline seen, yet */ -#define NEWLINE_CR 1 /* \r newline seen */ -#define NEWLINE_LF 2 /* \n newline seen */ -#define NEWLINE_CRLF 4 /* \r\n newline seen */ - -/* ===================================================================== */ -/* Structure definitions. */ - -typedef struct { - PyObject_HEAD - FILE *rawfp; - - char* f_buf; /* Allocated readahead buffer */ - char* f_bufend; /* Points after last occupied position */ - char* f_bufptr; /* Current buffer position */ - - BZFILE *fp; - int mode; - Py_off_t pos; - Py_off_t size; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2FileObject; - -typedef struct { - PyObject_HEAD - bz_stream bzs; - int running; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2CompObject; - -typedef struct { - PyObject_HEAD - bz_stream bzs; - int running; - PyObject *unused_data; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2DecompObject; - -/* ===================================================================== */ -/* Utility functions. */ - -/* Refuse regular I/O if there's data in the iteration-buffer. - * Mixing them would cause data to arrive out of order, as the read* - * methods don't use the iteration buffer. */ -static int -check_iterbuffered(BZ2FileObject *f) -{ - if (f->f_buf != NULL && - (f->f_bufend - f->f_bufptr) > 0 && - f->f_buf[0] != '\0') { - PyErr_SetString(PyExc_ValueError, - "Mixing iteration and read methods would lose data"); - return -1; - } - return 0; -} - -static int -Util_CatchBZ2Error(int bzerror) -{ - int ret = 0; - switch(bzerror) { - case BZ_OK: - case BZ_STREAM_END: - break; - -#ifdef BZ_CONFIG_ERROR - case BZ_CONFIG_ERROR: - PyErr_SetString(PyExc_SystemError, - "the bz2 library was not compiled " - "correctly"); - ret = 1; - break; -#endif - - case BZ_PARAM_ERROR: - PyErr_SetString(PyExc_ValueError, - "the bz2 library has received wrong " - "parameters"); - ret = 1; - break; - - case BZ_MEM_ERROR: - PyErr_NoMemory(); - ret = 1; - break; - - case BZ_DATA_ERROR: - case BZ_DATA_ERROR_MAGIC: - PyErr_SetString(PyExc_IOError, "invalid data stream"); - ret = 1; - break; - - case BZ_IO_ERROR: - PyErr_SetString(PyExc_IOError, "unknown IO error"); - ret = 1; - break; - - case BZ_UNEXPECTED_EOF: - PyErr_SetString(PyExc_EOFError, - "compressed file ended before the " - "logical end-of-stream was detected"); - ret = 1; - break; - - case BZ_SEQUENCE_ERROR: - PyErr_SetString(PyExc_RuntimeError, - "wrong sequence of bz2 library " - "commands used"); - ret = 1; - break; - } - return ret; -} - -#if BUFSIZ < 8192 -#define SMALLCHUNK 8192 -#else -#define SMALLCHUNK BUFSIZ -#endif - -/* This is a hacked version of Python's fileobject.c:new_buffersize(). */ -static size_t -Util_NewBufferSize(size_t currentsize) -{ - /* Expand the buffer by an amount proportional to the current size, - giving us amortized linear-time behavior. Use a less-than-double - growth factor to avoid excessive allocation. */ - return currentsize + (currentsize >> 3) + 6; -} - -/* This is a hacked version of Python's fileobject.c:get_line(). */ -static PyObject * -Util_GetLine(BZ2FileObject *f, int n) -{ - char c; - char *buf, *end; - size_t total_v_size; /* total # of slots in buffer */ - size_t used_v_size; /* # used slots in buffer */ - size_t increment; /* amount to increment the buffer */ - PyObject *v; - int bzerror; - int bytes_read; - - total_v_size = n > 0 ? n : 100; - v = PyBytes_FromStringAndSize((char *)NULL, total_v_size); - if (v == NULL) - return NULL; - - buf = BUF(v); - end = buf + total_v_size; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - do { - bytes_read = BZ2_bzRead(&bzerror, f->fp, &c, 1); - f->pos++; - if (bytes_read == 0) - break; - *buf++ = c; - } while (bzerror == BZ_OK && c != '\n' && buf != end); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - f->size = f->pos; - f->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(v); - return NULL; - } - if (c == '\n') - break; - /* Must be because buf == end */ - if (n > 0) - break; - used_v_size = total_v_size; - increment = total_v_size >> 2; /* mild exponential growth */ - total_v_size += increment; - if (total_v_size > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "line is longer than a Python string can hold"); - Py_DECREF(v); - return NULL; - } - if (_PyBytes_Resize(&v, total_v_size) < 0) { - return NULL; - } - buf = BUF(v) + used_v_size; - end = BUF(v) + total_v_size; - } - - used_v_size = buf - BUF(v); - if (used_v_size != total_v_size) { - if (_PyBytes_Resize(&v, used_v_size) < 0) { - v = NULL; - } - } - return v; -} - -/* This is a hacked version of Python's fileobject.c:drop_readahead(). */ -static void -Util_DropReadAhead(BZ2FileObject *f) -{ - if (f->f_buf != NULL) { - PyMem_Free(f->f_buf); - f->f_buf = NULL; - } -} - -/* This is a hacked version of Python's fileobject.c:readahead(). */ -static int -Util_ReadAhead(BZ2FileObject *f, int bufsize) -{ - int chunksize; - int bzerror; - - if (f->f_buf != NULL) { - if((f->f_bufend - f->f_bufptr) >= 1) - return 0; - else - Util_DropReadAhead(f); - } - if (f->mode == MODE_READ_EOF) { - f->f_bufptr = f->f_buf; - f->f_bufend = f->f_buf; - return 0; - } - if ((f->f_buf = PyMem_Malloc(bufsize)) == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, f->fp, f->f_buf, bufsize); - Py_END_ALLOW_THREADS - f->pos += chunksize; - if (bzerror == BZ_STREAM_END) { - f->size = f->pos; - f->mode = MODE_READ_EOF; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Util_DropReadAhead(f); - return -1; - } - f->f_bufptr = f->f_buf; - f->f_bufend = f->f_buf + chunksize; - return 0; -} - -/* This is a hacked version of Python's - * fileobject.c:readahead_get_line_skip(). */ -static PyBytesObject * -Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize) -{ - PyBytesObject* s; - char *bufptr; - char *buf; - int len; - - if (f->f_buf == NULL) - if (Util_ReadAhead(f, bufsize) < 0) - return NULL; - - len = f->f_bufend - f->f_bufptr; - if (len == 0) - return (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, skip); - bufptr = memchr(f->f_bufptr, '\n', len); - if (bufptr != NULL) { - bufptr++; /* Count the '\n' */ - len = bufptr - f->f_bufptr; - s = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, skip+len); - if (s == NULL) - return NULL; - memcpy(PyBytes_AS_STRING(s)+skip, f->f_bufptr, len); - f->f_bufptr = bufptr; - if (bufptr == f->f_bufend) - Util_DropReadAhead(f); - } else { - bufptr = f->f_bufptr; - buf = f->f_buf; - f->f_buf = NULL; /* Force new readahead buffer */ - s = Util_ReadAheadGetLineSkip(f, skip+len, - bufsize + (bufsize>>2)); - if (s == NULL) { - PyMem_Free(buf); - return NULL; - } - memcpy(PyBytes_AS_STRING(s)+skip, bufptr, len); - PyMem_Free(buf); - } - return s; -} - -/* ===================================================================== */ -/* Methods of BZ2File. */ - -PyDoc_STRVAR(BZ2File_read__doc__, -"read([size]) -> string\n\ -\n\ -Read at most size uncompressed bytes, returned as a string. If the size\n\ -argument is negative or omitted, read until EOF is reached.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_read(). */ -static PyObject * -BZ2File_read(BZ2FileObject *self, PyObject *args) -{ - long bytesrequested = -1; - size_t bytesread, buffersize, chunksize; - int bzerror; - PyObject *ret = NULL; - - if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - ret = PyBytes_FromStringAndSize("", 0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if (bytesrequested < 0) - buffersize = Util_NewBufferSize((size_t)0); - else - buffersize = bytesrequested; - if (buffersize > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "requested number of bytes is " - "more than a Python string can hold"); - goto cleanup; - } - ret = PyBytes_FromStringAndSize((char *)NULL, buffersize); - if (ret == NULL || buffersize == 0) - goto cleanup; - bytesread = 0; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, - BUF(ret)+bytesread, - buffersize-bytesread); - self->pos += chunksize; - Py_END_ALLOW_THREADS - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(ret); - ret = NULL; - goto cleanup; - } - if (bytesrequested < 0) { - buffersize = Util_NewBufferSize(buffersize); - if (_PyBytes_Resize(&ret, buffersize) < 0) { - ret = NULL; - goto cleanup; - } - } else { - break; - } - } - if (bytesread != buffersize) { - if (_PyBytes_Resize(&ret, bytesread) < 0) { - ret = NULL; - } - } - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_readline__doc__, -"readline([size]) -> string\n\ -\n\ -Return the next line from the file, as a string, retaining newline.\n\ -A non-negative size argument will limit the maximum number of bytes to\n\ -return (an incomplete line may be returned then). Return an empty\n\ -string at EOF.\n\ -"); - -static PyObject * -BZ2File_readline(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - int sizehint = -1; - - if (!PyArg_ParseTuple(args, "|i:readline", &sizehint)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - ret = PyBytes_FromStringAndSize("", 0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if (sizehint == 0) - ret = PyBytes_FromStringAndSize("", 0); - else - ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint); - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_readlines__doc__, -"readlines([size]) -> list\n\ -\n\ -Call readline() repeatedly and return a list of lines read.\n\ -The optional size argument, if given, is an approximate bound on the\n\ -total number of bytes in the lines returned.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_readlines(). */ -static PyObject * -BZ2File_readlines(BZ2FileObject *self, PyObject *args) -{ - long sizehint = 0; - PyObject *list = NULL; - PyObject *line; - char small_buffer[SMALLCHUNK]; - char *buffer = small_buffer; - size_t buffersize = SMALLCHUNK; - PyObject *big_buffer = NULL; - size_t nfilled = 0; - size_t nread; - size_t totalread = 0; - char *p, *q, *end; - int err; - int shortread = 0; - int bzerror; - - if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - list = PyList_New(0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if ((list = PyList_New(0)) == NULL) - goto cleanup; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - nread = BZ2_bzRead(&bzerror, self->fp, - buffer+nfilled, buffersize-nfilled); - self->pos += nread; - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - if (nread == 0) { - sizehint = 0; - break; - } - shortread = 1; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - error: - Py_DECREF(list); - list = NULL; - goto cleanup; - } - totalread += nread; - p = memchr(buffer+nfilled, '\n', nread); - if (!shortread && p == NULL) { - /* Need a larger buffer to fit this line */ - nfilled += nread; - buffersize *= 2; - if (buffersize > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "line is longer than a Python string can hold"); - goto error; - } - if (big_buffer == NULL) { - /* Create the big buffer */ - big_buffer = PyBytes_FromStringAndSize( - NULL, buffersize); - if (big_buffer == NULL) - goto error; - buffer = PyBytes_AS_STRING(big_buffer); - memcpy(buffer, small_buffer, nfilled); - } - else { - /* Grow the big buffer */ - if (_PyBytes_Resize(&big_buffer, buffersize) < 0){ - big_buffer = NULL; - goto error; - } - buffer = PyBytes_AS_STRING(big_buffer); - } - continue; - } - end = buffer+nfilled+nread; - q = buffer; - while (p != NULL) { - /* Process complete lines */ - p++; - line = PyBytes_FromStringAndSize(q, p-q); - if (line == NULL) - goto error; - err = PyList_Append(list, line); - Py_DECREF(line); - if (err != 0) - goto error; - q = p; - p = memchr(q, '\n', end-q); - } - /* Move the remaining incomplete line to the start */ - nfilled = end-q; - memmove(buffer, q, nfilled); - if (sizehint > 0) - if (totalread >= (size_t)sizehint) - break; - if (shortread) { - sizehint = 0; - break; - } - } - if (nfilled != 0) { - /* Partial last line */ - line = PyBytes_FromStringAndSize(buffer, nfilled); - if (line == NULL) - goto error; - if (sizehint > 0) { - /* Need to complete the last line */ - PyObject *rest = Util_GetLine(self, 0); - if (rest == NULL) { - Py_DECREF(line); - goto error; - } - PyBytes_Concat(&line, rest); - Py_DECREF(rest); - if (line == NULL) - goto error; - } - err = PyList_Append(list, line); - Py_DECREF(line); - if (err != 0) - goto error; - } - - cleanup: - RELEASE_LOCK(self); - if (big_buffer) { - Py_DECREF(big_buffer); - } - return list; -} - -PyDoc_STRVAR(BZ2File_write__doc__, -"write(data) -> None\n\ -\n\ -Write the 'data' string to file. Note that due to buffering, close() may\n\ -be needed before the file on disk reflects the data written.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_write(). */ -static PyObject * -BZ2File_write(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - Py_buffer pbuf; - char *buf; - int len; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:write", &pbuf)) - return NULL; - buf = pbuf.buf; - len = pbuf.len; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_WRITE: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for writing"); - goto cleanup; - } - - Py_BEGIN_ALLOW_THREADS - BZ2_bzWrite (&bzerror, self->fp, buf, len); - self->pos += len; - Py_END_ALLOW_THREADS - - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - - Py_INCREF(Py_None); - ret = Py_None; - -cleanup: - PyBuffer_Release(&pbuf); - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_writelines__doc__, -"writelines(sequence_of_strings) -> None\n\ -\n\ -Write the sequence of strings to the file. Note that newlines are not\n\ -added. The sequence can be any iterable object producing strings. This is\n\ -equivalent to calling write() for each string.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_writelines(). */ -static PyObject * -BZ2File_writelines(BZ2FileObject *self, PyObject *seq) -{ -#define CHUNKSIZE 1000 - PyObject *list = NULL; - PyObject *iter = NULL; - PyObject *ret = NULL; - PyObject *line; - int i, j, index, len, islist; - int bzerror; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_WRITE: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto error; - - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for writing"); - goto error; - } - - islist = PyList_Check(seq); - if (!islist) { - iter = PyObject_GetIter(seq); - if (iter == NULL) { - PyErr_SetString(PyExc_TypeError, - "writelines() requires an iterable argument"); - goto error; - } - list = PyList_New(CHUNKSIZE); - if (list == NULL) - goto error; - } - - /* Strategy: slurp CHUNKSIZE lines into a private list, - checking that they are all strings, then write that list - without holding the interpreter lock, then come back for more. */ - for (index = 0; ; index += CHUNKSIZE) { - if (islist) { - Py_XDECREF(list); - list = PyList_GetSlice(seq, index, index+CHUNKSIZE); - if (list == NULL) - goto error; - j = PyList_GET_SIZE(list); - } - else { - for (j = 0; j < CHUNKSIZE; j++) { - line = PyIter_Next(iter); - if (line == NULL) { - if (PyErr_Occurred()) - goto error; - break; - } - PyList_SetItem(list, j, line); - } - } - if (j == 0) - break; - - /* Check that all entries are indeed byte strings. If not, - apply the same rules as for file.write() and - convert the rets to strings. This is slow, but - seems to be the only way since all conversion APIs - could potentially execute Python code. */ - for (i = 0; i < j; i++) { - PyObject *v = PyList_GET_ITEM(list, i); - if (!PyBytes_Check(v)) { - const char *buffer; - Py_ssize_t len; - if (PyObject_AsCharBuffer(v, &buffer, &len)) { - PyErr_SetString(PyExc_TypeError, - "writelines() " - "argument must be " - "a sequence of " - "bytes objects"); - goto error; - } - line = PyBytes_FromStringAndSize(buffer, - len); - if (line == NULL) - goto error; - Py_DECREF(v); - PyList_SET_ITEM(list, i, line); - } - } - - /* Since we are releasing the global lock, the - following code may *not* execute Python code. */ - Py_BEGIN_ALLOW_THREADS - for (i = 0; i < j; i++) { - line = PyList_GET_ITEM(list, i); - len = PyBytes_GET_SIZE(line); - BZ2_bzWrite (&bzerror, self->fp, - PyBytes_AS_STRING(line), len); - if (bzerror != BZ_OK) { - Py_BLOCK_THREADS - Util_CatchBZ2Error(bzerror); - goto error; - } - } - Py_END_ALLOW_THREADS - - if (j < CHUNKSIZE) - break; - } - - Py_INCREF(Py_None); - ret = Py_None; - - error: - RELEASE_LOCK(self); - Py_XDECREF(list); - Py_XDECREF(iter); - return ret; -#undef CHUNKSIZE -} - -PyDoc_STRVAR(BZ2File_seek__doc__, -"seek(offset [, whence]) -> None\n\ -\n\ -Move to new file position. Argument offset is a byte count. Optional\n\ -argument whence defaults to 0 (offset from start of file, offset\n\ -should be >= 0); other values are 1 (move relative to current position,\n\ -positive or negative), and 2 (move relative to end of file, usually\n\ -negative, although many platforms allow seeking beyond the end of a file).\n\ -\n\ -Note that seeking of bz2 files is emulated, and depending on the parameters\n\ -the operation may be extremely slow.\n\ -"); - -static PyObject * -BZ2File_seek(BZ2FileObject *self, PyObject *args) -{ - int where = 0; - PyObject *offobj; - Py_off_t offset; - char small_buffer[SMALLCHUNK]; - char *buffer = small_buffer; - size_t buffersize = SMALLCHUNK; - Py_off_t bytesread = 0; - size_t readsize; - int chunksize; - int bzerror; - PyObject *ret = NULL; - - if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &where)) - return NULL; -#if !defined(HAVE_LARGEFILE_SUPPORT) - offset = PyLong_AsLong(offobj); -#else - offset = PyLong_Check(offobj) ? - PyLong_AsLongLong(offobj) : PyLong_AsLong(offobj); -#endif - if (PyErr_Occurred()) - return NULL; - - ACQUIRE_LOCK(self); - Util_DropReadAhead(self); - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - - default: - PyErr_SetString(PyExc_IOError, - "seek works only while reading"); - goto cleanup; - } - - if (where == 2) { - if (self->size == -1) { - assert(self->mode != MODE_READ_EOF); - for (;;) { - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, - buffer, buffersize); - self->pos += chunksize; - Py_END_ALLOW_THREADS - - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - } - self->mode = MODE_READ_EOF; - self->size = self->pos; - bytesread = 0; - } - offset = self->size + offset; - } else if (where == 1) { - offset = self->pos + offset; - } - - /* Before getting here, offset must be the absolute position the file - * pointer should be set to. */ - - if (offset >= self->pos) { - /* we can move forward */ - offset -= self->pos; - } else { - /* we cannot move back, so rewind the stream */ - BZ2_bzReadClose(&bzerror, self->fp); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - rewind(self->rawfp); - self->pos = 0; - self->fp = BZ2_bzReadOpen(&bzerror, self->rawfp, - 0, 0, NULL, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - self->mode = MODE_READ; - } - - if (offset <= 0 || self->mode == MODE_READ_EOF) - goto exit; - - /* Before getting here, offset must be set to the number of bytes - * to walk forward. */ - for (;;) { - if (offset-bytesread > buffersize) - readsize = buffersize; - else - /* offset might be wider that readsize, but the result - * of the subtraction is bound by buffersize (see the - * condition above). buffersize is 8192. */ - readsize = (size_t)(offset-bytesread); - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, buffer, readsize); - self->pos += chunksize; - Py_END_ALLOW_THREADS - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - if (bytesread == offset) - break; - } - -exit: - Py_INCREF(Py_None); - ret = Py_None; - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_tell__doc__, -"tell() -> int\n\ -\n\ -Return the current file position, an integer (may be a long integer).\n\ -"); - -static PyObject * -BZ2File_tell(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - } - -#if !defined(HAVE_LARGEFILE_SUPPORT) - ret = PyLong_FromLong(self->pos); -#else - ret = PyLong_FromLongLong(self->pos); -#endif - -cleanup: - return ret; -} - -PyDoc_STRVAR(BZ2File_close__doc__, -"close() -> None or (perhaps) an integer\n\ -\n\ -Close the file. Sets data attribute .closed to true. A closed file\n\ -cannot be used for further I/O operations. close() may be called more\n\ -than once without error.\n\ -"); - -static PyObject * -BZ2File_close(BZ2FileObject *self) -{ - PyObject *ret = NULL; - int bzerror = BZ_OK; - - if (self->mode == MODE_CLOSED) { - Py_RETURN_NONE; - } - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - BZ2_bzReadClose(&bzerror, self->fp); - break; - case MODE_WRITE: - BZ2_bzWriteClose(&bzerror, self->fp, - 0, NULL, NULL); - break; - } - self->mode = MODE_CLOSED; - fclose(self->rawfp); - self->rawfp = NULL; - if (bzerror == BZ_OK) { - Py_INCREF(Py_None); - ret = Py_None; - } - else { - Util_CatchBZ2Error(bzerror); - } - - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_enter_doc, -"__enter__() -> self."); - -static PyObject * -BZ2File_enter(BZ2FileObject *self) -{ - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - Py_INCREF(self); - return (PyObject *) self; -} - -PyDoc_STRVAR(BZ2File_exit_doc, -"__exit__(*excinfo) -> None. Closes the file."); - -static PyObject * -BZ2File_exit(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = PyObject_CallMethod((PyObject *) self, "close", NULL); - if (!ret) - /* If error occurred, pass through */ - return NULL; - Py_DECREF(ret); - Py_RETURN_NONE; -} - - -static PyObject *BZ2File_getiter(BZ2FileObject *self); - -static PyMethodDef BZ2File_methods[] = { - {"read", (PyCFunction)BZ2File_read, METH_VARARGS, BZ2File_read__doc__}, - {"readline", (PyCFunction)BZ2File_readline, METH_VARARGS, BZ2File_readline__doc__}, - {"readlines", (PyCFunction)BZ2File_readlines, METH_VARARGS, BZ2File_readlines__doc__}, - {"write", (PyCFunction)BZ2File_write, METH_VARARGS, BZ2File_write__doc__}, - {"writelines", (PyCFunction)BZ2File_writelines, METH_O, BZ2File_writelines__doc__}, - {"seek", (PyCFunction)BZ2File_seek, METH_VARARGS, BZ2File_seek__doc__}, - {"tell", (PyCFunction)BZ2File_tell, METH_NOARGS, BZ2File_tell__doc__}, - {"close", (PyCFunction)BZ2File_close, METH_NOARGS, BZ2File_close__doc__}, - {"__enter__", (PyCFunction)BZ2File_enter, METH_NOARGS, BZ2File_enter_doc}, - {"__exit__", (PyCFunction)BZ2File_exit, METH_VARARGS, BZ2File_exit_doc}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Getters and setters of BZ2File. */ - -static PyObject * -BZ2File_get_closed(BZ2FileObject *self, void *closure) -{ - return PyLong_FromLong(self->mode == MODE_CLOSED); -} - -static PyGetSetDef BZ2File_getset[] = { - {"closed", (getter)BZ2File_get_closed, NULL, - "True if the file is closed"}, - {NULL} /* Sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2File_Type. */ - -static int -BZ2File_init(BZ2FileObject *self, PyObject *args, PyObject *kwargs) -{ - static char *kwlist[] = {"filename", "mode", "buffering", - "compresslevel", 0}; - PyObject *name_obj = NULL; - char *name; - char *mode = "r"; - int buffering = -1; - int compresslevel = 9; - int bzerror; - int mode_char = 0; - - self->size = -1; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|sii:BZ2File", - kwlist, PyUnicode_FSConverter, &name_obj, - &mode, &buffering, - &compresslevel)) - return -1; - - name = PyBytes_AsString(name_obj); - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - Py_DECREF(name_obj); - return -1; - } - - for (;;) { - int error = 0; - switch (*mode) { - case 'r': - case 'w': - if (mode_char) - error = 1; - mode_char = *mode; - break; - - case 'b': - break; - - default: - error = 1; - break; - } - if (error) { - PyErr_Format(PyExc_ValueError, - "invalid mode char %c", *mode); - Py_DECREF(name_obj); - return -1; - } - mode++; - if (*mode == '\0') - break; - } - - if (mode_char == 0) { - mode_char = 'r'; - } - - mode = (mode_char == 'r') ? "rb" : "wb"; - - self->rawfp = fopen(name, mode); - Py_DECREF(name_obj); - if (self->rawfp == NULL) { - PyErr_SetFromErrno(PyExc_IOError); - return -1; - } - /* XXX Ignore buffering */ - - /* From now on, we have stuff to dealloc, so jump to error label - * instead of returning */ - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - if (mode_char == 'r') - self->fp = BZ2_bzReadOpen(&bzerror, self->rawfp, - 0, 0, NULL, 0); - else - self->fp = BZ2_bzWriteOpen(&bzerror, self->rawfp, - compresslevel, 0, 0); - - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->mode = (mode_char == 'r') ? MODE_READ : MODE_WRITE; - - return 0; - -error: - fclose(self->rawfp); - self->rawfp = NULL; -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - return -1; -} - -static void -BZ2File_dealloc(BZ2FileObject *self) -{ - int bzerror; -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - BZ2_bzReadClose(&bzerror, self->fp); - break; - case MODE_WRITE: - BZ2_bzWriteClose(&bzerror, self->fp, - 0, NULL, NULL); - break; - } - Util_DropReadAhead(self); - if (self->rawfp != NULL) - fclose(self->rawfp); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -/* This is a hacked version of Python's fileobject.c:file_getiter(). */ -static PyObject * -BZ2File_getiter(BZ2FileObject *self) -{ - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - Py_INCREF((PyObject*)self); - return (PyObject *)self; -} - -/* This is a hacked version of Python's fileobject.c:file_iternext(). */ -#define READAHEAD_BUFSIZE 8192 -static PyObject * -BZ2File_iternext(BZ2FileObject *self) -{ - PyBytesObject* ret; - ACQUIRE_LOCK(self); - if (self->mode == MODE_CLOSED) { - RELEASE_LOCK(self); - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); - RELEASE_LOCK(self); - if (ret == NULL || PyBytes_GET_SIZE(ret) == 0) { - Py_XDECREF(ret); - return NULL; - } - return (PyObject *)ret; -} - -/* ===================================================================== */ -/* BZ2File_Type definition. */ - -PyDoc_VAR(BZ2File__doc__) = -PyDoc_STR( -"BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object\n\ -\n\ -Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or\n\ -writing. When opened for writing, the file will be created if it doesn't\n\ -exist, and truncated otherwise. If the buffering argument is given, 0 means\n\ -unbuffered, and larger numbers specify the buffer size. If compresslevel\n\ -is given, must be a number between 1 and 9.\n\ -Data read is always returned in bytes; data written ought to be bytes.\n\ -"); - -static PyTypeObject BZ2File_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2File", /*tp_name*/ - sizeof(BZ2FileObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2File_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2File__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - (getiterfunc)BZ2File_getiter, /*tp_iter*/ - (iternextfunc)BZ2File_iternext, /*tp_iternext*/ - BZ2File_methods, /*tp_methods*/ - 0, /*tp_members*/ - BZ2File_getset, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2File_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Methods of BZ2Comp. */ - -PyDoc_STRVAR(BZ2Comp_compress__doc__, -"compress(data) -> string\n\ -\n\ -Provide more data to the compressor object. It will return chunks of\n\ -compressed data whenever possible. When you've finished providing data\n\ -to compress, call the flush() method to finish the compression process,\n\ -and return what is left in the internal buffers.\n\ -"); - -static PyObject * -BZ2Comp_compress(BZ2CompObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PY_LONG_LONG totalout; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:compress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (datasize == 0) { - PyBuffer_Release(&pdata); - return PyBytes_FromStringAndSize("", 0); - } - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_ValueError, - "this object was already flushed"); - goto error; - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_RUN); - Py_END_ALLOW_THREADS - if (bzerror != BZ_RUN_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_in == 0) - break; /* no more input data */ - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzCompressEnd(bzs); - goto error; - } - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - return ret; - -error: - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - Py_XDECREF(ret); - return NULL; -} - -PyDoc_STRVAR(BZ2Comp_flush__doc__, -"flush() -> string\n\ -\n\ -Finish the compression process and return what is left in internal buffers.\n\ -You must not use the compressor object after calling this method.\n\ -"); - -static PyObject * -BZ2Comp_flush(BZ2CompObject *self) -{ - int bufsize = SMALLCHUNK; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - PY_LONG_LONG totalout; - int bzerror; - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_ValueError, "object was already " - "flushed"); - goto error; - } - self->running = 0; - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_FINISH); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_FINISH_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) - goto error; - bzs->next_out = BUF(ret); - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - } - - RELEASE_LOCK(self); - return ret; - -error: - RELEASE_LOCK(self); - Py_XDECREF(ret); - return NULL; -} - -static PyMethodDef BZ2Comp_methods[] = { - {"compress", (PyCFunction)BZ2Comp_compress, METH_VARARGS, - BZ2Comp_compress__doc__}, - {"flush", (PyCFunction)BZ2Comp_flush, METH_NOARGS, - BZ2Comp_flush__doc__}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2Comp_Type. */ - -static int -BZ2Comp_init(BZ2CompObject *self, PyObject *args, PyObject *kwargs) -{ - int compresslevel = 9; - int bzerror; - static char *kwlist[] = {"compresslevel", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:BZ2Compressor", - kwlist, &compresslevel)) - return -1; - - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - goto error; - } - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - memset(&self->bzs, 0, sizeof(bz_stream)); - bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->running = 1; - - return 0; -error: -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - return -1; -} - -static void -BZ2Comp_dealloc(BZ2CompObject *self) -{ -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - BZ2_bzCompressEnd(&self->bzs); - Py_TYPE(self)->tp_free((PyObject *)self); -} - - -/* ===================================================================== */ -/* BZ2Comp_Type definition. */ - -PyDoc_STRVAR(BZ2Comp__doc__, -"BZ2Compressor([compresslevel=9]) -> compressor object\n\ -\n\ -Create a new compressor object. This object may be used to compress\n\ -data sequentially. If you want to compress data in one shot, use the\n\ -compress() function instead. The compresslevel parameter, if given,\n\ -must be a number between 1 and 9.\n\ -"); - -static PyTypeObject BZ2Comp_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2Compressor", /*tp_name*/ - sizeof(BZ2CompObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2Comp_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2Comp__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - BZ2Comp_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2Comp_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Members of BZ2Decomp. */ - -#undef OFF -#define OFF(x) offsetof(BZ2DecompObject, x) - -static PyMemberDef BZ2Decomp_members[] = { - {"unused_data", T_OBJECT, OFF(unused_data), READONLY}, - {NULL} /* Sentinel */ -}; - - -/* ===================================================================== */ -/* Methods of BZ2Decomp. */ - -PyDoc_STRVAR(BZ2Decomp_decompress__doc__, -"decompress(data) -> string\n\ -\n\ -Provide more data to the decompressor object. It will return chunks\n\ -of decompressed data whenever possible. If you try to decompress data\n\ -after the end of stream is found, EOFError will be raised. If any data\n\ -was found after the end of stream, it'll be ignored and saved in\n\ -unused_data attribute.\n\ -"); - -static PyObject * -BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PY_LONG_LONG totalout; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:decompress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_EOFError, "end of stream was " - "already found"); - goto error; - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzDecompress(bzs); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - if (bzs->avail_in != 0) { - Py_DECREF(self->unused_data); - self->unused_data = - PyBytes_FromStringAndSize(bzs->next_in, - bzs->avail_in); - } - self->running = 0; - break; - } - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_in == 0) - break; /* no more input data */ - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzDecompressEnd(bzs); - goto error; - } - bzs->next_out = BUF(ret); - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - } - - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - return ret; - -error: - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - Py_XDECREF(ret); - return NULL; -} - -static PyMethodDef BZ2Decomp_methods[] = { - {"decompress", (PyCFunction)BZ2Decomp_decompress, METH_VARARGS, BZ2Decomp_decompress__doc__}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2Decomp_Type. */ - -static int -BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs) -{ - int bzerror; - - if (!PyArg_ParseTuple(args, ":BZ2Decompressor")) - return -1; - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - self->unused_data = PyBytes_FromStringAndSize("", 0); - if (!self->unused_data) - goto error; - - memset(&self->bzs, 0, sizeof(bz_stream)); - bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->running = 1; - - return 0; - -error: -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - Py_CLEAR(self->unused_data); - return -1; -} - -static void -BZ2Decomp_dealloc(BZ2DecompObject *self) -{ -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - Py_XDECREF(self->unused_data); - BZ2_bzDecompressEnd(&self->bzs); - Py_TYPE(self)->tp_free((PyObject *)self); -} - - -/* ===================================================================== */ -/* BZ2Decomp_Type definition. */ - -PyDoc_STRVAR(BZ2Decomp__doc__, -"BZ2Decompressor() -> decompressor object\n\ -\n\ -Create a new decompressor object. This object may be used to decompress\n\ -data sequentially. If you want to decompress data in one shot, use the\n\ -decompress() function instead.\n\ -"); - -static PyTypeObject BZ2Decomp_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2Decompressor", /*tp_name*/ - sizeof(BZ2DecompObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2Decomp_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2Decomp__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - BZ2Decomp_methods, /*tp_methods*/ - BZ2Decomp_members, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2Decomp_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Module functions. */ - -PyDoc_STRVAR(bz2_compress__doc__, -"compress(data [, compresslevel=9]) -> string\n\ -\n\ -Compress data in one shot. If you want to compress data sequentially,\n\ -use an instance of BZ2Compressor instead. The compresslevel parameter, if\n\ -given, must be a number between 1 and 9.\n\ -"); - -static PyObject * -bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int compresslevel=9; - Py_buffer pdata; - char *data; - int datasize; - int bufsize; - PyObject *ret = NULL; - bz_stream _bzs; - bz_stream *bzs = &_bzs; - int bzerror; - static char *kwlist[] = {"data", "compresslevel", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|i", - kwlist, &pdata, - &compresslevel)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - PyBuffer_Release(&pdata); - return NULL; - } - - /* Conforming to bz2 manual, this is large enough to fit compressed - * data in one shot. We will check it later anyway. */ - bufsize = datasize + (datasize/100+1) + 600; - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) { - PyBuffer_Release(&pdata); - return NULL; - } - - memset(bzs, 0, sizeof(bz_stream)); - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_FINISH); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_FINISH_OK) { - BZ2_bzCompressEnd(bzs); - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzCompressEnd(bzs); - PyBuffer_Release(&pdata); - return NULL; - } - bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) { - ret = NULL; - } - } - BZ2_bzCompressEnd(bzs); - - PyBuffer_Release(&pdata); - return ret; -} - -PyDoc_STRVAR(bz2_decompress__doc__, -"decompress(data) -> decompressed data\n\ -\n\ -Decompress data in one shot. If you want to decompress data sequentially,\n\ -use an instance of BZ2Decompressor instead.\n\ -"); - -static PyObject * -bz2_decompress(PyObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PyObject *ret; - bz_stream _bzs; - bz_stream *bzs = &_bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:decompress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (datasize == 0) { - PyBuffer_Release(&pdata); - return PyBytes_FromStringAndSize("", 0); - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) { - PyBuffer_Release(&pdata); - return NULL; - } - - memset(bzs, 0, sizeof(bz_stream)); - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - bzerror = BZ2_bzDecompressInit(bzs, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(ret); - PyBuffer_Release(&pdata); - return NULL; - } - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzDecompress(bzs); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_OK) { - BZ2_bzDecompressEnd(bzs); - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_in == 0) { - BZ2_bzDecompressEnd(bzs); - PyErr_SetString(PyExc_ValueError, - "couldn't find end of stream"); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzDecompressEnd(bzs); - PyBuffer_Release(&pdata); - return NULL; - } - bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) { - ret = NULL; - } - } - BZ2_bzDecompressEnd(bzs); - PyBuffer_Release(&pdata); - - return ret; -} - -static PyMethodDef bz2_methods[] = { - {"compress", (PyCFunction) bz2_compress, METH_VARARGS|METH_KEYWORDS, - bz2_compress__doc__}, - {"decompress", (PyCFunction) bz2_decompress, METH_VARARGS, - bz2_decompress__doc__}, - {NULL, NULL} /* sentinel */ -}; - -/* ===================================================================== */ -/* Initialization function. */ - -PyDoc_STRVAR(bz2__doc__, -"The python bz2 module provides a comprehensive interface for\n\ -the bz2 compression library. It implements a complete file\n\ -interface, one shot (de)compression functions, and types for\n\ -sequential (de)compression.\n\ -"); - - -static struct PyModuleDef bz2module = { - PyModuleDef_HEAD_INIT, - "bz2", - bz2__doc__, - -1, - bz2_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit_bz2(void) -{ - PyObject *m; - - if (PyType_Ready(&BZ2File_Type) < 0) - return NULL; - if (PyType_Ready(&BZ2Comp_Type) < 0) - return NULL; - if (PyType_Ready(&BZ2Decomp_Type) < 0) - return NULL; - - m = PyModule_Create(&bz2module); - if (m == NULL) - return NULL; - - PyModule_AddObject(m, "__author__", PyUnicode_FromString(__author__)); - - Py_INCREF(&BZ2File_Type); - PyModule_AddObject(m, "BZ2File", (PyObject *)&BZ2File_Type); - - Py_INCREF(&BZ2Comp_Type); - PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Comp_Type); - - Py_INCREF(&BZ2Decomp_Type); - PyModule_AddObject(m, "BZ2Decompressor", (PyObject *)&BZ2Decomp_Type); - return m; -} diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index ab4e659..9e9e96c 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -85,7 +85,7 @@ DECODER(gb2312) TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80) { NEXT(2, 1) } - else return 2; + else return 1; } return 0; @@ -141,7 +141,7 @@ DECODER(gbk) REQUIRE_INBUF(2) GBK_DECODE(c, IN2, **outbuf) - else return 2; + else return 1; NEXT(2, 1) } @@ -267,7 +267,7 @@ DECODER(gb18030) c3 = IN3; c4 = IN4; if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) - return 4; + return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; @@ -292,12 +292,12 @@ DECODER(gb18030) continue; } } - return 4; + return 1; } GBK_DECODE(c, c2, **outbuf) else TRYMAP_DEC(gb18030ext, **outbuf, c, c2); - else return 2; + else return 1; NEXT(2, 1) } @@ -400,7 +400,7 @@ DECODER(hz) else if (c2 == '\n') ; /* line-continuation */ else - return 2; + return 1; NEXT(2, 0); continue; } @@ -419,7 +419,7 @@ DECODER(hz) NEXT(2, 1) } else - return 2; + return 1; } } diff --git a/Modules/cjkcodecs/_codecs_hk.c b/Modules/cjkcodecs/_codecs_hk.c index 558a42f..d3ad04b 100644 --- a/Modules/cjkcodecs/_codecs_hk.c +++ b/Modules/cjkcodecs/_codecs_hk.c @@ -161,7 +161,7 @@ DECODER(big5hkscs) case 0x8864: WRITE2(0x00ca, 0x030c); break; case 0x88a3: WRITE2(0x00ea, 0x0304); break; case 0x88a5: WRITE2(0x00ea, 0x030c); break; - default: return 2; + default: return 1; } NEXT(2, 2) /* all decoded codepoints are pairs, above. */ diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 25c1a36..cbc1542 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -123,7 +123,7 @@ struct iso2022_config { CODEC_INIT(iso2022) { - const struct iso2022_designation *desig = CONFIG_DESIGNATIONS; + const struct iso2022_designation *desig; for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++) if (desig->initializer != NULL && desig->initializer() != 0) return -1; diff --git a/Modules/cjkcodecs/_codecs_jp.c b/Modules/cjkcodecs/_codecs_jp.c index a05e01b..a500696 100644 --- a/Modules/cjkcodecs/_codecs_jp.c +++ b/Modules/cjkcodecs/_codecs_jp.c @@ -112,7 +112,7 @@ DECODER(cp932) TRYMAP_DEC(cp932ext, **outbuf, c, c2); else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){ if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -120,7 +120,7 @@ DECODER(cp932) c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21; TRYMAP_DEC(jisx0208, **outbuf, c, c2); - else return 2; + else return 1; } else if (c >= 0xf0 && c <= 0xf9) { if ((c2 >= 0x40 && c2 <= 0x7e) || @@ -128,10 +128,10 @@ DECODER(cp932) OUT1(0xe000 + 188 * (c - 0xf0) + (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41)) else - return 2; + return 1; } else - return 2; + return 1; NEXT(2, 1) } @@ -256,7 +256,7 @@ DECODER(euc_jis_2004) NEXT(2, 1) } else - return 2; + return 1; } else if (c == 0x8f) { unsigned char c2, c3; @@ -274,7 +274,7 @@ DECODER(euc_jis_2004) continue; } else TRYMAP_DEC(jisx0212, **outbuf, c2, c3) ; - else return 3; + else return 1; NEXT(3, 1) } else { @@ -300,7 +300,7 @@ DECODER(euc_jis_2004) NEXT(2, 2) continue; } - else return 2; + else return 1; NEXT(2, 1) } } @@ -388,7 +388,7 @@ DECODER(euc_jp) NEXT(2, 1) } else - return 2; + return 1; } else if (c == 0x8f) { unsigned char c2, c3; @@ -401,7 +401,7 @@ DECODER(euc_jp) NEXT(3, 1) } else - return 3; + return 1; } else { unsigned char c2; @@ -417,7 +417,7 @@ DECODER(euc_jp) #endif TRYMAP_DEC(jisx0208, **outbuf, c ^ 0x80, c2 ^ 0x80) ; - else return 2; + else return 1; NEXT(2, 1) } } @@ -502,7 +502,7 @@ DECODER(shift_jis) REQUIRE_INBUF(2) c2 = IN2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -522,10 +522,10 @@ DECODER(shift_jis) continue; } else - return 2; + return 1; } else - return 2; + return 1; NEXT(1, 1) /* JIS X 0201 */ } @@ -645,7 +645,7 @@ DECODER(shift_jis_2004) REQUIRE_INBUF(2) c2 = IN2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -671,7 +671,7 @@ DECODER(shift_jis_2004) NEXT_OUT(2) } else - return 2; + return 1; NEXT_IN(2) } else { /* Plane 2 */ @@ -689,13 +689,13 @@ DECODER(shift_jis_2004) continue; } else - return 2; + return 1; NEXT(2, 1) } continue; } else - return 2; + return 1; NEXT(1, 1) /* JIS X 0201 */ } diff --git a/Modules/cjkcodecs/_codecs_kr.c b/Modules/cjkcodecs/_codecs_kr.c index 9272e36..f5697dd 100644 --- a/Modules/cjkcodecs/_codecs_kr.c +++ b/Modules/cjkcodecs/_codecs_kr.c @@ -123,7 +123,7 @@ DECODER(euc_kr) if ((*inbuf)[2] != EUCKR_JAMO_FIRSTBYTE || (*inbuf)[4] != EUCKR_JAMO_FIRSTBYTE || (*inbuf)[6] != EUCKR_JAMO_FIRSTBYTE) - return 8; + return 1; c = (*inbuf)[3]; if (0xa1 <= c && c <= 0xbe) @@ -143,7 +143,7 @@ DECODER(euc_kr) jong = NONE; if (cho == NONE || jung == NONE || jong == NONE) - return 8; + return 1; OUT1(0xac00 + cho*588 + jung*28 + jong); NEXT(8, 1) @@ -152,7 +152,7 @@ DECODER(euc_kr) NEXT(2, 1) } else - return 2; + return 1; } return 0; @@ -208,7 +208,7 @@ DECODER(cp949) REQUIRE_INBUF(2) TRYMAP_DEC(ksx1001, **outbuf, c ^ 0x80, IN2 ^ 0x80); else TRYMAP_DEC(cp949ext, **outbuf, c, IN2); - else return 2; + else return 1; NEXT(2, 1) } @@ -375,7 +375,7 @@ DECODER(johab) i_jong = johabidx_jongseong[c_jong]; if (i_cho == NONE || i_jung == NONE || i_jong == NONE) - return 2; + return 1; /* we don't use U+1100 hangul jamo yet. */ if (i_cho == FILL) { @@ -391,7 +391,7 @@ DECODER(johab) OUT1(0x3100 | johabjamo_jungseong[c_jung]) else - return 2; + return 1; } } else { if (i_jung == FILL) { @@ -399,7 +399,7 @@ DECODER(johab) OUT1(0x3100 | johabjamo_choseong[c_cho]) else - return 2; + return 1; } else OUT1(0xac00 + @@ -414,7 +414,7 @@ DECODER(johab) c2 < 0x31 || (c2 >= 0x80 && c2 < 0x91) || (c2 & 0x7f) == 0x7f || (c == 0xda && (c2 >= 0xa1 && c2 <= 0xd3))) - return 2; + return 1; else { unsigned char t1, t2; @@ -425,7 +425,7 @@ DECODER(johab) t2 = (t2 < 0x5e ? t2 : t2 - 0x5e) + 0x21; TRYMAP_DEC(ksx1001, **outbuf, t1, t2); - else return 2; + else return 1; NEXT(2, 1) } } diff --git a/Modules/cjkcodecs/_codecs_tw.c b/Modules/cjkcodecs/_codecs_tw.c index 38cf723..916298d 100644 --- a/Modules/cjkcodecs/_codecs_tw.c +++ b/Modules/cjkcodecs/_codecs_tw.c @@ -55,7 +55,7 @@ DECODER(big5) TRYMAP_DEC(big5, **outbuf, c, IN2) { NEXT(2, 1) } - else return 2; + else return 1; } return 0; @@ -109,7 +109,7 @@ DECODER(cp950) TRYMAP_DEC(cp950ext, **outbuf, c, IN2); else TRYMAP_DEC(big5, **outbuf, c, IN2); - else return 2; + else return 1; NEXT(2, 1) } diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7b04f020..8635a56 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -483,6 +483,7 @@ multibytecodec_encode(MultibyteCodec *codec, return PyBytes_FromStringAndSize(NULL, 0); buf.excobj = NULL; + buf.outobj = NULL; buf.inbuf = buf.inbuf_top = *data; buf.inbuf_end = buf.inbuf_top + datalen; @@ -900,11 +901,17 @@ mbiencoder_encode(MultibyteIncrementalEncoderObject *self, static PyObject * mbiencoder_reset(MultibyteIncrementalEncoderObject *self) { - if (self->codec->decreset != NULL && - self->codec->decreset(&self->state, self->codec->config) != 0) - return NULL; + /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */ + unsigned char buffer[4], *outbuf; + Py_ssize_t r; + if (self->codec->encreset != NULL) { + outbuf = buffer; + r = self->codec->encreset(&self->state, self->codec->config, + &outbuf, sizeof(buffer)); + if (r != 0) + return NULL; + } self->pendingsize = 0; - Py_RETURN_NONE; } @@ -1572,12 +1579,13 @@ mbstreamwriter_iwrite(MultibyteStreamWriterObject *self, PyObject *unistr) { PyObject *str, *wr; + _Py_identifier(write); str = encoder_encode_stateful(STATEFUL_ECTX(self), unistr, 0); if (str == NULL) return -1; - wr = PyObject_CallMethod(self->stream, "write", "O", str); + wr = _PyObject_CallMethodId(self->stream, &PyId_write, "O", str); Py_DECREF(str); if (wr == NULL) return -1; @@ -1643,7 +1651,9 @@ mbstreamwriter_reset(MultibyteStreamWriterObject *self) assert(PyBytes_Check(pwrt)); if (PyBytes_Size(pwrt) > 0) { PyObject *wr; - wr = PyObject_CallMethod(self->stream, "write", "O", pwrt); + _Py_identifier(write); + + wr = _PyObject_CallMethodId(self->stream, &PyId_write, "O", pwrt); if (wr == NULL) { Py_DECREF(pwrt); return NULL; diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 86720af..86b0a01 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -84,6 +84,8 @@ PyInit_errno(void) * The names and comments are borrowed from linux/include/errno.h, * which should be pretty all-inclusive. However, the Solaris specific * names and comments are borrowed from sys/errno.h in Solaris. + * MacOSX specific names and comments are borrowed from sys/errno.h in + * MacOSX. */ #ifdef ENODEV @@ -848,6 +850,59 @@ PyInit_errno(void) inscode(d, ds, de, "ENOTACTIVE", ENOTACTIVE, "Facility is not active"); #endif + /* MacOSX specific errnos */ +#ifdef EAUTH + inscode(d, ds, de, "EAUTH", EAUTH, "Authentication error"); +#endif +#ifdef EBADARCH + inscode(d, ds, de, "EBADARCH", EBADARCH, "Bad CPU type in executable"); +#endif +#ifdef EBADEXEC + inscode(d, ds, de, "EBADEXEC", EBADEXEC, "Bad executable (or shared library)"); +#endif +#ifdef EBADMACHO + inscode(d, ds, de, "EBADMACHO", EBADMACHO, "Malformed Mach-o file"); +#endif +#ifdef EBADRPC + inscode(d, ds, de, "EBADRPC", EBADRPC, "RPC struct is bad"); +#endif +#ifdef EDEVERR + inscode(d, ds, de, "EDEVERR", EDEVERR, "Device error"); +#endif +#ifdef EFTYPE + inscode(d, ds, de, "EFTYPE", EFTYPE, "Inappropriate file type or format"); +#endif +#ifdef ENEEDAUTH + inscode(d, ds, de, "ENEEDAUTH", ENEEDAUTH, "Need authenticator"); +#endif +#ifdef ENOATTR + inscode(d, ds, de, "ENOATTR", ENOATTR, "Attribute not found"); +#endif +#ifdef ENOPOLICY + inscode(d, ds, de, "ENOPOLICY", ENOPOLICY, "Policy not found"); +#endif +#ifdef EPROCLIM + inscode(d, ds, de, "EPROCLIM", EPROCLIM, "Too many processes"); +#endif +#ifdef EPROCUNAVAIL + inscode(d, ds, de, "EPROCUNAVAIL", EPROCUNAVAIL, "Bad procedure for program"); +#endif +#ifdef EPROGMISMATCH + inscode(d, ds, de, "EPROGMISMATCH", EPROGMISMATCH, "Program version wrong"); +#endif +#ifdef EPROGUNAVAIL + inscode(d, ds, de, "EPROGUNAVAIL", EPROGUNAVAIL, "RPC prog. not avail"); +#endif +#ifdef EPWROFF + inscode(d, ds, de, "EPWROFF", EPWROFF, "Device power is off"); +#endif +#ifdef ERPCMISMATCH + inscode(d, ds, de, "ERPCMISMATCH", ERPCMISMATCH, "RPC version wrong"); +#endif +#ifdef ESHLIBVERS + inscode(d, ds, de, "ESHLIBVERS", ESHLIBVERS, "Shared library version mismatch"); +#endif + Py_DECREF(de); return m; } diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c new file mode 100644 index 0000000..11d5340 --- /dev/null +++ b/Modules/faulthandler.c @@ -0,0 +1,1329 @@ +#include "Python.h" +#include "pythread.h" +#include <signal.h> +#include <object.h> +#include <frameobject.h> +#include <signal.h> +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) +#include <pthread.h> +#endif + +/* Allocate at maximum 100 MB of the stack to raise the stack overflow */ +#define STACK_OVERFLOW_MAX_SIZE (100*1024*1024) + +#ifdef WITH_THREAD +# define FAULTHANDLER_LATER +# define FAULTHANDLER_WATCHDOG +#endif + +#ifndef MS_WINDOWS + /* register() is useless on Windows, because only SIGSEGV, SIGABRT and + SIGILL can be handled by the process, and these signals can only be used + with enable(), not using register() */ +# define FAULTHANDLER_USER +#endif + +#define PUTS(fd, str) write(fd, str, strlen(str)) + +#ifdef HAVE_SIGACTION +typedef struct sigaction _Py_sighandler_t; +#else +typedef PyOS_sighandler_t _Py_sighandler_t; +#endif + +typedef struct { + int signum; + int enabled; + const char* name; + _Py_sighandler_t previous; + int all_threads; +} fault_handler_t; + +static struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + PyInterpreterState *interp; +} fatal_error = {0, NULL, -1, 0}; + +#ifdef FAULTHANDLER_LATER +static struct { + PyObject *file; + int fd; + PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ + int repeat; + PyInterpreterState *interp; + int exit; + char *header; + size_t header_len; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + PyThread_type_lock cancel_event; + /* released by child thread when joined */ + PyThread_type_lock running; +} thread; +#endif + +#ifdef FAULTHANDLER_WATCHDOG +static struct { + int rfd; + int wfd; + PY_TIMEOUT_T period_us; /* period in microseconds */ + /* The main thread always holds this lock. It is only released when + faulthandler_watchdog() is interrupted before this thread exits, or at + Python exit. */ + PyThread_type_lock cancel_event; + /* released by child thread when joined */ + PyThread_type_lock running; +} watchdog; +#endif + +#ifdef FAULTHANDLER_USER +typedef struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; + PyInterpreterState *interp; +} user_signal_t; + +static user_signal_t *user_signals; + +/* the following macros come from Python: Modules/signalmodule.c */ +#if defined(PYOS_OS2) && !defined(PYCC_GCC) +#define NSIG 12 +#endif +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum); +#endif /* FAULTHANDLER_USER */ + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const unsigned char faulthandler_nsignals = \ + Py_ARRAY_LENGTH(faulthandler_handlers); + +#ifdef HAVE_SIGALTSTACK +static stack_t stack; +#endif + + +/* Get the file descriptor of a file by calling its fileno() method and then + call its flush() method. + + If file is NULL or Py_None, use sys.stderr as the new file. + + On success, return the new file and write the file descriptor into *p_fd. + On error, return NULL. */ + +static PyObject* +faulthandler_get_fileno(PyObject *file, int *p_fd) +{ + PyObject *result; + _Py_identifier(fileno); + _Py_identifier(flush); + long fd_long; + int fd; + + if (file == NULL || file == Py_None) { + file = PySys_GetObject("stderr"); + if (file == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.stderr"); + return NULL; + } + } + + result = _PyObject_CallMethodId(file, &PyId_fileno, ""); + if (result == NULL) + return NULL; + + fd = -1; + if (PyLong_Check(result)) { + fd_long = PyLong_AsLong(result); + if (0 <= fd_long && fd_long < INT_MAX) + fd = (int)fd_long; + } + Py_DECREF(result); + + if (fd == -1) { + PyErr_SetString(PyExc_RuntimeError, + "file.fileno() is not a valid file descriptor"); + return NULL; + } + + result = _PyObject_CallMethodId(file, &PyId_flush, ""); + if (result != NULL) + Py_DECREF(result); + else { + /* ignore flush() error */ + PyErr_Clear(); + } + *p_fd = fd; + return file; +} + +/* Get the state of the current thread: only call this function if the current + thread holds the GIL. Raise an exception on error. */ +static PyThreadState* +get_thread_state(void) +{ + PyThreadState *tstate = PyThreadState_Get(); + if (tstate == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get the current thread state"); + return NULL; + } + return tstate; +} + +static PyObject* +faulthandler_dump_traceback_py(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; + PyThreadState *tstate; + const char *errmsg; + int fd; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|Oi:dump_traceback", kwlist, + &file, &all_threads)) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + if (all_threads) { + errmsg = _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + if (errmsg != NULL) { + PyErr_SetString(PyExc_RuntimeError, errmsg); + return NULL; + } + } + else { + _Py_DumpTraceback(fd, tstate); + } + Py_RETURN_NONE; +} + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum) +{ + const int fd = fatal_error.fd; + unsigned int i; + fault_handler_t *handler = NULL; + PyThreadState *tstate; + int save_errno = errno; + + if (!fatal_error.enabled) + return; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + if (handler == NULL) { + /* faulthandler_nsignals == 0 (unlikely) */ + return; + } + + /* restore the previous handler */ +#ifdef HAVE_SIGACTION + (void)sigaction(signum, &handler->previous, NULL); +#else + (void)signal(signum, handler->previous); +#endif + handler->enabled = 0; + + PUTS(fd, "Fatal Python error: "); + PUTS(fd, handler->name); + PUTS(fd, "\n\n"); + +#ifdef WITH_THREAD + /* SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals and + are thus delivered to the thread that caused the fault. Get the Python + thread state of the current thread. + + PyThreadState_Get() doesn't give the state of the thread that caused the + fault if the thread released the GIL, and so this function cannot be + used. Read the thread local storage (TLS) instead: call + PyGILState_GetThisThreadState(). */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_Get(); +#endif + + if (fatal_error.all_threads) + _Py_DumpTracebackThreads(fd, fatal_error.interp, tstate); + else { + if (tstate != NULL) + _Py_DumpTraceback(fd, tstate); + } + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediatly if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + +/* Install the handler for fatal signals, faulthandler_fatal_error(). */ + +static PyObject* +faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; + unsigned int i; + fault_handler_t *handler; +#ifdef HAVE_SIGACTION + struct sigaction action; +#endif + int err; + int fd; + PyThreadState *tstate; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|Oi:enable", kwlist, &file, &all_threads)) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + Py_XDECREF(fatal_error.file); + Py_INCREF(file); + fatal_error.file = file; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + fatal_error.interp = tstate->interp; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; +#ifdef HAVE_SIGACTION + action.sa_handler = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER; +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + err = sigaction(handler->signum, &action, &handler->previous); +#else + handler->previous = signal(handler->signum, + faulthandler_fatal_error); + err = (handler->previous == SIG_ERR); +#endif + if (err) { + PyErr_SetFromErrno(PyExc_RuntimeError); + return NULL; + } + handler->enabled = 1; + } + } + Py_RETURN_NONE; +} + +static void +faulthandler_disable(void) +{ + unsigned int i; + fault_handler_t *handler; + + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; +#ifdef HAVE_SIGACTION + (void)sigaction(handler->signum, &handler->previous, NULL); +#else + (void)signal(handler->signum, handler->previous); +#endif + handler->enabled = 0; + } + } + + Py_CLEAR(fatal_error.file); +} + +static PyObject* +faulthandler_disable_py(PyObject *self) +{ + if (!fatal_error.enabled) { + Py_INCREF(Py_False); + return Py_False; + } + faulthandler_disable(); + Py_INCREF(Py_True); + return Py_True; +} + +static PyObject* +faulthandler_is_enabled(PyObject *self) +{ + return PyBool_FromLong(fatal_error.enabled); +} + +#ifdef FAULTHANDLER_LATER + +static void +faulthandler_thread(void *unused) +{ + PyLockStatus st; + const char* errmsg; + PyThreadState *current; + int ok; +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) + sigset_t set; + + /* we don't want to receive any signal */ + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + do { + st = PyThread_acquire_lock_timed(thread.cancel_event, + thread.timeout_us, 0); + if (st == PY_LOCK_ACQUIRED) { + PyThread_release_lock(thread.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == PY_LOCK_FAILURE); + + /* get the thread holding the GIL, NULL if no thread hold the GIL */ + current = _Py_atomic_load_relaxed(&_PyThreadState_Current); + + write(thread.fd, thread.header, thread.header_len); + + errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, current); + ok = (errmsg == NULL); + + if (thread.exit) + _exit(1); + } while (ok && thread.repeat); + + /* The only way out */ + PyThread_release_lock(thread.running); +} + +static void +cancel_dump_tracebacks_later(void) +{ + /* Notify cancellation */ + PyThread_release_lock(thread.cancel_event); + + /* Wait for thread to join */ + PyThread_acquire_lock(thread.running, 1); + PyThread_release_lock(thread.running); + + /* The main thread should always hold the cancel_event lock */ + PyThread_acquire_lock(thread.cancel_event, 1); + + Py_CLEAR(thread.file); + if (thread.header) { + free(thread.header); + thread.header = NULL; + } +} + +static char* +format_timeout(double timeout) +{ + unsigned long us, sec, min, hour; + double intpart, fracpart; + char buffer[100]; + + fracpart = modf(timeout, &intpart); + sec = (unsigned long)intpart; + us = (unsigned long)(fracpart * 1e6); + min = sec / 60; + sec %= 60; + hour = min / 60; + min %= 60; + + if (us != 0) + PyOS_snprintf(buffer, sizeof(buffer), + "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, min, sec, us); + else + PyOS_snprintf(buffer, sizeof(buffer), + "Timeout (%lu:%02lu:%02lu)!\n", + hour, min, sec); + + return strdup(buffer); +} + +static PyObject* +faulthandler_dump_tracebacks_later(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; + double timeout; + PY_TIMEOUT_T timeout_us; + int repeat = 0; + PyObject *file = NULL; + int fd; + int exit = 0; + PyThreadState *tstate; + char *header; + size_t header_len; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "d|iOi:dump_tracebacks_later", kwlist, + &timeout, &repeat, &file, &exit)) + return NULL; + if ((timeout * 1e6) >= (double) PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); + return NULL; + } + timeout_us = (PY_TIMEOUT_T)(timeout * 1e6); + if (timeout_us <= 0) { + PyErr_SetString(PyExc_ValueError, "timeout must be greater than 0"); + return NULL; + } + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + /* format the timeout */ + header = format_timeout(timeout); + if (header == NULL) + return PyErr_NoMemory(); + header_len = strlen(header); + + /* Cancel previous thread, if running */ + cancel_dump_tracebacks_later(); + + Py_XDECREF(thread.file); + Py_INCREF(file); + thread.file = file; + thread.fd = fd; + thread.timeout_us = timeout_us; + thread.repeat = repeat; + thread.interp = tstate->interp; + thread.exit = exit; + thread.header = header; + thread.header_len = header_len; + + /* Arm these locks to serve as events when released */ + PyThread_acquire_lock(thread.running, 1); + + if (PyThread_start_new_thread(faulthandler_thread, NULL) == -1) { + PyThread_release_lock(thread.running); + Py_CLEAR(thread.file); + free(header); + thread.header = NULL; + PyErr_SetString(PyExc_RuntimeError, + "unable to start watchdog thread"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +faulthandler_cancel_dump_tracebacks_later_py(PyObject *self) +{ + cancel_dump_tracebacks_later(); + Py_RETURN_NONE; +} +#endif /* FAULTHANDLER_LATER */ + +#ifdef FAULTHANDLER_WATCHDOG + +static void +file_watchdog(void *unused) +{ + PyLockStatus st; + PY_TIMEOUT_T timeout; + +#define MAXDATA 1024 + char buf1[MAXDATA], buf2[MAXDATA]; + char *data = buf1, *old_data = buf2; + Py_ssize_t data_len, old_data_len = -1; + +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) + sigset_t set; + + /* we don't want to receive any signal */ + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + /* On first pass, feed file contents immediately */ + timeout = 0; + do { + st = PyThread_acquire_lock_timed(watchdog.cancel_event, + timeout, 0); + timeout = watchdog.period_us; + if (st == PY_LOCK_ACQUIRED) { + PyThread_release_lock(watchdog.cancel_event); + break; + } + /* Timeout => read and write data */ + assert(st == PY_LOCK_FAILURE); + + if (lseek(watchdog.rfd, 0, SEEK_SET) < 0) { + break; + } + data_len = read(watchdog.rfd, data, MAXDATA); + if (data_len < 0) { + break; + } + if (data_len != old_data_len || memcmp(data, old_data, data_len)) { + char *tdata; + Py_ssize_t tlen; + /* Contents changed, feed them to wfd */ + long x = (long) data_len; + /* We can't do anything if the consumer is too slow, just bail out */ + if (write(watchdog.wfd, (void *) &x, sizeof(x)) < sizeof(x)) + break; + if (write(watchdog.wfd, data, data_len) < data_len) + break; + tdata = data; + data = old_data; + old_data = tdata; + tlen = data_len; + data_len = old_data_len; + old_data_len = tlen; + } + } while (1); + + close(watchdog.rfd); + close(watchdog.wfd); + + /* The only way out */ + PyThread_release_lock(watchdog.running); +#undef MAXDATA +} + +static void +cancel_file_watchdog(void) +{ + /* Notify cancellation */ + PyThread_release_lock(watchdog.cancel_event); + + /* Wait for thread to join */ + PyThread_acquire_lock(watchdog.running, 1); + PyThread_release_lock(watchdog.running); + + /* The main thread should always hold the cancel_event lock */ + PyThread_acquire_lock(watchdog.cancel_event, 1); +} + +static PyObject* +faulthandler_file_watchdog(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"rfd", "wfd", "period", NULL}; + double period; + PY_TIMEOUT_T period_us; + int rfd, wfd; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "iid:_file_watchdog", kwlist, + &rfd, &wfd, &period)) + return NULL; + if ((period * 1e6) >= (double) PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, "period value is too large"); + return NULL; + } + period_us = (PY_TIMEOUT_T)(period * 1e6); + if (period_us <= 0) { + PyErr_SetString(PyExc_ValueError, "period must be greater than 0"); + return NULL; + } + + /* Cancel previous thread, if running */ + cancel_file_watchdog(); + + watchdog.rfd = rfd; + watchdog.wfd = wfd; + watchdog.period_us = period_us; + + /* Arm these locks to serve as events when released */ + PyThread_acquire_lock(watchdog.running, 1); + + if (PyThread_start_new_thread(file_watchdog, NULL) == -1) { + PyThread_release_lock(watchdog.running); + PyErr_SetString(PyExc_RuntimeError, + "unable to start file watchdog thread"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +faulthandler_cancel_file_watchdog(PyObject *self) +{ + cancel_file_watchdog(); + Py_RETURN_NONE; +} +#endif /* FAULTHANDLER_WATCHDOG */ + +#ifdef FAULTHANDLER_USER +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ +#ifdef HAVE_SIGACTION + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + return sigaction(signum, &action, p_previous); +#else + _Py_sighandler_t previous; + previous = signal(signum, faulthandler_user); + if (p_previous != NULL) + *p_previous = previous; + return (previous == SIG_ERR); +#endif +} + +/* Handler of user signals (e.g. SIGUSR1). + + Dump the traceback of the current thread, or of all threads if + thread.all_threads is true. + + This function is signal safe and should only call signal safe functions. */ + +static void +faulthandler_user(int signum) +{ + user_signal_t *user; + PyThreadState *tstate; + int save_errno = errno; + + user = &user_signals[signum]; + if (!user->enabled) + return; + +#ifdef WITH_THREAD + /* PyThreadState_Get() doesn't give the state of the current thread if + the thread doesn't hold the GIL. Read the thread local storage (TLS) + instead: call PyGILState_GetThisThreadState(). */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_Get(); +#endif + + if (user->all_threads) + _Py_DumpTracebackThreads(user->fd, user->interp, tstate); + else { + if (tstate == NULL) + return; + _Py_DumpTraceback(user->fd, tstate); + } +#ifdef HAVE_SIGACTION + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + /* call the previous signal handler */ + raise(signum); + (void)faulthandler_register(signum, user->chain, NULL); + } +#else + if (user->chain) { + /* call the previous signal handler */ + user->previous(signum); + } +#endif + errno = save_errno; +} + +static int +check_signum(int signum) +{ + unsigned int i; + + for (i=0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + PyErr_Format(PyExc_RuntimeError, + "signal %i cannot be registered, " + "use enable() instead", + signum); + return 0; + } + } + if (signum < 1 || NSIG <= signum) { + PyErr_SetString(PyExc_ValueError, "signal number out of range"); + return 0; + } + return 1; +} + +static PyObject* +faulthandler_register_py(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"signum", "file", "all_threads", "chain", NULL}; + int signum; + PyObject *file = NULL; + int all_threads = 1; + int chain = 0; + int fd; + user_signal_t *user; + _Py_sighandler_t previous; + PyThreadState *tstate; + int err; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "i|Oii:register", kwlist, + &signum, &file, &all_threads, &chain)) + return NULL; + + if (!check_signum(signum)) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + if (user_signals == NULL) { + user_signals = calloc(NSIG, sizeof(user_signal_t)); + if (user_signals == NULL) + return PyErr_NoMemory(); + } + user = &user_signals[signum]; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + } + + Py_XDECREF(user->file); + Py_INCREF(file); + user->file = file; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + user->previous = previous; + user->interp = tstate->interp; + user->enabled = 1; + + Py_RETURN_NONE; +} + +static int +faulthandler_unregister(user_signal_t *user, int signum) +{ + if (!user->enabled) + return 0; + user->enabled = 0; +#ifdef HAVE_SIGACTION + (void)sigaction(signum, &user->previous, NULL); +#else + (void)signal(signum, user->previous); +#endif + Py_CLEAR(user->file); + user->fd = -1; + return 1; +} + +static PyObject* +faulthandler_unregister_py(PyObject *self, PyObject *args) +{ + int signum; + user_signal_t *user; + int change; + + if (!PyArg_ParseTuple(args, "i:unregister", &signum)) + return NULL; + + if (!check_signum(signum)) + return NULL; + + if (user_signals == NULL) + Py_RETURN_FALSE; + + user = &user_signals[signum]; + change = faulthandler_unregister(user, signum); + return PyBool_FromLong(change); +} +#endif /* FAULTHANDLER_USER */ + + +static PyObject * +faulthandler_read_null(PyObject *self, PyObject *args) +{ + int *x = NULL, y; + int release_gil = 0; + if (!PyArg_ParseTuple(args, "|i:_read_null", &release_gil)) + return NULL; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + y = *x; + Py_END_ALLOW_THREADS + } else + y = *x; + return PyLong_FromLong(y); + +} + +static PyObject * +faulthandler_sigsegv(PyObject *self, PyObject *args) +{ +#if defined(MS_WINDOWS) + /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal + handler and then gives back the execution flow to the program (without + explicitly calling the previous error handler). In a normal case, the + SIGSEGV was raised by the kernel because of a fault, and so if the + program retries to execute the same instruction, the fault will be + raised again. + + Here the fault is simulated by a fake SIGSEGV signal raised by the + application. We have to raise SIGSEGV at lease twice: once for + faulthandler_fatal_error(), and one more time for the previous signal + handler. */ + while(1) + raise(SIGSEGV); +#else + raise(SIGSEGV); +#endif + Py_RETURN_NONE; +} + +static PyObject * +faulthandler_sigfpe(PyObject *self, PyObject *args) +{ + /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on + PowerPC. Use volatile to disable compile-time optimizations. */ + volatile int x = 1, y = 0, z; + z = x / y; + /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), + raise it manually. */ + raise(SIGFPE); + /* This line is never reached, but we pretend to make something with z + to silence a compiler warning. */ + return PyLong_FromLong(z); +} + +static PyObject * +faulthandler_sigabrt(PyObject *self, PyObject *args) +{ +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif + abort(); + Py_RETURN_NONE; +} + +#ifdef SIGBUS +static PyObject * +faulthandler_sigbus(PyObject *self, PyObject *args) +{ + raise(SIGBUS); + Py_RETURN_NONE; +} +#endif + +#ifdef SIGILL +static PyObject * +faulthandler_sigill(PyObject *self, PyObject *args) +{ + raise(SIGILL); + Py_RETURN_NONE; +} +#endif + +static PyObject * +faulthandler_fatal_error_py(PyObject *self, PyObject *args) +{ + char *message; + if (!PyArg_ParseTuple(args, "y:fatal_error", &message)) + return NULL; + Py_FatalError(message); + Py_RETURN_NONE; +} + +#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +static void* +stack_overflow(void *min_sp, void *max_sp, size_t *depth) +{ + /* allocate 4096 bytes on the stack at each call */ + unsigned char buffer[4096]; + void *sp = &buffer; + *depth += 1; + if (sp < min_sp || max_sp < sp) + return sp; + buffer[0] = 1; + buffer[4095] = 0; + return stack_overflow(min_sp, max_sp, depth); +} + +static PyObject * +faulthandler_stack_overflow(PyObject *self) +{ + size_t depth, size; + char *sp = (char *)&depth, *stop; + + depth = 0; + stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, + sp + STACK_OVERFLOW_MAX_SIZE, + &depth); + if (sp < stop) + size = stop - sp; + else + size = sp - stop; + PyErr_Format(PyExc_RuntimeError, + "unable to raise a stack overflow (allocated %zu bytes " + "on the stack, %zu recursive calls)", + size, depth); + return NULL; +} +#endif + + +static int +faulthandler_traverse(PyObject *module, visitproc visit, void *arg) +{ +#ifdef FAULTHANDLER_USER + unsigned int signum; +#endif + +#ifdef FAULTHANDLER_LATER + Py_VISIT(thread.file); +#endif +#ifdef FAULTHANDLER_USER + if (user_signals != NULL) { + for (signum=0; signum < NSIG; signum++) + Py_VISIT(user_signals[signum].file); + } +#endif + Py_VISIT(fatal_error.file); + return 0; +} + +PyDoc_STRVAR(module_doc, +"faulthandler module."); + +static PyMethodDef module_methods[] = { + {"enable", + (PyCFunction)faulthandler_enable, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("enable(file=sys.stderr, all_threads=True): " + "enable the fault handler")}, + {"disable", (PyCFunction)faulthandler_disable_py, METH_NOARGS, + PyDoc_STR("disable(): disable the fault handler")}, + {"is_enabled", (PyCFunction)faulthandler_is_enabled, METH_NOARGS, + PyDoc_STR("is_enabled()->bool: check if the handler is enabled")}, + {"dump_traceback", + (PyCFunction)faulthandler_dump_traceback_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): " + "dump the traceback of the current thread, or of all threads " + "if all_threads is True, into file")}, +#ifdef FAULTHANDLER_LATER + {"dump_tracebacks_later", + (PyCFunction)faulthandler_dump_tracebacks_later, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("dump_tracebacks_later(timeout, repeat=False, file=sys.stderrn, exit=False):\n" + "dump the traceback of all threads in timeout seconds,\n" + "or each timeout seconds if repeat is True. If exit is True, " + "call _exit(1) which is not safe.")}, + {"cancel_dump_tracebacks_later", + (PyCFunction)faulthandler_cancel_dump_tracebacks_later_py, METH_NOARGS, + PyDoc_STR("cancel_dump_tracebacks_later():\ncancel the previous call " + "to dump_tracebacks_later().")}, +#endif + +#ifdef FAULTHANDLER_WATCHDOG + {"_file_watchdog", + (PyCFunction)faulthandler_file_watchdog, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("_file_watchdog(rfd, wfd, period):\n" + "feed the contents of 'rfd' to 'wfd', if changed,\n" + "every 'period seconds'.")}, + {"_cancel_file_watchdog", + (PyCFunction)faulthandler_cancel_file_watchdog, METH_NOARGS, + PyDoc_STR("_cancel_file_watchdog():\ncancel the previous call " + "to _file_watchdog().")}, +#endif + +#ifdef FAULTHANDLER_USER + {"register", + (PyCFunction)faulthandler_register_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("register(signum, file=sys.stderr, all_threads=True, chain=False): " + "register an handler for the signal 'signum': dump the " + "traceback of the current thread, or of all threads if " + "all_threads is True, into file")}, + {"unregister", + faulthandler_unregister_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("unregister(signum): unregister the handler of the signal " + "'signum' registered by register()")}, +#endif + + {"_read_null", faulthandler_read_null, METH_VARARGS, + PyDoc_STR("_read_null(release_gil=False): read from NULL, raise " + "a SIGSEGV or SIGBUS signal depending on the platform")}, + {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, + PyDoc_STR("_sigsegv(): raise a SIGSEGV signal")}, + {"_sigabrt", faulthandler_sigabrt, METH_VARARGS, + PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, + {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, + PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, +#ifdef SIGBUS + {"_sigbus", (PyCFunction)faulthandler_sigbus, METH_NOARGS, + PyDoc_STR("_sigbus(): raise a SIGBUS signal")}, +#endif +#ifdef SIGILL + {"_sigill", (PyCFunction)faulthandler_sigill, METH_NOARGS, + PyDoc_STR("_sigill(): raise a SIGILL signal")}, +#endif + {"_fatal_error", faulthandler_fatal_error_py, METH_VARARGS, + PyDoc_STR("_fatal_error(message): call Py_FatalError(message)")}, +#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) + {"_stack_overflow", (PyCFunction)faulthandler_stack_overflow, METH_NOARGS, + PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, +#endif + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "faulthandler", + module_doc, + 0, /* non-negative size to be able to unload the module */ + module_methods, + NULL, + faulthandler_traverse, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit_faulthandler(void) +{ + return PyModule_Create(&module_def); +} + +/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable + is defined, or if sys._xoptions has a 'faulthandler' key. */ + +static int +faulthandler_env_options(void) +{ + PyObject *xoptions, *key, *module, *res; + _Py_identifier(enable); + + if (!Py_GETENV("PYTHONFAULTHANDLER")) { + int has_key; + + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return -1; + + key = PyUnicode_FromString("faulthandler"); + if (key == NULL) + return -1; + + has_key = PyDict_Contains(xoptions, key); + Py_DECREF(key); + if (!has_key) + return 0; + } + + module = PyImport_ImportModule("faulthandler"); + if (module == NULL) { + return -1; + } + res = _PyObject_CallMethodId(module, &PyId_enable, ""); + Py_DECREF(module); + if (res == NULL) + return -1; + Py_DECREF(res); + return 0; +} + +int _PyFaulthandler_Init(void) +{ +#ifdef HAVE_SIGALTSTACK + int err; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = PyMem_Malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + err = sigaltstack(&stack, NULL); + if (err) { + PyMem_Free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +#endif +#ifdef FAULTHANDLER_LATER + thread.file = NULL; + thread.cancel_event = PyThread_allocate_lock(); + thread.running = PyThread_allocate_lock(); + if (!thread.cancel_event || !thread.running) { + PyErr_SetString(PyExc_RuntimeError, + "could not allocate locks for faulthandler"); + return -1; + } + PyThread_acquire_lock(thread.cancel_event, 1); +#endif +#ifdef FAULTHANDLER_WATCHDOG + watchdog.cancel_event = PyThread_allocate_lock(); + watchdog.running = PyThread_allocate_lock(); + if (!watchdog.cancel_event || !watchdog.running) { + PyErr_SetString(PyExc_RuntimeError, + "could not allocate locks for faulthandler"); + return -1; + } + PyThread_acquire_lock(watchdog.cancel_event, 1); +#endif + + return faulthandler_env_options(); +} + +void _PyFaulthandler_Fini(void) +{ +#ifdef FAULTHANDLER_USER + unsigned int signum; +#endif + +#ifdef FAULTHANDLER_LATER + /* later */ + cancel_dump_tracebacks_later(); + if (thread.cancel_event) { + PyThread_release_lock(thread.cancel_event); + PyThread_free_lock(thread.cancel_event); + thread.cancel_event = NULL; + } + if (thread.running) { + PyThread_free_lock(thread.running); + thread.running = NULL; + } +#endif + +#ifdef FAULTHANDLER_WATCHDOG + /* file watchdog */ + cancel_file_watchdog(); + if (watchdog.cancel_event) { + PyThread_release_lock(watchdog.cancel_event); + PyThread_free_lock(watchdog.cancel_event); + watchdog.cancel_event = NULL; + } + if (watchdog.running) { + PyThread_free_lock(watchdog.running); + watchdog.running = NULL; + } +#endif + +#ifdef FAULTHANDLER_USER + /* user */ + if (user_signals != NULL) { + for (signum=0; signum < NSIG; signum++) + faulthandler_unregister(&user_signals[signum], signum); + free(user_signals); + user_signals = NULL; + } +#endif + + /* fatal */ + faulthandler_disable(); +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + PyMem_Free(stack.ss_sp); + stack.ss_sp = NULL; + } +#endif +} diff --git a/Modules/fpectlmodule.c b/Modules/fpectlmodule.c index 1bb51cf..6af2f82 100644 --- a/Modules/fpectlmodule.c +++ b/Modules/fpectlmodule.c @@ -174,17 +174,6 @@ static void fpe_reset(Sigfunc *handler) fp_enable(TRP_INVALID | TRP_DIV_BY_ZERO | TRP_OVERFLOW); PyOS_setsig(SIGFPE, handler); -/*-- DEC ALPHA OSF --------------------------------------------------------*/ -#elif defined(__alpha) && defined(__osf__) - /* References: exception_intro, ieee man pages */ - /* cc -c -I/usr/local/python/include fpectlmodule.c */ - /* ld -shared -o fpectlmodule.so fpectlmodule.o */ -#include <machine/fpu.h> - unsigned long fp_control = - IEEE_TRAP_ENABLE_INV | IEEE_TRAP_ENABLE_DZE | IEEE_TRAP_ENABLE_OVF; - ieee_set_fp_control(fp_control); - PyOS_setsig(SIGFPE, handler); - /*-- DEC ALPHA LINUX ------------------------------------------------------*/ #elif defined(__alpha) && defined(linux) #include <asm/fpu.h> diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 10a4ed7..2533de6 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -680,8 +680,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) static void debug_cycle(char *msg, PyObject *op) { - PySys_WriteStderr("gc: %.100s <%.100s %p>\n", - msg, Py_TYPE(op)->tp_name, op); + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); } /* Handle uncollectable garbage (cycles with finalizers, and stuff reachable @@ -769,7 +769,9 @@ get_time(void) { double result = 0; if (tmod != NULL) { - PyObject *f = PyObject_CallMethod(tmod, "time", NULL); + _Py_identifier(time); + + PyObject *f = _PyObject_CallMethodId(tmod, &PyId_time, NULL); if (f == NULL) { PyErr_Clear(); } diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 7069b6e..0971a64 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -20,14 +20,6 @@ #endif #endif -/* on unix, SVNVERSION is passed on the command line. - * on Windows, the string is interpolated using - * subwcrev.exe - */ -#ifndef SVNVERSION -#define SVNVERSION "$WCRANGE$$WCMODS?M:$" -#endif - /* XXX Only unix build process has been tested */ #ifndef HGVERSION #define HGVERSION "" @@ -57,16 +49,6 @@ Py_GetBuildInfo(void) } const char * -_Py_svnversion(void) -{ - /* the following string can be modified by subwcrev.exe */ - static const char svnversion[] = SVNVERSION; - if (svnversion[0] != '$') - return svnversion; /* it was interpolated, or passed on command line */ - return "Unversioned directory"; -} - -const char * _Py_hgversion(void) { return HGVERSION; diff --git a/Modules/getpath.c b/Modules/getpath.c index b7f9573..7090879 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -406,7 +406,7 @@ calculate_path(void) static wchar_t delimiter[2] = {DELIM, '\0'}; static wchar_t separator[2] = {SEP, '\0'}; char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ - wchar_t rtpypath[MAXPATHLEN+1]; + wchar_t *rtpypath = NULL; wchar_t *home = Py_GetPythonHome(); char *_path = getenv("PATH"); wchar_t *path_buffer = NULL; @@ -606,12 +606,12 @@ calculate_path(void) bufsz = 0; if (_rtpypath) { - size_t s = mbstowcs(rtpypath, _rtpypath, sizeof(rtpypath)/sizeof(wchar_t)); - if (s == (size_t)-1 || s >=sizeof(rtpypath)) - /* XXX deal with errors more gracefully */ + size_t rtpypath_len; + rtpypath = _Py_char2wchar(_rtpypath, &rtpypath_len); + if (rtpypath != NULL) + bufsz += rtpypath_len + 1; + else _rtpypath = NULL; - if (_rtpypath) - bufsz += wcslen(rtpypath) + 1; } defpath = _pythonpath; @@ -645,7 +645,7 @@ calculate_path(void) } else { /* Run-time value of $PYTHONPATH goes first */ - if (_rtpypath) { + if (rtpypath) { wcscpy(buf, rtpypath); wcscat(buf, delimiter); } @@ -719,6 +719,8 @@ calculate_path(void) PyMem_Free(_pythonpath); PyMem_Free(_prefix); PyMem_Free(_exec_prefix); + if (rtpypath != NULL) + PyMem_Free(rtpypath); } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 8b6fa85..4f81fe2 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -654,7 +654,9 @@ tee(PyObject *self, PyObject *args) copyable = it; PyTuple_SET_ITEM(result, 0, copyable); for (i=1 ; i<n ; i++) { - copyable = PyObject_CallMethod(copyable, "__copy__", NULL); + _Py_identifier(__copy__); + + copyable = _PyObject_CallMethodId(copyable, &PyId___copy__, NULL); if (copyable == NULL) { Py_DECREF(result); return NULL; @@ -2592,6 +2594,7 @@ typedef struct { PyObject_HEAD PyObject *total; PyObject *it; + PyObject *binop; } accumulateobject; static PyTypeObject accumulate_type; @@ -2599,12 +2602,14 @@ static PyTypeObject accumulate_type; static PyObject * accumulate_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - static char *kwargs[] = {"iterable", NULL}; + static char *kwargs[] = {"iterable", "func", NULL}; PyObject *iterable; PyObject *it; + PyObject *binop = NULL; accumulateobject *lz; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:accumulate", kwargs, &iterable)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:accumulate", + kwargs, &iterable, &binop)) return NULL; /* Get iterator. */ @@ -2619,6 +2624,8 @@ accumulate_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + Py_XINCREF(binop); + lz->binop = binop; lz->total = NULL; lz->it = it; return (PyObject *)lz; @@ -2628,6 +2635,7 @@ static void accumulate_dealloc(accumulateobject *lz) { PyObject_GC_UnTrack(lz); + Py_XDECREF(lz->binop); Py_XDECREF(lz->total); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); @@ -2636,6 +2644,7 @@ accumulate_dealloc(accumulateobject *lz) static int accumulate_traverse(accumulateobject *lz, visitproc visit, void *arg) { + Py_VISIT(lz->binop); Py_VISIT(lz->it); Py_VISIT(lz->total); return 0; @@ -2655,8 +2664,11 @@ accumulate_next(accumulateobject *lz) lz->total = val; return lz->total; } - - newtotal = PyNumber_Add(lz->total, val); + + if (lz->binop == NULL) + newtotal = PyNumber_Add(lz->total, val); + else + newtotal = PyObject_CallFunctionObjArgs(lz->binop, lz->total, val, NULL); Py_DECREF(val); if (newtotal == NULL) return NULL; @@ -2670,9 +2682,9 @@ accumulate_next(accumulateobject *lz) } PyDoc_STRVAR(accumulate_doc, -"accumulate(iterable) --> accumulate object\n\ +"accumulate(iterable[, func]) --> accumulate object\n\ \n\ -Return series of accumulated sums."); +Return series of accumulated sums (or other binary function results)."); static PyTypeObject accumulate_type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -3633,7 +3645,7 @@ cycle(p) --> p0, p1, ... plast, p0, p1, ...\n\ repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times\n\ \n\ Iterators terminating on the shortest input sequence:\n\ -accumulate(p, start=0) --> p0, p0+p1, p0+p1+p2\n\ +accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ diff --git a/Modules/main.c b/Modules/main.c index fcd9330..747c12f 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -6,6 +6,7 @@ #include <locale.h> #ifdef __VMS +#error "PEP 11: VMS is now unsupported, code will be removed in Python 3.4" #include <unixlib.h> #endif @@ -100,6 +101,7 @@ static char *usage_5 = " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" ; static int @@ -577,7 +579,6 @@ Py_Main(int argc, wchar_t **argv) if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') { wchar_t* buffer; size_t len = strlen(p); - size_t r; buffer = malloc(len * sizeof(wchar_t)); if (buffer == NULL) { @@ -585,7 +586,7 @@ Py_Main(int argc, wchar_t **argv) "not enough memory to copy PYTHONEXECUTABLE"); } - r = mbstowcs(buffer, p, len); + mbstowcs(buffer, p, len); Py_SetProgramName(buffer); /* buffer is now handed off - do not free */ } else { diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 29c32a3..c4cc46e 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -55,11 +55,6 @@ raised for division by zero and mod by zero. #include "Python.h" #include "_math.h" -#ifdef _OSF_SOURCE -/* OSF1 5.1 doesn't make this available with XOPEN_SOURCE_EXTENDED defined */ -extern double copysign(double, double); -#endif - /* sin(pi*x), giving accurate results for all finite x (especially x integral or close to an integer). This is here for use in the @@ -244,7 +239,8 @@ m_tgamma(double x) } if (x == 0.0) { errno = EDOM; - return 1.0/x; /* tgamma(+-0.0) = +-inf, divide-by-zero */ + /* tgamma(+-0.0) = +-inf, divide-by-zero */ + return copysign(Py_HUGE_VAL, x); } /* integer arguments */ @@ -582,6 +578,61 @@ m_log(double x) } } +/* + log2: log to base 2. + + Uses an algorithm that should: + + (a) produce exact results for powers of 2, and + (b) give a monotonic log2 (for positive finite floats), + assuming that the system log is monotonic. +*/ + +static double +m_log2(double x) +{ + if (!Py_IS_FINITE(x)) { + if (Py_IS_NAN(x)) + return x; /* log2(nan) = nan */ + else if (x > 0.0) + return x; /* log2(+inf) = +inf */ + else { + errno = EDOM; + return Py_NAN; /* log2(-inf) = nan, invalid-operation */ + } + } + + if (x > 0.0) { +#ifdef HAVE_LOG2 + return log2(x); +#else + double m; + int e; + m = frexp(x, &e); + /* We want log2(m * 2**e) == log(m) / log(2) + e. Care is needed when + * x is just greater than 1.0: in that case e is 1, log(m) is negative, + * and we get significant cancellation error from the addition of + * log(m) / log(2) to e. The slight rewrite of the expression below + * avoids this problem. + */ + if (x >= 1.0) { + return log(2.0 * m) / log(2.0) + (e - 1); + } + else { + return log(m) / log(2.0) + e; + } +#endif + } + else if (x == 0.0) { + errno = EDOM; + return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + } + else { + errno = EDOM; + return Py_NAN; /* log2(-inf) = nan, invalid-operation */ + } +} + static double m_log10(double x) { @@ -1384,7 +1435,7 @@ math_factorial(PyObject *self, PyObject *arg) } /* use lookup table if x is small */ - if (x < (long)(sizeof(SmallFactorials)/sizeof(SmallFactorials[0]))) + if (x < (long)Py_ARRAY_LENGTH(SmallFactorials)) return PyLong_FromUnsignedLong(SmallFactorials[x]); /* else express in the form odd_part * 2**two_valuation, and compute as @@ -1628,6 +1679,15 @@ Return the logarithm of x to the given base.\n\ If the base not specified, returns the natural logarithm (base e) of x."); static PyObject * +math_log2(PyObject *self, PyObject *arg) +{ + return loghelper(arg, m_log2, "log2"); +} + +PyDoc_STRVAR(math_log2_doc, +"log2(x)\n\nReturn the base 2 logarithm of x."); + +static PyObject * math_log10(PyObject *self, PyObject *arg) { return loghelper(arg, m_log10, "log10"); @@ -1899,6 +1959,7 @@ static PyMethodDef math_methods[] = { {"log", math_log, METH_VARARGS, math_log_doc}, {"log1p", math_log1p, METH_O, math_log1p_doc}, {"log10", math_log10, METH_O, math_log10_doc}, + {"log2", math_log2, METH_O, math_log2_doc}, {"modf", math_modf, METH_O, math_modf_doc}, {"pow", math_pow, METH_VARARGS, math_pow_doc}, {"radians", math_radians, METH_O, math_radians_doc}, diff --git a/Modules/md5module.c b/Modules/md5module.c index 208930d..86f602e 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -243,7 +243,7 @@ void md5_process(struct md5_state *md5, in += MD5_BLOCKSIZE; inlen -= MD5_BLOCKSIZE; } else { - n = MIN(inlen, (MD5_BLOCKSIZE - md5->curlen)); + n = MIN(inlen, (Py_ssize_t)(MD5_BLOCKSIZE - md5->curlen)); memcpy(md5->buf + md5->curlen, in, (size_t)n); md5->curlen += n; in += n; @@ -376,7 +376,7 @@ MD5_hexdigest(MD5object *self, PyObject *unused) unsigned char digest[MD5_DIGESTSIZE]; struct md5_state temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -384,24 +384,18 @@ MD5_hexdigest(MD5object *self, PyObject *unused) md5_done(&temp, digest); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, MD5_DIGESTSIZE * 2); + retval = PyUnicode_New(MD5_DIGESTSIZE * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<MD5_DIGESTSIZE; i++) { - char c; + unsigned char c; c = (digest[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; c = (digest[i] & 0xf); - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; } return retval; } diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index da9283b..35d61ba 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -78,8 +78,6 @@ my_getpagesize(void) # define MAP_ANONYMOUS MAP_ANON #endif -static PyObject *mmap_module_error; - typedef enum { ACCESS_DEFAULT, @@ -240,15 +238,37 @@ mmap_read_line_method(mmap_object *self, return result; } +/* Basically the "n" format code with the ability to turn None into -1. */ +static int +mmap_convert_ssize_t(PyObject *obj, void *result) { + Py_ssize_t limit; + if (obj == Py_None) { + limit = -1; + } + else if (PyNumber_Check(obj)) { + limit = PyNumber_AsSsize_t(obj, PyExc_OverflowError); + if (limit == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_Format(PyExc_TypeError, + "integer argument expected, got '%.200s'", + Py_TYPE(obj)->tp_name); + return 0; + } + *((Py_ssize_t *)result) = limit; + return 1; +} + static PyObject * mmap_read_method(mmap_object *self, PyObject *args) { - Py_ssize_t num_bytes, n; + Py_ssize_t num_bytes = -1, n; PyObject *result; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n:read", &num_bytes)) + if (!PyArg_ParseTuple(args, "|O&:read", mmap_convert_ssize_t, &num_bytes)) return(NULL); /* silently 'adjust' out-of-range requests */ @@ -437,7 +457,7 @@ mmap_size_method(mmap_object *self, { struct stat buf; if (-1 == fstat(self->fd, &buf)) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } #ifdef HAVE_LARGEFILE_SUPPORT @@ -527,7 +547,7 @@ mmap_resize_method(mmap_object *self, void *newmap; if (ftruncate(self->fd, self->offset + new_size) == -1) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -542,7 +562,7 @@ mmap_resize_method(mmap_object *self, #endif if (newmap == (void *)-1) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } self->data = newmap; @@ -583,7 +603,7 @@ mmap_flush_method(mmap_object *self, PyObject *args) /* XXX semantics of return value? */ /* XXX flags for msync? */ if (-1 == msync(self->data + offset, size, MS_SYNC)) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } return PyLong_FromLong(0); @@ -645,9 +665,9 @@ mmap_move_method(mmap_object *self, PyObject *args) return NULL; } else { /* bounds check the values */ - if (cnt < 0 || (cnt + dest) < cnt || (cnt + src) < cnt || - src < 0 || src > self->size || (src + cnt) > self->size || - dest < 0 || dest > self->size || (dest + cnt) > self->size) { + if ((cnt + dest) < cnt || (cnt + src) < cnt || + src > self->size || (src + cnt) > self->size || + dest > self->size || (dest + cnt) > self->size) { PyErr_SetString(PyExc_ValueError, "source, destination, or count out of range"); return NULL; @@ -680,7 +700,9 @@ mmap__enter__method(mmap_object *self, PyObject *args) static PyObject * mmap__exit__method(PyObject *self, PyObject *args) { - return PyObject_CallMethod(self, "close", NULL); + _Py_identifier(close); + + return _PyObject_CallMethodId(self, &PyId_close, NULL); } static struct PyMethodDef mmap_object_methods[] = { @@ -1181,7 +1203,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) fd = devzero = open("/dev/zero", O_RDWR); if (devzero == -1) { Py_DECREF(m_obj); - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } #endif @@ -1189,7 +1211,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->fd = dup(fd); if (m_obj->fd == -1) { Py_DECREF(m_obj); - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } } @@ -1205,7 +1227,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (m_obj->data == (char *)-1) { m_obj->data = NULL; Py_DECREF(m_obj); - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } m_obj->access = (access_mode)access; @@ -1286,12 +1308,12 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) if (fileno != -1 && fileno != 0) { /* Ensure that fileno is within the CRT's valid range */ if (_PyVerify_fd(fileno) == 0) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } fh = (HANDLE)_get_osfhandle(fileno); if (fh==(HANDLE)-1) { - PyErr_SetFromErrno(mmap_module_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } /* Win9x appears to need us seeked to zero */ @@ -1445,11 +1467,7 @@ PyInit_mmap(void) dict = PyModule_GetDict(module); if (!dict) return NULL; - mmap_module_error = PyErr_NewException("mmap.error", - PyExc_EnvironmentError , NULL); - if (mmap_module_error == NULL) - return NULL; - PyDict_SetItemString(dict, "error", mmap_module_error); + PyDict_SetItemString(dict, "error", PyExc_OSError); PyDict_SetItemString(dict, "mmap", (PyObject*) &mmap_object_type); #ifdef PROT_EXEC setint(dict, "PROT_EXEC", PROT_EXEC); diff --git a/Modules/nismodule.c b/Modules/nismodule.c index a81ca8c..0af495f 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -411,7 +411,7 @@ nis_maps (PyObject *self, PyObject *args, PyObject *kwdict) return NULL; if ((list = PyList_New(0)) == NULL) return NULL; - for (maps = maps; maps; maps = maps->next) { + for (; maps; maps = maps->next) { PyObject *str = PyUnicode_FromString(maps->map); if (!str || PyList_Append(list, str) < 0) { diff --git a/Modules/operator.c b/Modules/operator.c index 866ec3a..2f47573 100644 --- a/Modules/operator.c +++ b/Modules/operator.c @@ -402,7 +402,8 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) for (idx = 0; idx < nattrs; ++idx) { PyObject *item = PyTuple_GET_ITEM(args, idx); Py_ssize_t item_len; - Py_UNICODE *item_buffer; + void *data; + unsigned int kind; int dot_count; if (!PyUnicode_Check(item)) { @@ -411,13 +412,18 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(attr); return NULL; } - item_len = PyUnicode_GET_SIZE(item); - item_buffer = PyUnicode_AS_UNICODE(item); + if (PyUnicode_READY(item)) { + Py_DECREF(attr); + return NULL; + } + item_len = PyUnicode_GET_LENGTH(item); + kind = PyUnicode_KIND(item); + data = PyUnicode_DATA(item); /* check whethere the string is dotted */ dot_count = 0; for (char_idx = 0; char_idx < item_len; ++char_idx) { - if (item_buffer[char_idx] == (Py_UNICODE)'.') + if (PyUnicode_READ(kind, data, char_idx) == '.') ++dot_count; } @@ -438,12 +444,12 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } for (; dot_count > 0; --dot_count) { - while (item_buffer[unibuff_till] != (Py_UNICODE)'.') { + while (PyUnicode_READ(kind, data, unibuff_till) != '.') { ++unibuff_till; } - attr_chain_item = PyUnicode_FromUnicode( - item_buffer + unibuff_from, - unibuff_till - unibuff_from); + attr_chain_item = PyUnicode_Substring(item, + unibuff_from, + unibuff_till); if (attr_chain_item == NULL) { Py_DECREF(attr_chain); Py_DECREF(attr); @@ -456,9 +462,8 @@ attrgetter_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /* now add the last dotless name */ - attr_chain_item = PyUnicode_FromUnicode( - item_buffer + unibuff_from, - item_len - unibuff_from); + attr_chain_item = PyUnicode_Substring(item, + unibuff_from, item_len); if (attr_chain_item == NULL) { Py_DECREF(attr_chain); Py_DECREF(attr); diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index b38ce52..7512c1d 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -214,6 +214,21 @@ oss_mixer_dealloc(oss_mixer_t *self) * Helper functions */ +/* Check if a given file descriptor is valid (i.e. hasn't been closed). + * If true, return 1. Otherwise, raise ValueError and return 0. + */ +static int _is_fd_valid(int fd) +{ + /* the FD is set to -1 in oss_close()/oss_mixer_close() */ + if (fd >= 0) { + return 1; + } else { + PyErr_SetString(PyExc_ValueError, + "Operation on closed OSS device."); + return 0; + } +} + /* _do_ioctl_1() is a private helper function used for the OSS ioctls -- SNDCTL_DSP_{SETFMT,CHANNELS,SPEED} -- that that are called from C like this: @@ -301,6 +316,9 @@ _do_ioctl_0(int fd, PyObject *args, char *fname, int cmd) static PyObject * oss_nonblock(oss_audio_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + /* Hmmm: it doesn't appear to be possible to return to blocking mode once we're in non-blocking mode! */ if (ioctl(self->fd, SNDCTL_DSP_NONBLOCK, NULL) == -1) @@ -312,6 +330,9 @@ oss_nonblock(oss_audio_t *self, PyObject *unused) static PyObject * oss_setfmt(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "setfmt", SNDCTL_DSP_SETFMT); } @@ -319,6 +340,10 @@ static PyObject * oss_getfmts(oss_audio_t *self, PyObject *unused) { int mask; + + if (!_is_fd_valid(self->fd)) + return NULL; + if (ioctl(self->fd, SNDCTL_DSP_GETFMTS, &mask) == -1) return PyErr_SetFromErrno(PyExc_IOError); return PyLong_FromLong(mask); @@ -327,30 +352,45 @@ oss_getfmts(oss_audio_t *self, PyObject *unused) static PyObject * oss_channels(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "channels", SNDCTL_DSP_CHANNELS); } static PyObject * oss_speed(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "speed", SNDCTL_DSP_SPEED); } static PyObject * oss_sync(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "sync", SNDCTL_DSP_SYNC); } static PyObject * oss_reset(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "reset", SNDCTL_DSP_RESET); } static PyObject * oss_post(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "post", SNDCTL_DSP_POST); } @@ -365,6 +405,9 @@ oss_read(oss_audio_t *self, PyObject *args) char *cp; PyObject *rv; + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "i:read", &size)) return NULL; rv = PyBytes_FromStringAndSize(NULL, size); @@ -392,6 +435,9 @@ oss_write(oss_audio_t *self, PyObject *args) char *cp; int rv, size; + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "y#:write", &cp, &size)) { return NULL; } @@ -423,6 +469,9 @@ oss_writeall(oss_audio_t *self, PyObject *args) mode, the behaviour of write() and writeall() from Python is indistinguishable. */ + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "y#:write", &cp, &size)) return NULL; @@ -482,10 +531,12 @@ oss_self(PyObject *self, PyObject *unused) return self; } -static PyObject * +static PyObject * oss_exit(PyObject *self, PyObject *unused) { - PyObject *ret = PyObject_CallMethod(self, "close", NULL); + _Py_identifier(close); + + PyObject *ret = _PyObject_CallMethodId(self, &PyId_close, NULL); if (!ret) return NULL; Py_DECREF(ret); @@ -495,6 +546,9 @@ oss_exit(PyObject *self, PyObject *unused) static PyObject * oss_fileno(oss_audio_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + return PyLong_FromLong(self->fd); } @@ -509,6 +563,9 @@ oss_setparameters(oss_audio_t *self, PyObject *args) int fmt, channels, rate; PyObject * rv; /* return tuple (fmt, channels, rate) */ + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "iii|i:setparameters", &wanted_fmt, &wanted_channels, &wanted_rate, &strict)) @@ -599,6 +656,9 @@ oss_bufsize(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -618,6 +678,9 @@ oss_obufcount(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -638,6 +701,9 @@ oss_obuffree(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -655,6 +721,9 @@ oss_getptr(oss_audio_t *self, PyObject *unused) count_info info; int req; + if (!_is_fd_valid(self->fd)) + return NULL; + if (self->mode == O_RDONLY) req = SNDCTL_DSP_GETIPTR; else @@ -685,6 +754,9 @@ oss_mixer_close(oss_mixer_t *self, PyObject *unused) static PyObject * oss_mixer_fileno(oss_mixer_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + return PyLong_FromLong(self->fd); } @@ -693,6 +765,9 @@ oss_mixer_fileno(oss_mixer_t *self, PyObject *unused) static PyObject * oss_mixer_controls(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "controls", SOUND_MIXER_READ_DEVMASK); } @@ -700,6 +775,9 @@ oss_mixer_controls(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_stereocontrols(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "stereocontrols", SOUND_MIXER_READ_STEREODEVS); } @@ -707,6 +785,9 @@ oss_mixer_stereocontrols(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_reccontrols(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "reccontrols", SOUND_MIXER_READ_RECMASK); } @@ -716,6 +797,9 @@ oss_mixer_get(oss_mixer_t *self, PyObject *args) { int channel, volume; + if (!_is_fd_valid(self->fd)) + return NULL; + /* Can't use _do_ioctl_1 because of encoded arg thingy. */ if (!PyArg_ParseTuple(args, "i:get", &channel)) return NULL; @@ -736,6 +820,9 @@ oss_mixer_set(oss_mixer_t *self, PyObject *args) { int channel, volume, leftVol, rightVol; + if (!_is_fd_valid(self->fd)) + return NULL; + /* Can't use _do_ioctl_1 because of encoded arg thingy. */ if (!PyArg_ParseTuple(args, "i(ii):set", &channel, &leftVol, &rightVol)) return NULL; @@ -761,6 +848,9 @@ oss_mixer_set(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_get_recsrc(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "get_recsrc", SOUND_MIXER_READ_RECSRC); } @@ -768,6 +858,9 @@ oss_mixer_get_recsrc(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_set_recsrc(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "set_recsrc", SOUND_MIXER_WRITE_RECSRC); } @@ -971,8 +1064,8 @@ build_namelists (PyObject *module) int num_controls; int i; - num_controls = sizeof(control_labels) / sizeof(control_labels[0]); - assert(num_controls == sizeof(control_names) / sizeof(control_names[0])); + num_controls = Py_ARRAY_LENGTH(control_labels); + assert(num_controls == Py_ARRAY_LENGTH(control_names)); labels = PyList_New(num_controls); names = PyList_New(num_controls); diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 3cdf135..3f03cee 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -581,8 +581,10 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) if (res) ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; } - else + else { PyParser_SetError(&err); + } + PyParser_ClearError(&err); } return (res); } @@ -3239,10 +3241,13 @@ PyInit_parser(void) copyreg = PyImport_ImportModuleNoBlock("copyreg"); if (copyreg != NULL) { PyObject *func, *pickler; + _Py_identifier(pickle); + _Py_identifier(sequence2st); + _Py_identifier(_pickler); - func = PyObject_GetAttrString(copyreg, "pickle"); - pickle_constructor = PyObject_GetAttrString(module, "sequence2st"); - pickler = PyObject_GetAttrString(module, "_pickler"); + func = _PyObject_GetAttrId(copyreg, &PyId_pickle); + pickle_constructor = _PyObject_GetAttrId(module, &PyId_sequence2st); + pickler = _PyObject_GetAttrId(module, &PyId__pickler); Py_XINCREF(pickle_constructor); if ((func != NULL) && (pickle_constructor != NULL) && (pickler != NULL)) { diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 702fec0..da0bef0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -28,6 +28,7 @@ #include "Python.h" #if defined(__VMS) +# error "PEP 11: VMS is now unsupported, code will be removed in Python 3.4" # include <unixio.h> #endif /* defined(__VMS) */ @@ -43,6 +44,7 @@ corresponding Unix manual entries for more information on calls."); #if defined(PYOS_OS2) +#error "PEP 11: OS/2 is now unsupported, code will be removed in Python 3.4" #define INCL_DOS #define INCL_DOSERRORS #define INCL_DOSPROCESS @@ -57,6 +59,10 @@ corresponding Unix manual entries for more information on calls."); #include "osdefs.h" #endif +#ifdef HAVE_SYS_UIO_H +#include <sys/uio.h> +#endif + #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif /* HAVE_SYS_TYPES_H */ @@ -93,6 +99,28 @@ corresponding Unix manual entries for more information on calls."); #include <langinfo.h> #endif +#ifdef HAVE_SYS_SENDFILE_H +#include <sys/sendfile.h> +#endif + +#ifdef HAVE_SCHED_H +#include <sched.h> +#endif + +#if defined(HAVE_SYS_XATTR_H) && defined(__GLIBC__) +#define USE_XATTRS +#endif + +#ifdef USE_XATTRS +#include <sys/xattr.h> +#endif + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif +#endif + /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ #if defined(PYCC_VACPP) && defined(PYOS_OS2) @@ -347,6 +375,22 @@ static int win32_can_symlink = 0; #endif #endif +/* A helper used by a number of POSIX-only functions */ +#ifndef MS_WINDOWS +static int +_parse_off_t(PyObject* arg, void* addr) +{ +#if !defined(HAVE_LARGEFILE_SUPPORT) + *((off_t*)addr) = PyLong_AsLong(arg); +#else + *((off_t*)addr) = PyLong_AsLongLong(arg); +#endif + if (PyErr_Occurred()) + return 0; + return 1; +} +#endif + #if defined _MSC_VER && _MSC_VER >= 1400 /* Microsoft CRT in VS2005 and higher will verify that a filehandle is * valid and throw an assertion if it isn't. @@ -653,7 +697,7 @@ win32_error(char* function, const char* filename) } static PyObject * -win32_error_unicode(char* function, Py_UNICODE* filename) +win32_error_unicode(char* function, wchar_t* filename) { /* XXX - see win32_error for comments on 'function' */ errno = GetLastError(); @@ -663,6 +707,20 @@ win32_error_unicode(char* function, Py_UNICODE* filename) return PyErr_SetFromWindowsErr(errno); } +static PyObject * +win32_error_object(char* function, PyObject* filename) +{ + /* XXX - see win32_error for comments on 'function' */ + errno = GetLastError(); + if (filename) + return PyErr_SetExcFromWindowsErrWithFilenameObject( + PyExc_WindowsError, + errno, + filename); + else + return PyErr_SetFromWindowsErr(errno); +} + static int convert_to_unicode(PyObject **param) { @@ -671,8 +729,7 @@ convert_to_unicode(PyObject **param) else if (PyUnicode_Check(*param)) /* For a Unicode subtype that's not a Unicode object, return a true Unicode object with the same data. */ - *param = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(*param), - PyUnicode_GET_SIZE(*param)); + *param = PyUnicode_Copy(*param); else *param = PyUnicode_FromEncodedObject(*param, Py_FileSystemDefaultEncoding, @@ -837,17 +894,21 @@ win32_1str(PyObject* args, char* func, char *ansi; BOOL result; - if (!PyArg_ParseTuple(args, wformat, &uni)) - PyErr_Clear(); - else { + if (PyArg_ParseTuple(args, wformat, &uni)) + { + wchar_t *wstr = PyUnicode_AsUnicode(uni); + if (wstr == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - result = funcW(PyUnicode_AsUnicode(uni)); + result = funcW(wstr); Py_END_ALLOW_THREADS if (!result) - return win32_error_unicode(func, PyUnicode_AsUnicode(uni)); + return win32_error_object(func, uni); Py_INCREF(Py_None); return Py_None; } + PyErr_Clear(); + if (!PyArg_ParseTuple(args, format, &ansi)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -1544,9 +1605,39 @@ static PyStructSequence_Desc statvfs_result_desc = { 10 }; +#if defined(HAVE_WAITID) && !defined(__APPLE__) +PyDoc_STRVAR(waitid_result__doc__, +"waitid_result: Result from waitid.\n\n\ +This object may be accessed either as a tuple of\n\ + (si_pid, si_uid, si_signo, si_status, si_code),\n\ +or via the attributes si_pid, si_uid, and so on.\n\ +\n\ +See os.waitid for more information."); + +static PyStructSequence_Field waitid_result_fields[] = { + {"si_pid", }, + {"si_uid", }, + {"si_signo", }, + {"si_status", }, + {"si_code", }, + {0} +}; + +static PyStructSequence_Desc waitid_result_desc = { + "waitid_result", /* name */ + waitid_result__doc__, /* doc */ + waitid_result_fields, + 5 +}; +static PyTypeObject WaitidResultType; +#endif + static int initialized; static PyTypeObject StatResultType; static PyTypeObject StatVFSResultType; +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) +static PyTypeObject SchedParamType; +#endif static newfunc structseq_new; static PyObject * @@ -1727,7 +1818,7 @@ posix_do_stat(PyObject *self, PyObject *args, int (*statfunc)(const char *, STRUCT_STAT *), #endif char *wformat, - int (*wstatfunc)(const Py_UNICODE *, STRUCT_STAT *)) + int (*wstatfunc)(const wchar_t *, STRUCT_STAT *)) { STRUCT_STAT st; PyObject *opath; @@ -1736,18 +1827,18 @@ posix_do_stat(PyObject *self, PyObject *args, PyObject *result; #ifdef MS_WINDOWS - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, wformat, &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE result OK without - thread lock as it is a simple dereference. */ res = wstatfunc(wpath, &st); Py_END_ALLOW_THREADS if (res != 0) - return win32_error_unicode("stat", wpath); + return win32_error_object("stat", po); return _pystat_fromstructstat(&st); } /* Drop the argument parsing error as narrow strings @@ -1796,12 +1887,13 @@ posix_access(PyObject *self, PyObject *args) #ifdef MS_WINDOWS DWORD attr; - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui:access", &po, &mode)) { + wchar_t* wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ - attr = GetFileAttributesW(PyUnicode_AS_UNICODE(po)); + attr = GetFileAttributesW(wpath); Py_END_ALLOW_THREADS goto finish; } @@ -1951,23 +2043,25 @@ posix_chmod(PyObject *self, PyObject *args) int res; #ifdef MS_WINDOWS DWORD attr; - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui|:chmod", &po, &i)) { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS - attr = GetFileAttributesW(PyUnicode_AS_UNICODE(po)); + attr = GetFileAttributesW(wpath); if (attr != 0xFFFFFFFF) { if (i & _S_IWRITE) attr &= ~FILE_ATTRIBUTE_READONLY; else attr |= FILE_ATTRIBUTE_READONLY; - res = SetFileAttributesW(PyUnicode_AS_UNICODE(po), attr); + res = SetFileAttributesW(wpath, attr); } else res = 0; Py_END_ALLOW_THREADS if (!res) - return win32_error_unicode("chmod", - PyUnicode_AS_UNICODE(po)); + return win32_error_object("chmod", po); Py_INCREF(Py_None); return Py_None; } @@ -2143,6 +2237,21 @@ posix_fsync(PyObject *self, PyObject *fdobj) } #endif /* HAVE_FSYNC */ +#ifdef HAVE_SYNC +PyDoc_STRVAR(posix_sync__doc__, +"sync()\n\n\ +Force write of everything to disk."); + +static PyObject * +posix_sync(PyObject *self, PyObject *noargs) +{ + Py_BEGIN_ALLOW_THREADS + sync(); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_FDATASYNC #ifdef __hpux @@ -2340,12 +2449,20 @@ win32_link(PyObject *self, PyObject *args) PyObject *osrc, *odst; char *src, *dst; BOOL rslt; + PyObject *usrc, *udst; + + if (PyArg_ParseTuple(args, "UU:link", &usrc, &udst)) + { + wchar_t *wsrc, *wdst; + wsrc = PyUnicode_AsUnicode(usrc); + if (wsrc == NULL) + return NULL; + wdst = PyUnicode_AsUnicode(udst); + if (wdst == NULL) + return NULL; - PyUnicodeObject *usrc, *udst; - if (PyArg_ParseTuple(args, "UU:link", &usrc, &udst)) { Py_BEGIN_ALLOW_THREADS - rslt = CreateHardLinkW(PyUnicode_AS_UNICODE(udst), - PyUnicode_AS_UNICODE(usrc), NULL); + rslt = CreateHardLinkW(wdst, wsrc, NULL); Py_END_ALLOW_THREADS if (rslt == 0) @@ -2406,14 +2523,15 @@ posix_listdir(PyObject *self, PyObject *args) PyObject *po = NULL; if (PyArg_ParseTuple(args, "|U:listdir", &po)) { WIN32_FIND_DATAW wFileData; - Py_UNICODE *wnamebuf, *po_wchars; + wchar_t *wnamebuf, *po_wchars; if (po == NULL) { /* Default arg: "." */ po_wchars = L"."; len = 1; } else { - po_wchars = PyUnicode_AS_UNICODE(po); - len = PyUnicode_GET_SIZE(po); + po_wchars = PyUnicode_AsUnicodeAndSize(po, &len); + if (po_wchars == NULL) + return NULL; } /* Overallocate for \\*.*\0 */ wnamebuf = malloc((len + 5) * sizeof(wchar_t)); @@ -2423,7 +2541,7 @@ posix_listdir(PyObject *self, PyObject *args) } wcscpy(wnamebuf, po_wchars); if (len > 0) { - Py_UNICODE wch = wnamebuf[len-1]; + wchar_t wch = wnamebuf[len-1]; if (wch != L'/' && wch != L'\\' && wch != L':') wnamebuf[len++] = L'\\'; wcscpy(wnamebuf + len, L"*.*"); @@ -2718,6 +2836,76 @@ posix_listdir(PyObject *self, PyObject *args) #endif /* which OS */ } /* end of posix_listdir */ +#ifdef HAVE_FDOPENDIR +PyDoc_STRVAR(posix_fdlistdir__doc__, +"fdlistdir(fd) -> list_of_strings\n\n\ +Like listdir(), but uses a file descriptor instead.\n\ +After succesful execution of this function, fd will be closed."); + +static PyObject * +posix_fdlistdir(PyObject *self, PyObject *args) +{ + PyObject *d, *v; + DIR *dirp; + struct dirent *ep; + int fd; + + errno = 0; + if (!PyArg_ParseTuple(args, "i:fdlistdir", &fd)) + return NULL; + Py_BEGIN_ALLOW_THREADS + dirp = fdopendir(fd); + Py_END_ALLOW_THREADS + if (dirp == NULL) { + close(fd); + return posix_error(); + } + if ((d = PyList_New(0)) == NULL) { + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + return NULL; + } + for (;;) { + errno = 0; + Py_BEGIN_ALLOW_THREADS + ep = readdir(dirp); + Py_END_ALLOW_THREADS + if (ep == NULL) { + if (errno == 0) { + break; + } else { + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + Py_DECREF(d); + return posix_error(); + } + } + if (ep->d_name[0] == '.' && + (NAMLEN(ep) == 1 || + (ep->d_name[1] == '.' && NAMLEN(ep) == 2))) + continue; + v = PyUnicode_DecodeFSDefaultAndSize(ep->d_name, NAMLEN(ep)); + if (v == NULL) { + Py_CLEAR(d); + break; + } + if (PyList_Append(d, v) != 0) { + Py_DECREF(v); + Py_CLEAR(d); + break; + } + Py_DECREF(v); + } + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + + return d; +} +#endif + #ifdef MS_WINDOWS /* A helper function for abspath on win32 */ static PyObject * @@ -2728,18 +2916,24 @@ posix__getfullpathname(PyObject *self, PyObject *args) char outbuf[MAX_PATH*2]; char *temp; #ifdef MS_WINDOWS - PyUnicodeObject *po; - if (PyArg_ParseTuple(args, "U|:_getfullpathname", &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); - Py_UNICODE woutbuf[MAX_PATH*2], *woutbufp = woutbuf; - Py_UNICODE *wtemp; + PyObject *po; + + if (PyArg_ParseTuple(args, "U|:_getfullpathname", &po)) + { + wchar_t *wpath; + wchar_t woutbuf[MAX_PATH*2], *woutbufp = woutbuf; + wchar_t *wtemp; DWORD result; PyObject *v; + + wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; result = GetFullPathNameW(wpath, - sizeof(woutbuf)/sizeof(woutbuf[0]), + Py_ARRAY_LENGTH(woutbuf), woutbuf, &wtemp); - if (result > sizeof(woutbuf)/sizeof(woutbuf[0])) { - woutbufp = malloc(result * sizeof(Py_UNICODE)); + if (result > Py_ARRAY_LENGTH(woutbuf)) { + woutbufp = malloc(result * sizeof(wchar_t)); if (!woutbufp) return PyErr_NoMemory(); result = GetFullPathNameW(wpath, result, woutbufp, &wtemp); @@ -2747,7 +2941,7 @@ posix__getfullpathname(PyObject *self, PyObject *args) if (result) v = PyUnicode_FromUnicode(woutbufp, wcslen(woutbufp)); else - v = win32_error_unicode("GetFullPathNameW", wpath); + v = win32_error_object("GetFullPathNameW", po); if (woutbufp != woutbuf) free(woutbufp); return v; @@ -2755,13 +2949,13 @@ posix__getfullpathname(PyObject *self, PyObject *args) /* Drop the argument parsing error as narrow strings are also valid. */ PyErr_Clear(); - #endif + if (!PyArg_ParseTuple (args, "O&:_getfullpathname", PyUnicode_FSConverter, &opath)) return NULL; path = PyBytes_AsString(opath); - if (!GetFullPathName(path, sizeof(outbuf)/sizeof(outbuf[0]), + if (!GetFullPathName(path, Py_ARRAY_LENGTH(outbuf), outbuf, &temp)) { win32_error("GetFullPathName", path); Py_DECREF(opath); @@ -2785,12 +2979,14 @@ posix__getfinalpathname(PyObject *self, PyObject *args) int buf_size; wchar_t *target_path; int result_length; - PyObject *result; + PyObject *po, *result; wchar_t *path; - if (!PyArg_ParseTuple(args, "u|:_getfinalpathname", &path)) { + if (!PyArg_ParseTuple(args, "U|:_getfinalpathname", &po)) + return NULL; + path = PyUnicode_AsUnicode(po); + if (path == NULL) return NULL; - } if(!check_GetFinalPathNameByHandle()) { /* If the OS doesn't have GetFinalPathNameByHandle, return a @@ -2809,18 +3005,15 @@ posix__getfinalpathname(PyObject *self, PyObject *args) FILE_FLAG_BACKUP_SEMANTICS, NULL); - if(hFile == INVALID_HANDLE_VALUE) { - return win32_error_unicode("GetFinalPathNamyByHandle", path); - return PyErr_Format(PyExc_RuntimeError, - "Could not get a handle to file."); - } + if(hFile == INVALID_HANDLE_VALUE) + return win32_error_object("CreateFileW", po); /* We have a good handle to the target, use it to determine the target path name. */ buf_size = Py_GetFinalPathNameByHandleW(hFile, 0, 0, VOLUME_NAME_NT); if(!buf_size) - return win32_error_unicode("GetFinalPathNameByHandle", path); + return win32_error_object("GetFinalPathNameByHandle", po); target_path = (wchar_t *)malloc((buf_size+1)*sizeof(wchar_t)); if(!target_path) @@ -2829,10 +3022,10 @@ posix__getfinalpathname(PyObject *self, PyObject *args) result_length = Py_GetFinalPathNameByHandleW(hFile, target_path, buf_size, VOLUME_NAME_DOS); if(!result_length) - return win32_error_unicode("GetFinalPathNamyByHandle", path); + return win32_error_object("GetFinalPathNamyByHandle", po); if(!CloseHandle(hFile)) - return win32_error_unicode("GetFinalPathNameByHandle", path); + return win32_error_object("CloseHandle", po); target_path[result_length] = 0; result = PyUnicode_FromUnicode(target_path, result_length); @@ -2874,11 +3067,13 @@ posix__isdir(PyObject *self, PyObject *args) { PyObject *opath; char *path; - PyUnicodeObject *po; + PyObject *po; DWORD attributes; if (PyArg_ParseTuple(args, "U|:_isdir", &po)) { - Py_UNICODE *wpath = PyUnicode_AS_UNICODE(po); + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; attributes = GetFileAttributesW(wpath); if (attributes == INVALID_FILE_ATTRIBUTES) @@ -2919,15 +3114,18 @@ posix_mkdir(PyObject *self, PyObject *args) int mode = 0777; #ifdef MS_WINDOWS - PyUnicodeObject *po; - if (PyArg_ParseTuple(args, "U|i:mkdir", &po, &mode)) { + PyObject *po; + if (PyArg_ParseTuple(args, "U|i:mkdir", &po, &mode)) + { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; + Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ - res = CreateDirectoryW(PyUnicode_AS_UNICODE(po), NULL); + res = CreateDirectoryW(wpath, NULL); Py_END_ALLOW_THREADS if (!res) - return win32_error_unicode("mkdir", PyUnicode_AS_UNICODE(po)); + return win32_error_object("mkdir", po); Py_INCREF(Py_None); return Py_None; } @@ -2939,8 +3137,6 @@ posix_mkdir(PyObject *self, PyObject *args) return NULL; path = PyBytes_AsString(opath); Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread lock as - it is a simple dereference. */ res = CreateDirectoryA(path, NULL); Py_END_ALLOW_THREADS if (!res) { @@ -3015,6 +3211,48 @@ posix_nice(PyObject *self, PyObject *args) } #endif /* HAVE_NICE */ + +#ifdef HAVE_GETPRIORITY +PyDoc_STRVAR(posix_getpriority__doc__, +"getpriority(which, who) -> current_priority\n\n\ +Get program scheduling priority."); + +static PyObject * +posix_getpriority(PyObject *self, PyObject *args) +{ + int which, who, retval; + + if (!PyArg_ParseTuple(args, "ii", &which, &who)) + return NULL; + errno = 0; + retval = getpriority(which, who); + if (errno != 0) + return posix_error(); + return PyLong_FromLong((long)retval); +} +#endif /* HAVE_GETPRIORITY */ + + +#ifdef HAVE_SETPRIORITY +PyDoc_STRVAR(posix_setpriority__doc__, +"setpriority(which, who, prio) -> None\n\n\ +Set program scheduling priority."); + +static PyObject * +posix_setpriority(PyObject *self, PyObject *args) +{ + int which, who, prio, retval; + + if (!PyArg_ParseTuple(args, "iii", &which, &who, &prio)) + return NULL; + retval = setpriority(which, who, prio); + if (retval == -1) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_SETPRIORITY */ + + PyDoc_STRVAR(posix_rename__doc__, "rename(old, new)\n\n\ Rename a file or directory."); @@ -3024,6 +3262,7 @@ posix_rename(PyObject *self, PyObject *args) { #ifdef MS_WINDOWS PyObject *o1, *o2; + wchar_t *w1, *w2; char *p1, *p2; BOOL result; if (!PyArg_ParseTuple(args, "OO:rename", &o1, &o2)) @@ -3034,9 +3273,14 @@ posix_rename(PyObject *self, PyObject *args) Py_DECREF(o1); goto error; } + w1 = PyUnicode_AsUnicode(o1); + if (w1 == NULL) + goto error; + w2 = PyUnicode_AsUnicode(o2); + if (w2 == NULL) + goto error; Py_BEGIN_ALLOW_THREADS - result = MoveFileW(PyUnicode_AsUnicode(o1), - PyUnicode_AsUnicode(o2)); + result = MoveFileW(w1, w2); Py_END_ALLOW_THREADS Py_DECREF(o1); Py_DECREF(o2); @@ -3223,6 +3467,30 @@ posix_uname(PyObject *self, PyObject *noargs) } #endif /* HAVE_UNAME */ + +/* + * Classic POSIX utime functions supported microseconds (1m/sec). + * Newer POSIX functions support nanoseconds (1 billion per sec). + * posixmodule now uses the new functions where possible. + * This improves accuracy in many situations, for example shutil.copy2(). + * + * The implementation isn't currently sophisticated enough to handle + * a platform where HAVE_UTIMENSAT is true but HAVE_FUTIMENS is false. + * Specifically, posix_futimes() would break. + * + * Supporting such a platform wouldn't be impossible; you'd need two + * extract_time() functions, or make its precision a parameter. + * Since such a platform seems unlikely we haven't bothered. + */ +#if defined(HAVE_UTIMENSAT) +#define EXTRACT_TIME_PRECISION (1e9) +#if !defined(HAVE_FUTIMENS) +#error You HAVE_UTIMENSAT but not HAVE_FUTIMENS... please see accompanying comment. +#endif +#else +#define EXTRACT_TIME_PRECISION (1e6) +#endif + static int extract_time(PyObject *t, time_t* sec, long* usec) { @@ -3241,7 +3509,8 @@ extract_time(PyObject *t, time_t* sec, long* usec) if (intval == -1 && PyErr_Occurred()) return -1; *sec = intval; - *usec = (long)((tval - intval) * 1e6); /* can't exceed 1000000 */ + + *usec = (long)((tval - intval) * EXTRACT_TIME_PRECISION); if (*usec < 0) /* If rounding gave us a negative number, truncate. */ @@ -3271,7 +3540,7 @@ posix_utime(PyObject *self, PyObject *args) { #ifdef MS_WINDOWS PyObject *arg; - PyUnicodeObject *obwpath; + PyObject *obwpath; wchar_t *wpath = NULL; PyObject *oapath; char *apath; @@ -3282,23 +3551,26 @@ posix_utime(PyObject *self, PyObject *args) PyObject *result = NULL; if (PyArg_ParseTuple(args, "UO|:utime", &obwpath, &arg)) { - wpath = PyUnicode_AS_UNICODE(obwpath); + wpath = PyUnicode_AsUnicode(obwpath); + if (wpath == NULL) + return NULL; Py_BEGIN_ALLOW_THREADS hFile = CreateFileW(wpath, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); Py_END_ALLOW_THREADS if (hFile == INVALID_HANDLE_VALUE) - return win32_error_unicode("utime", wpath); - } else + return win32_error_object("utime", obwpath); + } + else { /* Drop the argument parsing error as narrow strings are also valid. */ PyErr_Clear(); - if (!wpath) { if (!PyArg_ParseTuple(args, "O&O:utime", PyUnicode_FSConverter, &oapath, &arg)) return NULL; + apath = PyBytes_AsString(oapath); Py_BEGIN_ALLOW_THREADS hFile = CreateFileA(apath, FILE_WRITE_ATTRIBUTES, 0, @@ -3359,24 +3631,6 @@ done: int res; PyObject* arg; -#if defined(HAVE_UTIMES) - struct timeval buf[2]; -#define ATIME buf[0].tv_sec -#define MTIME buf[1].tv_sec -#elif defined(HAVE_UTIME_H) -/* XXX should define struct utimbuf instead, above */ - struct utimbuf buf; -#define ATIME buf.actime -#define MTIME buf.modtime -#define UTIME_ARG &buf -#else /* HAVE_UTIMES */ - time_t buf[2]; -#define ATIME buf[0] -#define MTIME buf[1] -#define UTIME_ARG buf -#endif /* HAVE_UTIMES */ - - if (!PyArg_ParseTuple(args, "O&O:utime", PyUnicode_FSConverter, &opath, &arg)) return NULL; @@ -3404,19 +3658,37 @@ done: Py_DECREF(opath); return NULL; } - ATIME = atime; - MTIME = mtime; -#ifdef HAVE_UTIMES + + Py_BEGIN_ALLOW_THREADS + { +#ifdef HAVE_UTIMENSAT + struct timespec buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_nsec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_nsec = musec; + res = utimensat(AT_FDCWD, path, buf, 0); +#elif defined(HAVE_UTIMES) + struct timeval buf[2]; + buf[0].tv_sec = atime; buf[0].tv_usec = ausec; + buf[1].tv_sec = mtime; buf[1].tv_usec = musec; - Py_BEGIN_ALLOW_THREADS res = utimes(path, buf); - Py_END_ALLOW_THREADS +#elif defined(HAVE_UTIME_H) + /* XXX should define struct utimbuf instead, above */ + struct utimbuf buf; + buf.actime = atime; + buf.modtime = mtime; + res = utime(path, &buf); #else - Py_BEGIN_ALLOW_THREADS - res = utime(path, UTIME_ARG); + time_t buf[2]; + buf[0] = atime; + buf[1] = mtime; + res = utime(path, buf); +#endif + } Py_END_ALLOW_THREADS -#endif /* HAVE_UTIMES */ } if (res < 0) { return posix_error_with_allocated_filename(opath); @@ -3424,12 +3696,199 @@ done: Py_DECREF(opath); Py_INCREF(Py_None); return Py_None; -#undef UTIME_ARG -#undef ATIME -#undef MTIME +#undef UTIME_EXTRACT #endif /* MS_WINDOWS */ } +#ifdef HAVE_FUTIMES +PyDoc_STRVAR(posix_futimes__doc__, +"futimes(fd, (atime, mtime))\n\ +futimes(fd, None)\n\n\ +Set the access and modified time of the file specified by the file\n\ +descriptor fd to the given values. If the second form is used, set the\n\ +access and modified times to the current time."); + +static PyObject * +posix_futimes(PyObject *self, PyObject *args) +{ + int res, fd; + PyObject* arg; + time_t atime, mtime; + long ausec, musec; + + if (!PyArg_ParseTuple(args, "iO:futimes", &fd, &arg)) + return NULL; + + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimes(fd, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimes() arg 2 must be a tuple (atime, mtime)"); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &atime, &ausec) == -1) { + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &mtime, &musec) == -1) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS + { +#ifdef HAVE_FUTIMENS + struct timespec buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_nsec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_nsec = musec; + res = futimens(fd, buf); +#else + struct timeval buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_usec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_usec = musec; + res = futimes(fd, buf); +#endif + } + Py_END_ALLOW_THREADS + } + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_LUTIMES +PyDoc_STRVAR(posix_lutimes__doc__, +"lutimes(path, (atime, mtime))\n\ +lutimes(path, None)\n\n\ +Like utime(), but if path is a symbolic link, it is not dereferenced."); + +static PyObject * +posix_lutimes(PyObject *self, PyObject *args) +{ + PyObject *opath, *arg; + const char *path; + int res; + time_t atime, mtime; + long ausec, musec; + + if (!PyArg_ParseTuple(args, "O&O:lutimes", + PyUnicode_FSConverter, &opath, &arg)) + return NULL; + path = PyBytes_AsString(opath); + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = lutimes(path, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "lutimes() arg 2 must be a tuple (atime, mtime)"); + Py_DECREF(opath); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &atime, &ausec) == -1) { + Py_DECREF(opath); + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &mtime, &musec) == -1) { + Py_DECREF(opath); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + { +#ifdef HAVE_UTIMENSAT + struct timespec buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_nsec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_nsec = musec; + res = utimensat(AT_FDCWD, path, buf, AT_SYMLINK_NOFOLLOW); +#else + struct timeval buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_usec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_usec = musec; + res = lutimes(path, buf); +#endif + } + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_FUTIMENS +PyDoc_STRVAR(posix_futimens__doc__, +"futimens(fd, (atime_sec, atime_nsec), (mtime_sec, mtime_nsec))\n\ +futimens(fd, None, None)\n\n\ +Updates the timestamps of a file specified by the file descriptor fd, with\n\ +nanosecond precision.\n\ +The second form sets atime and mtime to the current time.\n\ +If *_nsec is specified as UTIME_NOW, the timestamp is updated to the\n\ +current time.\n\ +If *_nsec is specified as UTIME_OMIT, the timestamp is not updated."); + +static PyObject * +posix_futimens(PyObject *self, PyObject *args) +{ + int res, fd; + PyObject *atime, *mtime; + struct timespec buf[2]; + + if (!PyArg_ParseTuple(args, "iOO:futimens", + &fd, &atime, &mtime)) + return NULL; + if (atime == Py_None && mtime == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimens(fd, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(atime) || PyTuple_Size(atime) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimens() arg 2 must be a tuple (atime_sec, atime_nsec)"); + return NULL; + } + else if (!PyTuple_Check(mtime) || PyTuple_Size(mtime) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimens() arg 3 must be a tuple (mtime_sec, mtime_nsec)"); + return NULL; + } + else { + if (!PyArg_ParseTuple(atime, "ll:futimens", + &(buf[0].tv_sec), &(buf[0].tv_nsec))) { + return NULL; + } + if (!PyArg_ParseTuple(mtime, "ll:futimens", + &(buf[1].tv_sec), &(buf[1].tv_nsec))) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS + res = futimens(fd, buf); + Py_END_ALLOW_THREADS + } + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* Process operations */ @@ -3474,79 +3933,7 @@ int fsconvert_strdup(PyObject *o, char**out) } #endif - -#ifdef HAVE_EXECV -PyDoc_STRVAR(posix_execv__doc__, -"execv(path, args)\n\n\ -Execute an executable path with arguments, replacing current process.\n\ -\n\ - path: path of executable file\n\ - args: tuple or list of strings"); - -static PyObject * -posix_execv(PyObject *self, PyObject *args) -{ - PyObject *opath; - char *path; - PyObject *argv; - char **argvlist; - Py_ssize_t i, argc; - PyObject *(*getitem)(PyObject *, Py_ssize_t); - - /* execv has two arguments: (path, argv), where - argv is a list or tuple of strings. */ - - if (!PyArg_ParseTuple(args, "O&O:execv", - PyUnicode_FSConverter, - &opath, &argv)) - return NULL; - path = PyBytes_AsString(opath); - if (PyList_Check(argv)) { - argc = PyList_Size(argv); - getitem = PyList_GetItem; - } - else if (PyTuple_Check(argv)) { - argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; - } - else { - PyErr_SetString(PyExc_TypeError, "execv() arg 2 must be a tuple or list"); - Py_DECREF(opath); - return NULL; - } - if (argc < 1) { - PyErr_SetString(PyExc_ValueError, "execv() arg 2 must not be empty"); - Py_DECREF(opath); - return NULL; - } - - argvlist = PyMem_NEW(char *, argc+1); - if (argvlist == NULL) { - Py_DECREF(opath); - return PyErr_NoMemory(); - } - for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) { - free_string_array(argvlist, i); - PyErr_SetString(PyExc_TypeError, - "execv() arg 2 must contain only strings"); - Py_DECREF(opath); - return NULL; - - } - } - argvlist[argc] = NULL; - - execv(path, argvlist); - - /* If we get here it's definitely an error */ - - free_string_array(argvlist, argc); - Py_DECREF(opath); - return posix_error(); -} - +#if defined(HAVE_EXECV) || defined (HAVE_FEXECVE) static char** parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) { @@ -3628,6 +4015,87 @@ error: return NULL; } +static char** +parse_arglist(PyObject* argv, Py_ssize_t *argc) +{ + int i; + char **argvlist = PyMem_NEW(char *, *argc+1); + if (argvlist == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (i = 0; i < *argc; i++) { + PyObject* item = PySequence_ITEM(argv, i); + if (item == NULL) + goto fail; + if (!fsconvert_strdup(item, &argvlist[i])) { + Py_DECREF(item); + goto fail; + } + Py_DECREF(item); + } + argvlist[*argc] = NULL; + return argvlist; +fail: + *argc = i; + free_string_array(argvlist, *argc); + return NULL; +} +#endif + +#ifdef HAVE_EXECV +PyDoc_STRVAR(posix_execv__doc__, +"execv(path, args)\n\n\ +Execute an executable path with arguments, replacing current process.\n\ +\n\ + path: path of executable file\n\ + args: tuple or list of strings"); + +static PyObject * +posix_execv(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + PyObject *argv; + char **argvlist; + Py_ssize_t argc; + + /* execv has two arguments: (path, argv), where + argv is a list or tuple of strings. */ + + if (!PyArg_ParseTuple(args, "O&O:execv", + PyUnicode_FSConverter, + &opath, &argv)) + return NULL; + path = PyBytes_AsString(opath); + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "execv() arg 2 must be a tuple or list"); + Py_DECREF(opath); + return NULL; + } + argc = PySequence_Size(argv); + if (argc < 1) { + PyErr_SetString(PyExc_ValueError, "execv() arg 2 must not be empty"); + Py_DECREF(opath); + return NULL; + } + + argvlist = parse_arglist(argv, &argc); + if (argvlist == NULL) { + Py_DECREF(opath); + return NULL; + } + + execv(path, argvlist); + + /* If we get here it's definitely an error */ + + free_string_array(argvlist, argc); + Py_DECREF(opath); + return posix_error(); +} + PyDoc_STRVAR(posix_execve__doc__, "execve(path, args, env)\n\n\ Execute a path with arguments and environment, replacing current process.\n\ @@ -3644,9 +4112,7 @@ posix_execve(PyObject *self, PyObject *args) PyObject *argv, *env; char **argvlist; char **envlist; - Py_ssize_t i, argc, envc; - PyObject *(*getitem)(PyObject *, Py_ssize_t); - Py_ssize_t lastarg = 0; + Py_ssize_t argc, envc; /* execve has three arguments: (path, argv, env), where argv is a list or tuple of strings and env is a dictionary @@ -3657,40 +4123,22 @@ posix_execve(PyObject *self, PyObject *args) &opath, &argv, &env)) return NULL; path = PyBytes_AsString(opath); - if (PyList_Check(argv)) { - argc = PyList_Size(argv); - getitem = PyList_GetItem; - } - else if (PyTuple_Check(argv)) { - argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; - } - else { + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { PyErr_SetString(PyExc_TypeError, "execve() arg 2 must be a tuple or list"); goto fail_0; } + argc = PySequence_Size(argv); if (!PyMapping_Check(env)) { PyErr_SetString(PyExc_TypeError, "execve() arg 3 must be a mapping object"); goto fail_0; } - argvlist = PyMem_NEW(char *, argc+1); + argvlist = parse_arglist(argv, &argc); if (argvlist == NULL) { - PyErr_NoMemory(); goto fail_0; } - for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) - { - lastarg = i; - goto fail_1; - } - } - lastarg = argc; - argvlist[argc] = NULL; envlist = parse_envlist(env, &envc); if (envlist == NULL) @@ -3706,13 +4154,69 @@ posix_execve(PyObject *self, PyObject *args) PyMem_DEL(envlist[envc]); PyMem_DEL(envlist); fail_1: - free_string_array(argvlist, lastarg); + free_string_array(argvlist, argc); fail_0: Py_DECREF(opath); return NULL; } #endif /* HAVE_EXECV */ +#ifdef HAVE_FEXECVE +PyDoc_STRVAR(posix_fexecve__doc__, +"fexecve(fd, args, env)\n\n\ +Execute the program specified by a file descriptor with arguments and\n\ +environment, replacing the current process.\n\ +\n\ + fd: file descriptor of executable\n\ + args: tuple or list of arguments\n\ + env: dictionary of strings mapping to strings"); + +static PyObject * +posix_fexecve(PyObject *self, PyObject *args) +{ + int fd; + PyObject *argv, *env; + char **argvlist; + char **envlist; + Py_ssize_t argc, envc; + + if (!PyArg_ParseTuple(args, "iOO:fexecve", + &fd, &argv, &env)) + return NULL; + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "fexecve() arg 2 must be a tuple or list"); + return NULL; + } + argc = PySequence_Size(argv); + if (!PyMapping_Check(env)) { + PyErr_SetString(PyExc_TypeError, + "fexecve() arg 3 must be a mapping object"); + return NULL; + } + + argvlist = parse_arglist(argv, &argc); + if (argvlist == NULL) + return NULL; + + envlist = parse_envlist(env, &envc); + if (envlist == NULL) + goto fail; + + fexecve(fd, argvlist, envlist); + + /* If we get here it's definitely an error */ + + (void) posix_error(); + + while (--envc >= 0) + PyMem_DEL(envlist[envc]); + PyMem_DEL(envlist); + fail: + free_string_array(argvlist, argc); + return NULL; +} +#endif /* HAVE_FEXECVE */ #ifdef HAVE_SPAWNV PyDoc_STRVAR(posix_spawnv__doc__, @@ -4153,6 +4657,574 @@ posix_fork(PyObject *self, PyObject *noargs) } #endif +#ifdef HAVE_SCHED_H + +#ifdef HAVE_SCHED_GET_PRIORITY_MAX + +PyDoc_STRVAR(posix_sched_get_priority_max__doc__, +"sched_get_priority_max(policy)\n\n\ +Get the maximum scheduling priority for *policy*."); + +static PyObject * +posix_sched_get_priority_max(PyObject *self, PyObject *args) +{ + int policy, max; + + if (!PyArg_ParseTuple(args, "i:sched_get_priority_max", &policy)) + return NULL; + max = sched_get_priority_max(policy); + if (max < 0) + return posix_error(); + return PyLong_FromLong(max); +} + +PyDoc_STRVAR(posix_sched_get_priority_min__doc__, +"sched_get_priority_min(policy)\n\n\ +Get the minimum scheduling priority for *policy*."); + +static PyObject * +posix_sched_get_priority_min(PyObject *self, PyObject *args) +{ + int policy, min; + + if (!PyArg_ParseTuple(args, "i:sched_get_priority_min", &policy)) + return NULL; + min = sched_get_priority_min(policy); + if (min < 0) + return posix_error(); + return PyLong_FromLong(min); +} + +#endif /* HAVE_SCHED_GET_PRIORITY_MAX */ + +#ifdef HAVE_SCHED_SETSCHEDULER + +PyDoc_STRVAR(posix_sched_getscheduler__doc__, +"sched_getscheduler(pid)\n\n\ +Get the scheduling policy for the process with a PID of *pid*.\n\ +Passing a PID of 0 returns the scheduling policy for the calling process."); + +static PyObject * +posix_sched_getscheduler(PyObject *self, PyObject *args) +{ + pid_t pid; + int policy; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_getscheduler", &pid)) + return NULL; + policy = sched_getscheduler(pid); + if (policy < 0) + return posix_error(); + return PyLong_FromLong(policy); +} + +#endif + +#if defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM) + +static PyObject * +sched_param_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res, *priority; + static char *kwlist[] = {"sched_priority", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:sched_param", kwlist, &priority)) + return NULL; + res = PyStructSequence_New(type); + if (!res) + return NULL; + Py_INCREF(priority); + PyStructSequence_SET_ITEM(res, 0, priority); + return res; +} + +PyDoc_STRVAR(sched_param__doc__, +"sched_param(sched_priority): A scheduling parameter.\n\n\ +Current has only one field: sched_priority"); + +static PyStructSequence_Field sched_param_fields[] = { + {"sched_priority", "the scheduling priority"}, + {0} +}; + +static PyStructSequence_Desc sched_param_desc = { + "sched_param", /* name */ + sched_param__doc__, /* doc */ + sched_param_fields, + 1 +}; + +static int +convert_sched_param(PyObject *param, struct sched_param *res) +{ + long priority; + + if (Py_TYPE(param) != &SchedParamType) { + PyErr_SetString(PyExc_TypeError, "must have a sched_param object"); + return 0; + } + priority = PyLong_AsLong(PyStructSequence_GET_ITEM(param, 0)); + if (priority == -1 && PyErr_Occurred()) + return 0; + if (priority > INT_MAX || priority < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, "sched_priority out of range"); + return 0; + } + res->sched_priority = Py_SAFE_DOWNCAST(priority, long, int); + return 1; +} + +#endif + +#ifdef HAVE_SCHED_SETSCHEDULER + +PyDoc_STRVAR(posix_sched_setscheduler__doc__, +"sched_setscheduler(pid, policy, param)\n\n\ +Set the scheduling policy, *policy*, for *pid*.\n\ +If *pid* is 0, the calling process is changed.\n\ +*param* is an instance of sched_param."); + +static PyObject * +posix_sched_setscheduler(PyObject *self, PyObject *args) +{ + pid_t pid; + int policy; + struct sched_param param; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "iO&:sched_setscheduler", + &pid, &policy, &convert_sched_param, ¶m)) + return NULL; + + /* + ** sched_setscheduler() returns 0 in Linux, but the previous + ** scheduling policy under Solaris/Illumos, and others. + ** On error, -1 is returned in all Operating Systems. + */ + if (sched_setscheduler(pid, policy, ¶m) == -1) + return posix_error(); + Py_RETURN_NONE; +} + +#endif + +#ifdef HAVE_SCHED_SETPARAM + +PyDoc_STRVAR(posix_sched_getparam__doc__, +"sched_getparam(pid) -> sched_param\n\n\ +Returns scheduling parameters for the process with *pid* as an instance of the\n\ +sched_param class. A PID of 0 means the calling process."); + +static PyObject * +posix_sched_getparam(PyObject *self, PyObject *args) +{ + pid_t pid; + struct sched_param param; + PyObject *res, *priority; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_getparam", &pid)) + return NULL; + if (sched_getparam(pid, ¶m)) + return posix_error(); + res = PyStructSequence_New(&SchedParamType); + if (!res) + return NULL; + priority = PyLong_FromLong(param.sched_priority); + if (!priority) { + Py_DECREF(res); + return NULL; + } + PyStructSequence_SET_ITEM(res, 0, priority); + return res; +} + +PyDoc_STRVAR(posix_sched_setparam__doc__, +"sched_setparam(pid, param)\n\n\ +Set scheduling parameters for a process with PID *pid*.\n\ +A PID of 0 means the calling process."); + +static PyObject * +posix_sched_setparam(PyObject *self, PyObject *args) +{ + pid_t pid; + struct sched_param param; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O&:sched_setparam", + &pid, &convert_sched_param, ¶m)) + return NULL; + if (sched_setparam(pid, ¶m)) + return posix_error(); + Py_RETURN_NONE; +} + +#endif + +#ifdef HAVE_SCHED_RR_GET_INTERVAL + +PyDoc_STRVAR(posix_sched_rr_get_interval__doc__, +"sched_rr_get_interval(pid) -> float\n\n\ +Return the round-robin quantum for the process with PID *pid* in seconds."); + +static PyObject * +posix_sched_rr_get_interval(PyObject *self, PyObject *args) +{ + pid_t pid; + struct timespec interval; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_rr_get_interval", &pid)) + return NULL; + if (sched_rr_get_interval(pid, &interval)) + return posix_error(); + return PyFloat_FromDouble((double)interval.tv_sec + 1e-9*interval.tv_nsec); +} + +#endif + +PyDoc_STRVAR(posix_sched_yield__doc__, +"sched_yield()\n\n\ +Voluntarily relinquish the CPU."); + +static PyObject * +posix_sched_yield(PyObject *self, PyObject *noargs) +{ + if (sched_yield()) + return posix_error(); + Py_RETURN_NONE; +} + +#ifdef HAVE_SCHED_SETAFFINITY + +typedef struct { + PyObject_HEAD; + Py_ssize_t size; + int ncpus; + cpu_set_t *set; +} Py_cpu_set; + +static PyTypeObject cpu_set_type; + +static void +cpu_set_dealloc(Py_cpu_set *set) +{ + assert(set->set); + CPU_FREE(set->set); + Py_TYPE(set)->tp_free(set); +} + +static Py_cpu_set * +make_new_cpu_set(PyTypeObject *type, Py_ssize_t size) +{ + Py_cpu_set *set; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative size"); + return NULL; + } + set = (Py_cpu_set *)type->tp_alloc(type, 0); + if (!set) + return NULL; + set->ncpus = size; + set->size = CPU_ALLOC_SIZE(size); + set->set = CPU_ALLOC(size); + if (!set->set) { + type->tp_free(set); + PyErr_NoMemory(); + return NULL; + } + CPU_ZERO_S(set->size, set->set); + return set; +} + +static PyObject * +cpu_set_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + int size; + + if (!_PyArg_NoKeywords("cpu_set()", kwargs) || + !PyArg_ParseTuple(args, "i:cpu_set", &size)) + return NULL; + return (PyObject *)make_new_cpu_set(type, size); +} + +static PyObject * +cpu_set_repr(Py_cpu_set *set) +{ + return PyUnicode_FromFormat("<cpu_set with %li entries>", set->ncpus); +} + +static Py_ssize_t +cpu_set_len(Py_cpu_set *set) +{ + return set->ncpus; +} + +static int +_get_cpu(Py_cpu_set *set, const char *requester, PyObject *args) +{ + int cpu; + if (!PyArg_ParseTuple(args, requester, &cpu)) + return -1; + if (cpu < 0) { + PyErr_SetString(PyExc_ValueError, "cpu < 0 not valid"); + return -1; + } + if (cpu >= set->ncpus) { + PyErr_SetString(PyExc_ValueError, "cpu too large for set"); + return -1; + } + return cpu; +} + +PyDoc_STRVAR(cpu_set_set_doc, +"cpu_set.set(i)\n\n\ +Add CPU *i* to the set."); + +static PyObject * +cpu_set_set(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|set", args); + if (cpu == -1) + return NULL; + CPU_SET_S(cpu, set->size, set->set); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(cpu_set_count_doc, +"cpu_set.count() -> int\n\n\ +Return the number of CPUs active in the set."); + +static PyObject * +cpu_set_count(Py_cpu_set *set, PyObject *noargs) +{ + return PyLong_FromLong(CPU_COUNT_S(set->size, set->set)); +} + +PyDoc_STRVAR(cpu_set_clear_doc, +"cpu_set.clear(i)\n\n\ +Remove CPU *i* from the set."); + +static PyObject * +cpu_set_clear(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|clear", args); + if (cpu == -1) + return NULL; + CPU_CLR_S(cpu, set->size, set->set); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(cpu_set_isset_doc, +"cpu_set.isset(i) -> bool\n\n\ +Test if CPU *i* is in the set."); + +static PyObject * +cpu_set_isset(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|isset", args); + if (cpu == -1) + return NULL; + if (CPU_ISSET_S(cpu, set->size, set->set)) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(cpu_set_zero_doc, +"cpu_set.zero()\n\n\ +Clear the cpu_set."); + +static PyObject * +cpu_set_zero(Py_cpu_set *set, PyObject *noargs) +{ + CPU_ZERO_S(set->size, set->set); + Py_RETURN_NONE; +} + +static PyObject * +cpu_set_richcompare(Py_cpu_set *set, Py_cpu_set *other, int op) +{ + int eq; + + if ((op != Py_EQ && op != Py_NE) || Py_TYPE(other) != &cpu_set_type) + Py_RETURN_NOTIMPLEMENTED; + + eq = set->ncpus == other->ncpus && CPU_EQUAL_S(set->size, set->set, other->set); + if ((op == Py_EQ) ? eq : !eq) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +#define CPU_SET_BINOP(name, op) \ + static PyObject * \ + do_cpu_set_##name(Py_cpu_set *left, Py_cpu_set *right, Py_cpu_set *res) { \ + if (res) { \ + Py_INCREF(res); \ + } \ + else { \ + res = make_new_cpu_set(&cpu_set_type, left->ncpus); \ + if (!res) \ + return NULL; \ + } \ + if (Py_TYPE(right) != &cpu_set_type || left->ncpus != right->ncpus) { \ + Py_DECREF(res); \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + assert(left->size == right->size && right->size == res->size); \ + op(res->size, res->set, left->set, right->set); \ + return (PyObject *)res; \ + } \ + static PyObject * \ + cpu_set_##name(Py_cpu_set *left, Py_cpu_set *right) { \ + return do_cpu_set_##name(left, right, NULL); \ + } \ + static PyObject * \ + cpu_set_i##name(Py_cpu_set *left, Py_cpu_set *right) { \ + return do_cpu_set_##name(left, right, left); \ + } \ + +CPU_SET_BINOP(and, CPU_AND_S) +CPU_SET_BINOP(or, CPU_OR_S) +CPU_SET_BINOP(xor, CPU_XOR_S) +#undef CPU_SET_BINOP + +PyDoc_STRVAR(cpu_set_doc, +"cpu_set(size)\n\n\ +Create an empty mask of CPUs."); + +static PyNumberMethods cpu_set_as_number = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_bool*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + (binaryfunc)cpu_set_and, /*nb_and*/ + (binaryfunc)cpu_set_xor, /*nb_xor*/ + (binaryfunc)cpu_set_or, /*nb_or*/ + 0, /*nb_int*/ + 0, /*nb_reserved*/ + 0, /*nb_float*/ + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + (binaryfunc)cpu_set_iand, /*nb_inplace_and*/ + (binaryfunc)cpu_set_ixor, /*nb_inplace_xor*/ + (binaryfunc)cpu_set_ior, /*nb_inplace_or*/ +}; + +static PySequenceMethods cpu_set_as_sequence = { + (lenfunc)cpu_set_len, /* sq_length */ +}; + +static PyMethodDef cpu_set_methods[] = { + {"clear", (PyCFunction)cpu_set_clear, METH_VARARGS, cpu_set_clear_doc}, + {"count", (PyCFunction)cpu_set_count, METH_NOARGS, cpu_set_count_doc}, + {"isset", (PyCFunction)cpu_set_isset, METH_VARARGS, cpu_set_isset_doc}, + {"set", (PyCFunction)cpu_set_set, METH_VARARGS, cpu_set_set_doc}, + {"zero", (PyCFunction)cpu_set_zero, METH_NOARGS, cpu_set_zero_doc}, + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject cpu_set_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "posix.cpu_set", /* tp_name */ + sizeof(Py_cpu_set), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)cpu_set_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)cpu_set_repr, /* tp_repr */ + &cpu_set_as_number, /* tp_as_number */ + &cpu_set_as_sequence, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + PyObject_HashNotImplemented, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + cpu_set_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)cpu_set_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + cpu_set_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + cpu_set_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + +PyDoc_STRVAR(posix_sched_setaffinity__doc__, +"sched_setaffinity(pid, cpu_set)\n\n\ +Set the affinity of the process with PID *pid* to *cpu_set*."); + +static PyObject * +posix_sched_setaffinity(PyObject *self, PyObject *args) +{ + pid_t pid; + Py_cpu_set *cpu_set; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O!:sched_setaffinity", + &pid, &cpu_set_type, &cpu_set)) + return NULL; + if (sched_setaffinity(pid, cpu_set->size, cpu_set->set)) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_sched_getaffinity__doc__, +"sched_getaffinity(pid, ncpus) -> cpu_set\n\n\ +Return the affinity of the process with PID *pid*.\n\ +The returned cpu_set will be of size *ncpus*."); + +static PyObject * +posix_sched_getaffinity(PyObject *self, PyObject *args) +{ + pid_t pid; + int ncpus; + Py_cpu_set *res; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "i:sched_getaffinity", + &pid, &ncpus)) + return NULL; + res = make_new_cpu_set(&cpu_set_type, ncpus); + if (!res) + return NULL; + if (sched_getaffinity(pid, res->size, res->set)) { + Py_DECREF(res); + return posix_error(); + } + return (PyObject *)res; +} + +#endif /* HAVE_SCHED_SETAFFINITY */ + +#endif /* HAVE_SCHED_H */ + /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ /* IRIX has both /dev/ptc and /dev/ptmx, use ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) @@ -4279,6 +5351,7 @@ posix_forkpty(PyObject *self, PyObject *noargs) } #endif + #ifdef HAVE_GETEGID PyDoc_STRVAR(posix_getegid__doc__, "getegid() -> egid\n\n\ @@ -4328,6 +5401,70 @@ posix_getpid(PyObject *self, PyObject *noargs) return PyLong_FromPid(getpid()); } +#ifdef HAVE_GETGROUPLIST +PyDoc_STRVAR(posix_getgrouplist__doc__, +"getgrouplist(user, group) -> list of groups to which a user belongs\n\n\ +Returns a list of groups to which a user belongs.\n\n\ + user: username to lookup\n\ + group: base group id of the user"); + +static PyObject * +posix_getgrouplist(PyObject *self, PyObject *args) +{ +#ifdef NGROUPS_MAX +#define MAX_GROUPS NGROUPS_MAX +#else + /* defined to be 16 on Solaris7, so this should be a small number */ +#define MAX_GROUPS 64 +#endif + + const char *user; + int i, ngroups; + PyObject *list; +#ifdef __APPLE__ + int *groups, basegid; +#else + gid_t *groups, basegid; +#endif + ngroups = MAX_GROUPS; + + if (!PyArg_ParseTuple(args, "si", &user, &basegid)) + return NULL; + +#ifdef __APPLE__ + groups = PyMem_Malloc(ngroups * sizeof(int)); +#else + groups = PyMem_Malloc(ngroups * sizeof(gid_t)); +#endif + if (groups == NULL) + return PyErr_NoMemory(); + + if (getgrouplist(user, basegid, groups, &ngroups) == -1) { + PyMem_Del(groups); + return posix_error(); + } + + list = PyList_New(ngroups); + if (list == NULL) { + PyMem_Del(groups); + return NULL; + } + + for (i = 0; i < ngroups; i++) { + PyObject *o = PyLong_FromUnsignedLong((unsigned long)groups[i]); + if (o == NULL) { + Py_DECREF(list); + PyMem_Del(groups); + return NULL; + } + PyList_SET_ITEM(list, i, o); + } + + PyMem_Del(groups); + + return list; +} +#endif #ifdef HAVE_GETGROUPS PyDoc_STRVAR(posix_getgroups__doc__, @@ -4565,7 +5702,7 @@ posix_getlogin(PyObject *self, PyObject *noargs) PyObject *result = NULL; #ifdef MS_WINDOWS wchar_t user_name[UNLEN + 1]; - DWORD num_chars = sizeof(user_name)/sizeof(user_name[0]); + DWORD num_chars = Py_ARRAY_LENGTH(user_name); if (GetUserNameW(user_name, &num_chars)) { /* num_chars is the number of unicode chars plus null terminator */ @@ -4965,6 +6102,7 @@ wait_helper(pid_t pid, int status, struct rusage *ru) { PyObject *result; static PyObject *struct_rusage; + _Py_identifier(struct_rusage); if (pid == -1) return posix_error(); @@ -4973,7 +6111,7 @@ wait_helper(pid_t pid, int status, struct rusage *ru) PyObject *m = PyImport_ImportModuleNoBlock("resource"); if (m == NULL) return NULL; - struct_rusage = PyObject_GetAttrString(m, "struct_rusage"); + struct_rusage = _PyObject_GetAttrId(m, &PyId_struct_rusage); Py_DECREF(m); if (struct_rusage == NULL) return NULL; @@ -5069,6 +6207,55 @@ posix_wait4(PyObject *self, PyObject *args) } #endif /* HAVE_WAIT4 */ +#if defined(HAVE_WAITID) && !defined(__APPLE__) +PyDoc_STRVAR(posix_waitid__doc__, +"waitid(idtype, id, options) -> waitid_result\n\n\ +Wait for the completion of one or more child processes.\n\n\ +idtype can be P_PID, P_PGID or P_ALL.\n\ +id specifies the pid to wait on.\n\ +options is constructed from the ORing of one or more of WEXITED, WSTOPPED\n\ +or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n\ +Returns either waitid_result or None if WNOHANG is specified and there are\n\ +no children in a waitable state."); + +static PyObject * +posix_waitid(PyObject *self, PyObject *args) +{ + PyObject *result; + idtype_t idtype; + id_t id; + int options, res; + siginfo_t si; + si.si_pid = 0; + if (!PyArg_ParseTuple(args, "i" _Py_PARSE_PID "i:waitid", &idtype, &id, &options)) + return NULL; + Py_BEGIN_ALLOW_THREADS + res = waitid(idtype, id, &si, options); + Py_END_ALLOW_THREADS + if (res == -1) + return posix_error(); + + if (si.si_pid == 0) + Py_RETURN_NONE; + + result = PyStructSequence_New(&WaitidResultType); + if (!result) + return NULL; + + PyStructSequence_SET_ITEM(result, 0, PyLong_FromPid(si.si_pid)); + PyStructSequence_SET_ITEM(result, 1, PyLong_FromPid(si.si_uid)); + PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si.si_signo))); + PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong((long)(si.si_status))); + PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si.si_code))); + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } + + return result; +} +#endif + #ifdef HAVE_WAITPID PyDoc_STRVAR(posix_waitpid__doc__, "waitpid(pid, options) -> (pid, status)\n\n\ @@ -5232,7 +6419,7 @@ win_readlink(PyObject *self, PyObject *args) wchar_t *path; DWORD n_bytes_returned; DWORD io_result; - PyObject *result; + PyObject *po, *result; HANDLE reparse_point_handle; char target_buffer[MAXIMUM_REPARSE_DATA_BUFFER_SIZE]; @@ -5240,8 +6427,11 @@ win_readlink(PyObject *self, PyObject *args) wchar_t *print_name; if (!PyArg_ParseTuple(args, - "u:readlink", - &path)) + "U:readlink", + &po)) + return NULL; + path = PyUnicode_AsUnicode(po); + if (path == NULL) return NULL; /* First get a handle to the reparse point */ @@ -5257,9 +6447,7 @@ win_readlink(PyObject *self, PyObject *args) Py_END_ALLOW_THREADS if (reparse_point_handle==INVALID_HANDLE_VALUE) - { - return win32_error_unicode("readlink", path); - } + return win32_error_object("readlink", po); Py_BEGIN_ALLOW_THREADS /* New call DeviceIoControl to read the reparse point */ @@ -5275,9 +6463,7 @@ win_readlink(PyObject *self, PyObject *args) Py_END_ALLOW_THREADS if (io_result==0) - { - return win32_error_unicode("readlink", path); - } + return win32_error_object("readlink", po); if (rdb->ReparseTag != IO_REPARSE_TAG_SYMLINK) { @@ -5328,6 +6514,7 @@ win_symlink(PyObject *self, PyObject *args, PyObject *kwargs) { static char *kwlist[] = {"src", "dest", "target_is_directory", NULL}; PyObject *src, *dest; + wchar_t *wsrc, *wdest; int target_is_directory = 0; DWORD res; WIN32_FILE_ATTRIBUTE_DATA src_info; @@ -5345,16 +6532,24 @@ win_symlink(PyObject *self, PyObject *args, PyObject *kwargs) if (win32_can_symlink == 0) return PyErr_Format(PyExc_OSError, "symbolic link privilege not held"); - if (!convert_to_unicode(&src)) { return NULL; } + if (!convert_to_unicode(&src)) + return NULL; if (!convert_to_unicode(&dest)) { Py_DECREF(src); return NULL; } + wsrc = PyUnicode_AsUnicode(src); + if (wsrc == NULL) + goto error; + wdest = PyUnicode_AsUnicode(dest); + if (wsrc == NULL) + goto error; + /* if src is a directory, ensure target_is_directory==1 */ if( GetFileAttributesExW( - PyUnicode_AsUnicode(src), GetFileExInfoStandard, &src_info + wsrc, GetFileExInfoStandard, &src_info )) { target_is_directory = target_is_directory || @@ -5362,20 +6557,21 @@ win_symlink(PyObject *self, PyObject *args, PyObject *kwargs) } Py_BEGIN_ALLOW_THREADS - res = Py_CreateSymbolicLinkW( - PyUnicode_AsUnicode(dest), - PyUnicode_AsUnicode(src), - target_is_directory); + res = Py_CreateSymbolicLinkW(wdest, wsrc, target_is_directory); Py_END_ALLOW_THREADS + Py_DECREF(src); Py_DECREF(dest); if (!res) - { - return win32_error_unicode("symlink", PyUnicode_AsUnicode(src)); - } + return win32_error_object("symlink", src); Py_INCREF(Py_None); return Py_None; + +error: + Py_DECREF(src); + Py_DECREF(dest); + return NULL; } #endif /* defined(HAVE_SYMLINK) && defined(MS_WINDOWS) */ @@ -5570,12 +6766,14 @@ posix_open(PyObject *self, PyObject *args) int fd; #ifdef MS_WINDOWS - PyUnicodeObject *po; + PyObject *po; if (PyArg_ParseTuple(args, "Ui|i:open", &po, &flag, &mode)) { + wchar_t *wpath = PyUnicode_AsUnicode(po); + if (wpath == NULL) + return NULL; + Py_BEGIN_ALLOW_THREADS - /* PyUnicode_AS_UNICODE OK without thread - lock as it is a simple dereference. */ - fd = _wopen(PyUnicode_AS_UNICODE(po), flag, mode); + fd = _wopen(wpath, flag, mode); Py_END_ALLOW_THREADS if (fd < 0) return posix_error(); @@ -5654,9 +6852,7 @@ posix_dup(PyObject *self, PyObject *args) return NULL; if (!_PyVerify_fd(fd)) return posix_error(); - Py_BEGIN_ALLOW_THREADS fd = dup(fd); - Py_END_ALLOW_THREADS if (fd < 0) return posix_error(); return PyLong_FromLong((long)fd); @@ -5675,15 +6871,42 @@ posix_dup2(PyObject *self, PyObject *args) return NULL; if (!_PyVerify_fd_dup2(fd, fd2)) return posix_error(); - Py_BEGIN_ALLOW_THREADS res = dup2(fd, fd2); - Py_END_ALLOW_THREADS if (res < 0) return posix_error(); Py_INCREF(Py_None); return Py_None; } +#ifdef HAVE_LOCKF +PyDoc_STRVAR(posix_lockf__doc__, +"lockf(fd, cmd, len)\n\n\ +Apply, test or remove a POSIX lock on an open file descriptor.\n\n\ +fd is an open file descriptor.\n\ +cmd specifies the command to use - one of F_LOCK, F_TLOCK, F_ULOCK or\n\ +F_TEST.\n\ +len specifies the section of the file to lock."); + +static PyObject * +posix_lockf(PyObject *self, PyObject *args) +{ + int fd, cmd, res; + off_t len; + if (!PyArg_ParseTuple(args, "iiO&:lockf", + &fd, &cmd, _parse_off_t, &len)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = lockf(fd, cmd, len); + Py_END_ALLOW_THREADS + + if (res < 0) + return posix_error(); + + Py_RETURN_NONE; +} +#endif + PyDoc_STRVAR(posix_lseek__doc__, "lseek(fd, pos, how) -> newpos\n\n\ @@ -5713,8 +6936,7 @@ posix_lseek(PyObject *self, PyObject *args) #if !defined(HAVE_LARGEFILE_SUPPORT) pos = PyLong_AsLong(posobj); #else - pos = PyLong_Check(posobj) ? - PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj); + pos = PyLong_AsLongLong(posobj); #endif if (PyErr_Occurred()) return NULL; @@ -5774,6 +6996,140 @@ posix_read(PyObject *self, PyObject *args) return buffer; } +#if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) +static Py_ssize_t +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +{ + int i, j; + Py_ssize_t blen, total = 0; + + *iov = PyMem_New(struct iovec, cnt); + if (*iov == NULL) { + PyErr_NoMemory(); + return total; + } + + *buf = PyMem_New(Py_buffer, cnt); + if (*buf == NULL) { + PyMem_Del(*iov); + PyErr_NoMemory(); + return total; + } + + for (i = 0; i < cnt; i++) { + PyObject *item = PySequence_GetItem(seq, i); + if (item == NULL) + goto fail; + if (PyObject_GetBuffer(item, &(*buf)[i], type) == -1) { + Py_DECREF(item); + goto fail; + } + Py_DECREF(item); + (*iov)[i].iov_base = (*buf)[i].buf; + blen = (*buf)[i].len; + (*iov)[i].iov_len = blen; + total += blen; + } + return total; + +fail: + PyMem_Del(*iov); + for (j = 0; j < i; j++) { + PyBuffer_Release(&(*buf)[j]); + } + PyMem_Del(*buf); + return 0; +} + +static void +iov_cleanup(struct iovec *iov, Py_buffer *buf, int cnt) +{ + int i; + PyMem_Del(iov); + for (i = 0; i < cnt; i++) { + PyBuffer_Release(&buf[i]); + } + PyMem_Del(buf); +} +#endif + +#ifdef HAVE_READV +PyDoc_STRVAR(posix_readv__doc__, +"readv(fd, buffers) -> bytesread\n\n\ +Read from a file descriptor into a number of writable buffers. buffers\n\ +is an arbitrary sequence of writable buffers.\n\ +Returns the total number of bytes read."); + +static PyObject * +posix_readv(PyObject *self, PyObject *args) +{ + int fd, cnt; + Py_ssize_t n; + PyObject *seq; + struct iovec *iov; + Py_buffer *buf; + + if (!PyArg_ParseTuple(args, "iO:readv", &fd, &seq)) + return NULL; + if (!PySequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, + "readv() arg 2 must be a sequence"); + return NULL; + } + cnt = PySequence_Size(seq); + + if (!iov_setup(&iov, &buf, seq, cnt, PyBUF_WRITABLE)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + n = readv(fd, iov, cnt); + Py_END_ALLOW_THREADS + + iov_cleanup(iov, buf, cnt); + return PyLong_FromSsize_t(n); +} +#endif + +#ifdef HAVE_PREAD +PyDoc_STRVAR(posix_pread__doc__, +"pread(fd, buffersize, offset) -> string\n\n\ +Read from a file descriptor, fd, at a position of offset. It will read up\n\ +to buffersize number of bytes. The file offset remains unchanged."); + +static PyObject * +posix_pread(PyObject *self, PyObject *args) +{ + int fd, size; + off_t offset; + Py_ssize_t n; + PyObject *buffer; + if (!PyArg_ParseTuple(args, "iiO&:pread", &fd, &size, _parse_off_t, &offset)) + return NULL; + + if (size < 0) { + errno = EINVAL; + return posix_error(); + } + buffer = PyBytes_FromStringAndSize((char *)NULL, size); + if (buffer == NULL) + return NULL; + if (!_PyVerify_fd(fd)) { + Py_DECREF(buffer); + return posix_error(); + } + Py_BEGIN_ALLOW_THREADS + n = pread(fd, PyBytes_AS_STRING(buffer), size, offset); + Py_END_ALLOW_THREADS + if (n < 0) { + Py_DECREF(buffer); + return posix_error(); + } + if (n != size) + _PyBytes_Resize(&buffer, n); + return buffer; +} +#endif PyDoc_STRVAR(posix_write__doc__, "write(fd, string) -> byteswritten\n\n\ @@ -5808,6 +7164,144 @@ posix_write(PyObject *self, PyObject *args) return PyLong_FromSsize_t(size); } +#ifdef HAVE_SENDFILE +PyDoc_STRVAR(posix_sendfile__doc__, +"sendfile(out, in, offset, nbytes) -> byteswritten\n\ +sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0)\n\ + -> byteswritten\n\ +Copy nbytes bytes from file descriptor in to file descriptor out."); + +static PyObject * +posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) +{ + int in, out; + Py_ssize_t ret; + off_t offset; + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifndef __APPLE__ + Py_ssize_t len; +#endif + PyObject *headers = NULL, *trailers = NULL; + Py_buffer *hbuf, *tbuf; + off_t sbytes; + struct sf_hdtr sf; + int flags = 0; + sf.headers = NULL; + sf.trailers = NULL; + static char *keywords[] = {"out", "in", + "offset", "count", + "headers", "trailers", "flags", NULL}; + +#ifdef __APPLE__ + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&O&|OOi:sendfile", + keywords, &out, &in, _parse_off_t, &offset, _parse_off_t, &sbytes, +#else + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&n|OOi:sendfile", + keywords, &out, &in, _parse_off_t, &offset, &len, +#endif + &headers, &trailers, &flags)) + return NULL; + if (headers != NULL) { + if (!PySequence_Check(headers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() headers must be a sequence or None"); + return NULL; + } else { + Py_ssize_t i = 0; /* Avoid uninitialized warning */ + sf.hdr_cnt = PySequence_Size(headers); + if (sf.hdr_cnt > 0 && + !(i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE))) + return NULL; +#ifdef __APPLE__ + sbytes += i; +#endif + } + } + if (trailers != NULL) { + if (!PySequence_Check(trailers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() trailers must be a sequence or None"); + return NULL; + } else { + Py_ssize_t i = 0; /* Avoid uninitialized warning */ + sf.trl_cnt = PySequence_Size(trailers); + if (sf.trl_cnt > 0 && + !(i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE))) + return NULL; +#ifdef __APPLE__ + sbytes += i; +#endif + } + } + + Py_BEGIN_ALLOW_THREADS +#ifdef __APPLE__ + ret = sendfile(in, out, offset, &sbytes, &sf, flags); +#else + ret = sendfile(in, out, offset, len, &sf, &sbytes, flags); +#endif + Py_END_ALLOW_THREADS + + if (sf.headers != NULL) + iov_cleanup(sf.headers, hbuf, sf.hdr_cnt); + if (sf.trailers != NULL) + iov_cleanup(sf.trailers, tbuf, sf.trl_cnt); + + if (ret < 0) { + if ((errno == EAGAIN) || (errno == EBUSY)) { + if (sbytes != 0) { + // some data has been sent + goto done; + } + else { + // no data has been sent; upper application is supposed + // to retry on EAGAIN or EBUSY + return posix_error(); + } + } + return posix_error(); + } + goto done; + +done: + #if !defined(HAVE_LARGEFILE_SUPPORT) + return Py_BuildValue("l", sbytes); + #else + return Py_BuildValue("L", sbytes); + #endif + +#else + Py_ssize_t count; + PyObject *offobj; + static char *keywords[] = {"out", "in", + "offset", "count", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiOn:sendfile", + keywords, &out, &in, &offobj, &count)) + return NULL; +#ifdef linux + if (offobj == Py_None) { + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, NULL, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); + return Py_BuildValue("n", ret); + } +#endif + if (!_parse_off_t(offobj, &offset)) + return NULL; + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, &offset, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); + return Py_BuildValue("n", ret); +#endif +} +#endif PyDoc_STRVAR(posix_fstat__doc__, "fstat(fd) -> stat result\n\n\ @@ -5869,9 +7363,7 @@ posix_pipe(PyObject *self, PyObject *noargs) HFILE read, write; APIRET rc; - Py_BEGIN_ALLOW_THREADS rc = DosCreatePipe( &read, &write, 4096); - Py_END_ALLOW_THREADS if (rc != NO_ERROR) return os2_error(rc); @@ -5880,9 +7372,7 @@ posix_pipe(PyObject *self, PyObject *noargs) #if !defined(MS_WINDOWS) int fds[2]; int res; - Py_BEGIN_ALLOW_THREADS res = pipe(fds); - Py_END_ALLOW_THREADS if (res != 0) return posix_error(); return Py_BuildValue("(ii)", fds[0], fds[1]); @@ -5890,9 +7380,7 @@ posix_pipe(PyObject *self, PyObject *noargs) HANDLE read, write; int read_fd, write_fd; BOOL ok; - Py_BEGIN_ALLOW_THREADS ok = CreatePipe(&read, &write, NULL, 0); - Py_END_ALLOW_THREADS if (!ok) return win32_error("CreatePipe", NULL); read_fd = _open_osfhandle((Py_intptr_t)read, 0); @@ -5903,6 +7391,99 @@ posix_pipe(PyObject *self, PyObject *noargs) } #endif /* HAVE_PIPE */ +#ifdef HAVE_PIPE2 +PyDoc_STRVAR(posix_pipe2__doc__, +"pipe2(flags) -> (read_end, write_end)\n\n\ +Create a pipe with flags set atomically.\n\ +flags can be constructed by ORing together one or more of these values:\n\ +O_NONBLOCK, O_CLOEXEC.\n\ +"); + +static PyObject * +posix_pipe2(PyObject *self, PyObject *arg) +{ + int flags; + int fds[2]; + int res; + + flags = PyLong_AsLong(arg); + if (flags == -1 && PyErr_Occurred()) + return NULL; + + res = pipe2(fds, flags); + if (res != 0) + return posix_error(); + return Py_BuildValue("(ii)", fds[0], fds[1]); +} +#endif /* HAVE_PIPE2 */ + +#ifdef HAVE_WRITEV +PyDoc_STRVAR(posix_writev__doc__, +"writev(fd, buffers) -> byteswritten\n\n\ +Write the contents of buffers to a file descriptor, where buffers is an\n\ +arbitrary sequence of buffers.\n\ +Returns the total bytes written."); + +static PyObject * +posix_writev(PyObject *self, PyObject *args) +{ + int fd, cnt; + Py_ssize_t res; + PyObject *seq; + struct iovec *iov; + Py_buffer *buf; + if (!PyArg_ParseTuple(args, "iO:writev", &fd, &seq)) + return NULL; + if (!PySequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, + "writev() arg 2 must be a sequence"); + return NULL; + } + cnt = PySequence_Size(seq); + + if (!iov_setup(&iov, &buf, seq, cnt, PyBUF_SIMPLE)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + res = writev(fd, iov, cnt); + Py_END_ALLOW_THREADS + + iov_cleanup(iov, buf, cnt); + return PyLong_FromSsize_t(res); +} +#endif + +#ifdef HAVE_PWRITE +PyDoc_STRVAR(posix_pwrite__doc__, +"pwrite(fd, string, offset) -> byteswritten\n\n\ +Write string to a file descriptor, fd, from offset, leaving the file\n\ +offset unchanged."); + +static PyObject * +posix_pwrite(PyObject *self, PyObject *args) +{ + Py_buffer pbuf; + int fd; + off_t offset; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "iy*O&:pwrite", &fd, &pbuf, _parse_off_t, &offset)) + return NULL; + + if (!_PyVerify_fd(fd)) { + PyBuffer_Release(&pbuf); + return posix_error(); + } + Py_BEGIN_ALLOW_THREADS + size = pwrite(fd, pbuf.buf, (size_t)pbuf.len, offset); + Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); + if (size < 0) + return posix_error(); + return PyLong_FromSsize_t(size); +} +#endif #ifdef HAVE_MKFIFO PyDoc_STRVAR(posix_mkfifo__doc__, @@ -6019,18 +7600,8 @@ posix_ftruncate(PyObject *self, PyObject *args) int fd; off_t length; int res; - PyObject *lenobj; - if (!PyArg_ParseTuple(args, "iO:ftruncate", &fd, &lenobj)) - return NULL; - -#if !defined(HAVE_LARGEFILE_SUPPORT) - length = PyLong_AsLong(lenobj); -#else - length = PyLong_Check(lenobj) ? - PyLong_AsLongLong(lenobj) : PyLong_AsLong(lenobj); -#endif - if (PyErr_Occurred()) + if (!PyArg_ParseTuple(args, "iO&:ftruncate", &fd, _parse_off_t, &length)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -6043,6 +7614,93 @@ posix_ftruncate(PyObject *self, PyObject *args) } #endif +#ifdef HAVE_TRUNCATE +PyDoc_STRVAR(posix_truncate__doc__, +"truncate(path, length)\n\n\ +Truncate the file given by path to length bytes."); + +static PyObject * +posix_truncate(PyObject *self, PyObject *args) +{ + PyObject *opath; + const char *path; + off_t length; + int res; + + if (!PyArg_ParseTuple(args, "O&O&:truncate", + PyUnicode_FSConverter, &opath, _parse_off_t, &length)) + return NULL; + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = truncate(path, length); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_POSIX_FALLOCATE +PyDoc_STRVAR(posix_posix_fallocate__doc__, +"posix_fallocate(fd, offset, len)\n\n\ +Ensures that enough disk space is allocated for the file specified by fd\n\ +starting from offset and continuing for len bytes."); + +static PyObject * +posix_posix_fallocate(PyObject *self, PyObject *args) +{ + off_t len, offset; + int res, fd; + + if (!PyArg_ParseTuple(args, "iO&O&:posix_fallocate", + &fd, _parse_off_t, &offset, _parse_off_t, &len)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = posix_fallocate(fd, offset, len); + Py_END_ALLOW_THREADS + if (res != 0) { + errno = res; + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_POSIX_FADVISE +PyDoc_STRVAR(posix_posix_fadvise__doc__, +"posix_fadvise(fd, offset, len, advice)\n\n\ +Announces an intention to access data in a specific pattern thus allowing\n\ +the kernel to make optimizations.\n\ +The advice applies to the region of the file specified by fd starting at\n\ +offset and continuing for len bytes.\n\ +advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\n\ +POSIX_FADV_RANDOM, POSIX_FADV_NOREUSE, POSIX_FADV_WILLNEED or\n\ +POSIX_FADV_DONTNEED."); + +static PyObject * +posix_posix_fadvise(PyObject *self, PyObject *args) +{ + off_t len, offset; + int res, fd, advice; + + if (!PyArg_ParseTuple(args, "iO&O&i:posix_fadvise", + &fd, _parse_off_t, &offset, _parse_off_t, &len, &advice)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = posix_fadvise(fd, offset, len, advice); + Py_END_ALLOW_THREADS + if (res != 0) { + errno = res; + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_PUTENV PyDoc_STRVAR(posix_putenv__doc__, "putenv(key, value)\n\n\ @@ -6117,6 +7775,8 @@ posix_putenv(PyObject *self, PyObject *args) } #ifdef MS_WINDOWS newenv = PyUnicode_AsUnicode(newstr); + if (newenv == NULL) + goto error; _snwprintf(newenv, len, L"%s=%s", s1, s2); if (_wputenv(newenv)) { posix_error(); @@ -6471,21 +8131,51 @@ Perform a statvfs system call on the given path."); static PyObject * posix_statvfs(PyObject *self, PyObject *args) { - char *path; + PyObject *path; int res; struct statvfs st; - if (!PyArg_ParseTuple(args, "s:statvfs", &path)) + if (!PyArg_ParseTuple(args, "O&:statvfs", PyUnicode_FSConverter, &path)) return NULL; Py_BEGIN_ALLOW_THREADS - res = statvfs(path, &st); + res = statvfs(PyBytes_AS_STRING(path), &st); Py_END_ALLOW_THREADS - if (res != 0) - return posix_error_with_filename(path); + if (res != 0) { + posix_error_with_filename(PyBytes_AS_STRING(path)); + Py_DECREF(path); + return NULL; + } + Py_DECREF(path); return _pystatvfs_fromstructstatvfs(st); } #endif /* HAVE_STATVFS */ +#ifdef MS_WINDOWS +PyDoc_STRVAR(win32__getdiskusage__doc__, +"_getdiskusage(path) -> (total, free)\n\n\ +Return disk usage statistics about the given path as (total, free) tuple."); + +static PyObject * +win32__getdiskusage(PyObject *self, PyObject *args) +{ + BOOL retval; + ULARGE_INTEGER _, total, free; + LPCTSTR path; + + if (! PyArg_ParseTuple(args, "s", &path)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + retval = GetDiskFreeSpaceEx(path, &_, &total, &free); + Py_END_ALLOW_THREADS + if (retval == 0) + return PyErr_SetFromWindowsErr(0); + + return Py_BuildValue("(LL)", total.QuadPart, free.QuadPart); +} +#endif + + /* This is used for fpathconf(), pathconf(), confstr() and sysconf(). * It maps strings representing configuration variable names to * integer values, allowing those functions to be called with the @@ -7548,9 +9238,10 @@ win32_startfile(PyObject *self, PyObject *args) PyObject *ofilepath; char *filepath; char *operation = NULL; + wchar_t *wpath, *woperation; HINSTANCE rc; - PyObject *unipath, *woperation = NULL; + PyObject *unipath, *uoperation = NULL; if (!PyArg_ParseTuple(args, "U|s:startfile", &unipath, &operation)) { PyErr_Clear(); @@ -7558,26 +9249,35 @@ win32_startfile(PyObject *self, PyObject *args) } if (operation) { - woperation = PyUnicode_DecodeASCII(operation, + uoperation = PyUnicode_DecodeASCII(operation, strlen(operation), NULL); - if (!woperation) { + if (!uoperation) { PyErr_Clear(); operation = NULL; goto normal; } } + wpath = PyUnicode_AsUnicode(unipath); + if (wpath == NULL) + goto normal; + if (uoperation) { + woperation = PyUnicode_AsUnicode(uoperation); + if (woperation == NULL) + goto normal; + } + else + woperation = NULL; + Py_BEGIN_ALLOW_THREADS - rc = ShellExecuteW((HWND)0, woperation ? PyUnicode_AS_UNICODE(woperation) : 0, - PyUnicode_AS_UNICODE(unipath), - NULL, NULL, SW_SHOWNORMAL); + rc = ShellExecuteW((HWND)0, woperation, wpath, + NULL, NULL, SW_SHOWNORMAL); Py_END_ALLOW_THREADS - Py_XDECREF(woperation); + Py_XDECREF(uoperation); if (rc <= (HINSTANCE)32) { - PyObject *errval = win32_error_unicode("startfile", - PyUnicode_AS_UNICODE(unipath)); - return errval; + win32_error_object("startfile", unipath); + return NULL; } Py_INCREF(Py_None); return Py_None; @@ -7707,6 +9407,9 @@ static PyObject * device_encoding(PyObject *self, PyObject *args) { int fd; +#if defined(MS_WINDOWS) || defined(MS_WIN64) + UINT cp; +#endif if (!PyArg_ParseTuple(args, "i:device_encoding", &fd)) return NULL; if (!_PyVerify_fd(fd) || !isatty(fd)) { @@ -7714,16 +9417,16 @@ device_encoding(PyObject *self, PyObject *args) return Py_None; } #if defined(MS_WINDOWS) || defined(MS_WIN64) - if (fd == 0) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleCP()); - return PyUnicode_FromString(buf); - } - if (fd == 1 || fd == 2) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleOutputCP()); - return PyUnicode_FromString(buf); - } + if (fd == 0) + cp = GetConsoleCP(); + else if (fd == 1 || fd == 2) + cp = GetConsoleOutputCP(); + else + cp = 0; + /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application + has no console */ + if (cp != 0) + return PyUnicode_FromFormat("cp%u", (unsigned int)cp); #elif defined(CODESET) { char *codeset = nl_langinfo(CODESET); @@ -7847,6 +9550,947 @@ posix_getresgid (PyObject *self, PyObject *noargs) } #endif +/* Posix *at family of functions: + faccessat, fchmodat, fchownat, fstatat, futimesat, + linkat, mkdirat, mknodat, openat, readlinkat, renameat, symlinkat, + unlinkat, utimensat, mkfifoat */ + +#ifdef HAVE_FACCESSAT +PyDoc_STRVAR(posix_faccessat__doc__, +"faccessat(dirfd, path, mode, flags=0) -> True if granted, False otherwise\n\n\ +Like access() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and can be constructed by ORing together zero or more\n\ +of these values: AT_SYMLINK_NOFOLLOW, AT_EACCESS.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_faccessat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int mode; + int res; + int dirfd, flags = 0; + if (!PyArg_ParseTuple(args, "iO&i|i:faccessat", + &dirfd, PyUnicode_FSConverter, &opath, &mode, &flags)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = faccessat(dirfd, path, mode, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + return PyBool_FromLong(res == 0); +} +#endif + +#ifdef HAVE_FCHMODAT +PyDoc_STRVAR(posix_fchmodat__doc__, +"fchmodat(dirfd, path, mode, flags=0)\n\n\ +Like chmod() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fchmodat(PyObject *self, PyObject *args) +{ + int dirfd, mode, res; + int flags = 0; + PyObject *opath; + char *path; + + if (!PyArg_ParseTuple(args, "iO&i|i:fchmodat", + &dirfd, PyUnicode_FSConverter, &opath, &mode, &flags)) + return NULL; + + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fchmodat(dirfd, path, mode, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_FCHMODAT */ + +#ifdef HAVE_FCHOWNAT +PyDoc_STRVAR(posix_fchownat__doc__, +"fchownat(dirfd, path, uid, gid, flags=0)\n\n\ +Like chown() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fchownat(PyObject *self, PyObject *args) +{ + PyObject *opath; + int dirfd, res; + long uid, gid; + int flags = 0; + char *path; + + if (!PyArg_ParseTuple(args, "iO&ll|i:fchownat", + &dirfd, PyUnicode_FSConverter, &opath, &uid, &gid, &flags)) + return NULL; + + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fchownat(dirfd, path, (uid_t) uid, (gid_t) gid, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_FCHOWNAT */ + +#ifdef HAVE_FSTATAT +PyDoc_STRVAR(posix_fstatat__doc__, +"fstatat(dirfd, path, flags=0) -> stat result\n\n\ +Like stat() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fstatat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + STRUCT_STAT st; + int dirfd, res, flags = 0; + + if (!PyArg_ParseTuple(args, "iO&|i:fstatat", + &dirfd, PyUnicode_FSConverter, &opath, &flags)) + return NULL; + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fstatat(dirfd, path, &st, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res != 0) + return posix_error(); + + return _pystat_fromstructstat(&st); +} +#endif + +#ifdef HAVE_FUTIMESAT +PyDoc_STRVAR(posix_futimesat__doc__, +"futimesat(dirfd, path, (atime, mtime))\n\ +futimesat(dirfd, path, None)\n\n\ +Like utime() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_futimesat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int res, dirfd; + PyObject* arg; + time_t atime, mtime; + long ausec, musec; + + if (!PyArg_ParseTuple(args, "iO&O:futimesat", + &dirfd, PyUnicode_FSConverter, &opath, &arg)) + return NULL; + path = PyBytes_AsString(opath); + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimesat(dirfd, path, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimesat() arg 3 must be a tuple (atime, mtime)"); + Py_DECREF(opath); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &atime, &ausec) == -1) { + Py_DECREF(opath); + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &mtime, &musec) == -1) { + Py_DECREF(opath); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + { +#ifdef HAVE_UTIMENSAT + struct timespec buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_nsec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_nsec = musec; + res = utimensat(dirfd, path, buf, 0); +#else + struct timeval buf[2]; + buf[0].tv_sec = atime; + buf[0].tv_usec = ausec; + buf[1].tv_sec = mtime; + buf[1].tv_usec = musec; + res = futimesat(dirfd, path, buf); +#endif + } + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_LINKAT +PyDoc_STRVAR(posix_linkat__doc__, +"linkat(srcfd, srcpath, dstfd, dstpath, flags=0)\n\n\ +Like link() but if srcpath is relative, it is taken as relative to srcfd\n\ +and if dstpath is relative, it is taken as relative to dstfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_FOLLOW.\n\ +If srcpath is relative and srcfd is the special value AT_FDCWD, then\n\ +srcpath is interpreted relative to the current working directory. This\n\ +also applies for dstpath."); + +static PyObject * +posix_linkat(PyObject *self, PyObject *args) +{ + PyObject *osrc, *odst; + char *src, *dst; + int res, srcfd, dstfd; + int flags = 0; + + if (!PyArg_ParseTuple(args, "iO&iO&|i:linkat", + &srcfd, PyUnicode_FSConverter, &osrc, &dstfd, PyUnicode_FSConverter, &odst, &flags)) + return NULL; + src = PyBytes_AsString(osrc); + dst = PyBytes_AsString(odst); + Py_BEGIN_ALLOW_THREADS + res = linkat(srcfd, src, dstfd, dst, flags); + Py_END_ALLOW_THREADS + Py_DECREF(osrc); + Py_DECREF(odst); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_LINKAT */ + +#ifdef HAVE_MKDIRAT +PyDoc_STRVAR(posix_mkdirat__doc__, +"mkdirat(dirfd, path, mode=0o777)\n\n\ +Like mkdir() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mkdirat(PyObject *self, PyObject *args) +{ + int res, dirfd; + PyObject *opath; + char *path; + int mode = 0777; + + if (!PyArg_ParseTuple(args, "iO&|i:mkdirat", + &dirfd, PyUnicode_FSConverter, &opath, &mode)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = mkdirat(dirfd, path, mode); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#if defined(HAVE_MKNODAT) && defined(HAVE_MAKEDEV) +PyDoc_STRVAR(posix_mknodat__doc__, +"mknodat(dirfd, path, mode=0o600, device=0)\n\n\ +Like mknod() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mknodat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *filename; + int mode = 0600; + int device = 0; + int res, dirfd; + if (!PyArg_ParseTuple(args, "iO&|ii:mknodat", &dirfd, + PyUnicode_FSConverter, &opath, &mode, &device)) + return NULL; + filename = PyBytes_AS_STRING(opath); + Py_BEGIN_ALLOW_THREADS + res = mknodat(dirfd, filename, mode, device); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_OPENAT +PyDoc_STRVAR(posix_openat__doc__, +"openat(dirfd, path, flag, mode=0o777) -> fd\n\n\ +Like open() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_openat(PyObject *self, PyObject *args) +{ + PyObject *ofile; + char *file; + int flag, dirfd, fd; + int mode = 0777; + + if (!PyArg_ParseTuple(args, "iO&i|i:openat", + &dirfd, PyUnicode_FSConverter, &ofile, + &flag, &mode)) + return NULL; + file = PyBytes_AsString(ofile); + Py_BEGIN_ALLOW_THREADS + fd = openat(dirfd, file, flag, mode); + Py_END_ALLOW_THREADS + Py_DECREF(ofile); + if (fd < 0) + return posix_error(); + return PyLong_FromLong((long)fd); +} +#endif + +#ifdef HAVE_READLINKAT +PyDoc_STRVAR(posix_readlinkat__doc__, +"readlinkat(dirfd, path) -> path\n\n\ +Like readlink() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_readlinkat(PyObject *self, PyObject *args) +{ + PyObject *v, *opath; + char buf[MAXPATHLEN]; + char *path; + int n, dirfd; + int arg_is_unicode = 0; + + if (!PyArg_ParseTuple(args, "iO&:readlinkat", + &dirfd, PyUnicode_FSConverter, &opath)) + return NULL; + path = PyBytes_AsString(opath); + v = PySequence_GetItem(args, 1); + if (v == NULL) { + Py_DECREF(opath); + return NULL; + } + + if (PyUnicode_Check(v)) { + arg_is_unicode = 1; + } + Py_DECREF(v); + + Py_BEGIN_ALLOW_THREADS + n = readlinkat(dirfd, path, buf, (int) sizeof buf); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (n < 0) + return posix_error(); + + if (arg_is_unicode) + return PyUnicode_DecodeFSDefaultAndSize(buf, n); + else + return PyBytes_FromStringAndSize(buf, n); +} +#endif /* HAVE_READLINKAT */ + +#ifdef HAVE_RENAMEAT +PyDoc_STRVAR(posix_renameat__doc__, +"renameat(olddirfd, oldpath, newdirfd, newpath)\n\n\ +Like rename() but if oldpath is relative, it is taken as relative to\n\ +olddirfd and if newpath is relative, it is taken as relative to newdirfd.\n\ +If oldpath is relative and olddirfd is the special value AT_FDCWD, then\n\ +oldpath is interpreted relative to the current working directory. This\n\ +also applies for newpath."); + +static PyObject * +posix_renameat(PyObject *self, PyObject *args) +{ + int res; + PyObject *opathold, *opathnew; + char *opath, *npath; + int oldfd, newfd; + + if (!PyArg_ParseTuple(args, "iO&iO&:renameat", + &oldfd, PyUnicode_FSConverter, &opathold, &newfd, PyUnicode_FSConverter, &opathnew)) + return NULL; + opath = PyBytes_AsString(opathold); + npath = PyBytes_AsString(opathnew); + Py_BEGIN_ALLOW_THREADS + res = renameat(oldfd, opath, newfd, npath); + Py_END_ALLOW_THREADS + Py_DECREF(opathold); + Py_DECREF(opathnew); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#if HAVE_SYMLINKAT +PyDoc_STRVAR(posix_symlinkat__doc__, +"symlinkat(src, dstfd, dst)\n\n\ +Like symlink() but if dst is relative, it is taken as relative to dstfd.\n\ +If dst is relative and dstfd is the special value AT_FDCWD, then dst\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_symlinkat(PyObject *self, PyObject *args) +{ + int res, dstfd; + PyObject *osrc, *odst; + char *src, *dst; + + if (!PyArg_ParseTuple(args, "O&iO&:symlinkat", + PyUnicode_FSConverter, &osrc, &dstfd, PyUnicode_FSConverter, &odst)) + return NULL; + src = PyBytes_AsString(osrc); + dst = PyBytes_AsString(odst); + Py_BEGIN_ALLOW_THREADS + res = symlinkat(src, dstfd, dst); + Py_END_ALLOW_THREADS + Py_DECREF(osrc); + Py_DECREF(odst); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_SYMLINKAT */ + +#ifdef HAVE_UNLINKAT +PyDoc_STRVAR(posix_unlinkat__doc__, +"unlinkat(dirfd, path, flags=0)\n\n\ +Like unlink() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_REMOVEDIR. If AT_REMOVEDIR is\n\ +specified, unlinkat() behaves like rmdir().\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_unlinkat(PyObject *self, PyObject *args) +{ + int dirfd, res, flags = 0; + PyObject *opath; + char *path; + + if (!PyArg_ParseTuple(args, "iO&|i:unlinkat", + &dirfd, PyUnicode_FSConverter, &opath, &flags)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = unlinkat(dirfd, path, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_UTIMENSAT +PyDoc_STRVAR(posix_utimensat__doc__, +"utimensat(dirfd, path, (atime_sec, atime_nsec),\n\ + (mtime_sec, mtime_nsec), flags)\n\ +utimensat(dirfd, path, None, None, flags)\n\n\ +Updates the timestamps of a file with nanosecond precision. If path is\n\ +relative, it is taken as relative to dirfd.\n\ +The second form sets atime and mtime to the current time.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory.\n\ +If *_nsec is specified as UTIME_NOW, the timestamp is updated to the\n\ +current time.\n\ +If *_nsec is specified as UTIME_OMIT, the timestamp is not updated."); + +static PyObject * +posix_utimensat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int res, dirfd, flags = 0; + PyObject *atime, *mtime; + + struct timespec buf[2]; + + if (!PyArg_ParseTuple(args, "iO&OO|i:utimensat", + &dirfd, PyUnicode_FSConverter, &opath, &atime, &mtime, &flags)) + return NULL; + path = PyBytes_AsString(opath); + if (atime == Py_None && mtime == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = utimensat(dirfd, path, NULL, flags); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(atime) || PyTuple_Size(atime) != 2) { + PyErr_SetString(PyExc_TypeError, + "utimensat() arg 3 must be a tuple (atime_sec, atime_nsec)"); + Py_DECREF(opath); + return NULL; + } + else if (!PyTuple_Check(mtime) || PyTuple_Size(mtime) != 2) { + PyErr_SetString(PyExc_TypeError, + "utimensat() arg 4 must be a tuple (mtime_sec, mtime_nsec)"); + Py_DECREF(opath); + return NULL; + } + else { + if (!PyArg_ParseTuple(atime, "ll:utimensat", + &(buf[0].tv_sec), &(buf[0].tv_nsec))) { + Py_DECREF(opath); + return NULL; + } + if (!PyArg_ParseTuple(mtime, "ll:utimensat", + &(buf[1].tv_sec), &(buf[1].tv_nsec))) { + Py_DECREF(opath); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + res = utimensat(dirfd, path, buf, flags); + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_MKFIFOAT +PyDoc_STRVAR(posix_mkfifoat__doc__, +"mkfifoat(dirfd, path, mode=0o666)\n\n\ +Like mkfifo() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mkfifoat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *filename; + int mode = 0666; + int res, dirfd; + if (!PyArg_ParseTuple(args, "iO&|i:mkfifoat", + &dirfd, PyUnicode_FSConverter, &opath, &mode)) + return NULL; + filename = PyBytes_AS_STRING(opath); + Py_BEGIN_ALLOW_THREADS + res = mkfifoat(dirfd, filename, mode); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef USE_XATTRS + +static int +try_getxattr(const char *path, const char *name, + ssize_t (*get)(const char *, const char *, void *, size_t), + Py_ssize_t buf_size, PyObject **res) +{ + PyObject *value; + Py_ssize_t len; + + assert(buf_size <= XATTR_SIZE_MAX); + value = PyBytes_FromStringAndSize(NULL, buf_size); + if (!value) + return 0; + Py_BEGIN_ALLOW_THREADS; + len = get(path, name, PyBytes_AS_STRING(value), buf_size); + Py_END_ALLOW_THREADS; + if (len < 0) { + Py_DECREF(value); + if (errno == ERANGE) { + value = NULL; + } + else { + posix_error(); + return 0; + } + } + else if (len != buf_size) { + /* Can only shrink. */ + _PyBytes_Resize(&value, len); + } + *res = value; + return 1; +} + +static PyObject * +getxattr_common(const char *path, PyObject *name_obj, + ssize_t (*get)(const char *, const char *, void *, size_t)) +{ + PyObject *value; + const char *name = PyBytes_AS_STRING(name_obj); + + /* Try a small value first. */ + if (!try_getxattr(path, name, get, 128, &value)) + return NULL; + if (value) + return value; + /* Now the maximum possible one. */ + if (!try_getxattr(path, name, get, XATTR_SIZE_MAX, &value)) + return NULL; + assert(value); + return value; +} + +PyDoc_STRVAR(posix_getxattr__doc__, +"getxattr(path, attr) -> value\n\n\ +Return the value of extended attribute *name* on *path*."); + +static PyObject * +posix_getxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *res, *name; + + if (!PyArg_ParseTuple(args, "O&O&:getxattr", PyUnicode_FSConverter, &path, + PyUnicode_FSConverter, &name)) + return NULL; + res = getxattr_common(PyBytes_AS_STRING(path), name, getxattr); + Py_DECREF(path); + Py_DECREF(name); + return res; +} + +PyDoc_STRVAR(posix_lgetxattr__doc__, +"lgetxattr(path, attr) -> value\n\n\ +Like getxattr but don't follow symlinks."); + +static PyObject * +posix_lgetxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *res, *name; + + if (!PyArg_ParseTuple(args, "O&O&:lgetxattr", PyUnicode_FSConverter, &path, + PyUnicode_FSConverter, &name)) + return NULL; + res = getxattr_common(PyBytes_AS_STRING(path), name, lgetxattr); + Py_DECREF(path); + Py_DECREF(name); + return res; +} + +static ssize_t +wrap_fgetxattr(const char *path, const char *name, void *value, size_t size) +{ + /* Hack to share code. */ + return fgetxattr((int)(Py_uintptr_t)path, name, value, size); +} + +PyDoc_STRVAR(posix_fgetxattr__doc__, +"fgetxattr(fd, attr) -> value\n\n\ +Like getxattr but operate on a fd instead of a path."); + +static PyObject * +posix_fgetxattr(PyObject *self, PyObject *args) +{ + PyObject *res, *name; + int fd; + + if (!PyArg_ParseTuple(args, "iO&:fgetxattr", &fd, PyUnicode_FSConverter, &name)) + return NULL; + res = getxattr_common((const char *)(Py_uintptr_t)fd, name, wrap_fgetxattr); + Py_DECREF(name); + return res; +} + +PyDoc_STRVAR(posix_setxattr__doc__, +"setxattr(path, attr, value, flags=0)\n\n\ +Set extended attribute *attr* on *path* to *value*."); + +static PyObject * +posix_setxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *name; + Py_buffer data; + int flags = 0, err; + + if (!PyArg_ParseTuple(args, "O&O&y*|i:setxattr", PyUnicode_FSConverter, + &path, PyUnicode_FSConverter, &name, &data, &flags)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = setxattr(PyBytes_AS_STRING(path), PyBytes_AS_STRING(name), + data.buf, data.len, flags); + Py_END_ALLOW_THREADS; + Py_DECREF(path); + Py_DECREF(name); + PyBuffer_Release(&data); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_lsetxattr__doc__, +"lsetxattr(path, attr, value, flags=0)\n\n\ +Like setxattr but don't follow symlinks."); + +static PyObject * +posix_lsetxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *name; + Py_buffer data; + int flags = 0, err; + + if (!PyArg_ParseTuple(args, "O&O&y*|i:lsetxattr", PyUnicode_FSConverter, + &path, PyUnicode_FSConverter, &name, &data, &flags)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = lsetxattr(PyBytes_AS_STRING(path), PyBytes_AS_STRING(name), + data.buf, data.len, flags); + Py_END_ALLOW_THREADS; + Py_DECREF(path); + Py_DECREF(name); + PyBuffer_Release(&data); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_fsetxattr__doc__, +"fsetxattr(fd, attr, value, flags=0)\n\n\ +Like setxattr but operates on *fd* instead of a path."); + +static PyObject * +posix_fsetxattr(PyObject *self, PyObject *args) +{ + Py_buffer data; + const char *name; + int fd, flags = 0, err; + + if (!PyArg_ParseTuple(args, "iO&y*|i:fsetxattr", &fd, PyUnicode_FSConverter, + &name, &data, &flags)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = fsetxattr(fd, PyBytes_AS_STRING(name), data.buf, data.len, flags); + Py_END_ALLOW_THREADS; + Py_DECREF(name); + PyBuffer_Release(&data); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_removexattr__doc__, +"removexattr(path, attr)\n\n\ +Remove extended attribute *attr* on *path*."); + +static PyObject * +posix_removexattr(PyObject *self, PyObject *args) +{ + PyObject *path, *name; + int err; + + if (!PyArg_ParseTuple(args, "O&O&:removexattr", PyUnicode_FSConverter, &path, + PyUnicode_FSConverter, &name)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = removexattr(PyBytes_AS_STRING(path), PyBytes_AS_STRING(name)); + Py_END_ALLOW_THREADS; + Py_DECREF(path); + Py_DECREF(name); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_lremovexattr__doc__, +"lremovexattr(path, attr)\n\n\ +Like removexattr but don't follow symlinks."); + +static PyObject * +posix_lremovexattr(PyObject *self, PyObject *args) +{ + PyObject *path, *name; + int err; + + if (!PyArg_ParseTuple(args, "O&O&:lremovexattr", PyUnicode_FSConverter, &path, + PyUnicode_FSConverter, &name)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = lremovexattr(PyBytes_AS_STRING(path), PyBytes_AS_STRING(name)); + Py_END_ALLOW_THREADS; + Py_DECREF(path); + Py_DECREF(name); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_fremovexattr__doc__, +"fremovexattr(fd, attr)\n\n\ +Like removexattr but operates on a file descriptor."); + +static PyObject * +posix_fremovexattr(PyObject *self, PyObject *args) +{ + PyObject *name; + int fd, err; + + if (!PyArg_ParseTuple(args, "iO&:fremovexattr", &fd, + PyUnicode_FSConverter, &name)) + return NULL; + Py_BEGIN_ALLOW_THREADS; + err = fremovexattr(fd, PyBytes_AS_STRING(name)); + Py_END_ALLOW_THREADS; + Py_DECREF(name); + if (err) + return posix_error(); + Py_RETURN_NONE; +} + +static Py_ssize_t +try_listxattr(const char *path, ssize_t (*list)(const char *, char *, size_t), + Py_ssize_t buf_size, char **buf) +{ + Py_ssize_t len; + + *buf = PyMem_MALLOC(buf_size); + if (!*buf) { + PyErr_NoMemory(); + return -1; + } + Py_BEGIN_ALLOW_THREADS; + len = list(path, *buf, buf_size); + Py_END_ALLOW_THREADS; + if (len < 0) { + PyMem_FREE(*buf); + if (errno != ERANGE) + posix_error(); + return -1; + } + return len; +} + +static PyObject * +listxattr_common(const char *path, ssize_t (*list)(const char *, char *, size_t)) +{ + PyObject *res, *attr; + Py_ssize_t len, err, start, i; + char *buf; + + len = try_listxattr(path, list, 256, &buf); + if (len < 0) { + if (PyErr_Occurred()) + return NULL; + len = try_listxattr(path, list, XATTR_LIST_MAX, &buf); + if (len < 0) + return NULL; + } + res = PyList_New(0); + if (!res) { + PyMem_FREE(buf); + return NULL; + } + for (start = i = 0; i < len; i++) { + if (!buf[i]) { + attr = PyUnicode_DecodeFSDefaultAndSize(&buf[start], i - start); + if (!attr) { + Py_DECREF(res); + PyMem_FREE(buf); + return NULL; + } + err = PyList_Append(res, attr); + Py_DECREF(attr); + if (err) { + Py_DECREF(res); + PyMem_FREE(buf); + return NULL; + } + start = i + 1; + } + } + PyMem_FREE(buf); + return res; +} + +PyDoc_STRVAR(posix_listxattr__doc__, +"listxattr(path)\n\n\ +Return a list of extended attributes on *path*."); + +static PyObject * +posix_listxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *res; + + if (!PyArg_ParseTuple(args, "O&:listxattr", PyUnicode_FSConverter, &path)) + return NULL; + res = listxattr_common(PyBytes_AS_STRING(path), listxattr); + Py_DECREF(path); + return res; +} + +PyDoc_STRVAR(posix_llistxattr__doc__, +"llistxattr(path)\n\n\ +Like listxattr but don't follow symlinks.."); + +static PyObject * +posix_llistxattr(PyObject *self, PyObject *args) +{ + PyObject *path, *res; + + if (!PyArg_ParseTuple(args, "O&:llistxattr", PyUnicode_FSConverter, &path)) + return NULL; + res = listxattr_common(PyBytes_AS_STRING(path), llistxattr); + Py_DECREF(path); + return res; +} + +static ssize_t +wrap_flistxattr(const char *path, char *buf, size_t len) +{ + /* Hack to share code. */ + return flistxattr((int)(Py_uintptr_t)path, buf, len); +} + +PyDoc_STRVAR(posix_flistxattr__doc__, +"flistxattr(path)\n\n\ +Like flistxattr but operates on a file descriptor."); + +static PyObject * +posix_flistxattr(PyObject *self, PyObject *args) +{ + long fd; + + if (!PyArg_ParseTuple(args, "i:flistxattr", &fd)) + return NULL; + return listxattr_common((const char *)(Py_uintptr_t)fd, wrap_flistxattr); +} + +#endif /* USE_XATTRS */ + static PyMethodDef posix_methods[] = { {"access", posix_access, METH_VARARGS, posix_access__doc__}, #ifdef HAVE_TTYNAME @@ -7891,11 +10535,20 @@ static PyMethodDef posix_methods[] = { {"link", posix_link, METH_VARARGS, posix_link__doc__}, #endif /* HAVE_LINK */ {"listdir", posix_listdir, METH_VARARGS, posix_listdir__doc__}, +#ifdef HAVE_FDOPENDIR + {"fdlistdir", posix_fdlistdir, METH_VARARGS, posix_fdlistdir__doc__}, +#endif {"lstat", posix_lstat, METH_VARARGS, posix_lstat__doc__}, {"mkdir", posix_mkdir, METH_VARARGS, posix_mkdir__doc__}, #ifdef HAVE_NICE {"nice", posix_nice, METH_VARARGS, posix_nice__doc__}, #endif /* HAVE_NICE */ +#ifdef HAVE_GETPRIORITY + {"getpriority", posix_getpriority, METH_VARARGS, posix_getpriority__doc__}, +#endif /* HAVE_GETPRIORITY */ +#ifdef HAVE_SETPRIORITY + {"setpriority", posix_setpriority, METH_VARARGS, posix_setpriority__doc__}, +#endif /* HAVE_SETPRIORITY */ #ifdef HAVE_READLINK {"readlink", posix_readlink, METH_VARARGS, posix_readlink__doc__}, #endif /* HAVE_READLINK */ @@ -7923,6 +10576,15 @@ static PyMethodDef posix_methods[] = { {"unlink", posix_unlink, METH_VARARGS, posix_unlink__doc__}, {"remove", posix_unlink, METH_VARARGS, posix_remove__doc__}, {"utime", posix_utime, METH_VARARGS, posix_utime__doc__}, +#ifdef HAVE_FUTIMES + {"futimes", posix_futimes, METH_VARARGS, posix_futimes__doc__}, +#endif +#ifdef HAVE_LUTIMES + {"lutimes", posix_lutimes, METH_VARARGS, posix_lutimes__doc__}, +#endif +#ifdef HAVE_FUTIMENS + {"futimens", posix_futimens, METH_VARARGS, posix_futimens__doc__}, +#endif #ifdef HAVE_TIMES {"times", posix_times, METH_NOARGS, posix_times__doc__}, #endif /* HAVE_TIMES */ @@ -7931,6 +10593,9 @@ static PyMethodDef posix_methods[] = { {"execv", posix_execv, METH_VARARGS, posix_execv__doc__}, {"execve", posix_execve, METH_VARARGS, posix_execve__doc__}, #endif /* HAVE_EXECV */ +#ifdef HAVE_FEXECVE + {"fexecve", posix_fexecve, METH_VARARGS, posix_fexecve__doc__}, +#endif #ifdef HAVE_SPAWNV {"spawnv", posix_spawnv, METH_VARARGS, posix_spawnv__doc__}, {"spawnve", posix_spawnve, METH_VARARGS, posix_spawnve__doc__}, @@ -7945,6 +10610,32 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FORK {"fork", posix_fork, METH_NOARGS, posix_fork__doc__}, #endif /* HAVE_FORK */ +#ifdef HAVE_SCHED_H +#ifdef HAVE_SCHED_GET_PRIORITY_MAX + {"sched_get_priority_max", posix_sched_get_priority_max, METH_VARARGS, posix_sched_get_priority_max__doc__}, + {"sched_get_priority_min", posix_sched_get_priority_min, METH_VARARGS, posix_sched_get_priority_min__doc__}, +#endif +#ifdef HAVE_SCHED_SETPARAM + {"sched_getparam", posix_sched_getparam, METH_VARARGS, posix_sched_getparam__doc__}, +#endif +#ifdef HAVE_SCHED_SETSCHEDULER + {"sched_getscheduler", posix_sched_getscheduler, METH_VARARGS, posix_sched_getscheduler__doc__}, +#endif +#ifdef HAVE_SCHED_RR_GET_INTERVAL + {"sched_rr_get_interval", posix_sched_rr_get_interval, METH_VARARGS, posix_sched_rr_get_interval__doc__}, +#endif +#ifdef HAVE_SCHED_SETPARAM + {"sched_setparam", posix_sched_setparam, METH_VARARGS, posix_sched_setparam__doc__}, +#endif +#ifdef HAVE_SCHED_SETSCHEDULER + {"sched_setscheduler", posix_sched_setscheduler, METH_VARARGS, posix_sched_setscheduler__doc__}, +#endif + {"sched_yield", posix_sched_yield, METH_NOARGS, posix_sched_yield__doc__}, +#ifdef HAVE_SCHED_SETAFFINITY + {"sched_setaffinity", posix_sched_setaffinity, METH_VARARGS, posix_sched_setaffinity__doc__}, + {"sched_getaffinity", posix_sched_getaffinity, METH_VARARGS, posix_sched_getaffinity__doc__}, +#endif +#endif /* HAVE_SCHED_H */ #if defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX) {"openpty", posix_openpty, METH_NOARGS, posix_openpty__doc__}, #endif /* HAVE_OPENPTY || HAVE__GETPTY || HAVE_DEV_PTMX */ @@ -7960,6 +10651,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_GETGID {"getgid", posix_getgid, METH_NOARGS, posix_getgid__doc__}, #endif /* HAVE_GETGID */ +#ifdef HAVE_GETGROUPLIST + {"getgrouplist", posix_getgrouplist, METH_VARARGS, posix_getgrouplist__doc__}, +#endif #ifdef HAVE_GETGROUPS {"getgroups", posix_getgroups, METH_NOARGS, posix_getgroups__doc__}, #endif @@ -8029,6 +10723,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_WAIT4 {"wait4", posix_wait4, METH_VARARGS, posix_wait4__doc__}, #endif /* HAVE_WAIT4 */ +#if defined(HAVE_WAITID) && !defined(__APPLE__) + {"waitid", posix_waitid, METH_VARARGS, posix_waitid__doc__}, +#endif #if defined(HAVE_WAITPID) || defined(HAVE_CWAIT) {"waitpid", posix_waitpid, METH_VARARGS, posix_waitpid__doc__}, #endif /* HAVE_WAITPID */ @@ -8053,14 +10750,36 @@ static PyMethodDef posix_methods[] = { {"device_encoding", device_encoding, METH_VARARGS, device_encoding__doc__}, {"dup", posix_dup, METH_VARARGS, posix_dup__doc__}, {"dup2", posix_dup2, METH_VARARGS, posix_dup2__doc__}, +#ifdef HAVE_LOCKF + {"lockf", posix_lockf, METH_VARARGS, posix_lockf__doc__}, +#endif {"lseek", posix_lseek, METH_VARARGS, posix_lseek__doc__}, {"read", posix_read, METH_VARARGS, posix_read__doc__}, +#ifdef HAVE_READV + {"readv", posix_readv, METH_VARARGS, posix_readv__doc__}, +#endif +#ifdef HAVE_PREAD + {"pread", posix_pread, METH_VARARGS, posix_pread__doc__}, +#endif {"write", posix_write, METH_VARARGS, posix_write__doc__}, +#ifdef HAVE_WRITEV + {"writev", posix_writev, METH_VARARGS, posix_writev__doc__}, +#endif +#ifdef HAVE_PWRITE + {"pwrite", posix_pwrite, METH_VARARGS, posix_pwrite__doc__}, +#endif +#ifdef HAVE_SENDFILE + {"sendfile", (PyCFunction)posix_sendfile, METH_VARARGS | METH_KEYWORDS, + posix_sendfile__doc__}, +#endif {"fstat", posix_fstat, METH_VARARGS, posix_fstat__doc__}, {"isatty", posix_isatty, METH_VARARGS, posix_isatty__doc__}, #ifdef HAVE_PIPE {"pipe", posix_pipe, METH_NOARGS, posix_pipe__doc__}, #endif +#ifdef HAVE_PIPE2 + {"pipe2", posix_pipe2, METH_O, posix_pipe2__doc__}, +#endif #ifdef HAVE_MKFIFO {"mkfifo", posix_mkfifo, METH_VARARGS, posix_mkfifo__doc__}, #endif @@ -8075,6 +10794,15 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FTRUNCATE {"ftruncate", posix_ftruncate, METH_VARARGS, posix_ftruncate__doc__}, #endif +#ifdef HAVE_TRUNCATE + {"truncate", posix_truncate, METH_VARARGS, posix_truncate__doc__}, +#endif +#ifdef HAVE_POSIX_FALLOCATE + {"posix_fallocate", posix_posix_fallocate, METH_VARARGS, posix_posix_fallocate__doc__}, +#endif +#ifdef HAVE_POSIX_FADVISE + {"posix_fadvise", posix_posix_fadvise, METH_VARARGS, posix_posix_fadvise__doc__}, +#endif #ifdef HAVE_PUTENV {"putenv", posix_putenv, METH_VARARGS, posix_putenv__doc__}, #endif @@ -8088,6 +10816,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FSYNC {"fsync", posix_fsync, METH_O, posix_fsync__doc__}, #endif +#ifdef HAVE_SYNC + {"sync", posix_sync, METH_NOARGS, posix_sync__doc__}, +#endif #ifdef HAVE_FDATASYNC {"fdatasync", posix_fdatasync, METH_O, posix_fdatasync__doc__}, #endif @@ -8141,6 +10872,7 @@ static PyMethodDef posix_methods[] = { {"_getfinalpathname", posix__getfinalpathname, METH_VARARGS, NULL}, {"_getfileinformation", posix__getfileinformation, METH_VARARGS, NULL}, {"_isdir", posix__isdir, METH_VARARGS, posix__isdir__doc__}, + {"_getdiskusage", win32__getdiskusage, METH_VARARGS, win32__getdiskusage__doc__}, #endif #ifdef HAVE_GETLOADAVG {"getloadavg", posix_getloadavg, METH_NOARGS, posix_getloadavg__doc__}, @@ -8164,6 +10896,66 @@ static PyMethodDef posix_methods[] = { {"getresgid", posix_getresgid, METH_NOARGS, posix_getresgid__doc__}, #endif +/* posix *at family of functions */ +#ifdef HAVE_FACCESSAT + {"faccessat", posix_faccessat, METH_VARARGS, posix_faccessat__doc__}, +#endif +#ifdef HAVE_FCHMODAT + {"fchmodat", posix_fchmodat, METH_VARARGS, posix_fchmodat__doc__}, +#endif /* HAVE_FCHMODAT */ +#ifdef HAVE_FCHOWNAT + {"fchownat", posix_fchownat, METH_VARARGS, posix_fchownat__doc__}, +#endif /* HAVE_FCHOWNAT */ +#ifdef HAVE_FSTATAT + {"fstatat", posix_fstatat, METH_VARARGS, posix_fstatat__doc__}, +#endif +#ifdef HAVE_FUTIMESAT + {"futimesat", posix_futimesat, METH_VARARGS, posix_futimesat__doc__}, +#endif +#ifdef HAVE_LINKAT + {"linkat", posix_linkat, METH_VARARGS, posix_linkat__doc__}, +#endif /* HAVE_LINKAT */ +#ifdef HAVE_MKDIRAT + {"mkdirat", posix_mkdirat, METH_VARARGS, posix_mkdirat__doc__}, +#endif +#if defined(HAVE_MKNODAT) && defined(HAVE_MAKEDEV) + {"mknodat", posix_mknodat, METH_VARARGS, posix_mknodat__doc__}, +#endif +#ifdef HAVE_OPENAT + {"openat", posix_openat, METH_VARARGS, posix_openat__doc__}, +#endif +#ifdef HAVE_READLINKAT + {"readlinkat", posix_readlinkat, METH_VARARGS, posix_readlinkat__doc__}, +#endif /* HAVE_READLINKAT */ +#ifdef HAVE_RENAMEAT + {"renameat", posix_renameat, METH_VARARGS, posix_renameat__doc__}, +#endif +#if HAVE_SYMLINKAT + {"symlinkat", posix_symlinkat, METH_VARARGS, posix_symlinkat__doc__}, +#endif /* HAVE_SYMLINKAT */ +#ifdef HAVE_UNLINKAT + {"unlinkat", posix_unlinkat, METH_VARARGS, posix_unlinkat__doc__}, +#endif +#ifdef HAVE_UTIMENSAT + {"utimensat", posix_utimensat, METH_VARARGS, posix_utimensat__doc__}, +#endif +#ifdef HAVE_MKFIFOAT + {"mkfifoat", posix_mkfifoat, METH_VARARGS, posix_mkfifoat__doc__}, +#endif +#ifdef USE_XATTRS + {"setxattr", posix_setxattr, METH_VARARGS, posix_setxattr__doc__}, + {"lsetxattr", posix_lsetxattr, METH_VARARGS, posix_lsetxattr__doc__}, + {"fsetxattr", posix_fsetxattr, METH_VARARGS, posix_fsetxattr__doc__}, + {"getxattr", posix_getxattr, METH_VARARGS, posix_getxattr__doc__}, + {"lgetxattr", posix_lgetxattr, METH_VARARGS, posix_lgetxattr__doc__}, + {"fgetxattr", posix_fgetxattr, METH_VARARGS, posix_fgetxattr__doc__}, + {"removexattr", posix_removexattr, METH_VARARGS, posix_removexattr__doc__}, + {"lremovexattr", posix_lremovexattr, METH_VARARGS, posix_lremovexattr__doc__}, + {"fremovexattr", posix_fremovexattr, METH_VARARGS, posix_fremovexattr__doc__}, + {"listxattr", posix_listxattr, METH_VARARGS, posix_listxattr__doc__}, + {"llistxattr", posix_llistxattr, METH_VARARGS, posix_llistxattr__doc__}, + {"flistxattr", posix_flistxattr, METH_VARARGS, posix_flistxattr__doc__}, +#endif {NULL, NULL} /* Sentinel */ }; @@ -8340,6 +11132,41 @@ all_ins(PyObject *d) #ifdef O_EXLOCK if (ins(d, "O_EXLOCK", (long)O_EXLOCK)) return -1; #endif +#ifdef PRIO_PROCESS + if (ins(d, "PRIO_PROCESS", (long)PRIO_PROCESS)) return -1; +#endif +#ifdef PRIO_PGRP + if (ins(d, "PRIO_PGRP", (long)PRIO_PGRP)) return -1; +#endif +#ifdef PRIO_USER + if (ins(d, "PRIO_USER", (long)PRIO_USER)) return -1; +#endif +#ifdef O_CLOEXEC + if (ins(d, "O_CLOEXEC", (long)O_CLOEXEC)) return -1; +#endif +/* posix - constants for *at functions */ +#ifdef AT_SYMLINK_NOFOLLOW + if (ins(d, "AT_SYMLINK_NOFOLLOW", (long)AT_SYMLINK_NOFOLLOW)) return -1; +#endif +#ifdef AT_EACCESS + if (ins(d, "AT_EACCESS", (long)AT_EACCESS)) return -1; +#endif +#ifdef AT_FDCWD + if (ins(d, "AT_FDCWD", (long)AT_FDCWD)) return -1; +#endif +#ifdef AT_REMOVEDIR + if (ins(d, "AT_REMOVEDIR", (long)AT_REMOVEDIR)) return -1; +#endif +#ifdef AT_SYMLINK_FOLLOW + if (ins(d, "AT_SYMLINK_FOLLOW", (long)AT_SYMLINK_FOLLOW)) return -1; +#endif +#ifdef UTIME_NOW + if (ins(d, "UTIME_NOW", (long)UTIME_NOW)) return -1; +#endif +#ifdef UTIME_OMIT + if (ins(d, "UTIME_OMIT", (long)UTIME_OMIT)) return -1; +#endif + /* MS Windows */ #ifdef O_NOINHERIT @@ -8448,6 +11275,87 @@ all_ins(PyObject *d) if (ins(d, "ST_NOSUID", (long)ST_NOSUID)) return -1; #endif /* ST_NOSUID */ + /* FreeBSD sendfile() constants */ +#ifdef SF_NODISKIO + if (ins(d, "SF_NODISKIO", (long)SF_NODISKIO)) return -1; +#endif +#ifdef SF_MNOWAIT + if (ins(d, "SF_MNOWAIT", (long)SF_MNOWAIT)) return -1; +#endif +#ifdef SF_SYNC + if (ins(d, "SF_SYNC", (long)SF_SYNC)) return -1; +#endif + + /* constants for posix_fadvise */ +#ifdef POSIX_FADV_NORMAL + if (ins(d, "POSIX_FADV_NORMAL", (long)POSIX_FADV_NORMAL)) return -1; +#endif +#ifdef POSIX_FADV_SEQUENTIAL + if (ins(d, "POSIX_FADV_SEQUENTIAL", (long)POSIX_FADV_SEQUENTIAL)) return -1; +#endif +#ifdef POSIX_FADV_RANDOM + if (ins(d, "POSIX_FADV_RANDOM", (long)POSIX_FADV_RANDOM)) return -1; +#endif +#ifdef POSIX_FADV_NOREUSE + if (ins(d, "POSIX_FADV_NOREUSE", (long)POSIX_FADV_NOREUSE)) return -1; +#endif +#ifdef POSIX_FADV_WILLNEED + if (ins(d, "POSIX_FADV_WILLNEED", (long)POSIX_FADV_WILLNEED)) return -1; +#endif +#ifdef POSIX_FADV_DONTNEED + if (ins(d, "POSIX_FADV_DONTNEED", (long)POSIX_FADV_DONTNEED)) return -1; +#endif + + /* constants for waitid */ +#if defined(HAVE_SYS_WAIT_H) && defined(HAVE_WAITID) + if (ins(d, "P_PID", (long)P_PID)) return -1; + if (ins(d, "P_PGID", (long)P_PGID)) return -1; + if (ins(d, "P_ALL", (long)P_ALL)) return -1; +#endif +#ifdef WEXITED + if (ins(d, "WEXITED", (long)WEXITED)) return -1; +#endif +#ifdef WNOWAIT + if (ins(d, "WNOWAIT", (long)WNOWAIT)) return -1; +#endif +#ifdef WSTOPPED + if (ins(d, "WSTOPPED", (long)WSTOPPED)) return -1; +#endif +#ifdef CLD_EXITED + if (ins(d, "CLD_EXITED", (long)CLD_EXITED)) return -1; +#endif +#ifdef CLD_DUMPED + if (ins(d, "CLD_DUMPED", (long)CLD_DUMPED)) return -1; +#endif +#ifdef CLD_TRAPPED + if (ins(d, "CLD_TRAPPED", (long)CLD_TRAPPED)) return -1; +#endif +#ifdef CLD_CONTINUED + if (ins(d, "CLD_CONTINUED", (long)CLD_CONTINUED)) return -1; +#endif + + /* constants for lockf */ +#ifdef F_LOCK + if (ins(d, "F_LOCK", (long)F_LOCK)) return -1; +#endif +#ifdef F_TLOCK + if (ins(d, "F_TLOCK", (long)F_TLOCK)) return -1; +#endif +#ifdef F_ULOCK + if (ins(d, "F_ULOCK", (long)F_ULOCK)) return -1; +#endif +#ifdef F_TEST + if (ins(d, "F_TEST", (long)F_TEST)) return -1; +#endif + + /* constants for futimens */ +#ifdef UTIME_NOW + if (ins(d, "UTIME_NOW", (long)UTIME_NOW)) return -1; +#endif +#ifdef UTIME_OMIT + if (ins(d, "UTIME_OMIT", (long)UTIME_OMIT)) return -1; +#endif + #ifdef HAVE_SPAWNV #if defined(PYOS_OS2) && defined(PYCC_GCC) if (ins(d, "P_WAIT", (long)P_WAIT)) return -1; @@ -8479,6 +11387,42 @@ all_ins(PyObject *d) #endif #endif +#ifdef HAVE_SCHED_H + if (ins(d, "SCHED_OTHER", (long)SCHED_OTHER)) return -1; + if (ins(d, "SCHED_FIFO", (long)SCHED_FIFO)) return -1; + if (ins(d, "SCHED_RR", (long)SCHED_RR)) return -1; +#ifdef SCHED_SPORADIC + if (ins(d, "SCHED_SPORADIC", (long)SCHED_SPORADIC) return -1; +#endif +#ifdef SCHED_BATCH + if (ins(d, "SCHED_BATCH", (long)SCHED_BATCH)) return -1; +#endif +#ifdef SCHED_IDLE + if (ins(d, "SCHED_IDLE", (long)SCHED_IDLE)) return -1; +#endif +#ifdef SCHED_RESET_ON_FORK + if (ins(d, "SCHED_RESET_ON_FORK", (long)SCHED_RESET_ON_FORK)) return -1; +#endif +#ifdef SCHED_SYS + if (ins(d, "SCHED_SYS", (long)SCHED_SYS)) return -1; +#endif +#ifdef SCHED_IA + if (ins(d, "SCHED_IA", (long)SCHED_IA)) return -1; +#endif +#ifdef SCHED_FSS + if (ins(d, "SCHED_FSS", (long)SCHED_FSS)) return -1; +#endif +#ifdef SCHED_FX + if (ins(d, "SCHED_FX", (long)SCHED_FSS)) return -1; +#endif +#endif + +#ifdef USE_XATTRS + if (ins(d, "XATTR_CREATE", (long)XATTR_CREATE)) return -1; + if (ins(d, "XATTR_REPLACE", (long)XATTR_REPLACE)) return -1; + if (ins(d, "XATTR_SIZE_MAX", (long)XATTR_SIZE_MAX)) return -1; +#endif + #if defined(PYOS_OS2) if (insertvalues(d)) return -1; #endif @@ -8541,12 +11485,24 @@ INITFUNC(void) Py_INCREF(PyExc_OSError); PyModule_AddObject(m, "error", PyExc_OSError); +#ifdef HAVE_SCHED_SETAFFINITY + if (PyType_Ready(&cpu_set_type) < 0) + return NULL; + Py_INCREF(&cpu_set_type); + PyModule_AddObject(m, "cpu_set", (PyObject *)&cpu_set_type); +#endif + #ifdef HAVE_PUTENV if (posix_putenv_garbage == NULL) posix_putenv_garbage = PyDict_New(); #endif if (!initialized) { +#if defined(HAVE_WAITID) && !defined(__APPLE__) + waitid_result_desc.name = MODNAME ".waitid_result"; + PyStructSequence_InitType(&WaitidResultType, &waitid_result_desc); +#endif + stat_result_desc.name = MODNAME ".stat_result"; stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; @@ -8566,12 +11522,27 @@ INITFUNC(void) ticks_per_second = 60; /* magic fallback value; may be bogus */ # endif #endif + +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) + sched_param_desc.name = MODNAME ".sched_param"; + PyStructSequence_InitType(&SchedParamType, &sched_param_desc); + SchedParamType.tp_new = sched_param_new; +#endif } +#if defined(HAVE_WAITID) && !defined(__APPLE__) + Py_INCREF((PyObject*) &WaitidResultType); + PyModule_AddObject(m, "waitid_result", (PyObject*) &WaitidResultType); +#endif Py_INCREF((PyObject*) &StatResultType); PyModule_AddObject(m, "stat_result", (PyObject*) &StatResultType); Py_INCREF((PyObject*) &StatVFSResultType); PyModule_AddObject(m, "statvfs_result", (PyObject*) &StatVFSResultType); + +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) + Py_INCREF(&SchedParamType); + PyModule_AddObject(m, "sched_param", (PyObject *)&SchedParamType); +#endif initialized = 1; #ifdef __APPLE__ diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 849423f..45ab70e 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -100,16 +100,17 @@ static PyObject * set_error(xmlparseobject *self, enum XML_Error code) { PyObject *err; - char buffer[256]; + PyObject *buffer; XML_Parser parser = self->itself; int lineno = XML_GetErrorLineNumber(parser); int column = XML_GetErrorColumnNumber(parser); - /* There is no risk of overflowing this buffer, since - even for 64-bit integers, there is sufficient space. */ - sprintf(buffer, "%.200s: line %i, column %i", - XML_ErrorString(code), lineno, column); - err = PyObject_CallFunction(ErrorObject, "s", buffer); + buffer = PyUnicode_FromFormat("%s: line %i, column %i", + XML_ErrorString(code), lineno, column); + if (buffer == NULL) + return NULL; + err = PyObject_CallFunction(ErrorObject, "O", buffer); + Py_DECREF(buffer); if ( err != NULL && set_error_attr(err, "code", code) && set_error_attr(err, "offset", column) @@ -842,9 +843,9 @@ xmlparse_ParseFile(xmlparseobject *self, PyObject *f) { int rv = 1; PyObject *readmethod = NULL; + _Py_identifier(read); - - readmethod = PyObject_GetAttrString(f, "read"); + readmethod = _PyObject_GetAttrId(f, &PyId_read); if (readmethod == NULL) { PyErr_SetString(PyExc_TypeError, "argument must have 'read' attribute"); @@ -1101,17 +1102,22 @@ PyUnknownEncodingHandler(void *encodingHandlerData, PyUnicodeObject *_u_string = NULL; int result = 0; int i; + int kind; + void *data; /* Yes, supports only 8bit encodings */ _u_string = (PyUnicodeObject *) PyUnicode_Decode(template_buffer, 256, name, "replace"); - if (_u_string == NULL) + if (_u_string == NULL || PyUnicode_READY(_u_string) == -1) return result; + kind = PyUnicode_KIND(_u_string); + data = PyUnicode_DATA(_u_string); + for (i = 0; i < 256; i++) { /* Stupid to access directly, but fast */ - Py_UNICODE c = _u_string->str[i]; + Py_UCS4 c = PyUnicode_READ(kind, data, i); if (c == Py_UNICODE_REPLACEMENT_CHARACTER) info->map[i] = -1; else @@ -1228,11 +1234,13 @@ get_pybool(int istrue) static PyObject * xmlparse_getattro(xmlparseobject *self, PyObject *nameobj) { - Py_UNICODE *name; + Py_UCS4 first_char; int handlernum = -1; if (!PyUnicode_Check(nameobj)) goto generic; + if (PyUnicode_READY(nameobj)) + return NULL; handlernum = handlername2int(nameobj); @@ -1244,8 +1252,8 @@ xmlparse_getattro(xmlparseobject *self, PyObject *nameobj) return result; } - name = PyUnicode_AS_UNICODE(nameobj); - if (name[0] == 'E') { + first_char = PyUnicode_READ_CHAR(nameobj, 0); + if (first_char == 'E') { if (PyUnicode_CompareWithASCIIString(nameobj, "ErrorCode") == 0) return PyLong_FromLong((long) XML_GetErrorCode(self->itself)); @@ -1259,7 +1267,7 @@ xmlparse_getattro(xmlparseobject *self, PyObject *nameobj) return PyLong_FromLong((long) XML_GetErrorByteIndex(self->itself)); } - if (name[0] == 'C') { + if (first_char == 'C') { if (PyUnicode_CompareWithASCIIString(nameobj, "CurrentLineNumber") == 0) return PyLong_FromLong((long) XML_GetCurrentLineNumber(self->itself)); @@ -1270,7 +1278,7 @@ xmlparse_getattro(xmlparseobject *self, PyObject *nameobj) return PyLong_FromLong((long) XML_GetCurrentByteIndex(self->itself)); } - if (name[0] == 'b') { + if (first_char == 'b') { if (PyUnicode_CompareWithASCIIString(nameobj, "buffer_size") == 0) return PyLong_FromLong((long) self->buffer_size); if (PyUnicode_CompareWithASCIIString(nameobj, "buffer_text") == 0) @@ -1661,7 +1669,6 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; - PyObject *version; PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; @@ -1698,10 +1705,6 @@ MODULE_INITFUNC(void) Py_INCREF(&Xmlparsetype); PyModule_AddObject(m, "XMLParserType", (PyObject *) &Xmlparsetype); - version = PyUnicode_FromString(PY_VERSION); - if (!version) - return NULL; - PyModule_AddObject(m, "__version__", version); PyModule_AddStringConstant(m, "EXPAT_VERSION", (char *) XML_ExpatVersion()); { diff --git a/Modules/readline.c b/Modules/readline.c index 8337956..a5e48ab 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -233,10 +233,9 @@ set_hook(const char *funcname, PyObject **hook_var, PyObject *args) Py_XDECREF(tmp); } else { - PyOS_snprintf(buf, sizeof(buf), - "set_%.50s(func): argument not callable", - funcname); - PyErr_SetString(PyExc_TypeError, buf); + PyErr_Format(PyExc_TypeError, + "set_%.50s(func): argument not callable", + funcname); return NULL; } Py_RETURN_NONE; @@ -890,7 +889,7 @@ setup_readline(void) #endif #ifdef __APPLE__ - /* the libedit readline emulation resets key bindings etc + /* the libedit readline emulation resets key bindings etc * when calling rl_initialize. So call it upfront */ if (using_libedit_emulation) @@ -930,11 +929,11 @@ setup_readline(void) */ #ifdef __APPLE__ if (using_libedit_emulation) - rl_read_init_file(NULL); + rl_read_init_file(NULL); else #endif /* __APPLE__ */ rl_initialize(); - + RESTORE_LOCALE(saved_locale) } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 2452a65..6071144 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -54,8 +54,6 @@ extern void bzero(void *, int); # endif #endif -static PyObject *SelectError; - /* list of Python objects and their file descriptor */ typedef struct { PyObject *obj; /* owned reference */ @@ -227,6 +225,11 @@ select_select(PyObject *self, PyObject *args) "timeout period too long"); return NULL; } + if (timeout < 0) { + PyErr_SetString(PyExc_ValueError, + "timeout must be non-negative"); + return NULL; + } seconds = (long)timeout; timeout = timeout - (double)seconds; tv.tv_sec = seconds; @@ -269,11 +272,11 @@ select_select(PyObject *self, PyObject *args) #ifdef MS_WINDOWS if (n == SOCKET_ERROR) { - PyErr_SetExcFromWindowsErr(SelectError, WSAGetLastError()); + PyErr_SetExcFromWindowsErr(PyExc_OSError, WSAGetLastError()); } #else if (n < 0) { - PyErr_SetFromErrno(SelectError); + PyErr_SetFromErrno(PyExc_OSError); } #endif else { @@ -420,7 +423,7 @@ poll_modify(pollObject *self, PyObject *args) return NULL; if (PyDict_GetItem(self->dict, key) == NULL) { errno = ENOENT; - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } value = PyLong_FromLong(events); @@ -519,7 +522,7 @@ poll_poll(pollObject *self, PyObject *args) Py_END_ALLOW_THREADS if (poll_result < 0) { - PyErr_SetFromErrno(SelectError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -759,7 +762,7 @@ newPyEpoll_Object(PyTypeObject *type, int sizehint, SOCKET fd) } if (self->epfd < 0) { Py_DECREF(self); - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } return (PyObject *)self; @@ -792,7 +795,7 @@ pyepoll_close(pyEpoll_Object *self) { errno = pyepoll_internal_close(self); if (errno < 0) { - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } Py_RETURN_NONE; @@ -885,7 +888,7 @@ pyepoll_internal_ctl(int epfd, int op, PyObject *pfd, unsigned int events) } if (result < 0) { - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } Py_RETURN_NONE; @@ -909,7 +912,7 @@ pyepoll_register(pyEpoll_Object *self, PyObject *args, PyObject *kwds) PyDoc_STRVAR(pyepoll_register_doc, "register(fd[, eventmask]) -> None\n\ \n\ -Registers a new fd or raises an IOError if the fd is already registered.\n\ +Registers a new fd or raises an OSError if the fd is already registered.\n\ fd is the target file descriptor of the operation.\n\ events is a bit set composed of the various EPOLL constants; the default\n\ is EPOLL_IN | EPOLL_OUT | EPOLL_PRI.\n\ @@ -1008,7 +1011,7 @@ pyepoll_poll(pyEpoll_Object *self, PyObject *args, PyObject *kwds) nfds = epoll_wait(self->epfd, evs, maxevents, timeout); Py_END_ALLOW_THREADS if (nfds < 0) { - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); goto error; } @@ -1399,7 +1402,7 @@ newKqueue_Object(PyTypeObject *type, SOCKET fd) } if (self->kqfd < 0) { Py_DECREF(self); - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } return (PyObject *)self; @@ -1431,7 +1434,7 @@ kqueue_queue_close(kqueue_queue_Object *self) { errno = kqueue_queue_internal_close(self); if (errno < 0) { - PyErr_SetFromErrno(PyExc_IOError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } Py_RETURN_NONE; @@ -1773,9 +1776,8 @@ PyInit_select(void) if (m == NULL) return NULL; - SelectError = PyErr_NewException("select.error", NULL, NULL); - Py_INCREF(SelectError); - PyModule_AddObject(m, "error", SelectError); + Py_INCREF(PyExc_OSError); + PyModule_AddObject(m, "error", PyExc_OSError); #ifdef PIPE_BUF #ifdef HAVE_BROKEN_PIPE_BUF diff --git a/Modules/sha1module.c b/Modules/sha1module.c index b25bd44..30e5c50 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -218,7 +218,7 @@ void sha1_process(struct sha1_state *sha1, in += SHA1_BLOCKSIZE; inlen -= SHA1_BLOCKSIZE; } else { - n = MIN(inlen, (SHA1_BLOCKSIZE - sha1->curlen)); + n = MIN(inlen, (Py_ssize_t)(SHA1_BLOCKSIZE - sha1->curlen)); memcpy(sha1->buf + sha1->curlen, in, (size_t)n); sha1->curlen += n; in += n; @@ -352,7 +352,7 @@ SHA1_hexdigest(SHA1object *self, PyObject *unused) unsigned char digest[SHA1_DIGESTSIZE]; struct sha1_state temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -360,24 +360,18 @@ SHA1_hexdigest(SHA1object *self, PyObject *unused) sha1_done(&temp, digest); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, SHA1_DIGESTSIZE * 2); + retval = PyUnicode_New(SHA1_DIGESTSIZE * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<SHA1_DIGESTSIZE; i++) { - char c; + unsigned char c; c = (digest[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; c = (digest[i] & 0xf); - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; } return retval; } diff --git a/Modules/sha256module.c b/Modules/sha256module.c index 8617210..f1ef329 100644 --- a/Modules/sha256module.c +++ b/Modules/sha256module.c @@ -445,7 +445,7 @@ SHA256_hexdigest(SHAobject *self, PyObject *unused) unsigned char digest[SHA_DIGESTSIZE]; SHAobject temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -453,24 +453,18 @@ SHA256_hexdigest(SHAobject *self, PyObject *unused) sha_final(digest, &temp); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, self->digestsize * 2); + retval = PyUnicode_New(self->digestsize * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for(i=j=0; i<self->digestsize; i++) { - char c; + unsigned char c; c = (digest[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; c = (digest[i] & 0xf); - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; } return retval; } diff --git a/Modules/sha512module.c b/Modules/sha512module.c index 0faaf5c..4f5a113 100644 --- a/Modules/sha512module.c +++ b/Modules/sha512module.c @@ -511,7 +511,7 @@ SHA512_hexdigest(SHAobject *self, PyObject *unused) unsigned char digest[SHA_DIGESTSIZE]; SHAobject temp; PyObject *retval; - Py_UNICODE *hex_digest; + Py_UCS1 *hex_digest; int i, j; /* Get the raw (binary) digest value */ @@ -519,24 +519,18 @@ SHA512_hexdigest(SHAobject *self, PyObject *unused) sha512_final(digest, &temp); /* Create a new string */ - retval = PyUnicode_FromStringAndSize(NULL, self->digestsize * 2); + retval = PyUnicode_New(self->digestsize * 2, 127); if (!retval) return NULL; - hex_digest = PyUnicode_AS_UNICODE(retval); - if (!hex_digest) { - Py_DECREF(retval); - return NULL; - } + hex_digest = PyUnicode_1BYTE_DATA(retval); /* Make hex version of the digest */ for (i=j=0; i<self->digestsize; i++) { - char c; + unsigned char c; c = (digest[i] >> 4) & 0xf; - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; c = (digest[i] & 0xf); - c = (c>9) ? c+'a'-10 : c + '0'; - hex_digest[j++] = c; + hex_digest[j++] = Py_hexdigits[c]; } return retval; } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 87c1c9a..45a7dfa 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -22,6 +22,14 @@ #include <sys/time.h> #endif +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) +# define PYPTHREAD_SIGMASK +#endif + +#if defined(PYPTHREAD_SIGMASK) && defined(HAVE_PTHREAD_H) +# include <pthread.h> +#endif + #ifndef SIG_ERR #define SIG_ERR ((PyOS_sighandler_t)(-1)) #endif @@ -168,15 +176,19 @@ checksignals_witharg(void * unused) static void trip_signal(int sig_num) { + unsigned char byte; + Handlers[sig_num].tripped = 1; + if (wakeup_fd != -1) { + byte = (unsigned char)sig_num; + write(wakeup_fd, &byte, 1); + } if (is_tripped) return; /* Set is_tripped after setting .tripped, as it gets cleared in PyErr_CheckSignals() before .tripped. */ is_tripped = 1; Py_AddPendingCall(checksignals_witharg, NULL); - if (wakeup_fd != -1) - write(wakeup_fd, "\0", 1); } static void @@ -313,7 +325,7 @@ signal_signal(PyObject *self, PyObject *args) else func = signal_handler; if (PyOS_setsig(sig_num, func) == SIG_ERR) { - PyErr_SetFromErrno(PyExc_RuntimeError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } old_handler = Handlers[sig_num].func; @@ -382,7 +394,7 @@ signal_siginterrupt(PyObject *self, PyObject *args) return NULL; } if (siginterrupt(sig_num, flag)<0) { - PyErr_SetFromErrno(PyExc_RuntimeError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -495,6 +507,346 @@ PyDoc_STRVAR(getitimer_doc, Returns current value of given itimer."); #endif +#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) || \ + defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) +/* Convert an iterable to a sigset. + Return 0 on success, return -1 and raise an exception on error. */ + +static int +iterable_to_sigset(PyObject *iterable, sigset_t *mask) +{ + int result = -1; + PyObject *iterator, *item; + long signum; + int err; + + sigemptyset(mask); + + iterator = PyObject_GetIter(iterable); + if (iterator == NULL) + goto error; + + while (1) + { + item = PyIter_Next(iterator); + if (item == NULL) { + if (PyErr_Occurred()) + goto error; + else + break; + } + + signum = PyLong_AsLong(item); + Py_DECREF(item); + if (signum == -1 && PyErr_Occurred()) + goto error; + if (0 < signum && signum < NSIG) + err = sigaddset(mask, (int)signum); + else + err = 1; + if (err) { + PyErr_Format(PyExc_ValueError, + "signal number %ld out of range", signum); + goto error; + } + } + result = 0; + +error: + Py_XDECREF(iterator); + return result; +} +#endif + +#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGPENDING) +static PyObject* +sigset_to_set(sigset_t mask) +{ + PyObject *signum, *result; + int sig; + + result = PySet_New(0); + if (result == NULL) + return NULL; + + for (sig = 1; sig < NSIG; sig++) { + if (sigismember(&mask, sig) != 1) + continue; + + /* Handle the case where it is a member by adding the signal to + the result list. Ignore the other cases because they mean the + signal isn't a member of the mask or the signal was invalid, + and an invalid signal must have been our fault in constructing + the loop boundaries. */ + signum = PyLong_FromLong(sig); + if (signum == NULL) { + Py_DECREF(result); + return NULL; + } + if (PySet_Add(result, signum) == -1) { + Py_DECREF(signum); + Py_DECREF(result); + return NULL; + } + Py_DECREF(signum); + } + return result; +} +#endif + +#ifdef PYPTHREAD_SIGMASK +static PyObject * +signal_pthread_sigmask(PyObject *self, PyObject *args) +{ + int how; + PyObject *signals; + sigset_t mask, previous; + int err; + + if (!PyArg_ParseTuple(args, "iO:pthread_sigmask", &how, &signals)) + return NULL; + + if (iterable_to_sigset(signals, &mask)) + return NULL; + + err = pthread_sigmask(how, &mask, &previous); + if (err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + /* if signals was unblocked, signal handlers have been called */ + if (PyErr_CheckSignals()) + return NULL; + + return sigset_to_set(previous); +} + +PyDoc_STRVAR(signal_pthread_sigmask_doc, +"pthread_sigmask(how, mask) -> old mask\n\ +\n\ +Fetch and/or change the signal mask of the calling thread."); +#endif /* #ifdef PYPTHREAD_SIGMASK */ + + +#ifdef HAVE_SIGPENDING +static PyObject * +signal_sigpending(PyObject *self) +{ + int err; + sigset_t mask; + err = sigpending(&mask); + if (err) + return PyErr_SetFromErrno(PyExc_OSError); + return sigset_to_set(mask); +} + +PyDoc_STRVAR(signal_sigpending_doc, +"sigpending() -> list\n\ +\n\ +Examine pending signals."); +#endif /* #ifdef HAVE_SIGPENDING */ + + +#ifdef HAVE_SIGWAIT +static PyObject * +signal_sigwait(PyObject *self, PyObject *args) +{ + PyObject *signals; + sigset_t set; + int err, signum; + + if (!PyArg_ParseTuple(args, "O:sigwait", &signals)) + return NULL; + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigwait(&set, &signum); + Py_END_ALLOW_THREADS + if (err) { + errno = err; + return PyErr_SetFromErrno(PyExc_OSError); + } + + return PyLong_FromLong(signum); +} + +PyDoc_STRVAR(signal_sigwait_doc, +"sigwait(sigset) -> signum\n\ +\n\ +Wait a signal."); +#endif /* #ifdef HAVE_SIGPENDING */ + +#if defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) +static int initialized; +static PyStructSequence_Field struct_siginfo_fields[] = { + {"si_signo", "signal number"}, + {"si_code", "signal code"}, + {"si_errno", "errno associated with this signal"}, + {"si_pid", "sending process ID"}, + {"si_uid", "real user ID of sending process"}, + {"si_status", "exit value or signal"}, + {"si_band", "band event for SIGPOLL"}, + {0} +}; + +PyDoc_STRVAR(struct_siginfo__doc__, +"struct_siginfo: Result from sigwaitinfo or sigtimedwait.\n\n\ +This object may be accessed either as a tuple of\n\ +(si_signo, si_code, si_errno, si_pid, si_uid, si_status, si_band),\n\ +or via the attributes si_signo, si_code, and so on."); + +static PyStructSequence_Desc struct_siginfo_desc = { + "signal.struct_siginfo", /* name */ + struct_siginfo__doc__, /* doc */ + struct_siginfo_fields, /* fields */ + 7 /* n_in_sequence */ +}; + +static PyTypeObject SiginfoType; + +static PyObject * +fill_siginfo(siginfo_t *si) +{ + PyObject *result = PyStructSequence_New(&SiginfoType); + if (!result) + return NULL; + + PyStructSequence_SET_ITEM(result, 0, PyLong_FromLong((long)(si->si_signo))); + PyStructSequence_SET_ITEM(result, 1, PyLong_FromLong((long)(si->si_code))); + PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si->si_errno))); + PyStructSequence_SET_ITEM(result, 3, PyLong_FromPid(si->si_pid)); + PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si->si_uid))); + PyStructSequence_SET_ITEM(result, 5, + PyLong_FromLong((long)(si->si_status))); + PyStructSequence_SET_ITEM(result, 6, PyLong_FromLong(si->si_band)); + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } + + return result; +} +#endif + +#ifdef HAVE_SIGWAITINFO +static PyObject * +signal_sigwaitinfo(PyObject *self, PyObject *args) +{ + PyObject *signals; + sigset_t set; + siginfo_t si; + int err; + + if (!PyArg_ParseTuple(args, "O:sigwaitinfo", &signals)) + return NULL; + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigwaitinfo(&set, &si); + Py_END_ALLOW_THREADS + if (err == -1) + return PyErr_SetFromErrno(PyExc_OSError); + + return fill_siginfo(&si); +} + +PyDoc_STRVAR(signal_sigwaitinfo_doc, +"sigwaitinfo(sigset) -> struct_siginfo\n\ +\n\ +Wait synchronously for a signal until one of the signals in *sigset* is\n\ +delivered.\n\ +Returns a struct_siginfo containing information about the signal."); +#endif /* #ifdef HAVE_SIGWAITINFO */ + +#ifdef HAVE_SIGTIMEDWAIT +static PyObject * +signal_sigtimedwait(PyObject *self, PyObject *args) +{ + PyObject *signals, *timeout; + struct timespec buf; + sigset_t set; + siginfo_t si; + int err; + + if (!PyArg_ParseTuple(args, "OO:sigtimedwait", &signals, &timeout)) + return NULL; + + if (!PyTuple_Check(timeout) || PyTuple_Size(timeout) != 2) { + PyErr_SetString(PyExc_TypeError, + "sigtimedwait() arg 2 must be a tuple " + "(timeout_sec, timeout_nsec)"); + return NULL; + } else if (!PyArg_ParseTuple(timeout, "ll:sigtimedwait", + &(buf.tv_sec), &(buf.tv_nsec))) + return NULL; + + if (buf.tv_sec < 0 || buf.tv_nsec < 0) { + PyErr_SetString(PyExc_ValueError, "timeout must be non-negative"); + return NULL; + } + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigtimedwait(&set, &si, &buf); + Py_END_ALLOW_THREADS + if (err == -1) { + if (errno == EAGAIN) + Py_RETURN_NONE; + else + return PyErr_SetFromErrno(PyExc_OSError); + } + + return fill_siginfo(&si); +} + +PyDoc_STRVAR(signal_sigtimedwait_doc, +"sigtimedwait(sigset, (timeout_sec, timeout_nsec)) -> struct_siginfo\n\ +\n\ +Like sigwaitinfo(), but with a timeout specified as a tuple of (seconds,\n\ +nanoseconds)."); +#endif /* #ifdef HAVE_SIGTIMEDWAIT */ + + +#if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) +static PyObject * +signal_pthread_kill(PyObject *self, PyObject *args) +{ + long tid; + int signum; + int err; + + if (!PyArg_ParseTuple(args, "li:pthread_kill", &tid, &signum)) + return NULL; + + err = pthread_kill((pthread_t)tid, signum); + if (err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + /* the signal may have been send to the current thread */ + if (PyErr_CheckSignals()) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(signal_pthread_kill_doc, +"pthread_kill(thread_id, signum)\n\ +\n\ +Send a signal to a thread."); +#endif /* #if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) */ + + /* List of functions defined in the module */ static PyMethodDef signal_methods[] = { @@ -515,10 +867,34 @@ static PyMethodDef signal_methods[] = { #endif #ifdef HAVE_PAUSE {"pause", (PyCFunction)signal_pause, - METH_NOARGS,pause_doc}, + METH_NOARGS, pause_doc}, #endif {"default_int_handler", signal_default_int_handler, METH_VARARGS, default_int_handler_doc}, +#if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) + {"pthread_kill", (PyCFunction)signal_pthread_kill, + METH_VARARGS, signal_pthread_kill_doc}, +#endif +#ifdef PYPTHREAD_SIGMASK + {"pthread_sigmask", (PyCFunction)signal_pthread_sigmask, + METH_VARARGS, signal_pthread_sigmask_doc}, +#endif +#ifdef HAVE_SIGPENDING + {"sigpending", (PyCFunction)signal_sigpending, + METH_NOARGS, signal_sigpending_doc}, +#endif +#ifdef HAVE_SIGWAIT + {"sigwait", (PyCFunction)signal_sigwait, + METH_VARARGS, signal_sigwait_doc}, +#endif +#ifdef HAVE_SIGWAITINFO + {"sigwaitinfo", (PyCFunction)signal_sigwaitinfo, + METH_VARARGS, signal_sigwaitinfo_doc}, +#endif +#ifdef HAVE_SIGTIMEDWAIT + {"sigtimedwait", (PyCFunction)signal_sigtimedwait, + METH_VARARGS, signal_sigtimedwait_doc}, +#endif {NULL, NULL} /* sentinel */ }; @@ -587,6 +963,15 @@ PyInit_signal(void) if (m == NULL) return NULL; +#if defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) + if (!initialized) + PyStructSequence_InitType(&SiginfoType, &struct_siginfo_desc); + + Py_INCREF((PyObject*) &SiginfoType); + PyModule_AddObject(m, "struct_siginfo", (PyObject*) &SiginfoType); + initialized = 1; +#endif + /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); @@ -603,6 +988,19 @@ PyInit_signal(void) goto finally; Py_DECREF(x); +#ifdef SIG_BLOCK + if (PyModule_AddIntMacro(m, SIG_BLOCK)) + goto finally; +#endif +#ifdef SIG_UNBLOCK + if (PyModule_AddIntMacro(m, SIG_UNBLOCK)) + goto finally; +#endif +#ifdef SIG_SETMASK + if (PyModule_AddIntMacro(m, SIG_SETMASK)) + goto finally; +#endif + x = IntHandler = PyDict_GetItemString(d, "default_int_handler"); if (!x) goto finally; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index d3e5c75..2056b61 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -42,6 +42,9 @@ Module interface: - socket.inet_ntoa(packed IP) -> IP address string - socket.getdefaulttimeout() -> None | float - socket.setdefaulttimeout(None | float) +- socket.if_nameindex() -> list of tuples (if_index, if_name) +- socket.if_nametoindex(name) -> corresponding interface index +- socket.if_indextoname(index) -> corresponding interface name - an Internet socket address is a pair (hostname, port) where hostname can be anything recognized by gethostbyname() (including the dd.dd.dd.dd notation) and port is in host byte order @@ -133,6 +136,9 @@ setblocking(0 | 1) -- set or clear the blocking I/O flag\n\ setsockopt(level, optname, value) -- set socket options\n\ settimeout(None | float) -- set or clear the timeout\n\ shutdown(how) -- shut down traffic in one or both directions\n\ +if_nameindex() -- return all network interface indices and names\n\ +if_nametoindex(name) -- return the corresponding interface index\n\ +if_indextoname(index) -- return the corresponding interface name\n\ \n\ [*] not available on all platforms!"); @@ -150,12 +156,16 @@ shutdown(how) -- shut down traffic in one or both directions\n\ # undef HAVE_GETHOSTBYNAME_R_6_ARG #endif +#if defined(__OpenBSD__) +# include <sys/uio.h> +#endif + #ifndef WITH_THREAD # undef HAVE_GETHOSTBYNAME_R #endif #ifdef HAVE_GETHOSTBYNAME_R -# if defined(_AIX) || defined(__osf__) +# if defined(_AIX) # define HAVE_GETHOSTBYNAME_R_3_ARG # elif defined(__sun) || defined(__sgi) # define HAVE_GETHOSTBYNAME_R_5_ARG @@ -250,6 +260,14 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #include <sys/types.h> #endif +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif + +#ifdef HAVE_NET_IF_H +#include <net/if.h> +#endif + /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" @@ -260,6 +278,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ /* Non-MS WINDOWS includes */ # include <netdb.h> +# include <unistd.h> /* Headers needed for inet_ntoa() and inet_addr() */ # if defined(PYOS_OS2) && defined(PYCC_VACPP) @@ -438,7 +457,6 @@ dup_socket(SOCKET handle) /* Global variable holding the exception type for errors detected by this module (but not argument type or memory errors, etc.). */ -static PyObject *socket_error; static PyObject *socket_herror; static PyObject *socket_gaierror; static PyObject *socket_timeout; @@ -455,6 +473,17 @@ static PyTypeObject sock_type; #include <sys/poll.h> #endif +/* Largest value to try to store in a socklen_t (used when handling + ancillary data). POSIX requires socklen_t to hold at least + (2**31)-1 and recommends against storing larger values, but + socklen_t was originally int in the BSD interface, so to be on the + safe side we use the smaller of (2**31)-1 and INT_MAX. */ +#if INT_MAX > 0x7fffffff +#define SOCKLEN_T_LIMIT 0x7fffffff +#else +#define SOCKLEN_T_LIMIT INT_MAX +#endif + #ifdef HAVE_POLL /* Instead of select(), we'll use poll() since poll() works on any fd. */ #define IS_SELECTABLE(s) 1 @@ -468,7 +497,7 @@ static PyTypeObject sock_type; static PyObject* select_error(void) { - PyErr_SetString(socket_error, "unable to select on socket"); + PyErr_SetString(PyExc_OSError, "unable to select on socket"); return NULL; } @@ -495,7 +524,7 @@ set_error(void) recognizes the error codes used by both GetLastError() and WSAGetLastError */ if (err_no) - return PyErr_SetExcFromWindowsErr(socket_error, err_no); + return PyErr_SetExcFromWindowsErr(PyExc_OSError, err_no); #endif #if defined(PYOS_OS2) && !defined(PYCC_GCC) @@ -526,7 +555,7 @@ set_error(void) } v = Py_BuildValue("(is)", myerrorcode, outbuf); if (v != NULL) { - PyErr_SetObject(socket_error, v); + PyErr_SetObject(PyExc_OSError, v); Py_DECREF(v); } return NULL; @@ -534,7 +563,7 @@ set_error(void) } #endif - return PyErr_SetFromErrno(socket_error); + return PyErr_SetFromErrno(PyExc_OSError); } @@ -853,13 +882,13 @@ setipaddr(char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) #endif default: freeaddrinfo(res); - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "unsupported address family"); return -1; } if (res->ai_next) { freeaddrinfo(res); - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "wildcard resolved to multiple address"); return -1; } @@ -872,7 +901,7 @@ setipaddr(char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) if (name[0] == '<' && strcmp(name, "<broadcast>") == 0) { struct sockaddr_in *sin; if (af != AF_INET && af != AF_UNSPEC) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "address family mismatched"); return -1; } @@ -930,7 +959,7 @@ setipaddr(char *name, struct sockaddr *addr_ret, size_t addr_ret_size, int af) return 16; #endif default: - PyErr_SetString(socket_error, "unknown address family"); + PyErr_SetString(PyExc_OSError, "unknown address family"); return -1; } } @@ -979,7 +1008,7 @@ setbdaddr(char *name, bdaddr_t *bdaddr) bdaddr->b[5] = b5; return 6; } else { - PyErr_SetString(socket_error, "bad bluetooth address"); + PyErr_SetString(PyExc_OSError, "bad bluetooth address"); return -1; } } @@ -1190,6 +1219,25 @@ makesockaddr(SOCKET_T sockfd, struct sockaddr *addr, size_t addrlen, int proto) } #endif +#ifdef AF_CAN + case AF_CAN: + { + struct sockaddr_can *a = (struct sockaddr_can *)addr; + char *ifname = ""; + struct ifreq ifr; + /* need to look up interface name given index */ + if (a->can_ifindex) { + ifr.ifr_ifindex = a->can_ifindex; + if (ioctl(sockfd, SIOCGIFNAME, &ifr) == 0) + ifname = ifr.ifr_name; + } + + return Py_BuildValue("O&h", PyUnicode_DecodeFSDefault, + ifname, + a->can_family); + } +#endif + /* More cases here... */ default: @@ -1229,7 +1277,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, if (len > 0 && path[0] == 0) { /* Linux abstract namespace extension */ if (len > sizeof addr->sun_path) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "AF_UNIX path too long"); return 0; } @@ -1239,7 +1287,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, { /* regular NULL-terminated string */ if (len >= sizeof addr->sun_path) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "AF_UNIX path too long"); return 0; } @@ -1369,7 +1417,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, _BT_L2_MEMB(addr, family) = AF_BLUETOOTH; if (!PyArg_ParseTuple(args, "si", &straddr, &_BT_L2_MEMB(addr, psm))) { - PyErr_SetString(socket_error, "getsockaddrarg: " + PyErr_SetString(PyExc_OSError, "getsockaddrarg: " "wrong format"); return 0; } @@ -1388,7 +1436,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, _BT_RC_MEMB(addr, family) = AF_BLUETOOTH; if (!PyArg_ParseTuple(args, "si", &straddr, &_BT_RC_MEMB(addr, channel))) { - PyErr_SetString(socket_error, "getsockaddrarg: " + PyErr_SetString(PyExc_OSError, "getsockaddrarg: " "wrong format"); return 0; } @@ -1406,7 +1454,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, _BT_HCI_MEMB(addr, family) = AF_BLUETOOTH; if (straddr == NULL) { - PyErr_SetString(socket_error, "getsockaddrarg: " + PyErr_SetString(PyExc_OSError, "getsockaddrarg: " "wrong format"); return 0; } @@ -1415,7 +1463,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, #else _BT_HCI_MEMB(addr, family) = AF_BLUETOOTH; if (!PyArg_ParseTuple(args, "i", &_BT_HCI_MEMB(addr, dev))) { - PyErr_SetString(socket_error, "getsockaddrarg: " + PyErr_SetString(PyExc_OSError, "getsockaddrarg: " "wrong format"); return 0; } @@ -1432,7 +1480,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, addr = (struct sockaddr_sco *)addr_ret; _BT_SCO_MEMB(addr, family) = AF_BLUETOOTH; if (!PyBytes_Check(args)) { - PyErr_SetString(socket_error, "getsockaddrarg: " + PyErr_SetString(PyExc_OSError, "getsockaddrarg: " "wrong format"); return 0; } @@ -1445,7 +1493,7 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } #endif default: - PyErr_SetString(socket_error, "getsockaddrarg: unknown Bluetooth protocol"); + PyErr_SetString(PyExc_OSError, "getsockaddrarg: unknown Bluetooth protocol"); return 0; } } @@ -1557,10 +1605,57 @@ getsockaddrarg(PySocketSockObject *s, PyObject *args, } #endif +#ifdef AF_CAN + case AF_CAN: + switch (s->sock_proto) { + case CAN_RAW: + { + struct sockaddr_can *addr; + PyObject *interfaceName; + struct ifreq ifr; + addr = (struct sockaddr_can *)addr_ret; + Py_ssize_t len; + + if (!PyArg_ParseTuple(args, "O&", PyUnicode_FSConverter, + &interfaceName)) + return 0; + + len = PyBytes_GET_SIZE(interfaceName); + + if (len == 0) { + ifr.ifr_ifindex = 0; + } else if (len < sizeof(ifr.ifr_name)) { + strcpy(ifr.ifr_name, PyBytes_AS_STRING(interfaceName)); + if (ioctl(s->sock_fd, SIOCGIFINDEX, &ifr) < 0) { + s->errorhandler(); + Py_DECREF(interfaceName); + return 0; + } + } else { + PyErr_SetString(PyExc_OSError, + "AF_CAN interface name too long"); + Py_DECREF(interfaceName); + return 0; + } + + addr->can_family = AF_CAN; + addr->can_ifindex = ifr.ifr_ifindex; + + *len_ret = sizeof(*addr); + Py_DECREF(interfaceName); + return 1; + } + default: + PyErr_SetString(PyExc_OSError, + "getsockaddrarg: unsupported CAN protocol"); + return 0; + } +#endif + /* More cases here... */ default: - PyErr_SetString(socket_error, "getsockaddrarg: bad family"); + PyErr_SetString(PyExc_OSError, "getsockaddrarg: bad family"); return 0; } @@ -1626,7 +1721,7 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) return 1; #endif default: - PyErr_SetString(socket_error, "getsockaddrlen: " + PyErr_SetString(PyExc_OSError, "getsockaddrlen: " "unknown BT protocol"); return 0; @@ -1650,14 +1745,134 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) } #endif +#ifdef AF_CAN + case AF_CAN: + { + *len_ret = sizeof (struct sockaddr_can); + return 1; + } +#endif + /* More cases here... */ default: - PyErr_SetString(socket_error, "getsockaddrlen: bad family"); + PyErr_SetString(PyExc_OSError, "getsockaddrlen: bad family"); + return 0; + + } +} + + +/* Support functions for the sendmsg() and recvmsg[_into]() methods. + Currently, these methods are only compiled if the RFC 2292/3542 + CMSG_LEN() macro is available. Older systems seem to have used + sizeof(struct cmsghdr) + (length) where CMSG_LEN() is used now, so + it may be possible to define CMSG_LEN() that way if it's not + provided. Some architectures might need extra padding after the + cmsghdr, however, and CMSG_LEN() would have to take account of + this. */ +#ifdef CMSG_LEN +/* If length is in range, set *result to CMSG_LEN(length) and return + true; otherwise, return false. */ +static int +get_CMSG_LEN(size_t length, size_t *result) +{ + size_t tmp; + + if (length > (SOCKLEN_T_LIMIT - CMSG_LEN(0))) + return 0; + tmp = CMSG_LEN(length); + if (tmp > SOCKLEN_T_LIMIT || tmp < length) + return 0; + *result = tmp; + return 1; +} + +#ifdef CMSG_SPACE +/* If length is in range, set *result to CMSG_SPACE(length) and return + true; otherwise, return false. */ +static int +get_CMSG_SPACE(size_t length, size_t *result) +{ + size_t tmp; + + /* Use CMSG_SPACE(1) here in order to take account of the padding + necessary before *and* after the data. */ + if (length > (SOCKLEN_T_LIMIT - CMSG_SPACE(1))) + return 0; + tmp = CMSG_SPACE(length); + if (tmp > SOCKLEN_T_LIMIT || tmp < length) + return 0; + *result = tmp; + return 1; +} +#endif + +/* Return true iff msg->msg_controllen is valid, cmsgh is a valid + pointer in msg->msg_control with at least "space" bytes after it, + and its cmsg_len member inside the buffer. */ +static int +cmsg_min_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t space) +{ + size_t cmsg_offset; + static const size_t cmsg_len_end = (offsetof(struct cmsghdr, cmsg_len) + + sizeof(cmsgh->cmsg_len)); + + /* Note that POSIX allows msg_controllen to be of signed type. */ + if (cmsgh == NULL || msg->msg_control == NULL || msg->msg_controllen < 0) + return 0; + if (space < cmsg_len_end) + space = cmsg_len_end; + cmsg_offset = (char *)cmsgh - (char *)msg->msg_control; + return (cmsg_offset <= (size_t)-1 - space && + cmsg_offset + space <= msg->msg_controllen); +} + +/* If pointer CMSG_DATA(cmsgh) is in buffer msg->msg_control, set + *space to number of bytes following it in the buffer and return + true; otherwise, return false. Assumes cmsgh, msg->msg_control and + msg->msg_controllen are valid. */ +static int +get_cmsg_data_space(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *space) +{ + size_t data_offset; + char *data_ptr; + + if ((data_ptr = (char *)CMSG_DATA(cmsgh)) == NULL) + return 0; + data_offset = data_ptr - (char *)msg->msg_control; + if (data_offset > msg->msg_controllen) return 0; + *space = msg->msg_controllen - data_offset; + return 1; +} + +/* If cmsgh is invalid or not contained in the buffer pointed to by + msg->msg_control, return -1. If cmsgh is valid and its associated + data is entirely contained in the buffer, set *data_len to the + length of the associated data and return 0. If only part of the + associated data is contained in the buffer but cmsgh is otherwise + valid, set *data_len to the length contained in the buffer and + return 1. */ +static int +get_cmsg_data_len(struct msghdr *msg, struct cmsghdr *cmsgh, size_t *data_len) +{ + size_t space, cmsg_data_len; + if (!cmsg_min_space(msg, cmsgh, CMSG_LEN(0)) || + cmsgh->cmsg_len < CMSG_LEN(0)) + return -1; + cmsg_data_len = cmsgh->cmsg_len - CMSG_LEN(0); + if (!get_cmsg_data_space(msg, cmsgh, &space)) + return -1; + if (space >= cmsg_data_len) { + *data_len = cmsg_data_len; + return 0; } + *data_len = space; + return 1; } +#endif /* CMSG_LEN */ /* s._accept() -> (fd, address) */ @@ -1882,7 +2097,7 @@ sock_getsockopt(PySocketSockObject *s, PyObject *args) #else if (buflen <= 0 || buflen > 1024) { #endif - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "getsockopt buflen out of range"); return NULL; } @@ -2613,6 +2828,331 @@ PyDoc_STRVAR(recvfrom_into_doc, Like recv_into(buffer[, nbytes[, flags]]) but also return the sender's address info."); +/* The sendmsg() and recvmsg[_into]() methods require a working + CMSG_LEN(). See the comment near get_CMSG_LEN(). */ +#ifdef CMSG_LEN +/* + * Call recvmsg() with the supplied iovec structures, flags, and + * ancillary data buffer size (controllen). Returns the tuple return + * value for recvmsg() or recvmsg_into(), with the first item provided + * by the supplied makeval() function. makeval() will be called with + * the length read and makeval_data as arguments, and must return a + * new reference (which will be decrefed if there is a subsequent + * error). On error, closes any file descriptors received via + * SCM_RIGHTS. + */ +static PyObject * +sock_recvmsg_guts(PySocketSockObject *s, struct iovec *iov, int iovlen, + int flags, Py_ssize_t controllen, + PyObject *(*makeval)(ssize_t, void *), void *makeval_data) +{ + ssize_t bytes_received = -1; + int timeout; + sock_addr_t addrbuf; + socklen_t addrbuflen; + struct msghdr msg = {0}; + PyObject *cmsg_list = NULL, *retval = NULL; + void *controlbuf = NULL; + struct cmsghdr *cmsgh; + size_t cmsgdatalen = 0; + int cmsg_status; + + /* XXX: POSIX says that msg_name and msg_namelen "shall be + ignored" when the socket is connected (Linux fills them in + anyway for AF_UNIX sockets at least). Normally msg_namelen + seems to be set to 0 if there's no address, but try to + initialize msg_name to something that won't be mistaken for a + real address if that doesn't happen. */ + if (!getsockaddrlen(s, &addrbuflen)) + return NULL; + memset(&addrbuf, 0, addrbuflen); + SAS2SA(&addrbuf)->sa_family = AF_UNSPEC; + + if (controllen < 0 || controllen > SOCKLEN_T_LIMIT) { + PyErr_SetString(PyExc_ValueError, + "invalid ancillary data buffer length"); + return NULL; + } + if (controllen > 0 && (controlbuf = PyMem_Malloc(controllen)) == NULL) + return PyErr_NoMemory(); + + /* Make the system call. */ + if (!IS_SELECTABLE(s)) { + select_error(); + goto finally; + } + + BEGIN_SELECT_LOOP(s) + Py_BEGIN_ALLOW_THREADS; + msg.msg_name = SAS2SA(&addrbuf); + msg.msg_namelen = addrbuflen; + msg.msg_iov = iov; + msg.msg_iovlen = iovlen; + msg.msg_control = controlbuf; + msg.msg_controllen = controllen; + timeout = internal_select_ex(s, 0, interval); + if (!timeout) + bytes_received = recvmsg(s->sock_fd, &msg, flags); + Py_END_ALLOW_THREADS; + if (timeout == 1) { + PyErr_SetString(socket_timeout, "timed out"); + goto finally; + } + END_SELECT_LOOP(s) + + if (bytes_received < 0) { + s->errorhandler(); + goto finally; + } + + /* Make list of (level, type, data) tuples from control messages. */ + if ((cmsg_list = PyList_New(0)) == NULL) + goto err_closefds; + /* Check for empty ancillary data as old CMSG_FIRSTHDR() + implementations didn't do so. */ + for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL); + cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) { + PyObject *bytes, *tuple; + int tmp; + + cmsg_status = get_cmsg_data_len(&msg, cmsgh, &cmsgdatalen); + if (cmsg_status != 0) { + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "received malformed or improperly-truncated " + "ancillary data", 1) == -1) + goto err_closefds; + } + if (cmsg_status < 0) + break; + if (cmsgdatalen > PY_SSIZE_T_MAX) { + PyErr_SetString(PyExc_OSError, "control message too long"); + goto err_closefds; + } + + bytes = PyBytes_FromStringAndSize((char *)CMSG_DATA(cmsgh), + cmsgdatalen); + tuple = Py_BuildValue("iiN", (int)cmsgh->cmsg_level, + (int)cmsgh->cmsg_type, bytes); + if (tuple == NULL) + goto err_closefds; + tmp = PyList_Append(cmsg_list, tuple); + Py_DECREF(tuple); + if (tmp != 0) + goto err_closefds; + + if (cmsg_status != 0) + break; + } + + retval = Py_BuildValue("NOiN", + (*makeval)(bytes_received, makeval_data), + cmsg_list, + (int)msg.msg_flags, + makesockaddr(s->sock_fd, SAS2SA(&addrbuf), + ((msg.msg_namelen > addrbuflen) ? + addrbuflen : msg.msg_namelen), + s->sock_proto)); + if (retval == NULL) + goto err_closefds; + +finally: + Py_XDECREF(cmsg_list); + PyMem_Free(controlbuf); + return retval; + +err_closefds: +#ifdef SCM_RIGHTS + /* Close all descriptors coming from SCM_RIGHTS, so they don't leak. */ + for (cmsgh = ((msg.msg_controllen > 0) ? CMSG_FIRSTHDR(&msg) : NULL); + cmsgh != NULL; cmsgh = CMSG_NXTHDR(&msg, cmsgh)) { + cmsg_status = get_cmsg_data_len(&msg, cmsgh, &cmsgdatalen); + if (cmsg_status < 0) + break; + if (cmsgh->cmsg_level == SOL_SOCKET && + cmsgh->cmsg_type == SCM_RIGHTS) { + size_t numfds; + int *fdp; + + numfds = cmsgdatalen / sizeof(int); + fdp = (int *)CMSG_DATA(cmsgh); + while (numfds-- > 0) + close(*fdp++); + } + if (cmsg_status != 0) + break; + } +#endif /* SCM_RIGHTS */ + goto finally; +} + + +static PyObject * +makeval_recvmsg(ssize_t received, void *data) +{ + PyObject **buf = data; + + if (received < PyBytes_GET_SIZE(*buf)) + _PyBytes_Resize(buf, received); + Py_XINCREF(*buf); + return *buf; +} + +/* s.recvmsg(bufsize[, ancbufsize[, flags]]) method */ + +static PyObject * +sock_recvmsg(PySocketSockObject *s, PyObject *args) +{ + Py_ssize_t bufsize, ancbufsize = 0; + int flags = 0; + struct iovec iov; + PyObject *buf = NULL, *retval = NULL; + + if (!PyArg_ParseTuple(args, "n|ni:recvmsg", &bufsize, &ancbufsize, &flags)) + return NULL; + + if (bufsize < 0) { + PyErr_SetString(PyExc_ValueError, "negative buffer size in recvmsg()"); + return NULL; + } + if ((buf = PyBytes_FromStringAndSize(NULL, bufsize)) == NULL) + return NULL; + iov.iov_base = PyBytes_AS_STRING(buf); + iov.iov_len = bufsize; + + /* Note that we're passing a pointer to *our pointer* to the bytes + object here (&buf); makeval_recvmsg() may incref the object, or + deallocate it and set our pointer to NULL. */ + retval = sock_recvmsg_guts(s, &iov, 1, flags, ancbufsize, + &makeval_recvmsg, &buf); + Py_XDECREF(buf); + return retval; +} + +PyDoc_STRVAR(recvmsg_doc, +"recvmsg(bufsize[, ancbufsize[, flags]]) -> (data, ancdata, msg_flags, address)\n\ +\n\ +Receive normal data (up to bufsize bytes) and ancillary data from the\n\ +socket. The ancbufsize argument sets the size in bytes of the\n\ +internal buffer used to receive the ancillary data; it defaults to 0,\n\ +meaning that no ancillary data will be received. Appropriate buffer\n\ +sizes for ancillary data can be calculated using CMSG_SPACE() or\n\ +CMSG_LEN(), and items which do not fit into the buffer might be\n\ +truncated or discarded. The flags argument defaults to 0 and has the\n\ +same meaning as for recv().\n\ +\n\ +The return value is a 4-tuple: (data, ancdata, msg_flags, address).\n\ +The data item is a bytes object holding the non-ancillary data\n\ +received. The ancdata item is a list of zero or more tuples\n\ +(cmsg_level, cmsg_type, cmsg_data) representing the ancillary data\n\ +(control messages) received: cmsg_level and cmsg_type are integers\n\ +specifying the protocol level and protocol-specific type respectively,\n\ +and cmsg_data is a bytes object holding the associated data. The\n\ +msg_flags item is the bitwise OR of various flags indicating\n\ +conditions on the received message; see your system documentation for\n\ +details. If the receiving socket is unconnected, address is the\n\ +address of the sending socket, if available; otherwise, its value is\n\ +unspecified.\n\ +\n\ +If recvmsg() raises an exception after the system call returns, it\n\ +will first attempt to close any file descriptors received via the\n\ +SCM_RIGHTS mechanism."); + + +static PyObject * +makeval_recvmsg_into(ssize_t received, void *data) +{ + return PyLong_FromSsize_t(received); +} + +/* s.recvmsg_into(buffers[, ancbufsize[, flags]]) method */ + +static PyObject * +sock_recvmsg_into(PySocketSockObject *s, PyObject *args) +{ + Py_ssize_t ancbufsize = 0; + int flags = 0; + struct iovec *iovs = NULL; + Py_ssize_t i, nitems, nbufs = 0; + Py_buffer *bufs = NULL; + PyObject *buffers_arg, *fast, *retval = NULL; + + if (!PyArg_ParseTuple(args, "O|ni:recvmsg_into", + &buffers_arg, &ancbufsize, &flags)) + return NULL; + + if ((fast = PySequence_Fast(buffers_arg, + "recvmsg_into() argument 1 must be an " + "iterable")) == NULL) + return NULL; + nitems = PySequence_Fast_GET_SIZE(fast); + if (nitems > INT_MAX) { + PyErr_SetString(PyExc_OSError, "recvmsg_into() argument 1 is too long"); + goto finally; + } + + /* Fill in an iovec for each item, and save the Py_buffer + structs to release afterwards. */ + if (nitems > 0 && ((iovs = PyMem_New(struct iovec, nitems)) == NULL || + (bufs = PyMem_New(Py_buffer, nitems)) == NULL)) { + PyErr_NoMemory(); + goto finally; + } + for (; nbufs < nitems; nbufs++) { + if (!PyArg_Parse(PySequence_Fast_GET_ITEM(fast, nbufs), + "w*;recvmsg_into() argument 1 must be an iterable " + "of single-segment read-write buffers", + &bufs[nbufs])) + goto finally; + iovs[nbufs].iov_base = bufs[nbufs].buf; + iovs[nbufs].iov_len = bufs[nbufs].len; + } + + retval = sock_recvmsg_guts(s, iovs, nitems, flags, ancbufsize, + &makeval_recvmsg_into, NULL); +finally: + for (i = 0; i < nbufs; i++) + PyBuffer_Release(&bufs[i]); + PyMem_Free(bufs); + PyMem_Free(iovs); + Py_DECREF(fast); + return retval; +} + +PyDoc_STRVAR(recvmsg_into_doc, +"recvmsg_into(buffers[, ancbufsize[, flags]]) -> (nbytes, ancdata, msg_flags, address)\n\ +\n\ +Receive normal data and ancillary data from the socket, scattering the\n\ +non-ancillary data into a series of buffers. The buffers argument\n\ +must be an iterable of objects that export writable buffers\n\ +(e.g. bytearray objects); these will be filled with successive chunks\n\ +of the non-ancillary data until it has all been written or there are\n\ +no more buffers. The ancbufsize argument sets the size in bytes of\n\ +the internal buffer used to receive the ancillary data; it defaults to\n\ +0, meaning that no ancillary data will be received. Appropriate\n\ +buffer sizes for ancillary data can be calculated using CMSG_SPACE()\n\ +or CMSG_LEN(), and items which do not fit into the buffer might be\n\ +truncated or discarded. The flags argument defaults to 0 and has the\n\ +same meaning as for recv().\n\ +\n\ +The return value is a 4-tuple: (nbytes, ancdata, msg_flags, address).\n\ +The nbytes item is the total number of bytes of non-ancillary data\n\ +written into the buffers. The ancdata item is a list of zero or more\n\ +tuples (cmsg_level, cmsg_type, cmsg_data) representing the ancillary\n\ +data (control messages) received: cmsg_level and cmsg_type are\n\ +integers specifying the protocol level and protocol-specific type\n\ +respectively, and cmsg_data is a bytes object holding the associated\n\ +data. The msg_flags item is the bitwise OR of various flags\n\ +indicating conditions on the received message; see your system\n\ +documentation for details. If the receiving socket is unconnected,\n\ +address is the address of the sending socket, if available; otherwise,\n\ +its value is unspecified.\n\ +\n\ +If recvmsg_into() raises an exception after the system call returns,\n\ +it will first attempt to close any file descriptors received via the\n\ +SCM_RIGHTS mechanism."); +#endif /* CMSG_LEN */ + + /* s.send(data [,flags]) method */ static PyObject * @@ -2764,6 +3304,7 @@ sock_sendto(PySocketSockObject *s, PyObject *args) PyErr_Format(PyExc_TypeError, "sendto() takes 2 or 3 arguments (%d given)", arglen); + return NULL; } if (PyErr_Occurred()) return NULL; @@ -2807,6 +3348,234 @@ Like send(data, flags) but allows specifying the destination address.\n\ For IP sockets, the address is a pair (hostaddr, port)."); +/* The sendmsg() and recvmsg[_into]() methods require a working + CMSG_LEN(). See the comment near get_CMSG_LEN(). */ +#ifdef CMSG_LEN +/* s.sendmsg(buffers[, ancdata[, flags[, address]]]) method */ + +static PyObject * +sock_sendmsg(PySocketSockObject *s, PyObject *args) +{ + Py_ssize_t i, ndataparts, ndatabufs = 0, ncmsgs, ncmsgbufs = 0; + Py_buffer *databufs = NULL; + struct iovec *iovs = NULL; + sock_addr_t addrbuf; + struct msghdr msg = {0}; + struct cmsginfo { + int level; + int type; + Py_buffer data; + } *cmsgs = NULL; + void *controlbuf = NULL; + size_t controllen, controllen_last; + ssize_t bytes_sent = -1; + int addrlen, timeout, flags = 0; + PyObject *data_arg, *cmsg_arg = NULL, *addr_arg = NULL, *data_fast = NULL, + *cmsg_fast = NULL, *retval = NULL; + + if (!PyArg_ParseTuple(args, "O|OiO:sendmsg", + &data_arg, &cmsg_arg, &flags, &addr_arg)) + return NULL; + + /* Parse destination address. */ + if (addr_arg != NULL && addr_arg != Py_None) { + if (!getsockaddrarg(s, addr_arg, SAS2SA(&addrbuf), &addrlen)) + goto finally; + msg.msg_name = &addrbuf; + msg.msg_namelen = addrlen; + } + + /* Fill in an iovec for each message part, and save the Py_buffer + structs to release afterwards. */ + if ((data_fast = PySequence_Fast(data_arg, + "sendmsg() argument 1 must be an " + "iterable")) == NULL) + goto finally; + ndataparts = PySequence_Fast_GET_SIZE(data_fast); + if (ndataparts > INT_MAX) { + PyErr_SetString(PyExc_OSError, "sendmsg() argument 1 is too long"); + goto finally; + } + msg.msg_iovlen = ndataparts; + if (ndataparts > 0 && + ((msg.msg_iov = iovs = PyMem_New(struct iovec, ndataparts)) == NULL || + (databufs = PyMem_New(Py_buffer, ndataparts)) == NULL)) { + PyErr_NoMemory(); + goto finally; + } + for (; ndatabufs < ndataparts; ndatabufs++) { + if (!PyArg_Parse(PySequence_Fast_GET_ITEM(data_fast, ndatabufs), + "y*;sendmsg() argument 1 must be an iterable of " + "buffer-compatible objects", + &databufs[ndatabufs])) + goto finally; + iovs[ndatabufs].iov_base = databufs[ndatabufs].buf; + iovs[ndatabufs].iov_len = databufs[ndatabufs].len; + } + + if (cmsg_arg == NULL) + ncmsgs = 0; + else { + if ((cmsg_fast = PySequence_Fast(cmsg_arg, + "sendmsg() argument 2 must be an " + "iterable")) == NULL) + goto finally; + ncmsgs = PySequence_Fast_GET_SIZE(cmsg_fast); + } + +#ifndef CMSG_SPACE + if (ncmsgs > 1) { + PyErr_SetString(PyExc_OSError, + "sending multiple control messages is not supported " + "on this system"); + goto finally; + } +#endif + /* Save level, type and Py_buffer for each control message, + and calculate total size. */ + if (ncmsgs > 0 && (cmsgs = PyMem_New(struct cmsginfo, ncmsgs)) == NULL) { + PyErr_NoMemory(); + goto finally; + } + controllen = controllen_last = 0; + while (ncmsgbufs < ncmsgs) { + size_t bufsize, space; + + if (!PyArg_Parse(PySequence_Fast_GET_ITEM(cmsg_fast, ncmsgbufs), + "(iiy*):[sendmsg() ancillary data items]", + &cmsgs[ncmsgbufs].level, + &cmsgs[ncmsgbufs].type, + &cmsgs[ncmsgbufs].data)) + goto finally; + bufsize = cmsgs[ncmsgbufs++].data.len; + +#ifdef CMSG_SPACE + if (!get_CMSG_SPACE(bufsize, &space)) { +#else + if (!get_CMSG_LEN(bufsize, &space)) { +#endif + PyErr_SetString(PyExc_OSError, "ancillary data item too large"); + goto finally; + } + controllen += space; + if (controllen > SOCKLEN_T_LIMIT || controllen < controllen_last) { + PyErr_SetString(PyExc_OSError, "too much ancillary data"); + goto finally; + } + controllen_last = controllen; + } + + /* Construct ancillary data block from control message info. */ + if (ncmsgbufs > 0) { + struct cmsghdr *cmsgh = NULL; + + if ((msg.msg_control = controlbuf = + PyMem_Malloc(controllen)) == NULL) { + PyErr_NoMemory(); + goto finally; + } + msg.msg_controllen = controllen; + + /* Need to zero out the buffer as a workaround for glibc's + CMSG_NXTHDR() implementation. After getting the pointer to + the next header, it checks its (uninitialized) cmsg_len + member to see if the "message" fits in the buffer, and + returns NULL if it doesn't. Zero-filling the buffer + ensures that that doesn't happen. */ + memset(controlbuf, 0, controllen); + + for (i = 0; i < ncmsgbufs; i++) { + size_t msg_len, data_len = cmsgs[i].data.len; + int enough_space = 0; + + cmsgh = (i == 0) ? CMSG_FIRSTHDR(&msg) : CMSG_NXTHDR(&msg, cmsgh); + if (cmsgh == NULL) { + PyErr_Format(PyExc_RuntimeError, + "unexpected NULL result from %s()", + (i == 0) ? "CMSG_FIRSTHDR" : "CMSG_NXTHDR"); + goto finally; + } + if (!get_CMSG_LEN(data_len, &msg_len)) { + PyErr_SetString(PyExc_RuntimeError, + "item size out of range for CMSG_LEN()"); + goto finally; + } + if (cmsg_min_space(&msg, cmsgh, msg_len)) { + size_t space; + + cmsgh->cmsg_len = msg_len; + if (get_cmsg_data_space(&msg, cmsgh, &space)) + enough_space = (space >= data_len); + } + if (!enough_space) { + PyErr_SetString(PyExc_RuntimeError, + "ancillary data does not fit in calculated " + "space"); + goto finally; + } + cmsgh->cmsg_level = cmsgs[i].level; + cmsgh->cmsg_type = cmsgs[i].type; + memcpy(CMSG_DATA(cmsgh), cmsgs[i].data.buf, data_len); + } + } + + /* Make the system call. */ + if (!IS_SELECTABLE(s)) { + select_error(); + goto finally; + } + + BEGIN_SELECT_LOOP(s) + Py_BEGIN_ALLOW_THREADS; + timeout = internal_select_ex(s, 1, interval); + if (!timeout) + bytes_sent = sendmsg(s->sock_fd, &msg, flags); + Py_END_ALLOW_THREADS; + if (timeout == 1) { + PyErr_SetString(socket_timeout, "timed out"); + goto finally; + } + END_SELECT_LOOP(s) + + if (bytes_sent < 0) { + s->errorhandler(); + goto finally; + } + retval = PyLong_FromSsize_t(bytes_sent); + +finally: + PyMem_Free(controlbuf); + for (i = 0; i < ncmsgbufs; i++) + PyBuffer_Release(&cmsgs[i].data); + PyMem_Free(cmsgs); + Py_XDECREF(cmsg_fast); + for (i = 0; i < ndatabufs; i++) + PyBuffer_Release(&databufs[i]); + PyMem_Free(databufs); + PyMem_Free(iovs); + Py_XDECREF(data_fast); + return retval; +} + +PyDoc_STRVAR(sendmsg_doc, +"sendmsg(buffers[, ancdata[, flags[, address]]]) -> count\n\ +\n\ +Send normal and ancillary data to the socket, gathering the\n\ +non-ancillary data from a series of buffers and concatenating it into\n\ +a single message. The buffers argument specifies the non-ancillary\n\ +data as an iterable of buffer-compatible objects (e.g. bytes objects).\n\ +The ancdata argument specifies the ancillary data (control messages)\n\ +as an iterable of zero or more tuples (cmsg_level, cmsg_type,\n\ +cmsg_data), where cmsg_level and cmsg_type are integers specifying the\n\ +protocol level and protocol-specific type respectively, and cmsg_data\n\ +is a buffer-compatible object holding the associated data. The flags\n\ +argument defaults to 0 and has the same meaning as for send(). If\n\ +address is supplied and not None, it sets a destination address for\n\ +the message. The return value is the number of bytes of non-ancillary\n\ +data sent."); +#endif /* CMSG_LEN */ + + /* s.shutdown(how) method */ static PyObject * @@ -2933,6 +3702,14 @@ static PyMethodDef sock_methods[] = { setsockopt_doc}, {"shutdown", (PyCFunction)sock_shutdown, METH_O, shutdown_doc}, +#ifdef CMSG_LEN + {"recvmsg", (PyCFunction)sock_recvmsg, METH_VARARGS, + recvmsg_doc}, + {"recvmsg_into", (PyCFunction)sock_recvmsg_into, METH_VARARGS, + recvmsg_into_doc,}, + {"sendmsg", (PyCFunction)sock_sendmsg, METH_VARARGS, + sendmsg_doc}, +#endif {NULL, NULL} /* sentinel */ }; @@ -3108,7 +3885,7 @@ socket_gethostname(PyObject *self, PyObject *unused) version of the hostname, whereas we need a Unicode string. Otherwise, gethostname apparently also returns the DNS name. */ wchar_t buf[MAX_COMPUTERNAME_LENGTH + 1]; - DWORD size = sizeof(buf) / sizeof(wchar_t); + DWORD size = Py_ARRAY_LENGTH(buf); PyObject *result; if (!GetComputerNameExW(ComputerNamePhysicalDnsHostname, buf, &size)) { if (GetLastError() == ERROR_MORE_DATA) { @@ -3126,7 +3903,7 @@ socket_gethostname(PyObject *self, PyObject *unused) } return PyErr_SetExcFromWindowsErr(PyExc_WindowsError, GetLastError()); } - return PyUnicode_FromUnicode(buf, size); + return PyUnicode_FromUnicode(buf, size); #else char buf[1024]; int res; @@ -3145,6 +3922,37 @@ PyDoc_STRVAR(gethostname_doc, \n\ Return the current host name."); +#ifdef HAVE_SETHOSTNAME +PyDoc_STRVAR(sethostname_doc, +"sethostname(name)\n\n\ +Sets the hostname to name."); + +static PyObject * +socket_sethostname(PyObject *self, PyObject *args) +{ + PyObject *hnobj; + Py_buffer buf; + int res, flag = 0; + + if (!PyArg_ParseTuple(args, "S:sethostname", &hnobj)) { + PyErr_Clear(); + if (!PyArg_ParseTuple(args, "O&:sethostname", + PyUnicode_FSConverter, &hnobj)) + return NULL; + flag = 1; + } + res = PyObject_GetBuffer(hnobj, &buf, PyBUF_SIMPLE); + if (!res) { + res = sethostname(buf.buf, buf.len); + PyBuffer_Release(&buf); + } + if (flag) + Py_DECREF(hnobj); + if (res) + return set_error(); + Py_RETURN_NONE; +} +#endif /* Python interface to gethostbyname(name). */ @@ -3191,7 +3999,7 @@ gethost_common(struct hostent *h, struct sockaddr *addr, int alen, int af) if (h->h_addrtype != af) { /* Let's get real error message to return */ - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, (char *)strerror(EAFNOSUPPORT)); return NULL; @@ -3276,7 +4084,7 @@ gethost_common(struct hostent *h, struct sockaddr *addr, int alen, int af) #endif default: /* can't happen */ - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "unsupported address family"); return NULL; } @@ -3417,7 +4225,7 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) goto finally; af = sa->sa_family; ap = NULL; - al = 0; + /* al = 0; */ switch (af) { case AF_INET: ap = (char *)&((struct sockaddr_in *)sa)->sin_addr; @@ -3430,7 +4238,7 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) break; #endif default: - PyErr_SetString(socket_error, "unsupported address family"); + PyErr_SetString(PyExc_OSError, "unsupported address family"); goto finally; } Py_BEGIN_ALLOW_THREADS @@ -3486,7 +4294,7 @@ socket_getservbyname(PyObject *self, PyObject *args) sp = getservbyname(name, proto); Py_END_ALLOW_THREADS if (sp == NULL) { - PyErr_SetString(socket_error, "service/proto not found"); + PyErr_SetString(PyExc_OSError, "service/proto not found"); return NULL; } return PyLong_FromLong((long) ntohs(sp->s_port)); @@ -3523,7 +4331,7 @@ socket_getservbyport(PyObject *self, PyObject *args) sp = getservbyport(htons((short)port), proto); Py_END_ALLOW_THREADS if (sp == NULL) { - PyErr_SetString(socket_error, "port/proto not found"); + PyErr_SetString(PyExc_OSError, "port/proto not found"); return NULL; } return PyUnicode_FromString(sp->s_name); @@ -3552,7 +4360,7 @@ socket_getprotobyname(PyObject *self, PyObject *args) sp = getprotobyname(name); Py_END_ALLOW_THREADS if (sp == NULL) { - PyErr_SetString(socket_error, "protocol not found"); + PyErr_SetString(PyExc_OSError, "protocol not found"); return NULL; } return PyLong_FromLong((long) sp->p_proto); @@ -3807,7 +4615,7 @@ socket_inet_aton(PyObject *self, PyObject *args) return PyBytes_FromStringAndSize((char *)(&buf), sizeof(buf)); - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "illegal IP address string passed to inet_aton"); return NULL; @@ -3828,7 +4636,7 @@ socket_inet_aton(PyObject *self, PyObject *args) packed_addr = inet_addr(ip_addr); if (packed_addr == INADDR_NONE) { /* invalid address */ - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "illegal IP address string passed to inet_aton"); return NULL; } @@ -3860,7 +4668,7 @@ socket_inet_ntoa(PyObject *self, PyObject *args) } if (addr_len != sizeof(packed_addr)) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "packed IP wrong length for inet_ntoa"); return NULL; } @@ -3895,7 +4703,7 @@ socket_inet_pton(PyObject *self, PyObject *args) #if !defined(ENABLE_IPV6) && defined(AF_INET6) if(af == AF_INET6) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "can't use AF_INET6, IPv6 is disabled"); return NULL; } @@ -3903,10 +4711,10 @@ socket_inet_pton(PyObject *self, PyObject *args) retval = inet_pton(af, ip, packed); if (retval < 0) { - PyErr_SetFromErrno(socket_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } else if (retval == 0) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "illegal IP address string passed to inet_pton"); return NULL; } else if (af == AF_INET) { @@ -3918,7 +4726,7 @@ socket_inet_pton(PyObject *self, PyObject *args) sizeof(struct in6_addr)); #endif } else { - PyErr_SetString(socket_error, "unknown address family"); + PyErr_SetString(PyExc_OSError, "unknown address family"); return NULL; } } @@ -3970,7 +4778,7 @@ socket_inet_ntop(PyObject *self, PyObject *args) retval = inet_ntop(af, packed, ip, sizeof(ip)); if (!retval) { - PyErr_SetFromErrno(socket_error); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } else { return PyUnicode_FromString(retval); @@ -3989,7 +4797,7 @@ socket_inet_ntop(PyObject *self, PyObject *args) static PyObject * socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) { - static char* kwnames[] = {"host", "port", "family", "type", "proto", + static char* kwnames[] = {"host", "port", "family", "type", "proto", "flags", 0}; struct addrinfo hints, *res; struct addrinfo *res0 = NULL; @@ -4004,7 +4812,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) family = socktype = protocol = flags = 0; family = AF_UNSPEC; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", kwnames, &hobj, &pobj, &family, &socktype, &protocol, &flags)) { return NULL; @@ -4012,7 +4820,9 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) if (hobj == Py_None) { hptr = NULL; } else if (PyUnicode_Check(hobj)) { - idna = PyObject_CallMethod(hobj, "encode", "s", "idna"); + _Py_identifier(encode); + + idna = _PyObject_CallMethodId(hobj, &PyId_encode, "s", "idna"); if (!idna) return NULL; assert(PyBytes_Check(idna)); @@ -4039,7 +4849,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) } else if (pobj == Py_None) { pptr = (char *)NULL; } else { - PyErr_SetString(socket_error, "Int or String expected"); + PyErr_SetString(PyExc_OSError, "Int or String expected"); goto err; } memset(&hints, 0, sizeof(hints)); @@ -4136,7 +4946,7 @@ socket_getnameinfo(PyObject *self, PyObject *args) goto fail; } if (res->ai_next) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "sockaddr resolved to multiple addresses"); goto fail; } @@ -4144,7 +4954,7 @@ socket_getnameinfo(PyObject *self, PyObject *args) case AF_INET: { if (PyTuple_GET_SIZE(sa) != 2) { - PyErr_SetString(socket_error, + PyErr_SetString(PyExc_OSError, "IPv4 sockaddr must be 2 tuple"); goto fail; } @@ -4231,6 +5041,163 @@ Set the default timeout in seconds (float) for new socket objects.\n\ A value of None indicates that new socket objects have no timeout.\n\ When the socket module is first imported, the default is None."); +#ifdef HAVE_IF_NAMEINDEX +/* Python API for getting interface indices and names */ + +static PyObject * +socket_if_nameindex(PyObject *self, PyObject *arg) +{ + PyObject *list; + int i; + struct if_nameindex *ni; + + ni = if_nameindex(); + if (ni == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + list = PyList_New(0); + if (list == NULL) { + if_freenameindex(ni); + return NULL; + } + + for (i = 0; ni[i].if_index != 0 && i < INT_MAX; i++) { + PyObject *ni_tuple = Py_BuildValue("IO&", + ni[i].if_index, PyUnicode_DecodeFSDefault, ni[i].if_name); + + if (ni_tuple == NULL || PyList_Append(list, ni_tuple) == -1) { + Py_XDECREF(ni_tuple); + Py_DECREF(list); + if_freenameindex(ni); + return NULL; + } + Py_DECREF(ni_tuple); + } + + if_freenameindex(ni); + return list; +} + +PyDoc_STRVAR(if_nameindex_doc, +"if_nameindex()\n\ +\n\ +Returns a list of network interface information (index, name) tuples."); + +static PyObject * +socket_if_nametoindex(PyObject *self, PyObject *args) +{ + PyObject *oname; + unsigned long index; + + if (!PyArg_ParseTuple(args, "O&:if_nametoindex", + PyUnicode_FSConverter, &oname)) + return NULL; + + index = if_nametoindex(PyBytes_AS_STRING(oname)); + Py_DECREF(oname); + if (index == 0) { + /* if_nametoindex() doesn't set errno */ + PyErr_SetString(PyExc_OSError, "no interface with this name"); + return NULL; + } + + return PyLong_FromUnsignedLong(index); +} + +PyDoc_STRVAR(if_nametoindex_doc, +"if_nametoindex(if_name)\n\ +\n\ +Returns the interface index corresponding to the interface name if_name."); + +static PyObject * +socket_if_indextoname(PyObject *self, PyObject *arg) +{ + unsigned long index; + char name[IF_NAMESIZE + 1]; + + index = PyLong_AsUnsignedLong(arg); + if (index == (unsigned long) -1) + return NULL; + + if (if_indextoname(index, name) == NULL) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + return PyUnicode_DecodeFSDefault(name); +} + +PyDoc_STRVAR(if_indextoname_doc, +"if_indextoname(if_index)\n\ +\n\ +Returns the interface name corresponding to the interface index if_index."); + +#endif /* HAVE_IF_NAMEINDEX */ + + +#ifdef CMSG_LEN +/* Python interface to CMSG_LEN(length). */ + +static PyObject * +socket_CMSG_LEN(PyObject *self, PyObject *args) +{ + Py_ssize_t length; + size_t result; + + if (!PyArg_ParseTuple(args, "n:CMSG_LEN", &length)) + return NULL; + if (length < 0 || !get_CMSG_LEN(length, &result)) { + PyErr_Format(PyExc_OverflowError, "CMSG_LEN() argument out of range"); + return NULL; + } + return PyLong_FromSize_t(result); +} + +PyDoc_STRVAR(CMSG_LEN_doc, +"CMSG_LEN(length) -> control message length\n\ +\n\ +Return the total length, without trailing padding, of an ancillary\n\ +data item with associated data of the given length. This value can\n\ +often be used as the buffer size for recvmsg() to receive a single\n\ +item of ancillary data, but RFC 3542 requires portable applications to\n\ +use CMSG_SPACE() and thus include space for padding, even when the\n\ +item will be the last in the buffer. Raises OverflowError if length\n\ +is outside the permissible range of values."); + + +#ifdef CMSG_SPACE +/* Python interface to CMSG_SPACE(length). */ + +static PyObject * +socket_CMSG_SPACE(PyObject *self, PyObject *args) +{ + Py_ssize_t length; + size_t result; + + if (!PyArg_ParseTuple(args, "n:CMSG_SPACE", &length)) + return NULL; + if (length < 0 || !get_CMSG_SPACE(length, &result)) { + PyErr_SetString(PyExc_OverflowError, + "CMSG_SPACE() argument out of range"); + return NULL; + } + return PyLong_FromSize_t(result); +} + +PyDoc_STRVAR(CMSG_SPACE_doc, +"CMSG_SPACE(length) -> buffer size\n\ +\n\ +Return the buffer size needed for recvmsg() to receive an ancillary\n\ +data item with associated data of the given length, along with any\n\ +trailing padding. The buffer space needed to receive multiple items\n\ +is the sum of the CMSG_SPACE() values for their associated data\n\ +lengths. Raises OverflowError if length is outside the permissible\n\ +range of values."); +#endif /* CMSG_SPACE */ +#endif /* CMSG_LEN */ + /* List of functions exported by this module. */ @@ -4243,6 +5210,10 @@ static PyMethodDef socket_methods[] = { METH_VARARGS, gethostbyaddr_doc}, {"gethostname", socket_gethostname, METH_NOARGS, gethostname_doc}, +#ifdef HAVE_SETHOSTNAME + {"sethostname", socket_sethostname, + METH_VARARGS, sethostname_doc}, +#endif {"getservbyname", socket_getservbyname, METH_VARARGS, getservbyname_doc}, {"getservbyport", socket_getservbyport, @@ -4283,6 +5254,22 @@ static PyMethodDef socket_methods[] = { METH_NOARGS, getdefaulttimeout_doc}, {"setdefaulttimeout", socket_setdefaulttimeout, METH_O, setdefaulttimeout_doc}, +#ifdef HAVE_IF_NAMEINDEX + {"if_nameindex", socket_if_nameindex, + METH_NOARGS, if_nameindex_doc}, + {"if_nametoindex", socket_if_nametoindex, + METH_VARARGS, if_nametoindex_doc}, + {"if_indextoname", socket_if_indextoname, + METH_O, if_indextoname_doc}, +#endif +#ifdef CMSG_LEN + {"CMSG_LEN", socket_CMSG_LEN, + METH_VARARGS, CMSG_LEN_doc}, +#ifdef CMSG_SPACE + {"CMSG_SPACE", socket_CMSG_SPACE, + METH_VARARGS, CMSG_SPACE_doc}, +#endif +#endif {NULL, NULL} /* Sentinel */ }; @@ -4415,27 +5402,24 @@ PyInit__socket(void) if (m == NULL) return NULL; - socket_error = PyErr_NewException("socket.error", - PyExc_IOError, NULL); - if (socket_error == NULL) - return NULL; - PySocketModuleAPI.error = socket_error; - Py_INCREF(socket_error); - PyModule_AddObject(m, "error", socket_error); + Py_INCREF(PyExc_OSError); + PySocketModuleAPI.error = PyExc_OSError; + Py_INCREF(PyExc_OSError); + PyModule_AddObject(m, "error", PyExc_OSError); socket_herror = PyErr_NewException("socket.herror", - socket_error, NULL); + PyExc_OSError, NULL); if (socket_herror == NULL) return NULL; Py_INCREF(socket_herror); PyModule_AddObject(m, "herror", socket_herror); - socket_gaierror = PyErr_NewException("socket.gaierror", socket_error, + socket_gaierror = PyErr_NewException("socket.gaierror", PyExc_OSError, NULL); if (socket_gaierror == NULL) return NULL; Py_INCREF(socket_gaierror); PyModule_AddObject(m, "gaierror", socket_gaierror); socket_timeout = PyErr_NewException("socket.timeout", - socket_error, NULL); + PyExc_OSError, NULL); if (socket_timeout == NULL) return NULL; PySocketModuleAPI.timeout_error = socket_timeout; @@ -4621,6 +5605,15 @@ PyInit__socket(void) PyModule_AddStringConstant(m, "BDADDR_LOCAL", "00:00:00:FF:FF:FF"); #endif +#ifdef AF_CAN + /* Controller Area Network */ + PyModule_AddIntConstant(m, "AF_CAN", AF_CAN); +#endif +#ifdef PF_CAN + /* Controller Area Network */ + PyModule_AddIntConstant(m, "PF_CAN", PF_CAN); +#endif + #ifdef AF_PACKET PyModule_AddIntMacro(m, AF_PACKET); #endif @@ -4770,6 +5763,15 @@ PyInit__socket(void) #ifdef SO_SETFIB PyModule_AddIntConstant(m, "SO_SETFIB", SO_SETFIB); #endif +#ifdef SO_PASSCRED + PyModule_AddIntConstant(m, "SO_PASSCRED", SO_PASSCRED); +#endif +#ifdef SO_PEERCRED + PyModule_AddIntConstant(m, "SO_PEERCRED", SO_PEERCRED); +#endif +#ifdef LOCAL_PEERCRED + PyModule_AddIntConstant(m, "LOCAL_PEERCRED", LOCAL_PEERCRED); +#endif /* Maximum number of connections for "listen" */ #ifdef SOMAXCONN @@ -4778,6 +5780,17 @@ PyInit__socket(void) PyModule_AddIntConstant(m, "SOMAXCONN", 5); /* Common value */ #endif + /* Ancilliary message types */ +#ifdef SCM_RIGHTS + PyModule_AddIntConstant(m, "SCM_RIGHTS", SCM_RIGHTS); +#endif +#ifdef SCM_CREDENTIALS + PyModule_AddIntConstant(m, "SCM_CREDENTIALS", SCM_CREDENTIALS); +#endif +#ifdef SCM_CREDS + PyModule_AddIntConstant(m, "SCM_CREDS", SCM_CREDS); +#endif + /* Flags for send, recv */ #ifdef MSG_OOB PyModule_AddIntConstant(m, "MSG_OOB", MSG_OOB); @@ -4809,6 +5822,33 @@ PyInit__socket(void) #ifdef MSG_ETAG PyModule_AddIntConstant(m, "MSG_ETAG", MSG_ETAG); #endif +#ifdef MSG_NOSIGNAL + PyModule_AddIntConstant(m, "MSG_NOSIGNAL", MSG_NOSIGNAL); +#endif +#ifdef MSG_NOTIFICATION + PyModule_AddIntConstant(m, "MSG_NOTIFICATION", MSG_NOTIFICATION); +#endif +#ifdef MSG_CMSG_CLOEXEC + PyModule_AddIntConstant(m, "MSG_CMSG_CLOEXEC", MSG_CMSG_CLOEXEC); +#endif +#ifdef MSG_ERRQUEUE + PyModule_AddIntConstant(m, "MSG_ERRQUEUE", MSG_ERRQUEUE); +#endif +#ifdef MSG_CONFIRM + PyModule_AddIntConstant(m, "MSG_CONFIRM", MSG_CONFIRM); +#endif +#ifdef MSG_MORE + PyModule_AddIntConstant(m, "MSG_MORE", MSG_MORE); +#endif +#ifdef MSG_EOF + PyModule_AddIntConstant(m, "MSG_EOF", MSG_EOF); +#endif +#ifdef MSG_BCAST + PyModule_AddIntConstant(m, "MSG_BCAST", MSG_BCAST); +#endif +#ifdef MSG_MCAST + PyModule_AddIntConstant(m, "MSG_MCAST", MSG_MCAST); +#endif /* Protocol level and numbers, usable for [gs]etsockopt */ #ifdef SOL_SOCKET @@ -4844,6 +5884,28 @@ PyInit__socket(void) #else PyModule_AddIntConstant(m, "SOL_UDP", 17); #endif +#ifdef SOL_CAN_BASE + PyModule_AddIntConstant(m, "SOL_CAN_BASE", SOL_CAN_BASE); +#endif +#ifdef SOL_CAN_RAW + PyModule_AddIntConstant(m, "SOL_CAN_RAW", SOL_CAN_RAW); + PyModule_AddIntConstant(m, "CAN_RAW", CAN_RAW); +#endif +#ifdef HAVE_LINUX_CAN_H + PyModule_AddIntConstant(m, "CAN_EFF_FLAG", CAN_EFF_FLAG); + PyModule_AddIntConstant(m, "CAN_RTR_FLAG", CAN_RTR_FLAG); + PyModule_AddIntConstant(m, "CAN_ERR_FLAG", CAN_ERR_FLAG); + + PyModule_AddIntConstant(m, "CAN_SFF_MASK", CAN_SFF_MASK); + PyModule_AddIntConstant(m, "CAN_EFF_MASK", CAN_EFF_MASK); + PyModule_AddIntConstant(m, "CAN_ERR_MASK", CAN_ERR_MASK); +#endif +#ifdef HAVE_LINUX_CAN_RAW_H + PyModule_AddIntConstant(m, "CAN_RAW_FILTER", CAN_RAW_FILTER); + PyModule_AddIntConstant(m, "CAN_RAW_ERR_FILTER", CAN_RAW_ERR_FILTER); + PyModule_AddIntConstant(m, "CAN_RAW_LOOPBACK", CAN_RAW_LOOPBACK); + PyModule_AddIntConstant(m, "CAN_RAW_RECV_OWN_MSGS", CAN_RAW_RECV_OWN_MSGS); +#endif #ifdef IPPROTO_IP PyModule_AddIntConstant(m, "IPPROTO_IP", IPPROTO_IP); #else @@ -4948,6 +6010,9 @@ PyInit__socket(void) #ifdef IPPROTO_VRRP PyModule_AddIntConstant(m, "IPPROTO_VRRP", IPPROTO_VRRP); #endif +#ifdef IPPROTO_SCTP + PyModule_AddIntConstant(m, "IPPROTO_SCTP", IPPROTO_SCTP); +#endif #ifdef IPPROTO_BIP PyModule_AddIntConstant(m, "IPPROTO_BIP", IPPROTO_BIP); #endif @@ -5319,7 +6384,7 @@ PyInit__socket(void) DWORD codes[] = {SIO_RCVALL, SIO_KEEPALIVE_VALS}; const char *names[] = {"SIO_RCVALL", "SIO_KEEPALIVE_VALS"}; int i; - for(i = 0; i<sizeof(codes)/sizeof(*codes); ++i) { + for(i = 0; i<Py_ARRAY_LENGTH(codes); ++i) { PyObject *tmp; tmp = PyLong_FromUnsignedLong(codes[i]); if (tmp == NULL) diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f064795..13e33d5 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -59,9 +59,12 @@ typedef int socklen_t; #include <bluetooth.h> #endif +#ifdef HAVE_NET_IF_H +# include <net/if.h> +#endif + #ifdef HAVE_NETPACKET_PACKET_H # include <sys/ioctl.h> -# include <net/if.h> # include <netpacket/packet.h> #endif @@ -69,6 +72,14 @@ typedef int socklen_t; # include <linux/tipc.h> #endif +#ifdef HAVE_LINUX_CAN_H +#include <linux/can.h> +#endif + +#ifdef HAVE_LINUX_CAN_RAW_H +#include <linux/can/raw.h> +#endif + #ifndef Py__SOCKET_H #define Py__SOCKET_H #ifdef __cplusplus @@ -123,6 +134,9 @@ typedef union sock_addr { #ifdef HAVE_NETPACKET_PACKET_H struct sockaddr_ll ll; #endif +#ifdef HAVE_LINUX_CAN_H + struct sockaddr_can can; +#endif } sock_addr_t; /* The object holding a socket. It holds some extra information, diff --git a/Modules/sre.h b/Modules/sre.h index 518c11d..0d91f25 100644 --- a/Modules/sre.h +++ b/Modules/sre.h @@ -30,7 +30,8 @@ typedef struct { PyObject* pattern; /* pattern source (or None) */ int flags; /* flags used when compiling pattern source */ PyObject *weakreflist; /* List of weak references */ - int charsize; /* pattern charsize (or -1) */ + int logical_charsize; /* pattern charsize (or -1) */ + int charsize; /* pattern code */ Py_ssize_t codesize; SRE_CODE code[1]; @@ -71,6 +72,7 @@ typedef struct { PyObject* string; Py_ssize_t pos, endpos; /* character size */ + int logical_charsize; /* kind of thing: 1 - bytes, 2/4 - unicode */ int charsize; /* registers */ Py_ssize_t lastindex; diff --git a/Modules/syslogmodule.c b/Modules/syslogmodule.c index 5b86963..c7a2487 100644 --- a/Modules/syslogmodule.c +++ b/Modules/syslogmodule.c @@ -70,7 +70,7 @@ syslog_get_argv(void) Py_ssize_t argv_len, scriptlen; PyObject *scriptobj; - Py_UNICODE *atslash, *atstart; + Py_ssize_t slash; PyObject *argv = PySys_GetObject("argv"); if (argv == NULL) { @@ -90,16 +90,16 @@ syslog_get_argv(void) if (!PyUnicode_Check(scriptobj)) { return(NULL); } - scriptlen = PyUnicode_GET_SIZE(scriptobj); + scriptlen = PyUnicode_GET_LENGTH(scriptobj); if (scriptlen == 0) { return(NULL); } - atstart = PyUnicode_AS_UNICODE(scriptobj); - atslash = Py_UNICODE_strrchr(atstart, SEP); - if (atslash) { - return(PyUnicode_FromUnicode(atslash + 1, - scriptlen - (atslash - atstart) - 1)); + slash = PyUnicode_FindChar(scriptobj, SEP, 0, scriptlen, -1); + if (slash == -2) + return NULL; + if (slash != -1) { + return PyUnicode_Substring(scriptobj, slash, scriptlen); } else { Py_INCREF(scriptobj); return(scriptobj); diff --git a/Modules/termios.c b/Modules/termios.c index edeb6f5..b78d33e 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -9,11 +9,6 @@ #endif #include <termios.h> -#ifdef __osf__ -/* On OSF, sys/ioctl.h requires that struct termio already be defined, - * so this needs to be included first on that platform. */ -#include <termio.h> -#endif #include <sys/ioctl.h> /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, diff --git a/Modules/testcapi_long.h b/Modules/testcapi_long.h index fa94fd6..5784452 100644 --- a/Modules/testcapi_long.h +++ b/Modules/testcapi_long.h @@ -177,6 +177,32 @@ TESTNAME(PyObject *error(const char*)) Py_DECREF(one); } + /* Test F_PY_TO_{S,U} on non-pylong input. This should raise a TypeError. */ + { + TYPENAME out; + unsigned TYPENAME uout; + + Py_INCREF(Py_None); + + out = F_PY_TO_S(Py_None); + if (out != (TYPENAME)-1 || !PyErr_Occurred()) + return error("PyLong_AsXXX(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return error("PyLong_AsXXX(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + uout = F_PY_TO_U(Py_None); + if (uout != (unsigned TYPENAME)-1 || !PyErr_Occurred()) + return error("PyLong_AsXXX(None) didn't complain"); + if (!PyErr_ExceptionMatches(PyExc_TypeError)) + return error("PyLong_AsXXX(None) raised " + "something other than TypeError"); + PyErr_Clear(); + + Py_DECREF(Py_None); + } + Py_INCREF(Py_None); return Py_None; } diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 34e4875..24066d8 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -3,8 +3,6 @@ #include "Python.h" #include "_time.h" -#define TZNAME_ENCODING "utf-8" - #include <ctype.h> #ifdef HAVE_SYS_TYPES_H @@ -45,12 +43,11 @@ static long main_thread; #endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ -#if defined(MS_WINDOWS) && !defined(__BORLANDC__) -/* Win32 has better clock replacement; we have our own version below. */ -#undef HAVE_CLOCK -#undef TZNAME_ENCODING -#define TZNAME_ENCODING "mbcs" -#endif /* MS_WINDOWS && !defined(__BORLANDC__) */ +#if defined(HAVE_MBCS) +# define TZNAME_ENCODING "mbcs" +#else +# define TZNAME_ENCODING "utf-8" +#endif #if defined(PYOS_OS2) #define INCL_DOS @@ -66,9 +63,6 @@ static long main_thread; static int floatsleep(double); static double floattime(void); -/* For Y2K check */ -static PyObject *moddict; - static PyObject * time_time(PyObject *self, PyObject *unused) { @@ -87,25 +81,9 @@ PyDoc_STRVAR(time_doc, Return the current time in seconds since the Epoch.\n\ Fractions of a second may be present if the system clock provides them."); -#ifdef HAVE_CLOCK - -#ifndef CLOCKS_PER_SEC -#ifdef CLK_TCK -#define CLOCKS_PER_SEC CLK_TCK -#else -#define CLOCKS_PER_SEC 1000000 -#endif -#endif - -static PyObject * -time_clock(PyObject *self, PyObject *unused) -{ - return PyFloat_FromDouble(((double)clock()) / CLOCKS_PER_SEC); -} -#endif /* HAVE_CLOCK */ - #if defined(MS_WINDOWS) && !defined(__BORLANDC__) -/* Due to Mark Hammond and Tim Peters */ +/* Win32 has better clock replacement; we have our own version, due to Mark + Hammond and Tim Peters */ static PyObject * time_clock(PyObject *self, PyObject *unused) { @@ -130,8 +108,23 @@ time_clock(PyObject *self, PyObject *unused) return PyFloat_FromDouble(diff / divisor); } -#define HAVE_CLOCK /* So it gets included in the methods */ -#endif /* MS_WINDOWS && !defined(__BORLANDC__) */ +#elif defined(HAVE_CLOCK) + +#ifndef CLOCKS_PER_SEC +#ifdef CLK_TCK +#define CLOCKS_PER_SEC CLK_TCK +#else +#define CLOCKS_PER_SEC 1000000 +#endif +#endif + +static PyObject * +time_clock(PyObject *self, PyObject *unused) +{ + return PyFloat_FromDouble(((double)clock()) / CLOCKS_PER_SEC); +} +#endif /* HAVE_CLOCK */ + #ifdef HAVE_CLOCK PyDoc_STRVAR(clock_doc, @@ -148,6 +141,11 @@ time_sleep(PyObject *self, PyObject *args) double secs; if (!PyArg_ParseTuple(args, "d:sleep", &secs)) return NULL; + if (secs < 0) { + PyErr_SetString(PyExc_ValueError, + "sleep length must be non-negative"); + return NULL; + } if (floatsleep(secs) != 0) return NULL; Py_INCREF(Py_None); @@ -311,49 +309,6 @@ gettmarg(PyObject *args, struct tm *p) &p->tm_hour, &p->tm_min, &p->tm_sec, &p->tm_wday, &p->tm_yday, &p->tm_isdst)) return 0; - - /* If year is specified with less than 4 digits, its interpretation - * depends on the accept2dyear value. - * - * If accept2dyear is true (default), a backward compatibility behavior is - * invoked as follows: - * - * - for 2-digit year, century is guessed according to POSIX rules for - * %y strptime format: 21st century for y < 69, 20th century - * otherwise. A deprecation warning is issued when century - * information is guessed in this way. - * - * - for 3-digit or negative year, a ValueError exception is raised. - * - * If accept2dyear is false (set by the program or as a result of a - * non-empty value assigned to PYTHONY2K environment variable) all year - * values are interpreted as given. - */ - if (y < 1000) { - PyObject *accept = PyDict_GetItemString(moddict, - "accept2dyear"); - if (accept != NULL) { - int acceptval = PyObject_IsTrue(accept); - if (acceptval == -1) - return 0; - if (acceptval) { - if (0 <= y && y < 69) - y += 2000; - else if (69 <= y && y < 100) - y += 1900; - else { - PyErr_SetString(PyExc_ValueError, - "year out of range"); - return 0; - } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Century info guessed for a 2-digit year.", 1) != 0) - return 0; - } - } - else - return 0; - } p->tm_year = y - 1900; p->tm_mon--; p->tm_wday = (p->tm_wday + 1) % 7; @@ -585,11 +540,12 @@ time_strptime(PyObject *self, PyObject *args) { PyObject *strptime_module = PyImport_ImportModuleNoBlock("_strptime"); PyObject *strptime_result; + _Py_identifier(_strptime_time); if (!strptime_module) return NULL; - strptime_result = PyObject_CallMethod(strptime_module, - "_strptime_time", "O", args); + strptime_result = _PyObject_CallMethodId(strptime_module, + &PyId__strptime_time, "O", args); Py_DECREF(strptime_module); return strptime_result; } @@ -606,31 +562,20 @@ _asctime(struct tm *timeptr) { /* Inspired by Open Group reference implementation available at * http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html */ - static char wday_name[7][3] = { + static char wday_name[7][4] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; - static char mon_name[12][3] = { + static char mon_name[12][4] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - char buf[20]; /* 'Sun Sep 16 01:03:52\0' */ - int n; - - n = PyOS_snprintf(buf, sizeof(buf), "%.3s %.3s%3d %.2d:%.2d:%.2d", - wday_name[timeptr->tm_wday], - mon_name[timeptr->tm_mon], - timeptr->tm_mday, timeptr->tm_hour, - timeptr->tm_min, timeptr->tm_sec); - /* XXX: since the fields used by snprintf above are validated in checktm, - * the following condition should never trigger. We keep the check because - * historically fixed size buffer used in asctime was the source of - * crashes. */ - if (n + 1 != sizeof(buf)) { - PyErr_SetString(PyExc_ValueError, "unconvertible time"); - return NULL; - } - - return PyUnicode_FromFormat("%s %d", buf, 1900 + timeptr->tm_year); + return PyUnicode_FromFormat( + "%s %s%3d %.2d:%.2d:%.2d %d", + wday_name[timeptr->tm_wday], + mon_name[timeptr->tm_mon], + timeptr->tm_mday, timeptr->tm_hour, + timeptr->tm_min, timeptr->tm_sec, + 1900 + timeptr->tm_year); } static PyObject * @@ -846,7 +791,7 @@ PyInit_timezone(PyObject *m) { static PyMethodDef time_methods[] = { {"time", time_time, METH_NOARGS, time_doc}, -#ifdef HAVE_CLOCK +#if (defined(MS_WINDOWS) && !defined(__BORLANDC__)) || defined(HAVE_CLOCK) {"clock", time_clock, METH_NOARGS, clock_doc}, #endif {"sleep", time_sleep, METH_VARARGS, sleep_doc}, @@ -879,7 +824,7 @@ The actual value can be retrieved by calling gmtime(0).\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ - year (four digits, e.g. 1998)\n\ + year (including century, e.g. 1998)\n\ month (1-12)\n\ day (1-31)\n\ hours (0-23)\n\ @@ -931,18 +876,10 @@ PyMODINIT_FUNC PyInit_time(void) { PyObject *m; - char *p; m = PyModule_Create(&timemodule); if (m == NULL) return NULL; - /* Accept 2-digit dates unless PYTHONY2K is set and non-empty */ - p = Py_GETENV("PYTHONY2K"); - PyModule_AddIntConstant(m, "accept2dyear", (long) (!p || !*p)); - /* Squirrel away the module's dictionary for the y2k check */ - moddict = PyModule_GetDict(m); - Py_INCREF(moddict); - /* Set, or reset, module variables like time.timezone */ PyInit_timezone(m); @@ -985,23 +922,28 @@ floatsleep(double secs) #if defined(HAVE_SELECT) && !defined(__EMX__) struct timeval t; double frac; + int err; + frac = fmod(secs, 1.0); secs = floor(secs); t.tv_sec = (long)secs; t.tv_usec = (long)(frac*1000000.0); Py_BEGIN_ALLOW_THREADS - if (select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t) != 0) { + err = select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t); + Py_END_ALLOW_THREADS + if (err != 0) { #ifdef EINTR - if (errno != EINTR) { -#else - if (1) { + if (errno == EINTR) { + if (PyErr_CheckSignals()) + return -1; + } + else #endif - Py_BLOCK_THREADS + { PyErr_SetFromErrno(PyExc_IOError); return -1; } } - Py_END_ALLOW_THREADS #elif defined(__WATCOMC__) && !defined(__QNX__) /* XXX Can't interrupt this sleep */ Py_BEGIN_ALLOW_THREADS diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index 887056c..db0f115 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -92,16 +92,13 @@ new_previous_version(const char*name, const change_record* (*getrecord)(Py_UCS4) static Py_UCS4 getuchar(PyUnicodeObject *obj) { - Py_UNICODE *v = PyUnicode_AS_UNICODE(obj); - - if (PyUnicode_GET_SIZE(obj) == 1) - return *v; -#ifndef Py_UNICODE_WIDE - else if ((PyUnicode_GET_SIZE(obj) == 2) && - (0xD800 <= v[0] && v[0] <= 0xDBFF) && - (0xDC00 <= v[1] && v[1] <= 0xDFFF)) - return (((v[0] & 0x3FF)<<10) | (v[1] & 0x3FF)) + 0x10000; -#endif + if (PyUnicode_READY(obj)) + return (Py_UCS4)-1; + if (PyUnicode_GET_LENGTH(obj) == 1) { + if (PyUnicode_READY(obj)) + return (Py_UCS4)-1; + return PyUnicode_READ_CHAR(obj, 0); + } PyErr_SetString(PyExc_TypeError, "need a single Unicode character as parameter"); return (Py_UCS4)-1; @@ -442,7 +439,7 @@ unicodedata_decomposition(PyObject *self, PyObject *args) from Tools/unicode/makeunicodedata.py, it should not be possible to overflow decomp_prefix. */ prefix_index = decomp_data[index] & 255; - assert(prefix_index < (sizeof(decomp_prefix)/sizeof(*decomp_prefix))); + assert(prefix_index < Py_ARRAY_LENGTH(decomp_prefix)); /* copy prefix */ i = strlen(decomp_prefix[prefix_index]); @@ -497,36 +494,44 @@ static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) { PyObject *result; - Py_UNICODE *i, *end, *o; + Py_UCS4 *output; + Py_ssize_t i, o, osize; + int kind; + void *data; /* Longest decomposition in Unicode 3.2: U+FDFA */ - Py_UNICODE stack[20]; + Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; stackptr = 0; - isize = PyUnicode_GET_SIZE(input); + isize = PyUnicode_GET_LENGTH(input); /* Overallocate atmost 10 characters. */ space = (isize > 10 ? 10 : isize) + isize; - result = PyUnicode_FromUnicode(NULL, space); - if (!result) + osize = space; + output = PyMem_Malloc(space * sizeof(Py_UCS4)); + if (!output) { + PyErr_NoMemory(); return NULL; - i = PyUnicode_AS_UNICODE(input); - end = i + isize; - o = PyUnicode_AS_UNICODE(result); + } + i = o = 0; + kind = PyUnicode_KIND(input); + data = PyUnicode_DATA(input); - while (i < end) { - stack[stackptr++] = *i++; + while (i < isize) { + stack[stackptr++] = PyUnicode_READ(kind, data, i++); while(stackptr) { - Py_UNICODE code = stack[--stackptr]; + Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in a single step, so we need atleast that much room. */ if (space < 3) { - Py_ssize_t newsize = PyUnicode_GET_SIZE(result) + 10; + osize += 10; space += 10; - if (PyUnicode_Resize(&result, newsize) == -1) + output = PyMem_Realloc(output, osize*sizeof(Py_UCS4)); + if (output == NULL) { + PyErr_NoMemory(); return NULL; - o = PyUnicode_AS_UNICODE(result) + newsize - space; + } } /* Hangul Decomposition. */ if (SBase <= code && code < (SBase+SCount)) { @@ -534,11 +539,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) int L = LBase + SIndex / NCount; int V = VBase + (SIndex % NCount) / TCount; int T = TBase + SIndex % TCount; - *o++ = L; - *o++ = V; + output[o++] = L; + output[o++] = V; space -= 2; if (T != TBase) { - *o++ = T; + output[o++] = T; space --; } continue; @@ -558,7 +563,7 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Copy character if it is not decomposable, or has a compatibility decomposition, but we do NFD. */ if (!count || (prefix && !k)) { - *o++ = code; + output[o++] = code; space--; continue; } @@ -571,15 +576,20 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) } } - /* Drop overallocation. Cannot fail. */ - PyUnicode_Resize(&result, PyUnicode_GET_SIZE(result) - space); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + output, o); + PyMem_Free(output); + if (!result) + return NULL; + /* result is guaranteed to be ready, as it is compact. */ + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); /* Sort canonically. */ - i = PyUnicode_AS_UNICODE(result); - prev = _getrecord_ex(*i)->combining; - end = i + PyUnicode_GET_SIZE(result); - for (i++; i < end; i++) { - cur = _getrecord_ex(*i)->combining; + i = 0; + prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + for (i++; i < PyUnicode_GET_LENGTH(result); i++) { + cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; if (prev == 0 || cur == 0 || prev <= cur) { prev = cur; continue; @@ -587,31 +597,32 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) /* Non-canonical order. Need to switch *i with previous. */ o = i - 1; while (1) { - Py_UNICODE tmp = o[1]; - o[1] = o[0]; - o[0] = tmp; + Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); + PyUnicode_WRITE(kind, data, o+1, + PyUnicode_READ(kind, data, o)); + PyUnicode_WRITE(kind, data, o, tmp); o--; - if (o < PyUnicode_AS_UNICODE(result)) + if (o < 0) break; - prev = _getrecord_ex(*o)->combining; + prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; if (prev == 0 || prev <= cur) break; } - prev = _getrecord_ex(*i)->combining; + prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; } return result; } static int -find_nfc_index(PyObject *self, struct reindex* nfc, Py_UNICODE code) +find_nfc_index(PyObject *self, struct reindex* nfc, Py_UCS4 code) { - int index; + unsigned int index; for (index = 0; nfc[index].start; index++) { - int start = nfc[index].start; + unsigned int start = nfc[index].start; if (code < start) return -1; if (code <= start + nfc[index].count) { - int delta = code - start; + unsigned int delta = code - start; return nfc[index].index + delta; } } @@ -622,27 +633,36 @@ static PyObject* nfc_nfkc(PyObject *self, PyObject *input, int k) { PyObject *result; - Py_UNICODE *i, *i1, *o, *end; + int kind; + void *data; + Py_UCS4 *output; + Py_ssize_t i, i1, o, len; int f,l,index,index1,comb; - Py_UNICODE code; - Py_UNICODE *skipped[20]; + Py_UCS4 code; + Py_ssize_t skipped[20]; int cskipped = 0; result = nfd_nfkd(self, input, k); if (!result) return NULL; - - /* We are going to modify result in-place. - If nfd_nfkd is changed to sometimes return the input, - this code needs to be reviewed. */ - assert(result != input); - - i = PyUnicode_AS_UNICODE(result); - end = i + PyUnicode_GET_SIZE(result); - o = PyUnicode_AS_UNICODE(result); + /* result will be "ready". */ + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + len = PyUnicode_GET_LENGTH(result); + + /* We allocate a buffer for the output. + If we find that we made no changes, we still return + the NFD result. */ + output = PyMem_Malloc(len * sizeof(Py_UCS4)); + if (!output) { + PyErr_NoMemory(); + Py_DECREF(result); + return 0; + } + i = o = 0; again: - while (i < end) { + while (i < len) { for (index = 0; index < cskipped; index++) { if (skipped[index] == i) { /* *i character is skipped. @@ -655,33 +675,41 @@ nfc_nfkc(PyObject *self, PyObject *input, int k) } /* Hangul Composition. We don't need to check for <LV,T> pairs, since we always have decomposed data. */ - if (LBase <= *i && *i < (LBase+LCount) && - i + 1 < end && - VBase <= i[1] && i[1] <= (VBase+VCount)) { + code = PyUnicode_READ(kind, data, i); + if (LBase <= code && code < (LBase+LCount) && + i + 1 < len && + VBase <= PyUnicode_READ(kind, data, i+1) && + PyUnicode_READ(kind, data, i+1) <= (VBase+VCount)) { int LIndex, VIndex; - LIndex = i[0] - LBase; - VIndex = i[1] - VBase; + LIndex = code - LBase; + VIndex = PyUnicode_READ(kind, data, i+1) - VBase; code = SBase + (LIndex*VCount+VIndex)*TCount; i+=2; - if (i < end && - TBase <= *i && *i <= (TBase+TCount)) { - code += *i-TBase; + if (i < len && + TBase <= PyUnicode_READ(kind, data, i) && + PyUnicode_READ(kind, data, i) <= (TBase+TCount)) { + code += PyUnicode_READ(kind, data, i)-TBase; i++; } - *o++ = code; + output[o++] = code; continue; } - f = find_nfc_index(self, nfc_first, *i); + /* code is still input[i] here */ + f = find_nfc_index(self, nfc_first, code); if (f == -1) { - *o++ = *i++; + output[o++] = code; + i++; continue; } /* Find next unblocked character. */ i1 = i+1; comb = 0; - while (i1 < end) { - int comb1 = _getrecord_ex(*i1)->combining; + /* output base character for now; might be updated later. */ + output[o] = PyUnicode_READ(kind, data, i); + while (i1 < len) { + Py_UCS4 code1 = PyUnicode_READ(kind, data, i1); + int comb1 = _getrecord_ex(code1)->combining; if (comb) { if (comb1 == 0) break; @@ -691,8 +719,8 @@ nfc_nfkc(PyObject *self, PyObject *input, int k) continue; } } - l = find_nfc_index(self, nfc_last, *i1); - /* *i1 cannot be combined with *i. If *i1 + l = find_nfc_index(self, nfc_last, code1); + /* i1 cannot be combined with i. If i1 is a starter, we don't need to look further. Otherwise, record the combining class. */ if (l == -1) { @@ -711,19 +739,28 @@ nfc_nfkc(PyObject *self, PyObject *input, int k) goto not_combinable; /* Replace the original character. */ - *i = code; + output[o] = code; /* Mark the second character unused. */ assert(cskipped < 20); skipped[cskipped++] = i1; i1++; - f = find_nfc_index(self, nfc_first, *i); + f = find_nfc_index(self, nfc_first, output[o]); if (f == -1) break; } - *o++ = *i++; + /* Output character was already written. + Just advance the indices. */ + o++; i++; + } + if (o == len) { + /* No changes. Return original string. */ + PyMem_Free(output); + return result; } - if (o != end) - PyUnicode_Resize(&result, o - PyUnicode_AS_UNICODE(result)); + Py_DECREF(result); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + output, o); + PyMem_Free(output); return result; } @@ -731,7 +768,9 @@ nfc_nfkc(PyObject *self, PyObject *input, int k) static int is_normalized(PyObject *self, PyObject *input, int nfc, int k) { - Py_UNICODE *i, *end; + Py_ssize_t i, len; + int kind; + void *data; unsigned char prev_combining = 0, quickcheck_mask; /* An older version of the database is requested, quickchecks must be @@ -743,10 +782,13 @@ is_normalized(PyObject *self, PyObject *input, int nfc, int k) as described in http://unicode.org/reports/tr15/#Annex8. */ quickcheck_mask = 3 << ((nfc ? 4 : 0) + (k ? 2 : 0)); - i = PyUnicode_AS_UNICODE(input); - end = i + PyUnicode_GET_SIZE(input); - while (i < end) { - const _PyUnicode_DatabaseRecord *record = _getrecord_ex(*i++); + i = 0; + kind = PyUnicode_KIND(input); + data = PyUnicode_DATA(input); + len = PyUnicode_GET_LENGTH(input); + while (i < len) { + Py_UCS4 ch = PyUnicode_READ(kind, data, i++); + const _PyUnicode_DatabaseRecord *record = _getrecord_ex(ch); unsigned char combining = record->combining; unsigned char quickcheck = record->normalization_quick_check; @@ -775,7 +817,10 @@ unicodedata_normalize(PyObject *self, PyObject *args) &form, &PyUnicode_Type, &input)) return NULL; - if (PyUnicode_GetSize(input) == 0) { + if (PyUnicode_READY(input) == -1) + return NULL; + + if (PyUnicode_GET_LENGTH(input) == 0) { /* Special case empty input strings, since resizing them later would cause internal errors. */ Py_INCREF(input); @@ -993,7 +1038,7 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) *len = -1; for (i = 0; i < count; i++) { char *s = hangul_syllables[i][column]; - len1 = strlen(s); + len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int); if (len1 <= *len) continue; if (strncmp(str, s, len1) == 0) { @@ -1142,7 +1187,6 @@ static PyObject * unicodedata_lookup(PyObject* self, PyObject* args) { Py_UCS4 code; - Py_UNICODE str[2]; char* name; int namelen; @@ -1155,15 +1199,7 @@ unicodedata_lookup(PyObject* self, PyObject* args) return NULL; } -#ifndef Py_UNICODE_WIDE - if (code >= 0x10000) { - str[0] = 0xd800 + ((code - 0x10000) >> 10); - str[1] = 0xdc00 + ((code - 0x10000) & 0x3ff); - return PyUnicode_FromUnicode(str, 2); - } -#endif - str[0] = (Py_UNICODE) code; - return PyUnicode_FromUnicode(str, 1); + return PyUnicode_FromOrdinal(code); } /* XXX Add doc strings. */ diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index ec924f2..661b6e2 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -187,8 +187,7 @@ static PyType_Spec Str_Type_spec = { static PyObject * null_richcompare(PyObject *self, PyObject *other, int op) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyType_Slot Null_Type_slots[] = { diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 68c2894..87dc0dbf 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -49,7 +49,7 @@ static PyObject *zip_directory_cache = NULL; /* forward decls */ static PyObject *read_directory(PyObject *archive); static PyObject *get_data(PyObject *archive, PyObject *toc_entry); -static PyObject *get_module_code(ZipImporter *self, char *fullname, +static PyObject *get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath); @@ -64,7 +64,7 @@ static int zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) { PyObject *pathobj, *files; - Py_UNICODE *path, *p, *prefix, buf[MAXPATHLEN+2]; + Py_UCS4 *path, *p, *prefix, buf[MAXPATHLEN+2]; Py_ssize_t len; if (!_PyArg_NoKeywords("zipimporter()", kwds)) @@ -74,8 +74,11 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) PyUnicode_FSDecoder, &pathobj)) return -1; + if (PyUnicode_READY(pathobj) == -1) + return -1; + /* copy path to buf */ - len = PyUnicode_GET_SIZE(pathobj); + len = PyUnicode_GET_LENGTH(pathobj); if (len == 0) { PyErr_SetString(ZipImportError, "archive path is empty"); goto error; @@ -85,7 +88,8 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) "archive path too long"); goto error; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(pathobj)); + if (!PyUnicode_AsUCS4(pathobj, buf, Py_ARRAY_LENGTH(buf), 1)) + goto error; #ifdef ALTSEP for (p = buf; *p; p++) { @@ -101,7 +105,8 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) int rv; if (pathobj == NULL) { - pathobj = PyUnicode_FromUnicode(buf, len); + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (pathobj == NULL) goto error; } @@ -116,7 +121,7 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) else if (PyErr_Occurred()) goto error; /* back up one path element */ - p = Py_UNICODE_strrchr(buf, SEP); + p = Py_UCS4_strrchr(buf, SEP); if (prefix != NULL) *prefix = SEP; if (p == NULL) @@ -148,7 +153,7 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) if (prefix != NULL) { prefix++; - len = Py_UNICODE_strlen(prefix); + len = Py_UCS4_strlen(prefix); if (prefix[len-1] != SEP) { /* add trailing SEP */ prefix[len] = SEP; @@ -158,7 +163,8 @@ zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) } else len = 0; - self->prefix = PyUnicode_FromUnicode(prefix, len); + self->prefix = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + prefix, len); if (self->prefix == NULL) goto error; @@ -193,7 +199,7 @@ zipimporter_repr(ZipImporter *self) { if (self->archive == NULL) return PyUnicode_FromString("<zipimporter object \"???\">"); - else if (self->prefix != NULL && PyUnicode_GET_SIZE(self->prefix) != 0) + else if (self->prefix != NULL && PyUnicode_GET_LENGTH(self->prefix) != 0) return PyUnicode_FromFormat("<zipimporter object \"%U%c%U\">", self->archive, SEP, self->prefix); else @@ -202,49 +208,64 @@ zipimporter_repr(ZipImporter *self) } /* return fullname.split(".")[-1] */ -static char * -get_subname(char *fullname) +static PyObject * +get_subname(PyObject *fullname) { - char *subname = strrchr(fullname, '.'); - if (subname == NULL) - subname = fullname; - else + Py_ssize_t len; + Py_UCS4 *subname, *fullname_ucs4; + fullname_ucs4 = PyUnicode_AsUCS4Copy(fullname); + if (!fullname_ucs4) + return NULL; + subname = Py_UCS4_strrchr(fullname_ucs4, '.'); + if (subname == NULL) { + PyMem_Free(fullname_ucs4); + Py_INCREF(fullname); + return fullname; + } else { + PyObject *result; subname++; - return subname; + len = PyUnicode_GET_LENGTH(fullname); + len -= subname - fullname_ucs4; + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); + PyMem_Free(fullname_ucs4); + return result; + } } /* Given a (sub)modulename, write the potential file path in the archive (without extension) to the path buffer. Return the - length of the resulting string. */ -static int -make_filename(PyObject *prefix_obj, char *name, char *path, size_t pathsize) -{ - size_t len; - char *p; - PyObject *prefix; + length of the resulting string. - prefix = PyUnicode_EncodeFSDefault(prefix_obj); - if (prefix == NULL) - return -1; - len = PyBytes_GET_SIZE(prefix); + return self.prefix + name.replace('.', os.sep) */ +static PyObject* +make_filename(PyObject *prefix, PyObject *name) +{ + PyObject *pathobj; + Py_UCS4 *p, *buf; + Py_ssize_t len; - /* self.prefix + name [+ SEP + "__init__"] + ".py[co]" */ - if (len + strlen(name) + 13 >= pathsize - 1) { - PyErr_SetString(ZipImportError, "path too long"); - Py_DECREF(prefix); - return -1; + len = PyUnicode_GET_LENGTH(prefix) + PyUnicode_GET_LENGTH(name) + 1; + p = buf = PyMem_Malloc(sizeof(Py_UCS4) * len); + if (buf == NULL) { + PyErr_NoMemory(); + return NULL; } - strcpy(path, PyBytes_AS_STRING(prefix)); - Py_DECREF(prefix); - strcpy(path + len, name); - for (p = path + len; *p; p++) { + if (!PyUnicode_AsUCS4(prefix, p, len, 0)) + return NULL; + p += PyUnicode_GET_LENGTH(prefix); + len -= PyUnicode_GET_LENGTH(prefix); + if (!PyUnicode_AsUCS4(name, p, len, 1)) + return NULL; + for (; *p; p++) { if (*p == '.') *p = SEP; } - len += strlen(name); - assert(len < INT_MAX); - return (int)len; + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, p-buf); + PyMem_Free(buf); + return pathobj; } enum zi_module_info { @@ -256,27 +277,38 @@ enum zi_module_info { /* Return some information about a module. */ static enum zi_module_info -get_module_info(ZipImporter *self, char *fullname) +get_module_info(ZipImporter *self, PyObject *fullname) { - char *subname, path[MAXPATHLEN + 1]; - int len; + PyObject *subname; + PyObject *path, *fullpath, *item; struct st_zip_searchorder *zso; subname = get_subname(fullname); + if (subname == NULL) + return MI_ERROR; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return MI_ERROR; for (zso = zip_searchorder; *zso->suffix; zso++) { - strcpy(path + len, zso->suffix); - if (PyDict_GetItemString(self->files, path) != NULL) { + fullpath = PyUnicode_FromFormat("%U%s", path, zso->suffix); + if (fullpath == NULL) { + Py_DECREF(path); + return MI_ERROR; + } + item = PyDict_GetItem(self->files, fullpath); + Py_DECREF(fullpath); + if (item != NULL) { + Py_DECREF(path); if (zso->type & IS_PACKAGE) return MI_PACKAGE; else return MI_MODULE; } } + Py_DECREF(path); return MI_NOT_FOUND; } @@ -287,10 +319,10 @@ zipimporter_find_module(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *path = NULL; - char *fullname; + PyObject *fullname; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s|O:zipimporter.find_module", + if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module", &fullname, &path)) return NULL; @@ -311,19 +343,21 @@ zipimporter_load_module(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *code = NULL, *mod, *dict; - char *fullname; - PyObject *modpath = NULL, *modpath_bytes; + PyObject *fullname; + PyObject *modpath = NULL; int ispackage; - if (!PyArg_ParseTuple(args, "s:zipimporter.load_module", + if (!PyArg_ParseTuple(args, "U:zipimporter.load_module", &fullname)) return NULL; + if (PyUnicode_READY(fullname) == -1) + return NULL; code = get_module_code(self, fullname, &ispackage, &modpath); if (code == NULL) goto error; - mod = PyImport_AddModule(fullname); + mod = PyImport_AddModuleObject(fullname); if (mod == NULL) goto error; dict = PyModule_GetDict(mod); @@ -336,17 +370,17 @@ zipimporter_load_module(PyObject *obj, PyObject *args) /* add __path__ to the module *before* the code gets executed */ PyObject *pkgpath, *fullpath; - char *subname = get_subname(fullname); + PyObject *subname = get_subname(fullname); int err; - fullpath = PyUnicode_FromFormat("%U%c%U%s", + fullpath = PyUnicode_FromFormat("%U%c%U%U", self->archive, SEP, self->prefix, subname); + Py_DECREF(subname); if (fullpath == NULL) goto error; - pkgpath = Py_BuildValue("[O]", fullpath); - Py_DECREF(fullpath); + pkgpath = Py_BuildValue("[N]", fullpath); if (pkgpath == NULL) goto error; err = PyDict_SetItemString(dict, "__path__", pkgpath); @@ -354,18 +388,13 @@ zipimporter_load_module(PyObject *obj, PyObject *args) if (err != 0) goto error; } - modpath_bytes = PyUnicode_EncodeFSDefault(modpath); - if (modpath_bytes == NULL) - goto error; - mod = PyImport_ExecCodeModuleEx(fullname, code, - PyBytes_AS_STRING(modpath_bytes)); - Py_DECREF(modpath_bytes); + mod = PyImport_ExecCodeModuleObject(fullname, code, modpath, NULL); Py_CLEAR(code); if (mod == NULL) goto error; if (Py_VerboseFlag) - PySys_FormatStderr("import %s # loaded from Zip %U\n", + PySys_FormatStderr("import %U # loaded from Zip %U\n", fullname, modpath); Py_DECREF(modpath); return mod; @@ -380,12 +409,10 @@ static PyObject * zipimporter_get_filename(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - PyObject *code; - char *fullname; - PyObject *modpath; + PyObject *fullname, *code, *modpath; int ispackage; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_filename", + if (!PyArg_ParseTuple(args, "U:zipimporter.get_filename", &fullname)) return NULL; @@ -404,10 +431,10 @@ static PyObject * zipimporter_is_package(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - char *fullname; + PyObject *fullname; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s:zipimporter.is_package", + if (!PyArg_ParseTuple(args, "U:zipimporter.is_package", &fullname)) return NULL; @@ -415,52 +442,59 @@ zipimporter_is_package(PyObject *obj, PyObject *args) if (mi == MI_ERROR) return NULL; if (mi == MI_NOT_FOUND) { - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); + PyErr_Format(ZipImportError, "can't find module %R", fullname); return NULL; } return PyBool_FromLong(mi == MI_PACKAGE); } + static PyObject * zipimporter_get_data(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *pathobj, *key; - const Py_UNICODE *path; + const Py_UCS4 *path; #ifdef ALTSEP - Py_UNICODE *p, buf[MAXPATHLEN + 1]; + Py_UCS4 *p; #endif - Py_UNICODE *archive; PyObject *toc_entry; Py_ssize_t path_len, len; + Py_UCS4 buf[MAXPATHLEN + 1], archive[MAXPATHLEN + 1]; if (!PyArg_ParseTuple(args, "U:zipimporter.get_data", &pathobj)) return NULL; - path_len = PyUnicode_GET_SIZE(pathobj); - path = PyUnicode_AS_UNICODE(pathobj); -#ifdef ALTSEP + if (PyUnicode_READY(pathobj) == -1) + return NULL; + + path_len = PyUnicode_GET_LENGTH(pathobj); if (path_len >= MAXPATHLEN) { PyErr_SetString(ZipImportError, "path too long"); return NULL; } - Py_UNICODE_strcpy(buf, path); + if (!PyUnicode_AsUCS4(pathobj, buf, Py_ARRAY_LENGTH(buf), 1)) + return NULL; + path = buf; +#ifdef ALTSEP for (p = buf; *p; p++) { if (*p == ALTSEP) *p = SEP; } - path = buf; #endif - archive = PyUnicode_AS_UNICODE(self->archive); - len = PyUnicode_GET_SIZE(self->archive); - if ((size_t)len < Py_UNICODE_strlen(path) && - Py_UNICODE_strncmp(path, archive, len) == 0 && - path[len] == SEP) { - path += len + 1; - path_len -= len + 1; + len = PyUnicode_GET_LENGTH(self->archive); + if ((size_t)len < Py_UCS4_strlen(path)) { + if (!PyUnicode_AsUCS4(self->archive, archive, Py_ARRAY_LENGTH(archive), 1)) + return NULL; + if (Py_UCS4_strncmp(path, archive, len) == 0 && + path[len] == SEP) { + path += len + 1; + path_len -= len + 1; + } } - key = PyUnicode_FromUnicode(path, path_len); + key = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + path, path_len); if (key == NULL) return NULL; toc_entry = PyDict_GetItem(self->files, key); @@ -477,9 +511,9 @@ static PyObject * zipimporter_get_code(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - char *fullname; + PyObject *fullname; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_code", &fullname)) + if (!PyArg_ParseTuple(args, "U:zipimporter.get_code", &fullname)) return NULL; return get_module_code(self, fullname, NULL, NULL); @@ -490,34 +524,39 @@ zipimporter_get_source(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *toc_entry; - char *fullname, *subname, path[MAXPATHLEN+1]; - int len; + PyObject *fullname, *subname, *path, *fullpath; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_source", &fullname)) + if (!PyArg_ParseTuple(args, "U:zipimporter.get_source", &fullname)) return NULL; mi = get_module_info(self, fullname); if (mi == MI_ERROR) return NULL; if (mi == MI_NOT_FOUND) { - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); + PyErr_Format(ZipImportError, "can't find module %R", fullname); return NULL; } + subname = get_subname(fullname); + if (subname == NULL) + return NULL; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return NULL; - if (mi == MI_PACKAGE) { - path[len] = SEP; - strcpy(path + len + 1, "__init__.py"); - } + if (mi == MI_PACKAGE) + fullpath = PyUnicode_FromFormat("%U%c__init__.py", path, SEP); else - strcpy(path + len, ".py"); + fullpath = PyUnicode_FromFormat("%U.py", path); + Py_DECREF(path); + if (fullpath == NULL) + return NULL; - toc_entry = PyDict_GetItemString(self->files, path); + toc_entry = PyDict_GetItem(self->files, fullpath); + Py_DECREF(fullpath); if (toc_entry != NULL) { PyObject *res, *bytes; bytes = get_data(self->archive, toc_entry); @@ -708,17 +747,17 @@ get_long(unsigned char *buf) { data_size and file_offset are 0. */ static PyObject * -read_directory(PyObject *archive_obj) +read_directory(PyObject *archive) { - /* FIXME: work on Py_UNICODE* instead of char* */ PyObject *files = NULL; FILE *fp; unsigned short flags; long compress, crc, data_size, file_size, file_offset, date, time; long header_offset, name_size, header_size, header_position; - long i, l, count; + long l, count; + Py_ssize_t i; size_t length; - Py_UNICODE path[MAXPATHLEN + 5]; + Py_UCS4 path[MAXPATHLEN + 5]; char name[MAXPATHLEN + 5]; PyObject *nameobj = NULL; char *p, endof_central_dir[22]; @@ -727,29 +766,30 @@ read_directory(PyObject *archive_obj) const char *charset; int bootstrap; - if (PyUnicode_GET_SIZE(archive_obj) > MAXPATHLEN) { + if (PyUnicode_GET_LENGTH(archive) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "Zip path name is too long"); return NULL; } - Py_UNICODE_strcpy(path, PyUnicode_AS_UNICODE(archive_obj)); + if (!PyUnicode_AsUCS4(archive, path, Py_ARRAY_LENGTH(path), 1)) + return NULL; - fp = _Py_fopen(archive_obj, "rb"); + fp = _Py_fopen(archive, "rb"); if (fp == NULL) { - PyErr_Format(ZipImportError, "can't open Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "can't open Zip file: %R", archive); return NULL; } fseek(fp, -22, SEEK_END); header_position = ftell(fp); if (fread(endof_central_dir, 1, 22, fp) != 22) { fclose(fp); - PyErr_Format(ZipImportError, "can't read Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "can't read Zip file: %R", archive); return NULL; } if (get_long((unsigned char *)endof_central_dir) != 0x06054B50) { /* Bad: End of Central Dir signature */ fclose(fp); - PyErr_Format(ZipImportError, "not a Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "not a Zip file: %R", archive); return NULL; } @@ -762,7 +802,7 @@ read_directory(PyObject *archive_obj) if (files == NULL) goto error; - length = Py_UNICODE_strlen(path); + length = Py_UCS4_strlen(path); path[length] = SEP; /* Start of Central Directory */ @@ -793,7 +833,7 @@ read_directory(PyObject *archive_obj) name_size = MAXPATHLEN; p = name; - for (i = 0; i < name_size; i++) { + for (i = 0; i < (Py_ssize_t)name_size; i++) { *p = (char)getc(fp); if (*p == '/') *p = SEP; @@ -818,6 +858,8 @@ read_directory(PyObject *archive_obj) else charset = "cp437"; nameobj = PyUnicode_Decode(name, name_size, charset, NULL); + if (PyUnicode_READY(nameobj) == -1) + goto error; if (nameobj == NULL) { if (bootstrap) PyErr_Format(PyExc_NotImplementedError, @@ -826,9 +868,12 @@ read_directory(PyObject *archive_obj) PY_MAJOR_VERSION, PY_MINOR_VERSION); goto error; } - Py_UNICODE_strncpy(path + length + 1, PyUnicode_AS_UNICODE(nameobj), MAXPATHLEN - length - 1); - - pathobj = PyUnicode_FromUnicode(path, Py_UNICODE_strlen(path)); + for (i = 0; (i < (MAXPATHLEN - (Py_ssize_t)length - 1)) && + (i < PyUnicode_GET_LENGTH(nameobj)); i++) + path[length + 1 + i] = PyUnicode_READ_CHAR(nameobj, i); + path[length + 1 + i] = 0; + pathobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + path, Py_UCS4_strlen(path)); if (pathobj == NULL) goto error; t = Py_BuildValue("Niiiiiii", pathobj, compress, data_size, @@ -844,8 +889,8 @@ read_directory(PyObject *archive_obj) } fclose(fp); if (Py_VerboseFlag) - PySys_FormatStderr("# zipimport: found %ld names in %U\n", - count, archive_obj); + PySys_FormatStderr("# zipimport: found %ld names in %R\n", + count, archive); return files; error: fclose(fp); @@ -863,6 +908,7 @@ get_decompress_func(void) static int importing_zlib = 0; PyObject *zlib; PyObject *decompress; + _Py_identifier(decompress); if (importing_zlib != 0) /* Someone has a zlib.py[co] in their Zip file; @@ -872,8 +918,8 @@ get_decompress_func(void) zlib = PyImport_ImportModuleNoBlock("zlib"); importing_zlib = 0; if (zlib != NULL) { - decompress = PyObject_GetAttrString(zlib, - "decompress"); + decompress = _PyObject_GetAttrId(zlib, + &PyId_decompress); Py_DECREF(zlib); } else { @@ -998,7 +1044,7 @@ eq_mtime(time_t t1, time_t t2) to .py if available and we don't want to mask other errors). Returns a new reference. */ static PyObject * -unmarshal_code(char *pathname, PyObject *data, time_t mtime) +unmarshal_code(PyObject *pathname, PyObject *data, time_t mtime) { PyObject *code; char *buf = PyBytes_AsString(data); @@ -1012,8 +1058,8 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (get_long((unsigned char *)buf) != PyImport_GetMagicNumber()) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", - pathname); + PySys_FormatStderr("# %R has bad magic\n", + pathname); Py_INCREF(Py_None); return Py_None; /* signal caller to try alternative */ } @@ -1021,8 +1067,8 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (mtime != 0 && !eq_mtime(get_long((unsigned char *)buf + 4), mtime)) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", - pathname); + PySys_FormatStderr("# %R has bad mtime\n", + pathname); Py_INCREF(Py_None); return Py_None; /* signal caller to try alternative */ } @@ -1033,7 +1079,7 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (!PyCode_Check(code)) { Py_DECREF(code); PyErr_Format(PyExc_TypeError, - "compiled module %s is not a code object", + "compiled module %R is not a code object", pathname); return NULL; } @@ -1047,11 +1093,12 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) static PyObject * normalize_line_endings(PyObject *source) { - char *buf, *q, *p = PyBytes_AsString(source); + char *buf, *q, *p; PyObject *fixed_source; int len = 0; - if (!p) { + p = PyBytes_AsString(source); + if (p == NULL) { return PyBytes_FromStringAndSize("\n\0", 2); } @@ -1084,16 +1131,24 @@ normalize_line_endings(PyObject *source) /* Given a string buffer containing Python source code, compile it return and return a code object as a new reference. */ static PyObject * -compile_source(char *pathname, PyObject *source) +compile_source(PyObject *pathname, PyObject *source) { - PyObject *code, *fixed_source; + PyObject *code, *fixed_source, *pathbytes; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) + return NULL; fixed_source = normalize_line_endings(source); - if (fixed_source == NULL) + if (fixed_source == NULL) { + Py_DECREF(pathbytes); return NULL; + } - code = Py_CompileString(PyBytes_AsString(fixed_source), pathname, + code = Py_CompileString(PyBytes_AsString(fixed_source), + PyBytes_AsString(pathbytes), Py_file_input); + Py_DECREF(pathbytes); Py_DECREF(fixed_source); return code; } @@ -1122,14 +1177,22 @@ parse_dostime(int dostime, int dosdate) modification time of the matching .py file, or 0 if no source is available. */ static time_t -get_mtime_of_source(ZipImporter *self, char *path) +get_mtime_of_source(ZipImporter *self, PyObject *path) { - PyObject *toc_entry; - time_t mtime = 0; - Py_ssize_t lastchar = strlen(path) - 1; - char savechar = path[lastchar]; - path[lastchar] = '\0'; /* strip 'c' or 'o' from *.py[co] */ - toc_entry = PyDict_GetItemString(self->files, path); + PyObject *toc_entry, *stripped; + time_t mtime; + + /* strip 'c' or 'o' from *.py[co] */ + if (PyUnicode_READY(path) == -1) + return (time_t)-1; + stripped = PyUnicode_FromKindAndData(PyUnicode_KIND(path), + PyUnicode_DATA(path), + PyUnicode_GET_LENGTH(path) - 1); + if (stripped == NULL) + return (time_t)-1; + + toc_entry = PyDict_GetItem(self->files, stripped); + Py_DECREF(stripped); if (toc_entry != NULL && PyTuple_Check(toc_entry) && PyTuple_Size(toc_entry) == 8) { /* fetch the time stamp of the .py file for comparison @@ -1138,8 +1201,8 @@ get_mtime_of_source(ZipImporter *self, char *path) time = PyLong_AsLong(PyTuple_GetItem(toc_entry, 5)); date = PyLong_AsLong(PyTuple_GetItem(toc_entry, 6)); mtime = parse_dostime(time, date); - } - path[lastchar] = savechar; + } else + mtime = 0; return mtime; } @@ -1149,24 +1212,17 @@ static PyObject * get_code_from_data(ZipImporter *self, int ispackage, int isbytecode, time_t mtime, PyObject *toc_entry) { - PyObject *data, *code; - PyObject *modpath; + PyObject *data, *modpath, *code; data = get_data(self->archive, toc_entry); if (data == NULL) return NULL; - modpath = PyUnicode_EncodeFSDefault(PyTuple_GetItem(toc_entry, 0)); - if (modpath == NULL) { - Py_DECREF(data); - return NULL; - } - + modpath = PyTuple_GetItem(toc_entry, 0); if (isbytecode) - code = unmarshal_code(PyBytes_AS_STRING(modpath), data, mtime); + code = unmarshal_code(modpath, data, mtime); else - code = compile_source(PyBytes_AS_STRING(modpath), data); - Py_DECREF(modpath); + code = compile_source(modpath, data); Py_DECREF(data); return code; } @@ -1174,35 +1230,45 @@ get_code_from_data(ZipImporter *self, int ispackage, int isbytecode, /* Get the code object associated with the module specified by 'fullname'. */ static PyObject * -get_module_code(ZipImporter *self, char *fullname, +get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath) { - PyObject *toc_entry; - char *subname, path[MAXPATHLEN + 1]; - int len; + PyObject *code = NULL, *toc_entry, *subname; + PyObject *path, *fullpath = NULL; struct st_zip_searchorder *zso; subname = get_subname(fullname); + if (subname == NULL) + return NULL; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return NULL; for (zso = zip_searchorder; *zso->suffix; zso++) { - PyObject *code = NULL; + code = NULL; + + fullpath = PyUnicode_FromFormat("%U%s", path, zso->suffix); + if (fullpath == NULL) + goto exit; - strcpy(path + len, zso->suffix); if (Py_VerboseFlag > 1) - PySys_FormatStderr("# trying %U%c%s\n", - self->archive, (int)SEP, path); - toc_entry = PyDict_GetItemString(self->files, path); + PySys_FormatStderr("# trying %U%c%U\n", + self->archive, (int)SEP, fullpath); + toc_entry = PyDict_GetItem(self->files, fullpath); if (toc_entry != NULL) { time_t mtime = 0; int ispackage = zso->type & IS_PACKAGE; int isbytecode = zso->type & IS_BYTECODE; - if (isbytecode) - mtime = get_mtime_of_source(self, path); + if (isbytecode) { + mtime = get_mtime_of_source(self, fullpath); + if (mtime == (time_t)-1 && PyErr_Occurred()) { + goto exit; + } + } + Py_CLEAR(fullpath); if (p_ispackage != NULL) *p_ispackage = ispackage; code = get_code_from_data(self, ispackage, @@ -1218,11 +1284,16 @@ get_module_code(ZipImporter *self, char *fullname, *p_modpath = PyTuple_GetItem(toc_entry, 0); Py_INCREF(*p_modpath); } - return code; + goto exit; } + else + Py_CLEAR(fullpath); } - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); - return NULL; + PyErr_Format(ZipImportError, "can't find module %R", fullname); +exit: + Py_DECREF(path); + Py_XDECREF(fullpath); + return code; } diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index a1e605b..a6da056 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -43,6 +43,7 @@ typedef struct z_stream zst; PyObject *unused_data; PyObject *unconsumed_tail; + char eof; int is_initialised; #ifdef WITH_THREAD PyThread_type_lock lock; @@ -95,6 +96,7 @@ newcompobject(PyTypeObject *type) self = PyObject_New(compobject, type); if (self == NULL) return NULL; + self->eof = 0; self->is_initialised = 0; self->unused_data = PyBytes_FromStringAndSize("", 0); if (self->unused_data == NULL) { @@ -297,7 +299,7 @@ PyZlib_decompress(PyObject *self, PyObject *args) err = inflateEnd(&zst); if (err != Z_OK) { - zlib_error(zst, err, "while finishing data decompression"); + zlib_error(zst, err, "while finishing decompression"); goto error; } @@ -482,7 +484,7 @@ PyZlib_objcompress(compobject *self, PyObject *args) */ if (err != Z_OK && err != Z_BUF_ERROR) { - zlib_error(self->zst, err, "while compressing"); + zlib_error(self->zst, err, "while compressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -617,12 +619,13 @@ PyZlib_objdecompress(compobject *self, PyObject *args) Py_DECREF(RetVal); goto error; } + self->eof = 1; /* We will only get Z_BUF_ERROR if the output buffer was full but there wasn't more output when we tried again, so it is not an error condition. */ } else if (err != Z_OK && err != Z_BUF_ERROR) { - zlib_error(self->zst, err, "while decompressing"); + zlib_error(self->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -703,7 +706,7 @@ PyZlib_flush(compobject *self, PyObject *args) if (err == Z_STREAM_END && flushmode == Z_FINISH) { err = deflateEnd(&(self->zst)); if (err != Z_OK) { - zlib_error(self->zst, err, "from deflateEnd()"); + zlib_error(self->zst, err, "while finishing compression"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -771,6 +774,7 @@ PyZlib_copy(compobject *self) Py_XDECREF(retval->unconsumed_tail); retval->unused_data = self->unused_data; retval->unconsumed_tail = self->unconsumed_tail; + retval->eof = self->eof; /* Mark it as being initialized */ retval->is_initialised = 1; @@ -822,6 +826,7 @@ PyZlib_uncopy(compobject *self) Py_XDECREF(retval->unconsumed_tail); retval->unused_data = self->unused_data; retval->unconsumed_tail = self->unconsumed_tail; + retval->eof = self->eof; /* Mark it as being initialized */ retval->is_initialised = 1; @@ -887,14 +892,13 @@ PyZlib_unflush(compobject *self, PyObject *args) Py_END_ALLOW_THREADS } - /* If flushmode is Z_FINISH, we also have to call deflateEnd() to free - various data structures. Note we should only get Z_STREAM_END when - flushmode is Z_FINISH */ + /* If at end of stream, clean up any memory allocated by zlib. */ if (err == Z_STREAM_END) { - err = inflateEnd(&(self->zst)); + self->eof = 1; self->is_initialised = 0; + err = inflateEnd(&(self->zst)); if (err != Z_OK) { - zlib_error(self->zst, err, "from inflateEnd()"); + zlib_error(self->zst, err, "while finishing decompression"); Py_DECREF(retval); retval = NULL; goto error; @@ -942,6 +946,7 @@ static PyMethodDef Decomp_methods[] = static PyMemberDef Decomp_members[] = { {"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY}, {"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY}, + {"eof", T_BOOL, COMP_OFF(eof), READONLY}, {NULL}, }; @@ -1164,6 +1169,10 @@ PyInit_zlib(void) if (ver != NULL) PyModule_AddObject(m, "ZLIB_VERSION", ver); + ver = PyUnicode_FromString(zlibVersion()); + if (ver != NULL) + PyModule_AddObject(m, "ZLIB_RUNTIME_VERSION", ver); + PyModule_AddStringConstant(m, "__version__", "1.0"); return m; |