diff options
Diffstat (limited to 'Modules')
75 files changed, 6986 insertions, 4079 deletions
diff --git a/Modules/Setup.dist b/Modules/Setup.dist index e1c8fe4..2859fa5 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -127,6 +127,9 @@ _io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesi # builtin module avoids some bootstrapping problems and reduces overhead. zipimport zipimport.c +# faulthandler module +faulthandler faulthandler.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If @@ -207,7 +210,7 @@ _symtable symtablemodule.c # # First, look at Setup.config; configure may have set this for you. -#crypt cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems +#_crypt _cryptmodule.c # -lcrypt # crypt(3); needs -lcrypt on some systems # Some more UNIX dependent modules -- off by default, since these diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c new file mode 100644 index 0000000..d329c14 --- /dev/null +++ b/Modules/_bz2module.c @@ -0,0 +1,598 @@ +/* _bz2 - Low-level Python interface to libbzip2. */ + +#define PY_SSIZE_T_CLEAN + +#include "Python.h" +#include "structmember.h" + +#ifdef WITH_THREAD +#include "pythread.h" +#endif + +#include <bzlib.h> +#include <stdio.h> + + +#ifndef BZ_CONFIG_ERROR +#define BZ2_bzCompress bzCompress +#define BZ2_bzCompressInit bzCompressInit +#define BZ2_bzCompressEnd bzCompressEnd +#define BZ2_bzDecompress bzDecompress +#define BZ2_bzDecompressInit bzDecompressInit +#define BZ2_bzDecompressEnd bzDecompressEnd +#endif /* ! BZ_CONFIG_ERROR */ + + +#ifdef WITH_THREAD +#define ACQUIRE_LOCK(obj) do { \ + if (!PyThread_acquire_lock((obj)->lock, 0)) { \ + Py_BEGIN_ALLOW_THREADS \ + PyThread_acquire_lock((obj)->lock, 1); \ + Py_END_ALLOW_THREADS \ + } } while (0) +#define RELEASE_LOCK(obj) PyThread_release_lock((obj)->lock) +#else +#define ACQUIRE_LOCK(obj) +#define RELEASE_LOCK(obj) +#endif + +#define MIN(X, Y) (((X) < (Y)) ? (X) : (Y)) + + +typedef struct { + PyObject_HEAD + bz_stream bzs; + int flushed; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2Compressor; + +typedef struct { + PyObject_HEAD + bz_stream bzs; + char eof; /* T_BOOL expects a char */ + PyObject *unused_data; +#ifdef WITH_THREAD + PyThread_type_lock lock; +#endif +} BZ2Decompressor; + + +/* Helper functions. */ + +static int +catch_bz2_error(int bzerror) +{ + switch(bzerror) { + case BZ_OK: + case BZ_RUN_OK: + case BZ_FLUSH_OK: + case BZ_FINISH_OK: + case BZ_STREAM_END: + return 0; + +#ifdef BZ_CONFIG_ERROR + case BZ_CONFIG_ERROR: + PyErr_SetString(PyExc_SystemError, + "libbzip2 was not compiled correctly"); + return 1; +#endif + case BZ_PARAM_ERROR: + PyErr_SetString(PyExc_ValueError, + "Internal error - " + "invalid parameters passed to libbzip2"); + return 1; + case BZ_MEM_ERROR: + PyErr_NoMemory(); + return 1; + case BZ_DATA_ERROR: + case BZ_DATA_ERROR_MAGIC: + PyErr_SetString(PyExc_IOError, "Invalid data stream"); + return 1; + case BZ_IO_ERROR: + PyErr_SetString(PyExc_IOError, "Unknown I/O error"); + return 1; + case BZ_UNEXPECTED_EOF: + PyErr_SetString(PyExc_EOFError, + "Compressed file ended before the logical " + "end-of-stream was detected"); + return 1; + case BZ_SEQUENCE_ERROR: + PyErr_SetString(PyExc_RuntimeError, + "Internal error - " + "Invalid sequence of commands sent to libbzip2"); + return 1; + default: + PyErr_Format(PyExc_IOError, + "Unrecognized error from libbzip2: %d", bzerror); + return 1; + } +} + +#if BUFSIZ < 8192 +#define SMALLCHUNK 8192 +#else +#define SMALLCHUNK BUFSIZ +#endif + +#if SIZEOF_INT < 4 +#define BIGCHUNK (512 * 32) +#else +#define BIGCHUNK (512 * 1024) +#endif + +static int +grow_buffer(PyObject **buf) +{ + size_t size = PyBytes_GET_SIZE(*buf); + if (size <= SMALLCHUNK) + return _PyBytes_Resize(buf, size + SMALLCHUNK); + else if (size <= BIGCHUNK) + return _PyBytes_Resize(buf, size * 2); + else + return _PyBytes_Resize(buf, size + BIGCHUNK); +} + + +/* BZ2Compressor class. */ + +static PyObject * +compress(BZ2Compressor *c, char *data, size_t len, int action) +{ + size_t data_size = 0; + PyObject *result; + + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); + if (result == NULL) + return NULL; + c->bzs.next_in = data; + /* On a 64-bit system, len might not fit in avail_in (an unsigned int). + Do compression in chunks of no more than UINT_MAX bytes each. */ + c->bzs.avail_in = MIN(len, UINT_MAX); + len -= c->bzs.avail_in; + c->bzs.next_out = PyBytes_AS_STRING(result); + c->bzs.avail_out = PyBytes_GET_SIZE(result); + for (;;) { + char *this_out; + int bzerror; + + Py_BEGIN_ALLOW_THREADS + this_out = c->bzs.next_out; + bzerror = BZ2_bzCompress(&c->bzs, action); + data_size += c->bzs.next_out - this_out; + Py_END_ALLOW_THREADS + if (catch_bz2_error(bzerror)) + goto error; + + if (c->bzs.avail_in == 0 && len > 0) { + c->bzs.avail_in = MIN(len, UINT_MAX); + len -= c->bzs.avail_in; + } + + /* In regular compression mode, stop when input data is exhausted. + In flushing mode, stop when all buffered data has been flushed. */ + if ((action == BZ_RUN && c->bzs.avail_in == 0) || + (action == BZ_FINISH && bzerror == BZ_STREAM_END)) + break; + + if (c->bzs.avail_out == 0) { + if (grow_buffer(&result) < 0) + goto error; + c->bzs.next_out = PyBytes_AS_STRING(result) + data_size; + c->bzs.avail_out = PyBytes_GET_SIZE(result) - data_size; + } + } + if (data_size != PyBytes_GET_SIZE(result)) + if (_PyBytes_Resize(&result, data_size) < 0) + goto error; + return result; + +error: + Py_XDECREF(result); + return NULL; +} + +PyDoc_STRVAR(BZ2Compressor_compress__doc__, +"compress(data) -> bytes\n" +"\n" +"Provide data to the compressor object. Returns a chunk of\n" +"compressed data if possible, or b'' otherwise.\n" +"\n" +"When you have finished providing data to the compressor, call the\n" +"flush() method to finish the compression process.\n"); + +static PyObject * +BZ2Compressor_compress(BZ2Compressor *self, PyObject *args) +{ + Py_buffer buffer; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(args, "y*:compress", &buffer)) + return NULL; + + ACQUIRE_LOCK(self); + if (self->flushed) + PyErr_SetString(PyExc_ValueError, "Compressor has been flushed"); + else + result = compress(self, buffer.buf, buffer.len, BZ_RUN); + RELEASE_LOCK(self); + PyBuffer_Release(&buffer); + return result; +} + +PyDoc_STRVAR(BZ2Compressor_flush__doc__, +"flush() -> bytes\n" +"\n" +"Finish the compression process. Returns the compressed data left\n" +"in internal buffers.\n" +"\n" +"The compressor object may not be used after this method is called.\n"); + +static PyObject * +BZ2Compressor_flush(BZ2Compressor *self, PyObject *noargs) +{ + PyObject *result = NULL; + + ACQUIRE_LOCK(self); + if (self->flushed) + PyErr_SetString(PyExc_ValueError, "Repeated call to flush()"); + else { + self->flushed = 1; + result = compress(self, NULL, 0, BZ_FINISH); + } + RELEASE_LOCK(self); + return result; +} + +static int +BZ2Compressor_init(BZ2Compressor *self, PyObject *args, PyObject *kwargs) +{ + int compresslevel = 9; + int bzerror; + + if (!PyArg_ParseTuple(args, "|i:BZ2Compressor", &compresslevel)) + return -1; + if (!(1 <= compresslevel && compresslevel <= 9)) { + PyErr_SetString(PyExc_ValueError, + "compresslevel must be between 1 and 9"); + return -1; + } + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return -1; + } +#endif + + bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0); + if (catch_bz2_error(bzerror)) + goto error; + + return 0; + +error: +#ifdef WITH_THREAD + PyThread_free_lock(self->lock); + self->lock = NULL; +#endif + return -1; +} + +static void +BZ2Compressor_dealloc(BZ2Compressor *self) +{ + BZ2_bzCompressEnd(&self->bzs); +#ifdef WITH_THREAD + if (self->lock != NULL) + PyThread_free_lock(self->lock); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef BZ2Compressor_methods[] = { + {"compress", (PyCFunction)BZ2Compressor_compress, METH_VARARGS, + BZ2Compressor_compress__doc__}, + {"flush", (PyCFunction)BZ2Compressor_flush, METH_NOARGS, + BZ2Compressor_flush__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Compressor__doc__, +"BZ2Compressor(compresslevel=9)\n" +"\n" +"Create a compressor object for compressing data incrementally.\n" +"\n" +"compresslevel, if given, must be a number between 1 and 9.\n" +"\n" +"For one-shot compression, use the compress() function instead.\n"); + +static PyTypeObject BZ2Compressor_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_bz2.BZ2Compressor", /* tp_name */ + sizeof(BZ2Compressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)BZ2Compressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + BZ2Compressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BZ2Compressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BZ2Compressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* BZ2Decompressor class. */ + +static PyObject * +decompress(BZ2Decompressor *d, char *data, size_t len) +{ + size_t data_size = 0; + PyObject *result; + + result = PyBytes_FromStringAndSize(NULL, SMALLCHUNK); + if (result == NULL) + return result; + d->bzs.next_in = data; + /* On a 64-bit system, len might not fit in avail_in (an unsigned int). + Do decompression in chunks of no more than UINT_MAX bytes each. */ + d->bzs.avail_in = MIN(len, UINT_MAX); + len -= d->bzs.avail_in; + d->bzs.next_out = PyBytes_AS_STRING(result); + d->bzs.avail_out = PyBytes_GET_SIZE(result); + for (;;) { + char *this_out; + int bzerror; + + Py_BEGIN_ALLOW_THREADS + this_out = d->bzs.next_out; + bzerror = BZ2_bzDecompress(&d->bzs); + data_size += d->bzs.next_out - this_out; + Py_END_ALLOW_THREADS + if (catch_bz2_error(bzerror)) + goto error; + if (bzerror == BZ_STREAM_END) { + d->eof = 1; + len += d->bzs.avail_in; + if (len > 0) { /* Save leftover input to unused_data */ + Py_CLEAR(d->unused_data); + d->unused_data = PyBytes_FromStringAndSize(d->bzs.next_in, len); + if (d->unused_data == NULL) + goto error; + } + break; + } + if (d->bzs.avail_in == 0) { + if (len == 0) + break; + d->bzs.avail_in = MIN(len, UINT_MAX); + len -= d->bzs.avail_in; + } + if (d->bzs.avail_out == 0) { + if (grow_buffer(&result) < 0) + goto error; + d->bzs.next_out = PyBytes_AS_STRING(result) + data_size; + d->bzs.avail_out = PyBytes_GET_SIZE(result) - data_size; + } + } + if (data_size != PyBytes_GET_SIZE(result)) + if (_PyBytes_Resize(&result, data_size) < 0) + goto error; + return result; + +error: + Py_XDECREF(result); + return NULL; +} + +PyDoc_STRVAR(BZ2Decompressor_decompress__doc__, +"decompress(data) -> bytes\n" +"\n" +"Provide data to the decompressor object. Returns a chunk of\n" +"decompressed data if possible, or b'' otherwise.\n" +"\n" +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream\n" +"is ignored and saved in the unused_data attribute.\n"); + +static PyObject * +BZ2Decompressor_decompress(BZ2Decompressor *self, PyObject *args) +{ + Py_buffer buffer; + PyObject *result = NULL; + + if (!PyArg_ParseTuple(args, "y*:decompress", &buffer)) + return NULL; + + ACQUIRE_LOCK(self); + if (self->eof) + PyErr_SetString(PyExc_EOFError, "End of stream already reached"); + else + result = decompress(self, buffer.buf, buffer.len); + RELEASE_LOCK(self); + PyBuffer_Release(&buffer); + return result; +} + +static int +BZ2Decompressor_init(BZ2Decompressor *self, PyObject *args, PyObject *kwargs) +{ + int bzerror; + + if (!PyArg_ParseTuple(args, ":BZ2Decompressor")) + return -1; + +#ifdef WITH_THREAD + self->lock = PyThread_allocate_lock(); + if (self->lock == NULL) { + PyErr_SetString(PyExc_MemoryError, "Unable to allocate lock"); + return -1; + } +#endif + + self->unused_data = PyBytes_FromStringAndSize("", 0); + if (self->unused_data == NULL) + goto error; + + bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); + if (catch_bz2_error(bzerror)) + goto error; + + return 0; + +error: + Py_CLEAR(self->unused_data); +#ifdef WITH_THREAD + PyThread_free_lock(self->lock); + self->lock = NULL; +#endif + return -1; +} + +static void +BZ2Decompressor_dealloc(BZ2Decompressor *self) +{ + BZ2_bzDecompressEnd(&self->bzs); + Py_CLEAR(self->unused_data); +#ifdef WITH_THREAD + if (self->lock != NULL) + PyThread_free_lock(self->lock); +#endif + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyMethodDef BZ2Decompressor_methods[] = { + {"decompress", (PyCFunction)BZ2Decompressor_decompress, METH_VARARGS, + BZ2Decompressor_decompress__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Decompressor_eof__doc__, +"True if the end-of-stream marker has been reached."); + +PyDoc_STRVAR(BZ2Decompressor_unused_data__doc__, +"Data found after the end of the compressed stream."); + +static PyMemberDef BZ2Decompressor_members[] = { + {"eof", T_BOOL, offsetof(BZ2Decompressor, eof), + READONLY, BZ2Decompressor_eof__doc__}, + {"unused_data", T_OBJECT_EX, offsetof(BZ2Decompressor, unused_data), + READONLY, BZ2Decompressor_unused_data__doc__}, + {NULL} +}; + +PyDoc_STRVAR(BZ2Decompressor__doc__, +"BZ2Decompressor()\n" +"\n" +"Create a decompressor object for decompressing data incrementally.\n" +"\n" +"For one-shot decompression, use the decompress() function instead.\n"); + +static PyTypeObject BZ2Decompressor_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_bz2.BZ2Decompressor", /* tp_name */ + sizeof(BZ2Decompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)BZ2Decompressor_dealloc,/* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + BZ2Decompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BZ2Decompressor_methods, /* tp_methods */ + BZ2Decompressor_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BZ2Decompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + + +/* Module initialization. */ + +static struct PyModuleDef _bz2module = { + PyModuleDef_HEAD_INIT, + "_bz2", + NULL, + -1, + NULL, + NULL, + NULL, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit__bz2(void) +{ + PyObject *m; + + if (PyType_Ready(&BZ2Compressor_Type) < 0) + return NULL; + if (PyType_Ready(&BZ2Decompressor_Type) < 0) + return NULL; + + m = PyModule_Create(&_bz2module); + if (m == NULL) + return NULL; + + Py_INCREF(&BZ2Compressor_Type); + PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Compressor_Type); + + Py_INCREF(&BZ2Decompressor_Type); + PyModule_AddObject(m, "BZ2Decompressor", + (PyObject *)&BZ2Decompressor_Type); + + return m; +} diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index eb740f9..4bc0482 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -588,7 +588,7 @@ charmap_decode(PyObject *self, return codec_tuple(unicode, pbuf.len); } -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS static PyObject * mbcs_decode(PyObject *self, @@ -613,7 +613,7 @@ mbcs_decode(PyObject *self, return codec_tuple(decoded, consumed); } -#endif /* MS_WINDOWS */ +#endif /* HAVE_MBCS */ /* --- Encoder ------------------------------------------------------------ */ @@ -989,7 +989,7 @@ charmap_build(PyObject *self, PyObject *args) return PyUnicode_BuildEncodingMap(map); } -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS static PyObject * mbcs_encode(PyObject *self, @@ -1014,7 +1014,7 @@ mbcs_encode(PyObject *self, return v; } -#endif /* MS_WINDOWS */ +#endif /* HAVE_MBCS */ /* --- Error handler registry --------------------------------------------- */ @@ -1101,7 +1101,7 @@ static PyMethodDef _codecs_functions[] = { {"charmap_decode", charmap_decode, METH_VARARGS}, {"charmap_build", charmap_build, METH_VARARGS}, {"readbuffer_encode", readbuffer_encode, METH_VARARGS}, -#if defined(MS_WINDOWS) && defined(HAVE_USABLE_WCHAR_T) +#ifdef HAVE_MBCS {"mbcs_encode", mbcs_encode, METH_VARARGS}, {"mbcs_decode", mbcs_decode, METH_VARARGS}, #endif diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index 5545d1e..156ad18 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -832,8 +832,7 @@ deque_richcompare(PyObject *v, PyObject *w, int op) if (!PyObject_TypeCheck(v, &deque_type) || !PyObject_TypeCheck(w, &deque_type)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } /* Shortcuts */ @@ -1552,12 +1551,8 @@ _count_elements(PyObject *self, PyObject *args) if (PyDict_CheckExact(mapping)) { while (1) { key = PyIter_Next(it); - if (key == NULL) { - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - break; - } + if (key == NULL) + break; oldval = PyDict_GetItem(mapping, key); if (oldval == NULL) { if (PyDict_SetItem(mapping, key, one) == -1) @@ -1575,12 +1570,8 @@ _count_elements(PyObject *self, PyObject *args) } else { while (1) { key = PyIter_Next(it); - if (key == NULL) { - if (PyErr_Occurred() && PyErr_ExceptionMatches(PyExc_StopIteration)) - PyErr_Clear(); - else - break; - } + if (key == NULL) + break; oldval = PyObject_GetItem(mapping, key); if (oldval == NULL) { if (!PyErr_Occurred() || !PyErr_ExceptionMatches(PyExc_KeyError)) diff --git a/Modules/cryptmodule.c b/Modules/_cryptmodule.c index d5a42ff..5100788 100644 --- a/Modules/cryptmodule.c +++ b/Modules/_cryptmodule.c @@ -45,7 +45,7 @@ static PyMethodDef crypt_methods[] = { static struct PyModuleDef cryptmodule = { PyModuleDef_HEAD_INIT, - "crypt", + "_crypt", NULL, -1, crypt_methods, @@ -56,7 +56,7 @@ static struct PyModuleDef cryptmodule = { }; PyMODINIT_FUNC -PyInit_crypt(void) +PyInit__crypt(void) { return PyModule_Create(&cryptmodule); } diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8e85980..79c8663 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1844,7 +1844,7 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } if (PyUnicode_Check(proto)) { - PyObject *v = _PyUnicode_AsDefaultEncodedString(proto, NULL); + PyObject *v = _PyUnicode_AsDefaultEncodedString(proto); if (!v) goto error; proto_str = PyBytes_AS_STRING(v); diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index ccaa3c9..0aa9f0b 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -52,7 +52,7 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, { CFieldObject *self; PyObject *proto; - Py_ssize_t size, align, length; + Py_ssize_t size, align; SETFUNC setfunc = NULL; GETFUNC getfunc = NULL; StgDictObject *dict; @@ -106,7 +106,6 @@ PyCField_FromDesc(PyObject *desc, Py_ssize_t index, } size = dict->size; - length = dict->length; proto = desc; /* Field descriptors for 'c_char * n' are be scpecial cased to diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index 092fb69..6d72024 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -105,10 +105,6 @@ char *PyCursesVersion = "2.2"; #include "Python.h" -#ifdef __osf__ -#define STRICT_SYSV_CURSES /* Don't use ncurses extensions */ -#endif - #ifdef __hpux #define STRICT_SYSV_CURSES #endif @@ -910,6 +906,40 @@ PyCursesWindow_GetKey(PyCursesWindowObject *self, PyObject *args) } } +#ifdef HAVE_NCURSESW +static PyObject * +PyCursesWindow_Get_WCh(PyCursesWindowObject *self, PyObject *args) +{ + int x, y; + int ct; + wint_t rtn; + + switch (PyTuple_Size(args)) { + case 0: + Py_BEGIN_ALLOW_THREADS + ct = wget_wch(self->win,&rtn); + Py_END_ALLOW_THREADS + break; + case 2: + if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) + return NULL; + Py_BEGIN_ALLOW_THREADS + ct = mvwget_wch(self->win,y,x,&rtn); + Py_END_ALLOW_THREADS + break; + default: + PyErr_SetString(PyExc_TypeError, "get_wch requires 0 or 2 arguments"); + return NULL; + } + if (ct == ERR) { + /* get_wch() returns ERR in nodelay mode */ + PyErr_SetString(PyCursesError, "no input"); + return NULL; + } + return PyLong_FromLong(rtn); +} +#endif + static PyObject * PyCursesWindow_GetStr(PyCursesWindowObject *self, PyObject *args) { @@ -1608,6 +1638,9 @@ static PyMethodDef PyCursesWindow_Methods[] = { {"getbkgd", (PyCFunction)PyCursesWindow_GetBkgd, METH_NOARGS}, {"getch", (PyCFunction)PyCursesWindow_GetCh, METH_VARARGS}, {"getkey", (PyCFunction)PyCursesWindow_GetKey, METH_VARARGS}, +#ifdef HAVE_NCURSESW + {"get_wch", (PyCFunction)PyCursesWindow_Get_WCh, METH_VARARGS}, +#endif {"getmaxyx", (PyCFunction)PyCursesWindow_getmaxyx, METH_NOARGS}, {"getparyx", (PyCFunction)PyCursesWindow_getparyx, METH_NOARGS}, {"getstr", (PyCFunction)PyCursesWindow_GetStr, METH_VARARGS}, diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 6ee5317..718dfe2 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -766,7 +766,7 @@ typedef struct PyObject *name; } PyDateTime_TimeZone; -/* The interned UTC timezone instance */ +/* The interned UTC timezone instance */ static PyObject *PyDateTime_TimeZone_UTC; /* Create new timezone instance checking offset range. This @@ -1166,31 +1166,6 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, if (!pin) return NULL; - /* Give up if the year is before 1000. - * Python strftime() plays games with the year, and different - * games depending on whether envar PYTHON2K is set. This makes - * years before 1000 a nightmare, even if the platform strftime - * supports them (and not all do). - * We could get a lot farther here by avoiding Python's strftime - * wrapper and calling the C strftime() directly, but that isn't - * an option in the Python implementation of this module. - */ - { - long year; - PyObject *pyyear = PySequence_GetItem(timetuple, 0); - if (pyyear == NULL) return NULL; - assert(PyLong_Check(pyyear)); - year = PyLong_AsLong(pyyear); - Py_DECREF(pyyear); - if (year < 1000) { - PyErr_Format(PyExc_ValueError, "year=%ld is before " - "1000; the datetime strftime() " - "methods require year >= 1000", - year); - return NULL; - } - } - /* Scan the input format, looking for %z/%Z/%f escapes, building * a new format. Since computing the replacements for those codes * is expensive, don't unless they're actually used. @@ -1461,7 +1436,7 @@ delta_to_microseconds(PyDateTime_Delta *self) goto Done; Py_DECREF(x1); Py_DECREF(x2); - x1 = x2 = NULL; + /* x1 = */ x2 = NULL; /* x3 has days+seconds in seconds */ x1 = PyNumber_Multiply(x3, us_per_second); /* us */ @@ -1837,8 +1812,7 @@ delta_richcompare(PyObject *self, PyObject *other, int op) return diff_to_bool(diff, op); } else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } } @@ -1936,10 +1910,8 @@ delta_remainder(PyObject *left, PyObject *right) PyObject *pyus_remainder; PyObject *remainder; - if (!PyDelta_Check(left) || !PyDelta_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyDelta_Check(left) || !PyDelta_Check(right)) + Py_RETURN_NOTIMPLEMENTED; pyus_left = delta_to_microseconds((PyDateTime_Delta *)left); if (pyus_left == NULL) @@ -1974,10 +1946,8 @@ delta_divmod(PyObject *left, PyObject *right) PyObject *delta; PyObject *result; - if (!PyDelta_Check(left) || !PyDelta_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!PyDelta_Check(left) || !PyDelta_Check(right)) + Py_RETURN_NOTIMPLEMENTED; pyus_left = delta_to_microseconds((PyDateTime_Delta *)left); if (pyus_left == NULL) @@ -2571,10 +2541,9 @@ add_date_timedelta(PyDateTime_Date *date, PyDateTime_Delta *delta, int negate) static PyObject * date_add(PyObject *left, PyObject *right) { - if (PyDateTime_Check(left) || PyDateTime_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (PyDateTime_Check(left) || PyDateTime_Check(right)) + Py_RETURN_NOTIMPLEMENTED; + if (PyDate_Check(left)) { /* date + ??? */ if (PyDelta_Check(right)) @@ -2593,17 +2562,15 @@ date_add(PyObject *left, PyObject *right) (PyDateTime_Delta *) left, 0); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * date_subtract(PyObject *left, PyObject *right) { - if (PyDateTime_Check(left) || PyDateTime_Check(right)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (PyDateTime_Check(left) || PyDateTime_Check(right)) + Py_RETURN_NOTIMPLEMENTED; + if (PyDate_Check(left)) { if (PyDate_Check(right)) { /* date - date */ @@ -2622,8 +2589,7 @@ date_subtract(PyObject *left, PyObject *right) 1); } } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } @@ -2740,10 +2706,8 @@ date_richcompare(PyObject *self, PyObject *other, int op) _PyDateTime_DATE_DATASIZE); return diff_to_bool(diff, op); } - else { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + else + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -3240,10 +3204,8 @@ static PyObject * timezone_richcompare(PyDateTime_TimeZone *self, PyDateTime_TimeZone *other, int op) { - if (op != Py_EQ && op != Py_NE) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (op != Py_EQ && op != Py_NE) + Py_RETURN_NOTIMPLEMENTED; return delta_richcompare(self->offset, other->offset, op); } @@ -3288,7 +3250,6 @@ timezone_repr(PyDateTime_TimeZone *self) static PyObject * timezone_str(PyDateTime_TimeZone *self) { - char buf[10]; int hours, minutes, seconds; PyObject *offset; char sign; @@ -3314,11 +3275,9 @@ timezone_str(PyDateTime_TimeZone *self) Py_DECREF(offset); minutes = divmod(seconds, 60, &seconds); hours = divmod(minutes, 60, &minutes); - assert(seconds == 0); /* XXX ignore sub-minute data, curently not allowed. */ - PyOS_snprintf(buf, sizeof(buf), "UTC%c%02d:%02d", sign, hours, minutes); - - return PyUnicode_FromString(buf); + assert(seconds == 0); + return PyUnicode_FromFormat("UTC%c%02d:%02d", sign, hours, minutes); } static PyObject * @@ -3692,10 +3651,8 @@ time_richcompare(PyObject *self, PyObject *other, int op) PyObject *offset1, *offset2; int diff; - if (! PyTime_Check(other)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (! PyTime_Check(other)) + Py_RETURN_NOTIMPLEMENTED; if (GET_TIME_TZINFO(self) == GET_TIME_TZINFO(other)) { diff = memcmp(((PyDateTime_Time *)self)->data, @@ -4384,8 +4341,7 @@ datetime_add(PyObject *left, PyObject *right) (PyDateTime_Delta *) left, 1); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyObject * @@ -4587,8 +4543,7 @@ datetime_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_TRUE; return cmperror(self, other); } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } if (GET_DT_TZINFO(self) == GET_DT_TZINFO(other)) { diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index 827acce..69a7112 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -219,7 +219,7 @@ dbm_contains(PyObject *self, PyObject *arg) return -1; } if (PyUnicode_Check(arg)) { - arg = _PyUnicode_AsDefaultEncodedString(arg, NULL); + arg = _PyUnicode_AsDefaultEncodedString(arg); if (arg == NULL) return -1; } diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 3061d8e..6373c48 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -226,7 +226,7 @@ typedef struct { PyObject* *children; PyObject* _children[STATIC_CHILDREN]; - + } ElementObjectExtra; typedef struct { @@ -354,14 +354,14 @@ element_resize(ElementObject* self, int extra) /* use Python 2.4's list growth strategy */ size = (size >> 3) + (size < 9 ? 3 : 6) + size; /* Coverity CID #182 size_error: Allocating 1 bytes to pointer "children" - * which needs at least 4 bytes. - * Although it's a false alarm always assume at least one child to + * which needs at least 4 bytes. + * Although it's a false alarm always assume at least one child to * be safe. */ size = size ? size : 1; if (self->extra->children != self->extra->_children) { /* Coverity CID #182 size_error: Allocating 1 bytes to pointer - * "children", which needs at least 4 bytes. Although it's a + * "children", which needs at least 4 bytes. Although it's a * false alarm always assume at least one child to be safe. */ children = PyObject_Realloc(self->extra->children, @@ -606,7 +606,7 @@ element_copy(ElementObject* self, PyObject* args) Py_INCREF(JOIN_OBJ(element->tail)); if (self->extra) { - + if (element_resize(element, self->extra->length) < 0) { Py_DECREF(element); return NULL; @@ -618,7 +618,7 @@ element_copy(ElementObject* self, PyObject* args) } element->extra->length = self->extra->length; - + } return (PyObject*) element; @@ -661,7 +661,7 @@ element_deepcopy(ElementObject* self, PyObject* args) if (!element) return NULL; - + text = deepcopy(JOIN_OBJ(self->text), memo); if (!text) goto error; @@ -675,7 +675,7 @@ element_deepcopy(ElementObject* self, PyObject* args) element->tail = JOIN_SET(tail, JOIN_GET(self->tail)); if (self->extra) { - + if (element_resize(element, self->extra->length) < 0) goto error; @@ -689,7 +689,7 @@ element_deepcopy(ElementObject* self, PyObject* args) } element->extra->length = self->extra->length; - + } /* add object to memo dictionary (so deepcopy won't visit it again) */ @@ -800,7 +800,7 @@ element_find(ElementObject* self, PyObject* args) if (!self->extra) Py_RETURN_NONE; - + for (i = 0; i < self->extra->length; i++) { PyObject* item = self->extra->children[i]; if (Element_CheckExact(item) && @@ -953,7 +953,7 @@ static PyObject* element_iter(ElementObject* self, PyObject* args) { PyObject* result; - + PyObject* tag = Py_None; if (!PyArg_ParseTuple(args, "|O:iter", &tag)) return NULL; @@ -985,7 +985,7 @@ static PyObject* element_itertext(ElementObject* self, PyObject* args) { PyObject* result; - + if (!PyArg_ParseTuple(args, ":itertext")) return NULL; @@ -1483,7 +1483,7 @@ element_getattro(ElementObject* self, PyObject* nameobj) if (PyUnicode_Check(nameobj)) name = _PyUnicode_AsString(nameobj); - + if (name == NULL) return NULL; @@ -2113,7 +2113,7 @@ makeuniversal(XMLParserObject* self, const char* string) Py_INCREF(key); tag = key; } - + /* decode universal name */ p = PyBytes_AS_STRING(tag); value = PyUnicode_DecodeUTF8(p, size, "strict"); @@ -2138,13 +2138,15 @@ makeuniversal(XMLParserObject* self, const char* string) static void expat_set_error(const char* message, int line, int column) { - PyObject *error; - PyObject *position; - char buffer[256]; + PyObject *errmsg, *error, *position; - sprintf(buffer, "%.100s: line %d, column %d", message, line, column); + errmsg = PyUnicode_FromFormat("%s: line %d, column %d", + message, line, column); + if (errmsg == NULL) + return; - error = PyObject_CallFunction(elementtree_parseerror_obj, "s", buffer); + error = PyObject_CallFunction(elementtree_parseerror_obj, "O", errmsg); + Py_DECREF(errmsg); if (!error) return; @@ -2407,7 +2409,7 @@ expat_unknown_encoding_handler(XMLParserObject *self, const XML_Char *name, for (i = 0; i < 256; i++) s[i] = i; - + u = PyUnicode_Decode((char*) s, 256, name, "replace"); if (!u) return XML_STATUS_ERROR; @@ -2466,7 +2468,7 @@ xmlparser(PyObject* self_, PyObject* args, PyObject* kw) PyObject_Del(self); return NULL; } - + self->names = PyDict_New(); if (!self->names) { PyObject_Del(self->entity); @@ -2645,7 +2647,7 @@ xmlparser_parse(XMLParserObject* self, PyObject* args) reader = PyObject_GetAttrString(fileobj, "read"); if (!reader) return NULL; - + /* read from open file object */ for (;;) { @@ -2796,7 +2798,7 @@ static PyMethodDef xmlparser_methods[] = { {NULL, NULL} }; -static PyObject* +static PyObject* xmlparser_getattro(XMLParserObject* self, PyObject* nameobj) { if (PyUnicode_Check(nameobj)) { @@ -2957,7 +2959,7 @@ PyInit__elementtree(void) " break\n" " parser.feed(data)\n" " self._root = parser.close()\n" - " else:\n" + " else:\n" " parser = cElementTree.XMLParser()\n" " self._root = parser._parse(source)\n" " return self._root\n" diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d8a283b..0882d36 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -330,6 +330,176 @@ static PyTypeObject partial_type = { }; +/* cmp_to_key ***************************************************************/ + +typedef struct { + PyObject_HEAD + PyObject *cmp; + PyObject *object; +} keyobject; + +static void +keyobject_dealloc(keyobject *ko) +{ + Py_DECREF(ko->cmp); + Py_XDECREF(ko->object); + PyObject_FREE(ko); +} + +static int +keyobject_traverse(keyobject *ko, visitproc visit, void *arg) +{ + Py_VISIT(ko->cmp); + if (ko->object) + Py_VISIT(ko->object); + return 0; +} + +static int +keyobject_clear(keyobject *ko) +{ + Py_CLEAR(ko->cmp); + if (ko->object) + Py_CLEAR(ko->object); + return 0; +} + +static PyMemberDef keyobject_members[] = { + {"obj", T_OBJECT, + offsetof(keyobject, object), 0, + PyDoc_STR("Value wrapped by a key function.")}, + {NULL} +}; + +static PyObject * +keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds); + +static PyObject * +keyobject_richcompare(PyObject *ko, PyObject *other, int op); + +static PyTypeObject keyobject_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "functools.KeyWrapper", /* tp_name */ + sizeof(keyobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)keyobject_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + (ternaryfunc)keyobject_call, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + 0, /* tp_doc */ + (traverseproc)keyobject_traverse, /* tp_traverse */ + (inquiry)keyobject_clear, /* tp_clear */ + keyobject_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + keyobject_members, /* tp_members */ + 0, /* tp_getset */ +}; + +static PyObject * +keyobject_call(keyobject *ko, PyObject *args, PyObject *kwds) +{ + PyObject *object; + keyobject *result; + static char *kwargs[] = {"obj", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:K", kwargs, &object)) + return NULL; + result = PyObject_New(keyobject, &keyobject_type); + if (!result) + return NULL; + Py_INCREF(ko->cmp); + result->cmp = ko->cmp; + Py_INCREF(object); + result->object = object; + return (PyObject *)result; +} + +static PyObject * +keyobject_richcompare(PyObject *ko, PyObject *other, int op) +{ + PyObject *res; + PyObject *args; + PyObject *x; + PyObject *y; + PyObject *compare; + PyObject *answer; + static PyObject *zero; + + if (zero == NULL) { + zero = PyLong_FromLong(0); + if (!zero) + return NULL; + } + + if (Py_TYPE(other) != &keyobject_type){ + PyErr_Format(PyExc_TypeError, "other argument must be K instance"); + return NULL; + } + compare = ((keyobject *) ko)->cmp; + assert(compare != NULL); + x = ((keyobject *) ko)->object; + y = ((keyobject *) other)->object; + if (!x || !y){ + PyErr_Format(PyExc_AttributeError, "object"); + return NULL; + } + + /* Call the user's comparison function and translate the 3-way + * result into true or false (or error). + */ + args = PyTuple_New(2); + if (args == NULL) + return NULL; + Py_INCREF(x); + Py_INCREF(y); + PyTuple_SET_ITEM(args, 0, x); + PyTuple_SET_ITEM(args, 1, y); + res = PyObject_Call(compare, args, NULL); + Py_DECREF(args); + if (res == NULL) + return NULL; + answer = PyObject_RichCompare(res, zero, op); + Py_DECREF(res); + return answer; +} + +static PyObject * +functools_cmp_to_key(PyObject *self, PyObject *args, PyObject *kwds) +{ + PyObject *cmp; + static char *kwargs[] = {"mycmp", NULL}; + keyobject *object; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:cmp_to_key", kwargs, &cmp)) + return NULL; + object = PyObject_New(keyobject, &keyobject_type); + if (!object) + return NULL; + Py_INCREF(cmp); + object->cmp = cmp; + object->object = NULL; + return (PyObject *)object; +} + +PyDoc_STRVAR(functools_cmp_to_key_doc, +"Convert a cmp= function into a key= function."); + /* reduce (used to be a builtin) ********************************************/ static PyObject * @@ -413,6 +583,8 @@ PyDoc_STRVAR(module_doc, static PyMethodDef module_methods[] = { {"reduce", functools_reduce, METH_VARARGS, functools_reduce_doc}, + {"cmp_to_key", (PyCFunction)functools_cmp_to_key, + METH_VARARGS | METH_KEYWORDS, functools_cmp_to_key_doc}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 44bdac6..6f5bd48 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -36,6 +36,7 @@ PyObject *_PyIO_str_nl; PyObject *_PyIO_str_read; PyObject *_PyIO_str_read1; PyObject *_PyIO_str_readable; +PyObject *_PyIO_str_readall; PyObject *_PyIO_str_readinto; PyObject *_PyIO_str_readline; PyObject *_PyIO_str_reset; @@ -767,6 +768,8 @@ PyInit__io(void) goto fail; if (!(_PyIO_str_readable = PyUnicode_InternFromString("readable"))) goto fail; + if (!(_PyIO_str_readall = PyUnicode_InternFromString("readall"))) + goto fail; if (!(_PyIO_str_readinto = PyUnicode_InternFromString("readinto"))) goto fail; if (!(_PyIO_str_readline = PyUnicode_InternFromString("readline"))) diff --git a/Modules/_io/_iomodule.h b/Modules/_io/_iomodule.h index 925e4f2..9174bdd 100644 --- a/Modules/_io/_iomodule.h +++ b/Modules/_io/_iomodule.h @@ -155,6 +155,7 @@ extern PyObject *_PyIO_str_nl; extern PyObject *_PyIO_str_read; extern PyObject *_PyIO_str_read1; extern PyObject *_PyIO_str_readable; +extern PyObject *_PyIO_str_readall; extern PyObject *_PyIO_str_readinto; extern PyObject *_PyIO_str_readline; extern PyObject *_PyIO_str_reset; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index d6f0c9c..a8631e0 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -1,9 +1,9 @@ /* An implementation of Buffered I/O as defined by PEP 3116 - "New I/O" - + Classes defined here: BufferedIOBase, BufferedReader, BufferedWriter, BufferedRandom. - + Written by Amaury Forgeot d'Arc and Antoine Pitrou */ @@ -198,7 +198,7 @@ typedef struct { int readable; int writable; int deallocating; - + /* True if this is a vanilla Buffered object (rather than a user derived class) *and* the raw stream is a vanilla FileIO object. */ int fast_closed_checks; @@ -237,7 +237,7 @@ typedef struct { /* Implementation notes: - + * BufferedReader, BufferedWriter and BufferedRandom try to share most methods (this is helped by the members `readable` and `writable`, which are initialized in the respective constructors) @@ -255,7 +255,7 @@ typedef struct { NOTE: we should try to maintain block alignment of reads and writes to the raw stream (according to the buffer size), but for now it is only done in read() and friends. - + */ /* These macros protect the buffered object against concurrent operations. */ @@ -589,14 +589,15 @@ _bufferedreader_reset_buf(buffered *self); static void _bufferedwriter_reset_buf(buffered *self); static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t); +_bufferedreader_peek_unlocked(buffered *self); static PyObject * _bufferedreader_read_all(buffered *self); static PyObject * _bufferedreader_read_fast(buffered *self, Py_ssize_t); static PyObject * _bufferedreader_read_generic(buffered *self, Py_ssize_t); - +static Py_ssize_t +_bufferedreader_raw_read(buffered *self, char *start, Py_ssize_t len); /* * Helpers @@ -635,7 +636,7 @@ _buffered_raw_tell(buffered *self) if (!PyErr_Occurred()) PyErr_Format(PyExc_IOError, "Raw stream returned invalid position %" PY_PRIdOFF, - (PY_OFF_T_COMPAT)n); + (PY_OFF_T_COMPAT)n); return -1; } self->abs_pos = n; @@ -668,7 +669,7 @@ _buffered_raw_seek(buffered *self, Py_off_t target, int whence) if (!PyErr_Occurred()) PyErr_Format(PyExc_IOError, "Raw stream returned invalid position %" PY_PRIdOFF, - (PY_OFF_T_COMPAT)n); + (PY_OFF_T_COMPAT)n); return -1; } self->abs_pos = n; @@ -809,7 +810,7 @@ buffered_peek(buffered *self, PyObject *args) goto end; Py_CLEAR(res); } - res = _bufferedreader_peek_unlocked(self, n); + res = _bufferedreader_peek_unlocked(self); end: LEAVE_BUFFERED(self) @@ -875,7 +876,7 @@ buffered_read1(buffered *self, PyObject *args) if (!ENTER_BUFFERED(self)) return NULL; - + /* Return up to n bytes. If at least one byte is buffered, we only return buffered bytes. Otherwise, we do one raw read. */ @@ -924,10 +925,78 @@ end: static PyObject * buffered_readinto(buffered *self, PyObject *args) { + Py_buffer buf; + Py_ssize_t n, written = 0, remaining; + PyObject *res = NULL; + CHECK_INITIALIZED(self) - - /* TODO: use raw.readinto() (or a direct copy from our buffer) instead! */ - return bufferediobase_readinto((PyObject *)self, args); + + if (!PyArg_ParseTuple(args, "w*:readinto", &buf)) + return NULL; + + n = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); + if (n > 0) { + if (n >= buf.len) { + memcpy(buf.buf, self->buffer + self->pos, buf.len); + self->pos += buf.len; + res = PyLong_FromSsize_t(buf.len); + goto end_unlocked; + } + memcpy(buf.buf, self->buffer + self->pos, n); + self->pos += n; + written = n; + } + + if (!ENTER_BUFFERED(self)) + goto end_unlocked; + + if (self->writable) { + res = buffered_flush_and_rewind_unlocked(self); + if (res == NULL) + goto end; + Py_CLEAR(res); + } + + _bufferedreader_reset_buf(self); + self->pos = 0; + + for (remaining = buf.len - written; + remaining > 0; + written += n, remaining -= n) { + /* If remaining bytes is larger than internal buffer size, copy + * directly into caller's buffer. */ + if (remaining > self->buffer_size) { + n = _bufferedreader_raw_read(self, (char *) buf.buf + written, + remaining); + } + else { + n = _bufferedreader_fill_buffer(self); + if (n > 0) { + if (n > remaining) + n = remaining; + memcpy((char *) buf.buf + written, + self->buffer + self->pos, n); + self->pos += n; + continue; /* short circuit */ + } + } + if (n == 0 || (n == -2 && written > 0)) + break; + if (n < 0) { + if (n == -2) { + Py_INCREF(Py_None); + res = Py_None; + } + goto end; + } + } + res = PyLong_FromSsize_t(written); + +end: + LEAVE_BUFFERED(self); +end_unlocked: + PyBuffer_Release(&buf); + return res; } static PyObject * @@ -1342,33 +1411,58 @@ static PyObject * _bufferedreader_read_all(buffered *self) { Py_ssize_t current_size; - PyObject *res, *data = NULL; - PyObject *chunks = PyList_New(0); - - if (chunks == NULL) - return NULL; + PyObject *res, *data = NULL, *chunk, *chunks; /* First copy what we have in the current buffer. */ current_size = Py_SAFE_DOWNCAST(READAHEAD(self), Py_off_t, Py_ssize_t); if (current_size) { data = PyBytes_FromStringAndSize( self->buffer + self->pos, current_size); - if (data == NULL) { - Py_DECREF(chunks); + if (data == NULL) return NULL; - } self->pos += current_size; } /* We're going past the buffer's bounds, flush it */ if (self->writable) { res = buffered_flush_and_rewind_unlocked(self); - if (res == NULL) { - Py_DECREF(chunks); + if (res == NULL) return NULL; - } Py_CLEAR(res); } _bufferedreader_reset_buf(self); + + if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) { + chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL); + if (chunk == NULL) + return NULL; + if (chunk != Py_None && !PyBytes_Check(chunk)) { + Py_XDECREF(data); + Py_DECREF(chunk); + PyErr_SetString(PyExc_TypeError, "readall() should return bytes"); + return NULL; + } + if (chunk == Py_None) { + if (current_size == 0) + return chunk; + else { + Py_DECREF(chunk); + return data; + } + } + else if (current_size) { + PyBytes_Concat(&data, chunk); + Py_DECREF(chunk); + if (data == NULL) + return NULL; + return data; + } else + return chunk; + } + + chunks = PyList_New(0); + if (chunks == NULL) + return NULL; + while (1) { if (data) { if (PyList_Append(chunks, data) < 0) { @@ -1530,7 +1624,7 @@ error: } static PyObject * -_bufferedreader_peek_unlocked(buffered *self, Py_ssize_t n) +_bufferedreader_peek_unlocked(buffered *self) { Py_ssize_t have, r; @@ -1572,6 +1666,7 @@ static PyMethodDef bufferedreader_methods[] = { {"read", (PyCFunction)buffered_read, METH_VARARGS}, {"peek", (PyCFunction)buffered_peek, METH_VARARGS}, {"read1", (PyCFunction)buffered_read1, METH_VARARGS}, + {"readinto", (PyCFunction)buffered_readinto, METH_VARARGS}, {"readline", (PyCFunction)buffered_readline, METH_VARARGS}, {"seek", (PyCFunction)buffered_seek, METH_VARARGS}, {"tell", (PyCFunction)buffered_tell, METH_NOARGS}, diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index b40513f..65ec931 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -938,13 +938,11 @@ static int bytesiobuf_getbuffer(bytesiobuf *obj, Py_buffer *view, int flags) { int ret; - void *ptr; bytesio *b = (bytesio *) obj->source; if (view == NULL) { b->exports++; return 0; } - ptr = (void *) obj; ret = PyBuffer_FillInfo(view, (PyObject*)obj, b->buf, b->string_size, 0, flags); if (ret >= 0) { diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index b1d492b..3de1ff5 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -547,14 +547,14 @@ fileio_readinto(fileio *self, PyObject *args) } static size_t -new_buffersize(fileio *self, size_t currentsize) +new_buffersize(fileio *self, size_t currentsize +#ifdef HAVE_FSTAT + , off_t pos, off_t end +#endif + ) { #ifdef HAVE_FSTAT - off_t pos, end; - struct stat st; - if (fstat(self->fd, &st) == 0) { - end = st.st_size; - pos = lseek(self->fd, 0L, SEEK_CUR); + if (end != (off_t)-1) { /* Files claiming a size smaller than SMALLCHUNK may actually be streaming pseudo-files. In this case, we apply the more aggressive algorithm below. @@ -579,9 +579,14 @@ new_buffersize(fileio *self, size_t currentsize) static PyObject * fileio_readall(fileio *self) { +#ifdef HAVE_FSTAT + struct stat st; + off_t pos, end; +#endif PyObject *result; Py_ssize_t total = 0; int n; + size_t newsize; if (self->fd < 0) return err_closed(); @@ -592,8 +597,23 @@ fileio_readall(fileio *self) if (result == NULL) return NULL; +#ifdef HAVE_FSTAT +#if defined(MS_WIN64) || defined(MS_WINDOWS) + pos = _lseeki64(self->fd, 0L, SEEK_CUR); +#else + pos = lseek(self->fd, 0L, SEEK_CUR); +#endif + if (fstat(self->fd, &st) == 0) + end = st.st_size; + else + end = (off_t)-1; +#endif while (1) { - size_t newsize = new_buffersize(self, total); +#ifdef HAVE_FSTAT + newsize = new_buffersize(self, total, pos, end); +#else + newsize = new_buffersize(self, total); +#endif if (newsize > PY_SSIZE_T_MAX || newsize <= 0) { PyErr_SetString(PyExc_OverflowError, "unbounded read returned more bytes " @@ -632,6 +652,9 @@ fileio_readall(fileio *self) return NULL; } total += n; +#ifdef HAVE_FSTAT + pos += n; +#endif } if (PyBytes_GET_SIZE(result) > total) { diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 9c5f441..13d4bd9 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -680,12 +680,16 @@ typedef struct PyObject *pending_bytes; /* list of bytes objects waiting to be written, or NULL */ Py_ssize_t pending_bytes_count; - PyObject *snapshot; + /* snapshot is either None, or a tuple (dec_flags, next_input) where * dec_flags is the second (integer) item of the decoder state and * next_input is the chunk of input bytes that comes next after the * snapshot point. We use this to reconstruct decoder states in tell(). */ + PyObject *snapshot; + /* Bytes-to-characters ratio for the current chunk. Serves as input for + the heuristic in tell(). */ + double b2cratio; /* Cache raw object if it's a FileIO object */ PyObject *raw; @@ -852,6 +856,7 @@ textiowrapper_init(textio *self, PyObject *args, PyObject *kwds) self->decoded_chars_used = 0; self->pending_bytes_count = 0; self->encodefunc = NULL; + self->b2cratio = 0.0; if (encoding == NULL) { /* Try os.device_encoding(fileno) */ @@ -1397,6 +1402,7 @@ textiowrapper_read_chunk(textio *self) PyObject *dec_flags = NULL; PyObject *input_chunk = NULL; PyObject *decoded_chars, *chunk_size; + Py_ssize_t nbytes, nchars; int eof; /* The return value is True unless EOF was reached. The decoded string is @@ -1443,7 +1449,8 @@ textiowrapper_read_chunk(textio *self) goto fail; assert(PyBytes_Check(input_chunk)); - eof = (PyBytes_Size(input_chunk) == 0); + nbytes = PyBytes_Size(input_chunk); + eof = (nbytes == 0); if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) { decoded_chars = _PyIncrementalNewlineDecoder_decode( @@ -1458,7 +1465,12 @@ textiowrapper_read_chunk(textio *self) if (decoded_chars == NULL) goto fail; textiowrapper_set_decoded_chars(self, decoded_chars); - if (PyUnicode_GET_SIZE(decoded_chars) > 0) + nchars = PyUnicode_GET_SIZE(decoded_chars); + if (nchars > 0) + self->b2cratio = (double) nbytes / nchars; + else + self->b2cratio = 0.0; + if (nchars > 0) eof = 0; if (self->telling) { @@ -1509,8 +1521,13 @@ textiowrapper_read(textio *self, PyObject *args) PyObject *decoded; if (bytes == NULL) goto fail; - decoded = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_decode, - bytes, Py_True, NULL); + + if (Py_TYPE(self->decoder) == &PyIncrementalNewlineDecoder_Type) + decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, + bytes, 1); + else + decoded = PyObject_CallMethodObjArgs( + self->decoder, _PyIO_str_decode, bytes, Py_True, NULL); Py_DECREF(bytes); if (decoded == NULL) goto fail; @@ -2147,8 +2164,12 @@ textiowrapper_tell(textio *self, PyObject *args) cookie_type cookie = {0,0,0,0,0}; PyObject *next_input; Py_ssize_t chars_to_skip, chars_decoded; + Py_ssize_t skip_bytes, skip_back; PyObject *saved_state = NULL; char *input, *input_end; + char *dec_buffer; + Py_ssize_t dec_buffer_len; + int dec_flags; CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -2184,6 +2205,7 @@ textiowrapper_tell(textio *self, PyObject *args) #else cookie.start_pos = PyLong_AsLong(posobj); #endif + Py_DECREF(posobj); if (PyErr_Occurred()) goto fail; @@ -2198,57 +2220,99 @@ textiowrapper_tell(textio *self, PyObject *args) /* How many decoded characters have been used up since the snapshot? */ if (self->decoded_chars_used == 0) { /* We haven't moved from the snapshot point. */ - Py_DECREF(posobj); return textiowrapper_build_cookie(&cookie); } chars_to_skip = self->decoded_chars_used; - /* Starting from the snapshot position, we will walk the decoder - * forward until it gives us enough decoded characters. - */ + /* Decoder state will be restored at the end */ saved_state = PyObject_CallMethodObjArgs(self->decoder, _PyIO_str_getstate, NULL); if (saved_state == NULL) goto fail; - /* Note our initial start point. */ - if (_textiowrapper_decoder_setstate(self, &cookie) < 0) - goto fail; +#define DECODER_GETSTATE() do { \ + PyObject *_state = PyObject_CallMethodObjArgs(self->decoder, \ + _PyIO_str_getstate, NULL); \ + if (_state == NULL) \ + goto fail; \ + if (!PyArg_Parse(_state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { \ + Py_DECREF(_state); \ + goto fail; \ + } \ + Py_DECREF(_state); \ + } while (0) + + /* TODO: replace assert with exception */ +#define DECODER_DECODE(start, len, res) do { \ + PyObject *_decoded = PyObject_CallMethod( \ + self->decoder, "decode", "y#", start, len); \ + if (_decoded == NULL) \ + goto fail; \ + assert (PyUnicode_Check(_decoded)); \ + res = PyUnicode_GET_SIZE(_decoded); \ + Py_DECREF(_decoded); \ + } while (0) + + /* Fast search for an acceptable start point, close to our + current pos */ + skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); + skip_back = 1; + assert(skip_back <= PyBytes_GET_SIZE(next_input)); + input = PyBytes_AS_STRING(next_input); + while (skip_bytes > 0) { + /* Decode up to temptative start point */ + if (_textiowrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + DECODER_DECODE(input, skip_bytes, chars_decoded); + if (chars_decoded <= chars_to_skip) { + DECODER_GETSTATE(); + if (dec_buffer_len == 0) { + /* Before pos and no bytes buffered in decoder => OK */ + cookie.dec_flags = dec_flags; + chars_to_skip -= chars_decoded; + break; + } + /* Skip back by buffered amount and reset heuristic */ + skip_bytes -= dec_buffer_len; + skip_back = 1; + } + else { + /* We're too far ahead, skip back a bit */ + skip_bytes -= skip_back; + skip_back *= 2; + } + } + if (skip_bytes <= 0) { + skip_bytes = 0; + if (_textiowrapper_decoder_setstate(self, &cookie) < 0) + goto fail; + } - /* Feed the decoder one byte at a time. As we go, note the - * nearest "safe start point" before the current location - * (a point where the decoder has nothing buffered, so seek() + /* Note our initial start point. */ + cookie.start_pos += skip_bytes; + cookie.chars_to_skip = chars_to_skip; + if (chars_to_skip == 0) + goto finally; + + /* We should be close to the desired position. Now feed the decoder one + * byte at a time until we reach the `chars_to_skip` target. + * As we go, note the nearest "safe start point" before the current + * location (a point where the decoder has nothing buffered, so seek() * can safely start from there and advance to this location). */ chars_decoded = 0; input = PyBytes_AS_STRING(next_input); input_end = input + PyBytes_GET_SIZE(next_input); + input += skip_bytes; while (input < input_end) { - PyObject *state; - char *dec_buffer; - Py_ssize_t dec_buffer_len; - int dec_flags; - - PyObject *decoded = PyObject_CallMethod( - self->decoder, "decode", "y#", input, 1); - if (decoded == NULL) - goto fail; - assert (PyUnicode_Check(decoded)); - chars_decoded += PyUnicode_GET_SIZE(decoded); - Py_DECREF(decoded); + Py_ssize_t n; + DECODER_DECODE(input, 1, n); + /* We got n chars for 1 byte */ + chars_decoded += n; cookie.bytes_to_feed += 1; - - state = PyObject_CallMethodObjArgs(self->decoder, - _PyIO_str_getstate, NULL); - if (state == NULL) - goto fail; - if (!PyArg_Parse(state, "(y#i)", &dec_buffer, &dec_buffer_len, &dec_flags)) { - Py_DECREF(state); - goto fail; - } - Py_DECREF(state); + DECODER_GETSTATE(); if (dec_buffer_len == 0 && chars_decoded <= chars_to_skip) { /* Decoder buffer is empty, so this is a safe start point. */ @@ -2280,8 +2344,7 @@ textiowrapper_tell(textio *self, PyObject *args) } } - /* finally */ - Py_XDECREF(posobj); +finally: res = PyObject_CallMethod(self->decoder, "setstate", "(O)", saved_state); Py_DECREF(saved_state); if (res == NULL) @@ -2292,8 +2355,7 @@ textiowrapper_tell(textio *self, PyObject *args) cookie.chars_to_skip = Py_SAFE_DOWNCAST(chars_to_skip, Py_ssize_t, int); return textiowrapper_build_cookie(&cookie); - fail: - Py_XDECREF(posobj); +fail: if (saved_state) { PyObject *type, *value, *traceback; PyErr_Fetch(&type, &value, &traceback); diff --git a/Modules/_json.c b/Modules/_json.c index 0924873..d5120fa 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -75,6 +75,122 @@ static PyMemberDef encoder_members[] = { {NULL} }; +/* + * A two-level accumulator of unicode objects that avoids both the overhead + * of keeping a huge number of small separate objects, and the quadratic + * behaviour of using a naive repeated concatenation scheme. + */ + +typedef struct { + PyObject *large; /* A list of previously accumulated large strings */ + PyObject *small; /* Pending small strings */ +} accumulator; + +static PyObject * +join_list_unicode(PyObject *lst) +{ + /* return u''.join(lst) */ + static PyObject *sep = NULL; + if (sep == NULL) { + sep = PyUnicode_FromStringAndSize("", 0); + if (sep == NULL) + return NULL; + } + return PyUnicode_Join(sep, lst); +} + +static int +init_accumulator(accumulator *acc) +{ + acc->large = PyList_New(0); + if (acc->large == NULL) + return -1; + acc->small = PyList_New(0); + if (acc->small == NULL) { + Py_CLEAR(acc->large); + return -1; + } + return 0; +} + +static int +flush_accumulator(accumulator *acc) +{ + Py_ssize_t nsmall = PyList_GET_SIZE(acc->small); + if (nsmall) { + int ret; + PyObject *joined = join_list_unicode(acc->small); + if (joined == NULL) + return -1; + if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) { + Py_DECREF(joined); + return -1; + } + ret = PyList_Append(acc->large, joined); + Py_DECREF(joined); + return ret; + } + return 0; +} + +static int +accumulate_unicode(accumulator *acc, PyObject *obj) +{ + int ret; + Py_ssize_t nsmall; + PyObject *joined; + assert(PyUnicode_Check(obj)); + + if (PyList_Append(acc->small, obj)) + return -1; + nsmall = PyList_GET_SIZE(acc->small); + /* Each item in a list of unicode objects has an overhead (in 64-bit + * builds) of: + * - 8 bytes for the list slot + * - 56 bytes for the header of the unicode object + * that is, 64 bytes. 100000 such objects waste more than 6MB + * compared to a single concatenated string. + */ + if (nsmall < 100000) + return 0; + joined = join_list_unicode(acc->small); + if (joined == NULL) + return -1; + if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) { + Py_DECREF(joined); + return -1; + } + ret = PyList_Append(acc->large, joined); + Py_DECREF(joined); + return ret; +} + +static PyObject * +finish_accumulator(accumulator *acc) +{ + int ret; + PyObject *res; + + ret = flush_accumulator(acc); + Py_CLEAR(acc->small); + if (ret) { + Py_CLEAR(acc->large); + return NULL; + } + res = acc->large; + acc->large = NULL; + return res; +} + +static void +destroy_accumulator(accumulator *acc) +{ + Py_CLEAR(acc->small); + Py_CLEAR(acc->large); +} + +/* Forward decls */ + static PyObject * ascii_escape_unicode(PyObject *pystr); static PyObject * @@ -101,11 +217,11 @@ encoder_dealloc(PyObject *self); static int encoder_clear(PyObject *self); static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level); +encoder_listencode_list(PyEncoderObject *s, accumulator *acc, PyObject *seq, Py_ssize_t indent_level); static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level); +encoder_listencode_obj(PyEncoderObject *s, accumulator *acc, PyObject *obj, Py_ssize_t indent_level); static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level); +encoder_listencode_dict(PyEncoderObject *s, accumulator *acc, PyObject *dct, Py_ssize_t indent_level); static PyObject * _encoded_const(PyObject *obj); static void @@ -267,19 +383,6 @@ raise_errmsg(char *msg, PyObject *s, Py_ssize_t end) } static PyObject * -join_list_unicode(PyObject *lst) -{ - /* return u''.join(lst) */ - static PyObject *sep = NULL; - if (sep == NULL) { - sep = PyUnicode_FromStringAndSize("", 0); - if (sep == NULL) - return NULL; - } - return PyUnicode_Join(sep, lst); -} - -static PyObject * _build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) { /* return (rval, idx) tuple, stealing reference to rval */ PyObject *tpl; @@ -335,7 +438,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next PyObject *rval = NULL; Py_ssize_t len = PyUnicode_GET_SIZE(pystr); Py_ssize_t begin = end - 1; - Py_ssize_t next = begin; + Py_ssize_t next /* = begin */; const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr); PyObject *chunks = NULL; PyObject *chunk = NULL; @@ -842,7 +945,8 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ Py_ssize_t idx = start; int is_float = 0; PyObject *rval; - PyObject *numstr; + PyObject *numstr = NULL; + PyObject *custom_func; /* read a sign if it's there, make sure it's not the end of the string */ if (str[idx] == '-') { @@ -895,22 +999,37 @@ _match_number_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t start, Py_ } } - /* copy the section we determined to be a number */ - numstr = PyUnicode_FromUnicode(&str[start], idx - start); - if (numstr == NULL) - return NULL; - if (is_float) { - /* parse as a float using a fast path if available, otherwise call user defined method */ - if (s->parse_float != (PyObject *)&PyFloat_Type) { - rval = PyObject_CallFunctionObjArgs(s->parse_float, numstr, NULL); - } - else { - rval = PyFloat_FromString(numstr); - } + if (is_float && s->parse_float != (PyObject *)&PyFloat_Type) + custom_func = s->parse_float; + else if (!is_float && s->parse_int != (PyObject *) &PyLong_Type) + custom_func = s->parse_int; + else + custom_func = NULL; + + if (custom_func) { + /* copy the section we determined to be a number */ + numstr = PyUnicode_FromUnicode(&str[start], idx - start); + if (numstr == NULL) + return NULL; + rval = PyObject_CallFunctionObjArgs(custom_func, numstr, NULL); } else { - /* no fast path for unicode -> int, just call */ - rval = PyObject_CallFunctionObjArgs(s->parse_int, numstr, NULL); + Py_ssize_t i, n; + char *buf; + /* Straight conversion to ASCII, to avoid costly conversion of + decimal unicode digits (which cannot appear here) */ + n = idx - start; + numstr = PyBytes_FromStringAndSize(NULL, n); + if (numstr == NULL) + return NULL; + buf = PyBytes_AS_STRING(numstr); + for (i = 0; i < n; i++) { + buf[i] = (char) str[i + start]; + } + if (is_float) + rval = PyFloat_FromString(numstr); + else + rval = PyLong_FromString(buf, NULL, 10); } Py_DECREF(numstr); *next_idx_ptr = idx; @@ -1210,22 +1329,22 @@ encoder_call(PyObject *self, PyObject *args, PyObject *kwds) /* Python callable interface to encode_listencode_obj */ static char *kwlist[] = {"obj", "_current_indent_level", NULL}; PyObject *obj; - PyObject *rval; Py_ssize_t indent_level; PyEncoderObject *s; + accumulator acc; + assert(PyEncoder_Check(self)); s = (PyEncoderObject *)self; if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist, &obj, _convertPyInt_AsSsize_t, &indent_level)) return NULL; - rval = PyList_New(0); - if (rval == NULL) + if (init_accumulator(&acc)) return NULL; - if (encoder_listencode_obj(s, rval, obj, indent_level)) { - Py_DECREF(rval); + if (encoder_listencode_obj(s, &acc, obj, indent_level)) { + destroy_accumulator(&acc); return NULL; } - return rval; + return finish_accumulator(&acc); } static PyObject * @@ -1297,18 +1416,19 @@ encoder_encode_string(PyEncoderObject *s, PyObject *obj) } static int -_steal_list_append(PyObject *lst, PyObject *stolen) +_steal_accumulate(accumulator *acc, PyObject *stolen) { /* Append stolen and then decrement its reference count */ - int rval = PyList_Append(lst, stolen); + int rval = accumulate_unicode(acc, stolen); Py_DECREF(stolen); return rval; } static int -encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level) +encoder_listencode_obj(PyEncoderObject *s, accumulator *acc, + PyObject *obj, Py_ssize_t indent_level) { - /* Encode Python object obj to a JSON term, rval is a PyList */ + /* Encode Python object obj to a JSON term */ PyObject *newobj; int rv; @@ -1316,38 +1436,38 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi PyObject *cstr = _encoded_const(obj); if (cstr == NULL) return -1; - return _steal_list_append(rval, cstr); + return _steal_accumulate(acc, cstr); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyLong_Check(obj)) { PyObject *encoded = PyObject_Str(obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyFloat_Check(obj)) { PyObject *encoded = encoder_encode_float(s, obj); if (encoded == NULL) return -1; - return _steal_list_append(rval, encoded); + return _steal_accumulate(acc, encoded); } else if (PyList_Check(obj) || PyTuple_Check(obj)) { if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_list(s, rval, obj, indent_level); + rv = encoder_listencode_list(s, acc, obj, indent_level); Py_LeaveRecursiveCall(); return rv; } else if (PyDict_Check(obj)) { if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_dict(s, rval, obj, indent_level); + rv = encoder_listencode_dict(s, acc, obj, indent_level); Py_LeaveRecursiveCall(); return rv; } @@ -1378,7 +1498,7 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi if (Py_EnterRecursiveCall(" while encoding a JSON object")) return -1; - rv = encoder_listencode_obj(s, rval, newobj, indent_level); + rv = encoder_listencode_obj(s, acc, newobj, indent_level); Py_LeaveRecursiveCall(); Py_DECREF(newobj); @@ -1398,9 +1518,10 @@ encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssi } static int -encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level) +encoder_listencode_dict(PyEncoderObject *s, accumulator *acc, + PyObject *dct, Py_ssize_t indent_level) { - /* Encode Python dict dct a JSON term, rval is a PyList */ + /* Encode Python dict dct a JSON term */ static PyObject *open_dict = NULL; static PyObject *close_dict = NULL; static PyObject *empty_dict = NULL; @@ -1420,7 +1541,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss return -1; } if (Py_SIZE(dct) == 0) - return PyList_Append(rval, empty_dict); + return accumulate_unicode(acc, empty_dict); if (s->markers != Py_None) { int has_key; @@ -1438,7 +1559,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss } } - if (PyList_Append(rval, open_dict)) + if (accumulate_unicode(acc, open_dict)) goto bail; if (s->indent != Py_None) { @@ -1525,7 +1646,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss } if (idx) { - if (PyList_Append(rval, s->item_separator)) + if (accumulate_unicode(acc, s->item_separator)) goto bail; } @@ -1533,16 +1654,16 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss Py_CLEAR(kstr); if (encoded == NULL) goto bail; - if (PyList_Append(rval, encoded)) { + if (accumulate_unicode(acc, encoded)) { Py_DECREF(encoded); goto bail; } Py_DECREF(encoded); - if (PyList_Append(rval, s->key_separator)) + if (accumulate_unicode(acc, s->key_separator)) goto bail; value = PyTuple_GET_ITEM(item, 1); - if (encoder_listencode_obj(s, rval, value, indent_level)) + if (encoder_listencode_obj(s, acc, value, indent_level)) goto bail; idx += 1; Py_DECREF(item); @@ -1556,14 +1677,13 @@ encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ss goto bail; Py_CLEAR(ident); } + /* TODO DOES NOT RUN; dead code if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ indent_level -= 1; - /* - yield '\n' + (' ' * (_indent * _current_indent_level)) - */ - } - if (PyList_Append(rval, close_dict)) + + yield '\n' + (' ' * (_indent * _current_indent_level)) + }*/ + if (accumulate_unicode(acc, close_dict)) goto bail; return 0; @@ -1577,9 +1697,10 @@ bail: static int -encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level) +encoder_listencode_list(PyEncoderObject *s, accumulator *acc, + PyObject *seq, Py_ssize_t indent_level) { - /* Encode Python list seq to a JSON term, rval is a PyList */ + /* Encode Python list seq to a JSON term */ static PyObject *open_array = NULL; static PyObject *close_array = NULL; static PyObject *empty_array = NULL; @@ -1603,7 +1724,7 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss num_items = PySequence_Fast_GET_SIZE(s_fast); if (num_items == 0) { Py_DECREF(s_fast); - return PyList_Append(rval, empty_array); + return accumulate_unicode(acc, empty_array); } if (s->markers != Py_None) { @@ -1623,7 +1744,7 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss } seq_items = PySequence_Fast_ITEMS(s_fast); - if (PyList_Append(rval, open_array)) + if (accumulate_unicode(acc, open_array)) goto bail; if (s->indent != Py_None) { /* TODO: DOES NOT RUN */ @@ -1637,10 +1758,10 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss for (i = 0; i < num_items; i++) { PyObject *obj = seq_items[i]; if (i) { - if (PyList_Append(rval, s->item_separator)) + if (accumulate_unicode(acc, s->item_separator)) goto bail; } - if (encoder_listencode_obj(s, rval, obj, indent_level)) + if (encoder_listencode_obj(s, acc, obj, indent_level)) goto bail; } if (ident != NULL) { @@ -1648,14 +1769,14 @@ encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ss goto bail; Py_CLEAR(ident); } + + /* TODO: DOES NOT RUN if (s->indent != Py_None) { - /* TODO: DOES NOT RUN */ indent_level -= 1; - /* - yield '\n' + (' ' * (_indent * _current_indent_level)) - */ - } - if (PyList_Append(rval, close_array)) + + yield '\n' + (' ' * (_indent * _current_indent_level)) + }*/ + if (accumulate_unicode(acc, close_array)) goto bail; Py_DECREF(s_fast); return 0; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index cc412bf..b0a226b 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -176,36 +176,29 @@ normalizeUserObj(PyObject *obj) if (fn->m_self == NULL) { /* built-in function: look up the module name */ PyObject *mod = fn->m_module; - const char *modname; - if (mod && PyUnicode_Check(mod)) { - /* XXX: The following will truncate module names with embedded - * null-characters. It is unlikely that this can happen in - * practice and the concequences are not serious enough to - * introduce extra checks here. - */ - modname = _PyUnicode_AsString(mod); - if (modname == NULL) { - modname = "<encoding error>"; - PyErr_Clear(); + PyObject *modname = NULL; + if (mod != NULL) { + if (PyUnicode_Check(mod)) { + modname = mod; + Py_INCREF(modname); } - } - else if (mod && PyModule_Check(mod)) { - modname = PyModule_GetName(mod); - if (modname == NULL) { - PyErr_Clear(); - modname = "builtins"; + else if (PyModule_Check(mod)) { + modname = PyModule_GetNameObject(mod); + if (modname == NULL) + PyErr_Clear(); } } - else { - modname = "builtins"; + if (modname != NULL) { + if (PyUnicode_CompareWithASCIIString(modname, "builtins") != 0) { + PyObject *result; + result = PyUnicode_FromFormat("<%U.%s>", modname, + fn->m_ml->ml_name); + Py_DECREF(modname); + return result; + } + Py_DECREF(modname); } - if (strcmp(modname, "builtins") != 0) - return PyUnicode_FromFormat("<%s.%s>", - modname, - fn->m_ml->ml_name); - else - return PyUnicode_FromFormat("<%s>", - fn->m_ml->ml_name); + return PyUnicode_FromFormat("<%s>", fn->m_ml->ml_name); } else { /* built-in method: try to return diff --git a/Modules/_multiprocessing/connection.h b/Modules/_multiprocessing/connection.h deleted file mode 100644 index 002d5aa..0000000 --- a/Modules/_multiprocessing/connection.h +++ /dev/null @@ -1,527 +0,0 @@ -/* - * Definition of a `Connection` type. - * Used by `socket_connection.c` and `pipe_connection.c`. - * - * connection.h - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#ifndef CONNECTION_H -#define CONNECTION_H - -/* - * Read/write flags - */ - -#define READABLE 1 -#define WRITABLE 2 - -#define CHECK_READABLE(self) \ - if (!(self->flags & READABLE)) { \ - PyErr_SetString(PyExc_IOError, "connection is write-only"); \ - return NULL; \ - } - -#define CHECK_WRITABLE(self) \ - if (!(self->flags & WRITABLE)) { \ - PyErr_SetString(PyExc_IOError, "connection is read-only"); \ - return NULL; \ - } - -/* - * Allocation and deallocation - */ - -static PyObject * -connection_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - ConnectionObject *self; - HANDLE handle; - BOOL readable = TRUE, writable = TRUE; - - static char *kwlist[] = {"handle", "readable", "writable", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, F_HANDLE "|ii", kwlist, - &handle, &readable, &writable)) - return NULL; - - if (handle == INVALID_HANDLE_VALUE || (Py_ssize_t)handle < 0) { - PyErr_Format(PyExc_IOError, "invalid handle %zd", - (Py_ssize_t)handle); - return NULL; - } - - if (!readable && !writable) { - PyErr_SetString(PyExc_ValueError, - "either readable or writable must be true"); - return NULL; - } - - self = PyObject_New(ConnectionObject, type); - if (self == NULL) - return NULL; - - self->weakreflist = NULL; - self->handle = handle; - self->flags = 0; - - if (readable) - self->flags |= READABLE; - if (writable) - self->flags |= WRITABLE; - assert(self->flags >= 1 && self->flags <= 3); - - return (PyObject*)self; -} - -static void -connection_dealloc(ConnectionObject* self) -{ - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject*)self); - - if (self->handle != INVALID_HANDLE_VALUE) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - } - PyObject_Del(self); -} - -/* - * Functions for transferring buffers - */ - -static PyObject * -connection_sendbytes(ConnectionObject *self, PyObject *args) -{ - Py_buffer pbuffer; - char *buffer; - Py_ssize_t length, offset=0, size=PY_SSIZE_T_MIN; - int res; - - if (!PyArg_ParseTuple(args, F_RBUFFER "*|" F_PY_SSIZE_T F_PY_SSIZE_T, - &pbuffer, &offset, &size)) - return NULL; - buffer = pbuffer.buf; - length = pbuffer.len; - - CHECK_WRITABLE(self); /* XXX release buffer in case of failure */ - - if (offset < 0) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "offset is negative"); - return NULL; - } - if (length < offset) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "buffer length < offset"); - return NULL; - } - - if (size == PY_SSIZE_T_MIN) { - size = length - offset; - } else { - if (size < 0) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, "size is negative"); - return NULL; - } - if (offset + size > length) { - PyBuffer_Release(&pbuffer); - PyErr_SetString(PyExc_ValueError, - "buffer length < offset + size"); - return NULL; - } - } - - res = conn_send_string(self, buffer + offset, size); - - PyBuffer_Release(&pbuffer); - if (res < 0) { - if (PyErr_Occurred()) - return NULL; - else - return mp_SetError(PyExc_IOError, res); - } - - Py_RETURN_NONE; -} - -static PyObject * -connection_recvbytes(ConnectionObject *self, PyObject *args) -{ - char *freeme = NULL; - Py_ssize_t res, maxlength = PY_SSIZE_T_MAX; - PyObject *result = NULL; - - if (!PyArg_ParseTuple(args, "|" F_PY_SSIZE_T, &maxlength)) - return NULL; - - CHECK_READABLE(self); - - if (maxlength < 0) { - PyErr_SetString(PyExc_ValueError, "maxlength < 0"); - return NULL; - } - - res = conn_recv_string(self, self->buffer, CONNECTION_BUFFER_SIZE, - &freeme, maxlength); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - result = PyBytes_FromStringAndSize(self->buffer, res); - } else { - result = PyBytes_FromStringAndSize(freeme, res); - PyMem_Free(freeme); - } - } - - return result; -} - -static PyObject * -connection_recvbytes_into(ConnectionObject *self, PyObject *args) -{ - char *freeme = NULL, *buffer = NULL; - Py_ssize_t res, length, offset = 0; - PyObject *result = NULL; - Py_buffer pbuf; - - CHECK_READABLE(self); - - if (!PyArg_ParseTuple(args, "w*|" F_PY_SSIZE_T, - &pbuf, &offset)) - return NULL; - - buffer = pbuf.buf; - length = pbuf.len; - - if (offset < 0) { - PyErr_SetString(PyExc_ValueError, "negative offset"); - goto _error; - } - - if (offset > length) { - PyErr_SetString(PyExc_ValueError, "offset too large"); - goto _error; - } - - res = conn_recv_string(self, buffer+offset, length-offset, - &freeme, PY_SSIZE_T_MAX); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - result = PyInt_FromSsize_t(res); - } else { - result = PyObject_CallFunction(BufferTooShort, - F_RBUFFER "#", - freeme, res); - PyMem_Free(freeme); - if (result) { - PyErr_SetObject(BufferTooShort, result); - Py_DECREF(result); - } - goto _error; - } - } - -_cleanup: - PyBuffer_Release(&pbuf); - return result; - -_error: - result = NULL; - goto _cleanup; -} - -/* - * Functions for transferring objects - */ - -static PyObject * -connection_send_obj(ConnectionObject *self, PyObject *obj) -{ - char *buffer; - int res; - Py_ssize_t length; - PyObject *pickled_string = NULL; - - CHECK_WRITABLE(self); - - pickled_string = PyObject_CallFunctionObjArgs(pickle_dumps, obj, - pickle_protocol, NULL); - if (!pickled_string) - goto failure; - - if (PyBytes_AsStringAndSize(pickled_string, &buffer, &length) < 0) - goto failure; - - res = conn_send_string(self, buffer, (int)length); - - if (res < 0) { - mp_SetError(PyExc_IOError, res); - goto failure; - } - - Py_XDECREF(pickled_string); - Py_RETURN_NONE; - - failure: - Py_XDECREF(pickled_string); - return NULL; -} - -static PyObject * -connection_recv_obj(ConnectionObject *self) -{ - char *freeme = NULL; - Py_ssize_t res; - PyObject *temp = NULL, *result = NULL; - - CHECK_READABLE(self); - - res = conn_recv_string(self, self->buffer, CONNECTION_BUFFER_SIZE, - &freeme, PY_SSIZE_T_MAX); - - if (res < 0) { - if (res == MP_BAD_MESSAGE_LENGTH) { - if ((self->flags & WRITABLE) == 0) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } else { - self->flags = WRITABLE; - } - } - mp_SetError(PyExc_IOError, res); - } else { - if (freeme == NULL) { - temp = PyBytes_FromStringAndSize(self->buffer, res); - } else { - temp = PyBytes_FromStringAndSize(freeme, res); - PyMem_Free(freeme); - } - } - - if (temp) - result = PyObject_CallFunctionObjArgs(pickle_loads, - temp, NULL); - Py_XDECREF(temp); - return result; -} - -/* - * Other functions - */ - -static PyObject * -connection_poll(ConnectionObject *self, PyObject *args) -{ - PyObject *timeout_obj = NULL; - double timeout = 0.0; - int res; - - CHECK_READABLE(self); - - if (!PyArg_ParseTuple(args, "|O", &timeout_obj)) - return NULL; - - if (timeout_obj == NULL) { - timeout = 0.0; - } else if (timeout_obj == Py_None) { - timeout = -1.0; /* block forever */ - } else { - timeout = PyFloat_AsDouble(timeout_obj); - if (PyErr_Occurred()) - return NULL; - if (timeout < 0.0) - timeout = 0.0; - } - - Py_BEGIN_ALLOW_THREADS - res = conn_poll(self, timeout, _save); - Py_END_ALLOW_THREADS - - switch (res) { - case TRUE: - Py_RETURN_TRUE; - case FALSE: - Py_RETURN_FALSE; - default: - return mp_SetError(PyExc_IOError, res); - } -} - -static PyObject * -connection_fileno(ConnectionObject* self) -{ - if (self->handle == INVALID_HANDLE_VALUE) { - PyErr_SetString(PyExc_IOError, "handle is invalid"); - return NULL; - } - return PyInt_FromLong((long)self->handle); -} - -static PyObject * -connection_close(ConnectionObject *self) -{ - if (self->handle != INVALID_HANDLE_VALUE) { - Py_BEGIN_ALLOW_THREADS - CLOSE(self->handle); - Py_END_ALLOW_THREADS - self->handle = INVALID_HANDLE_VALUE; - } - - Py_RETURN_NONE; -} - -static PyObject * -connection_repr(ConnectionObject *self) -{ - static char *conn_type[] = {"read-only", "write-only", "read-write"}; - - assert(self->flags >= 1 && self->flags <= 3); - return FROM_FORMAT("<%s %s, handle %zd>", - conn_type[self->flags - 1], - CONNECTION_NAME, (Py_ssize_t)self->handle); -} - -/* - * Getters and setters - */ - -static PyObject * -connection_closed(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->handle == INVALID_HANDLE_VALUE)); -} - -static PyObject * -connection_readable(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->flags & READABLE)); -} - -static PyObject * -connection_writable(ConnectionObject *self, void *closure) -{ - return PyBool_FromLong((long)(self->flags & WRITABLE)); -} - -/* - * Tables - */ - -static PyMethodDef connection_methods[] = { - {"send_bytes", (PyCFunction)connection_sendbytes, METH_VARARGS, - "send the byte data from a readable buffer-like object"}, - {"recv_bytes", (PyCFunction)connection_recvbytes, METH_VARARGS, - "receive byte data as a string"}, - {"recv_bytes_into",(PyCFunction)connection_recvbytes_into,METH_VARARGS, - "receive byte data into a writeable buffer-like object\n" - "returns the number of bytes read"}, - - {"send", (PyCFunction)connection_send_obj, METH_O, - "send a (picklable) object"}, - {"recv", (PyCFunction)connection_recv_obj, METH_NOARGS, - "receive a (picklable) object"}, - - {"poll", (PyCFunction)connection_poll, METH_VARARGS, - "whether there is any input available to be read"}, - {"fileno", (PyCFunction)connection_fileno, METH_NOARGS, - "file descriptor or handle of the connection"}, - {"close", (PyCFunction)connection_close, METH_NOARGS, - "close the connection"}, - - {NULL} /* Sentinel */ -}; - -static PyGetSetDef connection_getset[] = { - {"closed", (getter)connection_closed, NULL, - "True if the connection is closed", NULL}, - {"readable", (getter)connection_readable, NULL, - "True if the connection is readable", NULL}, - {"writable", (getter)connection_writable, NULL, - "True if the connection is writable", NULL}, - {NULL} -}; - -/* - * Connection type - */ - -PyDoc_STRVAR(connection_doc, - "Connection type whose constructor signature is\n\n" - " Connection(handle, readable=True, writable=True).\n\n" - "The constructor does *not* duplicate the handle."); - -PyTypeObject CONNECTION_TYPE = { - PyVarObject_HEAD_INIT(NULL, 0) - /* tp_name */ "_multiprocessing." CONNECTION_NAME, - /* tp_basicsize */ sizeof(ConnectionObject), - /* tp_itemsize */ 0, - /* tp_dealloc */ (destructor)connection_dealloc, - /* tp_print */ 0, - /* tp_getattr */ 0, - /* tp_setattr */ 0, - /* tp_reserved */ 0, - /* tp_repr */ (reprfunc)connection_repr, - /* tp_as_number */ 0, - /* tp_as_sequence */ 0, - /* tp_as_mapping */ 0, - /* tp_hash */ 0, - /* tp_call */ 0, - /* tp_str */ 0, - /* tp_getattro */ 0, - /* tp_setattro */ 0, - /* tp_as_buffer */ 0, - /* tp_flags */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_WEAKREFS, - /* tp_doc */ connection_doc, - /* tp_traverse */ 0, - /* tp_clear */ 0, - /* tp_richcompare */ 0, - /* tp_weaklistoffset */ offsetof(ConnectionObject, weakreflist), - /* tp_iter */ 0, - /* tp_iternext */ 0, - /* tp_methods */ connection_methods, - /* tp_members */ 0, - /* tp_getset */ connection_getset, - /* tp_base */ 0, - /* tp_dict */ 0, - /* tp_descr_get */ 0, - /* tp_descr_set */ 0, - /* tp_dictoffset */ 0, - /* tp_init */ 0, - /* tp_alloc */ 0, - /* tp_new */ connection_new, -}; - -#endif /* CONNECTION_H */ diff --git a/Modules/_multiprocessing/multiprocessing.c b/Modules/_multiprocessing/multiprocessing.c index 7c4f52d..5d1cf56 100644 --- a/Modules/_multiprocessing/multiprocessing.c +++ b/Modules/_multiprocessing/multiprocessing.c @@ -16,7 +16,6 @@ PyObject *create_win32_namespace(void); -PyObject *pickle_dumps, *pickle_loads, *pickle_protocol; PyObject *ProcessError, *BufferTooShort; /* @@ -49,16 +48,6 @@ mp_SetError(PyObject *Type, int num) case MP_MEMORY_ERROR: PyErr_NoMemory(); break; - case MP_END_OF_FILE: - PyErr_SetNone(PyExc_EOFError); - break; - case MP_EARLY_END_OF_FILE: - PyErr_SetString(PyExc_IOError, - "got end of file during message"); - break; - case MP_BAD_MESSAGE_LENGTH: - PyErr_SetString(PyExc_IOError, "bad message length"); - break; case MP_EXCEPTION_HAS_BEEN_SET: break; default: @@ -187,7 +176,7 @@ multiprocessing_address_of_buffer(PyObject *self, PyObject *obj) if (PyObject_AsWriteBuffer(obj, &buffer, &buffer_len) < 0) return NULL; - return Py_BuildValue("N" F_PY_SSIZE_T, + return Py_BuildValue("Nn", PyLong_FromVoidPtr(buffer), buffer_len); } @@ -241,15 +230,6 @@ PyInit__multiprocessing(void) if (!module) return NULL; - /* Get copy of objects from pickle */ - temp = PyImport_ImportModule(PICKLE_MODULE); - if (!temp) - return NULL; - pickle_dumps = PyObject_GetAttrString(temp, "dumps"); - pickle_loads = PyObject_GetAttrString(temp, "loads"); - pickle_protocol = PyObject_GetAttrString(temp, "HIGHEST_PROTOCOL"); - Py_XDECREF(temp); - /* Get copy of BufferTooShort */ temp = PyImport_ImportModule("multiprocessing"); if (!temp) @@ -257,12 +237,6 @@ PyInit__multiprocessing(void) BufferTooShort = PyObject_GetAttrString(temp, "BufferTooShort"); Py_XDECREF(temp); - /* Add connection type to module */ - if (PyType_Ready(&ConnectionType) < 0) - return NULL; - Py_INCREF(&ConnectionType); - PyModule_AddObject(module, "Connection", (PyObject*)&ConnectionType); - #if defined(MS_WINDOWS) || \ (defined(HAVE_SEM_OPEN) && !defined(POSIX_SEMAPHORES_NOT_ENABLED)) /* Add SemLock type to module */ @@ -286,13 +260,6 @@ PyInit__multiprocessing(void) #endif #ifdef MS_WINDOWS - /* Add PipeConnection to module */ - if (PyType_Ready(&PipeConnectionType) < 0) - return NULL; - Py_INCREF(&PipeConnectionType); - PyModule_AddObject(module, "PipeConnection", - (PyObject*)&PipeConnectionType); - /* Initialize win32 class and add to multiprocessing */ temp = create_win32_namespace(); if (!temp) diff --git a/Modules/_multiprocessing/multiprocessing.h b/Modules/_multiprocessing/multiprocessing.h index 14425de..c303447 100644 --- a/Modules/_multiprocessing/multiprocessing.h +++ b/Modules/_multiprocessing/multiprocessing.h @@ -4,7 +4,7 @@ #define PY_SSIZE_T_CLEAN #ifdef __sun -/* The control message API is only available on Solaris +/* The control message API is only available on Solaris if XPG 4.2 or later is requested. */ #define _XOPEN_SOURCE 500 #endif @@ -64,20 +64,6 @@ /* - * Make sure Py_ssize_t available - */ - -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) - typedef int Py_ssize_t; -# define PY_SSIZE_T_MAX INT_MAX -# define PY_SSIZE_T_MIN INT_MIN -# define F_PY_SSIZE_T "i" -# define PyInt_FromSsize_t(n) PyInt_FromLong((long)n) -#else -# define F_PY_SSIZE_T "n" -#endif - -/* * Format codes */ @@ -105,12 +91,6 @@ # define T_SEM_HANDLE T_POINTER #endif -#if PY_VERSION_HEX >= 0x03000000 -# define F_RBUFFER "y" -#else -# define F_RBUFFER "s" -#endif - /* * Error codes which can be returned by functions called without GIL */ @@ -118,11 +98,8 @@ #define MP_SUCCESS (0) #define MP_STANDARD_ERROR (-1) #define MP_MEMORY_ERROR (-1001) -#define MP_END_OF_FILE (-1002) -#define MP_EARLY_END_OF_FILE (-1003) -#define MP_BAD_MESSAGE_LENGTH (-1004) -#define MP_SOCKET_ERROR (-1005) -#define MP_EXCEPTION_HAS_BEEN_SET (-1006) +#define MP_SOCKET_ERROR (-1002) +#define MP_EXCEPTION_HAS_BEEN_SET (-1003) PyObject *mp_SetError(PyObject *Type, int num); @@ -130,57 +107,15 @@ PyObject *mp_SetError(PyObject *Type, int num); * Externs - not all will really exist on all platforms */ -extern PyObject *pickle_dumps; -extern PyObject *pickle_loads; -extern PyObject *pickle_protocol; extern PyObject *BufferTooShort; extern PyTypeObject SemLockType; -extern PyTypeObject ConnectionType; extern PyTypeObject PipeConnectionType; extern HANDLE sigint_event; /* - * Py3k compatibility - */ - -#if PY_VERSION_HEX >= 0x03000000 -# define PICKLE_MODULE "pickle" -# define FROM_FORMAT PyUnicode_FromFormat -# define PyInt_FromLong PyLong_FromLong -# define PyInt_FromSsize_t PyLong_FromSsize_t -#else -# define PICKLE_MODULE "cPickle" -# define FROM_FORMAT PyString_FromFormat -#endif - -#ifndef PyVarObject_HEAD_INIT -# define PyVarObject_HEAD_INIT(type, size) PyObject_HEAD_INIT(type) size, -#endif - -#ifndef Py_TPFLAGS_HAVE_WEAKREFS -# define Py_TPFLAGS_HAVE_WEAKREFS 0 -#endif - -/* - * Connection definition - */ - -#define CONNECTION_BUFFER_SIZE 1024 - -typedef struct { - PyObject_HEAD - HANDLE handle; - int flags; - PyObject *weakreflist; - char buffer[CONNECTION_BUFFER_SIZE]; -} ConnectionObject; - -/* * Miscellaneous */ -#define MAX_MESSAGE_LENGTH 0x7fffffff - #ifndef MIN # define MIN(x, y) ((x) < (y) ? x : y) # define MAX(x, y) ((x) > (y) ? x : y) diff --git a/Modules/_multiprocessing/pipe_connection.c b/Modules/_multiprocessing/pipe_connection.c deleted file mode 100644 index 05dde0c..0000000 --- a/Modules/_multiprocessing/pipe_connection.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * A type which wraps a pipe handle in message oriented mode - * - * pipe_connection.c - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#include "multiprocessing.h" - -#define CLOSE(h) CloseHandle(h) - -/* - * Send string to the pipe; assumes in message oriented mode - */ - -static Py_ssize_t -conn_send_string(ConnectionObject *conn, char *string, size_t length) -{ - DWORD amount_written; - BOOL ret; - - Py_BEGIN_ALLOW_THREADS - ret = WriteFile(conn->handle, string, length, &amount_written, NULL); - Py_END_ALLOW_THREADS - - if (ret == 0 && GetLastError() == ERROR_NO_SYSTEM_RESOURCES) { - PyErr_Format(PyExc_ValueError, "Cannnot send %" PY_FORMAT_SIZE_T "d bytes over connection", length); - return MP_STANDARD_ERROR; - } - - return ret ? MP_SUCCESS : MP_STANDARD_ERROR; -} - -/* - * Attempts to read into buffer, or if buffer too small into *newbuffer. - * - * Returns number of bytes read. Assumes in message oriented mode. - */ - -static Py_ssize_t -conn_recv_string(ConnectionObject *conn, char *buffer, - size_t buflength, char **newbuffer, size_t maxlength) -{ - DWORD left, length, full_length, err; - BOOL ret; - *newbuffer = NULL; - - Py_BEGIN_ALLOW_THREADS - ret = ReadFile(conn->handle, buffer, MIN(buflength, maxlength), - &length, NULL); - Py_END_ALLOW_THREADS - if (ret) - return length; - - err = GetLastError(); - if (err != ERROR_MORE_DATA) { - if (err == ERROR_BROKEN_PIPE) - return MP_END_OF_FILE; - return MP_STANDARD_ERROR; - } - - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, NULL, &left)) - return MP_STANDARD_ERROR; - - full_length = length + left; - if (full_length > maxlength) - return MP_BAD_MESSAGE_LENGTH; - - *newbuffer = PyMem_Malloc(full_length); - if (*newbuffer == NULL) - return MP_MEMORY_ERROR; - - memcpy(*newbuffer, buffer, length); - - Py_BEGIN_ALLOW_THREADS - ret = ReadFile(conn->handle, *newbuffer+length, left, &length, NULL); - Py_END_ALLOW_THREADS - if (ret) { - assert(length == left); - return full_length; - } else { - PyMem_Free(*newbuffer); - return MP_STANDARD_ERROR; - } -} - -/* - * Check whether any data is available for reading - */ - -static int -conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) -{ - DWORD bytes, deadline, delay; - int difference, res; - BOOL block = FALSE; - - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, &bytes, NULL)) - return MP_STANDARD_ERROR; - - if (timeout == 0.0) - return bytes > 0; - - if (timeout < 0.0) - block = TRUE; - else - /* XXX does not check for overflow */ - deadline = GetTickCount() + (DWORD)(1000 * timeout + 0.5); - - Sleep(0); - - for (delay = 1 ; ; delay += 1) { - if (!PeekNamedPipe(conn->handle, NULL, 0, NULL, &bytes, NULL)) - return MP_STANDARD_ERROR; - else if (bytes > 0) - return TRUE; - - if (!block) { - difference = deadline - GetTickCount(); - if (difference < 0) - return FALSE; - if ((int)delay > difference) - delay = difference; - } - - if (delay > 20) - delay = 20; - - Sleep(delay); - - /* check for signals */ - Py_BLOCK_THREADS - res = PyErr_CheckSignals(); - Py_UNBLOCK_THREADS - - if (res) - return MP_EXCEPTION_HAS_BEEN_SET; - } -} - -/* - * "connection.h" defines the PipeConnection type using the definitions above - */ - -#define CONNECTION_NAME "PipeConnection" -#define CONNECTION_TYPE PipeConnectionType - -#include "connection.h" diff --git a/Modules/_multiprocessing/semaphore.c b/Modules/_multiprocessing/semaphore.c index c2cd914..6749f23 100644 --- a/Modules/_multiprocessing/semaphore.c +++ b/Modules/_multiprocessing/semaphore.c @@ -481,7 +481,7 @@ semlock_dealloc(SemLockObject* self) static PyObject * semlock_count(SemLockObject *self) { - return PyInt_FromLong((long)self->count); + return PyLong_FromLong((long)self->count); } static PyObject * @@ -505,7 +505,7 @@ semlock_getvalue(SemLockObject *self) the number of waiting threads */ if (sval < 0) sval = 0; - return PyInt_FromLong((long)sval); + return PyLong_FromLong((long)sval); #endif } diff --git a/Modules/_multiprocessing/socket_connection.c b/Modules/_multiprocessing/socket_connection.c deleted file mode 100644 index 7ebf338..0000000 --- a/Modules/_multiprocessing/socket_connection.c +++ /dev/null @@ -1,202 +0,0 @@ -/* - * A type which wraps a socket - * - * socket_connection.c - * - * Copyright (c) 2006-2008, R Oudkerk --- see COPYING.txt - */ - -#include "multiprocessing.h" - -#ifdef MS_WINDOWS -# define WRITE(h, buffer, length) send((SOCKET)h, buffer, length, 0) -# define READ(h, buffer, length) recv((SOCKET)h, buffer, length, 0) -# define CLOSE(h) closesocket((SOCKET)h) -#else -# define WRITE(h, buffer, length) write(h, buffer, length) -# define READ(h, buffer, length) read(h, buffer, length) -# define CLOSE(h) close(h) -#endif - -/* - * Send string to file descriptor - */ - -static Py_ssize_t -_conn_sendall(HANDLE h, char *string, size_t length) -{ - char *p = string; - Py_ssize_t res; - - while (length > 0) { - res = WRITE(h, p, length); - if (res < 0) - return MP_SOCKET_ERROR; - length -= res; - p += res; - } - - return MP_SUCCESS; -} - -/* - * Receive string of exact length from file descriptor - */ - -static Py_ssize_t -_conn_recvall(HANDLE h, char *buffer, size_t length) -{ - size_t remaining = length; - Py_ssize_t temp; - char *p = buffer; - - while (remaining > 0) { - temp = READ(h, p, remaining); - if (temp <= 0) { - if (temp == 0) - return remaining == length ? - MP_END_OF_FILE : MP_EARLY_END_OF_FILE; - else - return temp; - } - remaining -= temp; - p += temp; - } - - return MP_SUCCESS; -} - -/* - * Send a string prepended by the string length in network byte order - */ - -static Py_ssize_t -conn_send_string(ConnectionObject *conn, char *string, size_t length) -{ - Py_ssize_t res; - /* The "header" of the message is a 32 bit unsigned number (in - network order) which specifies the length of the "body". If - the message is shorter than about 16kb then it is quicker to - combine the "header" and the "body" of the message and send - them at once. */ - if (length < (16*1024)) { - char *message; - - message = PyMem_Malloc(length+4); - if (message == NULL) - return MP_MEMORY_ERROR; - - *(UINT32*)message = htonl((UINT32)length); - memcpy(message+4, string, length); - Py_BEGIN_ALLOW_THREADS - res = _conn_sendall(conn->handle, message, length+4); - Py_END_ALLOW_THREADS - PyMem_Free(message); - } else { - UINT32 lenbuff; - - if (length > MAX_MESSAGE_LENGTH) - return MP_BAD_MESSAGE_LENGTH; - - lenbuff = htonl((UINT32)length); - Py_BEGIN_ALLOW_THREADS - res = _conn_sendall(conn->handle, (char*)&lenbuff, 4) || - _conn_sendall(conn->handle, string, length); - Py_END_ALLOW_THREADS - } - return res; -} - -/* - * Attempts to read into buffer, or failing that into *newbuffer - * - * Returns number of bytes read. - */ - -static Py_ssize_t -conn_recv_string(ConnectionObject *conn, char *buffer, - size_t buflength, char **newbuffer, size_t maxlength) -{ - int res; - UINT32 ulength; - - *newbuffer = NULL; - - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, (char*)&ulength, 4); - Py_END_ALLOW_THREADS - if (res < 0) - return res; - - ulength = ntohl(ulength); - if (ulength > maxlength) - return MP_BAD_MESSAGE_LENGTH; - - if (ulength <= buflength) { - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, buffer, (size_t)ulength); - Py_END_ALLOW_THREADS - return res < 0 ? res : ulength; - } else { - *newbuffer = PyMem_Malloc((size_t)ulength); - if (*newbuffer == NULL) - return MP_MEMORY_ERROR; - Py_BEGIN_ALLOW_THREADS - res = _conn_recvall(conn->handle, *newbuffer, (size_t)ulength); - Py_END_ALLOW_THREADS - return res < 0 ? (Py_ssize_t)res : (Py_ssize_t)ulength; - } -} - -/* - * Check whether any data is available for reading -- neg timeout blocks - */ - -static int -conn_poll(ConnectionObject *conn, double timeout, PyThreadState *_save) -{ - int res; - fd_set rfds; - - /* - * Verify the handle, issue 3321. Not required for windows. - */ - #ifndef MS_WINDOWS - if (((int)conn->handle) < 0 || ((int)conn->handle) >= FD_SETSIZE) { - Py_BLOCK_THREADS - PyErr_SetString(PyExc_IOError, "handle out of range in select()"); - Py_UNBLOCK_THREADS - return MP_EXCEPTION_HAS_BEEN_SET; - } - #endif - - FD_ZERO(&rfds); - FD_SET((SOCKET)conn->handle, &rfds); - - if (timeout < 0.0) { - res = select((int)conn->handle+1, &rfds, NULL, NULL, NULL); - } else { - struct timeval tv; - tv.tv_sec = (long)timeout; - tv.tv_usec = (long)((timeout - tv.tv_sec) * 1e6 + 0.5); - res = select((int)conn->handle+1, &rfds, NULL, NULL, &tv); - } - - if (res < 0) { - return MP_SOCKET_ERROR; - } else if (FD_ISSET(conn->handle, &rfds)) { - return TRUE; - } else { - assert(res == 0); - return FALSE; - } -} - -/* - * "connection.h" defines the Connection type using defs above - */ - -#define CONNECTION_NAME "Connection" -#define CONNECTION_TYPE ConnectionType - -#include "connection.h" diff --git a/Modules/_multiprocessing/win32_functions.c b/Modules/_multiprocessing/win32_functions.c index 452d608..c017b2a 100644 --- a/Modules/_multiprocessing/win32_functions.c +++ b/Modules/_multiprocessing/win32_functions.c @@ -12,10 +12,223 @@ #define WIN32_FUNCTION(func) \ {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_STATIC, ""} +#define WIN32_KWARGS_FUNCTION(func) \ + {#func, (PyCFunction)win32_ ## func, METH_VARARGS | METH_KEYWORDS | METH_STATIC, ""} + #define WIN32_CONSTANT(fmt, con) \ PyDict_SetItemString(Win32Type.tp_dict, #con, Py_BuildValue(fmt, con)) +/* Grab CancelIoEx dynamically from kernel32 */ +static int has_CancelIoEx = -1; +static BOOL (CALLBACK *Py_CancelIoEx)(HANDLE, LPOVERLAPPED); + +static int +check_CancelIoEx() +{ + if (has_CancelIoEx == -1) + { + HINSTANCE hKernel32 = GetModuleHandle("KERNEL32"); + * (FARPROC *) &Py_CancelIoEx = GetProcAddress(hKernel32, + "CancelIoEx"); + has_CancelIoEx = (Py_CancelIoEx != NULL); + } + return has_CancelIoEx; +} + + +/* + * A Python object wrapping an OVERLAPPED structure and other useful data + * for overlapped I/O + */ + +typedef struct { + PyObject_HEAD + OVERLAPPED overlapped; + /* For convenience, we store the file handle too */ + HANDLE handle; + /* Whether there's I/O in flight */ + int pending; + /* Whether I/O completed successfully */ + int completed; + /* Buffer used for reading (optional) */ + PyObject *read_buffer; + /* Buffer used for writing (optional) */ + Py_buffer write_buffer; +} OverlappedObject; + +static void +overlapped_dealloc(OverlappedObject *self) +{ + int err = GetLastError(); + if (self->pending) { + if (check_CancelIoEx()) + Py_CancelIoEx(self->handle, &self->overlapped); + else { + PyErr_SetString(PyExc_RuntimeError, + "I/O operations still in flight while destroying " + "Overlapped object, the process may crash"); + PyErr_WriteUnraisable(NULL); + } + } + CloseHandle(self->overlapped.hEvent); + SetLastError(err); + if (self->write_buffer.obj) + PyBuffer_Release(&self->write_buffer); + Py_CLEAR(self->read_buffer); + PyObject_Del(self); +} + +static PyObject * +overlapped_GetOverlappedResult(OverlappedObject *self, PyObject *waitobj) +{ + int wait; + BOOL res; + DWORD transferred = 0; + + wait = PyObject_IsTrue(waitobj); + if (wait < 0) + return NULL; + Py_BEGIN_ALLOW_THREADS + res = GetOverlappedResult(self->handle, &self->overlapped, &transferred, + wait != 0); + Py_END_ALLOW_THREADS + + if (!res) { + int err = GetLastError(); + if (err == ERROR_IO_INCOMPLETE) + Py_RETURN_NONE; + if (err != ERROR_MORE_DATA) { + self->pending = 0; + return PyErr_SetExcFromWindowsErr(PyExc_IOError, err); + } + } + self->pending = 0; + self->completed = 1; + if (self->read_buffer) { + assert(PyBytes_CheckExact(self->read_buffer)); + if (_PyBytes_Resize(&self->read_buffer, transferred)) + return NULL; + } + return Py_BuildValue("lN", (long) transferred, PyBool_FromLong(res)); +} + +static PyObject * +overlapped_getbuffer(OverlappedObject *self) +{ + PyObject *res; + if (!self->completed) { + PyErr_SetString(PyExc_ValueError, + "can't get read buffer before GetOverlappedResult() " + "signals the operation completed"); + return NULL; + } + res = self->read_buffer ? self->read_buffer : Py_None; + Py_INCREF(res); + return res; +} + +static PyObject * +overlapped_cancel(OverlappedObject *self) +{ + BOOL res = TRUE; + + if (self->pending) { + Py_BEGIN_ALLOW_THREADS + if (check_CancelIoEx()) + res = Py_CancelIoEx(self->handle, &self->overlapped); + else + res = CancelIo(self->handle); + Py_END_ALLOW_THREADS + } + + /* CancelIoEx returns ERROR_NOT_FOUND if the I/O completed in-between */ + if (!res && GetLastError() != ERROR_NOT_FOUND) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + self->pending = 0; + Py_RETURN_NONE; +} + +static PyMethodDef overlapped_methods[] = { + {"GetOverlappedResult", (PyCFunction) overlapped_GetOverlappedResult, + METH_O, NULL}, + {"getbuffer", (PyCFunction) overlapped_getbuffer, METH_NOARGS, NULL}, + {"cancel", (PyCFunction) overlapped_cancel, METH_NOARGS, NULL}, + {NULL} +}; + +static PyMemberDef overlapped_members[] = { + {"event", T_HANDLE, + offsetof(OverlappedObject, overlapped) + offsetof(OVERLAPPED, hEvent), + READONLY, "overlapped event handle"}, + {NULL} +}; + +PyTypeObject OverlappedType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* tp_name */ "_multiprocessing.win32.Overlapped", + /* tp_basicsize */ sizeof(OverlappedObject), + /* tp_itemsize */ 0, + /* tp_dealloc */ (destructor) overlapped_dealloc, + /* tp_print */ 0, + /* tp_getattr */ 0, + /* tp_setattr */ 0, + /* tp_reserved */ 0, + /* tp_repr */ 0, + /* tp_as_number */ 0, + /* tp_as_sequence */ 0, + /* tp_as_mapping */ 0, + /* tp_hash */ 0, + /* tp_call */ 0, + /* tp_str */ 0, + /* tp_getattro */ 0, + /* tp_setattro */ 0, + /* tp_as_buffer */ 0, + /* tp_flags */ Py_TPFLAGS_DEFAULT, + /* tp_doc */ "OVERLAPPED structure wrapper", + /* tp_traverse */ 0, + /* tp_clear */ 0, + /* tp_richcompare */ 0, + /* tp_weaklistoffset */ 0, + /* tp_iter */ 0, + /* tp_iternext */ 0, + /* tp_methods */ overlapped_methods, + /* tp_members */ overlapped_members, + /* tp_getset */ 0, + /* tp_base */ 0, + /* tp_dict */ 0, + /* tp_descr_get */ 0, + /* tp_descr_set */ 0, + /* tp_dictoffset */ 0, + /* tp_init */ 0, + /* tp_alloc */ 0, + /* tp_new */ 0, +}; + +static OverlappedObject * +new_overlapped(HANDLE handle) +{ + OverlappedObject *self; + + self = PyObject_New(OverlappedObject, &OverlappedType); + if (!self) + return NULL; + self->handle = handle; + self->read_buffer = NULL; + self->pending = 0; + self->completed = 0; + memset(&self->overlapped, 0, sizeof(OVERLAPPED)); + memset(&self->write_buffer, 0, sizeof(Py_buffer)); + /* Manual reset, initially non-signalled */ + self->overlapped.hEvent = CreateEvent(NULL, TRUE, FALSE, NULL); + return self; +} + + +/* + * Module functions + */ + static PyObject * win32_CloseHandle(PyObject *self, PyObject *args) { @@ -36,20 +249,44 @@ win32_CloseHandle(PyObject *self, PyObject *args) } static PyObject * -win32_ConnectNamedPipe(PyObject *self, PyObject *args) +win32_ConnectNamedPipe(PyObject *self, PyObject *args, PyObject *kwds) { HANDLE hNamedPipe; - LPOVERLAPPED lpOverlapped; + int use_overlapped = 0; BOOL success; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "overlapped", NULL}; - if (!PyArg_ParseTuple(args, F_HANDLE F_POINTER, - &hNamedPipe, &lpOverlapped)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "|i", kwlist, + &hNamedPipe, &use_overlapped)) return NULL; + if (use_overlapped) { + overlapped = new_overlapped(hNamedPipe); + if (!overlapped) + return NULL; + } + Py_BEGIN_ALLOW_THREADS - success = ConnectNamedPipe(hNamedPipe, lpOverlapped); + success = ConnectNamedPipe(hNamedPipe, + overlapped ? &overlapped->overlapped : NULL); Py_END_ALLOW_THREADS + if (overlapped) { + int err = GetLastError(); + /* Overlapped ConnectNamedPipe never returns a success code */ + assert(success == 0); + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err == ERROR_PIPE_CONNECTED) + SetEvent(overlapped->overlapped.hEvent); + else { + Py_DECREF(overlapped); + return PyErr_SetFromWindowsErr(err); + } + return (PyObject *) overlapped; + } if (!success) return PyErr_SetFromWindowsErr(0); @@ -215,16 +452,295 @@ win32_WaitNamedPipe(PyObject *self, PyObject *args) Py_RETURN_NONE; } +static PyObject * +win32_closesocket(PyObject *self, PyObject *args) +{ + HANDLE handle; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE ":closesocket" , &handle)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = closesocket((SOCKET) handle); + Py_END_ALLOW_THREADS + + if (ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + Py_RETURN_NONE; +} + +static PyObject * +win32_recv(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size, nread; + PyObject *buf; + + if (!PyArg_ParseTuple(args, F_HANDLE "i:recv" , &handle, &size)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + + Py_BEGIN_ALLOW_THREADS + nread = recv((SOCKET) handle, PyBytes_AS_STRING(buf), size, 0); + Py_END_ALLOW_THREADS + + if (nread < 0) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + } + _PyBytes_Resize(&buf, nread); + return buf; +} + +static PyObject * +win32_send(PyObject *self, PyObject *args) +{ + HANDLE handle; + Py_buffer buf; + int ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "y*:send" , &handle, &buf)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + ret = send((SOCKET) handle, buf.buf, buf.len, 0); + Py_END_ALLOW_THREADS + + PyBuffer_Release(&buf); + if (ret < 0) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, WSAGetLastError()); + return PyLong_FromLong(ret); +} + +static PyObject * +win32_WriteFile(PyObject *self, PyObject *args, PyObject *kwds) +{ + HANDLE handle; + Py_buffer _buf, *buf; + PyObject *bufobj; + int written; + BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "buffer", "overlapped", NULL}; + + /* First get handle and use_overlapped to know which Py_buffer to use */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "O|i:WriteFile", kwlist, + &handle, &bufobj, &use_overlapped)) + return NULL; + + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) + return NULL; + buf = &overlapped->write_buffer; + } + else + buf = &_buf; + + if (!PyArg_Parse(bufobj, "y*", buf)) { + Py_XDECREF(overlapped); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + ret = WriteFile(handle, buf->buf, buf->len, &written, + overlapped ? &overlapped->overlapped : NULL); + Py_END_ALLOW_THREADS + + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + + PyBuffer_Release(buf); + if (!ret) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + return PyLong_FromLong(written); +} + +static PyObject * +win32_ReadFile(PyObject *self, PyObject *args, PyObject *kwds) +{ + HANDLE handle; + int size; + DWORD nread; + PyObject *buf; + BOOL ret; + int use_overlapped = 0; + OverlappedObject *overlapped = NULL; + static char *kwlist[] = {"handle", "size", "overlapped", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, + F_HANDLE "i|i:ReadFile", kwlist, + &handle, &size, &use_overlapped)) + return NULL; + + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + if (use_overlapped) { + overlapped = new_overlapped(handle); + if (!overlapped) { + Py_DECREF(buf); + return NULL; + } + /* Steals reference to buf */ + overlapped->read_buffer = buf; + } + + Py_BEGIN_ALLOW_THREADS + ret = ReadFile(handle, PyBytes_AS_STRING(buf), size, &nread, + overlapped ? &overlapped->overlapped : NULL); + Py_END_ALLOW_THREADS + + if (overlapped) { + int err = GetLastError(); + if (!ret) { + if (err == ERROR_IO_PENDING) + overlapped->pending = 1; + else if (err != ERROR_MORE_DATA) { + Py_DECREF(overlapped); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + } + return (PyObject *) overlapped; + } + + if (!ret && GetLastError() != ERROR_MORE_DATA) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("NN", buf, PyBool_FromLong(ret)); +} + +static PyObject * +win32_PeekNamedPipe(PyObject *self, PyObject *args) +{ + HANDLE handle; + int size = 0; + PyObject *buf = NULL; + DWORD nread, navail, nleft; + BOOL ret; + + if (!PyArg_ParseTuple(args, F_HANDLE "|i:PeekNamedPipe" , &handle, &size)) + return NULL; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative size"); + return NULL; + } + + if (size) { + buf = PyBytes_FromStringAndSize(NULL, size); + if (!buf) + return NULL; + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, PyBytes_AS_STRING(buf), size, &nread, + &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + Py_DECREF(buf); + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + if (_PyBytes_Resize(&buf, nread)) + return NULL; + return Py_BuildValue("Nii", buf, navail, nleft); + } + else { + Py_BEGIN_ALLOW_THREADS + ret = PeekNamedPipe(handle, NULL, 0, NULL, &navail, &nleft); + Py_END_ALLOW_THREADS + if (!ret) { + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + } + return Py_BuildValue("ii", navail, nleft); + } +} + +static PyObject * +win32_WaitForMultipleObjects(PyObject* self, PyObject* args) +{ + DWORD result; + PyObject *handle_seq; + HANDLE handles[MAXIMUM_WAIT_OBJECTS]; + Py_ssize_t nhandles, i; + int wait_flag; + int milliseconds = INFINITE; + + if (!PyArg_ParseTuple(args, "Oi|i:WaitForMultipleObjects", + &handle_seq, &wait_flag, &milliseconds)) + return NULL; + + if (!PySequence_Check(handle_seq)) { + PyErr_Format(PyExc_TypeError, + "sequence type expected, got '%s'", + Py_TYPE(handle_seq)->tp_doc); + return NULL; + } + nhandles = PySequence_Length(handle_seq); + if (nhandles == -1) + return NULL; + if (nhandles < 0 || nhandles >= MAXIMUM_WAIT_OBJECTS) { + PyErr_Format(PyExc_ValueError, + "need at most %zd handles, got a sequence of length %zd", + MAXIMUM_WAIT_OBJECTS, nhandles); + return NULL; + } + for (i = 0; i < nhandles; i++) { + HANDLE h; + PyObject *v = PySequence_GetItem(handle_seq, i); + if (v == NULL) + return NULL; + if (!PyArg_Parse(v, F_HANDLE, &h)) + return NULL; + handles[i] = h; + } + + Py_BEGIN_ALLOW_THREADS + result = WaitForMultipleObjects((DWORD) nhandles, handles, + (BOOL) wait_flag, (DWORD) milliseconds); + Py_END_ALLOW_THREADS + + if (result == WAIT_FAILED) + return PyErr_SetExcFromWindowsErr(PyExc_IOError, 0); + + return PyLong_FromLong((int) result); +} + + static PyMethodDef win32_methods[] = { WIN32_FUNCTION(CloseHandle), WIN32_FUNCTION(GetLastError), WIN32_FUNCTION(OpenProcess), WIN32_FUNCTION(ExitProcess), - WIN32_FUNCTION(ConnectNamedPipe), + WIN32_KWARGS_FUNCTION(ConnectNamedPipe), WIN32_FUNCTION(CreateFile), WIN32_FUNCTION(CreateNamedPipe), + WIN32_KWARGS_FUNCTION(ReadFile), + WIN32_FUNCTION(PeekNamedPipe), WIN32_FUNCTION(SetNamedPipeHandleState), + WIN32_FUNCTION(WaitForMultipleObjects), WIN32_FUNCTION(WaitNamedPipe), + WIN32_KWARGS_FUNCTION(WriteFile), + WIN32_FUNCTION(closesocket), + WIN32_FUNCTION(recv), + WIN32_FUNCTION(send), {NULL} }; @@ -243,10 +759,18 @@ create_win32_namespace(void) return NULL; Py_INCREF(&Win32Type); + if (PyType_Ready(&OverlappedType) < 0) + return NULL; + PyDict_SetItemString(Win32Type.tp_dict, "Overlapped", + (PyObject *) &OverlappedType); + WIN32_CONSTANT(F_DWORD, ERROR_ALREADY_EXISTS); + WIN32_CONSTANT(F_DWORD, ERROR_BROKEN_PIPE); + WIN32_CONSTANT(F_DWORD, ERROR_NO_SYSTEM_RESOURCES); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_BUSY); WIN32_CONSTANT(F_DWORD, ERROR_PIPE_CONNECTED); WIN32_CONSTANT(F_DWORD, ERROR_SEM_TIMEOUT); + WIN32_CONSTANT(F_DWORD, FILE_FLAG_OVERLAPPED); WIN32_CONSTANT(F_DWORD, GENERIC_READ); WIN32_CONSTANT(F_DWORD, GENERIC_WRITE); WIN32_CONSTANT(F_DWORD, INFINITE); diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 001360b..adc35f1 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1541,7 +1541,10 @@ save_long(PicklerObject *self, PyObject *obj) PyErr_Clear(); } else - return save_int(self, val); +#if SIZEOF_LONG > 4 + if (val <= 0x7fffffffL && val >= -0x80000000L) +#endif + return save_int(self, val); if (self->proto >= 2) { /* Linear-time pickling. */ @@ -2841,6 +2844,28 @@ save_pers(PicklerObject *self, PyObject *obj, PyObject *func) return status; } +static PyObject * +get_class(PyObject *obj) +{ + PyObject *cls; + static PyObject *str_class; + + if (str_class == NULL) { + str_class = PyUnicode_InternFromString("__class__"); + if (str_class == NULL) + return NULL; + } + cls = PyObject_GetAttr(obj, str_class); + if (cls == NULL) { + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + PyErr_Clear(); + cls = (PyObject *) Py_TYPE(obj); + Py_INCREF(cls); + } + } + return cls; +} + /* We're saving obj, and args is the 2-thru-5 tuple returned by the * appropriate __reduce__ method for obj. */ @@ -2906,17 +2931,18 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) /* Protocol 2 special case: if callable's name is __newobj__, use NEWOBJ. */ if (use_newobj) { - static PyObject *newobj_str = NULL; - PyObject *name_str; + static PyObject *newobj_str = NULL, *name_str = NULL; + PyObject *name; if (newobj_str == NULL) { newobj_str = PyUnicode_InternFromString("__newobj__"); - if (newobj_str == NULL) + name_str = PyUnicode_InternFromString("__name__"); + if (newobj_str == NULL || name_str == NULL) return -1; } - name_str = PyObject_GetAttrString(callable, "__name__"); - if (name_str == NULL) { + name = PyObject_GetAttr(callable, name_str); + if (name == NULL) { if (PyErr_ExceptionMatches(PyExc_AttributeError)) PyErr_Clear(); else @@ -2924,9 +2950,9 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) use_newobj = 0; } else { - use_newobj = PyUnicode_Check(name_str) && - PyUnicode_Compare(name_str, newobj_str) == 0; - Py_DECREF(name_str); + use_newobj = PyUnicode_Check(name) && + PyUnicode_Compare(name, newobj_str) == 0; + Py_DECREF(name); } } if (use_newobj) { @@ -2942,20 +2968,14 @@ save_reduce(PicklerObject *self, PyObject *args, PyObject *obj) } cls = PyTuple_GET_ITEM(argtup, 0); - if (!PyObject_HasAttrString(cls, "__new__")) { + if (!PyType_Check(cls)) { PyErr_SetString(PicklingError, "args[0] from " - "__newobj__ args has no __new__"); + "__newobj__ args is not a type"); return -1; } if (obj != NULL) { - obj_class = PyObject_GetAttrString(obj, "__class__"); - if (obj_class == NULL) { - if (PyErr_ExceptionMatches(PyExc_AttributeError)) - PyErr_Clear(); - else - return -1; - } + obj_class = get_class(obj); p = obj_class != cls; /* true iff a problem */ Py_DECREF(obj_class); if (p) { diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index 11b24a0..bc2f72c 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -1,7 +1,7 @@ /* Authors: Gregory P. Smith & Jeffrey Yasskin */ #include "Python.h" -#ifdef HAVE_PIPE2 -#define _GNU_SOURCE +#if defined(HAVE_PIPE2) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE #endif #include <unistd.h> #include <fcntl.h> diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 7d12d5e..3c83c7c 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -675,7 +675,6 @@ void _pysqlite_final_callback(sqlite3_context* context) { PyObject* function_result = NULL; PyObject** aggregate_instance; - PyObject* aggregate_class; #ifdef WITH_THREAD PyGILState_STATE threadstate; @@ -683,8 +682,6 @@ void _pysqlite_final_callback(sqlite3_context* context) threadstate = PyGILState_Ensure(); #endif - aggregate_class = (PyObject*)sqlite3_user_data(context); - aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); if (!*aggregate_instance) { /* this branch is executed if there was an exception in the aggregate's @@ -909,6 +906,38 @@ static int _progress_handler(void* user_arg) return rc; } +static void _trace_callback(void* user_arg, const char* statement_string) +{ + PyObject *py_statement = NULL; + PyObject *ret = NULL; + +#ifdef WITH_THREAD + PyGILState_STATE gilstate; + + gilstate = PyGILState_Ensure(); +#endif + py_statement = PyUnicode_DecodeUTF8(statement_string, + strlen(statement_string), "replace"); + if (py_statement) { + ret = PyObject_CallFunctionObjArgs((PyObject*)user_arg, py_statement, NULL); + Py_DECREF(py_statement); + } + + if (ret) { + Py_DECREF(ret); + } else { + if (_enable_callback_tracebacks) { + PyErr_Print(); + } else { + PyErr_Clear(); + } + } + +#ifdef WITH_THREAD + PyGILState_Release(gilstate); +#endif +} + static PyObject* pysqlite_connection_set_authorizer(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) { PyObject* authorizer_cb; @@ -968,6 +997,34 @@ static PyObject* pysqlite_connection_set_progress_handler(pysqlite_Connection* s return Py_None; } +static PyObject* pysqlite_connection_set_trace_callback(pysqlite_Connection* self, PyObject* args, PyObject* kwargs) +{ + PyObject* trace_callback; + + static char *kwlist[] = { "trace_callback", NULL }; + + if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:set_trace_callback", + kwlist, &trace_callback)) { + return NULL; + } + + if (trace_callback == Py_None) { + /* None clears the trace callback previously set */ + sqlite3_trace(self->db, 0, (void*)0); + } else { + if (PyDict_SetItem(self->function_pinboard, trace_callback, Py_None) == -1) + return NULL; + sqlite3_trace(self->db, _trace_callback, trace_callback); + } + + Py_INCREF(Py_None); + return Py_None; +} + #ifdef HAVE_LOAD_EXTENSION static PyObject* pysqlite_enable_load_extension(pysqlite_Connection* self, PyObject* args) { @@ -1521,6 +1578,8 @@ static PyMethodDef connection_methods[] = { #endif {"set_progress_handler", (PyCFunction)pysqlite_connection_set_progress_handler, METH_VARARGS|METH_KEYWORDS, PyDoc_STR("Sets progress handler callback. Non-standard.")}, + {"set_trace_callback", (PyCFunction)pysqlite_connection_set_trace_callback, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("Sets a trace callback called for each SQL statement (passed as unicode). Non-standard.")}, {"execute", (PyCFunction)pysqlite_connection_execute, METH_VARARGS, PyDoc_STR("Executes a SQL statement. Non-standard.")}, {"executemany", (PyCFunction)pysqlite_connection_executemany, METH_VARARGS, diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 97908a3..7f5c213 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -126,11 +126,9 @@ static int pysqlite_cursor_init(pysqlite_Cursor* self, PyObject* args, PyObject* static void pysqlite_cursor_dealloc(pysqlite_Cursor* self) { - int rc; - /* Reset the statement if the user has not closed the cursor */ if (self->statement) { - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); Py_DECREF(self->statement); } @@ -432,9 +430,14 @@ static int check_cursor(pysqlite_Cursor* cur) if (cur->closed) { PyErr_SetString(pysqlite_ProgrammingError, "Cannot operate on a closed cursor."); return 0; - } else { - return pysqlite_check_thread(cur->connection) && pysqlite_check_connection(cur->connection); } + + if (cur->locked) { + PyErr_SetString(pysqlite_ProgrammingError, "Recursive use of cursors not allowed."); + return 0; + } + + return pysqlite_check_thread(cur->connection) && pysqlite_check_connection(cur->connection); } PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* args) @@ -457,9 +460,10 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* int allow_8bit_chars; if (!check_cursor(self)) { - return NULL; + goto error; } + self->locked = 1; self->reset = 0; /* Make shooting yourself in the foot with not utf-8 decodable 8-bit-strings harder */ @@ -472,12 +476,12 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* if (multiple) { /* executemany() */ if (!PyArg_ParseTuple(args, "OO", &operation, &second_argument)) { - return NULL; + goto error; } if (!PyUnicode_Check(operation)) { PyErr_SetString(PyExc_ValueError, "operation parameter must be str"); - return NULL; + goto error; } if (PyIter_Check(second_argument)) { @@ -488,23 +492,23 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* /* sequence */ parameters_iter = PyObject_GetIter(second_argument); if (!parameters_iter) { - return NULL; + goto error; } } } else { /* execute() */ if (!PyArg_ParseTuple(args, "O|O", &operation, &second_argument)) { - return NULL; + goto error; } if (!PyUnicode_Check(operation)) { PyErr_SetString(PyExc_ValueError, "operation parameter must be str"); - return NULL; + goto error; } parameters_list = PyList_New(0); if (!parameters_list) { - return NULL; + goto error; } if (second_argument == NULL) { @@ -529,7 +533,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* if (self->statement != NULL) { /* There is an active statement */ - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); } operation_cstr = _PyUnicode_AsStringAndSize(operation, &operation_len); @@ -734,7 +738,7 @@ PyObject* _pysqlite_query_execute(pysqlite_Cursor* self, int multiple, PyObject* } if (multiple) { - rc = pysqlite_statement_reset(self->statement); + pysqlite_statement_reset(self->statement); } Py_XDECREF(parameters); } @@ -744,7 +748,8 @@ error: * ROLLBACK could have happened */ #ifdef SQLITE_VERSION_NUMBER #if SQLITE_VERSION_NUMBER >= 3002002 - self->connection->inTransaction = !sqlite3_get_autocommit(self->connection->db); + if (self->connection && self->connection->db) + self->connection->inTransaction = !sqlite3_get_autocommit(self->connection->db); #endif #endif @@ -752,6 +757,8 @@ error: Py_XDECREF(parameters_iter); Py_XDECREF(parameters_list); + self->locked = 0; + if (PyErr_Occurred()) { self->rowcount = -1L; return NULL; diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h index 5d8b5c1..118ba38 100644 --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -42,6 +42,7 @@ typedef struct pysqlite_Statement* statement; int closed; int reset; + int locked; int initialized; /* the next row to be returned, NULL if no next row available */ diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c index 3d44094..b50658c 100644 --- a/Modules/_sqlite/row.c +++ b/Modules/_sqlite/row.c @@ -173,10 +173,9 @@ static Py_hash_t pysqlite_row_hash(pysqlite_Row *self) static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other, int opid) { - if (opid != Py_EQ && opid != Py_NE) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (opid != Py_EQ && opid != Py_NE) + Py_RETURN_NOTIMPLEMENTED; + if (PyType_IsSubtype(Py_TYPE(_other), &pysqlite_RowType)) { pysqlite_Row *other = (pysqlite_Row *)_other; PyObject *res = PyObject_RichCompare(self->description, other->description, opid); @@ -186,8 +185,7 @@ static PyObject* pysqlite_row_richcompare(pysqlite_Row *self, PyObject *_other, return PyObject_RichCompare(self->data, other->data, opid); } } - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } PyMappingMethods pysqlite_row_as_mapping = { diff --git a/Modules/_sqlite/statement.c b/Modules/_sqlite/statement.c index f89fc9a..4e039c1 100644 --- a/Modules/_sqlite/statement.c +++ b/Modules/_sqlite/statement.c @@ -369,11 +369,9 @@ void pysqlite_statement_mark_dirty(pysqlite_Statement* self) void pysqlite_statement_dealloc(pysqlite_Statement* self) { - int rc; - if (self->st) { Py_BEGIN_ALLOW_THREADS - rc = sqlite3_finalize(self->st); + sqlite3_finalize(self->st); Py_END_ALLOW_THREADS } diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 27dcdbc..1a367f2 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -124,6 +124,17 @@ static unsigned int _ssl_locks_count = 0; # undef HAVE_SSL_CTX_CLEAR_OPTIONS #endif +/* In case of 'tls-unique' it will be 12 bytes for TLS, 36 bytes for + * older SSL, but let's be safe */ +#define PySSL_CB_MAXLEN 128 + +/* SSL_get_finished got added to OpenSSL in 0.9.5 */ +#if OPENSSL_VERSION_NUMBER >= 0x0090500fL +# define HAVE_OPENSSL_FINISHED 1 +#else +# define HAVE_OPENSSL_FINISHED 0 +#endif + typedef struct { PyObject_HEAD SSL_CTX *ctx; @@ -135,6 +146,7 @@ typedef struct { SSL *ssl; X509 *peer_cert; int shutdown_seen_zero; + enum py_ssl_server_or_client socket_type; } PySSLSocket; static PyTypeObject PySSLContext_Type; @@ -328,6 +340,7 @@ newPySSLSocket(SSL_CTX *ctx, PySocketSockObject *sock, SSL_set_accept_state(self->ssl); PySSL_END_ALLOW_THREADS + self->socket_type = socket_type; self->Socket = PyWeakref_NewRef((PyObject *) sock, NULL); return self; } @@ -356,7 +369,6 @@ static PyObject *PySSL_SSLdo_handshake(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ - sockstate = 0; do { PySSL_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); @@ -1092,7 +1104,6 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) goto error; } do { - err = 0; PySSL_BEGIN_ALLOW_THREADS len = SSL_write(self->ssl, buf.buf, buf.len); err = SSL_get_error(self->ssl, len); @@ -1228,7 +1239,6 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) } } do { - err = 0; PySSL_BEGIN_ALLOW_THREADS count = SSL_read(self->ssl, mem, len); err = SSL_get_error(self->ssl, count); @@ -1380,6 +1390,41 @@ PyDoc_STRVAR(PySSL_SSLshutdown_doc, Does the SSL shutdown handshake with the remote end, and returns\n\ the underlying socket object."); +#if HAVE_OPENSSL_FINISHED +static PyObject * +PySSL_tls_unique_cb(PySSLSocket *self) +{ + PyObject *retval = NULL; + char buf[PySSL_CB_MAXLEN]; + int len; + + if (SSL_session_reused(self->ssl) ^ !self->socket_type) { + /* if session is resumed XOR we are the client */ + len = SSL_get_finished(self->ssl, buf, PySSL_CB_MAXLEN); + } + else { + /* if a new session XOR we are the server */ + len = SSL_get_peer_finished(self->ssl, buf, PySSL_CB_MAXLEN); + } + + /* It cannot be negative in current OpenSSL version as of July 2011 */ + assert(len >= 0); + if (len == 0) + Py_RETURN_NONE; + + retval = PyBytes_FromStringAndSize(buf, len); + + return retval; +} + +PyDoc_STRVAR(PySSL_tls_unique_cb_doc, +"tls_unique_cb() -> bytes\n\ +\n\ +Returns the 'tls-unique' channel binding data, as defined by RFC 5929.\n\ +\n\ +If the TLS handshake is not yet complete, None is returned"); + +#endif /* HAVE_OPENSSL_FINISHED */ static PyMethodDef PySSLMethods[] = { {"do_handshake", (PyCFunction)PySSL_SSLdo_handshake, METH_NOARGS}, @@ -1394,6 +1439,10 @@ static PyMethodDef PySSLMethods[] = { {"cipher", (PyCFunction)PySSL_cipher, METH_NOARGS}, {"shutdown", (PyCFunction)PySSL_SSLshutdown, METH_NOARGS, PySSL_SSLshutdown_doc}, +#if HAVE_OPENSSL_FINISHED + {"tls_unique_cb", (PyCFunction)PySSL_tls_unique_cb, METH_NOARGS, + PySSL_tls_unique_cb_doc}, +#endif {NULL, NULL} }; @@ -1890,6 +1939,69 @@ Mix string into the OpenSSL PRNG state. entropy (a float) is a lower\n\ bound on the entropy contained in string. See RFC 1750."); static PyObject * +PySSL_RAND(int len, int pseudo) +{ + int ok; + PyObject *bytes; + unsigned long err; + const char *errstr; + PyObject *v; + + bytes = PyBytes_FromStringAndSize(NULL, len); + if (bytes == NULL) + return NULL; + if (pseudo) { + ok = RAND_pseudo_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 0 || ok == 1) + return Py_BuildValue("NO", bytes, ok == 1 ? Py_True : Py_False); + } + else { + ok = RAND_bytes((unsigned char*)PyBytes_AS_STRING(bytes), len); + if (ok == 1) + return bytes; + } + Py_DECREF(bytes); + + err = ERR_get_error(); + errstr = ERR_reason_error_string(err); + v = Py_BuildValue("(ks)", err, errstr); + if (v != NULL) { + PyErr_SetObject(PySSLErrorObject, v); + Py_DECREF(v); + } + return NULL; +} + +static PyObject * +PySSL_RAND_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_bytes", &len)) + return NULL; + return PySSL_RAND(len, 0); +} + +PyDoc_STRVAR(PySSL_RAND_bytes_doc, +"RAND_bytes(n) -> bytes\n\ +\n\ +Generate n cryptographically strong pseudo-random bytes."); + +static PyObject * +PySSL_RAND_pseudo_bytes(PyObject *self, PyObject *args) +{ + int len; + if (!PyArg_ParseTuple(args, "i:RAND_pseudo_bytes", &len)) + return NULL; + return PySSL_RAND(len, 1); +} + +PyDoc_STRVAR(PySSL_RAND_pseudo_bytes_doc, +"RAND_pseudo_bytes(n) -> (bytes, is_cryptographic)\n\ +\n\ +Generate n pseudo-random bytes. is_cryptographic is True if the bytes\ +generated are cryptographically strong."); + +static PyObject * PySSL_RAND_status(PyObject *self) { return PyLong_FromLong(RAND_status()); @@ -1942,6 +2054,10 @@ static PyMethodDef PySSL_methods[] = { #ifdef HAVE_OPENSSL_RAND {"RAND_add", PySSL_RAND_add, METH_VARARGS, PySSL_RAND_add_doc}, + {"RAND_bytes", PySSL_RAND_bytes, METH_VARARGS, + PySSL_RAND_bytes_doc}, + {"RAND_pseudo_bytes", PySSL_RAND_pseudo_bytes, METH_VARARGS, + PySSL_RAND_pseudo_bytes_doc}, {"RAND_egd", PySSL_RAND_egd, METH_VARARGS, PySSL_RAND_egd_doc}, {"RAND_status", (PyCFunction)PySSL_RAND_status, METH_NOARGS, @@ -2157,6 +2273,14 @@ PyInit__ssl(void) Py_INCREF(r); PyModule_AddObject(m, "HAS_SNI", r); +#if HAVE_OPENSSL_FINISHED + r = Py_True; +#else + r = Py_False; +#endif + Py_INCREF(r); + PyModule_AddObject(m, "HAS_TLS_UNIQUE", r); + /* OpenSSL version */ /* SSLeay() gives us the version of the library linked against, which could be different from the headers version. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index f19d0df..51c79c9 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -22,14 +22,7 @@ static PyObject *TestError; /* set to exception object in init */ static PyObject * raiseTestError(const char* test_name, const char* msg) { - char buf[2048]; - - if (strlen(test_name) + strlen(msg) > sizeof(buf) - 50) - PyErr_SetString(TestError, "internal error msg too large"); - else { - PyOS_snprintf(buf, sizeof(buf), "%s: %s", test_name, msg); - PyErr_SetString(TestError, buf); - } + PyErr_Format(TestError, "%s: %s", test_name, msg); return NULL; } @@ -43,11 +36,9 @@ static PyObject* sizeof_error(const char* fatname, const char* typname, int expected, int got) { - char buf[1024]; - PyOS_snprintf(buf, sizeof(buf), - "%.200s #define == %d but sizeof(%.200s) == %d", + PyErr_Format(TestError, + "%s #define == %d but sizeof(%s) == %d", fatname, expected, typname, got); - PyErr_SetString(TestError, buf); return (PyObject*)NULL; } @@ -1012,6 +1003,15 @@ test_k_code(PyObject *self) } static PyObject * +getargs_c(PyObject *self, PyObject *args) +{ + char c; + if (!PyArg_ParseTuple(args, "c", &c)) + return NULL; + return PyBytes_FromStringAndSize(&c, 1); +} + +static PyObject * getargs_s(PyObject *self, PyObject *args) { char *str; @@ -2298,6 +2298,7 @@ static PyMethodDef TestMethods[] = { (PyCFunction)test_long_long_and_overflow, METH_NOARGS}, {"test_L_code", (PyCFunction)test_L_code, METH_NOARGS}, #endif + {"getargs_c", getargs_c, METH_VARARGS}, {"getargs_s", getargs_s, METH_VARARGS}, {"getargs_s_star", getargs_s_star, METH_VARARGS}, {"getargs_s_hash", getargs_s_hash, METH_VARARGS}, diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index ea038de..6e39ca0 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -53,6 +53,7 @@ acquire_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds) _PyTime_timeval curtime; _PyTime_timeval endtime; + if (microseconds > 0) { _PyTime_gettimeofday(&endtime); endtime.tv_sec += microseconds / (1000 * 1000); @@ -75,7 +76,7 @@ acquire_timed(PyThread_type_lock lock, PY_TIMEOUT_T microseconds) /* If we're using a timeout, recompute the timeout after processing * signals, since those can take time. */ - if (microseconds >= 0) { + if (microseconds > 0) { _PyTime_gettimeofday(&curtime); microseconds = ((endtime.tv_sec - curtime.tv_sec) * 1000000 + (endtime.tv_usec - curtime.tv_usec)); @@ -413,6 +414,12 @@ rlock_release_save(rlockobject *self) long owner; unsigned long count; + if (self->rlock_count == 0) { + PyErr_SetString(PyExc_RuntimeError, + "cannot release un-acquired lock"); + return NULL; + } + owner = self->rlock_owner; count = self->rlock_count; self->rlock_count = 0; @@ -1222,11 +1229,9 @@ the suggested approach in the absence of more specific information)."); static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, - start_new_doc}, + METH_VARARGS, start_new_doc}, {"start_new", (PyCFunction)thread_PyThread_start_new_thread, - METH_VARARGS, - start_new_doc}, + METH_VARARGS, start_new_doc}, {"allocate_lock", (PyCFunction)thread_PyThread_allocate_lock, METH_NOARGS, allocate_doc}, {"allocate", (PyCFunction)thread_PyThread_allocate_lock, @@ -1242,8 +1247,7 @@ static PyMethodDef thread_methods[] = { {"_count", (PyCFunction)thread__count, METH_NOARGS, _count_doc}, {"stack_size", (PyCFunction)thread_stack_size, - METH_VARARGS, - stack_size_doc}, + METH_VARARGS, stack_size_doc}, {NULL, NULL} /* sentinel */ }; @@ -1307,7 +1311,9 @@ PyInit__thread(void) /* Add a symbolic constant */ d = PyModule_GetDict(m); - ThreadError = PyErr_NewException("_thread.error", NULL, NULL); + ThreadError = PyExc_RuntimeError; + Py_INCREF(ThreadError); + PyDict_SetItemString(d, "error", ThreadError); Locktype.tp_doc = lock_doc; Py_INCREF(&Locktype); diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index cb2795f..6879975 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -2005,7 +2005,7 @@ static int PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, char *argv[]) { PythonCmd_ClientData *data = (PythonCmd_ClientData *)clientData; - PyObject *self, *func, *arg, *res; + PyObject *func, *arg, *res; int i, rv; Tcl_Obj *obj_res; @@ -2014,7 +2014,6 @@ PythonCmd(ClientData clientData, Tcl_Interp *interp, int argc, char *argv[]) /* TBD: no error checking here since we know, via the * Tkapp_CreateCommand() that the client data is a two-tuple */ - self = data->self; func = data->func; /* Create argument list (argv1, ..., argvN) */ @@ -2413,11 +2412,9 @@ static PyObject * Tktt_Repr(PyObject *self) { TkttObject *v = (TkttObject *)self; - char buf[100]; - - PyOS_snprintf(buf, sizeof(buf), "<tktimertoken at %p%s>", v, - v->func == NULL ? ", handler deleted" : ""); - return PyUnicode_FromString(buf); + return PyUnicode_FromFormat("<tktimertoken at %p%s>", + v, + v->func == NULL ? ", handler deleted" : ""); } static PyTypeObject Tktt_Type = diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index fe6106c..81c9c36 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -514,10 +514,8 @@ array_richcompare(PyObject *v, PyObject *w, int op) Py_ssize_t i, k; PyObject *res; - if (!array_Check(v) || !array_Check(w)) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; - } + if (!array_Check(v) || !array_Check(w)) + Py_RETURN_NOTIMPLEMENTED; va = (arrayobject *)v; wa = (arrayobject *)w; @@ -876,7 +874,6 @@ array_inplace_repeat(arrayobject *self, Py_ssize_t n) if (Py_SIZE(self) > 0) { if (n < 0) n = 0; - items = self->ob_item; if ((self->ob_descr->itemsize != 0) && (Py_SIZE(self) > PY_SSIZE_T_MAX / self->ob_descr->itemsize)) { return PyErr_NoMemory(); @@ -2092,7 +2089,7 @@ array_repr(arrayobject *a) if (len == 0) { return PyUnicode_FromFormat("array('%c')", (int)typecode); } - if ((typecode == 'u')) + if (typecode == 'u') v = array_tounicode(a, NULL); else v = array_tolist(a, NULL); diff --git a/Modules/audioop.c b/Modules/audioop.c index a031d42..daf70dc 100644 --- a/Modules/audioop.c +++ b/Modules/audioop.c @@ -513,7 +513,6 @@ audioop_findfit(PyObject *self, PyObject *args) best_result = result; best_j = 0; - j = 0; for ( j=1; j<=len1-len2; j++) { aj_m1 = (double)cp1[j-1]; @@ -599,7 +598,6 @@ audioop_findmax(PyObject *self, PyObject *args) best_result = result; best_j = 0; - j = 0; for ( j=1; j<=len1-len2; j++) { aj_m1 = (double)cp1[j-1]; @@ -1433,7 +1431,6 @@ audioop_lin2adpcm(PyObject *self, PyObject *args) if ( state == Py_None ) { /* First time, it seems. Set defaults */ valpred = 0; - step = 7; index = 0; } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) ) return 0; @@ -1534,7 +1531,6 @@ audioop_adpcm2lin(PyObject *self, PyObject *args) if ( state == Py_None ) { /* First time, it seems. Set defaults */ valpred = 0; - step = 7; index = 0; } else if ( !PyArg_ParseTuple(state, "ii", &valpred, &index) ) return 0; diff --git a/Modules/bz2module.c b/Modules/bz2module.c deleted file mode 100644 index 3e55202..0000000 --- a/Modules/bz2module.c +++ /dev/null @@ -1,2180 +0,0 @@ -/* - -python-bz2 - python bz2 library interface - -Copyright (c) 2002 Gustavo Niemeyer <niemeyer@conectiva.com> -Copyright (c) 2002 Python Software Foundation; All Rights Reserved - -*/ - -#include "Python.h" -#include <stdio.h> -#include <bzlib.h> -#include "structmember.h" - -#ifdef WITH_THREAD -#include "pythread.h" -#endif - -static char __author__[] = -"The bz2 python module was written by:\n\ -\n\ - Gustavo Niemeyer <niemeyer@conectiva.com>\n\ -"; - -/* Our very own off_t-like type, 64-bit if possible */ -/* copied from Objects/fileobject.c */ -#if !defined(HAVE_LARGEFILE_SUPPORT) -typedef off_t Py_off_t; -#elif SIZEOF_OFF_T >= 8 -typedef off_t Py_off_t; -#elif SIZEOF_FPOS_T >= 8 -typedef fpos_t Py_off_t; -#else -#error "Large file support, but neither off_t nor fpos_t is large enough." -#endif - -#define BUF(v) PyBytes_AS_STRING(v) - -#define MODE_CLOSED 0 -#define MODE_READ 1 -#define MODE_READ_EOF 2 -#define MODE_WRITE 3 - -#define BZ2FileObject_Check(v) (Py_TYPE(v) == &BZ2File_Type) - - -#ifdef BZ_CONFIG_ERROR - -#if SIZEOF_LONG >= 8 -#define BZS_TOTAL_OUT(bzs) \ - (((long)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) -#elif SIZEOF_LONG_LONG >= 8 -#define BZS_TOTAL_OUT(bzs) \ - (((PY_LONG_LONG)bzs->total_out_hi32 << 32) + bzs->total_out_lo32) -#else -#define BZS_TOTAL_OUT(bzs) \ - bzs->total_out_lo32 -#endif - -#else /* ! BZ_CONFIG_ERROR */ - -#define BZ2_bzRead bzRead -#define BZ2_bzReadOpen bzReadOpen -#define BZ2_bzReadClose bzReadClose -#define BZ2_bzWrite bzWrite -#define BZ2_bzWriteOpen bzWriteOpen -#define BZ2_bzWriteClose bzWriteClose -#define BZ2_bzCompress bzCompress -#define BZ2_bzCompressInit bzCompressInit -#define BZ2_bzCompressEnd bzCompressEnd -#define BZ2_bzDecompress bzDecompress -#define BZ2_bzDecompressInit bzDecompressInit -#define BZ2_bzDecompressEnd bzDecompressEnd - -#define BZS_TOTAL_OUT(bzs) bzs->total_out - -#endif /* ! BZ_CONFIG_ERROR */ - - -#ifdef WITH_THREAD -#define ACQUIRE_LOCK(obj) do { \ - if (!PyThread_acquire_lock(obj->lock, 0)) { \ - Py_BEGIN_ALLOW_THREADS \ - PyThread_acquire_lock(obj->lock, 1); \ - Py_END_ALLOW_THREADS \ - } } while(0) -#define RELEASE_LOCK(obj) PyThread_release_lock(obj->lock) -#else -#define ACQUIRE_LOCK(obj) -#define RELEASE_LOCK(obj) -#endif - -/* Bits in f_newlinetypes */ -#define NEWLINE_UNKNOWN 0 /* No newline seen, yet */ -#define NEWLINE_CR 1 /* \r newline seen */ -#define NEWLINE_LF 2 /* \n newline seen */ -#define NEWLINE_CRLF 4 /* \r\n newline seen */ - -/* ===================================================================== */ -/* Structure definitions. */ - -typedef struct { - PyObject_HEAD - FILE *rawfp; - - char* f_buf; /* Allocated readahead buffer */ - char* f_bufend; /* Points after last occupied position */ - char* f_bufptr; /* Current buffer position */ - - BZFILE *fp; - int mode; - Py_off_t pos; - Py_off_t size; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2FileObject; - -typedef struct { - PyObject_HEAD - bz_stream bzs; - int running; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2CompObject; - -typedef struct { - PyObject_HEAD - bz_stream bzs; - int running; - PyObject *unused_data; -#ifdef WITH_THREAD - PyThread_type_lock lock; -#endif -} BZ2DecompObject; - -/* ===================================================================== */ -/* Utility functions. */ - -/* Refuse regular I/O if there's data in the iteration-buffer. - * Mixing them would cause data to arrive out of order, as the read* - * methods don't use the iteration buffer. */ -static int -check_iterbuffered(BZ2FileObject *f) -{ - if (f->f_buf != NULL && - (f->f_bufend - f->f_bufptr) > 0 && - f->f_buf[0] != '\0') { - PyErr_SetString(PyExc_ValueError, - "Mixing iteration and read methods would lose data"); - return -1; - } - return 0; -} - -static int -Util_CatchBZ2Error(int bzerror) -{ - int ret = 0; - switch(bzerror) { - case BZ_OK: - case BZ_STREAM_END: - break; - -#ifdef BZ_CONFIG_ERROR - case BZ_CONFIG_ERROR: - PyErr_SetString(PyExc_SystemError, - "the bz2 library was not compiled " - "correctly"); - ret = 1; - break; -#endif - - case BZ_PARAM_ERROR: - PyErr_SetString(PyExc_ValueError, - "the bz2 library has received wrong " - "parameters"); - ret = 1; - break; - - case BZ_MEM_ERROR: - PyErr_NoMemory(); - ret = 1; - break; - - case BZ_DATA_ERROR: - case BZ_DATA_ERROR_MAGIC: - PyErr_SetString(PyExc_IOError, "invalid data stream"); - ret = 1; - break; - - case BZ_IO_ERROR: - PyErr_SetString(PyExc_IOError, "unknown IO error"); - ret = 1; - break; - - case BZ_UNEXPECTED_EOF: - PyErr_SetString(PyExc_EOFError, - "compressed file ended before the " - "logical end-of-stream was detected"); - ret = 1; - break; - - case BZ_SEQUENCE_ERROR: - PyErr_SetString(PyExc_RuntimeError, - "wrong sequence of bz2 library " - "commands used"); - ret = 1; - break; - } - return ret; -} - -#if BUFSIZ < 8192 -#define SMALLCHUNK 8192 -#else -#define SMALLCHUNK BUFSIZ -#endif - -#if SIZEOF_INT < 4 -#define BIGCHUNK (512 * 32) -#else -#define BIGCHUNK (512 * 1024) -#endif - -/* This is a hacked version of Python's fileobject.c:new_buffersize(). */ -static size_t -Util_NewBufferSize(size_t currentsize) -{ - if (currentsize > SMALLCHUNK) { - /* Keep doubling until we reach BIGCHUNK; - then keep adding BIGCHUNK. */ - if (currentsize <= BIGCHUNK) - return currentsize + currentsize; - else - return currentsize + BIGCHUNK; - } - return currentsize + SMALLCHUNK; -} - -/* This is a hacked version of Python's fileobject.c:get_line(). */ -static PyObject * -Util_GetLine(BZ2FileObject *f, int n) -{ - char c; - char *buf, *end; - size_t total_v_size; /* total # of slots in buffer */ - size_t used_v_size; /* # used slots in buffer */ - size_t increment; /* amount to increment the buffer */ - PyObject *v; - int bzerror; - int bytes_read; - - total_v_size = n > 0 ? n : 100; - v = PyBytes_FromStringAndSize((char *)NULL, total_v_size); - if (v == NULL) - return NULL; - - buf = BUF(v); - end = buf + total_v_size; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - do { - bytes_read = BZ2_bzRead(&bzerror, f->fp, &c, 1); - f->pos++; - if (bytes_read == 0) - break; - *buf++ = c; - } while (bzerror == BZ_OK && c != '\n' && buf != end); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - f->size = f->pos; - f->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(v); - return NULL; - } - if (c == '\n') - break; - /* Must be because buf == end */ - if (n > 0) - break; - used_v_size = total_v_size; - increment = total_v_size >> 2; /* mild exponential growth */ - total_v_size += increment; - if (total_v_size > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "line is longer than a Python string can hold"); - Py_DECREF(v); - return NULL; - } - if (_PyBytes_Resize(&v, total_v_size) < 0) { - return NULL; - } - buf = BUF(v) + used_v_size; - end = BUF(v) + total_v_size; - } - - used_v_size = buf - BUF(v); - if (used_v_size != total_v_size) { - if (_PyBytes_Resize(&v, used_v_size) < 0) { - v = NULL; - } - } - return v; -} - -/* This is a hacked version of Python's fileobject.c:drop_readahead(). */ -static void -Util_DropReadAhead(BZ2FileObject *f) -{ - if (f->f_buf != NULL) { - PyMem_Free(f->f_buf); - f->f_buf = NULL; - } -} - -/* This is a hacked version of Python's fileobject.c:readahead(). */ -static int -Util_ReadAhead(BZ2FileObject *f, int bufsize) -{ - int chunksize; - int bzerror; - - if (f->f_buf != NULL) { - if((f->f_bufend - f->f_bufptr) >= 1) - return 0; - else - Util_DropReadAhead(f); - } - if (f->mode == MODE_READ_EOF) { - f->f_bufptr = f->f_buf; - f->f_bufend = f->f_buf; - return 0; - } - if ((f->f_buf = PyMem_Malloc(bufsize)) == NULL) { - PyErr_NoMemory(); - return -1; - } - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, f->fp, f->f_buf, bufsize); - Py_END_ALLOW_THREADS - f->pos += chunksize; - if (bzerror == BZ_STREAM_END) { - f->size = f->pos; - f->mode = MODE_READ_EOF; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Util_DropReadAhead(f); - return -1; - } - f->f_bufptr = f->f_buf; - f->f_bufend = f->f_buf + chunksize; - return 0; -} - -/* This is a hacked version of Python's - * fileobject.c:readahead_get_line_skip(). */ -static PyBytesObject * -Util_ReadAheadGetLineSkip(BZ2FileObject *f, int skip, int bufsize) -{ - PyBytesObject* s; - char *bufptr; - char *buf; - int len; - - if (f->f_buf == NULL) - if (Util_ReadAhead(f, bufsize) < 0) - return NULL; - - len = f->f_bufend - f->f_bufptr; - if (len == 0) - return (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, skip); - bufptr = memchr(f->f_bufptr, '\n', len); - if (bufptr != NULL) { - bufptr++; /* Count the '\n' */ - len = bufptr - f->f_bufptr; - s = (PyBytesObject *) - PyBytes_FromStringAndSize(NULL, skip+len); - if (s == NULL) - return NULL; - memcpy(PyBytes_AS_STRING(s)+skip, f->f_bufptr, len); - f->f_bufptr = bufptr; - if (bufptr == f->f_bufend) - Util_DropReadAhead(f); - } else { - bufptr = f->f_bufptr; - buf = f->f_buf; - f->f_buf = NULL; /* Force new readahead buffer */ - s = Util_ReadAheadGetLineSkip(f, skip+len, - bufsize + (bufsize>>2)); - if (s == NULL) { - PyMem_Free(buf); - return NULL; - } - memcpy(PyBytes_AS_STRING(s)+skip, bufptr, len); - PyMem_Free(buf); - } - return s; -} - -/* ===================================================================== */ -/* Methods of BZ2File. */ - -PyDoc_STRVAR(BZ2File_read__doc__, -"read([size]) -> string\n\ -\n\ -Read at most size uncompressed bytes, returned as a string. If the size\n\ -argument is negative or omitted, read until EOF is reached.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_read(). */ -static PyObject * -BZ2File_read(BZ2FileObject *self, PyObject *args) -{ - long bytesrequested = -1; - size_t bytesread, buffersize, chunksize; - int bzerror; - PyObject *ret = NULL; - - if (!PyArg_ParseTuple(args, "|l:read", &bytesrequested)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - ret = PyBytes_FromStringAndSize("", 0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if (bytesrequested < 0) - buffersize = Util_NewBufferSize((size_t)0); - else - buffersize = bytesrequested; - if (buffersize > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "requested number of bytes is " - "more than a Python string can hold"); - goto cleanup; - } - ret = PyBytes_FromStringAndSize((char *)NULL, buffersize); - if (ret == NULL || buffersize == 0) - goto cleanup; - bytesread = 0; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, - BUF(ret)+bytesread, - buffersize-bytesread); - self->pos += chunksize; - Py_END_ALLOW_THREADS - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(ret); - ret = NULL; - goto cleanup; - } - if (bytesrequested < 0) { - buffersize = Util_NewBufferSize(buffersize); - if (_PyBytes_Resize(&ret, buffersize) < 0) { - ret = NULL; - goto cleanup; - } - } else { - break; - } - } - if (bytesread != buffersize) { - if (_PyBytes_Resize(&ret, bytesread) < 0) { - ret = NULL; - } - } - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_readline__doc__, -"readline([size]) -> string\n\ -\n\ -Return the next line from the file, as a string, retaining newline.\n\ -A non-negative size argument will limit the maximum number of bytes to\n\ -return (an incomplete line may be returned then). Return an empty\n\ -string at EOF.\n\ -"); - -static PyObject * -BZ2File_readline(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - int sizehint = -1; - - if (!PyArg_ParseTuple(args, "|i:readline", &sizehint)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - ret = PyBytes_FromStringAndSize("", 0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if (sizehint == 0) - ret = PyBytes_FromStringAndSize("", 0); - else - ret = Util_GetLine(self, (sizehint < 0) ? 0 : sizehint); - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_readlines__doc__, -"readlines([size]) -> list\n\ -\n\ -Call readline() repeatedly and return a list of lines read.\n\ -The optional size argument, if given, is an approximate bound on the\n\ -total number of bytes in the lines returned.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_readlines(). */ -static PyObject * -BZ2File_readlines(BZ2FileObject *self, PyObject *args) -{ - long sizehint = 0; - PyObject *list = NULL; - PyObject *line; - char small_buffer[SMALLCHUNK]; - char *buffer = small_buffer; - size_t buffersize = SMALLCHUNK; - PyObject *big_buffer = NULL; - size_t nfilled = 0; - size_t nread; - size_t totalread = 0; - char *p, *q, *end; - int err; - int shortread = 0; - int bzerror; - - if (!PyArg_ParseTuple(args, "|l:readlines", &sizehint)) - return NULL; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - break; - case MODE_READ_EOF: - list = PyList_New(0); - goto cleanup; - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for reading"); - goto cleanup; - } - - /* refuse to mix with f.next() */ - if (check_iterbuffered(self)) - goto cleanup; - - if ((list = PyList_New(0)) == NULL) - goto cleanup; - - for (;;) { - Py_BEGIN_ALLOW_THREADS - nread = BZ2_bzRead(&bzerror, self->fp, - buffer+nfilled, buffersize-nfilled); - self->pos += nread; - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - if (nread == 0) { - sizehint = 0; - break; - } - shortread = 1; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - error: - Py_DECREF(list); - list = NULL; - goto cleanup; - } - totalread += nread; - p = memchr(buffer+nfilled, '\n', nread); - if (!shortread && p == NULL) { - /* Need a larger buffer to fit this line */ - nfilled += nread; - buffersize *= 2; - if (buffersize > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, - "line is longer than a Python string can hold"); - goto error; - } - if (big_buffer == NULL) { - /* Create the big buffer */ - big_buffer = PyBytes_FromStringAndSize( - NULL, buffersize); - if (big_buffer == NULL) - goto error; - buffer = PyBytes_AS_STRING(big_buffer); - memcpy(buffer, small_buffer, nfilled); - } - else { - /* Grow the big buffer */ - if (_PyBytes_Resize(&big_buffer, buffersize) < 0){ - big_buffer = NULL; - goto error; - } - buffer = PyBytes_AS_STRING(big_buffer); - } - continue; - } - end = buffer+nfilled+nread; - q = buffer; - while (p != NULL) { - /* Process complete lines */ - p++; - line = PyBytes_FromStringAndSize(q, p-q); - if (line == NULL) - goto error; - err = PyList_Append(list, line); - Py_DECREF(line); - if (err != 0) - goto error; - q = p; - p = memchr(q, '\n', end-q); - } - /* Move the remaining incomplete line to the start */ - nfilled = end-q; - memmove(buffer, q, nfilled); - if (sizehint > 0) - if (totalread >= (size_t)sizehint) - break; - if (shortread) { - sizehint = 0; - break; - } - } - if (nfilled != 0) { - /* Partial last line */ - line = PyBytes_FromStringAndSize(buffer, nfilled); - if (line == NULL) - goto error; - if (sizehint > 0) { - /* Need to complete the last line */ - PyObject *rest = Util_GetLine(self, 0); - if (rest == NULL) { - Py_DECREF(line); - goto error; - } - PyBytes_Concat(&line, rest); - Py_DECREF(rest); - if (line == NULL) - goto error; - } - err = PyList_Append(list, line); - Py_DECREF(line); - if (err != 0) - goto error; - } - - cleanup: - RELEASE_LOCK(self); - if (big_buffer) { - Py_DECREF(big_buffer); - } - return list; -} - -PyDoc_STRVAR(BZ2File_write__doc__, -"write(data) -> None\n\ -\n\ -Write the 'data' string to file. Note that due to buffering, close() may\n\ -be needed before the file on disk reflects the data written.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_write(). */ -static PyObject * -BZ2File_write(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - Py_buffer pbuf; - char *buf; - int len; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:write", &pbuf)) - return NULL; - buf = pbuf.buf; - len = pbuf.len; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_WRITE: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for writing"); - goto cleanup; - } - - Py_BEGIN_ALLOW_THREADS - BZ2_bzWrite (&bzerror, self->fp, buf, len); - self->pos += len; - Py_END_ALLOW_THREADS - - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - - Py_INCREF(Py_None); - ret = Py_None; - -cleanup: - PyBuffer_Release(&pbuf); - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_writelines__doc__, -"writelines(sequence_of_strings) -> None\n\ -\n\ -Write the sequence of strings to the file. Note that newlines are not\n\ -added. The sequence can be any iterable object producing strings. This is\n\ -equivalent to calling write() for each string.\n\ -"); - -/* This is a hacked version of Python's fileobject.c:file_writelines(). */ -static PyObject * -BZ2File_writelines(BZ2FileObject *self, PyObject *seq) -{ -#define CHUNKSIZE 1000 - PyObject *list = NULL; - PyObject *iter = NULL; - PyObject *ret = NULL; - PyObject *line; - int i, j, index, len, islist; - int bzerror; - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_WRITE: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto error; - - default: - PyErr_SetString(PyExc_IOError, - "file is not ready for writing"); - goto error; - } - - islist = PyList_Check(seq); - if (!islist) { - iter = PyObject_GetIter(seq); - if (iter == NULL) { - PyErr_SetString(PyExc_TypeError, - "writelines() requires an iterable argument"); - goto error; - } - list = PyList_New(CHUNKSIZE); - if (list == NULL) - goto error; - } - - /* Strategy: slurp CHUNKSIZE lines into a private list, - checking that they are all strings, then write that list - without holding the interpreter lock, then come back for more. */ - for (index = 0; ; index += CHUNKSIZE) { - if (islist) { - Py_XDECREF(list); - list = PyList_GetSlice(seq, index, index+CHUNKSIZE); - if (list == NULL) - goto error; - j = PyList_GET_SIZE(list); - } - else { - for (j = 0; j < CHUNKSIZE; j++) { - line = PyIter_Next(iter); - if (line == NULL) { - if (PyErr_Occurred()) - goto error; - break; - } - PyList_SetItem(list, j, line); - } - } - if (j == 0) - break; - - /* Check that all entries are indeed byte strings. If not, - apply the same rules as for file.write() and - convert the rets to strings. This is slow, but - seems to be the only way since all conversion APIs - could potentially execute Python code. */ - for (i = 0; i < j; i++) { - PyObject *v = PyList_GET_ITEM(list, i); - if (!PyBytes_Check(v)) { - const char *buffer; - Py_ssize_t len; - if (PyObject_AsCharBuffer(v, &buffer, &len)) { - PyErr_SetString(PyExc_TypeError, - "writelines() " - "argument must be " - "a sequence of " - "bytes objects"); - goto error; - } - line = PyBytes_FromStringAndSize(buffer, - len); - if (line == NULL) - goto error; - Py_DECREF(v); - PyList_SET_ITEM(list, i, line); - } - } - - /* Since we are releasing the global lock, the - following code may *not* execute Python code. */ - Py_BEGIN_ALLOW_THREADS - for (i = 0; i < j; i++) { - line = PyList_GET_ITEM(list, i); - len = PyBytes_GET_SIZE(line); - BZ2_bzWrite (&bzerror, self->fp, - PyBytes_AS_STRING(line), len); - if (bzerror != BZ_OK) { - Py_BLOCK_THREADS - Util_CatchBZ2Error(bzerror); - goto error; - } - } - Py_END_ALLOW_THREADS - - if (j < CHUNKSIZE) - break; - } - - Py_INCREF(Py_None); - ret = Py_None; - - error: - RELEASE_LOCK(self); - Py_XDECREF(list); - Py_XDECREF(iter); - return ret; -#undef CHUNKSIZE -} - -PyDoc_STRVAR(BZ2File_seek__doc__, -"seek(offset [, whence]) -> None\n\ -\n\ -Move to new file position. Argument offset is a byte count. Optional\n\ -argument whence defaults to 0 (offset from start of file, offset\n\ -should be >= 0); other values are 1 (move relative to current position,\n\ -positive or negative), and 2 (move relative to end of file, usually\n\ -negative, although many platforms allow seeking beyond the end of a file).\n\ -\n\ -Note that seeking of bz2 files is emulated, and depending on the parameters\n\ -the operation may be extremely slow.\n\ -"); - -static PyObject * -BZ2File_seek(BZ2FileObject *self, PyObject *args) -{ - int where = 0; - PyObject *offobj; - Py_off_t offset; - char small_buffer[SMALLCHUNK]; - char *buffer = small_buffer; - size_t buffersize = SMALLCHUNK; - Py_off_t bytesread = 0; - size_t readsize; - int chunksize; - int bzerror; - PyObject *ret = NULL; - - if (!PyArg_ParseTuple(args, "O|i:seek", &offobj, &where)) - return NULL; -#if !defined(HAVE_LARGEFILE_SUPPORT) - offset = PyLong_AsLong(offobj); -#else - offset = PyLong_Check(offobj) ? - PyLong_AsLongLong(offobj) : PyLong_AsLong(offobj); -#endif - if (PyErr_Occurred()) - return NULL; - - ACQUIRE_LOCK(self); - Util_DropReadAhead(self); - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - break; - - case MODE_CLOSED: - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - - default: - PyErr_SetString(PyExc_IOError, - "seek works only while reading"); - goto cleanup; - } - - if (where == 2) { - if (self->size == -1) { - assert(self->mode != MODE_READ_EOF); - for (;;) { - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, - buffer, buffersize); - self->pos += chunksize; - Py_END_ALLOW_THREADS - - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - } - self->mode = MODE_READ_EOF; - self->size = self->pos; - bytesread = 0; - } - offset = self->size + offset; - } else if (where == 1) { - offset = self->pos + offset; - } - - /* Before getting here, offset must be the absolute position the file - * pointer should be set to. */ - - if (offset >= self->pos) { - /* we can move forward */ - offset -= self->pos; - } else { - /* we cannot move back, so rewind the stream */ - BZ2_bzReadClose(&bzerror, self->fp); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - rewind(self->rawfp); - self->pos = 0; - self->fp = BZ2_bzReadOpen(&bzerror, self->rawfp, - 0, 0, NULL, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - self->mode = MODE_READ; - } - - if (offset <= 0 || self->mode == MODE_READ_EOF) - goto exit; - - /* Before getting here, offset must be set to the number of bytes - * to walk forward. */ - for (;;) { - if (offset-bytesread > buffersize) - readsize = buffersize; - else - /* offset might be wider that readsize, but the result - * of the subtraction is bound by buffersize (see the - * condition above). buffersize is 8192. */ - readsize = (size_t)(offset-bytesread); - Py_BEGIN_ALLOW_THREADS - chunksize = BZ2_bzRead(&bzerror, self->fp, buffer, readsize); - self->pos += chunksize; - Py_END_ALLOW_THREADS - bytesread += chunksize; - if (bzerror == BZ_STREAM_END) { - self->size = self->pos; - self->mode = MODE_READ_EOF; - break; - } else if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto cleanup; - } - if (bytesread == offset) - break; - } - -exit: - Py_INCREF(Py_None); - ret = Py_None; - -cleanup: - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_tell__doc__, -"tell() -> int\n\ -\n\ -Return the current file position, an integer (may be a long integer).\n\ -"); - -static PyObject * -BZ2File_tell(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = NULL; - - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - goto cleanup; - } - -#if !defined(HAVE_LARGEFILE_SUPPORT) - ret = PyLong_FromLong(self->pos); -#else - ret = PyLong_FromLongLong(self->pos); -#endif - -cleanup: - return ret; -} - -PyDoc_STRVAR(BZ2File_close__doc__, -"close() -> None or (perhaps) an integer\n\ -\n\ -Close the file. Sets data attribute .closed to true. A closed file\n\ -cannot be used for further I/O operations. close() may be called more\n\ -than once without error.\n\ -"); - -static PyObject * -BZ2File_close(BZ2FileObject *self) -{ - PyObject *ret = NULL; - int bzerror = BZ_OK; - - if (self->mode == MODE_CLOSED) { - Py_RETURN_NONE; - } - - ACQUIRE_LOCK(self); - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - BZ2_bzReadClose(&bzerror, self->fp); - break; - case MODE_WRITE: - BZ2_bzWriteClose(&bzerror, self->fp, - 0, NULL, NULL); - break; - } - self->mode = MODE_CLOSED; - fclose(self->rawfp); - self->rawfp = NULL; - if (bzerror == BZ_OK) { - Py_INCREF(Py_None); - ret = Py_None; - } - else { - Util_CatchBZ2Error(bzerror); - } - - RELEASE_LOCK(self); - return ret; -} - -PyDoc_STRVAR(BZ2File_enter_doc, -"__enter__() -> self."); - -static PyObject * -BZ2File_enter(BZ2FileObject *self) -{ - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - Py_INCREF(self); - return (PyObject *) self; -} - -PyDoc_STRVAR(BZ2File_exit_doc, -"__exit__(*excinfo) -> None. Closes the file."); - -static PyObject * -BZ2File_exit(BZ2FileObject *self, PyObject *args) -{ - PyObject *ret = PyObject_CallMethod((PyObject *) self, "close", NULL); - if (!ret) - /* If error occurred, pass through */ - return NULL; - Py_DECREF(ret); - Py_RETURN_NONE; -} - - -static PyObject *BZ2File_getiter(BZ2FileObject *self); - -static PyMethodDef BZ2File_methods[] = { - {"read", (PyCFunction)BZ2File_read, METH_VARARGS, BZ2File_read__doc__}, - {"readline", (PyCFunction)BZ2File_readline, METH_VARARGS, BZ2File_readline__doc__}, - {"readlines", (PyCFunction)BZ2File_readlines, METH_VARARGS, BZ2File_readlines__doc__}, - {"write", (PyCFunction)BZ2File_write, METH_VARARGS, BZ2File_write__doc__}, - {"writelines", (PyCFunction)BZ2File_writelines, METH_O, BZ2File_writelines__doc__}, - {"seek", (PyCFunction)BZ2File_seek, METH_VARARGS, BZ2File_seek__doc__}, - {"tell", (PyCFunction)BZ2File_tell, METH_NOARGS, BZ2File_tell__doc__}, - {"close", (PyCFunction)BZ2File_close, METH_NOARGS, BZ2File_close__doc__}, - {"__enter__", (PyCFunction)BZ2File_enter, METH_NOARGS, BZ2File_enter_doc}, - {"__exit__", (PyCFunction)BZ2File_exit, METH_VARARGS, BZ2File_exit_doc}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Getters and setters of BZ2File. */ - -static PyObject * -BZ2File_get_closed(BZ2FileObject *self, void *closure) -{ - return PyLong_FromLong(self->mode == MODE_CLOSED); -} - -static PyGetSetDef BZ2File_getset[] = { - {"closed", (getter)BZ2File_get_closed, NULL, - "True if the file is closed"}, - {NULL} /* Sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2File_Type. */ - -static int -BZ2File_init(BZ2FileObject *self, PyObject *args, PyObject *kwargs) -{ - static char *kwlist[] = {"filename", "mode", "buffering", - "compresslevel", 0}; - PyObject *name_obj = NULL; - char *name; - char *mode = "r"; - int buffering = -1; - int compresslevel = 9; - int bzerror; - int mode_char = 0; - - self->size = -1; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|sii:BZ2File", - kwlist, PyUnicode_FSConverter, &name_obj, - &mode, &buffering, - &compresslevel)) - return -1; - - name = PyBytes_AsString(name_obj); - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - Py_DECREF(name_obj); - return -1; - } - - for (;;) { - int error = 0; - switch (*mode) { - case 'r': - case 'w': - if (mode_char) - error = 1; - mode_char = *mode; - break; - - case 'b': - break; - - default: - error = 1; - break; - } - if (error) { - PyErr_Format(PyExc_ValueError, - "invalid mode char %c", *mode); - Py_DECREF(name_obj); - return -1; - } - mode++; - if (*mode == '\0') - break; - } - - if (mode_char == 0) { - mode_char = 'r'; - } - - mode = (mode_char == 'r') ? "rb" : "wb"; - - self->rawfp = fopen(name, mode); - Py_DECREF(name_obj); - if (self->rawfp == NULL) { - PyErr_SetFromErrno(PyExc_IOError); - return -1; - } - /* XXX Ignore buffering */ - - /* From now on, we have stuff to dealloc, so jump to error label - * instead of returning */ - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - if (mode_char == 'r') - self->fp = BZ2_bzReadOpen(&bzerror, self->rawfp, - 0, 0, NULL, 0); - else - self->fp = BZ2_bzWriteOpen(&bzerror, self->rawfp, - compresslevel, 0, 0); - - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->mode = (mode_char == 'r') ? MODE_READ : MODE_WRITE; - - return 0; - -error: - fclose(self->rawfp); - self->rawfp = NULL; -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - return -1; -} - -static void -BZ2File_dealloc(BZ2FileObject *self) -{ - int bzerror; -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - switch (self->mode) { - case MODE_READ: - case MODE_READ_EOF: - BZ2_bzReadClose(&bzerror, self->fp); - break; - case MODE_WRITE: - BZ2_bzWriteClose(&bzerror, self->fp, - 0, NULL, NULL); - break; - } - Util_DropReadAhead(self); - if (self->rawfp != NULL) - fclose(self->rawfp); - Py_TYPE(self)->tp_free((PyObject *)self); -} - -/* This is a hacked version of Python's fileobject.c:file_getiter(). */ -static PyObject * -BZ2File_getiter(BZ2FileObject *self) -{ - if (self->mode == MODE_CLOSED) { - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - Py_INCREF((PyObject*)self); - return (PyObject *)self; -} - -/* This is a hacked version of Python's fileobject.c:file_iternext(). */ -#define READAHEAD_BUFSIZE 8192 -static PyObject * -BZ2File_iternext(BZ2FileObject *self) -{ - PyBytesObject* ret; - ACQUIRE_LOCK(self); - if (self->mode == MODE_CLOSED) { - RELEASE_LOCK(self); - PyErr_SetString(PyExc_ValueError, - "I/O operation on closed file"); - return NULL; - } - ret = Util_ReadAheadGetLineSkip(self, 0, READAHEAD_BUFSIZE); - RELEASE_LOCK(self); - if (ret == NULL || PyBytes_GET_SIZE(ret) == 0) { - Py_XDECREF(ret); - return NULL; - } - return (PyObject *)ret; -} - -/* ===================================================================== */ -/* BZ2File_Type definition. */ - -PyDoc_VAR(BZ2File__doc__) = -PyDoc_STR( -"BZ2File(name [, mode='r', buffering=0, compresslevel=9]) -> file object\n\ -\n\ -Open a bz2 file. The mode can be 'r' or 'w', for reading (default) or\n\ -writing. When opened for writing, the file will be created if it doesn't\n\ -exist, and truncated otherwise. If the buffering argument is given, 0 means\n\ -unbuffered, and larger numbers specify the buffer size. If compresslevel\n\ -is given, must be a number between 1 and 9.\n\ -Data read is always returned in bytes; data written ought to be bytes.\n\ -"); - -static PyTypeObject BZ2File_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2File", /*tp_name*/ - sizeof(BZ2FileObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2File_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2File__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - (getiterfunc)BZ2File_getiter, /*tp_iter*/ - (iternextfunc)BZ2File_iternext, /*tp_iternext*/ - BZ2File_methods, /*tp_methods*/ - 0, /*tp_members*/ - BZ2File_getset, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2File_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Methods of BZ2Comp. */ - -PyDoc_STRVAR(BZ2Comp_compress__doc__, -"compress(data) -> string\n\ -\n\ -Provide more data to the compressor object. It will return chunks of\n\ -compressed data whenever possible. When you've finished providing data\n\ -to compress, call the flush() method to finish the compression process,\n\ -and return what is left in the internal buffers.\n\ -"); - -static PyObject * -BZ2Comp_compress(BZ2CompObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PY_LONG_LONG totalout; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:compress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (datasize == 0) { - PyBuffer_Release(&pdata); - return PyBytes_FromStringAndSize("", 0); - } - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_ValueError, - "this object was already flushed"); - goto error; - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_RUN); - Py_END_ALLOW_THREADS - if (bzerror != BZ_RUN_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_in == 0) - break; /* no more input data */ - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzCompressEnd(bzs); - goto error; - } - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - return ret; - -error: - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - Py_XDECREF(ret); - return NULL; -} - -PyDoc_STRVAR(BZ2Comp_flush__doc__, -"flush() -> string\n\ -\n\ -Finish the compression process and return what is left in internal buffers.\n\ -You must not use the compressor object after calling this method.\n\ -"); - -static PyObject * -BZ2Comp_flush(BZ2CompObject *self) -{ - int bufsize = SMALLCHUNK; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - PY_LONG_LONG totalout; - int bzerror; - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_ValueError, "object was already " - "flushed"); - goto error; - } - self->running = 0; - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_FINISH); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_FINISH_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) - goto error; - bzs->next_out = BUF(ret); - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - } - - RELEASE_LOCK(self); - return ret; - -error: - RELEASE_LOCK(self); - Py_XDECREF(ret); - return NULL; -} - -static PyMethodDef BZ2Comp_methods[] = { - {"compress", (PyCFunction)BZ2Comp_compress, METH_VARARGS, - BZ2Comp_compress__doc__}, - {"flush", (PyCFunction)BZ2Comp_flush, METH_NOARGS, - BZ2Comp_flush__doc__}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2Comp_Type. */ - -static int -BZ2Comp_init(BZ2CompObject *self, PyObject *args, PyObject *kwargs) -{ - int compresslevel = 9; - int bzerror; - static char *kwlist[] = {"compresslevel", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|i:BZ2Compressor", - kwlist, &compresslevel)) - return -1; - - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - goto error; - } - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - memset(&self->bzs, 0, sizeof(bz_stream)); - bzerror = BZ2_bzCompressInit(&self->bzs, compresslevel, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->running = 1; - - return 0; -error: -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - return -1; -} - -static void -BZ2Comp_dealloc(BZ2CompObject *self) -{ -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - BZ2_bzCompressEnd(&self->bzs); - Py_TYPE(self)->tp_free((PyObject *)self); -} - - -/* ===================================================================== */ -/* BZ2Comp_Type definition. */ - -PyDoc_STRVAR(BZ2Comp__doc__, -"BZ2Compressor([compresslevel=9]) -> compressor object\n\ -\n\ -Create a new compressor object. This object may be used to compress\n\ -data sequentially. If you want to compress data in one shot, use the\n\ -compress() function instead. The compresslevel parameter, if given,\n\ -must be a number between 1 and 9.\n\ -"); - -static PyTypeObject BZ2Comp_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2Compressor", /*tp_name*/ - sizeof(BZ2CompObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2Comp_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2Comp__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - BZ2Comp_methods, /*tp_methods*/ - 0, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2Comp_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Members of BZ2Decomp. */ - -#undef OFF -#define OFF(x) offsetof(BZ2DecompObject, x) - -static PyMemberDef BZ2Decomp_members[] = { - {"unused_data", T_OBJECT, OFF(unused_data), READONLY}, - {NULL} /* Sentinel */ -}; - - -/* ===================================================================== */ -/* Methods of BZ2Decomp. */ - -PyDoc_STRVAR(BZ2Decomp_decompress__doc__, -"decompress(data) -> string\n\ -\n\ -Provide more data to the decompressor object. It will return chunks\n\ -of decompressed data whenever possible. If you try to decompress data\n\ -after the end of stream is found, EOFError will be raised. If any data\n\ -was found after the end of stream, it'll be ignored and saved in\n\ -unused_data attribute.\n\ -"); - -static PyObject * -BZ2Decomp_decompress(BZ2DecompObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PY_LONG_LONG totalout; - PyObject *ret = NULL; - bz_stream *bzs = &self->bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:decompress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - ACQUIRE_LOCK(self); - if (!self->running) { - PyErr_SetString(PyExc_EOFError, "end of stream was " - "already found"); - goto error; - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) - goto error; - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - totalout = BZS_TOTAL_OUT(bzs); - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzDecompress(bzs); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - if (bzs->avail_in != 0) { - Py_DECREF(self->unused_data); - self->unused_data = - PyBytes_FromStringAndSize(bzs->next_in, - bzs->avail_in); - } - self->running = 0; - break; - } - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - if (bzs->avail_in == 0) - break; /* no more input data */ - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzDecompressEnd(bzs); - goto error; - } - bzs->next_out = BUF(ret); - bzs->next_out = BUF(ret) + (BZS_TOTAL_OUT(bzs) - - totalout); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, - (Py_ssize_t)(BZS_TOTAL_OUT(bzs) - totalout)) < 0) - goto error; - } - - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - return ret; - -error: - RELEASE_LOCK(self); - PyBuffer_Release(&pdata); - Py_XDECREF(ret); - return NULL; -} - -static PyMethodDef BZ2Decomp_methods[] = { - {"decompress", (PyCFunction)BZ2Decomp_decompress, METH_VARARGS, BZ2Decomp_decompress__doc__}, - {NULL, NULL} /* sentinel */ -}; - - -/* ===================================================================== */ -/* Slot definitions for BZ2Decomp_Type. */ - -static int -BZ2Decomp_init(BZ2DecompObject *self, PyObject *args, PyObject *kwargs) -{ - int bzerror; - - if (!PyArg_ParseTuple(args, ":BZ2Decompressor")) - return -1; - -#ifdef WITH_THREAD - self->lock = PyThread_allocate_lock(); - if (!self->lock) { - PyErr_SetString(PyExc_MemoryError, "unable to allocate lock"); - goto error; - } -#endif - - self->unused_data = PyBytes_FromStringAndSize("", 0); - if (!self->unused_data) - goto error; - - memset(&self->bzs, 0, sizeof(bz_stream)); - bzerror = BZ2_bzDecompressInit(&self->bzs, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - goto error; - } - - self->running = 1; - - return 0; - -error: -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - Py_CLEAR(self->unused_data); - return -1; -} - -static void -BZ2Decomp_dealloc(BZ2DecompObject *self) -{ -#ifdef WITH_THREAD - if (self->lock) - PyThread_free_lock(self->lock); -#endif - Py_XDECREF(self->unused_data); - BZ2_bzDecompressEnd(&self->bzs); - Py_TYPE(self)->tp_free((PyObject *)self); -} - - -/* ===================================================================== */ -/* BZ2Decomp_Type definition. */ - -PyDoc_STRVAR(BZ2Decomp__doc__, -"BZ2Decompressor() -> decompressor object\n\ -\n\ -Create a new decompressor object. This object may be used to decompress\n\ -data sequentially. If you want to decompress data in one shot, use the\n\ -decompress() function instead.\n\ -"); - -static PyTypeObject BZ2Decomp_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "bz2.BZ2Decompressor", /*tp_name*/ - sizeof(BZ2DecompObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)BZ2Decomp_dealloc, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - PyObject_GenericGetAttr,/*tp_getattro*/ - PyObject_GenericSetAttr,/*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ - BZ2Decomp__doc__, /*tp_doc*/ - 0, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - BZ2Decomp_methods, /*tp_methods*/ - BZ2Decomp_members, /*tp_members*/ - 0, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - (initproc)BZ2Decomp_init, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_Free, /*tp_free*/ - 0, /*tp_is_gc*/ -}; - - -/* ===================================================================== */ -/* Module functions. */ - -PyDoc_STRVAR(bz2_compress__doc__, -"compress(data [, compresslevel=9]) -> string\n\ -\n\ -Compress data in one shot. If you want to compress data sequentially,\n\ -use an instance of BZ2Compressor instead. The compresslevel parameter, if\n\ -given, must be a number between 1 and 9.\n\ -"); - -static PyObject * -bz2_compress(PyObject *self, PyObject *args, PyObject *kwargs) -{ - int compresslevel=9; - Py_buffer pdata; - char *data; - int datasize; - int bufsize; - PyObject *ret = NULL; - bz_stream _bzs; - bz_stream *bzs = &_bzs; - int bzerror; - static char *kwlist[] = {"data", "compresslevel", 0}; - - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y*|i", - kwlist, &pdata, - &compresslevel)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (compresslevel < 1 || compresslevel > 9) { - PyErr_SetString(PyExc_ValueError, - "compresslevel must be between 1 and 9"); - PyBuffer_Release(&pdata); - return NULL; - } - - /* Conforming to bz2 manual, this is large enough to fit compressed - * data in one shot. We will check it later anyway. */ - bufsize = datasize + (datasize/100+1) + 600; - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) { - PyBuffer_Release(&pdata); - return NULL; - } - - memset(bzs, 0, sizeof(bz_stream)); - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - bzerror = BZ2_bzCompressInit(bzs, compresslevel, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzCompress(bzs, BZ_FINISH); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_FINISH_OK) { - BZ2_bzCompressEnd(bzs); - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzCompressEnd(bzs); - PyBuffer_Release(&pdata); - return NULL; - } - bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) { - ret = NULL; - } - } - BZ2_bzCompressEnd(bzs); - - PyBuffer_Release(&pdata); - return ret; -} - -PyDoc_STRVAR(bz2_decompress__doc__, -"decompress(data) -> decompressed data\n\ -\n\ -Decompress data in one shot. If you want to decompress data sequentially,\n\ -use an instance of BZ2Decompressor instead.\n\ -"); - -static PyObject * -bz2_decompress(PyObject *self, PyObject *args) -{ - Py_buffer pdata; - char *data; - int datasize; - int bufsize = SMALLCHUNK; - PyObject *ret; - bz_stream _bzs; - bz_stream *bzs = &_bzs; - int bzerror; - - if (!PyArg_ParseTuple(args, "y*:decompress", &pdata)) - return NULL; - data = pdata.buf; - datasize = pdata.len; - - if (datasize == 0) { - PyBuffer_Release(&pdata); - return PyBytes_FromStringAndSize("", 0); - } - - ret = PyBytes_FromStringAndSize(NULL, bufsize); - if (!ret) { - PyBuffer_Release(&pdata); - return NULL; - } - - memset(bzs, 0, sizeof(bz_stream)); - - bzs->next_in = data; - bzs->avail_in = datasize; - bzs->next_out = BUF(ret); - bzs->avail_out = bufsize; - - bzerror = BZ2_bzDecompressInit(bzs, 0, 0); - if (bzerror != BZ_OK) { - Util_CatchBZ2Error(bzerror); - Py_DECREF(ret); - PyBuffer_Release(&pdata); - return NULL; - } - - for (;;) { - Py_BEGIN_ALLOW_THREADS - bzerror = BZ2_bzDecompress(bzs); - Py_END_ALLOW_THREADS - if (bzerror == BZ_STREAM_END) { - break; - } else if (bzerror != BZ_OK) { - BZ2_bzDecompressEnd(bzs); - Util_CatchBZ2Error(bzerror); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_in == 0) { - BZ2_bzDecompressEnd(bzs); - PyErr_SetString(PyExc_ValueError, - "couldn't find end of stream"); - PyBuffer_Release(&pdata); - Py_DECREF(ret); - return NULL; - } - if (bzs->avail_out == 0) { - bufsize = Util_NewBufferSize(bufsize); - if (_PyBytes_Resize(&ret, bufsize) < 0) { - BZ2_bzDecompressEnd(bzs); - PyBuffer_Release(&pdata); - return NULL; - } - bzs->next_out = BUF(ret) + BZS_TOTAL_OUT(bzs); - bzs->avail_out = bufsize - (bzs->next_out - BUF(ret)); - } - } - - if (bzs->avail_out != 0) { - if (_PyBytes_Resize(&ret, (Py_ssize_t)BZS_TOTAL_OUT(bzs)) < 0) { - ret = NULL; - } - } - BZ2_bzDecompressEnd(bzs); - PyBuffer_Release(&pdata); - - return ret; -} - -static PyMethodDef bz2_methods[] = { - {"compress", (PyCFunction) bz2_compress, METH_VARARGS|METH_KEYWORDS, - bz2_compress__doc__}, - {"decompress", (PyCFunction) bz2_decompress, METH_VARARGS, - bz2_decompress__doc__}, - {NULL, NULL} /* sentinel */ -}; - -/* ===================================================================== */ -/* Initialization function. */ - -PyDoc_STRVAR(bz2__doc__, -"The python bz2 module provides a comprehensive interface for\n\ -the bz2 compression library. It implements a complete file\n\ -interface, one shot (de)compression functions, and types for\n\ -sequential (de)compression.\n\ -"); - - -static struct PyModuleDef bz2module = { - PyModuleDef_HEAD_INIT, - "bz2", - bz2__doc__, - -1, - bz2_methods, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit_bz2(void) -{ - PyObject *m; - - if (PyType_Ready(&BZ2File_Type) < 0) - return NULL; - if (PyType_Ready(&BZ2Comp_Type) < 0) - return NULL; - if (PyType_Ready(&BZ2Decomp_Type) < 0) - return NULL; - - m = PyModule_Create(&bz2module); - if (m == NULL) - return NULL; - - PyModule_AddObject(m, "__author__", PyUnicode_FromString(__author__)); - - Py_INCREF(&BZ2File_Type); - PyModule_AddObject(m, "BZ2File", (PyObject *)&BZ2File_Type); - - Py_INCREF(&BZ2Comp_Type); - PyModule_AddObject(m, "BZ2Compressor", (PyObject *)&BZ2Comp_Type); - - Py_INCREF(&BZ2Decomp_Type); - PyModule_AddObject(m, "BZ2Decompressor", (PyObject *)&BZ2Decomp_Type); - return m; -} diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c index ab4e659..9e9e96c 100644 --- a/Modules/cjkcodecs/_codecs_cn.c +++ b/Modules/cjkcodecs/_codecs_cn.c @@ -85,7 +85,7 @@ DECODER(gb2312) TRYMAP_DEC(gb2312, **outbuf, c ^ 0x80, IN2 ^ 0x80) { NEXT(2, 1) } - else return 2; + else return 1; } return 0; @@ -141,7 +141,7 @@ DECODER(gbk) REQUIRE_INBUF(2) GBK_DECODE(c, IN2, **outbuf) - else return 2; + else return 1; NEXT(2, 1) } @@ -267,7 +267,7 @@ DECODER(gb18030) c3 = IN3; c4 = IN4; if (c < 0x81 || c3 < 0x81 || c4 < 0x30 || c4 > 0x39) - return 4; + return 1; c -= 0x81; c2 -= 0x30; c3 -= 0x81; c4 -= 0x30; @@ -292,12 +292,12 @@ DECODER(gb18030) continue; } } - return 4; + return 1; } GBK_DECODE(c, c2, **outbuf) else TRYMAP_DEC(gb18030ext, **outbuf, c, c2); - else return 2; + else return 1; NEXT(2, 1) } @@ -400,7 +400,7 @@ DECODER(hz) else if (c2 == '\n') ; /* line-continuation */ else - return 2; + return 1; NEXT(2, 0); continue; } @@ -419,7 +419,7 @@ DECODER(hz) NEXT(2, 1) } else - return 2; + return 1; } } diff --git a/Modules/cjkcodecs/_codecs_hk.c b/Modules/cjkcodecs/_codecs_hk.c index 558a42f..d3ad04b 100644 --- a/Modules/cjkcodecs/_codecs_hk.c +++ b/Modules/cjkcodecs/_codecs_hk.c @@ -161,7 +161,7 @@ DECODER(big5hkscs) case 0x8864: WRITE2(0x00ca, 0x030c); break; case 0x88a3: WRITE2(0x00ea, 0x0304); break; case 0x88a5: WRITE2(0x00ea, 0x030c); break; - default: return 2; + default: return 1; } NEXT(2, 2) /* all decoded codepoints are pairs, above. */ diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index 25c1a36..cbc1542 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -123,7 +123,7 @@ struct iso2022_config { CODEC_INIT(iso2022) { - const struct iso2022_designation *desig = CONFIG_DESIGNATIONS; + const struct iso2022_designation *desig; for (desig = CONFIG_DESIGNATIONS; desig->mark; desig++) if (desig->initializer != NULL && desig->initializer() != 0) return -1; diff --git a/Modules/cjkcodecs/_codecs_jp.c b/Modules/cjkcodecs/_codecs_jp.c index a05e01b..a500696 100644 --- a/Modules/cjkcodecs/_codecs_jp.c +++ b/Modules/cjkcodecs/_codecs_jp.c @@ -112,7 +112,7 @@ DECODER(cp932) TRYMAP_DEC(cp932ext, **outbuf, c, c2); else if ((c >= 0x81 && c <= 0x9f) || (c >= 0xe0 && c <= 0xea)){ if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -120,7 +120,7 @@ DECODER(cp932) c2 = (c2 < 0x5e ? c2 : c2 - 0x5e) + 0x21; TRYMAP_DEC(jisx0208, **outbuf, c, c2); - else return 2; + else return 1; } else if (c >= 0xf0 && c <= 0xf9) { if ((c2 >= 0x40 && c2 <= 0x7e) || @@ -128,10 +128,10 @@ DECODER(cp932) OUT1(0xe000 + 188 * (c - 0xf0) + (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41)) else - return 2; + return 1; } else - return 2; + return 1; NEXT(2, 1) } @@ -256,7 +256,7 @@ DECODER(euc_jis_2004) NEXT(2, 1) } else - return 2; + return 1; } else if (c == 0x8f) { unsigned char c2, c3; @@ -274,7 +274,7 @@ DECODER(euc_jis_2004) continue; } else TRYMAP_DEC(jisx0212, **outbuf, c2, c3) ; - else return 3; + else return 1; NEXT(3, 1) } else { @@ -300,7 +300,7 @@ DECODER(euc_jis_2004) NEXT(2, 2) continue; } - else return 2; + else return 1; NEXT(2, 1) } } @@ -388,7 +388,7 @@ DECODER(euc_jp) NEXT(2, 1) } else - return 2; + return 1; } else if (c == 0x8f) { unsigned char c2, c3; @@ -401,7 +401,7 @@ DECODER(euc_jp) NEXT(3, 1) } else - return 3; + return 1; } else { unsigned char c2; @@ -417,7 +417,7 @@ DECODER(euc_jp) #endif TRYMAP_DEC(jisx0208, **outbuf, c ^ 0x80, c2 ^ 0x80) ; - else return 2; + else return 1; NEXT(2, 1) } } @@ -502,7 +502,7 @@ DECODER(shift_jis) REQUIRE_INBUF(2) c2 = IN2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -522,10 +522,10 @@ DECODER(shift_jis) continue; } else - return 2; + return 1; } else - return 2; + return 1; NEXT(1, 1) /* JIS X 0201 */ } @@ -645,7 +645,7 @@ DECODER(shift_jis_2004) REQUIRE_INBUF(2) c2 = IN2; if (c2 < 0x40 || (c2 > 0x7e && c2 < 0x80) || c2 > 0xfc) - return 2; + return 1; c1 = (c < 0xe0 ? c - 0x81 : c - 0xc1); c2 = (c2 < 0x80 ? c2 - 0x40 : c2 - 0x41); @@ -671,7 +671,7 @@ DECODER(shift_jis_2004) NEXT_OUT(2) } else - return 2; + return 1; NEXT_IN(2) } else { /* Plane 2 */ @@ -689,13 +689,13 @@ DECODER(shift_jis_2004) continue; } else - return 2; + return 1; NEXT(2, 1) } continue; } else - return 2; + return 1; NEXT(1, 1) /* JIS X 0201 */ } diff --git a/Modules/cjkcodecs/_codecs_kr.c b/Modules/cjkcodecs/_codecs_kr.c index 9272e36..f5697dd 100644 --- a/Modules/cjkcodecs/_codecs_kr.c +++ b/Modules/cjkcodecs/_codecs_kr.c @@ -123,7 +123,7 @@ DECODER(euc_kr) if ((*inbuf)[2] != EUCKR_JAMO_FIRSTBYTE || (*inbuf)[4] != EUCKR_JAMO_FIRSTBYTE || (*inbuf)[6] != EUCKR_JAMO_FIRSTBYTE) - return 8; + return 1; c = (*inbuf)[3]; if (0xa1 <= c && c <= 0xbe) @@ -143,7 +143,7 @@ DECODER(euc_kr) jong = NONE; if (cho == NONE || jung == NONE || jong == NONE) - return 8; + return 1; OUT1(0xac00 + cho*588 + jung*28 + jong); NEXT(8, 1) @@ -152,7 +152,7 @@ DECODER(euc_kr) NEXT(2, 1) } else - return 2; + return 1; } return 0; @@ -208,7 +208,7 @@ DECODER(cp949) REQUIRE_INBUF(2) TRYMAP_DEC(ksx1001, **outbuf, c ^ 0x80, IN2 ^ 0x80); else TRYMAP_DEC(cp949ext, **outbuf, c, IN2); - else return 2; + else return 1; NEXT(2, 1) } @@ -375,7 +375,7 @@ DECODER(johab) i_jong = johabidx_jongseong[c_jong]; if (i_cho == NONE || i_jung == NONE || i_jong == NONE) - return 2; + return 1; /* we don't use U+1100 hangul jamo yet. */ if (i_cho == FILL) { @@ -391,7 +391,7 @@ DECODER(johab) OUT1(0x3100 | johabjamo_jungseong[c_jung]) else - return 2; + return 1; } } else { if (i_jung == FILL) { @@ -399,7 +399,7 @@ DECODER(johab) OUT1(0x3100 | johabjamo_choseong[c_cho]) else - return 2; + return 1; } else OUT1(0xac00 + @@ -414,7 +414,7 @@ DECODER(johab) c2 < 0x31 || (c2 >= 0x80 && c2 < 0x91) || (c2 & 0x7f) == 0x7f || (c == 0xda && (c2 >= 0xa1 && c2 <= 0xd3))) - return 2; + return 1; else { unsigned char t1, t2; @@ -425,7 +425,7 @@ DECODER(johab) t2 = (t2 < 0x5e ? t2 : t2 - 0x5e) + 0x21; TRYMAP_DEC(ksx1001, **outbuf, t1, t2); - else return 2; + else return 1; NEXT(2, 1) } } diff --git a/Modules/cjkcodecs/_codecs_tw.c b/Modules/cjkcodecs/_codecs_tw.c index 38cf723..916298d 100644 --- a/Modules/cjkcodecs/_codecs_tw.c +++ b/Modules/cjkcodecs/_codecs_tw.c @@ -55,7 +55,7 @@ DECODER(big5) TRYMAP_DEC(big5, **outbuf, c, IN2) { NEXT(2, 1) } - else return 2; + else return 1; } return 0; @@ -109,7 +109,7 @@ DECODER(cp950) TRYMAP_DEC(cp950ext, **outbuf, c, IN2); else TRYMAP_DEC(big5, **outbuf, c, IN2); - else return 2; + else return 1; NEXT(2, 1) } diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 7b04f020..1b37845 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -483,6 +483,7 @@ multibytecodec_encode(MultibyteCodec *codec, return PyBytes_FromStringAndSize(NULL, 0); buf.excobj = NULL; + buf.outobj = NULL; buf.inbuf = buf.inbuf_top = *data; buf.inbuf_end = buf.inbuf_top + datalen; @@ -900,11 +901,17 @@ mbiencoder_encode(MultibyteIncrementalEncoderObject *self, static PyObject * mbiencoder_reset(MultibyteIncrementalEncoderObject *self) { - if (self->codec->decreset != NULL && - self->codec->decreset(&self->state, self->codec->config) != 0) - return NULL; + /* Longest output: 4 bytes (b'\x0F\x1F(B') with ISO 2022 */ + unsigned char buffer[4], *outbuf; + Py_ssize_t r; + if (self->codec->encreset != NULL) { + outbuf = buffer; + r = self->codec->encreset(&self->state, self->codec->config, + &outbuf, sizeof(buffer)); + if (r != 0) + return NULL; + } self->pendingsize = 0; - Py_RETURN_NONE; } diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 86720af..86b0a01 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -84,6 +84,8 @@ PyInit_errno(void) * The names and comments are borrowed from linux/include/errno.h, * which should be pretty all-inclusive. However, the Solaris specific * names and comments are borrowed from sys/errno.h in Solaris. + * MacOSX specific names and comments are borrowed from sys/errno.h in + * MacOSX. */ #ifdef ENODEV @@ -848,6 +850,59 @@ PyInit_errno(void) inscode(d, ds, de, "ENOTACTIVE", ENOTACTIVE, "Facility is not active"); #endif + /* MacOSX specific errnos */ +#ifdef EAUTH + inscode(d, ds, de, "EAUTH", EAUTH, "Authentication error"); +#endif +#ifdef EBADARCH + inscode(d, ds, de, "EBADARCH", EBADARCH, "Bad CPU type in executable"); +#endif +#ifdef EBADEXEC + inscode(d, ds, de, "EBADEXEC", EBADEXEC, "Bad executable (or shared library)"); +#endif +#ifdef EBADMACHO + inscode(d, ds, de, "EBADMACHO", EBADMACHO, "Malformed Mach-o file"); +#endif +#ifdef EBADRPC + inscode(d, ds, de, "EBADRPC", EBADRPC, "RPC struct is bad"); +#endif +#ifdef EDEVERR + inscode(d, ds, de, "EDEVERR", EDEVERR, "Device error"); +#endif +#ifdef EFTYPE + inscode(d, ds, de, "EFTYPE", EFTYPE, "Inappropriate file type or format"); +#endif +#ifdef ENEEDAUTH + inscode(d, ds, de, "ENEEDAUTH", ENEEDAUTH, "Need authenticator"); +#endif +#ifdef ENOATTR + inscode(d, ds, de, "ENOATTR", ENOATTR, "Attribute not found"); +#endif +#ifdef ENOPOLICY + inscode(d, ds, de, "ENOPOLICY", ENOPOLICY, "Policy not found"); +#endif +#ifdef EPROCLIM + inscode(d, ds, de, "EPROCLIM", EPROCLIM, "Too many processes"); +#endif +#ifdef EPROCUNAVAIL + inscode(d, ds, de, "EPROCUNAVAIL", EPROCUNAVAIL, "Bad procedure for program"); +#endif +#ifdef EPROGMISMATCH + inscode(d, ds, de, "EPROGMISMATCH", EPROGMISMATCH, "Program version wrong"); +#endif +#ifdef EPROGUNAVAIL + inscode(d, ds, de, "EPROGUNAVAIL", EPROGUNAVAIL, "RPC prog. not avail"); +#endif +#ifdef EPWROFF + inscode(d, ds, de, "EPWROFF", EPWROFF, "Device power is off"); +#endif +#ifdef ERPCMISMATCH + inscode(d, ds, de, "ERPCMISMATCH", ERPCMISMATCH, "RPC version wrong"); +#endif +#ifdef ESHLIBVERS + inscode(d, ds, de, "ESHLIBVERS", ESHLIBVERS, "Shared library version mismatch"); +#endif + Py_DECREF(de); return m; } diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c new file mode 100644 index 0000000..b2ac83f --- /dev/null +++ b/Modules/faulthandler.c @@ -0,0 +1,1142 @@ +#include "Python.h" +#include "pythread.h" +#include <signal.h> +#include <object.h> +#include <frameobject.h> +#include <signal.h> +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) +#include <pthread.h> +#endif + +/* Allocate at maximum 100 MB of the stack to raise the stack overflow */ +#define STACK_OVERFLOW_MAX_SIZE (100*1024*1024) + +#ifdef WITH_THREAD +# define FAULTHANDLER_LATER +#endif + +#ifndef MS_WINDOWS + /* register() is useless on Windows, because only SIGSEGV, SIGABRT and + SIGILL can be handled by the process, and these signals can only be used + with enable(), not using register() */ +# define FAULTHANDLER_USER +#endif + +#define PUTS(fd, str) write(fd, str, strlen(str)) + +#ifdef HAVE_SIGACTION +typedef struct sigaction _Py_sighandler_t; +#else +typedef PyOS_sighandler_t _Py_sighandler_t; +#endif + +typedef struct { + int signum; + int enabled; + const char* name; + _Py_sighandler_t previous; + int all_threads; +} fault_handler_t; + +static struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + PyInterpreterState *interp; +} fatal_error = {0, NULL, -1, 0}; + +#ifdef FAULTHANDLER_LATER +static struct { + PyObject *file; + int fd; + PY_TIMEOUT_T timeout_us; /* timeout in microseconds */ + int repeat; + PyInterpreterState *interp; + int exit; + char *header; + size_t header_len; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + PyThread_type_lock cancel_event; + /* released by child thread when joined */ + PyThread_type_lock running; +} thread; +#endif + +#ifdef FAULTHANDLER_USER +typedef struct { + int enabled; + PyObject *file; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; + PyInterpreterState *interp; +} user_signal_t; + +static user_signal_t *user_signals; + +/* the following macros come from Python: Modules/signalmodule.c */ +#if defined(PYOS_OS2) && !defined(PYCC_GCC) +#define NSIG 12 +#endif +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum); +#endif /* FAULTHANDLER_USER */ + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const unsigned char faulthandler_nsignals = \ + sizeof(faulthandler_handlers) / sizeof(faulthandler_handlers[0]); + +#ifdef HAVE_SIGALTSTACK +static stack_t stack; +#endif + + +/* Get the file descriptor of a file by calling its fileno() method and then + call its flush() method. + + If file is NULL or Py_None, use sys.stderr as the new file. + + On success, return the new file and write the file descriptor into *p_fd. + On error, return NULL. */ + +static PyObject* +faulthandler_get_fileno(PyObject *file, int *p_fd) +{ + PyObject *result; + long fd_long; + int fd; + + if (file == NULL || file == Py_None) { + file = PySys_GetObject("stderr"); + if (file == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.stderr"); + return NULL; + } + } + + result = PyObject_CallMethod(file, "fileno", ""); + if (result == NULL) + return NULL; + + fd = -1; + if (PyLong_Check(result)) { + fd_long = PyLong_AsLong(result); + if (0 <= fd_long && fd_long < INT_MAX) + fd = (int)fd_long; + } + Py_DECREF(result); + + if (fd == -1) { + PyErr_SetString(PyExc_RuntimeError, + "file.fileno() is not a valid file descriptor"); + return NULL; + } + + result = PyObject_CallMethod(file, "flush", ""); + if (result != NULL) + Py_DECREF(result); + else { + /* ignore flush() error */ + PyErr_Clear(); + } + *p_fd = fd; + return file; +} + +/* Get the state of the current thread: only call this function if the current + thread holds the GIL. Raise an exception on error. */ +static PyThreadState* +get_thread_state(void) +{ + PyThreadState *tstate = PyThreadState_Get(); + if (tstate == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "unable to get the current thread state"); + return NULL; + } + return tstate; +} + +static PyObject* +faulthandler_dump_traceback_py(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; + PyThreadState *tstate; + const char *errmsg; + int fd; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|Oi:dump_traceback", kwlist, + &file, &all_threads)) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + if (all_threads) { + errmsg = _Py_DumpTracebackThreads(fd, tstate->interp, tstate); + if (errmsg != NULL) { + PyErr_SetString(PyExc_RuntimeError, errmsg); + return NULL; + } + } + else { + _Py_DumpTraceback(fd, tstate); + } + Py_RETURN_NONE; +} + + +/* Handler for SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals. + + Display the current Python traceback, restore the previous handler and call + the previous handler. + + On Windows, don't explicitly call the previous handler, because the Windows + signal handler would not be called (for an unknown reason). The execution of + the program continues at faulthandler_fatal_error() exit, but the same + instruction will raise the same fault (signal), and so the previous handler + will be called. + + This function is signal-safe and should only call signal-safe functions. */ + +static void +faulthandler_fatal_error(int signum) +{ + const int fd = fatal_error.fd; + unsigned int i; + fault_handler_t *handler = NULL; + PyThreadState *tstate; + int save_errno = errno; + + if (!fatal_error.enabled) + return; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (handler->signum == signum) + break; + } + if (handler == NULL) { + /* faulthandler_nsignals == 0 (unlikely) */ + return; + } + + /* restore the previous handler */ +#ifdef HAVE_SIGACTION + (void)sigaction(signum, &handler->previous, NULL); +#else + (void)signal(signum, handler->previous); +#endif + handler->enabled = 0; + + PUTS(fd, "Fatal Python error: "); + PUTS(fd, handler->name); + PUTS(fd, "\n\n"); + +#ifdef WITH_THREAD + /* SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL are synchronous signals and + are thus delivered to the thread that caused the fault. Get the Python + thread state of the current thread. + + PyThreadState_Get() doesn't give the state of the thread that caused the + fault if the thread released the GIL, and so this function cannot be + used. Read the thread local storage (TLS) instead: call + PyGILState_GetThisThreadState(). */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_Get(); +#endif + + if (fatal_error.all_threads) + _Py_DumpTracebackThreads(fd, fatal_error.interp, tstate); + else { + if (tstate != NULL) + _Py_DumpTraceback(fd, tstate); + } + + errno = save_errno; +#ifdef MS_WINDOWS + if (signum == SIGSEGV) { + /* don't explicitly call the previous handler for SIGSEGV in this signal + handler, because the Windows signal handler would not be called */ + return; + } +#endif + /* call the previous signal handler: it is called immediatly if we use + sigaction() thanks to SA_NODEFER flag, otherwise it is deferred */ + raise(signum); +} + +/* Install the handler for fatal signals, faulthandler_fatal_error(). */ + +static PyObject* +faulthandler_enable(PyObject *self, PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"file", "all_threads", NULL}; + PyObject *file = NULL; + int all_threads = 1; + unsigned int i; + fault_handler_t *handler; +#ifdef HAVE_SIGACTION + struct sigaction action; +#endif + int err; + int fd; + PyThreadState *tstate; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "|Oi:enable", kwlist, &file, &all_threads)) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + Py_XDECREF(fatal_error.file); + Py_INCREF(file); + fatal_error.file = file; + fatal_error.fd = fd; + fatal_error.all_threads = all_threads; + fatal_error.interp = tstate->interp; + + if (!fatal_error.enabled) { + fatal_error.enabled = 1; + + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; +#ifdef HAVE_SIGACTION + action.sa_handler = faulthandler_fatal_error; + sigemptyset(&action.sa_mask); + /* Do not prevent the signal from being received from within + its own signal handler */ + action.sa_flags = SA_NODEFER; +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + err = sigaction(handler->signum, &action, &handler->previous); +#else + handler->previous = signal(handler->signum, + faulthandler_fatal_error); + err = (handler->previous == SIG_ERR); +#endif + if (err) { + PyErr_SetFromErrno(PyExc_RuntimeError); + return NULL; + } + handler->enabled = 1; + } + } + Py_RETURN_NONE; +} + +static void +faulthandler_disable(void) +{ + unsigned int i; + fault_handler_t *handler; + + if (fatal_error.enabled) { + fatal_error.enabled = 0; + for (i=0; i < faulthandler_nsignals; i++) { + handler = &faulthandler_handlers[i]; + if (!handler->enabled) + continue; +#ifdef HAVE_SIGACTION + (void)sigaction(handler->signum, &handler->previous, NULL); +#else + (void)signal(handler->signum, handler->previous); +#endif + handler->enabled = 0; + } + } + + Py_CLEAR(fatal_error.file); +} + +static PyObject* +faulthandler_disable_py(PyObject *self) +{ + if (!fatal_error.enabled) { + Py_INCREF(Py_False); + return Py_False; + } + faulthandler_disable(); + Py_INCREF(Py_True); + return Py_True; +} + +static PyObject* +faulthandler_is_enabled(PyObject *self) +{ + return PyBool_FromLong(fatal_error.enabled); +} + +#ifdef FAULTHANDLER_LATER + +static void +faulthandler_thread(void *unused) +{ + PyLockStatus st; + const char* errmsg; + PyThreadState *current; + int ok; +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) + sigset_t set; + + /* we don't want to receive any signal */ + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + do { + st = PyThread_acquire_lock_timed(thread.cancel_event, + thread.timeout_us, 0); + if (st == PY_LOCK_ACQUIRED) { + PyThread_release_lock(thread.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == PY_LOCK_FAILURE); + + /* get the thread holding the GIL, NULL if no thread hold the GIL */ + current = _Py_atomic_load_relaxed(&_PyThreadState_Current); + + write(thread.fd, thread.header, thread.header_len); + + errmsg = _Py_DumpTracebackThreads(thread.fd, thread.interp, current); + ok = (errmsg == NULL); + + if (thread.exit) + _exit(1); + } while (ok && thread.repeat); + + /* The only way out */ + PyThread_release_lock(thread.running); +} + +static void +cancel_dump_tracebacks_later(void) +{ + /* Notify cancellation */ + PyThread_release_lock(thread.cancel_event); + + /* Wait for thread to join */ + PyThread_acquire_lock(thread.running, 1); + PyThread_release_lock(thread.running); + + /* The main thread should always hold the cancel_event lock */ + PyThread_acquire_lock(thread.cancel_event, 1); + + Py_CLEAR(thread.file); + if (thread.header) { + free(thread.header); + thread.header = NULL; + } +} + +static char* +format_timeout(double timeout) +{ + unsigned long us, sec, min, hour; + double intpart, fracpart; + char buffer[100]; + + fracpart = modf(timeout, &intpart); + sec = (unsigned long)intpart; + us = (unsigned long)(fracpart * 1e6); + min = sec / 60; + sec %= 60; + hour = min / 60; + min %= 60; + + if (us != 0) + PyOS_snprintf(buffer, sizeof(buffer), + "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, min, sec, us); + else + PyOS_snprintf(buffer, sizeof(buffer), + "Timeout (%lu:%02lu:%02lu)!\n", + hour, min, sec); + + return strdup(buffer); +} + +static PyObject* +faulthandler_dump_tracebacks_later(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"timeout", "repeat", "file", "exit", NULL}; + double timeout; + PY_TIMEOUT_T timeout_us; + int repeat = 0; + PyObject *file = NULL; + int fd; + int exit = 0; + PyThreadState *tstate; + char *header; + size_t header_len; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "d|iOi:dump_tracebacks_later", kwlist, + &timeout, &repeat, &file, &exit)) + return NULL; + if ((timeout * 1e6) >= (double) PY_TIMEOUT_MAX) { + PyErr_SetString(PyExc_OverflowError, "timeout value is too large"); + return NULL; + } + timeout_us = (PY_TIMEOUT_T)(timeout * 1e6); + if (timeout_us <= 0) { + PyErr_SetString(PyExc_ValueError, "timeout must be greater than 0"); + return NULL; + } + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + /* format the timeout */ + header = format_timeout(timeout); + if (header == NULL) + return PyErr_NoMemory(); + header_len = strlen(header); + + /* Cancel previous thread, if running */ + cancel_dump_tracebacks_later(); + + Py_XDECREF(thread.file); + Py_INCREF(file); + thread.file = file; + thread.fd = fd; + thread.timeout_us = timeout_us; + thread.repeat = repeat; + thread.interp = tstate->interp; + thread.exit = exit; + thread.header = header; + thread.header_len = header_len; + + /* Arm these locks to serve as events when released */ + PyThread_acquire_lock(thread.running, 1); + + if (PyThread_start_new_thread(faulthandler_thread, NULL) == -1) { + PyThread_release_lock(thread.running); + Py_CLEAR(thread.file); + free(header); + thread.header = NULL; + PyErr_SetString(PyExc_RuntimeError, + "unable to start watchdog thread"); + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject* +faulthandler_cancel_dump_tracebacks_later_py(PyObject *self) +{ + cancel_dump_tracebacks_later(); + Py_RETURN_NONE; +} +#endif /* FAULTHANDLER_LATER */ + +#ifdef FAULTHANDLER_USER +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ +#ifdef HAVE_SIGACTION + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } +#endif + return sigaction(signum, &action, p_previous); +#else + _Py_sighandler_t previous; + previous = signal(signum, faulthandler_user); + if (p_previous != NULL) + *p_previous = previous; + return (previous == SIG_ERR); +#endif +} + +/* Handler of user signals (e.g. SIGUSR1). + + Dump the traceback of the current thread, or of all threads if + thread.all_threads is true. + + This function is signal safe and should only call signal safe functions. */ + +static void +faulthandler_user(int signum) +{ + user_signal_t *user; + PyThreadState *tstate; + int save_errno = errno; + + user = &user_signals[signum]; + if (!user->enabled) + return; + +#ifdef WITH_THREAD + /* PyThreadState_Get() doesn't give the state of the current thread if + the thread doesn't hold the GIL. Read the thread local storage (TLS) + instead: call PyGILState_GetThisThreadState(). */ + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_Get(); +#endif + + if (user->all_threads) + _Py_DumpTracebackThreads(user->fd, user->interp, tstate); + else { + if (tstate == NULL) + return; + _Py_DumpTraceback(user->fd, tstate); + } +#ifdef HAVE_SIGACTION + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + /* call the previous signal handler */ + raise(signum); + (void)faulthandler_register(signum, user->chain, NULL); + } +#else + if (user->chain) { + /* call the previous signal handler */ + user->previous(signum); + } +#endif + errno = save_errno; +} + +static int +check_signum(int signum) +{ + unsigned int i; + + for (i=0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + PyErr_Format(PyExc_RuntimeError, + "signal %i cannot be registered, " + "use enable() instead", + signum); + return 0; + } + } + if (signum < 1 || NSIG <= signum) { + PyErr_SetString(PyExc_ValueError, "signal number out of range"); + return 0; + } + return 1; +} + +static PyObject* +faulthandler_register_py(PyObject *self, + PyObject *args, PyObject *kwargs) +{ + static char *kwlist[] = {"signum", "file", "all_threads", "chain", NULL}; + int signum; + PyObject *file = NULL; + int all_threads = 1; + int chain = 0; + int fd; + user_signal_t *user; + _Py_sighandler_t previous; + PyThreadState *tstate; + int err; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "i|Oii:register", kwlist, + &signum, &file, &all_threads, &chain)) + return NULL; + + if (!check_signum(signum)) + return NULL; + + tstate = get_thread_state(); + if (tstate == NULL) + return NULL; + + file = faulthandler_get_fileno(file, &fd); + if (file == NULL) + return NULL; + + if (user_signals == NULL) { + user_signals = calloc(NSIG, sizeof(user_signal_t)); + if (user_signals == NULL) + return PyErr_NoMemory(); + } + user = &user_signals[signum]; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); + if (err) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + } + + Py_XDECREF(user->file); + Py_INCREF(file); + user->file = file; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + user->previous = previous; + user->interp = tstate->interp; + user->enabled = 1; + + Py_RETURN_NONE; +} + +static int +faulthandler_unregister(user_signal_t *user, int signum) +{ + if (!user->enabled) + return 0; + user->enabled = 0; +#ifdef HAVE_SIGACTION + (void)sigaction(signum, &user->previous, NULL); +#else + (void)signal(signum, user->previous); +#endif + Py_CLEAR(user->file); + user->fd = -1; + return 1; +} + +static PyObject* +faulthandler_unregister_py(PyObject *self, PyObject *args) +{ + int signum; + user_signal_t *user; + int change; + + if (!PyArg_ParseTuple(args, "i:unregister", &signum)) + return NULL; + + if (!check_signum(signum)) + return NULL; + + if (user_signals == NULL) + Py_RETURN_FALSE; + + user = &user_signals[signum]; + change = faulthandler_unregister(user, signum); + return PyBool_FromLong(change); +} +#endif /* FAULTHANDLER_USER */ + + +static PyObject * +faulthandler_read_null(PyObject *self, PyObject *args) +{ + int *x = NULL, y; + int release_gil = 0; + if (!PyArg_ParseTuple(args, "|i:_read_null", &release_gil)) + return NULL; + if (release_gil) { + Py_BEGIN_ALLOW_THREADS + y = *x; + Py_END_ALLOW_THREADS + } else + y = *x; + return PyLong_FromLong(y); + +} + +static PyObject * +faulthandler_sigsegv(PyObject *self, PyObject *args) +{ +#if defined(MS_WINDOWS) + /* For SIGSEGV, faulthandler_fatal_error() restores the previous signal + handler and then gives back the execution flow to the program (without + explicitly calling the previous error handler). In a normal case, the + SIGSEGV was raised by the kernel because of a fault, and so if the + program retries to execute the same instruction, the fault will be + raised again. + + Here the fault is simulated by a fake SIGSEGV signal raised by the + application. We have to raise SIGSEGV at lease twice: once for + faulthandler_fatal_error(), and one more time for the previous signal + handler. */ + while(1) + raise(SIGSEGV); +#else + raise(SIGSEGV); +#endif + Py_RETURN_NONE; +} + +static PyObject * +faulthandler_sigfpe(PyObject *self, PyObject *args) +{ + /* Do an integer division by zero: raise a SIGFPE on Intel CPU, but not on + PowerPC. Use volatile to disable compile-time optimizations. */ + volatile int x = 1, y = 0, z; + z = x / y; + /* If the division by zero didn't raise a SIGFPE (e.g. on PowerPC), + raise it manually. */ + raise(SIGFPE); + /* This line is never reached, but we pretend to make something with z + to silence a compiler warning. */ + return PyLong_FromLong(z); +} + +static PyObject * +faulthandler_sigabrt(PyObject *self, PyObject *args) +{ +#ifdef _MSC_VER + /* Visual Studio: configure abort() to not display an error message nor + open a popup asking to report the fault. */ + _set_abort_behavior(0, _WRITE_ABORT_MSG | _CALL_REPORTFAULT); +#endif + abort(); + Py_RETURN_NONE; +} + +#ifdef SIGBUS +static PyObject * +faulthandler_sigbus(PyObject *self, PyObject *args) +{ + raise(SIGBUS); + Py_RETURN_NONE; +} +#endif + +#ifdef SIGILL +static PyObject * +faulthandler_sigill(PyObject *self, PyObject *args) +{ + raise(SIGILL); + Py_RETURN_NONE; +} +#endif + +static PyObject * +faulthandler_fatal_error_py(PyObject *self, PyObject *args) +{ + char *message; + if (!PyArg_ParseTuple(args, "y:fatal_error", &message)) + return NULL; + Py_FatalError(message); + Py_RETURN_NONE; +} + +#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) +static void* +stack_overflow(void *min_sp, void *max_sp, size_t *depth) +{ + /* allocate 4096 bytes on the stack at each call */ + unsigned char buffer[4096]; + void *sp = &buffer; + *depth += 1; + if (sp < min_sp || max_sp < sp) + return sp; + buffer[0] = 1; + buffer[4095] = 0; + return stack_overflow(min_sp, max_sp, depth); +} + +static PyObject * +faulthandler_stack_overflow(PyObject *self) +{ + size_t depth, size; + void *sp = &depth, *stop; + + depth = 0; + stop = stack_overflow(sp - STACK_OVERFLOW_MAX_SIZE, + sp + STACK_OVERFLOW_MAX_SIZE, + &depth); + if (sp < stop) + size = stop - sp; + else + size = sp - stop; + PyErr_Format(PyExc_RuntimeError, + "unable to raise a stack overflow (allocated %zu bytes " + "on the stack, %zu recursive calls)", + size, depth); + return NULL; +} +#endif + + +static int +faulthandler_traverse(PyObject *module, visitproc visit, void *arg) +{ +#ifdef FAULTHANDLER_USER + unsigned int signum; +#endif + +#ifdef FAULTHANDLER_LATER + Py_VISIT(thread.file); +#endif +#ifdef FAULTHANDLER_USER + if (user_signals != NULL) { + for (signum=0; signum < NSIG; signum++) + Py_VISIT(user_signals[signum].file); + } +#endif + Py_VISIT(fatal_error.file); + return 0; +} + +PyDoc_STRVAR(module_doc, +"faulthandler module."); + +static PyMethodDef module_methods[] = { + {"enable", + (PyCFunction)faulthandler_enable, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("enable(file=sys.stderr, all_threads=True): " + "enable the fault handler")}, + {"disable", (PyCFunction)faulthandler_disable_py, METH_NOARGS, + PyDoc_STR("disable(): disable the fault handler")}, + {"is_enabled", (PyCFunction)faulthandler_is_enabled, METH_NOARGS, + PyDoc_STR("is_enabled()->bool: check if the handler is enabled")}, + {"dump_traceback", + (PyCFunction)faulthandler_dump_traceback_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("dump_traceback(file=sys.stderr, all_threads=True): " + "dump the traceback of the current thread, or of all threads " + "if all_threads is True, into file")}, +#ifdef FAULTHANDLER_LATER + {"dump_tracebacks_later", + (PyCFunction)faulthandler_dump_tracebacks_later, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("dump_tracebacks_later(timeout, repeat=False, file=sys.stderrn, exit=False):\n" + "dump the traceback of all threads in timeout seconds,\n" + "or each timeout seconds if repeat is True. If exit is True, " + "call _exit(1) which is not safe.")}, + {"cancel_dump_tracebacks_later", + (PyCFunction)faulthandler_cancel_dump_tracebacks_later_py, METH_NOARGS, + PyDoc_STR("cancel_dump_tracebacks_later():\ncancel the previous call " + "to dump_tracebacks_later().")}, +#endif + +#ifdef FAULTHANDLER_USER + {"register", + (PyCFunction)faulthandler_register_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("register(signum, file=sys.stderr, all_threads=True, chain=False): " + "register an handler for the signal 'signum': dump the " + "traceback of the current thread, or of all threads if " + "all_threads is True, into file")}, + {"unregister", + faulthandler_unregister_py, METH_VARARGS|METH_KEYWORDS, + PyDoc_STR("unregister(signum): unregister the handler of the signal " + "'signum' registered by register()")}, +#endif + + {"_read_null", faulthandler_read_null, METH_VARARGS, + PyDoc_STR("_read_null(release_gil=False): read from NULL, raise " + "a SIGSEGV or SIGBUS signal depending on the platform")}, + {"_sigsegv", faulthandler_sigsegv, METH_VARARGS, + PyDoc_STR("_sigsegv(): raise a SIGSEGV signal")}, + {"_sigabrt", faulthandler_sigabrt, METH_VARARGS, + PyDoc_STR("_sigabrt(): raise a SIGABRT signal")}, + {"_sigfpe", (PyCFunction)faulthandler_sigfpe, METH_NOARGS, + PyDoc_STR("_sigfpe(): raise a SIGFPE signal")}, +#ifdef SIGBUS + {"_sigbus", (PyCFunction)faulthandler_sigbus, METH_NOARGS, + PyDoc_STR("_sigbus(): raise a SIGBUS signal")}, +#endif +#ifdef SIGILL + {"_sigill", (PyCFunction)faulthandler_sigill, METH_NOARGS, + PyDoc_STR("_sigill(): raise a SIGILL signal")}, +#endif + {"_fatal_error", faulthandler_fatal_error_py, METH_VARARGS, + PyDoc_STR("_fatal_error(message): call Py_FatalError(message)")}, +#if defined(HAVE_SIGALTSTACK) && defined(HAVE_SIGACTION) + {"_stack_overflow", (PyCFunction)faulthandler_stack_overflow, METH_NOARGS, + PyDoc_STR("_stack_overflow(): recursive call to raise a stack overflow")}, +#endif + {NULL, NULL} /* sentinel */ +}; + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "faulthandler", + module_doc, + 0, /* non-negative size to be able to unload the module */ + module_methods, + NULL, + faulthandler_traverse, + NULL, + NULL +}; + +PyMODINIT_FUNC +PyInit_faulthandler(void) +{ + return PyModule_Create(&module_def); +} + +/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable + is defined, or if sys._xoptions has a 'faulthandler' key. */ + +static int +faulthandler_env_options(void) +{ + PyObject *xoptions, *key, *module, *res; + + if (!Py_GETENV("PYTHONFAULTHANDLER")) { + int has_key; + + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return -1; + + key = PyUnicode_FromString("faulthandler"); + if (key == NULL) + return -1; + + has_key = PyDict_Contains(xoptions, key); + Py_DECREF(key); + if (!has_key) + return 0; + } + + module = PyImport_ImportModule("faulthandler"); + if (module == NULL) { + return -1; + } + res = PyObject_CallMethod(module, "enable", ""); + Py_DECREF(module); + if (res == NULL) + return -1; + Py_DECREF(res); + return 0; +} + +int _PyFaulthandler_Init(void) +{ +#ifdef HAVE_SIGALTSTACK + int err; + + /* Try to allocate an alternate stack for faulthandler() signal handler to + * be able to allocate memory on the stack, even on a stack overflow. If it + * fails, ignore the error. */ + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + stack.ss_sp = PyMem_Malloc(stack.ss_size); + if (stack.ss_sp != NULL) { + err = sigaltstack(&stack, NULL); + if (err) { + PyMem_Free(stack.ss_sp); + stack.ss_sp = NULL; + } + } +#endif +#ifdef FAULTHANDLER_LATER + thread.file = NULL; + thread.cancel_event = PyThread_allocate_lock(); + thread.running = PyThread_allocate_lock(); + if (!thread.cancel_event || !thread.running) { + PyErr_SetString(PyExc_RuntimeError, + "could not allocate locks for faulthandler"); + return -1; + } + PyThread_acquire_lock(thread.cancel_event, 1); +#endif + + return faulthandler_env_options(); +} + +void _PyFaulthandler_Fini(void) +{ +#ifdef FAULTHANDLER_USER + unsigned int signum; +#endif + +#ifdef FAULTHANDLER_LATER + /* later */ + cancel_dump_tracebacks_later(); + if (thread.cancel_event) { + PyThread_release_lock(thread.cancel_event); + PyThread_free_lock(thread.cancel_event); + thread.cancel_event = NULL; + } + if (thread.running) { + PyThread_free_lock(thread.running); + thread.running = NULL; + } +#endif + +#ifdef FAULTHANDLER_USER + /* user */ + if (user_signals != NULL) { + for (signum=0; signum < NSIG; signum++) + faulthandler_unregister(&user_signals[signum], signum); + free(user_signals); + user_signals = NULL; + } +#endif + + /* fatal */ + faulthandler_disable(); +#ifdef HAVE_SIGALTSTACK + if (stack.ss_sp != NULL) { + PyMem_Free(stack.ss_sp); + stack.ss_sp = NULL; + } +#endif +} diff --git a/Modules/fpectlmodule.c b/Modules/fpectlmodule.c index 1bb51cf..6af2f82 100644 --- a/Modules/fpectlmodule.c +++ b/Modules/fpectlmodule.c @@ -174,17 +174,6 @@ static void fpe_reset(Sigfunc *handler) fp_enable(TRP_INVALID | TRP_DIV_BY_ZERO | TRP_OVERFLOW); PyOS_setsig(SIGFPE, handler); -/*-- DEC ALPHA OSF --------------------------------------------------------*/ -#elif defined(__alpha) && defined(__osf__) - /* References: exception_intro, ieee man pages */ - /* cc -c -I/usr/local/python/include fpectlmodule.c */ - /* ld -shared -o fpectlmodule.so fpectlmodule.o */ -#include <machine/fpu.h> - unsigned long fp_control = - IEEE_TRAP_ENABLE_INV | IEEE_TRAP_ENABLE_DZE | IEEE_TRAP_ENABLE_OVF; - ieee_set_fp_control(fp_control); - PyOS_setsig(SIGFPE, handler); - /*-- DEC ALPHA LINUX ------------------------------------------------------*/ #elif defined(__alpha) && defined(linux) #include <asm/fpu.h> diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 10a4ed7..b05675c 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -680,8 +680,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) static void debug_cycle(char *msg, PyObject *op) { - PySys_WriteStderr("gc: %.100s <%.100s %p>\n", - msg, Py_TYPE(op)->tp_name, op); + PySys_FormatStderr("gc: %s <%s %p>\n", + msg, Py_TYPE(op)->tp_name, op); } /* Handle uncollectable garbage (cycles with finalizers, and stuff reachable diff --git a/Modules/getbuildinfo.c b/Modules/getbuildinfo.c index 7069b6e..0971a64 100644 --- a/Modules/getbuildinfo.c +++ b/Modules/getbuildinfo.c @@ -20,14 +20,6 @@ #endif #endif -/* on unix, SVNVERSION is passed on the command line. - * on Windows, the string is interpolated using - * subwcrev.exe - */ -#ifndef SVNVERSION -#define SVNVERSION "$WCRANGE$$WCMODS?M:$" -#endif - /* XXX Only unix build process has been tested */ #ifndef HGVERSION #define HGVERSION "" @@ -57,16 +49,6 @@ Py_GetBuildInfo(void) } const char * -_Py_svnversion(void) -{ - /* the following string can be modified by subwcrev.exe */ - static const char svnversion[] = SVNVERSION; - if (svnversion[0] != '$') - return svnversion; /* it was interpolated, or passed on command line */ - return "Unversioned directory"; -} - -const char * _Py_hgversion(void) { return HGVERSION; diff --git a/Modules/getpath.c b/Modules/getpath.c index b7f9573..7090879 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -406,7 +406,7 @@ calculate_path(void) static wchar_t delimiter[2] = {DELIM, '\0'}; static wchar_t separator[2] = {SEP, '\0'}; char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */ - wchar_t rtpypath[MAXPATHLEN+1]; + wchar_t *rtpypath = NULL; wchar_t *home = Py_GetPythonHome(); char *_path = getenv("PATH"); wchar_t *path_buffer = NULL; @@ -606,12 +606,12 @@ calculate_path(void) bufsz = 0; if (_rtpypath) { - size_t s = mbstowcs(rtpypath, _rtpypath, sizeof(rtpypath)/sizeof(wchar_t)); - if (s == (size_t)-1 || s >=sizeof(rtpypath)) - /* XXX deal with errors more gracefully */ + size_t rtpypath_len; + rtpypath = _Py_char2wchar(_rtpypath, &rtpypath_len); + if (rtpypath != NULL) + bufsz += rtpypath_len + 1; + else _rtpypath = NULL; - if (_rtpypath) - bufsz += wcslen(rtpypath) + 1; } defpath = _pythonpath; @@ -645,7 +645,7 @@ calculate_path(void) } else { /* Run-time value of $PYTHONPATH goes first */ - if (_rtpypath) { + if (rtpypath) { wcscpy(buf, rtpypath); wcscat(buf, delimiter); } @@ -719,6 +719,8 @@ calculate_path(void) PyMem_Free(_pythonpath); PyMem_Free(_prefix); PyMem_Free(_exec_prefix); + if (rtpypath != NULL) + PyMem_Free(rtpypath); } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 71d5bb6..ad22ec7 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -2590,6 +2590,7 @@ typedef struct { PyObject_HEAD PyObject *total; PyObject *it; + PyObject *binop; } accumulateobject; static PyTypeObject accumulate_type; @@ -2597,12 +2598,14 @@ static PyTypeObject accumulate_type; static PyObject * accumulate_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { - static char *kwargs[] = {"iterable", NULL}; + static char *kwargs[] = {"iterable", "func", NULL}; PyObject *iterable; PyObject *it; + PyObject *binop = NULL; accumulateobject *lz; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O:accumulate", kwargs, &iterable)) + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O:accumulate", + kwargs, &iterable, &binop)) return NULL; /* Get iterator. */ @@ -2617,6 +2620,8 @@ accumulate_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + Py_XINCREF(binop); + lz->binop = binop; lz->total = NULL; lz->it = it; return (PyObject *)lz; @@ -2626,6 +2631,7 @@ static void accumulate_dealloc(accumulateobject *lz) { PyObject_GC_UnTrack(lz); + Py_XDECREF(lz->binop); Py_XDECREF(lz->total); Py_XDECREF(lz->it); Py_TYPE(lz)->tp_free(lz); @@ -2634,6 +2640,7 @@ accumulate_dealloc(accumulateobject *lz) static int accumulate_traverse(accumulateobject *lz, visitproc visit, void *arg) { + Py_VISIT(lz->binop); Py_VISIT(lz->it); Py_VISIT(lz->total); return 0; @@ -2653,8 +2660,11 @@ accumulate_next(accumulateobject *lz) lz->total = val; return lz->total; } - - newtotal = PyNumber_Add(lz->total, val); + + if (lz->binop == NULL) + newtotal = PyNumber_Add(lz->total, val); + else + newtotal = PyObject_CallFunctionObjArgs(lz->binop, lz->total, val, NULL); Py_DECREF(val); if (newtotal == NULL) return NULL; @@ -2668,9 +2678,9 @@ accumulate_next(accumulateobject *lz) } PyDoc_STRVAR(accumulate_doc, -"accumulate(iterable) --> accumulate object\n\ +"accumulate(iterable[, func]) --> accumulate object\n\ \n\ -Return series of accumulated sums."); +Return series of accumulated sums (or other binary function results)."); static PyTypeObject accumulate_type = { PyVarObject_HEAD_INIT(NULL, 0) @@ -3631,7 +3641,7 @@ cycle(p) --> p0, p1, ... plast, p0, p1, ...\n\ repeat(elem [,n]) --> elem, elem, elem, ... endlessly or up to n times\n\ \n\ Iterators terminating on the shortest input sequence:\n\ -accumulate(p, start=0) --> p0, p0+p1, p0+p1+p2\n\ +accumulate(p[, func]) --> p0, p0+p1, p0+p1+p2\n\ chain(p, q, ...) --> p0, p1, ... plast, q0, q1, ... \n\ compress(data, selectors) --> (d[0] if s[0]), (d[1] if s[1]), ...\n\ dropwhile(pred, seq) --> seq[n], seq[n+1], starting when pred fails\n\ diff --git a/Modules/main.c b/Modules/main.c index fcd9330..747c12f 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -6,6 +6,7 @@ #include <locale.h> #ifdef __VMS +#error "PEP 11: VMS is now unsupported, code will be removed in Python 3.4" #include <unixlib.h> #endif @@ -100,6 +101,7 @@ static char *usage_5 = " The default module search path uses %s.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows).\n" "PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n" +"PYTHONFAULTHANDLER: dump the Python traceback on fatal errors.\n" ; static int @@ -577,7 +579,6 @@ Py_Main(int argc, wchar_t **argv) if ((p = Py_GETENV("PYTHONEXECUTABLE")) && *p != '\0') { wchar_t* buffer; size_t len = strlen(p); - size_t r; buffer = malloc(len * sizeof(wchar_t)); if (buffer == NULL) { @@ -585,7 +586,7 @@ Py_Main(int argc, wchar_t **argv) "not enough memory to copy PYTHONEXECUTABLE"); } - r = mbstowcs(buffer, p, len); + mbstowcs(buffer, p, len); Py_SetProgramName(buffer); /* buffer is now handed off - do not free */ } else { diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 29c32a3..cebb4ff 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -55,11 +55,6 @@ raised for division by zero and mod by zero. #include "Python.h" #include "_math.h" -#ifdef _OSF_SOURCE -/* OSF1 5.1 doesn't make this available with XOPEN_SOURCE_EXTENDED defined */ -extern double copysign(double, double); -#endif - /* sin(pi*x), giving accurate results for all finite x (especially x integral or close to an integer). This is here for use in the @@ -582,6 +577,61 @@ m_log(double x) } } +/* + log2: log to base 2. + + Uses an algorithm that should: + + (a) produce exact results for powers of 2, and + (b) give a monotonic log2 (for positive finite floats), + assuming that the system log is monotonic. +*/ + +static double +m_log2(double x) +{ + if (!Py_IS_FINITE(x)) { + if (Py_IS_NAN(x)) + return x; /* log2(nan) = nan */ + else if (x > 0.0) + return x; /* log2(+inf) = +inf */ + else { + errno = EDOM; + return Py_NAN; /* log2(-inf) = nan, invalid-operation */ + } + } + + if (x > 0.0) { +#ifdef HAVE_LOG2 + return log2(x); +#else + double m; + int e; + m = frexp(x, &e); + /* We want log2(m * 2**e) == log(m) / log(2) + e. Care is needed when + * x is just greater than 1.0: in that case e is 1, log(m) is negative, + * and we get significant cancellation error from the addition of + * log(m) / log(2) to e. The slight rewrite of the expression below + * avoids this problem. + */ + if (x >= 1.0) { + return log(2.0 * m) / log(2.0) + (e - 1); + } + else { + return log(m) / log(2.0) + e; + } +#endif + } + else if (x == 0.0) { + errno = EDOM; + return -Py_HUGE_VAL; /* log2(0) = -inf, divide-by-zero */ + } + else { + errno = EDOM; + return Py_NAN; /* log2(-inf) = nan, invalid-operation */ + } +} + static double m_log10(double x) { @@ -1628,6 +1678,15 @@ Return the logarithm of x to the given base.\n\ If the base not specified, returns the natural logarithm (base e) of x."); static PyObject * +math_log2(PyObject *self, PyObject *arg) +{ + return loghelper(arg, m_log2, "log2"); +} + +PyDoc_STRVAR(math_log2_doc, +"log2(x)\n\nReturn the base 2 logarithm of x."); + +static PyObject * math_log10(PyObject *self, PyObject *arg) { return loghelper(arg, m_log10, "log10"); @@ -1899,6 +1958,7 @@ static PyMethodDef math_methods[] = { {"log", math_log, METH_VARARGS, math_log_doc}, {"log1p", math_log1p, METH_O, math_log1p_doc}, {"log10", math_log10, METH_O, math_log10_doc}, + {"log2", math_log2, METH_O, math_log2_doc}, {"modf", math_modf, METH_O, math_modf_doc}, {"pow", math_pow, METH_VARARGS, math_pow_doc}, {"radians", math_radians, METH_O, math_radians_doc}, diff --git a/Modules/md5module.c b/Modules/md5module.c index 208930d..de43f1c 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -243,7 +243,7 @@ void md5_process(struct md5_state *md5, in += MD5_BLOCKSIZE; inlen -= MD5_BLOCKSIZE; } else { - n = MIN(inlen, (MD5_BLOCKSIZE - md5->curlen)); + n = MIN(inlen, (Py_ssize_t)(MD5_BLOCKSIZE - md5->curlen)); memcpy(md5->buf + md5->curlen, in, (size_t)n); md5->curlen += n; in += n; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index da9283b..5d086a7 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -240,15 +240,37 @@ mmap_read_line_method(mmap_object *self, return result; } +/* Basically the "n" format code with the ability to turn None into -1. */ +static int +mmap_convert_ssize_t(PyObject *obj, void *result) { + Py_ssize_t limit; + if (obj == Py_None) { + limit = -1; + } + else if (PyNumber_Check(obj)) { + limit = PyNumber_AsSsize_t(obj, PyExc_OverflowError); + if (limit == -1 && PyErr_Occurred()) + return 0; + } + else { + PyErr_Format(PyExc_TypeError, + "integer argument expected, got '%.200s'", + Py_TYPE(obj)->tp_name); + return 0; + } + *((Py_ssize_t *)result) = limit; + return 1; +} + static PyObject * mmap_read_method(mmap_object *self, PyObject *args) { - Py_ssize_t num_bytes, n; + Py_ssize_t num_bytes = -1, n; PyObject *result; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n:read", &num_bytes)) + if (!PyArg_ParseTuple(args, "|O&:read", mmap_convert_ssize_t, &num_bytes)) return(NULL); /* silently 'adjust' out-of-range requests */ @@ -645,9 +667,9 @@ mmap_move_method(mmap_object *self, PyObject *args) return NULL; } else { /* bounds check the values */ - if (cnt < 0 || (cnt + dest) < cnt || (cnt + src) < cnt || - src < 0 || src > self->size || (src + cnt) > self->size || - dest < 0 || dest > self->size || (dest + cnt) > self->size) { + if ((cnt + dest) < cnt || (cnt + src) < cnt || + src > self->size || (src + cnt) > self->size || + dest > self->size || (dest + cnt) > self->size) { PyErr_SetString(PyExc_ValueError, "source, destination, or count out of range"); return NULL; diff --git a/Modules/nismodule.c b/Modules/nismodule.c index a81ca8c..0af495f 100644 --- a/Modules/nismodule.c +++ b/Modules/nismodule.c @@ -411,7 +411,7 @@ nis_maps (PyObject *self, PyObject *args, PyObject *kwdict) return NULL; if ((list = PyList_New(0)) == NULL) return NULL; - for (maps = maps; maps; maps = maps->next) { + for (; maps; maps = maps->next) { PyObject *str = PyUnicode_FromString(maps->map); if (!str || PyList_Append(list, str) < 0) { diff --git a/Modules/ossaudiodev.c b/Modules/ossaudiodev.c index e660e50..1505731 100644 --- a/Modules/ossaudiodev.c +++ b/Modules/ossaudiodev.c @@ -213,6 +213,21 @@ oss_mixer_dealloc(oss_mixer_t *self) * Helper functions */ +/* Check if a given file descriptor is valid (i.e. hasn't been closed). + * If true, return 1. Otherwise, raise ValueError and return 0. + */ +static int _is_fd_valid(int fd) +{ + /* the FD is set to -1 in oss_close()/oss_mixer_close() */ + if (fd >= 0) { + return 1; + } else { + PyErr_SetString(PyExc_ValueError, + "Operation on closed OSS device."); + return 0; + } +} + /* _do_ioctl_1() is a private helper function used for the OSS ioctls -- SNDCTL_DSP_{SETFMT,CHANNELS,SPEED} -- that that are called from C like this: @@ -300,6 +315,9 @@ _do_ioctl_0(int fd, PyObject *args, char *fname, int cmd) static PyObject * oss_nonblock(oss_audio_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + /* Hmmm: it doesn't appear to be possible to return to blocking mode once we're in non-blocking mode! */ if (ioctl(self->fd, SNDCTL_DSP_NONBLOCK, NULL) == -1) @@ -311,6 +329,9 @@ oss_nonblock(oss_audio_t *self, PyObject *unused) static PyObject * oss_setfmt(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "setfmt", SNDCTL_DSP_SETFMT); } @@ -318,6 +339,10 @@ static PyObject * oss_getfmts(oss_audio_t *self, PyObject *unused) { int mask; + + if (!_is_fd_valid(self->fd)) + return NULL; + if (ioctl(self->fd, SNDCTL_DSP_GETFMTS, &mask) == -1) return PyErr_SetFromErrno(PyExc_IOError); return PyLong_FromLong(mask); @@ -326,30 +351,45 @@ oss_getfmts(oss_audio_t *self, PyObject *unused) static PyObject * oss_channels(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "channels", SNDCTL_DSP_CHANNELS); } static PyObject * oss_speed(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "speed", SNDCTL_DSP_SPEED); } static PyObject * oss_sync(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "sync", SNDCTL_DSP_SYNC); } static PyObject * oss_reset(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "reset", SNDCTL_DSP_RESET); } static PyObject * oss_post(oss_audio_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_0(self->fd, args, "post", SNDCTL_DSP_POST); } @@ -364,6 +404,9 @@ oss_read(oss_audio_t *self, PyObject *args) char *cp; PyObject *rv; + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "i:read", &size)) return NULL; rv = PyBytes_FromStringAndSize(NULL, size); @@ -391,6 +434,9 @@ oss_write(oss_audio_t *self, PyObject *args) char *cp; int rv, size; + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "y#:write", &cp, &size)) { return NULL; } @@ -422,6 +468,9 @@ oss_writeall(oss_audio_t *self, PyObject *args) mode, the behaviour of write() and writeall() from Python is indistinguishable. */ + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "y#:write", &cp, &size)) return NULL; @@ -489,6 +538,9 @@ oss_exit(PyObject *self, PyObject *unused) static PyObject * oss_fileno(oss_audio_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + return PyLong_FromLong(self->fd); } @@ -503,6 +555,9 @@ oss_setparameters(oss_audio_t *self, PyObject *args) int fmt, channels, rate; PyObject * rv; /* return tuple (fmt, channels, rate) */ + if (!_is_fd_valid(self->fd)) + return NULL; + if (!PyArg_ParseTuple(args, "iii|i:setparameters", &wanted_fmt, &wanted_channels, &wanted_rate, &strict)) @@ -593,6 +648,9 @@ oss_bufsize(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -612,6 +670,9 @@ oss_obufcount(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -632,6 +693,9 @@ oss_obuffree(oss_audio_t *self, PyObject *unused) audio_buf_info ai; int nchannels=0, ssize=0; + if (!_is_fd_valid(self->fd)) + return NULL; + if (_ssize(self, &nchannels, &ssize) < 0 || !nchannels || !ssize) { PyErr_SetFromErrno(PyExc_IOError); return NULL; @@ -649,6 +713,9 @@ oss_getptr(oss_audio_t *self, PyObject *unused) count_info info; int req; + if (!_is_fd_valid(self->fd)) + return NULL; + if (self->mode == O_RDONLY) req = SNDCTL_DSP_GETIPTR; else @@ -679,6 +746,9 @@ oss_mixer_close(oss_mixer_t *self, PyObject *unused) static PyObject * oss_mixer_fileno(oss_mixer_t *self, PyObject *unused) { + if (!_is_fd_valid(self->fd)) + return NULL; + return PyLong_FromLong(self->fd); } @@ -687,6 +757,9 @@ oss_mixer_fileno(oss_mixer_t *self, PyObject *unused) static PyObject * oss_mixer_controls(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "controls", SOUND_MIXER_READ_DEVMASK); } @@ -694,6 +767,9 @@ oss_mixer_controls(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_stereocontrols(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "stereocontrols", SOUND_MIXER_READ_STEREODEVS); } @@ -701,6 +777,9 @@ oss_mixer_stereocontrols(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_reccontrols(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "reccontrols", SOUND_MIXER_READ_RECMASK); } @@ -710,6 +789,9 @@ oss_mixer_get(oss_mixer_t *self, PyObject *args) { int channel, volume; + if (!_is_fd_valid(self->fd)) + return NULL; + /* Can't use _do_ioctl_1 because of encoded arg thingy. */ if (!PyArg_ParseTuple(args, "i:get", &channel)) return NULL; @@ -730,6 +812,9 @@ oss_mixer_set(oss_mixer_t *self, PyObject *args) { int channel, volume, leftVol, rightVol; + if (!_is_fd_valid(self->fd)) + return NULL; + /* Can't use _do_ioctl_1 because of encoded arg thingy. */ if (!PyArg_ParseTuple(args, "i(ii):set", &channel, &leftVol, &rightVol)) return NULL; @@ -755,6 +840,9 @@ oss_mixer_set(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_get_recsrc(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1_internal(self->fd, args, "get_recsrc", SOUND_MIXER_READ_RECSRC); } @@ -762,6 +850,9 @@ oss_mixer_get_recsrc(oss_mixer_t *self, PyObject *args) static PyObject * oss_mixer_set_recsrc(oss_mixer_t *self, PyObject *args) { + if (!_is_fd_valid(self->fd)) + return NULL; + return _do_ioctl_1(self->fd, args, "set_recsrc", SOUND_MIXER_WRITE_RECSRC); } diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 3cdf135..f1679d7 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -581,8 +581,10 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) if (res) ((PyST_Object *)res)->st_flags.cf_flags = flags & PyCF_MASK; } - else + else { PyParser_SetError(&err); + } + PyParser_ClearError(&err); } return (res); } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 702fec0..d701690 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -28,6 +28,7 @@ #include "Python.h" #if defined(__VMS) +# error "PEP 11: VMS is now unsupported, code will be removed in Python 3.4" # include <unixio.h> #endif /* defined(__VMS) */ @@ -43,6 +44,7 @@ corresponding Unix manual entries for more information on calls."); #if defined(PYOS_OS2) +#error "PEP 11: OS/2 is now unsupported, code will be removed in Python 3.4" #define INCL_DOS #define INCL_DOSERRORS #define INCL_DOSPROCESS @@ -57,6 +59,10 @@ corresponding Unix manual entries for more information on calls."); #include "osdefs.h" #endif +#ifdef HAVE_SYS_UIO_H +#include <sys/uio.h> +#endif + #ifdef HAVE_SYS_TYPES_H #include <sys/types.h> #endif /* HAVE_SYS_TYPES_H */ @@ -93,6 +99,20 @@ corresponding Unix manual entries for more information on calls."); #include <langinfo.h> #endif +#ifdef HAVE_SYS_SENDFILE_H +#include <sys/sendfile.h> +#endif + +#ifdef HAVE_SCHED_H +#include <sched.h> +#endif + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif +#endif + /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ #if defined(PYCC_VACPP) && defined(PYOS_OS2) @@ -347,6 +367,19 @@ static int win32_can_symlink = 0; #endif #endif +static int +_parse_off_t(PyObject* arg, void* addr) +{ +#if !defined(HAVE_LARGEFILE_SUPPORT) + *((off_t*)addr) = PyLong_AsLong(arg); +#else + *((off_t*)addr) = PyLong_AsLongLong(arg); +#endif + if (PyErr_Occurred()) + return 0; + return 1; +} + #if defined _MSC_VER && _MSC_VER >= 1400 /* Microsoft CRT in VS2005 and higher will verify that a filehandle is * valid and throw an assertion if it isn't. @@ -1544,9 +1577,39 @@ static PyStructSequence_Desc statvfs_result_desc = { 10 }; +#if defined(HAVE_WAITID) && !defined(__APPLE__) +PyDoc_STRVAR(waitid_result__doc__, +"waitid_result: Result from waitid.\n\n\ +This object may be accessed either as a tuple of\n\ + (si_pid, si_uid, si_signo, si_status, si_code),\n\ +or via the attributes si_pid, si_uid, and so on.\n\ +\n\ +See os.waitid for more information."); + +static PyStructSequence_Field waitid_result_fields[] = { + {"si_pid", }, + {"si_uid", }, + {"si_signo", }, + {"si_status", }, + {"si_code", }, + {0} +}; + +static PyStructSequence_Desc waitid_result_desc = { + "waitid_result", /* name */ + waitid_result__doc__, /* doc */ + waitid_result_fields, + 5 +}; +static PyTypeObject WaitidResultType; +#endif + static int initialized; static PyTypeObject StatResultType; static PyTypeObject StatVFSResultType; +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) +static PyTypeObject SchedParamType; +#endif static newfunc structseq_new; static PyObject * @@ -2143,6 +2206,21 @@ posix_fsync(PyObject *self, PyObject *fdobj) } #endif /* HAVE_FSYNC */ +#ifdef HAVE_SYNC +PyDoc_STRVAR(posix_sync__doc__, +"sync()\n\n\ +Force write of everything to disk."); + +static PyObject * +posix_sync(PyObject *self, PyObject *noargs) +{ + Py_BEGIN_ALLOW_THREADS + sync(); + Py_END_ALLOW_THREADS + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_FDATASYNC #ifdef __hpux @@ -2718,6 +2796,76 @@ posix_listdir(PyObject *self, PyObject *args) #endif /* which OS */ } /* end of posix_listdir */ +#ifdef HAVE_FDOPENDIR +PyDoc_STRVAR(posix_fdlistdir__doc__, +"fdlistdir(fd) -> list_of_strings\n\n\ +Like listdir(), but uses a file descriptor instead.\n\ +After succesful execution of this function, fd will be closed."); + +static PyObject * +posix_fdlistdir(PyObject *self, PyObject *args) +{ + PyObject *d, *v; + DIR *dirp; + struct dirent *ep; + int fd; + + errno = 0; + if (!PyArg_ParseTuple(args, "i:fdlistdir", &fd)) + return NULL; + Py_BEGIN_ALLOW_THREADS + dirp = fdopendir(fd); + Py_END_ALLOW_THREADS + if (dirp == NULL) { + close(fd); + return posix_error(); + } + if ((d = PyList_New(0)) == NULL) { + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + return NULL; + } + for (;;) { + errno = 0; + Py_BEGIN_ALLOW_THREADS + ep = readdir(dirp); + Py_END_ALLOW_THREADS + if (ep == NULL) { + if (errno == 0) { + break; + } else { + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + Py_DECREF(d); + return posix_error(); + } + } + if (ep->d_name[0] == '.' && + (NAMLEN(ep) == 1 || + (ep->d_name[1] == '.' && NAMLEN(ep) == 2))) + continue; + v = PyUnicode_DecodeFSDefaultAndSize(ep->d_name, NAMLEN(ep)); + if (v == NULL) { + Py_CLEAR(d); + break; + } + if (PyList_Append(d, v) != 0) { + Py_DECREF(v); + Py_CLEAR(d); + break; + } + Py_DECREF(v); + } + Py_BEGIN_ALLOW_THREADS + closedir(dirp); + Py_END_ALLOW_THREADS + + return d; +} +#endif + #ifdef MS_WINDOWS /* A helper function for abspath on win32 */ static PyObject * @@ -3015,6 +3163,48 @@ posix_nice(PyObject *self, PyObject *args) } #endif /* HAVE_NICE */ + +#ifdef HAVE_GETPRIORITY +PyDoc_STRVAR(posix_getpriority__doc__, +"getpriority(which, who) -> current_priority\n\n\ +Get program scheduling priority."); + +static PyObject * +posix_getpriority(PyObject *self, PyObject *args) +{ + int which, who, retval; + + if (!PyArg_ParseTuple(args, "ii", &which, &who)) + return NULL; + errno = 0; + retval = getpriority(which, who); + if (errno != 0) + return posix_error(); + return PyLong_FromLong((long)retval); +} +#endif /* HAVE_GETPRIORITY */ + + +#ifdef HAVE_SETPRIORITY +PyDoc_STRVAR(posix_setpriority__doc__, +"setpriority(which, who, prio) -> None\n\n\ +Set program scheduling priority."); + +static PyObject * +posix_setpriority(PyObject *self, PyObject *args) +{ + int which, who, prio, retval; + + if (!PyArg_ParseTuple(args, "iii", &which, &who, &prio)) + return NULL; + retval = setpriority(which, who, prio); + if (retval == -1) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_SETPRIORITY */ + + PyDoc_STRVAR(posix_rename__doc__, "rename(old, new)\n\n\ Rename a file or directory."); @@ -3430,6 +3620,167 @@ done: #endif /* MS_WINDOWS */ } +#ifdef HAVE_FUTIMES +PyDoc_STRVAR(posix_futimes__doc__, +"futimes(fd, (atime, mtime))\n\ +futimes(fd, None)\n\n\ +Set the access and modified time of the file specified by the file\n\ +descriptor fd to the given values. If the second form is used, set the\n\ +access and modified times to the current time."); + +static PyObject * +posix_futimes(PyObject *self, PyObject *args) +{ + int res, fd; + PyObject* arg; + struct timeval buf[2]; + long ausec, musec; + + if (!PyArg_ParseTuple(args, "iO:futimes", &fd, &arg)) + return NULL; + + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimes(fd, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimes() arg 2 must be a tuple (atime, mtime)"); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &(buf[0].tv_sec), &ausec) == -1) { + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &(buf[1].tv_sec), &musec) == -1) { + return NULL; + } + buf[0].tv_usec = ausec; + buf[1].tv_usec = musec; + Py_BEGIN_ALLOW_THREADS + res = futimes(fd, buf); + Py_END_ALLOW_THREADS + } + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_LUTIMES +PyDoc_STRVAR(posix_lutimes__doc__, +"lutimes(path, (atime, mtime))\n\ +lutimes(path, None)\n\n\ +Like utime(), but if path is a symbolic link, it is not dereferenced."); + +static PyObject * +posix_lutimes(PyObject *self, PyObject *args) +{ + PyObject *opath, *arg; + const char *path; + int res; + struct timeval buf[2]; + long ausec, musec; + + if (!PyArg_ParseTuple(args, "O&O:lutimes", + PyUnicode_FSConverter, &opath, &arg)) + return NULL; + path = PyBytes_AsString(opath); + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = lutimes(path, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "lutimes() arg 2 must be a tuple (atime, mtime)"); + Py_DECREF(opath); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &(buf[0].tv_sec), &ausec) == -1) { + Py_DECREF(opath); + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &(buf[1].tv_sec), &musec) == -1) { + Py_DECREF(opath); + return NULL; + } + buf[0].tv_usec = ausec; + buf[1].tv_usec = musec; + Py_BEGIN_ALLOW_THREADS + res = lutimes(path, buf); + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_FUTIMENS +PyDoc_STRVAR(posix_futimens__doc__, +"futimens(fd, (atime_sec, atime_nsec), (mtime_sec, mtime_nsec))\n\ +futimens(fd, None, None)\n\n\ +Updates the timestamps of a file specified by the file descriptor fd, with\n\ +nanosecond precision.\n\ +The second form sets atime and mtime to the current time.\n\ +If *_nsec is specified as UTIME_NOW, the timestamp is updated to the\n\ +current time.\n\ +If *_nsec is specified as UTIME_OMIT, the timestamp is not updated."); + +static PyObject * +posix_futimens(PyObject *self, PyObject *args) +{ + int res, fd; + PyObject *atime, *mtime; + struct timespec buf[2]; + + if (!PyArg_ParseTuple(args, "iOO:futimens", + &fd, &atime, &mtime)) + return NULL; + if (atime == Py_None && mtime == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimens(fd, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(atime) || PyTuple_Size(atime) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimens() arg 2 must be a tuple (atime_sec, atime_nsec)"); + return NULL; + } + else if (!PyTuple_Check(mtime) || PyTuple_Size(mtime) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimens() arg 3 must be a tuple (mtime_sec, mtime_nsec)"); + return NULL; + } + else { + if (!PyArg_ParseTuple(atime, "ll:futimens", + &(buf[0].tv_sec), &(buf[0].tv_nsec))) { + return NULL; + } + if (!PyArg_ParseTuple(mtime, "ll:futimens", + &(buf[1].tv_sec), &(buf[1].tv_nsec))) { + return NULL; + } + Py_BEGIN_ALLOW_THREADS + res = futimens(fd, buf); + Py_END_ALLOW_THREADS + } + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* Process operations */ @@ -3474,79 +3825,7 @@ int fsconvert_strdup(PyObject *o, char**out) } #endif - -#ifdef HAVE_EXECV -PyDoc_STRVAR(posix_execv__doc__, -"execv(path, args)\n\n\ -Execute an executable path with arguments, replacing current process.\n\ -\n\ - path: path of executable file\n\ - args: tuple or list of strings"); - -static PyObject * -posix_execv(PyObject *self, PyObject *args) -{ - PyObject *opath; - char *path; - PyObject *argv; - char **argvlist; - Py_ssize_t i, argc; - PyObject *(*getitem)(PyObject *, Py_ssize_t); - - /* execv has two arguments: (path, argv), where - argv is a list or tuple of strings. */ - - if (!PyArg_ParseTuple(args, "O&O:execv", - PyUnicode_FSConverter, - &opath, &argv)) - return NULL; - path = PyBytes_AsString(opath); - if (PyList_Check(argv)) { - argc = PyList_Size(argv); - getitem = PyList_GetItem; - } - else if (PyTuple_Check(argv)) { - argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; - } - else { - PyErr_SetString(PyExc_TypeError, "execv() arg 2 must be a tuple or list"); - Py_DECREF(opath); - return NULL; - } - if (argc < 1) { - PyErr_SetString(PyExc_ValueError, "execv() arg 2 must not be empty"); - Py_DECREF(opath); - return NULL; - } - - argvlist = PyMem_NEW(char *, argc+1); - if (argvlist == NULL) { - Py_DECREF(opath); - return PyErr_NoMemory(); - } - for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) { - free_string_array(argvlist, i); - PyErr_SetString(PyExc_TypeError, - "execv() arg 2 must contain only strings"); - Py_DECREF(opath); - return NULL; - - } - } - argvlist[argc] = NULL; - - execv(path, argvlist); - - /* If we get here it's definitely an error */ - - free_string_array(argvlist, argc); - Py_DECREF(opath); - return posix_error(); -} - +#if defined(HAVE_EXECV) || defined (HAVE_FEXECVE) static char** parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) { @@ -3628,6 +3907,87 @@ error: return NULL; } +static char** +parse_arglist(PyObject* argv, Py_ssize_t *argc) +{ + int i; + char **argvlist = PyMem_NEW(char *, *argc+1); + if (argvlist == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (i = 0; i < *argc; i++) { + PyObject* item = PySequence_ITEM(argv, i); + if (item == NULL) + goto fail; + if (!fsconvert_strdup(item, &argvlist[i])) { + Py_DECREF(item); + goto fail; + } + Py_DECREF(item); + } + argvlist[*argc] = NULL; + return argvlist; +fail: + *argc = i; + free_string_array(argvlist, *argc); + return NULL; +} +#endif + +#ifdef HAVE_EXECV +PyDoc_STRVAR(posix_execv__doc__, +"execv(path, args)\n\n\ +Execute an executable path with arguments, replacing current process.\n\ +\n\ + path: path of executable file\n\ + args: tuple or list of strings"); + +static PyObject * +posix_execv(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + PyObject *argv; + char **argvlist; + Py_ssize_t argc; + + /* execv has two arguments: (path, argv), where + argv is a list or tuple of strings. */ + + if (!PyArg_ParseTuple(args, "O&O:execv", + PyUnicode_FSConverter, + &opath, &argv)) + return NULL; + path = PyBytes_AsString(opath); + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "execv() arg 2 must be a tuple or list"); + Py_DECREF(opath); + return NULL; + } + argc = PySequence_Size(argv); + if (argc < 1) { + PyErr_SetString(PyExc_ValueError, "execv() arg 2 must not be empty"); + Py_DECREF(opath); + return NULL; + } + + argvlist = parse_arglist(argv, &argc); + if (argvlist == NULL) { + Py_DECREF(opath); + return NULL; + } + + execv(path, argvlist); + + /* If we get here it's definitely an error */ + + free_string_array(argvlist, argc); + Py_DECREF(opath); + return posix_error(); +} + PyDoc_STRVAR(posix_execve__doc__, "execve(path, args, env)\n\n\ Execute a path with arguments and environment, replacing current process.\n\ @@ -3644,9 +4004,7 @@ posix_execve(PyObject *self, PyObject *args) PyObject *argv, *env; char **argvlist; char **envlist; - Py_ssize_t i, argc, envc; - PyObject *(*getitem)(PyObject *, Py_ssize_t); - Py_ssize_t lastarg = 0; + Py_ssize_t argc, envc; /* execve has three arguments: (path, argv, env), where argv is a list or tuple of strings and env is a dictionary @@ -3657,40 +4015,22 @@ posix_execve(PyObject *self, PyObject *args) &opath, &argv, &env)) return NULL; path = PyBytes_AsString(opath); - if (PyList_Check(argv)) { - argc = PyList_Size(argv); - getitem = PyList_GetItem; - } - else if (PyTuple_Check(argv)) { - argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; - } - else { + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { PyErr_SetString(PyExc_TypeError, "execve() arg 2 must be a tuple or list"); goto fail_0; } + argc = PySequence_Size(argv); if (!PyMapping_Check(env)) { PyErr_SetString(PyExc_TypeError, "execve() arg 3 must be a mapping object"); goto fail_0; } - argvlist = PyMem_NEW(char *, argc+1); + argvlist = parse_arglist(argv, &argc); if (argvlist == NULL) { - PyErr_NoMemory(); goto fail_0; } - for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) - { - lastarg = i; - goto fail_1; - } - } - lastarg = argc; - argvlist[argc] = NULL; envlist = parse_envlist(env, &envc); if (envlist == NULL) @@ -3706,13 +4046,69 @@ posix_execve(PyObject *self, PyObject *args) PyMem_DEL(envlist[envc]); PyMem_DEL(envlist); fail_1: - free_string_array(argvlist, lastarg); + free_string_array(argvlist, argc); fail_0: Py_DECREF(opath); return NULL; } #endif /* HAVE_EXECV */ +#ifdef HAVE_FEXECVE +PyDoc_STRVAR(posix_fexecve__doc__, +"fexecve(fd, args, env)\n\n\ +Execute the program specified by a file descriptor with arguments and\n\ +environment, replacing the current process.\n\ +\n\ + fd: file descriptor of executable\n\ + args: tuple or list of arguments\n\ + env: dictionary of strings mapping to strings"); + +static PyObject * +posix_fexecve(PyObject *self, PyObject *args) +{ + int fd; + PyObject *argv, *env; + char **argvlist; + char **envlist; + Py_ssize_t argc, envc; + + if (!PyArg_ParseTuple(args, "iOO:fexecve", + &fd, &argv, &env)) + return NULL; + if (!PyList_Check(argv) && !PyTuple_Check(argv)) { + PyErr_SetString(PyExc_TypeError, + "fexecve() arg 2 must be a tuple or list"); + return NULL; + } + argc = PySequence_Size(argv); + if (!PyMapping_Check(env)) { + PyErr_SetString(PyExc_TypeError, + "fexecve() arg 3 must be a mapping object"); + return NULL; + } + + argvlist = parse_arglist(argv, &argc); + if (argvlist == NULL) + return NULL; + + envlist = parse_envlist(env, &envc); + if (envlist == NULL) + goto fail; + + fexecve(fd, argvlist, envlist); + + /* If we get here it's definitely an error */ + + (void) posix_error(); + + while (--envc >= 0) + PyMem_DEL(envlist[envc]); + PyMem_DEL(envlist); + fail: + free_string_array(argvlist, argc); + return NULL; +} +#endif /* HAVE_FEXECVE */ #ifdef HAVE_SPAWNV PyDoc_STRVAR(posix_spawnv__doc__, @@ -4153,6 +4549,564 @@ posix_fork(PyObject *self, PyObject *noargs) } #endif +#ifdef HAVE_SCHED_H + +PyDoc_STRVAR(posix_sched_get_priority_max__doc__, +"sched_get_priority_max(policy)\n\n\ +Get the maximum scheduling priority for *policy*."); + +static PyObject * +posix_sched_get_priority_max(PyObject *self, PyObject *args) +{ + int policy, max; + + if (!PyArg_ParseTuple(args, "i:sched_get_priority_max", &policy)) + return NULL; + max = sched_get_priority_max(policy); + if (max < 0) + return posix_error(); + return PyLong_FromLong(max); +} + +PyDoc_STRVAR(posix_sched_get_priority_min__doc__, +"sched_get_priority_min(policy)\n\n\ +Get the minimum scheduling priority for *policy*."); + +static PyObject * +posix_sched_get_priority_min(PyObject *self, PyObject *args) +{ + int policy, min; + + if (!PyArg_ParseTuple(args, "i:sched_get_priority_min", &policy)) + return NULL; + min = sched_get_priority_min(policy); + if (min < 0) + return posix_error(); + return PyLong_FromLong(min); +} + +#ifdef HAVE_SCHED_SETSCHEDULER + +PyDoc_STRVAR(posix_sched_getscheduler__doc__, +"sched_getscheduler(pid)\n\n\ +Get the scheduling policy for the process with a PID of *pid*.\n\ +Passing a PID of 0 returns the scheduling policy for the calling process."); + +static PyObject * +posix_sched_getscheduler(PyObject *self, PyObject *args) +{ + pid_t pid; + int policy; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_getscheduler", &pid)) + return NULL; + policy = sched_getscheduler(pid); + if (policy < 0) + return posix_error(); + return PyLong_FromLong(policy); +} + +#endif + +#if defined(HAVE_SCHED_SETSCHEDULER) || defined(HAVE_SCHED_SETPARAM) + +static PyObject * +sched_param_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *res, *priority; + static char *kwlist[] = {"sched_priority", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O:sched_param", kwlist, &priority)) + return NULL; + res = PyStructSequence_New(type); + if (!res) + return NULL; + Py_INCREF(priority); + PyStructSequence_SET_ITEM(res, 0, priority); + return res; +} + +PyDoc_STRVAR(sched_param__doc__, +"sched_param(sched_priority): A scheduling parameter.\n\n\ +Current has only one field: sched_priority"); + +static PyStructSequence_Field sched_param_fields[] = { + {"sched_priority", "the scheduling priority"}, + {0} +}; + +static PyStructSequence_Desc sched_param_desc = { + "sched_param", /* name */ + sched_param__doc__, /* doc */ + sched_param_fields, + 1 +}; + +static int +convert_sched_param(PyObject *param, struct sched_param *res) +{ + long priority; + + if (Py_TYPE(param) != &SchedParamType) { + PyErr_SetString(PyExc_TypeError, "must have a sched_param object"); + return 0; + } + priority = PyLong_AsLong(PyStructSequence_GET_ITEM(param, 0)); + if (priority == -1 && PyErr_Occurred()) + return 0; + if (priority > INT_MAX || priority < INT_MIN) { + PyErr_SetString(PyExc_OverflowError, "sched_priority out of range"); + return 0; + } + res->sched_priority = Py_SAFE_DOWNCAST(priority, long, int); + return 1; +} + +#endif + +#ifdef HAVE_SCHED_SETSCHEDULER + +PyDoc_STRVAR(posix_sched_setscheduler__doc__, +"sched_setscheduler(pid, policy, param)\n\n\ +Set the scheduling policy, *policy*, for *pid*.\n\ +If *pid* is 0, the calling process is changed.\n\ +*param* is an instance of sched_param."); + +static PyObject * +posix_sched_setscheduler(PyObject *self, PyObject *args) +{ + pid_t pid; + int policy; + struct sched_param param; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "iO&:sched_setscheduler", + &pid, &policy, &convert_sched_param, ¶m)) + return NULL; + if (sched_setscheduler(pid, policy, ¶m)) + return posix_error(); + Py_RETURN_NONE; +} + +#endif + +#ifdef HAVE_SCHED_SETPARAM + +PyDoc_STRVAR(posix_sched_getparam__doc__, +"sched_getparam(pid) -> sched_param\n\n\ +Returns scheduling parameters for the process with *pid* as an instance of the\n\ +sched_param class. A PID of 0 means the calling process."); + +static PyObject * +posix_sched_getparam(PyObject *self, PyObject *args) +{ + pid_t pid; + struct sched_param param; + PyObject *res, *priority; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_getparam", &pid)) + return NULL; + if (sched_getparam(pid, ¶m)) + return posix_error(); + res = PyStructSequence_New(&SchedParamType); + if (!res) + return NULL; + priority = PyLong_FromLong(param.sched_priority); + if (!priority) { + Py_DECREF(res); + return NULL; + } + PyStructSequence_SET_ITEM(res, 0, priority); + return res; +} + +PyDoc_STRVAR(posix_sched_setparam__doc__, +"sched_setparam(pid, param)\n\n\ +Set scheduling parameters for a process with PID *pid*.\n\ +A PID of 0 means the calling process."); + +static PyObject * +posix_sched_setparam(PyObject *self, PyObject *args) +{ + pid_t pid; + struct sched_param param; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O&:sched_setparam", + &pid, &convert_sched_param, ¶m)) + return NULL; + if (sched_setparam(pid, ¶m)) + return posix_error(); + Py_RETURN_NONE; +} + +#endif + +#ifdef HAVE_SCHED_RR_GET_INTERVAL + +PyDoc_STRVAR(posix_sched_rr_get_interval__doc__, +"sched_rr_get_interval(pid) -> float\n\n\ +Return the round-robin quantum for the process with PID *pid* in seconds."); + +static PyObject * +posix_sched_rr_get_interval(PyObject *self, PyObject *args) +{ + pid_t pid; + struct timespec interval; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID ":sched_rr_get_interval", &pid)) + return NULL; + if (sched_rr_get_interval(pid, &interval)) + return posix_error(); + return PyFloat_FromDouble((double)interval.tv_sec + 1e-9*interval.tv_nsec); +} + +#endif + +PyDoc_STRVAR(posix_sched_yield__doc__, +"sched_yield()\n\n\ +Voluntarily relinquish the CPU."); + +static PyObject * +posix_sched_yield(PyObject *self, PyObject *noargs) +{ + if (sched_yield()) + return posix_error(); + Py_RETURN_NONE; +} + +#ifdef HAVE_SCHED_SETAFFINITY + +typedef struct { + PyObject_HEAD; + Py_ssize_t size; + int ncpus; + cpu_set_t *set; +} Py_cpu_set; + +static PyTypeObject cpu_set_type; + +static void +cpu_set_dealloc(Py_cpu_set *set) +{ + assert(set->set); + CPU_FREE(set->set); + Py_TYPE(set)->tp_free(set); +} + +static Py_cpu_set * +make_new_cpu_set(PyTypeObject *type, Py_ssize_t size) +{ + Py_cpu_set *set; + + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "negative size"); + return NULL; + } + set = (Py_cpu_set *)type->tp_alloc(type, 0); + if (!set) + return NULL; + set->ncpus = size; + set->size = CPU_ALLOC_SIZE(size); + set->set = CPU_ALLOC(size); + if (!set->set) { + type->tp_free(set); + PyErr_NoMemory(); + return NULL; + } + CPU_ZERO_S(set->size, set->set); + return set; +} + +static PyObject * +cpu_set_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + int size; + + if (!_PyArg_NoKeywords("cpu_set()", kwargs) || + !PyArg_ParseTuple(args, "i:cpu_set", &size)) + return NULL; + return (PyObject *)make_new_cpu_set(type, size); +} + +static PyObject * +cpu_set_repr(Py_cpu_set *set) +{ + return PyUnicode_FromFormat("<cpu_set with %li entries>", set->ncpus); +} + +static Py_ssize_t +cpu_set_len(Py_cpu_set *set) +{ + return set->ncpus; +} + +static int +_get_cpu(Py_cpu_set *set, const char *requester, PyObject *args) +{ + int cpu; + if (!PyArg_ParseTuple(args, requester, &cpu)) + return -1; + if (cpu < 0) { + PyErr_SetString(PyExc_ValueError, "cpu < 0 not valid"); + return -1; + } + if (cpu >= set->ncpus) { + PyErr_SetString(PyExc_ValueError, "cpu too large for set"); + return -1; + } + return cpu; +} + +PyDoc_STRVAR(cpu_set_set_doc, +"cpu_set.set(i)\n\n\ +Add CPU *i* to the set."); + +static PyObject * +cpu_set_set(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|set", args); + if (cpu == -1) + return NULL; + CPU_SET_S(cpu, set->size, set->set); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(cpu_set_count_doc, +"cpu_set.count() -> int\n\n\ +Return the number of CPUs active in the set."); + +static PyObject * +cpu_set_count(Py_cpu_set *set, PyObject *noargs) +{ + return PyLong_FromLong(CPU_COUNT_S(set->size, set->set)); +} + +PyDoc_STRVAR(cpu_set_clear_doc, +"cpu_set.clear(i)\n\n\ +Remove CPU *i* from the set."); + +static PyObject * +cpu_set_clear(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|clear", args); + if (cpu == -1) + return NULL; + CPU_CLR_S(cpu, set->size, set->set); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(cpu_set_isset_doc, +"cpu_set.isset(i) -> bool\n\n\ +Test if CPU *i* is in the set."); + +static PyObject * +cpu_set_isset(Py_cpu_set *set, PyObject *args) +{ + int cpu = _get_cpu(set, "i|isset", args); + if (cpu == -1) + return NULL; + if (CPU_ISSET_S(cpu, set->size, set->set)) + Py_RETURN_TRUE; + Py_RETURN_FALSE; +} + +PyDoc_STRVAR(cpu_set_zero_doc, +"cpu_set.zero()\n\n\ +Clear the cpu_set."); + +static PyObject * +cpu_set_zero(Py_cpu_set *set, PyObject *noargs) +{ + CPU_ZERO_S(set->size, set->set); + Py_RETURN_NONE; +} + +static PyObject * +cpu_set_richcompare(Py_cpu_set *set, Py_cpu_set *other, int op) +{ + int eq; + + if ((op != Py_EQ && op != Py_NE) || Py_TYPE(other) != &cpu_set_type) + Py_RETURN_NOTIMPLEMENTED; + + eq = set->ncpus == other->ncpus && CPU_EQUAL_S(set->size, set->set, other->set); + if ((op == Py_EQ) ? eq : !eq) + Py_RETURN_TRUE; + else + Py_RETURN_FALSE; +} + +#define CPU_SET_BINOP(name, op) \ + static PyObject * \ + do_cpu_set_##name(Py_cpu_set *left, Py_cpu_set *right, Py_cpu_set *res) { \ + if (res) { \ + Py_INCREF(res); \ + } \ + else { \ + res = make_new_cpu_set(&cpu_set_type, left->ncpus); \ + if (!res) \ + return NULL; \ + } \ + if (Py_TYPE(right) != &cpu_set_type || left->ncpus != right->ncpus) { \ + Py_DECREF(res); \ + Py_RETURN_NOTIMPLEMENTED; \ + } \ + assert(left->size == right->size && right->size == res->size); \ + op(res->size, res->set, left->set, right->set); \ + return (PyObject *)res; \ + } \ + static PyObject * \ + cpu_set_##name(Py_cpu_set *left, Py_cpu_set *right) { \ + return do_cpu_set_##name(left, right, NULL); \ + } \ + static PyObject * \ + cpu_set_i##name(Py_cpu_set *left, Py_cpu_set *right) { \ + return do_cpu_set_##name(left, right, left); \ + } \ + +CPU_SET_BINOP(and, CPU_AND_S) +CPU_SET_BINOP(or, CPU_OR_S) +CPU_SET_BINOP(xor, CPU_XOR_S) +#undef CPU_SET_BINOP + +PyDoc_STRVAR(cpu_set_doc, +"cpu_set(size)\n\n\ +Create an empty mask of CPUs."); + +static PyNumberMethods cpu_set_as_number = { + 0, /*nb_add*/ + 0, /*nb_subtract*/ + 0, /*nb_multiply*/ + 0, /*nb_remainder*/ + 0, /*nb_divmod*/ + 0, /*nb_power*/ + 0, /*nb_negative*/ + 0, /*nb_positive*/ + 0, /*nb_absolute*/ + 0, /*nb_bool*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + (binaryfunc)cpu_set_and, /*nb_and*/ + (binaryfunc)cpu_set_xor, /*nb_xor*/ + (binaryfunc)cpu_set_or, /*nb_or*/ + 0, /*nb_int*/ + 0, /*nb_reserved*/ + 0, /*nb_float*/ + 0, /*nb_inplace_add*/ + 0, /*nb_inplace_subtract*/ + 0, /*nb_inplace_multiply*/ + 0, /*nb_inplace_remainder*/ + 0, /*nb_inplace_power*/ + 0, /*nb_inplace_lshift*/ + 0, /*nb_inplace_rshift*/ + (binaryfunc)cpu_set_iand, /*nb_inplace_and*/ + (binaryfunc)cpu_set_ixor, /*nb_inplace_xor*/ + (binaryfunc)cpu_set_ior, /*nb_inplace_or*/ +}; + +static PySequenceMethods cpu_set_as_sequence = { + (lenfunc)cpu_set_len, /* sq_length */ +}; + +static PyMethodDef cpu_set_methods[] = { + {"clear", (PyCFunction)cpu_set_clear, METH_VARARGS, cpu_set_clear_doc}, + {"count", (PyCFunction)cpu_set_count, METH_NOARGS, cpu_set_count_doc}, + {"isset", (PyCFunction)cpu_set_isset, METH_VARARGS, cpu_set_isset_doc}, + {"set", (PyCFunction)cpu_set_set, METH_VARARGS, cpu_set_set_doc}, + {"zero", (PyCFunction)cpu_set_zero, METH_NOARGS, cpu_set_zero_doc}, + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject cpu_set_type = { + PyVarObject_HEAD_INIT(&PyType_Type, 0) + "posix.cpu_set", /* tp_name */ + sizeof(Py_cpu_set), /* tp_basicsize */ + 0, /* tp_itemsize */ + /* methods */ + (destructor)cpu_set_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_reserved */ + (reprfunc)cpu_set_repr, /* tp_repr */ + &cpu_set_as_number, /* tp_as_number */ + &cpu_set_as_sequence, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + PyObject_HashNotImplemented, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + cpu_set_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)cpu_set_richcompare, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + cpu_set_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + cpu_set_new, /* tp_new */ + PyObject_Del, /* tp_free */ +}; + +PyDoc_STRVAR(posix_sched_setaffinity__doc__, +"sched_setaffinity(pid, cpu_set)\n\n\ +Set the affinity of the process with PID *pid* to *cpu_set*."); + +static PyObject * +posix_sched_setaffinity(PyObject *self, PyObject *args) +{ + pid_t pid; + Py_cpu_set *cpu_set; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "O!:sched_setaffinity", + &pid, &cpu_set_type, &cpu_set)) + return NULL; + if (sched_setaffinity(pid, cpu_set->size, cpu_set->set)) + return posix_error(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(posix_sched_getaffinity__doc__, +"sched_getaffinity(pid, ncpus) -> cpu_set\n\n\ +Return the affinity of the process with PID *pid*.\n\ +The returned cpu_set will be of size *ncpus*."); + +static PyObject * +posix_sched_getaffinity(PyObject *self, PyObject *args) +{ + pid_t pid; + int ncpus; + Py_cpu_set *res; + + if (!PyArg_ParseTuple(args, _Py_PARSE_PID "i:sched_getaffinity", + &pid, &ncpus)) + return NULL; + res = make_new_cpu_set(&cpu_set_type, ncpus); + if (!res) + return NULL; + if (sched_getaffinity(pid, res->size, res->set)) { + Py_DECREF(res); + return posix_error(); + } + return (PyObject *)res; +} + +#endif /* HAVE_SCHED_SETAFFINITY */ + +#endif /* HAVE_SCHED_H */ + /* AIX uses /dev/ptc but is otherwise the same as /dev/ptmx */ /* IRIX has both /dev/ptc and /dev/ptmx, use ptmx */ #if defined(HAVE_DEV_PTC) && !defined(HAVE_DEV_PTMX) @@ -4279,6 +5233,7 @@ posix_forkpty(PyObject *self, PyObject *noargs) } #endif + #ifdef HAVE_GETEGID PyDoc_STRVAR(posix_getegid__doc__, "getegid() -> egid\n\n\ @@ -4328,6 +5283,70 @@ posix_getpid(PyObject *self, PyObject *noargs) return PyLong_FromPid(getpid()); } +#ifdef HAVE_GETGROUPLIST +PyDoc_STRVAR(posix_getgrouplist__doc__, +"getgrouplist(user, group) -> list of groups to which a user belongs\n\n\ +Returns a list of groups to which a user belongs.\n\n\ + user: username to lookup\n\ + group: base group id of the user"); + +static PyObject * +posix_getgrouplist(PyObject *self, PyObject *args) +{ +#ifdef NGROUPS_MAX +#define MAX_GROUPS NGROUPS_MAX +#else + /* defined to be 16 on Solaris7, so this should be a small number */ +#define MAX_GROUPS 64 +#endif + + const char *user; + int i, ngroups; + PyObject *list; +#ifdef __APPLE__ + int *groups, basegid; +#else + gid_t *groups, basegid; +#endif + ngroups = MAX_GROUPS; + + if (!PyArg_ParseTuple(args, "si", &user, &basegid)) + return NULL; + +#ifdef __APPLE__ + groups = PyMem_Malloc(ngroups * sizeof(int)); +#else + groups = PyMem_Malloc(ngroups * sizeof(gid_t)); +#endif + if (groups == NULL) + return PyErr_NoMemory(); + + if (getgrouplist(user, basegid, groups, &ngroups) == -1) { + PyMem_Del(groups); + return posix_error(); + } + + list = PyList_New(ngroups); + if (list == NULL) { + PyMem_Del(groups); + return NULL; + } + + for (i = 0; i < ngroups; i++) { + PyObject *o = PyLong_FromUnsignedLong((unsigned long)groups[i]); + if (o == NULL) { + Py_DECREF(list); + PyMem_Del(groups); + return NULL; + } + PyList_SET_ITEM(list, i, o); + } + + PyMem_Del(groups); + + return list; +} +#endif #ifdef HAVE_GETGROUPS PyDoc_STRVAR(posix_getgroups__doc__, @@ -5069,6 +6088,55 @@ posix_wait4(PyObject *self, PyObject *args) } #endif /* HAVE_WAIT4 */ +#if defined(HAVE_WAITID) && !defined(__APPLE__) +PyDoc_STRVAR(posix_waitid__doc__, +"waitid(idtype, id, options) -> waitid_result\n\n\ +Wait for the completion of one or more child processes.\n\n\ +idtype can be P_PID, P_PGID or P_ALL.\n\ +id specifies the pid to wait on.\n\ +options is constructed from the ORing of one or more of WEXITED, WSTOPPED\n\ +or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n\ +Returns either waitid_result or None if WNOHANG is specified and there are\n\ +no children in a waitable state."); + +static PyObject * +posix_waitid(PyObject *self, PyObject *args) +{ + PyObject *result; + idtype_t idtype; + id_t id; + int options, res; + siginfo_t si; + si.si_pid = 0; + if (!PyArg_ParseTuple(args, "i" _Py_PARSE_PID "i:waitid", &idtype, &id, &options)) + return NULL; + Py_BEGIN_ALLOW_THREADS + res = waitid(idtype, id, &si, options); + Py_END_ALLOW_THREADS + if (res == -1) + return posix_error(); + + if (si.si_pid == 0) + Py_RETURN_NONE; + + result = PyStructSequence_New(&WaitidResultType); + if (!result) + return NULL; + + PyStructSequence_SET_ITEM(result, 0, PyLong_FromPid(si.si_pid)); + PyStructSequence_SET_ITEM(result, 1, PyLong_FromPid(si.si_uid)); + PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si.si_signo))); + PyStructSequence_SET_ITEM(result, 3, PyLong_FromLong((long)(si.si_status))); + PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si.si_code))); + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } + + return result; +} +#endif + #ifdef HAVE_WAITPID PyDoc_STRVAR(posix_waitpid__doc__, "waitpid(pid, options) -> (pid, status)\n\n\ @@ -5654,9 +6722,7 @@ posix_dup(PyObject *self, PyObject *args) return NULL; if (!_PyVerify_fd(fd)) return posix_error(); - Py_BEGIN_ALLOW_THREADS fd = dup(fd); - Py_END_ALLOW_THREADS if (fd < 0) return posix_error(); return PyLong_FromLong((long)fd); @@ -5675,15 +6741,42 @@ posix_dup2(PyObject *self, PyObject *args) return NULL; if (!_PyVerify_fd_dup2(fd, fd2)) return posix_error(); - Py_BEGIN_ALLOW_THREADS res = dup2(fd, fd2); - Py_END_ALLOW_THREADS if (res < 0) return posix_error(); Py_INCREF(Py_None); return Py_None; } +#ifdef HAVE_LOCKF +PyDoc_STRVAR(posix_lockf__doc__, +"lockf(fd, cmd, len)\n\n\ +Apply, test or remove a POSIX lock on an open file descriptor.\n\n\ +fd is an open file descriptor.\n\ +cmd specifies the command to use - one of F_LOCK, F_TLOCK, F_ULOCK or\n\ +F_TEST.\n\ +len specifies the section of the file to lock."); + +static PyObject * +posix_lockf(PyObject *self, PyObject *args) +{ + int fd, cmd, res; + off_t len; + if (!PyArg_ParseTuple(args, "iiO&:lockf", + &fd, &cmd, _parse_off_t, &len)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = lockf(fd, cmd, len); + Py_END_ALLOW_THREADS + + if (res < 0) + return posix_error(); + + Py_RETURN_NONE; +} +#endif + PyDoc_STRVAR(posix_lseek__doc__, "lseek(fd, pos, how) -> newpos\n\n\ @@ -5713,8 +6806,7 @@ posix_lseek(PyObject *self, PyObject *args) #if !defined(HAVE_LARGEFILE_SUPPORT) pos = PyLong_AsLong(posobj); #else - pos = PyLong_Check(posobj) ? - PyLong_AsLongLong(posobj) : PyLong_AsLong(posobj); + pos = PyLong_AsLongLong(posobj); #endif if (PyErr_Occurred()) return NULL; @@ -5774,6 +6866,140 @@ posix_read(PyObject *self, PyObject *args) return buffer; } +#if (defined(HAVE_SENDFILE) && (defined(__FreeBSD__) || defined(__DragonFly__) \ + || defined(__APPLE__))) || defined(HAVE_READV) || defined(HAVE_WRITEV) +static Py_ssize_t +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +{ + int i, j; + Py_ssize_t blen, total = 0; + + *iov = PyMem_New(struct iovec, cnt); + if (*iov == NULL) { + PyErr_NoMemory(); + return total; + } + + *buf = PyMem_New(Py_buffer, cnt); + if (*buf == NULL) { + PyMem_Del(*iov); + PyErr_NoMemory(); + return total; + } + + for (i = 0; i < cnt; i++) { + PyObject *item = PySequence_GetItem(seq, i); + if (item == NULL) + goto fail; + if (PyObject_GetBuffer(item, &(*buf)[i], type) == -1) { + Py_DECREF(item); + goto fail; + } + Py_DECREF(item); + (*iov)[i].iov_base = (*buf)[i].buf; + blen = (*buf)[i].len; + (*iov)[i].iov_len = blen; + total += blen; + } + return total; + +fail: + PyMem_Del(*iov); + for (j = 0; j < i; j++) { + PyBuffer_Release(&(*buf)[j]); + } + PyMem_Del(*buf); + return 0; +} + +static void +iov_cleanup(struct iovec *iov, Py_buffer *buf, int cnt) +{ + int i; + PyMem_Del(iov); + for (i = 0; i < cnt; i++) { + PyBuffer_Release(&buf[i]); + } + PyMem_Del(buf); +} +#endif + +#ifdef HAVE_READV +PyDoc_STRVAR(posix_readv__doc__, +"readv(fd, buffers) -> bytesread\n\n\ +Read from a file descriptor into a number of writable buffers. buffers\n\ +is an arbitrary sequence of writable buffers.\n\ +Returns the total number of bytes read."); + +static PyObject * +posix_readv(PyObject *self, PyObject *args) +{ + int fd, cnt; + Py_ssize_t n; + PyObject *seq; + struct iovec *iov; + Py_buffer *buf; + + if (!PyArg_ParseTuple(args, "iO:readv", &fd, &seq)) + return NULL; + if (!PySequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, + "readv() arg 2 must be a sequence"); + return NULL; + } + cnt = PySequence_Size(seq); + + if (!iov_setup(&iov, &buf, seq, cnt, PyBUF_WRITABLE)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + n = readv(fd, iov, cnt); + Py_END_ALLOW_THREADS + + iov_cleanup(iov, buf, cnt); + return PyLong_FromSsize_t(n); +} +#endif + +#ifdef HAVE_PREAD +PyDoc_STRVAR(posix_pread__doc__, +"pread(fd, buffersize, offset) -> string\n\n\ +Read from a file descriptor, fd, at a position of offset. It will read up\n\ +to buffersize number of bytes. The file offset remains unchanged."); + +static PyObject * +posix_pread(PyObject *self, PyObject *args) +{ + int fd, size; + off_t offset; + Py_ssize_t n; + PyObject *buffer; + if (!PyArg_ParseTuple(args, "iiO&:pread", &fd, &size, _parse_off_t, &offset)) + return NULL; + + if (size < 0) { + errno = EINVAL; + return posix_error(); + } + buffer = PyBytes_FromStringAndSize((char *)NULL, size); + if (buffer == NULL) + return NULL; + if (!_PyVerify_fd(fd)) { + Py_DECREF(buffer); + return posix_error(); + } + Py_BEGIN_ALLOW_THREADS + n = pread(fd, PyBytes_AS_STRING(buffer), size, offset); + Py_END_ALLOW_THREADS + if (n < 0) { + Py_DECREF(buffer); + return posix_error(); + } + if (n != size) + _PyBytes_Resize(&buffer, n); + return buffer; +} +#endif PyDoc_STRVAR(posix_write__doc__, "write(fd, string) -> byteswritten\n\n\ @@ -5808,6 +7034,144 @@ posix_write(PyObject *self, PyObject *args) return PyLong_FromSsize_t(size); } +#ifdef HAVE_SENDFILE +PyDoc_STRVAR(posix_sendfile__doc__, +"sendfile(out, in, offset, nbytes) -> byteswritten\n\ +sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0)\n\ + -> byteswritten\n\ +Copy nbytes bytes from file descriptor in to file descriptor out."); + +static PyObject * +posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) +{ + int in, out; + Py_ssize_t ret; + off_t offset; + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifndef __APPLE__ + Py_ssize_t len; +#endif + PyObject *headers = NULL, *trailers = NULL; + Py_buffer *hbuf, *tbuf; + off_t sbytes; + struct sf_hdtr sf; + int flags = 0; + sf.headers = NULL; + sf.trailers = NULL; + static char *keywords[] = {"out", "in", + "offset", "count", + "headers", "trailers", "flags", NULL}; + +#ifdef __APPLE__ + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&O&|OOi:sendfile", + keywords, &out, &in, _parse_off_t, &offset, _parse_off_t, &sbytes, +#else + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&n|OOi:sendfile", + keywords, &out, &in, _parse_off_t, &offset, &len, +#endif + &headers, &trailers, &flags)) + return NULL; + if (headers != NULL) { + if (!PySequence_Check(headers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() headers must be a sequence or None"); + return NULL; + } else { + Py_ssize_t i = 0; /* Avoid uninitialized warning */ + sf.hdr_cnt = PySequence_Size(headers); + if (sf.hdr_cnt > 0 && + !(i = iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE))) + return NULL; +#ifdef __APPLE__ + sbytes += i; +#endif + } + } + if (trailers != NULL) { + if (!PySequence_Check(trailers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() trailers must be a sequence or None"); + return NULL; + } else { + Py_ssize_t i = 0; /* Avoid uninitialized warning */ + sf.trl_cnt = PySequence_Size(trailers); + if (sf.trl_cnt > 0 && + !(i = iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE))) + return NULL; +#ifdef __APPLE__ + sbytes += i; +#endif + } + } + + Py_BEGIN_ALLOW_THREADS +#ifdef __APPLE__ + ret = sendfile(in, out, offset, &sbytes, &sf, flags); +#else + ret = sendfile(in, out, offset, len, &sf, &sbytes, flags); +#endif + Py_END_ALLOW_THREADS + + if (sf.headers != NULL) + iov_cleanup(sf.headers, hbuf, sf.hdr_cnt); + if (sf.trailers != NULL) + iov_cleanup(sf.trailers, tbuf, sf.trl_cnt); + + if (ret < 0) { + if ((errno == EAGAIN) || (errno == EBUSY)) { + if (sbytes != 0) { + // some data has been sent + goto done; + } + else { + // no data has been sent; upper application is supposed + // to retry on EAGAIN or EBUSY + return posix_error(); + } + } + return posix_error(); + } + goto done; + +done: + #if !defined(HAVE_LARGEFILE_SUPPORT) + return Py_BuildValue("l", sbytes); + #else + return Py_BuildValue("L", sbytes); + #endif + +#else + Py_ssize_t count; + PyObject *offobj; + static char *keywords[] = {"out", "in", + "offset", "count", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiOn:sendfile", + keywords, &out, &in, &offobj, &count)) + return NULL; +#ifdef linux + if (offobj == Py_None) { + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, NULL, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); + return Py_BuildValue("n", ret); + } +#endif + if (!_parse_off_t(offobj, &offset)) + return NULL; + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, &offset, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); + return Py_BuildValue("n", ret); +#endif +} +#endif PyDoc_STRVAR(posix_fstat__doc__, "fstat(fd) -> stat result\n\n\ @@ -5869,9 +7233,7 @@ posix_pipe(PyObject *self, PyObject *noargs) HFILE read, write; APIRET rc; - Py_BEGIN_ALLOW_THREADS rc = DosCreatePipe( &read, &write, 4096); - Py_END_ALLOW_THREADS if (rc != NO_ERROR) return os2_error(rc); @@ -5880,9 +7242,7 @@ posix_pipe(PyObject *self, PyObject *noargs) #if !defined(MS_WINDOWS) int fds[2]; int res; - Py_BEGIN_ALLOW_THREADS res = pipe(fds); - Py_END_ALLOW_THREADS if (res != 0) return posix_error(); return Py_BuildValue("(ii)", fds[0], fds[1]); @@ -5890,9 +7250,7 @@ posix_pipe(PyObject *self, PyObject *noargs) HANDLE read, write; int read_fd, write_fd; BOOL ok; - Py_BEGIN_ALLOW_THREADS ok = CreatePipe(&read, &write, NULL, 0); - Py_END_ALLOW_THREADS if (!ok) return win32_error("CreatePipe", NULL); read_fd = _open_osfhandle((Py_intptr_t)read, 0); @@ -5903,6 +7261,99 @@ posix_pipe(PyObject *self, PyObject *noargs) } #endif /* HAVE_PIPE */ +#ifdef HAVE_PIPE2 +PyDoc_STRVAR(posix_pipe2__doc__, +"pipe2(flags) -> (read_end, write_end)\n\n\ +Create a pipe with flags set atomically.\n\ +flags can be constructed by ORing together one or more of these values:\n\ +O_NONBLOCK, O_CLOEXEC.\n\ +"); + +static PyObject * +posix_pipe2(PyObject *self, PyObject *arg) +{ + int flags; + int fds[2]; + int res; + + flags = PyLong_AsLong(arg); + if (flags == -1 && PyErr_Occurred()) + return NULL; + + res = pipe2(fds, flags); + if (res != 0) + return posix_error(); + return Py_BuildValue("(ii)", fds[0], fds[1]); +} +#endif /* HAVE_PIPE2 */ + +#ifdef HAVE_WRITEV +PyDoc_STRVAR(posix_writev__doc__, +"writev(fd, buffers) -> byteswritten\n\n\ +Write the contents of buffers to a file descriptor, where buffers is an\n\ +arbitrary sequence of buffers.\n\ +Returns the total bytes written."); + +static PyObject * +posix_writev(PyObject *self, PyObject *args) +{ + int fd, cnt; + Py_ssize_t res; + PyObject *seq; + struct iovec *iov; + Py_buffer *buf; + if (!PyArg_ParseTuple(args, "iO:writev", &fd, &seq)) + return NULL; + if (!PySequence_Check(seq)) { + PyErr_SetString(PyExc_TypeError, + "writev() arg 2 must be a sequence"); + return NULL; + } + cnt = PySequence_Size(seq); + + if (!iov_setup(&iov, &buf, seq, cnt, PyBUF_SIMPLE)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + res = writev(fd, iov, cnt); + Py_END_ALLOW_THREADS + + iov_cleanup(iov, buf, cnt); + return PyLong_FromSsize_t(res); +} +#endif + +#ifdef HAVE_PWRITE +PyDoc_STRVAR(posix_pwrite__doc__, +"pwrite(fd, string, offset) -> byteswritten\n\n\ +Write string to a file descriptor, fd, from offset, leaving the file\n\ +offset unchanged."); + +static PyObject * +posix_pwrite(PyObject *self, PyObject *args) +{ + Py_buffer pbuf; + int fd; + off_t offset; + Py_ssize_t size; + + if (!PyArg_ParseTuple(args, "iy*O&:pwrite", &fd, &pbuf, _parse_off_t, &offset)) + return NULL; + + if (!_PyVerify_fd(fd)) { + PyBuffer_Release(&pbuf); + return posix_error(); + } + Py_BEGIN_ALLOW_THREADS + size = pwrite(fd, pbuf.buf, (size_t)pbuf.len, offset); + Py_END_ALLOW_THREADS + PyBuffer_Release(&pbuf); + if (size < 0) + return posix_error(); + return PyLong_FromSsize_t(size); +} +#endif #ifdef HAVE_MKFIFO PyDoc_STRVAR(posix_mkfifo__doc__, @@ -6019,18 +7470,8 @@ posix_ftruncate(PyObject *self, PyObject *args) int fd; off_t length; int res; - PyObject *lenobj; - if (!PyArg_ParseTuple(args, "iO:ftruncate", &fd, &lenobj)) - return NULL; - -#if !defined(HAVE_LARGEFILE_SUPPORT) - length = PyLong_AsLong(lenobj); -#else - length = PyLong_Check(lenobj) ? - PyLong_AsLongLong(lenobj) : PyLong_AsLong(lenobj); -#endif - if (PyErr_Occurred()) + if (!PyArg_ParseTuple(args, "iO&:ftruncate", &fd, _parse_off_t, &length)) return NULL; Py_BEGIN_ALLOW_THREADS @@ -6043,6 +7484,93 @@ posix_ftruncate(PyObject *self, PyObject *args) } #endif +#ifdef HAVE_TRUNCATE +PyDoc_STRVAR(posix_truncate__doc__, +"truncate(path, length)\n\n\ +Truncate the file given by path to length bytes."); + +static PyObject * +posix_truncate(PyObject *self, PyObject *args) +{ + PyObject *opath; + const char *path; + off_t length; + int res; + + if (!PyArg_ParseTuple(args, "O&O&:truncate", + PyUnicode_FSConverter, &opath, _parse_off_t, &length)) + return NULL; + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = truncate(path, length); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_POSIX_FALLOCATE +PyDoc_STRVAR(posix_posix_fallocate__doc__, +"posix_fallocate(fd, offset, len)\n\n\ +Ensures that enough disk space is allocated for the file specified by fd\n\ +starting from offset and continuing for len bytes."); + +static PyObject * +posix_posix_fallocate(PyObject *self, PyObject *args) +{ + off_t len, offset; + int res, fd; + + if (!PyArg_ParseTuple(args, "iO&O&:posix_fallocate", + &fd, _parse_off_t, &offset, _parse_off_t, &len)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = posix_fallocate(fd, offset, len); + Py_END_ALLOW_THREADS + if (res != 0) { + errno = res; + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_POSIX_FADVISE +PyDoc_STRVAR(posix_posix_fadvise__doc__, +"posix_fadvise(fd, offset, len, advice)\n\n\ +Announces an intention to access data in a specific pattern thus allowing\n\ +the kernel to make optimizations.\n\ +The advice applies to the region of the file specified by fd starting at\n\ +offset and continuing for len bytes.\n\ +advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\n\ +POSIX_FADV_RANDOM, POSIX_FADV_NOREUSE, POSIX_FADV_WILLNEED or\n\ +POSIX_FADV_DONTNEED."); + +static PyObject * +posix_posix_fadvise(PyObject *self, PyObject *args) +{ + off_t len, offset; + int res, fd, advice; + + if (!PyArg_ParseTuple(args, "iO&O&i:posix_fadvise", + &fd, _parse_off_t, &offset, _parse_off_t, &len, &advice)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + res = posix_fadvise(fd, offset, len, advice); + Py_END_ALLOW_THREADS + if (res != 0) { + errno = res; + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + #ifdef HAVE_PUTENV PyDoc_STRVAR(posix_putenv__doc__, "putenv(key, value)\n\n\ @@ -6486,6 +8014,32 @@ posix_statvfs(PyObject *self, PyObject *args) } #endif /* HAVE_STATVFS */ +#ifdef MS_WINDOWS +PyDoc_STRVAR(win32__getdiskusage__doc__, +"_getdiskusage(path) -> (total, free)\n\n\ +Return disk usage statistics about the given path as (total, free) tuple."); + +static PyObject * +win32__getdiskusage(PyObject *self, PyObject *args) +{ + BOOL retval; + ULARGE_INTEGER _, total, free; + LPCTSTR path; + + if (! PyArg_ParseTuple(args, "s", &path)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + retval = GetDiskFreeSpaceEx(path, &_, &total, &free); + Py_END_ALLOW_THREADS + if (retval == 0) + return PyErr_SetFromWindowsErr(0); + + return Py_BuildValue("(LL)", total.QuadPart, free.QuadPart); +} +#endif + + /* This is used for fpathconf(), pathconf(), confstr() and sysconf(). * It maps strings representing configuration variable names to * integer values, allowing those functions to be called with the @@ -7707,6 +9261,9 @@ static PyObject * device_encoding(PyObject *self, PyObject *args) { int fd; +#if defined(MS_WINDOWS) || defined(MS_WIN64) + UINT cp; +#endif if (!PyArg_ParseTuple(args, "i:device_encoding", &fd)) return NULL; if (!_PyVerify_fd(fd) || !isatty(fd)) { @@ -7714,16 +9271,16 @@ device_encoding(PyObject *self, PyObject *args) return Py_None; } #if defined(MS_WINDOWS) || defined(MS_WIN64) - if (fd == 0) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleCP()); - return PyUnicode_FromString(buf); - } - if (fd == 1 || fd == 2) { - char buf[100]; - sprintf(buf, "cp%d", GetConsoleOutputCP()); - return PyUnicode_FromString(buf); - } + if (fd == 0) + cp = GetConsoleCP(); + else if (fd == 1 || fd == 2) + cp = GetConsoleOutputCP(); + else + cp = 0; + /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application + has no console */ + if (cp != 0) + return PyUnicode_FromFormat("cp%u", (unsigned int)cp); #elif defined(CODESET) { char *codeset = nl_langinfo(CODESET); @@ -7847,6 +9404,552 @@ posix_getresgid (PyObject *self, PyObject *noargs) } #endif +/* Posix *at family of functions: + faccessat, fchmodat, fchownat, fstatat, futimesat, + linkat, mkdirat, mknodat, openat, readlinkat, renameat, symlinkat, + unlinkat, utimensat, mkfifoat */ + +#ifdef HAVE_FACCESSAT +PyDoc_STRVAR(posix_faccessat__doc__, +"faccessat(dirfd, path, mode, flags=0) -> True if granted, False otherwise\n\n\ +Like access() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and can be constructed by ORing together zero or more\n\ +of these values: AT_SYMLINK_NOFOLLOW, AT_EACCESS.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_faccessat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int mode; + int res; + int dirfd, flags = 0; + if (!PyArg_ParseTuple(args, "iO&i|i:faccessat", + &dirfd, PyUnicode_FSConverter, &opath, &mode, &flags)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = faccessat(dirfd, path, mode, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + return PyBool_FromLong(res == 0); +} +#endif + +#ifdef HAVE_FCHMODAT +PyDoc_STRVAR(posix_fchmodat__doc__, +"fchmodat(dirfd, path, mode, flags=0)\n\n\ +Like chmod() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fchmodat(PyObject *self, PyObject *args) +{ + int dirfd, mode, res; + int flags = 0; + PyObject *opath; + char *path; + + if (!PyArg_ParseTuple(args, "iO&i|i:fchmodat", + &dirfd, PyUnicode_FSConverter, &opath, &mode, &flags)) + return NULL; + + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fchmodat(dirfd, path, mode, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_FCHMODAT */ + +#ifdef HAVE_FCHOWNAT +PyDoc_STRVAR(posix_fchownat__doc__, +"fchownat(dirfd, path, uid, gid, flags=0)\n\n\ +Like chown() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fchownat(PyObject *self, PyObject *args) +{ + PyObject *opath; + int dirfd, res; + long uid, gid; + int flags = 0; + char *path; + + if (!PyArg_ParseTuple(args, "iO&ll|i:fchownat", + &dirfd, PyUnicode_FSConverter, &opath, &uid, &gid, &flags)) + return NULL; + + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fchownat(dirfd, path, (uid_t) uid, (gid_t) gid, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_FCHOWNAT */ + +#ifdef HAVE_FSTATAT +PyDoc_STRVAR(posix_fstatat__doc__, +"fstatat(dirfd, path, flags=0) -> stat result\n\n\ +Like stat() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_fstatat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + STRUCT_STAT st; + int dirfd, res, flags = 0; + + if (!PyArg_ParseTuple(args, "iO&|i:fstatat", + &dirfd, PyUnicode_FSConverter, &opath, &flags)) + return NULL; + path = PyBytes_AsString(opath); + + Py_BEGIN_ALLOW_THREADS + res = fstatat(dirfd, path, &st, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res != 0) + return posix_error(); + + return _pystat_fromstructstat(&st); +} +#endif + +#ifdef HAVE_FUTIMESAT +PyDoc_STRVAR(posix_futimesat__doc__, +"futimesat(dirfd, path, (atime, mtime))\n\ +futimesat(dirfd, path, None)\n\n\ +Like utime() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_futimesat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int res, dirfd; + PyObject* arg; + + struct timeval buf[2]; + + if (!PyArg_ParseTuple(args, "iO&O:futimesat", + &dirfd, PyUnicode_FSConverter, &opath, &arg)) + return NULL; + path = PyBytes_AsString(opath); + if (arg == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = futimesat(dirfd, path, NULL); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(arg) || PyTuple_Size(arg) != 2) { + PyErr_SetString(PyExc_TypeError, + "futimesat() arg 3 must be a tuple (atime, mtime)"); + Py_DECREF(opath); + return NULL; + } + else { + if (extract_time(PyTuple_GET_ITEM(arg, 0), + &(buf[0].tv_sec), &(buf[0].tv_usec)) == -1) { + Py_DECREF(opath); + return NULL; + } + if (extract_time(PyTuple_GET_ITEM(arg, 1), + &(buf[1].tv_sec), &(buf[1].tv_usec)) == -1) { + Py_DECREF(opath); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + res = futimesat(dirfd, path, buf); + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_LINKAT +PyDoc_STRVAR(posix_linkat__doc__, +"linkat(srcfd, srcpath, dstfd, dstpath, flags=0)\n\n\ +Like link() but if srcpath is relative, it is taken as relative to srcfd\n\ +and if dstpath is relative, it is taken as relative to dstfd.\n\ +flags is optional and may be 0 or AT_SYMLINK_FOLLOW.\n\ +If srcpath is relative and srcfd is the special value AT_FDCWD, then\n\ +srcpath is interpreted relative to the current working directory. This\n\ +also applies for dstpath."); + +static PyObject * +posix_linkat(PyObject *self, PyObject *args) +{ + PyObject *osrc, *odst; + char *src, *dst; + int res, srcfd, dstfd; + int flags = 0; + + if (!PyArg_ParseTuple(args, "iO&iO&|i:linkat", + &srcfd, PyUnicode_FSConverter, &osrc, &dstfd, PyUnicode_FSConverter, &odst, &flags)) + return NULL; + src = PyBytes_AsString(osrc); + dst = PyBytes_AsString(odst); + Py_BEGIN_ALLOW_THREADS + res = linkat(srcfd, src, dstfd, dst, flags); + Py_END_ALLOW_THREADS + Py_DECREF(osrc); + Py_DECREF(odst); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_LINKAT */ + +#ifdef HAVE_MKDIRAT +PyDoc_STRVAR(posix_mkdirat__doc__, +"mkdirat(dirfd, path, mode=0o777)\n\n\ +Like mkdir() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mkdirat(PyObject *self, PyObject *args) +{ + int res, dirfd; + PyObject *opath; + char *path; + int mode = 0777; + + if (!PyArg_ParseTuple(args, "iO&|i:mkdirat", + &dirfd, PyUnicode_FSConverter, &opath, &mode)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = mkdirat(dirfd, path, mode); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#if defined(HAVE_MKNODAT) && defined(HAVE_MAKEDEV) +PyDoc_STRVAR(posix_mknodat__doc__, +"mknodat(dirfd, path, mode=0o600, device=0)\n\n\ +Like mknod() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mknodat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *filename; + int mode = 0600; + int device = 0; + int res, dirfd; + if (!PyArg_ParseTuple(args, "iO&|ii:mknodat", &dirfd, + PyUnicode_FSConverter, &opath, &mode, &device)) + return NULL; + filename = PyBytes_AS_STRING(opath); + Py_BEGIN_ALLOW_THREADS + res = mknodat(dirfd, filename, mode, device); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_OPENAT +PyDoc_STRVAR(posix_openat__doc__, +"openat(dirfd, path, flag, mode=0o777) -> fd\n\n\ +Like open() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_openat(PyObject *self, PyObject *args) +{ + PyObject *ofile; + char *file; + int flag, dirfd, fd; + int mode = 0777; + + if (!PyArg_ParseTuple(args, "iO&i|i:openat", + &dirfd, PyUnicode_FSConverter, &ofile, + &flag, &mode)) + return NULL; + file = PyBytes_AsString(ofile); + Py_BEGIN_ALLOW_THREADS + fd = openat(dirfd, file, flag, mode); + Py_END_ALLOW_THREADS + Py_DECREF(ofile); + if (fd < 0) + return posix_error(); + return PyLong_FromLong((long)fd); +} +#endif + +#ifdef HAVE_READLINKAT +PyDoc_STRVAR(posix_readlinkat__doc__, +"readlinkat(dirfd, path) -> path\n\n\ +Like readlink() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_readlinkat(PyObject *self, PyObject *args) +{ + PyObject *v, *opath; + char buf[MAXPATHLEN]; + char *path; + int n, dirfd; + int arg_is_unicode = 0; + + if (!PyArg_ParseTuple(args, "iO&:readlinkat", + &dirfd, PyUnicode_FSConverter, &opath)) + return NULL; + path = PyBytes_AsString(opath); + v = PySequence_GetItem(args, 1); + if (v == NULL) { + Py_DECREF(opath); + return NULL; + } + + if (PyUnicode_Check(v)) { + arg_is_unicode = 1; + } + Py_DECREF(v); + + Py_BEGIN_ALLOW_THREADS + n = readlinkat(dirfd, path, buf, (int) sizeof buf); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (n < 0) + return posix_error(); + + if (arg_is_unicode) + return PyUnicode_DecodeFSDefaultAndSize(buf, n); + else + return PyBytes_FromStringAndSize(buf, n); +} +#endif /* HAVE_READLINKAT */ + +#ifdef HAVE_RENAMEAT +PyDoc_STRVAR(posix_renameat__doc__, +"renameat(olddirfd, oldpath, newdirfd, newpath)\n\n\ +Like rename() but if oldpath is relative, it is taken as relative to\n\ +olddirfd and if newpath is relative, it is taken as relative to newdirfd.\n\ +If oldpath is relative and olddirfd is the special value AT_FDCWD, then\n\ +oldpath is interpreted relative to the current working directory. This\n\ +also applies for newpath."); + +static PyObject * +posix_renameat(PyObject *self, PyObject *args) +{ + int res; + PyObject *opathold, *opathnew; + char *opath, *npath; + int oldfd, newfd; + + if (!PyArg_ParseTuple(args, "iO&iO&:renameat", + &oldfd, PyUnicode_FSConverter, &opathold, &newfd, PyUnicode_FSConverter, &opathnew)) + return NULL; + opath = PyBytes_AsString(opathold); + npath = PyBytes_AsString(opathnew); + Py_BEGIN_ALLOW_THREADS + res = renameat(oldfd, opath, newfd, npath); + Py_END_ALLOW_THREADS + Py_DECREF(opathold); + Py_DECREF(opathnew); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#if HAVE_SYMLINKAT +PyDoc_STRVAR(posix_symlinkat__doc__, +"symlinkat(src, dstfd, dst)\n\n\ +Like symlink() but if dst is relative, it is taken as relative to dstfd.\n\ +If dst is relative and dstfd is the special value AT_FDCWD, then dst\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_symlinkat(PyObject *self, PyObject *args) +{ + int res, dstfd; + PyObject *osrc, *odst; + char *src, *dst; + + if (!PyArg_ParseTuple(args, "O&iO&:symlinkat", + PyUnicode_FSConverter, &osrc, &dstfd, PyUnicode_FSConverter, &odst)) + return NULL; + src = PyBytes_AsString(osrc); + dst = PyBytes_AsString(odst); + Py_BEGIN_ALLOW_THREADS + res = symlinkat(src, dstfd, dst); + Py_END_ALLOW_THREADS + Py_DECREF(osrc); + Py_DECREF(odst); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif /* HAVE_SYMLINKAT */ + +#ifdef HAVE_UNLINKAT +PyDoc_STRVAR(posix_unlinkat__doc__, +"unlinkat(dirfd, path, flags=0)\n\n\ +Like unlink() but if path is relative, it is taken as relative to dirfd.\n\ +flags is optional and may be 0 or AT_REMOVEDIR. If AT_REMOVEDIR is\n\ +specified, unlinkat() behaves like rmdir().\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_unlinkat(PyObject *self, PyObject *args) +{ + int dirfd, res, flags = 0; + PyObject *opath; + char *path; + + if (!PyArg_ParseTuple(args, "iO&|i:unlinkat", + &dirfd, PyUnicode_FSConverter, &opath, &flags)) + return NULL; + path = PyBytes_AsString(opath); + Py_BEGIN_ALLOW_THREADS + res = unlinkat(dirfd, path, flags); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_UTIMENSAT +PyDoc_STRVAR(posix_utimensat__doc__, +"utimensat(dirfd, path, (atime_sec, atime_nsec),\n\ + (mtime_sec, mtime_nsec), flags)\n\ +utimensat(dirfd, path, None, None, flags)\n\n\ +Updates the timestamps of a file with nanosecond precision. If path is\n\ +relative, it is taken as relative to dirfd.\n\ +The second form sets atime and mtime to the current time.\n\ +flags is optional and may be 0 or AT_SYMLINK_NOFOLLOW.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory.\n\ +If *_nsec is specified as UTIME_NOW, the timestamp is updated to the\n\ +current time.\n\ +If *_nsec is specified as UTIME_OMIT, the timestamp is not updated."); + +static PyObject * +posix_utimensat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *path; + int res, dirfd, flags = 0; + PyObject *atime, *mtime; + + struct timespec buf[2]; + + if (!PyArg_ParseTuple(args, "iO&OO|i:utimensat", + &dirfd, PyUnicode_FSConverter, &opath, &atime, &mtime, &flags)) + return NULL; + path = PyBytes_AsString(opath); + if (atime == Py_None && mtime == Py_None) { + /* optional time values not given */ + Py_BEGIN_ALLOW_THREADS + res = utimensat(dirfd, path, NULL, flags); + Py_END_ALLOW_THREADS + } + else if (!PyTuple_Check(atime) || PyTuple_Size(atime) != 2) { + PyErr_SetString(PyExc_TypeError, + "utimensat() arg 3 must be a tuple (atime_sec, atime_nsec)"); + Py_DECREF(opath); + return NULL; + } + else if (!PyTuple_Check(mtime) || PyTuple_Size(mtime) != 2) { + PyErr_SetString(PyExc_TypeError, + "utimensat() arg 4 must be a tuple (mtime_sec, mtime_nsec)"); + Py_DECREF(opath); + return NULL; + } + else { + if (!PyArg_ParseTuple(atime, "ll:utimensat", + &(buf[0].tv_sec), &(buf[0].tv_nsec))) { + Py_DECREF(opath); + return NULL; + } + if (!PyArg_ParseTuple(mtime, "ll:utimensat", + &(buf[1].tv_sec), &(buf[1].tv_nsec))) { + Py_DECREF(opath); + return NULL; + } + Py_BEGIN_ALLOW_THREADS + res = utimensat(dirfd, path, buf, flags); + Py_END_ALLOW_THREADS + } + Py_DECREF(opath); + if (res < 0) { + return posix_error(); + } + Py_RETURN_NONE; +} +#endif + +#ifdef HAVE_MKFIFOAT +PyDoc_STRVAR(posix_mkfifoat__doc__, +"mkfifoat(dirfd, path, mode=0o666)\n\n\ +Like mkfifo() but if path is relative, it is taken as relative to dirfd.\n\ +If path is relative and dirfd is the special value AT_FDCWD, then path\n\ +is interpreted relative to the current working directory."); + +static PyObject * +posix_mkfifoat(PyObject *self, PyObject *args) +{ + PyObject *opath; + char *filename; + int mode = 0666; + int res, dirfd; + if (!PyArg_ParseTuple(args, "iO&|i:mkfifoat", + &dirfd, PyUnicode_FSConverter, &opath, &mode)) + return NULL; + filename = PyBytes_AS_STRING(opath); + Py_BEGIN_ALLOW_THREADS + res = mkfifoat(dirfd, filename, mode); + Py_END_ALLOW_THREADS + Py_DECREF(opath); + if (res < 0) + return posix_error(); + Py_RETURN_NONE; +} +#endif + static PyMethodDef posix_methods[] = { {"access", posix_access, METH_VARARGS, posix_access__doc__}, #ifdef HAVE_TTYNAME @@ -7891,11 +9994,20 @@ static PyMethodDef posix_methods[] = { {"link", posix_link, METH_VARARGS, posix_link__doc__}, #endif /* HAVE_LINK */ {"listdir", posix_listdir, METH_VARARGS, posix_listdir__doc__}, +#ifdef HAVE_FDOPENDIR + {"fdlistdir", posix_fdlistdir, METH_VARARGS, posix_fdlistdir__doc__}, +#endif {"lstat", posix_lstat, METH_VARARGS, posix_lstat__doc__}, {"mkdir", posix_mkdir, METH_VARARGS, posix_mkdir__doc__}, #ifdef HAVE_NICE {"nice", posix_nice, METH_VARARGS, posix_nice__doc__}, #endif /* HAVE_NICE */ +#ifdef HAVE_GETPRIORITY + {"getpriority", posix_getpriority, METH_VARARGS, posix_getpriority__doc__}, +#endif /* HAVE_GETPRIORITY */ +#ifdef HAVE_SETPRIORITY + {"setpriority", posix_setpriority, METH_VARARGS, posix_setpriority__doc__}, +#endif /* HAVE_SETPRIORITY */ #ifdef HAVE_READLINK {"readlink", posix_readlink, METH_VARARGS, posix_readlink__doc__}, #endif /* HAVE_READLINK */ @@ -7923,6 +10035,15 @@ static PyMethodDef posix_methods[] = { {"unlink", posix_unlink, METH_VARARGS, posix_unlink__doc__}, {"remove", posix_unlink, METH_VARARGS, posix_remove__doc__}, {"utime", posix_utime, METH_VARARGS, posix_utime__doc__}, +#ifdef HAVE_FUTIMES + {"futimes", posix_futimes, METH_VARARGS, posix_futimes__doc__}, +#endif +#ifdef HAVE_LUTIMES + {"lutimes", posix_lutimes, METH_VARARGS, posix_lutimes__doc__}, +#endif +#ifdef HAVE_FUTIMENS + {"futimens", posix_futimens, METH_VARARGS, posix_futimens__doc__}, +#endif #ifdef HAVE_TIMES {"times", posix_times, METH_NOARGS, posix_times__doc__}, #endif /* HAVE_TIMES */ @@ -7931,6 +10052,9 @@ static PyMethodDef posix_methods[] = { {"execv", posix_execv, METH_VARARGS, posix_execv__doc__}, {"execve", posix_execve, METH_VARARGS, posix_execve__doc__}, #endif /* HAVE_EXECV */ +#ifdef HAVE_FEXECVE + {"fexecve", posix_fexecve, METH_VARARGS, posix_fexecve__doc__}, +#endif #ifdef HAVE_SPAWNV {"spawnv", posix_spawnv, METH_VARARGS, posix_spawnv__doc__}, {"spawnve", posix_spawnve, METH_VARARGS, posix_spawnve__doc__}, @@ -7945,6 +10069,30 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FORK {"fork", posix_fork, METH_NOARGS, posix_fork__doc__}, #endif /* HAVE_FORK */ +#ifdef HAVE_SCHED_H + {"sched_get_priority_max", posix_sched_get_priority_max, METH_VARARGS, posix_sched_get_priority_max__doc__}, + {"sched_get_priority_min", posix_sched_get_priority_min, METH_VARARGS, posix_sched_get_priority_min__doc__}, +#ifdef HAVE_SCHED_SETPARAM + {"sched_getparam", posix_sched_getparam, METH_VARARGS, posix_sched_getparam__doc__}, +#endif +#ifdef HAVE_SCHED_SETSCHEDULER + {"sched_getscheduler", posix_sched_getscheduler, METH_VARARGS, posix_sched_getscheduler__doc__}, +#endif +#ifdef HAVE_SCHED_RR_GET_INTERVAL + {"sched_rr_get_interval", posix_sched_rr_get_interval, METH_VARARGS, posix_sched_rr_get_interval__doc__}, +#endif +#ifdef HAVE_SCHED_SETPARAM + {"sched_setparam", posix_sched_setparam, METH_VARARGS, posix_sched_setparam__doc__}, +#endif +#ifdef HAVE_SCHED_SETSCHEDULER + {"sched_setscheduler", posix_sched_setscheduler, METH_VARARGS, posix_sched_setscheduler__doc__}, +#endif + {"sched_yield", posix_sched_yield, METH_NOARGS, posix_sched_yield__doc__}, +#ifdef HAVE_SCHED_SETAFFINITY + {"sched_setaffinity", posix_sched_setaffinity, METH_VARARGS, posix_sched_setaffinity__doc__}, + {"sched_getaffinity", posix_sched_getaffinity, METH_VARARGS, posix_sched_getaffinity__doc__}, +#endif +#endif #if defined(HAVE_OPENPTY) || defined(HAVE__GETPTY) || defined(HAVE_DEV_PTMX) {"openpty", posix_openpty, METH_NOARGS, posix_openpty__doc__}, #endif /* HAVE_OPENPTY || HAVE__GETPTY || HAVE_DEV_PTMX */ @@ -7960,6 +10108,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_GETGID {"getgid", posix_getgid, METH_NOARGS, posix_getgid__doc__}, #endif /* HAVE_GETGID */ +#ifdef HAVE_GETGROUPLIST + {"getgrouplist", posix_getgrouplist, METH_VARARGS, posix_getgrouplist__doc__}, +#endif #ifdef HAVE_GETGROUPS {"getgroups", posix_getgroups, METH_NOARGS, posix_getgroups__doc__}, #endif @@ -8029,6 +10180,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_WAIT4 {"wait4", posix_wait4, METH_VARARGS, posix_wait4__doc__}, #endif /* HAVE_WAIT4 */ +#if defined(HAVE_WAITID) && !defined(__APPLE__) + {"waitid", posix_waitid, METH_VARARGS, posix_waitid__doc__}, +#endif #if defined(HAVE_WAITPID) || defined(HAVE_CWAIT) {"waitpid", posix_waitpid, METH_VARARGS, posix_waitpid__doc__}, #endif /* HAVE_WAITPID */ @@ -8053,14 +10207,36 @@ static PyMethodDef posix_methods[] = { {"device_encoding", device_encoding, METH_VARARGS, device_encoding__doc__}, {"dup", posix_dup, METH_VARARGS, posix_dup__doc__}, {"dup2", posix_dup2, METH_VARARGS, posix_dup2__doc__}, +#ifdef HAVE_LOCKF + {"lockf", posix_lockf, METH_VARARGS, posix_lockf__doc__}, +#endif {"lseek", posix_lseek, METH_VARARGS, posix_lseek__doc__}, {"read", posix_read, METH_VARARGS, posix_read__doc__}, +#ifdef HAVE_READV + {"readv", posix_readv, METH_VARARGS, posix_readv__doc__}, +#endif +#ifdef HAVE_PREAD + {"pread", posix_pread, METH_VARARGS, posix_pread__doc__}, +#endif {"write", posix_write, METH_VARARGS, posix_write__doc__}, +#ifdef HAVE_WRITEV + {"writev", posix_writev, METH_VARARGS, posix_writev__doc__}, +#endif +#ifdef HAVE_PWRITE + {"pwrite", posix_pwrite, METH_VARARGS, posix_pwrite__doc__}, +#endif +#ifdef HAVE_SENDFILE + {"sendfile", (PyCFunction)posix_sendfile, METH_VARARGS | METH_KEYWORDS, + posix_sendfile__doc__}, +#endif {"fstat", posix_fstat, METH_VARARGS, posix_fstat__doc__}, {"isatty", posix_isatty, METH_VARARGS, posix_isatty__doc__}, #ifdef HAVE_PIPE {"pipe", posix_pipe, METH_NOARGS, posix_pipe__doc__}, #endif +#ifdef HAVE_PIPE2 + {"pipe2", posix_pipe2, METH_O, posix_pipe2__doc__}, +#endif #ifdef HAVE_MKFIFO {"mkfifo", posix_mkfifo, METH_VARARGS, posix_mkfifo__doc__}, #endif @@ -8075,6 +10251,15 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FTRUNCATE {"ftruncate", posix_ftruncate, METH_VARARGS, posix_ftruncate__doc__}, #endif +#ifdef HAVE_TRUNCATE + {"truncate", posix_truncate, METH_VARARGS, posix_truncate__doc__}, +#endif +#ifdef HAVE_POSIX_FALLOCATE + {"posix_fallocate", posix_posix_fallocate, METH_VARARGS, posix_posix_fallocate__doc__}, +#endif +#ifdef HAVE_POSIX_FADVISE + {"posix_fadvise", posix_posix_fadvise, METH_VARARGS, posix_posix_fadvise__doc__}, +#endif #ifdef HAVE_PUTENV {"putenv", posix_putenv, METH_VARARGS, posix_putenv__doc__}, #endif @@ -8088,6 +10273,9 @@ static PyMethodDef posix_methods[] = { #ifdef HAVE_FSYNC {"fsync", posix_fsync, METH_O, posix_fsync__doc__}, #endif +#ifdef HAVE_SYNC + {"sync", posix_sync, METH_NOARGS, posix_sync__doc__}, +#endif #ifdef HAVE_FDATASYNC {"fdatasync", posix_fdatasync, METH_O, posix_fdatasync__doc__}, #endif @@ -8141,6 +10329,7 @@ static PyMethodDef posix_methods[] = { {"_getfinalpathname", posix__getfinalpathname, METH_VARARGS, NULL}, {"_getfileinformation", posix__getfileinformation, METH_VARARGS, NULL}, {"_isdir", posix__isdir, METH_VARARGS, posix__isdir__doc__}, + {"_getdiskusage", win32__getdiskusage, METH_VARARGS, win32__getdiskusage__doc__}, #endif #ifdef HAVE_GETLOADAVG {"getloadavg", posix_getloadavg, METH_NOARGS, posix_getloadavg__doc__}, @@ -8164,6 +10353,52 @@ static PyMethodDef posix_methods[] = { {"getresgid", posix_getresgid, METH_NOARGS, posix_getresgid__doc__}, #endif +/* posix *at family of functions */ +#ifdef HAVE_FACCESSAT + {"faccessat", posix_faccessat, METH_VARARGS, posix_faccessat__doc__}, +#endif +#ifdef HAVE_FCHMODAT + {"fchmodat", posix_fchmodat, METH_VARARGS, posix_fchmodat__doc__}, +#endif /* HAVE_FCHMODAT */ +#ifdef HAVE_FCHOWNAT + {"fchownat", posix_fchownat, METH_VARARGS, posix_fchownat__doc__}, +#endif /* HAVE_FCHOWNAT */ +#ifdef HAVE_FSTATAT + {"fstatat", posix_fstatat, METH_VARARGS, posix_fstatat__doc__}, +#endif +#ifdef HAVE_FUTIMESAT + {"futimesat", posix_futimesat, METH_VARARGS, posix_futimesat__doc__}, +#endif +#ifdef HAVE_LINKAT + {"linkat", posix_linkat, METH_VARARGS, posix_linkat__doc__}, +#endif /* HAVE_LINKAT */ +#ifdef HAVE_MKDIRAT + {"mkdirat", posix_mkdirat, METH_VARARGS, posix_mkdirat__doc__}, +#endif +#if defined(HAVE_MKNODAT) && defined(HAVE_MAKEDEV) + {"mknodat", posix_mknodat, METH_VARARGS, posix_mknodat__doc__}, +#endif +#ifdef HAVE_OPENAT + {"openat", posix_openat, METH_VARARGS, posix_openat__doc__}, +#endif +#ifdef HAVE_READLINKAT + {"readlinkat", posix_readlinkat, METH_VARARGS, posix_readlinkat__doc__}, +#endif /* HAVE_READLINKAT */ +#ifdef HAVE_RENAMEAT + {"renameat", posix_renameat, METH_VARARGS, posix_renameat__doc__}, +#endif +#if HAVE_SYMLINKAT + {"symlinkat", posix_symlinkat, METH_VARARGS, posix_symlinkat__doc__}, +#endif /* HAVE_SYMLINKAT */ +#ifdef HAVE_UNLINKAT + {"unlinkat", posix_unlinkat, METH_VARARGS, posix_unlinkat__doc__}, +#endif +#ifdef HAVE_UTIMENSAT + {"utimensat", posix_utimensat, METH_VARARGS, posix_utimensat__doc__}, +#endif +#ifdef HAVE_MKFIFOAT + {"mkfifoat", posix_mkfifoat, METH_VARARGS, posix_mkfifoat__doc__}, +#endif {NULL, NULL} /* Sentinel */ }; @@ -8340,6 +10575,41 @@ all_ins(PyObject *d) #ifdef O_EXLOCK if (ins(d, "O_EXLOCK", (long)O_EXLOCK)) return -1; #endif +#ifdef PRIO_PROCESS + if (ins(d, "PRIO_PROCESS", (long)PRIO_PROCESS)) return -1; +#endif +#ifdef PRIO_PGRP + if (ins(d, "PRIO_PGRP", (long)PRIO_PGRP)) return -1; +#endif +#ifdef PRIO_USER + if (ins(d, "PRIO_USER", (long)PRIO_USER)) return -1; +#endif +#ifdef O_CLOEXEC + if (ins(d, "O_CLOEXEC", (long)O_CLOEXEC)) return -1; +#endif +/* posix - constants for *at functions */ +#ifdef AT_SYMLINK_NOFOLLOW + if (ins(d, "AT_SYMLINK_NOFOLLOW", (long)AT_SYMLINK_NOFOLLOW)) return -1; +#endif +#ifdef AT_EACCESS + if (ins(d, "AT_EACCESS", (long)AT_EACCESS)) return -1; +#endif +#ifdef AT_FDCWD + if (ins(d, "AT_FDCWD", (long)AT_FDCWD)) return -1; +#endif +#ifdef AT_REMOVEDIR + if (ins(d, "AT_REMOVEDIR", (long)AT_REMOVEDIR)) return -1; +#endif +#ifdef AT_SYMLINK_FOLLOW + if (ins(d, "AT_SYMLINK_FOLLOW", (long)AT_SYMLINK_FOLLOW)) return -1; +#endif +#ifdef UTIME_NOW + if (ins(d, "UTIME_NOW", (long)UTIME_NOW)) return -1; +#endif +#ifdef UTIME_OMIT + if (ins(d, "UTIME_OMIT", (long)UTIME_OMIT)) return -1; +#endif + /* MS Windows */ #ifdef O_NOINHERIT @@ -8448,6 +10718,87 @@ all_ins(PyObject *d) if (ins(d, "ST_NOSUID", (long)ST_NOSUID)) return -1; #endif /* ST_NOSUID */ + /* FreeBSD sendfile() constants */ +#ifdef SF_NODISKIO + if (ins(d, "SF_NODISKIO", (long)SF_NODISKIO)) return -1; +#endif +#ifdef SF_MNOWAIT + if (ins(d, "SF_MNOWAIT", (long)SF_MNOWAIT)) return -1; +#endif +#ifdef SF_SYNC + if (ins(d, "SF_SYNC", (long)SF_SYNC)) return -1; +#endif + + /* constants for posix_fadvise */ +#ifdef POSIX_FADV_NORMAL + if (ins(d, "POSIX_FADV_NORMAL", (long)POSIX_FADV_NORMAL)) return -1; +#endif +#ifdef POSIX_FADV_SEQUENTIAL + if (ins(d, "POSIX_FADV_SEQUENTIAL", (long)POSIX_FADV_SEQUENTIAL)) return -1; +#endif +#ifdef POSIX_FADV_RANDOM + if (ins(d, "POSIX_FADV_RANDOM", (long)POSIX_FADV_RANDOM)) return -1; +#endif +#ifdef POSIX_FADV_NOREUSE + if (ins(d, "POSIX_FADV_NOREUSE", (long)POSIX_FADV_NOREUSE)) return -1; +#endif +#ifdef POSIX_FADV_WILLNEED + if (ins(d, "POSIX_FADV_WILLNEED", (long)POSIX_FADV_WILLNEED)) return -1; +#endif +#ifdef POSIX_FADV_DONTNEED + if (ins(d, "POSIX_FADV_DONTNEED", (long)POSIX_FADV_DONTNEED)) return -1; +#endif + + /* constants for waitid */ +#if defined(HAVE_SYS_WAIT_H) && defined(HAVE_WAITID) + if (ins(d, "P_PID", (long)P_PID)) return -1; + if (ins(d, "P_PGID", (long)P_PGID)) return -1; + if (ins(d, "P_ALL", (long)P_ALL)) return -1; +#endif +#ifdef WEXITED + if (ins(d, "WEXITED", (long)WEXITED)) return -1; +#endif +#ifdef WNOWAIT + if (ins(d, "WNOWAIT", (long)WNOWAIT)) return -1; +#endif +#ifdef WSTOPPED + if (ins(d, "WSTOPPED", (long)WSTOPPED)) return -1; +#endif +#ifdef CLD_EXITED + if (ins(d, "CLD_EXITED", (long)CLD_EXITED)) return -1; +#endif +#ifdef CLD_DUMPED + if (ins(d, "CLD_DUMPED", (long)CLD_DUMPED)) return -1; +#endif +#ifdef CLD_TRAPPED + if (ins(d, "CLD_TRAPPED", (long)CLD_TRAPPED)) return -1; +#endif +#ifdef CLD_CONTINUED + if (ins(d, "CLD_CONTINUED", (long)CLD_CONTINUED)) return -1; +#endif + + /* constants for lockf */ +#ifdef F_LOCK + if (ins(d, "F_LOCK", (long)F_LOCK)) return -1; +#endif +#ifdef F_TLOCK + if (ins(d, "F_TLOCK", (long)F_TLOCK)) return -1; +#endif +#ifdef F_ULOCK + if (ins(d, "F_ULOCK", (long)F_ULOCK)) return -1; +#endif +#ifdef F_TEST + if (ins(d, "F_TEST", (long)F_TEST)) return -1; +#endif + + /* constants for futimens */ +#ifdef UTIME_NOW + if (ins(d, "UTIME_NOW", (long)UTIME_NOW)) return -1; +#endif +#ifdef UTIME_OMIT + if (ins(d, "UTIME_OMIT", (long)UTIME_OMIT)) return -1; +#endif + #ifdef HAVE_SPAWNV #if defined(PYOS_OS2) && defined(PYCC_GCC) if (ins(d, "P_WAIT", (long)P_WAIT)) return -1; @@ -8479,6 +10830,24 @@ all_ins(PyObject *d) #endif #endif +#ifdef HAVE_SCHED_H + if (ins(d, "SCHED_FIFO", (long)SCHED_FIFO)) return -1; + if (ins(d, "SCHED_RR", (long)SCHED_RR)) return -1; +#ifdef SCHED_SPORADIC + if (ins(d, "SCHED_SPORADIC", (long)SCHED_SPORADIC) return -1; +#endif + if (ins(d, "SCHED_OTHER", (long)SCHED_OTHER)) return -1; +#ifdef SCHED_BATCH + if (ins(d, "SCHED_BATCH", (long)SCHED_BATCH)) return -1; +#endif +#ifdef SCHED_IDLE + if (ins(d, "SCHED_IDLE", (long)SCHED_IDLE)) return -1; +#endif +#ifdef SCHED_RESET_ON_FORK + if (ins(d, "SCHED_RESET_ON_FORK", (long)SCHED_RESET_ON_FORK)) return -1; +#endif +#endif + #if defined(PYOS_OS2) if (insertvalues(d)) return -1; #endif @@ -8541,12 +10910,24 @@ INITFUNC(void) Py_INCREF(PyExc_OSError); PyModule_AddObject(m, "error", PyExc_OSError); +#ifdef HAVE_SCHED_SETAFFINITY + if (PyType_Ready(&cpu_set_type) < 0) + return NULL; + Py_INCREF(&cpu_set_type); + PyModule_AddObject(m, "cpu_set", (PyObject *)&cpu_set_type); +#endif + #ifdef HAVE_PUTENV if (posix_putenv_garbage == NULL) posix_putenv_garbage = PyDict_New(); #endif if (!initialized) { +#if defined(HAVE_WAITID) && !defined(__APPLE__) + waitid_result_desc.name = MODNAME ".waitid_result"; + PyStructSequence_InitType(&WaitidResultType, &waitid_result_desc); +#endif + stat_result_desc.name = MODNAME ".stat_result"; stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; @@ -8566,12 +10947,27 @@ INITFUNC(void) ticks_per_second = 60; /* magic fallback value; may be bogus */ # endif #endif + +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) + sched_param_desc.name = MODNAME ".sched_param"; + PyStructSequence_InitType(&SchedParamType, &sched_param_desc); + SchedParamType.tp_new = sched_param_new; +#endif } +#if defined(HAVE_WAITID) && !defined(__APPLE__) + Py_INCREF((PyObject*) &WaitidResultType); + PyModule_AddObject(m, "waitid_result", (PyObject*) &WaitidResultType); +#endif Py_INCREF((PyObject*) &StatResultType); PyModule_AddObject(m, "stat_result", (PyObject*) &StatResultType); Py_INCREF((PyObject*) &StatVFSResultType); PyModule_AddObject(m, "statvfs_result", (PyObject*) &StatVFSResultType); + +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) + Py_INCREF(&SchedParamType); + PyModule_AddObject(m, "sched_param", (PyObject *)&SchedParamType); +#endif initialized = 1; #ifdef __APPLE__ diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 849423f..d923eeb 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -100,16 +100,17 @@ static PyObject * set_error(xmlparseobject *self, enum XML_Error code) { PyObject *err; - char buffer[256]; + PyObject *buffer; XML_Parser parser = self->itself; int lineno = XML_GetErrorLineNumber(parser); int column = XML_GetErrorColumnNumber(parser); - /* There is no risk of overflowing this buffer, since - even for 64-bit integers, there is sufficient space. */ - sprintf(buffer, "%.200s: line %i, column %i", - XML_ErrorString(code), lineno, column); - err = PyObject_CallFunction(ErrorObject, "s", buffer); + buffer = PyUnicode_FromFormat("%s: line %i, column %i", + XML_ErrorString(code), lineno, column); + if (buffer == NULL) + return NULL; + err = PyObject_CallFunction(ErrorObject, "O", buffer); + Py_DECREF(buffer); if ( err != NULL && set_error_attr(err, "code", code) && set_error_attr(err, "offset", column) @@ -1661,7 +1662,6 @@ MODULE_INITFUNC(void) PyObject *errors_module; PyObject *modelmod_name; PyObject *model_module; - PyObject *version; PyObject *sys_modules; PyObject *tmpnum, *tmpstr; PyObject *codes_dict; @@ -1698,10 +1698,6 @@ MODULE_INITFUNC(void) Py_INCREF(&Xmlparsetype); PyModule_AddObject(m, "XMLParserType", (PyObject *) &Xmlparsetype); - version = PyUnicode_FromString(PY_VERSION); - if (!version) - return NULL; - PyModule_AddObject(m, "__version__", version); PyModule_AddStringConstant(m, "EXPAT_VERSION", (char *) XML_ExpatVersion()); { diff --git a/Modules/readline.c b/Modules/readline.c index 8337956..a5e48ab 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -233,10 +233,9 @@ set_hook(const char *funcname, PyObject **hook_var, PyObject *args) Py_XDECREF(tmp); } else { - PyOS_snprintf(buf, sizeof(buf), - "set_%.50s(func): argument not callable", - funcname); - PyErr_SetString(PyExc_TypeError, buf); + PyErr_Format(PyExc_TypeError, + "set_%.50s(func): argument not callable", + funcname); return NULL; } Py_RETURN_NONE; @@ -890,7 +889,7 @@ setup_readline(void) #endif #ifdef __APPLE__ - /* the libedit readline emulation resets key bindings etc + /* the libedit readline emulation resets key bindings etc * when calling rl_initialize. So call it upfront */ if (using_libedit_emulation) @@ -930,11 +929,11 @@ setup_readline(void) */ #ifdef __APPLE__ if (using_libedit_emulation) - rl_read_init_file(NULL); + rl_read_init_file(NULL); else #endif /* __APPLE__ */ rl_initialize(); - + RESTORE_LOCALE(saved_locale) } diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 1285e65..20f23d9 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -234,6 +234,11 @@ select_select(PyObject *self, PyObject *args) "timeout period too long"); return NULL; } + if (timeout < 0) { + PyErr_SetString(PyExc_ValueError, + "timeout must be non-negative"); + return NULL; + } seconds = (long)timeout; timeout = timeout - (double)seconds; tv.tv_sec = seconds; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index b25bd44..1cace54 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -218,7 +218,7 @@ void sha1_process(struct sha1_state *sha1, in += SHA1_BLOCKSIZE; inlen -= SHA1_BLOCKSIZE; } else { - n = MIN(inlen, (SHA1_BLOCKSIZE - sha1->curlen)); + n = MIN(inlen, (Py_ssize_t)(SHA1_BLOCKSIZE - sha1->curlen)); memcpy(sha1->buf + sha1->curlen, in, (size_t)n); sha1->curlen += n; in += n; diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 87c1c9a..45a7dfa 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -22,6 +22,14 @@ #include <sys/time.h> #endif +#if defined(HAVE_PTHREAD_SIGMASK) && !defined(HAVE_BROKEN_PTHREAD_SIGMASK) +# define PYPTHREAD_SIGMASK +#endif + +#if defined(PYPTHREAD_SIGMASK) && defined(HAVE_PTHREAD_H) +# include <pthread.h> +#endif + #ifndef SIG_ERR #define SIG_ERR ((PyOS_sighandler_t)(-1)) #endif @@ -168,15 +176,19 @@ checksignals_witharg(void * unused) static void trip_signal(int sig_num) { + unsigned char byte; + Handlers[sig_num].tripped = 1; + if (wakeup_fd != -1) { + byte = (unsigned char)sig_num; + write(wakeup_fd, &byte, 1); + } if (is_tripped) return; /* Set is_tripped after setting .tripped, as it gets cleared in PyErr_CheckSignals() before .tripped. */ is_tripped = 1; Py_AddPendingCall(checksignals_witharg, NULL); - if (wakeup_fd != -1) - write(wakeup_fd, "\0", 1); } static void @@ -313,7 +325,7 @@ signal_signal(PyObject *self, PyObject *args) else func = signal_handler; if (PyOS_setsig(sig_num, func) == SIG_ERR) { - PyErr_SetFromErrno(PyExc_RuntimeError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } old_handler = Handlers[sig_num].func; @@ -382,7 +394,7 @@ signal_siginterrupt(PyObject *self, PyObject *args) return NULL; } if (siginterrupt(sig_num, flag)<0) { - PyErr_SetFromErrno(PyExc_RuntimeError); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -495,6 +507,346 @@ PyDoc_STRVAR(getitimer_doc, Returns current value of given itimer."); #endif +#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGWAIT) || \ + defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) +/* Convert an iterable to a sigset. + Return 0 on success, return -1 and raise an exception on error. */ + +static int +iterable_to_sigset(PyObject *iterable, sigset_t *mask) +{ + int result = -1; + PyObject *iterator, *item; + long signum; + int err; + + sigemptyset(mask); + + iterator = PyObject_GetIter(iterable); + if (iterator == NULL) + goto error; + + while (1) + { + item = PyIter_Next(iterator); + if (item == NULL) { + if (PyErr_Occurred()) + goto error; + else + break; + } + + signum = PyLong_AsLong(item); + Py_DECREF(item); + if (signum == -1 && PyErr_Occurred()) + goto error; + if (0 < signum && signum < NSIG) + err = sigaddset(mask, (int)signum); + else + err = 1; + if (err) { + PyErr_Format(PyExc_ValueError, + "signal number %ld out of range", signum); + goto error; + } + } + result = 0; + +error: + Py_XDECREF(iterator); + return result; +} +#endif + +#if defined(PYPTHREAD_SIGMASK) || defined(HAVE_SIGPENDING) +static PyObject* +sigset_to_set(sigset_t mask) +{ + PyObject *signum, *result; + int sig; + + result = PySet_New(0); + if (result == NULL) + return NULL; + + for (sig = 1; sig < NSIG; sig++) { + if (sigismember(&mask, sig) != 1) + continue; + + /* Handle the case where it is a member by adding the signal to + the result list. Ignore the other cases because they mean the + signal isn't a member of the mask or the signal was invalid, + and an invalid signal must have been our fault in constructing + the loop boundaries. */ + signum = PyLong_FromLong(sig); + if (signum == NULL) { + Py_DECREF(result); + return NULL; + } + if (PySet_Add(result, signum) == -1) { + Py_DECREF(signum); + Py_DECREF(result); + return NULL; + } + Py_DECREF(signum); + } + return result; +} +#endif + +#ifdef PYPTHREAD_SIGMASK +static PyObject * +signal_pthread_sigmask(PyObject *self, PyObject *args) +{ + int how; + PyObject *signals; + sigset_t mask, previous; + int err; + + if (!PyArg_ParseTuple(args, "iO:pthread_sigmask", &how, &signals)) + return NULL; + + if (iterable_to_sigset(signals, &mask)) + return NULL; + + err = pthread_sigmask(how, &mask, &previous); + if (err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + /* if signals was unblocked, signal handlers have been called */ + if (PyErr_CheckSignals()) + return NULL; + + return sigset_to_set(previous); +} + +PyDoc_STRVAR(signal_pthread_sigmask_doc, +"pthread_sigmask(how, mask) -> old mask\n\ +\n\ +Fetch and/or change the signal mask of the calling thread."); +#endif /* #ifdef PYPTHREAD_SIGMASK */ + + +#ifdef HAVE_SIGPENDING +static PyObject * +signal_sigpending(PyObject *self) +{ + int err; + sigset_t mask; + err = sigpending(&mask); + if (err) + return PyErr_SetFromErrno(PyExc_OSError); + return sigset_to_set(mask); +} + +PyDoc_STRVAR(signal_sigpending_doc, +"sigpending() -> list\n\ +\n\ +Examine pending signals."); +#endif /* #ifdef HAVE_SIGPENDING */ + + +#ifdef HAVE_SIGWAIT +static PyObject * +signal_sigwait(PyObject *self, PyObject *args) +{ + PyObject *signals; + sigset_t set; + int err, signum; + + if (!PyArg_ParseTuple(args, "O:sigwait", &signals)) + return NULL; + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigwait(&set, &signum); + Py_END_ALLOW_THREADS + if (err) { + errno = err; + return PyErr_SetFromErrno(PyExc_OSError); + } + + return PyLong_FromLong(signum); +} + +PyDoc_STRVAR(signal_sigwait_doc, +"sigwait(sigset) -> signum\n\ +\n\ +Wait a signal."); +#endif /* #ifdef HAVE_SIGPENDING */ + +#if defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) +static int initialized; +static PyStructSequence_Field struct_siginfo_fields[] = { + {"si_signo", "signal number"}, + {"si_code", "signal code"}, + {"si_errno", "errno associated with this signal"}, + {"si_pid", "sending process ID"}, + {"si_uid", "real user ID of sending process"}, + {"si_status", "exit value or signal"}, + {"si_band", "band event for SIGPOLL"}, + {0} +}; + +PyDoc_STRVAR(struct_siginfo__doc__, +"struct_siginfo: Result from sigwaitinfo or sigtimedwait.\n\n\ +This object may be accessed either as a tuple of\n\ +(si_signo, si_code, si_errno, si_pid, si_uid, si_status, si_band),\n\ +or via the attributes si_signo, si_code, and so on."); + +static PyStructSequence_Desc struct_siginfo_desc = { + "signal.struct_siginfo", /* name */ + struct_siginfo__doc__, /* doc */ + struct_siginfo_fields, /* fields */ + 7 /* n_in_sequence */ +}; + +static PyTypeObject SiginfoType; + +static PyObject * +fill_siginfo(siginfo_t *si) +{ + PyObject *result = PyStructSequence_New(&SiginfoType); + if (!result) + return NULL; + + PyStructSequence_SET_ITEM(result, 0, PyLong_FromLong((long)(si->si_signo))); + PyStructSequence_SET_ITEM(result, 1, PyLong_FromLong((long)(si->si_code))); + PyStructSequence_SET_ITEM(result, 2, PyLong_FromLong((long)(si->si_errno))); + PyStructSequence_SET_ITEM(result, 3, PyLong_FromPid(si->si_pid)); + PyStructSequence_SET_ITEM(result, 4, PyLong_FromLong((long)(si->si_uid))); + PyStructSequence_SET_ITEM(result, 5, + PyLong_FromLong((long)(si->si_status))); + PyStructSequence_SET_ITEM(result, 6, PyLong_FromLong(si->si_band)); + if (PyErr_Occurred()) { + Py_DECREF(result); + return NULL; + } + + return result; +} +#endif + +#ifdef HAVE_SIGWAITINFO +static PyObject * +signal_sigwaitinfo(PyObject *self, PyObject *args) +{ + PyObject *signals; + sigset_t set; + siginfo_t si; + int err; + + if (!PyArg_ParseTuple(args, "O:sigwaitinfo", &signals)) + return NULL; + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigwaitinfo(&set, &si); + Py_END_ALLOW_THREADS + if (err == -1) + return PyErr_SetFromErrno(PyExc_OSError); + + return fill_siginfo(&si); +} + +PyDoc_STRVAR(signal_sigwaitinfo_doc, +"sigwaitinfo(sigset) -> struct_siginfo\n\ +\n\ +Wait synchronously for a signal until one of the signals in *sigset* is\n\ +delivered.\n\ +Returns a struct_siginfo containing information about the signal."); +#endif /* #ifdef HAVE_SIGWAITINFO */ + +#ifdef HAVE_SIGTIMEDWAIT +static PyObject * +signal_sigtimedwait(PyObject *self, PyObject *args) +{ + PyObject *signals, *timeout; + struct timespec buf; + sigset_t set; + siginfo_t si; + int err; + + if (!PyArg_ParseTuple(args, "OO:sigtimedwait", &signals, &timeout)) + return NULL; + + if (!PyTuple_Check(timeout) || PyTuple_Size(timeout) != 2) { + PyErr_SetString(PyExc_TypeError, + "sigtimedwait() arg 2 must be a tuple " + "(timeout_sec, timeout_nsec)"); + return NULL; + } else if (!PyArg_ParseTuple(timeout, "ll:sigtimedwait", + &(buf.tv_sec), &(buf.tv_nsec))) + return NULL; + + if (buf.tv_sec < 0 || buf.tv_nsec < 0) { + PyErr_SetString(PyExc_ValueError, "timeout must be non-negative"); + return NULL; + } + + if (iterable_to_sigset(signals, &set)) + return NULL; + + Py_BEGIN_ALLOW_THREADS + err = sigtimedwait(&set, &si, &buf); + Py_END_ALLOW_THREADS + if (err == -1) { + if (errno == EAGAIN) + Py_RETURN_NONE; + else + return PyErr_SetFromErrno(PyExc_OSError); + } + + return fill_siginfo(&si); +} + +PyDoc_STRVAR(signal_sigtimedwait_doc, +"sigtimedwait(sigset, (timeout_sec, timeout_nsec)) -> struct_siginfo\n\ +\n\ +Like sigwaitinfo(), but with a timeout specified as a tuple of (seconds,\n\ +nanoseconds)."); +#endif /* #ifdef HAVE_SIGTIMEDWAIT */ + + +#if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) +static PyObject * +signal_pthread_kill(PyObject *self, PyObject *args) +{ + long tid; + int signum; + int err; + + if (!PyArg_ParseTuple(args, "li:pthread_kill", &tid, &signum)) + return NULL; + + err = pthread_kill((pthread_t)tid, signum); + if (err != 0) { + errno = err; + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + /* the signal may have been send to the current thread */ + if (PyErr_CheckSignals()) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(signal_pthread_kill_doc, +"pthread_kill(thread_id, signum)\n\ +\n\ +Send a signal to a thread."); +#endif /* #if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) */ + + /* List of functions defined in the module */ static PyMethodDef signal_methods[] = { @@ -515,10 +867,34 @@ static PyMethodDef signal_methods[] = { #endif #ifdef HAVE_PAUSE {"pause", (PyCFunction)signal_pause, - METH_NOARGS,pause_doc}, + METH_NOARGS, pause_doc}, #endif {"default_int_handler", signal_default_int_handler, METH_VARARGS, default_int_handler_doc}, +#if defined(HAVE_PTHREAD_KILL) && defined(WITH_THREAD) + {"pthread_kill", (PyCFunction)signal_pthread_kill, + METH_VARARGS, signal_pthread_kill_doc}, +#endif +#ifdef PYPTHREAD_SIGMASK + {"pthread_sigmask", (PyCFunction)signal_pthread_sigmask, + METH_VARARGS, signal_pthread_sigmask_doc}, +#endif +#ifdef HAVE_SIGPENDING + {"sigpending", (PyCFunction)signal_sigpending, + METH_NOARGS, signal_sigpending_doc}, +#endif +#ifdef HAVE_SIGWAIT + {"sigwait", (PyCFunction)signal_sigwait, + METH_VARARGS, signal_sigwait_doc}, +#endif +#ifdef HAVE_SIGWAITINFO + {"sigwaitinfo", (PyCFunction)signal_sigwaitinfo, + METH_VARARGS, signal_sigwaitinfo_doc}, +#endif +#ifdef HAVE_SIGTIMEDWAIT + {"sigtimedwait", (PyCFunction)signal_sigtimedwait, + METH_VARARGS, signal_sigtimedwait_doc}, +#endif {NULL, NULL} /* sentinel */ }; @@ -587,6 +963,15 @@ PyInit_signal(void) if (m == NULL) return NULL; +#if defined(HAVE_SIGWAITINFO) || defined(HAVE_SIGTIMEDWAIT) + if (!initialized) + PyStructSequence_InitType(&SiginfoType, &struct_siginfo_desc); + + Py_INCREF((PyObject*) &SiginfoType); + PyModule_AddObject(m, "struct_siginfo", (PyObject*) &SiginfoType); + initialized = 1; +#endif + /* Add some symbolic constants to the module */ d = PyModule_GetDict(m); @@ -603,6 +988,19 @@ PyInit_signal(void) goto finally; Py_DECREF(x); +#ifdef SIG_BLOCK + if (PyModule_AddIntMacro(m, SIG_BLOCK)) + goto finally; +#endif +#ifdef SIG_UNBLOCK + if (PyModule_AddIntMacro(m, SIG_UNBLOCK)) + goto finally; +#endif +#ifdef SIG_SETMASK + if (PyModule_AddIntMacro(m, SIG_SETMASK)) + goto finally; +#endif + x = IntHandler = PyDict_GetItemString(d, "default_int_handler"); if (!x) goto finally; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index fcdbf8a..8de84b7 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -42,6 +42,9 @@ Module interface: - socket.inet_ntoa(packed IP) -> IP address string - socket.getdefaulttimeout() -> None | float - socket.setdefaulttimeout(None | float) +- socket.if_nameindex() -> list of tuples (if_index, if_name) +- socket.if_nametoindex(name) -> corresponding interface index +- socket.if_indextoname(index) -> corresponding interface name - an Internet socket address is a pair (hostname, port) where hostname can be anything recognized by gethostbyname() (including the dd.dd.dd.dd notation) and port is in host byte order @@ -133,6 +136,9 @@ setblocking(0 | 1) -- set or clear the blocking I/O flag\n\ setsockopt(level, optname, value) -- set socket options\n\ settimeout(None | float) -- set or clear the timeout\n\ shutdown(how) -- shut down traffic in one or both directions\n\ +if_nameindex() -- return all network interface indices and names\n\ +if_nametoindex(name) -- return the corresponding interface index\n\ +if_indextoname(index) -- return the corresponding interface name\n\ \n\ [*] not available on all platforms!"); @@ -155,7 +161,7 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #endif #ifdef HAVE_GETHOSTBYNAME_R -# if defined(_AIX) || defined(__osf__) +# if defined(_AIX) # define HAVE_GETHOSTBYNAME_R_3_ARG # elif defined(__sun) || defined(__sgi) # define HAVE_GETHOSTBYNAME_R_5_ARG @@ -250,6 +256,14 @@ shutdown(how) -- shut down traffic in one or both directions\n\ #include <sys/types.h> #endif +#ifdef HAVE_SYS_SOCKET_H +#include <sys/socket.h> +#endif + +#ifdef HAVE_NET_IF_H +#include <net/if.h> +#endif + /* Generic socket object definitions and includes */ #define PySocket_BUILDING_SOCKET #include "socketmodule.h" @@ -2768,6 +2782,7 @@ sock_sendto(PySocketSockObject *s, PyObject *args) PyErr_Format(PyExc_TypeError, "sendto() takes 2 or 3 arguments (%d given)", arglen); + return NULL; } if (PyErr_Occurred()) return NULL; @@ -3130,7 +3145,7 @@ socket_gethostname(PyObject *self, PyObject *unused) } return PyErr_SetExcFromWindowsErr(PyExc_WindowsError, GetLastError()); } - return PyUnicode_FromUnicode(buf, size); + return PyUnicode_FromUnicode(buf, size); #else char buf[1024]; int res; @@ -3149,6 +3164,37 @@ PyDoc_STRVAR(gethostname_doc, \n\ Return the current host name."); +#ifdef HAVE_SETHOSTNAME +PyDoc_STRVAR(sethostname_doc, +"sethostname(name)\n\n\ +Sets the hostname to name."); + +static PyObject * +socket_sethostname(PyObject *self, PyObject *args) +{ + PyObject *hnobj; + Py_buffer buf; + int res, flag = 0; + + if (!PyArg_ParseTuple(args, "S:sethostname", &hnobj)) { + PyErr_Clear(); + if (!PyArg_ParseTuple(args, "O&:sethostname", + PyUnicode_FSConverter, &hnobj)) + return NULL; + flag = 1; + } + res = PyObject_GetBuffer(hnobj, &buf, PyBUF_SIMPLE); + if (!res) { + res = sethostname(buf.buf, buf.len); + PyBuffer_Release(&buf); + } + if (flag) + Py_DECREF(hnobj); + if (res) + return set_error(); + Py_RETURN_NONE; +} +#endif /* Python interface to gethostbyname(name). */ @@ -3421,7 +3467,7 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) goto finally; af = sa->sa_family; ap = NULL; - al = 0; + /* al = 0; */ switch (af) { case AF_INET: ap = (char *)&((struct sockaddr_in *)sa)->sin_addr; @@ -3993,7 +4039,7 @@ socket_inet_ntop(PyObject *self, PyObject *args) static PyObject * socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) { - static char* kwnames[] = {"host", "port", "family", "type", "proto", + static char* kwnames[] = {"host", "port", "family", "type", "proto", "flags", 0}; struct addrinfo hints, *res; struct addrinfo *res0 = NULL; @@ -4008,7 +4054,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) family = socktype = protocol = flags = 0; family = AF_UNSPEC; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|iiii:getaddrinfo", kwnames, &hobj, &pobj, &family, &socktype, &protocol, &flags)) { return NULL; @@ -4235,6 +4281,101 @@ Set the default timeout in seconds (float) for new socket objects.\n\ A value of None indicates that new socket objects have no timeout.\n\ When the socket module is first imported, the default is None."); +#ifdef HAVE_IF_NAMEINDEX +/* Python API for getting interface indices and names */ + +static PyObject * +socket_if_nameindex(PyObject *self, PyObject *arg) +{ + PyObject *list; + int i; + struct if_nameindex *ni; + + ni = if_nameindex(); + if (ni == NULL) { + PyErr_SetFromErrno(socket_error); + return NULL; + } + + list = PyList_New(0); + if (list == NULL) { + if_freenameindex(ni); + return NULL; + } + + for (i = 0; ni[i].if_index != 0 && i < INT_MAX; i++) { + PyObject *ni_tuple = Py_BuildValue("IO&", + ni[i].if_index, PyUnicode_DecodeFSDefault, ni[i].if_name); + + if (ni_tuple == NULL || PyList_Append(list, ni_tuple) == -1) { + Py_XDECREF(ni_tuple); + Py_DECREF(list); + if_freenameindex(ni); + return NULL; + } + Py_DECREF(ni_tuple); + } + + if_freenameindex(ni); + return list; +} + +PyDoc_STRVAR(if_nameindex_doc, +"if_nameindex()\n\ +\n\ +Returns a list of network interface information (index, name) tuples."); + +static PyObject * +socket_if_nametoindex(PyObject *self, PyObject *args) +{ + PyObject *oname; + unsigned long index; + + if (!PyArg_ParseTuple(args, "O&:if_nametoindex", + PyUnicode_FSConverter, &oname)) + return NULL; + + index = if_nametoindex(PyBytes_AS_STRING(oname)); + Py_DECREF(oname); + if (index == 0) { + /* if_nametoindex() doesn't set errno */ + PyErr_SetString(socket_error, "no interface with this name"); + return NULL; + } + + return PyLong_FromUnsignedLong(index); +} + +PyDoc_STRVAR(if_nametoindex_doc, +"if_nametoindex(if_name)\n\ +\n\ +Returns the interface index corresponding to the interface name if_name."); + +static PyObject * +socket_if_indextoname(PyObject *self, PyObject *arg) +{ + unsigned long index; + char name[IF_NAMESIZE + 1]; + + index = PyLong_AsUnsignedLong(arg); + if (index == (unsigned long) -1) + return NULL; + + if (if_indextoname(index, name) == NULL) { + PyErr_SetFromErrno(socket_error); + return NULL; + } + + return PyUnicode_DecodeFSDefault(name); +} + +PyDoc_STRVAR(if_indextoname_doc, +"if_indextoname(if_index)\n\ +\n\ +Returns the interface name corresponding to the interface index if_index."); + +#endif /* HAVE_IF_NAMEINDEX */ + /* List of functions exported by this module. */ @@ -4247,6 +4388,10 @@ static PyMethodDef socket_methods[] = { METH_VARARGS, gethostbyaddr_doc}, {"gethostname", socket_gethostname, METH_NOARGS, gethostname_doc}, +#ifdef HAVE_SETHOSTNAME + {"sethostname", socket_sethostname, + METH_VARARGS, sethostname_doc}, +#endif {"getservbyname", socket_getservbyname, METH_VARARGS, getservbyname_doc}, {"getservbyport", socket_getservbyport, @@ -4287,6 +4432,14 @@ static PyMethodDef socket_methods[] = { METH_NOARGS, getdefaulttimeout_doc}, {"setdefaulttimeout", socket_setdefaulttimeout, METH_O, setdefaulttimeout_doc}, +#ifdef HAVE_IF_NAMEINDEX + {"if_nameindex", socket_if_nameindex, + METH_NOARGS, if_nameindex_doc}, + {"if_nametoindex", socket_if_nametoindex, + METH_VARARGS, if_nametoindex_doc}, + {"if_indextoname", socket_if_indextoname, + METH_O, if_indextoname_doc}, +#endif {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index f064795..db44fd3 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -59,9 +59,12 @@ typedef int socklen_t; #include <bluetooth.h> #endif +#ifdef HAVE_NET_IF_H +# include <net/if.h> +#endif + #ifdef HAVE_NETPACKET_PACKET_H # include <sys/ioctl.h> -# include <net/if.h> # include <netpacket/packet.h> #endif diff --git a/Modules/termios.c b/Modules/termios.c index edeb6f5..b78d33e 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -9,11 +9,6 @@ #endif #include <termios.h> -#ifdef __osf__ -/* On OSF, sys/ioctl.h requires that struct termio already be defined, - * so this needs to be included first on that platform. */ -#include <termio.h> -#endif #include <sys/ioctl.h> /* HP-UX requires that this be included to pick up MDCD, MCTS, MDSR, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 3af041a..4dc82a0 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -3,8 +3,6 @@ #include "Python.h" #include "_time.h" -#define TZNAME_ENCODING "utf-8" - #include <ctype.h> #ifdef HAVE_SYS_TYPES_H @@ -45,12 +43,11 @@ static long main_thread; #endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ -#if defined(MS_WINDOWS) && !defined(__BORLANDC__) -/* Win32 has better clock replacement; we have our own version below. */ -#undef HAVE_CLOCK -#undef TZNAME_ENCODING -#define TZNAME_ENCODING "mbcs" -#endif /* MS_WINDOWS && !defined(__BORLANDC__) */ +#if defined(HAVE_MBCS) +# define TZNAME_ENCODING "mbcs" +#else +# define TZNAME_ENCODING "utf-8" +#endif #if defined(PYOS_OS2) #define INCL_DOS @@ -66,9 +63,6 @@ static long main_thread; static int floatsleep(double); static double floattime(void); -/* For Y2K check */ -static PyObject *moddict; - static PyObject * time_time(PyObject *self, PyObject *unused) { @@ -87,25 +81,9 @@ PyDoc_STRVAR(time_doc, Return the current time in seconds since the Epoch.\n\ Fractions of a second may be present if the system clock provides them."); -#ifdef HAVE_CLOCK - -#ifndef CLOCKS_PER_SEC -#ifdef CLK_TCK -#define CLOCKS_PER_SEC CLK_TCK -#else -#define CLOCKS_PER_SEC 1000000 -#endif -#endif - -static PyObject * -time_clock(PyObject *self, PyObject *unused) -{ - return PyFloat_FromDouble(((double)clock()) / CLOCKS_PER_SEC); -} -#endif /* HAVE_CLOCK */ - #if defined(MS_WINDOWS) && !defined(__BORLANDC__) -/* Due to Mark Hammond and Tim Peters */ +/* Win32 has better clock replacement; we have our own version, due to Mark + Hammond and Tim Peters */ static PyObject * time_clock(PyObject *self, PyObject *unused) { @@ -130,8 +108,23 @@ time_clock(PyObject *self, PyObject *unused) return PyFloat_FromDouble(diff / divisor); } -#define HAVE_CLOCK /* So it gets included in the methods */ -#endif /* MS_WINDOWS && !defined(__BORLANDC__) */ +#elif defined(HAVE_CLOCK) + +#ifndef CLOCKS_PER_SEC +#ifdef CLK_TCK +#define CLOCKS_PER_SEC CLK_TCK +#else +#define CLOCKS_PER_SEC 1000000 +#endif +#endif + +static PyObject * +time_clock(PyObject *self, PyObject *unused) +{ + return PyFloat_FromDouble(((double)clock()) / CLOCKS_PER_SEC); +} +#endif /* HAVE_CLOCK */ + #ifdef HAVE_CLOCK PyDoc_STRVAR(clock_doc, @@ -148,6 +141,11 @@ time_sleep(PyObject *self, PyObject *args) double secs; if (!PyArg_ParseTuple(args, "d:sleep", &secs)) return NULL; + if (secs < 0) { + PyErr_SetString(PyExc_ValueError, + "sleep length must be non-negative"); + return NULL; + } if (floatsleep(secs) != 0) return NULL; Py_INCREF(Py_None); @@ -311,49 +309,6 @@ gettmarg(PyObject *args, struct tm *p) &p->tm_hour, &p->tm_min, &p->tm_sec, &p->tm_wday, &p->tm_yday, &p->tm_isdst)) return 0; - - /* If year is specified with less than 4 digits, its interpretation - * depends on the accept2dyear value. - * - * If accept2dyear is true (default), a backward compatibility behavior is - * invoked as follows: - * - * - for 2-digit year, century is guessed according to POSIX rules for - * %y strptime format: 21st century for y < 69, 20th century - * otherwise. A deprecation warning is issued when century - * information is guessed in this way. - * - * - for 3-digit or negative year, a ValueError exception is raised. - * - * If accept2dyear is false (set by the program or as a result of a - * non-empty value assigned to PYTHONY2K environment variable) all year - * values are interpreted as given. - */ - if (y < 1000) { - PyObject *accept = PyDict_GetItemString(moddict, - "accept2dyear"); - if (accept != NULL) { - int acceptval = PyObject_IsTrue(accept); - if (acceptval == -1) - return 0; - if (acceptval) { - if (0 <= y && y < 69) - y += 2000; - else if (69 <= y && y < 100) - y += 1900; - else { - PyErr_SetString(PyExc_ValueError, - "year out of range"); - return 0; - } - if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Century info guessed for a 2-digit year.", 1) != 0) - return 0; - } - } - else - return 0; - } p->tm_year = y - 1900; p->tm_mon--; p->tm_wday = (p->tm_wday + 1) % 7; @@ -601,31 +556,20 @@ _asctime(struct tm *timeptr) { /* Inspired by Open Group reference implementation available at * http://pubs.opengroup.org/onlinepubs/009695399/functions/asctime.html */ - static char wday_name[7][3] = { + static char wday_name[7][4] = { "Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat" }; - static char mon_name[12][3] = { + static char mon_name[12][4] = { "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec" }; - char buf[20]; /* 'Sun Sep 16 01:03:52\0' */ - int n; - - n = PyOS_snprintf(buf, sizeof(buf), "%.3s %.3s%3d %.2d:%.2d:%.2d", - wday_name[timeptr->tm_wday], - mon_name[timeptr->tm_mon], - timeptr->tm_mday, timeptr->tm_hour, - timeptr->tm_min, timeptr->tm_sec); - /* XXX: since the fields used by snprintf above are validated in checktm, - * the following condition should never trigger. We keep the check because - * historically fixed size buffer used in asctime was the source of - * crashes. */ - if (n + 1 != sizeof(buf)) { - PyErr_SetString(PyExc_ValueError, "unconvertible time"); - return NULL; - } - - return PyUnicode_FromFormat("%s %d", buf, 1900 + timeptr->tm_year); + return PyUnicode_FromFormat( + "%s %s%3d %.2d:%.2d:%.2d %d", + wday_name[timeptr->tm_wday], + mon_name[timeptr->tm_mon], + timeptr->tm_mday, timeptr->tm_hour, + timeptr->tm_min, timeptr->tm_sec, + 1900 + timeptr->tm_year); } static PyObject * @@ -841,7 +785,7 @@ PyInit_timezone(PyObject *m) { static PyMethodDef time_methods[] = { {"time", time_time, METH_NOARGS, time_doc}, -#ifdef HAVE_CLOCK +#if (defined(MS_WINDOWS) && !defined(__BORLANDC__)) || defined(HAVE_CLOCK) {"clock", time_clock, METH_NOARGS, clock_doc}, #endif {"sleep", time_sleep, METH_VARARGS, sleep_doc}, @@ -874,7 +818,7 @@ The actual value can be retrieved by calling gmtime(0).\n\ \n\ The other representation is a tuple of 9 integers giving local time.\n\ The tuple items are:\n\ - year (four digits, e.g. 1998)\n\ + year (including century, e.g. 1998)\n\ month (1-12)\n\ day (1-31)\n\ hours (0-23)\n\ @@ -926,18 +870,10 @@ PyMODINIT_FUNC PyInit_time(void) { PyObject *m; - char *p; m = PyModule_Create(&timemodule); if (m == NULL) return NULL; - /* Accept 2-digit dates unless PYTHONY2K is set and non-empty */ - p = Py_GETENV("PYTHONY2K"); - PyModule_AddIntConstant(m, "accept2dyear", (long) (!p || !*p)); - /* Squirrel away the module's dictionary for the y2k check */ - moddict = PyModule_GetDict(m); - Py_INCREF(moddict); - /* Set, or reset, module variables like time.timezone */ PyInit_timezone(m); @@ -980,23 +916,28 @@ floatsleep(double secs) #if defined(HAVE_SELECT) && !defined(__EMX__) struct timeval t; double frac; + int err; + frac = fmod(secs, 1.0); secs = floor(secs); t.tv_sec = (long)secs; t.tv_usec = (long)(frac*1000000.0); Py_BEGIN_ALLOW_THREADS - if (select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t) != 0) { + err = select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &t); + Py_END_ALLOW_THREADS + if (err != 0) { #ifdef EINTR - if (errno != EINTR) { -#else - if (1) { + if (errno == EINTR) { + if (PyErr_CheckSignals()) + return -1; + } + else #endif - Py_BLOCK_THREADS + { PyErr_SetFromErrno(PyExc_IOError); return -1; } } - Py_END_ALLOW_THREADS #elif defined(__WATCOMC__) && !defined(__QNX__) /* XXX Can't interrupt this sleep */ Py_BEGIN_ALLOW_THREADS diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index ec924f2..661b6e2 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -187,8 +187,7 @@ static PyType_Spec Str_Type_spec = { static PyObject * null_richcompare(PyObject *self, PyObject *other, int op) { - Py_INCREF(Py_NotImplemented); - return Py_NotImplemented; + Py_RETURN_NOTIMPLEMENTED; } static PyType_Slot Null_Type_slots[] = { diff --git a/Modules/zipimport.c b/Modules/zipimport.c index 68c2894..a83bf8b 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -49,7 +49,7 @@ static PyObject *zip_directory_cache = NULL; /* forward decls */ static PyObject *read_directory(PyObject *archive); static PyObject *get_data(PyObject *archive, PyObject *toc_entry); -static PyObject *get_module_code(ZipImporter *self, char *fullname, +static PyObject *get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath); @@ -202,49 +202,50 @@ zipimporter_repr(ZipImporter *self) } /* return fullname.split(".")[-1] */ -static char * -get_subname(char *fullname) +static PyObject * +get_subname(PyObject *fullname) { - char *subname = strrchr(fullname, '.'); - if (subname == NULL) - subname = fullname; - else + Py_ssize_t len; + Py_UNICODE *subname; + subname = Py_UNICODE_strrchr(PyUnicode_AS_UNICODE(fullname), '.'); + if (subname == NULL) { + Py_INCREF(fullname); + return fullname; + } else { subname++; - return subname; + len = PyUnicode_GET_SIZE(fullname); + len -= subname - PyUnicode_AS_UNICODE(fullname); + return PyUnicode_FromUnicode(subname, len); + } } /* Given a (sub)modulename, write the potential file path in the archive (without extension) to the path buffer. Return the - length of the resulting string. */ -static int -make_filename(PyObject *prefix_obj, char *name, char *path, size_t pathsize) + length of the resulting string. + + return self.prefix + name.replace('.', os.sep) */ +static PyObject* +make_filename(PyObject *prefix, PyObject *name) { - size_t len; - char *p; - PyObject *prefix; + PyObject *pathobj; + Py_UNICODE *p; - prefix = PyUnicode_EncodeFSDefault(prefix_obj); - if (prefix == NULL) - return -1; - len = PyBytes_GET_SIZE(prefix); + pathobj = PyUnicode_FromUnicode(NULL, + PyUnicode_GET_SIZE(prefix) + + PyUnicode_GET_SIZE(name)); + if (pathobj == NULL) + return NULL; - /* self.prefix + name [+ SEP + "__init__"] + ".py[co]" */ - if (len + strlen(name) + 13 >= pathsize - 1) { - PyErr_SetString(ZipImportError, "path too long"); - Py_DECREF(prefix); - return -1; - } + p = PyUnicode_AS_UNICODE(pathobj); - strcpy(path, PyBytes_AS_STRING(prefix)); - Py_DECREF(prefix); - strcpy(path + len, name); - for (p = path + len; *p; p++) { + Py_UNICODE_strcpy(p, PyUnicode_AS_UNICODE(prefix)); + p += PyUnicode_GET_SIZE(prefix); + Py_UNICODE_strcpy(p, PyUnicode_AS_UNICODE(name)); + for (; *p; p++) { if (*p == '.') *p = SEP; } - len += strlen(name); - assert(len < INT_MAX); - return (int)len; + return pathobj; } enum zi_module_info { @@ -256,27 +257,38 @@ enum zi_module_info { /* Return some information about a module. */ static enum zi_module_info -get_module_info(ZipImporter *self, char *fullname) +get_module_info(ZipImporter *self, PyObject *fullname) { - char *subname, path[MAXPATHLEN + 1]; - int len; + PyObject *subname; + PyObject *path, *fullpath, *item; struct st_zip_searchorder *zso; subname = get_subname(fullname); + if (subname == NULL) + return MI_ERROR; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return MI_ERROR; for (zso = zip_searchorder; *zso->suffix; zso++) { - strcpy(path + len, zso->suffix); - if (PyDict_GetItemString(self->files, path) != NULL) { + fullpath = PyUnicode_FromFormat("%U%s", path, zso->suffix); + if (fullpath == NULL) { + Py_DECREF(path); + return MI_ERROR; + } + item = PyDict_GetItem(self->files, fullpath); + Py_DECREF(fullpath); + if (item != NULL) { + Py_DECREF(path); if (zso->type & IS_PACKAGE) return MI_PACKAGE; else return MI_MODULE; } } + Py_DECREF(path); return MI_NOT_FOUND; } @@ -287,10 +299,10 @@ zipimporter_find_module(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *path = NULL; - char *fullname; + PyObject *fullname; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s|O:zipimporter.find_module", + if (!PyArg_ParseTuple(args, "U|O:zipimporter.find_module", &fullname, &path)) return NULL; @@ -311,11 +323,11 @@ zipimporter_load_module(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *code = NULL, *mod, *dict; - char *fullname; - PyObject *modpath = NULL, *modpath_bytes; + PyObject *fullname; + PyObject *modpath = NULL; int ispackage; - if (!PyArg_ParseTuple(args, "s:zipimporter.load_module", + if (!PyArg_ParseTuple(args, "U:zipimporter.load_module", &fullname)) return NULL; @@ -323,7 +335,7 @@ zipimporter_load_module(PyObject *obj, PyObject *args) if (code == NULL) goto error; - mod = PyImport_AddModule(fullname); + mod = PyImport_AddModuleObject(fullname); if (mod == NULL) goto error; dict = PyModule_GetDict(mod); @@ -336,17 +348,17 @@ zipimporter_load_module(PyObject *obj, PyObject *args) /* add __path__ to the module *before* the code gets executed */ PyObject *pkgpath, *fullpath; - char *subname = get_subname(fullname); + PyObject *subname = get_subname(fullname); int err; - fullpath = PyUnicode_FromFormat("%U%c%U%s", + fullpath = PyUnicode_FromFormat("%U%c%U%U", self->archive, SEP, self->prefix, subname); + Py_DECREF(subname); if (fullpath == NULL) goto error; - pkgpath = Py_BuildValue("[O]", fullpath); - Py_DECREF(fullpath); + pkgpath = Py_BuildValue("[N]", fullpath); if (pkgpath == NULL) goto error; err = PyDict_SetItemString(dict, "__path__", pkgpath); @@ -354,18 +366,13 @@ zipimporter_load_module(PyObject *obj, PyObject *args) if (err != 0) goto error; } - modpath_bytes = PyUnicode_EncodeFSDefault(modpath); - if (modpath_bytes == NULL) - goto error; - mod = PyImport_ExecCodeModuleEx(fullname, code, - PyBytes_AS_STRING(modpath_bytes)); - Py_DECREF(modpath_bytes); + mod = PyImport_ExecCodeModuleObject(fullname, code, modpath, NULL); Py_CLEAR(code); if (mod == NULL) goto error; if (Py_VerboseFlag) - PySys_FormatStderr("import %s # loaded from Zip %U\n", + PySys_FormatStderr("import %U # loaded from Zip %U\n", fullname, modpath); Py_DECREF(modpath); return mod; @@ -380,12 +387,10 @@ static PyObject * zipimporter_get_filename(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - PyObject *code; - char *fullname; - PyObject *modpath; + PyObject *fullname, *code, *modpath; int ispackage; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_filename", + if (!PyArg_ParseTuple(args, "U:zipimporter.get_filename", &fullname)) return NULL; @@ -404,10 +409,10 @@ static PyObject * zipimporter_is_package(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - char *fullname; + PyObject *fullname; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s:zipimporter.is_package", + if (!PyArg_ParseTuple(args, "U:zipimporter.is_package", &fullname)) return NULL; @@ -415,7 +420,7 @@ zipimporter_is_package(PyObject *obj, PyObject *args) if (mi == MI_ERROR) return NULL; if (mi == MI_NOT_FOUND) { - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); + PyErr_Format(ZipImportError, "can't find module %R", fullname); return NULL; } return PyBool_FromLong(mi == MI_PACKAGE); @@ -477,9 +482,9 @@ static PyObject * zipimporter_get_code(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; - char *fullname; + PyObject *fullname; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_code", &fullname)) + if (!PyArg_ParseTuple(args, "U:zipimporter.get_code", &fullname)) return NULL; return get_module_code(self, fullname, NULL, NULL); @@ -490,34 +495,39 @@ zipimporter_get_source(PyObject *obj, PyObject *args) { ZipImporter *self = (ZipImporter *)obj; PyObject *toc_entry; - char *fullname, *subname, path[MAXPATHLEN+1]; - int len; + PyObject *fullname, *subname, *path, *fullpath; enum zi_module_info mi; - if (!PyArg_ParseTuple(args, "s:zipimporter.get_source", &fullname)) + if (!PyArg_ParseTuple(args, "U:zipimporter.get_source", &fullname)) return NULL; mi = get_module_info(self, fullname); if (mi == MI_ERROR) return NULL; if (mi == MI_NOT_FOUND) { - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); + PyErr_Format(ZipImportError, "can't find module %R", fullname); return NULL; } + subname = get_subname(fullname); + if (subname == NULL) + return NULL; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return NULL; - if (mi == MI_PACKAGE) { - path[len] = SEP; - strcpy(path + len + 1, "__init__.py"); - } + if (mi == MI_PACKAGE) + fullpath = PyUnicode_FromFormat("%U%c__init__.py", path, SEP); else - strcpy(path + len, ".py"); + fullpath = PyUnicode_FromFormat("%U.py", path); + Py_DECREF(path); + if (fullpath == NULL) + return NULL; - toc_entry = PyDict_GetItemString(self->files, path); + toc_entry = PyDict_GetItem(self->files, fullpath); + Py_DECREF(fullpath); if (toc_entry != NULL) { PyObject *res, *bytes; bytes = get_data(self->archive, toc_entry); @@ -708,9 +718,8 @@ get_long(unsigned char *buf) { data_size and file_offset are 0. */ static PyObject * -read_directory(PyObject *archive_obj) +read_directory(PyObject *archive) { - /* FIXME: work on Py_UNICODE* instead of char* */ PyObject *files = NULL; FILE *fp; unsigned short flags; @@ -727,29 +736,29 @@ read_directory(PyObject *archive_obj) const char *charset; int bootstrap; - if (PyUnicode_GET_SIZE(archive_obj) > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(archive) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "Zip path name is too long"); return NULL; } - Py_UNICODE_strcpy(path, PyUnicode_AS_UNICODE(archive_obj)); + Py_UNICODE_strcpy(path, PyUnicode_AS_UNICODE(archive)); - fp = _Py_fopen(archive_obj, "rb"); + fp = _Py_fopen(archive, "rb"); if (fp == NULL) { - PyErr_Format(ZipImportError, "can't open Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "can't open Zip file: %R", archive); return NULL; } fseek(fp, -22, SEEK_END); header_position = ftell(fp); if (fread(endof_central_dir, 1, 22, fp) != 22) { fclose(fp); - PyErr_Format(ZipImportError, "can't read Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "can't read Zip file: %R", archive); return NULL; } if (get_long((unsigned char *)endof_central_dir) != 0x06054B50) { /* Bad: End of Central Dir signature */ fclose(fp); - PyErr_Format(ZipImportError, "not a Zip file: '%U'", archive_obj); + PyErr_Format(ZipImportError, "not a Zip file: %R", archive); return NULL; } @@ -826,7 +835,9 @@ read_directory(PyObject *archive_obj) PY_MAJOR_VERSION, PY_MINOR_VERSION); goto error; } - Py_UNICODE_strncpy(path + length + 1, PyUnicode_AS_UNICODE(nameobj), MAXPATHLEN - length - 1); + Py_UNICODE_strncpy(path + length + 1, + PyUnicode_AS_UNICODE(nameobj), + MAXPATHLEN - length - 1); pathobj = PyUnicode_FromUnicode(path, Py_UNICODE_strlen(path)); if (pathobj == NULL) @@ -844,8 +855,8 @@ read_directory(PyObject *archive_obj) } fclose(fp); if (Py_VerboseFlag) - PySys_FormatStderr("# zipimport: found %ld names in %U\n", - count, archive_obj); + PySys_FormatStderr("# zipimport: found %ld names in %R\n", + count, archive); return files; error: fclose(fp); @@ -998,7 +1009,7 @@ eq_mtime(time_t t1, time_t t2) to .py if available and we don't want to mask other errors). Returns a new reference. */ static PyObject * -unmarshal_code(char *pathname, PyObject *data, time_t mtime) +unmarshal_code(PyObject *pathname, PyObject *data, time_t mtime) { PyObject *code; char *buf = PyBytes_AsString(data); @@ -1012,8 +1023,8 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (get_long((unsigned char *)buf) != PyImport_GetMagicNumber()) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", - pathname); + PySys_FormatStderr("# %R has bad magic\n", + pathname); Py_INCREF(Py_None); return Py_None; /* signal caller to try alternative */ } @@ -1021,8 +1032,8 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (mtime != 0 && !eq_mtime(get_long((unsigned char *)buf + 4), mtime)) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", - pathname); + PySys_FormatStderr("# %R has bad mtime\n", + pathname); Py_INCREF(Py_None); return Py_None; /* signal caller to try alternative */ } @@ -1033,7 +1044,7 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) if (!PyCode_Check(code)) { Py_DECREF(code); PyErr_Format(PyExc_TypeError, - "compiled module %s is not a code object", + "compiled module %R is not a code object", pathname); return NULL; } @@ -1047,11 +1058,12 @@ unmarshal_code(char *pathname, PyObject *data, time_t mtime) static PyObject * normalize_line_endings(PyObject *source) { - char *buf, *q, *p = PyBytes_AsString(source); + char *buf, *q, *p; PyObject *fixed_source; int len = 0; - if (!p) { + p = PyBytes_AsString(source); + if (p == NULL) { return PyBytes_FromStringAndSize("\n\0", 2); } @@ -1084,16 +1096,24 @@ normalize_line_endings(PyObject *source) /* Given a string buffer containing Python source code, compile it return and return a code object as a new reference. */ static PyObject * -compile_source(char *pathname, PyObject *source) +compile_source(PyObject *pathname, PyObject *source) { - PyObject *code, *fixed_source; + PyObject *code, *fixed_source, *pathbytes; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) + return NULL; fixed_source = normalize_line_endings(source); - if (fixed_source == NULL) + if (fixed_source == NULL) { + Py_DECREF(pathbytes); return NULL; + } - code = Py_CompileString(PyBytes_AsString(fixed_source), pathname, + code = Py_CompileString(PyBytes_AsString(fixed_source), + PyBytes_AsString(pathbytes), Py_file_input); + Py_DECREF(pathbytes); Py_DECREF(fixed_source); return code; } @@ -1122,14 +1142,19 @@ parse_dostime(int dostime, int dosdate) modification time of the matching .py file, or 0 if no source is available. */ static time_t -get_mtime_of_source(ZipImporter *self, char *path) +get_mtime_of_source(ZipImporter *self, PyObject *path) { - PyObject *toc_entry; - time_t mtime = 0; - Py_ssize_t lastchar = strlen(path) - 1; - char savechar = path[lastchar]; - path[lastchar] = '\0'; /* strip 'c' or 'o' from *.py[co] */ - toc_entry = PyDict_GetItemString(self->files, path); + PyObject *toc_entry, *stripped; + time_t mtime; + + /* strip 'c' or 'o' from *.py[co] */ + stripped = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(path), + PyUnicode_GET_SIZE(path) - 1); + if (stripped == NULL) + return (time_t)-1; + + toc_entry = PyDict_GetItem(self->files, stripped); + Py_DECREF(stripped); if (toc_entry != NULL && PyTuple_Check(toc_entry) && PyTuple_Size(toc_entry) == 8) { /* fetch the time stamp of the .py file for comparison @@ -1138,8 +1163,8 @@ get_mtime_of_source(ZipImporter *self, char *path) time = PyLong_AsLong(PyTuple_GetItem(toc_entry, 5)); date = PyLong_AsLong(PyTuple_GetItem(toc_entry, 6)); mtime = parse_dostime(time, date); - } - path[lastchar] = savechar; + } else + mtime = 0; return mtime; } @@ -1149,24 +1174,17 @@ static PyObject * get_code_from_data(ZipImporter *self, int ispackage, int isbytecode, time_t mtime, PyObject *toc_entry) { - PyObject *data, *code; - PyObject *modpath; + PyObject *data, *modpath, *code; data = get_data(self->archive, toc_entry); if (data == NULL) return NULL; - modpath = PyUnicode_EncodeFSDefault(PyTuple_GetItem(toc_entry, 0)); - if (modpath == NULL) { - Py_DECREF(data); - return NULL; - } - + modpath = PyTuple_GetItem(toc_entry, 0); if (isbytecode) - code = unmarshal_code(PyBytes_AS_STRING(modpath), data, mtime); + code = unmarshal_code(modpath, data, mtime); else - code = compile_source(PyBytes_AS_STRING(modpath), data); - Py_DECREF(modpath); + code = compile_source(modpath, data); Py_DECREF(data); return code; } @@ -1174,35 +1192,45 @@ get_code_from_data(ZipImporter *self, int ispackage, int isbytecode, /* Get the code object associated with the module specified by 'fullname'. */ static PyObject * -get_module_code(ZipImporter *self, char *fullname, +get_module_code(ZipImporter *self, PyObject *fullname, int *p_ispackage, PyObject **p_modpath) { - PyObject *toc_entry; - char *subname, path[MAXPATHLEN + 1]; - int len; + PyObject *code = NULL, *toc_entry, *subname; + PyObject *path, *fullpath = NULL; struct st_zip_searchorder *zso; subname = get_subname(fullname); + if (subname == NULL) + return NULL; - len = make_filename(self->prefix, subname, path, sizeof(path)); - if (len < 0) + path = make_filename(self->prefix, subname); + Py_DECREF(subname); + if (path == NULL) return NULL; for (zso = zip_searchorder; *zso->suffix; zso++) { - PyObject *code = NULL; + code = NULL; + + fullpath = PyUnicode_FromFormat("%U%s", path, zso->suffix); + if (fullpath == NULL) + goto exit; - strcpy(path + len, zso->suffix); if (Py_VerboseFlag > 1) - PySys_FormatStderr("# trying %U%c%s\n", - self->archive, (int)SEP, path); - toc_entry = PyDict_GetItemString(self->files, path); + PySys_FormatStderr("# trying %U%c%U\n", + self->archive, (int)SEP, fullpath); + toc_entry = PyDict_GetItem(self->files, fullpath); if (toc_entry != NULL) { time_t mtime = 0; int ispackage = zso->type & IS_PACKAGE; int isbytecode = zso->type & IS_BYTECODE; - if (isbytecode) - mtime = get_mtime_of_source(self, path); + if (isbytecode) { + mtime = get_mtime_of_source(self, fullpath); + if (mtime == (time_t)-1 && PyErr_Occurred()) { + goto exit; + } + } + Py_CLEAR(fullpath); if (p_ispackage != NULL) *p_ispackage = ispackage; code = get_code_from_data(self, ispackage, @@ -1218,11 +1246,16 @@ get_module_code(ZipImporter *self, char *fullname, *p_modpath = PyTuple_GetItem(toc_entry, 0); Py_INCREF(*p_modpath); } - return code; + goto exit; } + else + Py_CLEAR(fullpath); } - PyErr_Format(ZipImportError, "can't find module '%s'", fullname); - return NULL; + PyErr_Format(ZipImportError, "can't find module %R", fullname); +exit: + Py_DECREF(path); + Py_XDECREF(fullpath); + return code; } diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index ba0e59c..711004e 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -43,6 +43,7 @@ typedef struct z_stream zst; PyObject *unused_data; PyObject *unconsumed_tail; + char eof; int is_initialised; #ifdef WITH_THREAD PyThread_type_lock lock; @@ -89,6 +90,7 @@ newcompobject(PyTypeObject *type) self = PyObject_New(compobject, type); if (self == NULL) return NULL; + self->eof = 0; self->is_initialised = 0; self->unused_data = PyBytes_FromStringAndSize("", 0); if (self->unused_data == NULL) { @@ -291,7 +293,7 @@ PyZlib_decompress(PyObject *self, PyObject *args) err = inflateEnd(&zst); if (err != Z_OK) { - zlib_error(zst, err, "while finishing data decompression"); + zlib_error(zst, err, "while finishing decompression"); goto error; } @@ -476,7 +478,7 @@ PyZlib_objcompress(compobject *self, PyObject *args) */ if (err != Z_OK && err != Z_BUF_ERROR) { - zlib_error(self->zst, err, "while compressing"); + zlib_error(self->zst, err, "while compressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -611,12 +613,13 @@ PyZlib_objdecompress(compobject *self, PyObject *args) Py_DECREF(RetVal); goto error; } + self->eof = 1; /* We will only get Z_BUF_ERROR if the output buffer was full but there wasn't more output when we tried again, so it is not an error condition. */ } else if (err != Z_OK && err != Z_BUF_ERROR) { - zlib_error(self->zst, err, "while decompressing"); + zlib_error(self->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -697,7 +700,7 @@ PyZlib_flush(compobject *self, PyObject *args) if (err == Z_STREAM_END && flushmode == Z_FINISH) { err = deflateEnd(&(self->zst)); if (err != Z_OK) { - zlib_error(self->zst, err, "from deflateEnd()"); + zlib_error(self->zst, err, "while finishing compression"); Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -765,6 +768,7 @@ PyZlib_copy(compobject *self) Py_XDECREF(retval->unconsumed_tail); retval->unused_data = self->unused_data; retval->unconsumed_tail = self->unconsumed_tail; + retval->eof = self->eof; /* Mark it as being initialized */ retval->is_initialised = 1; @@ -816,6 +820,7 @@ PyZlib_uncopy(compobject *self) Py_XDECREF(retval->unconsumed_tail); retval->unused_data = self->unused_data; retval->unconsumed_tail = self->unconsumed_tail; + retval->eof = self->eof; /* Mark it as being initialized */ retval->is_initialised = 1; @@ -881,14 +886,13 @@ PyZlib_unflush(compobject *self, PyObject *args) Py_END_ALLOW_THREADS } - /* If flushmode is Z_FINISH, we also have to call deflateEnd() to free - various data structures. Note we should only get Z_STREAM_END when - flushmode is Z_FINISH */ + /* If at end of stream, clean up any memory allocated by zlib. */ if (err == Z_STREAM_END) { - err = inflateEnd(&(self->zst)); + self->eof = 1; self->is_initialised = 0; + err = inflateEnd(&(self->zst)); if (err != Z_OK) { - zlib_error(self->zst, err, "from inflateEnd()"); + zlib_error(self->zst, err, "while finishing decompression"); Py_DECREF(retval); retval = NULL; goto error; @@ -936,6 +940,7 @@ static PyMethodDef Decomp_methods[] = static PyMemberDef Decomp_members[] = { {"unused_data", T_OBJECT, COMP_OFF(unused_data), READONLY}, {"unconsumed_tail", T_OBJECT, COMP_OFF(unconsumed_tail), READONLY}, + {"eof", T_BOOL, COMP_OFF(eof), READONLY}, {NULL}, }; |