From 4f199eaa6ef767737e3b84bc71181a4450fd0e58 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 9 Apr 1998 20:56:35 +0000 Subject: Threading speedup patches by Christopher Lindblad . --- Modules/bsddbmodule.c | 170 ++++++++++++++++++++++++++++++++++++++++++------- Modules/socketmodule.c | 46 ++++++++++++- 2 files changed, 192 insertions(+), 24 deletions(-) diff --git a/Modules/bsddbmodule.c b/Modules/bsddbmodule.c index ee979b8..b3474c4 100644 --- a/Modules/bsddbmodule.c +++ b/Modules/bsddbmodule.c @@ -41,6 +41,9 @@ PERFORMANCE OF THIS SOFTWARE. */ #include "Python.h" +#ifdef WITH_THREAD +#include "thread.h" +#endif #include #include @@ -53,6 +56,9 @@ typedef struct { PyObject_HEAD DB *di_bsddb; int di_size; /* -1 means recompute */ +#ifdef WITH_THREAD + type_lock di_lock; +#endif } bsddbobject; staticforward PyTypeObject Bsddbtype; @@ -93,14 +99,27 @@ newdbhashobject(file, flags, mode, #ifdef O_BINARY flags |= O_BINARY; #endif - if ((dp->di_bsddb = dbopen(file, flags, - mode, DB_HASH, &info)) == NULL) { + Py_BEGIN_ALLOW_THREADS + dp->di_bsddb = dbopen(file, flags, mode, DB_HASH, &info); + Py_END_ALLOW_THREADS + if (dp->di_bsddb == NULL) { PyErr_SetFromErrno(BsddbError); +#ifdef WITH_THREAD + dp->di_lock = NULL; +#endif Py_DECREF(dp); return NULL; } dp->di_size = -1; +#ifdef WITH_THREAD + dp->di_lock = allocate_lock(); + if (dp->di_lock == NULL) { + PyErr_SetString(BsddbError, "can't allocate lock"); + Py_DECREF(dp); + return NULL; + } +#endif return (PyObject *)dp; } @@ -136,14 +155,27 @@ newdbbtobject(file, flags, mode, #ifdef O_BINARY flags |= O_BINARY; #endif - if ((dp->di_bsddb = dbopen(file, flags, - mode, DB_BTREE, &info)) == NULL) { + Py_BEGIN_ALLOW_THREADS + dp->di_bsddb = dbopen(file, flags, mode, DB_BTREE, &info); + Py_END_ALLOW_THREADS + if (dp->di_bsddb == NULL) { PyErr_SetFromErrno(BsddbError); +#ifdef WITH_THREAD + dp->di_lock = NULL; +#endif Py_DECREF(dp); return NULL; } dp->di_size = -1; +#ifdef WITH_THREAD + dp->di_lock = allocate_lock(); + if (dp->di_lock == NULL) { + PyErr_SetString(BsddbError, "can't allocate lock"); + Py_DECREF(dp); + return NULL; + } +#endif return (PyObject *)dp; } @@ -179,25 +211,49 @@ newdbrnobject(file, flags, mode, #ifdef O_BINARY flags |= O_BINARY; #endif - if ((dp->di_bsddb = dbopen(file, flags, mode, - DB_RECNO, &info)) == NULL) { + Py_BEGIN_ALLOW_THREADS + dp->di_bsddb = dbopen(file, flags, mode, DB_RECNO, &info); + Py_END_ALLOW_THREADS + if (dp->di_bsddb == NULL) { PyErr_SetFromErrno(BsddbError); +#ifdef WITH_THREAD + dp->di_lock = NULL; +#endif Py_DECREF(dp); return NULL; } dp->di_size = -1; +#ifdef WITH_THREAD + dp->di_lock = allocate_lock(); + if (dp->di_lock == NULL) { + PyErr_SetString(BsddbError, "can't allocate lock"); + Py_DECREF(dp); + return NULL; + } +#endif return (PyObject *)dp; } - static void bsddb_dealloc(dp) bsddbobject *dp; { +#ifdef WITH_THREAD + if (dp->di_lock) { + acquire_lock(dp->di_lock, 0); + release_lock(dp->di_lock); + free_lock(dp->di_lock); + dp->di_lock = NULL; + } +#endif if (dp->di_bsddb != NULL) { - if ((dp->di_bsddb->close)(dp->di_bsddb) != 0) + int status; + Py_BEGIN_ALLOW_THREADS + status = (dp->di_bsddb->close)(dp->di_bsddb); + Py_END_ALLOW_THREADS + if (status != 0) fprintf(stderr, "Python bsddb: close errno %d in dealloc\n", errno); @@ -205,6 +261,14 @@ bsddb_dealloc(dp) PyMem_DEL(dp); } +#ifdef WITH_THREAD +#define BSDDB_BGN_SAVE(_dp) Py_BEGIN_ALLOW_THREADS acquire_lock(_dp->di_lock,1); +#define BSDDB_END_SAVE(_dp) release_lock(_dp->di_lock); Py_END_ALLOW_THREADS +#else +#define BSDDB_BGN_SAVE(_dp) Py_BEGIN_ALLOW_THREADS +#define BSDDB_END_SAVE(_dp) Py_END_ALLOW_THREADS +#endif + static int bsddb_length(dp) bsddbobject *dp; @@ -217,12 +281,14 @@ bsddb_length(dp) DBT krec, drec; int status; int size = 0; + BSDDB_BGN_SAVE(dp) for (status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec,R_FIRST); status == 0; status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_NEXT)) size++; + BSDDB_END_SAVE(dp) if (status < 0) { PyErr_SetFromErrno(BsddbError); return -1; @@ -239,8 +305,9 @@ bsddb_subscript(dp, key) { int status; DBT krec, drec; - char *data; + char *data,buf[4096]; int size; + PyObject *result; if (!PyArg_Parse(key, "s#", &data, &size)) return NULL; @@ -249,7 +316,14 @@ bsddb_subscript(dp, key) krec.data = data; krec.size = size; + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->get)(dp->di_bsddb, &krec, &drec, 0); + if (status == 0) { + if (drec.size > sizeof(buf)) data = malloc(drec.size); + else data = buf; + memcpy(data,drec.data,drec.size); + } + BSDDB_END_SAVE(dp) if (status != 0) { if (status < 0) PyErr_SetFromErrno(BsddbError); @@ -258,7 +332,9 @@ bsddb_subscript(dp, key) return NULL; } - return PyString_FromStringAndSize((char *)drec.data, (int)drec.size); + result = PyString_FromStringAndSize(data, (int)drec.size); + if (data != buf) free(data); + return result; } static int @@ -284,7 +360,9 @@ bsddb_ass_sub(dp, key, value) krec.size = size; dp->di_size = -1; if (value == NULL) { + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->del)(dp->di_bsddb, &krec, 0); + BSDDB_END_SAVE(dp) } else { if (!PyArg_Parse(value, "s#", &data, &size)) { @@ -305,7 +383,9 @@ bsddb_ass_sub(dp, key, value) printf("before put key= '%s', size= %d\n", krec.data, krec.size); #endif + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->put)(dp->di_bsddb, &krec, &drec, 0); + BSDDB_END_SAVE(dp) } if (status != 0) { if (status < 0) @@ -331,7 +411,11 @@ bsddb_close(dp, args) if (!PyArg_NoArgs(args)) return NULL; if (dp->di_bsddb != NULL) { - if ((dp->di_bsddb->close)(dp->di_bsddb) != 0) { + int status; + BSDDB_BGN_SAVE(dp) + status = (dp->di_bsddb->close)(dp->di_bsddb); + BSDDB_END_SAVE(dp) + if (status != 0) { dp->di_bsddb = NULL; PyErr_SetFromErrno(BsddbError); return NULL; @@ -349,6 +433,7 @@ bsddb_keys(dp, args) { PyObject *list, *item; DBT krec, drec; + char *data,buf[4096]; int status; int err; @@ -358,12 +443,17 @@ bsddb_keys(dp, args) list = PyList_New(0); if (list == NULL) return NULL; - for (status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_FIRST); - status == 0; - status = (dp->di_bsddb->seq)(dp->di_bsddb, - &krec, &drec, R_NEXT)) { - item = PyString_FromStringAndSize((char *)krec.data, - (int)krec.size); + BSDDB_BGN_SAVE(dp) + status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_FIRST); + if (status == 0) { + if (krec.size > sizeof(buf)) data = malloc(krec.size); + else data = buf; + memcpy(data,krec.data,krec.size); + } + BSDDB_END_SAVE(dp) + while (status == 0) { + item = PyString_FromStringAndSize(data, (int)krec.size); + if (data != buf) free(data); if (item == NULL) { Py_DECREF(list); return NULL; @@ -374,6 +464,14 @@ bsddb_keys(dp, args) Py_DECREF(list); return NULL; } + BSDDB_BGN_SAVE(dp) + status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_NEXT); + if (status == 0) { + if (krec.size > sizeof(buf)) data = malloc(krec.size); + else data = buf; + memcpy(data,krec.data,krec.size); + } + BSDDB_END_SAVE(dp) } if (status < 0) { PyErr_SetFromErrno(BsddbError); @@ -401,7 +499,9 @@ bsddb_has_key(dp, args) krec.data = data; krec.size = size; + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->get)(dp->di_bsddb, &krec, &drec, 0); + BSDDB_END_SAVE(dp) if (status < 0) { PyErr_SetFromErrno(BsddbError); return NULL; @@ -417,8 +517,9 @@ bsddb_set_location(dp, key) { int status; DBT krec, drec; - char *data; + char *data,buf[4096]; int size; + PyObject *result; if (!PyArg_Parse(key, "s#", &data, &size)) return NULL; @@ -426,7 +527,14 @@ bsddb_set_location(dp, key) krec.data = data; krec.size = size; + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, R_CURSOR); + if (status == 0) { + if (drec.size > sizeof(buf)) data = malloc(drec.size); + else data = buf; + memcpy(data,drec.data,drec.size); + } + BSDDB_END_SAVE(dp) if (status != 0) { if (status < 0) PyErr_SetFromErrno(BsddbError); @@ -435,8 +543,9 @@ bsddb_set_location(dp, key) return NULL; } - return Py_BuildValue("s#s#", krec.data, krec.size, - drec.data, drec.size); + result = Py_BuildValue("s#s#", krec.data, krec.size, data, drec.size); + if (data != buf) free(data); + return result; } static PyObject * @@ -447,6 +556,9 @@ bsddb_seq(dp, args, sequence_request) { int status; DBT krec, drec; + char *kdata,kbuf[4096]; + char *ddata,dbuf[4096]; + PyObject *result; if (!PyArg_NoArgs(args)) return NULL; @@ -455,8 +567,18 @@ bsddb_seq(dp, args, sequence_request) krec.data = 0; krec.size = 0; + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->seq)(dp->di_bsddb, &krec, &drec, sequence_request); + if (status == 0) { + if (krec.size > sizeof(kbuf)) kdata = malloc(krec.size); + else kdata = kbuf; + memcpy(kdata,krec.data,krec.size); + if (drec.size > sizeof(dbuf)) ddata = malloc(drec.size); + else ddata = dbuf; + memcpy(ddata,drec.data,drec.size); + } + BSDDB_END_SAVE(dp) if (status != 0) { if (status < 0) PyErr_SetFromErrno(BsddbError); @@ -465,8 +587,10 @@ bsddb_seq(dp, args, sequence_request) return NULL; } - return Py_BuildValue("s#s#", krec.data, krec.size, - drec.data, drec.size); + result = Py_BuildValue("s#s#", kdata, krec.size, ddata, drec.size); + if (kdata != kbuf) free(kdata); + if (ddata != dbuf) free(ddata); + return result; } static PyObject * @@ -507,7 +631,9 @@ bsddb_sync(dp, args) if (!PyArg_NoArgs(args)) return NULL; check_bsddbobject_open(dp); + BSDDB_BGN_SAVE(dp) status = (dp->di_bsddb->sync)(dp->di_bsddb, 0); + BSDDB_END_SAVE(dp) if (status != 0) { PyErr_SetFromErrno(BsddbError); return NULL; diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 98ca52b..92001df 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -86,6 +86,9 @@ Socket methods: */ #include "Python.h" +#if defined(WITH_THREAD) && !defined(HAVE_GETHOSTBYNAME_R) && !defined(MS_WINDOWS) +#include "thread.h" +#endif #ifdef HAVE_UNISTD_H #include @@ -301,6 +304,13 @@ BUILD_FUNC_DEF_4(PySocketSock_New,int,fd, int,family, int,type, int,proto) } +/* Lock to allow python interpreter to continue, but only allow one + thread to be in gethostbyname */ +#if defined(WITH_THREAD) && !defined(HAVE_GETHOSTBYNAME_R) && !defined(MS_WINDOWS) +type_lock gethostbyname_lock; +#endif + + /* Convert a string specifying a host name or one of a few symbolic names to a numeric IP address. This usually calls gethostbyname() to do the work; the names "" and "" are special. @@ -337,13 +347,19 @@ BUILD_FUNC_DEF_2(setipaddr, char*,name, struct sockaddr_in *,addr_ret) ((long) d3 << 8) | ((long) d4 << 0)); return 4; } -#ifdef HAVE_GETHOSTBYNAME_R Py_BEGIN_ALLOW_THREADS +#ifdef HAVE_GETHOSTBYNAME_R hp = gethostbyname_r(name, &hp_allocated, buf, buf_len, &errnop); - Py_END_ALLOW_THREADS #else /* not HAVE_GETHOSTBYNAME_R */ +#if defined(WITH_THREAD) && !defined(MS_WINDOWS) + acquire_lock(gethostbyname_lock,1); +#endif hp = gethostbyname(name); +#if defined(WITH_THREAD) && !defined(MS_WINDOWS) + release_lock(gethostbyname_lock); +#endif #endif /* HAVE_GETHOSTBYNAME_R */ + Py_END_ALLOW_THREADS if (hp == NULL) { #ifdef HAVE_HSTRERROR @@ -1162,14 +1178,35 @@ BUILD_FUNC_DEF_2(PySocket_gethostbyaddr,PyObject *,self, PyObject *, args) PyObject *name_list = (PyObject *)NULL; PyObject *addr_list = (PyObject *)NULL; PyObject *tmp; +#ifdef HAVE_GETHOSTBYNAME_R + struct hostent hp_allocated; + char buf[16384]; + int buf_len = (sizeof buf) - 1; + int errnop; +#endif /* HAVE_GETHOSTBYNAME_R */ if (!PyArg_Parse(args, "s", &ip_num)) return NULL; if (setipaddr(ip_num, &addr) < 0) return NULL; + Py_BEGIN_ALLOW_THREADS +#ifdef HAVE_GETHOSTBYNAME_R + h = gethostbyaddr_r((char *)&addr.sin_addr, + sizeof(addr.sin_addr), + AF_INET, + &hp_allocated, buf, buf_len, &errnop); +#else /* not HAVE_GETHOSTBYNAME_R */ +#if defined(WITH_THREAD) && !defined(MS_WINDOWS) + acquire_lock(gethostbyname_lock,1); +#endif h = gethostbyaddr((char *)&addr.sin_addr, sizeof(addr.sin_addr), AF_INET); +#if defined(WITH_THREAD) && !defined(MS_WINDOWS) + release_lock(gethostbyname_lock); +#endif +#endif /* HAVE_GETHOSTBYNAME_R */ + Py_END_ALLOW_THREADS if (h == NULL) { #ifdef HAVE_HSTRERROR /* Let's get real error message to return */ @@ -1776,4 +1813,9 @@ initsocket() #ifdef IP_DROP_MEMBERSHIP insint(d, "IP_DROP_MEMBERSHIP", IP_DROP_MEMBERSHIP); #endif + + /* Initialize gethostbyname lock */ +#if defined(WITH_THREAD) && !defined(HAVE_GETHOSTBYNAME_R) && !defined(MS_WINDOWS) + gethostbyname_lock = allocate_lock(); +#endif } -- cgit v0.12