From 9291332de137141057591386b4ba449ae3a5ed48 Mon Sep 17 00:00:00 2001 From: Andrew MacIntyre Date: Tue, 13 Jun 2006 15:04:24 +0000 Subject: Patch #1454481: Make thread stack size runtime tunable. Heavily revised, comprising revisions: 46640 - original trunk revision (backed out in r46655) 46647 - markup fix (backed out in r46655) 46692:46918 merged from branch aimacintyre-sf1454481 branch tested on buildbots (Windows buildbots had problems not related to these changes). --- Doc/lib/libthread.tex | 20 ++++++++++++ Doc/lib/libthreading.tex | 20 ++++++++++++ Include/pythread.h | 3 ++ Lib/dummy_thread.py | 8 +++++ Lib/test/output/test_thread | 12 +++++++ Lib/test/test_thread.py | 43 +++++++++++++++++++++++++ Lib/test/test_threading.py | 16 ++++++++++ Lib/threading.py | 4 ++- Misc/NEWS | 3 ++ Modules/threadmodule.c | 58 ++++++++++++++++++++++++++++++++++ Python/thread.c | 27 ++++++++++++++++ Python/thread_nt.h | 29 ++++++++++++++++- Python/thread_os2.h | 35 ++++++++++++++++++++- Python/thread_pthread.h | 77 +++++++++++++++++++++++++++++++++++++++++++-- 14 files changed, 349 insertions(+), 6 deletions(-) diff --git a/Doc/lib/libthread.tex b/Doc/lib/libthread.tex index 9573ab3..d007eec 100644 --- a/Doc/lib/libthread.tex +++ b/Doc/lib/libthread.tex @@ -74,6 +74,26 @@ data. Thread identifiers may be recycled when a thread exits and another thread is created. \end{funcdesc} +\begin{funcdesc}{stack_size}{\optional{size}} +Return the thread stack size used when creating new threads. The +optional \var{size} argument specifies the stack size to be used for +subsequently created threads, and must be 0 (use platform or +configured default) or a positive integer value of at least 32,768 (32kB). +If changing the thread stack size is unsupported, a \exception{ThreadError} +is raised. If the specified stack size is invalid, a \exception{ValueError} +is raised and the stack size is unmodified. 32kB is currently the minimum +supported stack size value to guarantee sufficient stack space for the +interpreter itself. Note that some platforms may have particular +restrictions on values for the stack size, such as requiring a minimum +stack size > 32kB or requiring allocation in multiples of the system +memory page size - platform documentation should be referred to for +more information (4kB pages are common; using multiples of 4096 for +the stack size is the suggested approach in the absence of more +specific information). +Availability: Windows, systems with \POSIX{} threads. +\versionadded{2.5} +\end{funcdesc} + Lock objects have the following methods: diff --git a/Doc/lib/libthreading.tex b/Doc/lib/libthreading.tex index 8fb3137..0334750 100644 --- a/Doc/lib/libthreading.tex +++ b/Doc/lib/libthreading.tex @@ -125,6 +125,26 @@ method is called. \versionadded{2.3} \end{funcdesc} +\begin{funcdesc}{stack_size}{\optional{size}} +Return the thread stack size used when creating new threads. The +optional \var{size} argument specifies the stack size to be used for +subsequently created threads, and must be 0 (use platform or +configured default) or a positive integer value of at least 32,768 (32kB). +If changing the thread stack size is unsupported, a \exception{ThreadError} +is raised. If the specified stack size is invalid, a \exception{ValueError} +is raised and the stack size is unmodified. 32kB is currently the minimum +supported stack size value to guarantee sufficient stack space for the +interpreter itself. Note that some platforms may have particular +restrictions on values for the stack size, such as requiring a minimum +stack size > 32kB or requiring allocation in multiples of the system +memory page size - platform documentation should be referred to for +more information (4kB pages are common; using multiples of 4096 for +the stack size is the suggested approach in the absence of more +specific information). +Availability: Windows, systems with \POSIX{} threads. +\versionadded{2.5} +\end{funcdesc} + Detailed interfaces for the objects are documented below. The design of this module is loosely based on Java's threading model. diff --git a/Include/pythread.h b/Include/pythread.h index 0fa8db0..f26db16 100644 --- a/Include/pythread.h +++ b/Include/pythread.h @@ -25,6 +25,9 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int); #define NOWAIT_LOCK 0 PyAPI_FUNC(void) PyThread_release_lock(PyThread_type_lock); +PyAPI_FUNC(size_t) PyThread_get_stacksize(void); +PyAPI_FUNC(int) PyThread_set_stacksize(size_t); + #ifndef NO_EXIT_PROG PyAPI_FUNC(void) PyThread_exit_prog(int); PyAPI_FUNC(void) PyThread__PyThread_exit_prog(int); diff --git a/Lib/dummy_thread.py b/Lib/dummy_thread.py index 21fd03f..7c26f9e 100644 --- a/Lib/dummy_thread.py +++ b/Lib/dummy_thread.py @@ -20,6 +20,7 @@ __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock', 'interrupt_main', 'LockType'] import traceback as _traceback +import warnings class error(Exception): """Dummy implementation of thread.error.""" @@ -75,6 +76,13 @@ def allocate_lock(): """Dummy implementation of thread.allocate_lock().""" return LockType() +def stack_size(size=None): + """Dummy implementation of thread.stack_size().""" + if size is not None: + msg = "setting thread stack size not supported on this platform" + warnings.warn(msg, RuntimeWarning) + return 0 + class LockType(object): """Class implementing dummy implementation of thread.LockType. diff --git a/Lib/test/output/test_thread b/Lib/test/output/test_thread index d49651d..d8174ab 100644 --- a/Lib/test/output/test_thread +++ b/Lib/test/output/test_thread @@ -4,3 +4,15 @@ all tasks done *** Barrier Test *** all tasks done + +*** Changing thread stack size *** +caught expected ValueError setting stack_size(4096) +successfully set stack_size(32768) +successfully set stack_size(1048576) +successfully set stack_size(0) +trying stack_size = 32768 +waiting for all tasks to complete +all tasks done +trying stack_size = 1048576 +waiting for all tasks to complete +all tasks done diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index ea345b6..7b523e2 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -115,3 +115,46 @@ for i in range(numtasks): thread.start_new_thread(task2, (i,)) done.acquire() print 'all tasks done' + +# not all platforms support changing thread stack size +print '\n*** Changing thread stack size ***' +if thread.stack_size() != 0: + raise ValueError, "initial stack_size not 0" + +thread.stack_size(0) +if thread.stack_size() != 0: + raise ValueError, "stack_size not reset to default" + +from os import name as os_name +if os_name in ("nt", "os2", "posix"): + + tss_supported = 1 + try: + thread.stack_size(4096) + except ValueError: + print 'caught expected ValueError setting stack_size(4096)' + except thread.ThreadError: + tss_supported = 0 + print 'platform does not support changing thread stack size' + + if tss_supported: + failed = lambda s, e: s != e + fail_msg = "stack_size(%d) failed - should succeed" + for tss in (32768, 0x100000, 0): + thread.stack_size(tss) + if failed(thread.stack_size(), tss): + raise ValueError, fail_msg % tss + print 'successfully set stack_size(%d)' % tss + + for tss in (32768, 0x100000): + print 'trying stack_size = %d' % tss + next_ident = 0 + for i in range(numtasks): + newtask() + + print 'waiting for all tasks to complete' + done.acquire() + print 'all tasks done' + + # reset stack size to default + thread.stack_size(0) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 7eb9758..02f338a 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -85,6 +85,22 @@ class ThreadTests(unittest.TestCase): print 'all tasks done' self.assertEqual(numrunning.get(), 0) + # run with a minimum thread stack size (32kB) + def test_various_ops_small_stack(self): + if verbose: + print 'with 32kB thread stack size...' + threading.stack_size(0x8000) + self.test_various_ops() + threading.stack_size(0) + + # run with a large thread stack size (1MB) + def test_various_ops_large_stack(self): + if verbose: + print 'with 1MB thread stack size...' + threading.stack_size(0x100000) + self.test_various_ops() + threading.stack_size(0) + def test_foreign_thread(self): # Check that a "foreign" thread can use the threading module. def f(mutex): diff --git a/Lib/threading.py b/Lib/threading.py index c27140d..5655dde 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -15,7 +15,7 @@ from collections import deque # Rename some stuff so "from threading import *" is safe __all__ = ['activeCount', 'Condition', 'currentThread', 'enumerate', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', - 'Timer', 'setprofile', 'settrace', 'local'] + 'Timer', 'setprofile', 'settrace', 'local', 'stack_size'] _start_new_thread = thread.start_new_thread _allocate_lock = thread.allocate_lock @@ -713,6 +713,8 @@ def enumerate(): _active_limbo_lock.release() return active +from thread import stack_size + # Create the main thread object _MainThread() diff --git a/Misc/NEWS b/Misc/NEWS index 04240e8..82838f3 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -112,6 +112,9 @@ Extension Modules - Patch #1435422: zlib's compress and decompress objects now have a copy() method. +- Patch #1454481: thread stack size is now tunable at runtime for thread + enabled builds on Windows and systems with Posix threads support. + - On Win32, os.listdir now supports arbitrarily-long Unicode path names (up to the system limit of 32K characters). diff --git a/Modules/threadmodule.c b/Modules/threadmodule.c index 6169658..fd0bd08 100644 --- a/Modules/threadmodule.c +++ b/Modules/threadmodule.c @@ -586,6 +586,61 @@ allocated consecutive numbers starting at 1, this behavior should not\n\ be relied upon, and the number should be seen purely as a magic cookie.\n\ A thread's identity may be reused for another thread after it exits."); +static PyObject * +thread_stack_size(PyObject *self, PyObject *args) +{ + size_t old_size; + Py_ssize_t new_size = 0; + PyObject *set_size = NULL; + int rc; + + if (!PyArg_ParseTuple(args, "|n:stack_size", &new_size)) + return NULL; + + if (new_size < 0) { + PyErr_SetString(PyExc_ValueError, + "size must be 0 or a positive value"); + return NULL; + } + + old_size = PyThread_get_stacksize(); + + rc = PyThread_set_stacksize((size_t) new_size); + if (rc == -1) { + PyErr_Format(PyExc_ValueError, + "size not valid: %zd bytes", + new_size); + return NULL; + } + if (rc == -2) { + PyErr_SetString(ThreadError, + "setting stack size not supported"); + return NULL; + } + + return PyInt_FromSsize_t((Py_ssize_t) old_size); +} + +PyDoc_STRVAR(stack_size_doc, +"stack_size([size]) -> size\n\ +\n\ +Return the thread stack size used when creating new threads. The\n\ +optional size argument specifies the stack size (in bytes) to be used\n\ +for subsequently created threads, and must be 0 (use platform or\n\ +configured default) or a positive integer value of at least 32,768 (32k).\n\ +If changing the thread stack size is unsupported, a ThreadError\n\ +exception is raised. If the specified size is invalid, a ValueError\n\ +exception is raised, and the stack size is unmodified. 32k bytes\n\ + currently the minimum supported stack size value to guarantee\n\ +sufficient stack space for the interpreter itself.\n\ +\n\ +Note that some platforms may have particular restrictions on values for\n\ +the stack size, such as requiring a minimum stack size larger than 32kB or\n\ +requiring allocation in multiples of the system memory page size\n\ +- platform documentation should be referred to for more information\n\ +(4kB pages are common; using multiples of 4096 for the stack size is\n\ +the suggested approach in the absence of more specific information)."); + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, @@ -605,6 +660,9 @@ static PyMethodDef thread_methods[] = { METH_NOARGS, interrupt_doc}, {"get_ident", (PyCFunction)thread_get_ident, METH_NOARGS, get_ident_doc}, + {"stack_size", (PyCFunction)thread_stack_size, + METH_VARARGS, + stack_size_doc}, #ifndef NO_EXIT_PROG {"exit_prog", (PyCFunction)thread_PyThread_exit_prog, METH_VARARGS}, diff --git a/Python/thread.c b/Python/thread.c index c9356dc..bc501822 100644 --- a/Python/thread.c +++ b/Python/thread.c @@ -95,6 +95,11 @@ PyThread_init_thread(void) PyThread__init_thread(); } +/* Support for runtime thread stack size tuning. + A value of 0 means using the platform's default stack size + or the size specified by the THREAD_STACK_SIZE macro. */ +static size_t _pythread_stacksize = 0; + #ifdef SGI_THREADS #include "thread_sgi.h" #endif @@ -150,6 +155,28 @@ PyThread_init_thread(void) #endif */ +/* return the current thread stack size */ +size_t +PyThread_get_stacksize(void) +{ + return _pythread_stacksize; +} + +/* Only platforms defining a THREAD_SET_STACKSIZE() macro + in thread_.h support changing the stack size. + Return 0 if stack size is valid, + -1 if stack size value is invalid, + -2 if setting stack size is not supported. */ +int +PyThread_set_stacksize(size_t size) +{ +#if defined(THREAD_SET_STACKSIZE) + return THREAD_SET_STACKSIZE(size); +#else + return -2; +#endif +} + #ifndef Py_HAVE_NATIVE_TLS /* If the platform has not supplied a platform specific TLS implementation, provide our own. diff --git a/Python/thread_nt.h b/Python/thread_nt.h index 4dc6d6c..67f5ed5 100644 --- a/Python/thread_nt.h +++ b/Python/thread_nt.h @@ -196,7 +196,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) if (obj.done == NULL) return -1; - rv = _beginthread(bootstrap, 0, &obj); /* use default stack size */ + rv = _beginthread(bootstrap, _pythread_stacksize, &obj); if (rv == (Py_uintptr_t)-1) { /* I've seen errno == EAGAIN here, which means "there are * too many threads". @@ -335,3 +335,30 @@ PyThread_release_lock(PyThread_type_lock aLock) if (!(aLock && LeaveNonRecursiveMutex((PNRMUTEX) aLock))) dprintf(("%ld: Could not PyThread_release_lock(%p) error: %l\n", PyThread_get_thread_ident(), aLock, GetLastError())); } + +/* minimum/maximum thread stack sizes supported */ +#define THREAD_MIN_STACKSIZE 0x8000 /* 32kB */ +#define THREAD_MAX_STACKSIZE 0x10000000 /* 256MB */ + +/* set the thread stack size. + * Return 0 if size is valid, -1 otherwise. + */ +static int +_pythread_nt_set_stacksize(size_t size) +{ + /* set to default */ + if (size == 0) { + _pythread_stacksize = 0; + return 0; + } + + /* valid range? */ + if (size >= THREAD_MIN_STACKSIZE && size < THREAD_MAX_STACKSIZE) { + _pythread_stacksize = size; + return 0; + } + + return -1; +} + +#define THREAD_SET_STACKSIZE(x) _pythread_nt_set_stacksize(x) diff --git a/Python/thread_os2.h b/Python/thread_os2.h index 86e91c1..3ed9d08 100644 --- a/Python/thread_os2.h +++ b/Python/thread_os2.h @@ -14,10 +14,13 @@ long PyThread_get_thread_ident(void); #endif +/* default thread stack size of 64kB */ #if !defined(THREAD_STACK_SIZE) #define THREAD_STACK_SIZE 0x10000 #endif +#define OS2_STACKSIZE(x) (x ? x : THREAD_STACK_SIZE) + /* * Initialization of the C package, should not be needed. */ @@ -35,7 +38,10 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) int aThread; int success = 0; - aThread = _beginthread(func, NULL, THREAD_STACK_SIZE, arg); + aThread = _beginthread(func, + NULL, + OS2_STACKSIZE(_pythread_stacksize), + arg); if (aThread == -1) { success = -1; @@ -275,3 +281,30 @@ PyThread_release_lock(PyThread_type_lock aLock) DosExitCritSec(); #endif } + +/* minimum/maximum thread stack sizes supported */ +#define THREAD_MIN_STACKSIZE 0x8000 /* 32kB */ +#define THREAD_MAX_STACKSIZE 0x2000000 /* 32MB */ + +/* set the thread stack size. + * Return 0 if size is valid, -1 otherwise. + */ +static int +_pythread_os2_set_stacksize(size_t size) +{ + /* set to default */ + if (size == 0) { + _pythread_stacksize = 0; + return 0; + } + + /* valid range? */ + if (size >= THREAD_MIN_STACKSIZE && size < THREAD_MAX_STACKSIZE) { + _pythread_stacksize = size; + return 0; + } + + return -1; +} + +#define THREAD_SET_STACKSIZE(x) _pythread_os2_set_stacksize(x) diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index c29a61c..60d2fb2 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -12,6 +12,20 @@ #endif #include +/* The POSIX spec requires that use of pthread_attr_setstacksize + be conditional on _POSIX_THREAD_ATTR_STACKSIZE being defined. */ +#ifdef _POSIX_THREAD_ATTR_STACKSIZE +#ifndef THREAD_STACK_SIZE +#define THREAD_STACK_SIZE 0 /* use default stack size */ +#endif +/* for safety, ensure a viable minimum stacksize */ +#define THREAD_STACK_MIN 0x8000 /* 32kB */ +#else /* !_POSIX_THREAD_ATTR_STACKSIZE */ +#ifdef THREAD_STACK_SIZE +#error "THREAD_STACK_SIZE defined but _POSIX_THREAD_ATTR_STACKSIZE undefined" +#endif +#endif + /* The POSIX spec says that implementations supporting the sem_* family of functions must indicate this by defining _POSIX_SEMAPHORES. */ @@ -138,15 +152,27 @@ PyThread_start_new_thread(void (*func)(void *), void *arg) #if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED) pthread_attr_t attrs; #endif +#if defined(THREAD_STACK_SIZE) + size_t tss; +#endif + dprintf(("PyThread_start_new_thread called\n")); if (!initialized) PyThread_init_thread(); #if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED) - pthread_attr_init(&attrs); + if (pthread_attr_init(&attrs) != 0) + return -1; #endif -#ifdef THREAD_STACK_SIZE - pthread_attr_setstacksize(&attrs, THREAD_STACK_SIZE); +#if defined(THREAD_STACK_SIZE) + tss = (_pythread_stacksize != 0) ? _pythread_stacksize + : THREAD_STACK_SIZE; + if (tss != 0) { + if (pthread_attr_setstacksize(&attrs, tss) != 0) { + pthread_attr_destroy(&attrs); + return -1; + } + } #endif #if defined(PTHREAD_SYSTEM_SCHED_SUPPORTED) pthread_attr_setscope(&attrs, PTHREAD_SCOPE_SYSTEM); @@ -460,3 +486,48 @@ PyThread_release_lock(PyThread_type_lock lock) } #endif /* USE_SEMAPHORES */ + +/* set the thread stack size. + * Return 0 if size is valid, -1 if size is invalid, + * -2 if setting stack size is not supported. + */ +static int +_pythread_pthread_set_stacksize(size_t size) +{ +#if defined(THREAD_STACK_SIZE) + pthread_attr_t attrs; + size_t tss_min; + int rc = 0; +#endif + + /* set to default */ + if (size == 0) { + _pythread_stacksize = 0; + return 0; + } + +#if defined(THREAD_STACK_SIZE) +#if defined(PTHREAD_STACK_MIN) + tss_min = PTHREAD_STACK_MIN > THREAD_STACK_MIN ? PTHREAD_STACK_MIN + : THREAD_STACK_MIN; +#else + tss_min = THREAD_STACK_MIN; +#endif + if (size >= tss_min) { + /* validate stack size by setting thread attribute */ + if (pthread_attr_init(&attrs) == 0) { + rc = pthread_attr_setstacksize(&attrs, size); + pthread_attr_destroy(&attrs); + if (rc == 0) { + _pythread_stacksize = size; + return 0; + } + } + } + return -1; +#else + return -2; +#endif +} + +#define THREAD_SET_STACKSIZE(x) _pythread_pthread_set_stacksize(x) -- cgit v0.12