Patch #1454481: Make thread stack size runtime tunable.

14 files changed, 332 insertions, 5 deletions

diff --git a/Doc/lib/libthread.tex b/Doc/lib/libthread.tex
index 9573ab3..4a2d872 100644
--- a/Doc/lib/libthread.tex
+++ b/Doc/lib/libthread.tex
@@ -74,6 +74,25 @@ data.  Thread identifiers may be recycled when a thread exits and
 another thread is created.
 \end{funcdesc}
 
+\begin{funcdesc}{stack_size}{\optional{size}}
+Return the thread stack size used when creating new threads.  The
+optional \var{size} argument specifies the stack size to be used for
+subsequently created threads, and must be 0 (use platform or
+configured default) or a positive integer value of at least 32,768 (32kB).
+If changing the thread stack size is unsupported, or the specified size
+is invalid, a RuntimeWarning is issued and the stack size is unmodified.
+32kB is currently the minimum supported stack size value, to guarantee
+sufficient stack space for the interpreter itself.
+Note that some platforms may have particular restrictions on values for
+the stack size, such as requiring allocation in multiples of the system
+memory page size - platform documentation should be referred to for more
+information (4kB pages are common; using multiples of 4096 for the
+stack size is the suggested approach in the absence of more specific
+information).
+Availability: Windows, systems with \POSIX{} threads.
+\versionadded{2.5}
+\end{funcdesc}
+
 
 Lock objects have the following methods:
 
diff --git a/Doc/lib/libthreading.tex b/Doc/lib/libthreading.tex
index 8fb3137..86330a7 100644
--- a/Doc/lib/libthreading.tex
+++ b/Doc/lib/libthreading.tex
@@ -125,6 +125,25 @@ method is called.
 \versionadded{2.3}
 \end{funcdesc}
 
+\begin{funcdesc}{stack_size}{\optional{size}}
+Return the thread stack size used when creating new threads.  The
+optional \var{size} argument specifies the stack size to be used for
+subsequently created threads, and must be 0 (use platform or
+configured default) or a positive integer value of at least 32,768 (32kB).
+If changing the thread stack size is unsupported, or the specified size
+is invalid, a RuntimeWarning is issued and the stack size is unmodified.
+32kB is currently the minimum supported stack size value, to guarantee
+sufficient stack space for the interpreter itself.
+Note that some platforms may have particular restrictions on values for
+the stack size, such as requiring allocation in multiples of the system
+memory page size - platform documentation should be referred to for more
+information (4kB pages are common; using multiples of 4096 for the
+stack size is the suggested approach in the absence of more specific
+information).
+Availability: Windows, systems with \POSIX{} threads.
+\versionadded{2.5}
+\end{funcdesc}
+
 Detailed interfaces for the objects are documented below.  
 
 The design of this module is loosely based on Java's threading model.
diff --git a/Include/pythread.h b/Include/pythread.h
index 0fa8db0..f26db16 100644
--- a/Include/pythread.h
+++ b/Include/pythread.h
@@ -25,6 +25,9 @@ PyAPI_FUNC(int) PyThread_acquire_lock(PyThread_type_lock, int);
 #define NOWAIT_LOCK	0
 PyAPI_FUNC(void) PyThread_release_lock(PyThread_type_lock);
 
+PyAPI_FUNC(size_t) PyThread_get_stacksize(void);
+PyAPI_FUNC(int) PyThread_set_stacksize(size_t);
+
 #ifndef NO_EXIT_PROG
 PyAPI_FUNC(void) PyThread_exit_prog(int);
 PyAPI_FUNC(void) PyThread__PyThread_exit_prog(int);
diff --git a/Lib/dummy_thread.py b/Lib/dummy_thread.py
index 21fd03f..7c26f9e 100644
--- a/Lib/dummy_thread.py
+++ b/Lib/dummy_thread.py
@@ -20,6 +20,7 @@ __all__ = ['error', 'start_new_thread', 'exit', 'get_ident', 'allocate_lock',
            'interrupt_main', 'LockType']
 
 import traceback as _traceback
+import warnings
 
 class error(Exception):
     """Dummy implementation of thread.error."""
@@ -75,6 +76,13 @@ def allocate_lock():
     """Dummy implementation of thread.allocate_lock()."""
     return LockType()
 
+def stack_size(size=None):
+    """Dummy implementation of thread.stack_size()."""
+    if size is not None:
+        msg = "setting thread stack size not supported on this platform"
+        warnings.warn(msg, RuntimeWarning)
+    return 0
+
 class LockType(object):
     """Class implementing dummy implementation of thread.LockType.
 
diff --git a/Lib/test/output/test_thread b/Lib/test/output/test_thread
index d49651d..ec58d73 100644
--- a/Lib/test/output/test_thread
+++ b/Lib/test/output/test_thread
@@ -4,3 +4,11 @@ all tasks done
 
 *** Barrier Test ***
 all tasks done
+
+*** Changing thread stack size ***
+trying stack_size = 32768
+waiting for all tasks to complete
+all tasks done
+trying stack_size = 4194304
+waiting for all tasks to complete
+all tasks done
diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py
index ea345b6..d970107 100644
--- a/Lib/test/test_thread.py
+++ b/Lib/test/test_thread.py
@@ -115,3 +115,38 @@ for i in range(numtasks):
     thread.start_new_thread(task2, (i,))
 done.acquire()
 print 'all tasks done'
+
+# not all platforms support changing thread stack size
+print '\n*** Changing thread stack size ***'
+if thread.stack_size() != 0:
+    raise ValueError, "initial stack_size not 0"
+
+thread.stack_size(0) 
+if thread.stack_size() != 0:
+    raise ValueError, "stack_size not reset to default"
+
+from os import name as os_name
+if os_name in ("nt", "os2", "posix"):
+
+    for tss, ok in ((4096, 0), (32768, 1), (0x400000, 1), (0, 1)):
+        if ok:
+            failed = lambda s, e: s != e
+            fail_msg = "stack_size(%d) failed - should succeed"
+        else:
+            failed = lambda s, e: s == e
+            fail_msg = "stack_size(%d) succeeded - should fail"
+        thread.stack_size(tss)
+        if failed(thread.stack_size(), tss):
+            raise ValueError, fail_msg % tss
+    
+    for tss in (32768, 0x400000):
+        print 'trying stack_size = %d' % tss
+        next_ident = 0
+        for i in range(numtasks):
+            newtask()
+
+        print 'waiting for all tasks to complete'
+        done.acquire()
+        print 'all tasks done'
+
+    thread.stack_size(0)
diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py
index 7eb9758..09e84f4 100644
--- a/Lib/test/test_threading.py
+++ b/Lib/test/test_threading.py
@@ -85,6 +85,22 @@ class ThreadTests(unittest.TestCase):
             print 'all tasks done'
         self.assertEqual(numrunning.get(), 0)
 
+    # run with a minimum thread stack size (32kB)
+    def test_various_ops_small_stack(self):
+        if verbose:
+            print 'with 32kB thread stack size...'
+        threading.stack_size(0x8000)
+        self.test_various_ops()
+        threading.stack_size(0)
+
+    # run with a large thread stack size (16MB)
+    def test_various_ops_large_stack(self):
+        if verbose:
+            print 'with 16MB thread stack size...'
+        threading.stack_size(0x1000000)
+        self.test_various_ops()
+        threading.stack_size(0)
+
     def test_foreign_thread(self):
         # Check that a "foreign" thread can use the threading module.
         def f(mutex):
diff --git a/Lib/threading.py b/Lib/threading.py
index c27140d..5655dde 100644
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -15,7 +15,7 @@ from collections import deque
 # Rename some stuff so "from threading import *" is safe
 __all__ = ['activeCount', 'Condition', 'currentThread', 'enumerate', 'Event',
            'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread',
-           'Timer', 'setprofile', 'settrace', 'local']
+           'Timer', 'setprofile', 'settrace', 'local', 'stack_size']
 
 _start_new_thread = thread.start_new_thread
 _allocate_lock = thread.allocate_lock
@@ -713,6 +713,8 @@ def enumerate():
     _active_limbo_lock.release()
     return active
 
+from thread import stack_size
+
 # Create the main thread object
 
 _MainThread()
diff --git a/Misc/NEWS b/Misc/NEWS
index 088e245..176fa3a 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -84,6 +84,9 @@ Extension Modules
 - Patch #1435422: zlib's compress and decompress objects now have a
   copy() method.
 
+- Patch #1454481: thread stack size is now tunable at runtime for thread
+  enabled builds on Windows and systems with Posix threads support.
+
 - On Win32, os.listdir now supports arbitrarily-long Unicode path names
   (up to the system limit of 32K characters).
 
diff --git a/Modules/threadmodule.c b/Modules/threadmodule.c
index 6169658..c9227c6 100644
--- a/Modules/threadmodule.c
+++ b/Modules/threadmodule.c
@@ -586,6 +586,51 @@ allocated consecutive numbers starting at 1, this behavior should not\n\
 be relied upon, and the number should be seen purely as a magic cookie.\n\
 A thread's identity may be reused for another thread after it exits.");
 
+static PyObject *
+thread_stack_size(PyObject *self, PyObject *args)
+{
+	size_t old_size, new_size;
+	PyObject *set_size = NULL;
+
+	if (!PyArg_UnpackTuple(args, "stack_size", 0, 1, &set_size))
+		return NULL;
+
+	old_size = PyThread_get_stacksize();
+
+	if (set_size != NULL) {
+		if (PyInt_Check(set_size))
+			new_size = (size_t) PyInt_AsLong(set_size);
+		else {
+			PyErr_SetString(PyExc_TypeError,
+					"size must be an integer");
+			return NULL;
+		}
+		if (PyThread_set_stacksize(new_size))
+			return NULL;
+	}
+
+	return PyInt_FromLong((long) old_size);
+}
+
+PyDoc_STRVAR(stack_size_doc,
+"stack_size([size]) -> size\n\
+\n\
+Return the thread stack size used when creating new threads.  The\n\
+optional size argument specifies the stack size (in bytes) to be used\n\
+for subsequently created threads, and must be 0 (use platform or\n\
+configured default) or a positive integer value of at least 32,768 (32kB).\n\
+If changing the thread stack size is unsupported, or the specified size\n\
+is invalid, a RuntimeWarning is issued and the stack size is unmodified.\n\
+32kB is currently the minimum supported stack size value, to guarantee\n\
+sufficient stack space for the interpreter itself.\n\
+\n\
+Note that some platforms may have particular restrictions on values for\n\
+the stack size, such as requiring allocation in multiples of the system\n\
+memory page size - platform documentation should be referred to for more\n\
+information (4kB pages are common; using multiples of 4096 for the\n\
+stack size is the suggested approach in the absence of more specific\n\
+information).");
+
 static PyMethodDef thread_methods[] = {
 	{"start_new_thread",	(PyCFunction)thread_PyThread_start_new_thread,
 	                        METH_VARARGS,
@@ -605,6 +650,9 @@ static PyMethodDef thread_methods[] = {
 	 METH_NOARGS, interrupt_doc},
 	{"get_ident",		(PyCFunction)thread_get_ident, 
 	 METH_NOARGS, get_ident_doc},
+	{"stack_size",		(PyCFunction)thread_stack_size,
+				METH_VARARGS,
+				stack_size_doc},
 #ifndef NO_EXIT_PROG
 	{"exit_prog",		(PyCFunction)thread_PyThread_exit_prog,
 	 METH_VARARGS},
diff --git a/Python/thread.c b/Python/thread.c
index 5e7fc6c..dd9c3ad 100644
--- a/Python/thread.c
+++ b/Python/thread.c
@@ -94,6 +94,31 @@ void PyThread_init_thread(void)
 	PyThread__init_thread();
 }
 
+/* Support for runtime thread stack size tuning.
+   A value of 0 means using the platform's default stack size
+   or the size specified by the THREAD_STACK_SIZE macro. */
+static size_t _pythread_stacksize = 0;
+
+size_t
+PyThread_get_stacksize(void)
+{
+	return _pythread_stacksize;
+}
+
+static int
+_pythread_unsupported_set_stacksize(size_t size)
+{
+	return PyErr_Warn(PyExc_RuntimeWarning,
+			  "setting thread stack size not supported on "
+                          "this platform");
+}
+
+/* Only platforms with THREAD_SET_STACKSIZE() defined in
+   pthread_<platform>.h, overriding this default definition,
+   will support changing the stack size.
+   Return 1 if an exception is pending, 0 otherwise. */
+#define THREAD_SET_STACKSIZE(x)	_pythread_unsupported_set_stacksize(x)
+
 #ifdef SGI_THREADS
 #include "thread_sgi.h"
 #endif
@@ -149,6 +174,14 @@ void PyThread_init_thread(void)
 #endif
 */
 
+/* use appropriate thread stack size setting routine.
+   Return 1 if an exception is pending, 0 otherwise. */
+int
+PyThread_set_stacksize(size_t size)
+{
+	return THREAD_SET_STACKSIZE(size);
+}
+
 #ifndef Py_HAVE_NATIVE_TLS
 /* If the platform has not supplied a platform specific
    TLS implementation, provide our own.
diff --git a/Python/thread_nt.h b/Python/thread_nt.h
index 5141053..8f5f996 100644
--- a/Python/thread_nt.h
+++ b/Python/thread_nt.h
@@ -185,7 +185,7 @@ PyThread_start_new_thread(void (*func)(void *), void *arg)
 	if (obj.done == NULL)
 		return -1;
 
-	rv = _beginthread(bootstrap, 0, &obj); /* use default stack size */
+	rv = _beginthread(bootstrap, _pythread_stacksize, &obj);
 	if (rv == (Py_uintptr_t)-1) {
 		/* I've seen errno == EAGAIN here, which means "there are
 		 * too many threads".
@@ -313,3 +313,37 @@ void PyThread_release_lock(PyThread_type_lock aLock)
 	if (!(aLock && LeaveNonRecursiveMutex((PNRMUTEX) aLock)))
 		dprintf(("%ld: Could not PyThread_release_lock(%p) error: %l\n", PyThread_get_thread_ident(), aLock, GetLastError()));
 }
+
+/* minimum/maximum thread stack sizes supported */
+#define THREAD_MIN_STACKSIZE	0x8000		/* 32kB */
+#define THREAD_MAX_STACKSIZE	0x10000000	/* 256MB */
+
+/* set the thread stack size.
+ * Return 1 if an exception is pending, 0 otherwise.
+ */
+static int
+_pythread_nt_set_stacksize(size_t size)
+{
+	/* set to default */
+	if (size == 0) {
+		_pythread_stacksize = 0;
+		return 0;
+	}
+
+	/* valid range? */
+	if (size >= THREAD_MIN_STACKSIZE && size < THREAD_MAX_STACKSIZE) {
+		_pythread_stacksize = size;
+		return 0;
+	}
+	else {
+		char warning[128];
+		snprintf(warning,
+			 128,
+			 "thread stack size of %#x bytes not supported on Win32",
+			 size);
+		return PyErr_Warn(PyExc_RuntimeWarning, warning);
+	}
+}
+
+#undef THREAD_SET_STACKSIZE
+#define THREAD_SET_STACKSIZE(x)	_pythread_nt_set_stacksize(x)
diff --git a/Python/thread_os2.h b/Python/thread_os2.h
index a18ce6f..91959a0 100644
--- a/Python/thread_os2.h
+++ b/Python/thread_os2.h
@@ -14,10 +14,13 @@
 long PyThread_get_thread_ident(void);
 #endif
 
+/* default thread stack size of 64kB */
 #if !defined(THREAD_STACK_SIZE)
 #define	THREAD_STACK_SIZE	0x10000
 #endif
 
+#define OS2_STACKSIZE(x)	(x ? x : THREAD_STACK_SIZE)
+
 /*
  * Initialization of the C package, should not be needed.
  */
@@ -35,7 +38,10 @@ PyThread_start_new_thread(void (*func)(void *), void *arg)
 	int aThread;
 	int success = 0;
 
-	aThread = _beginthread(func, NULL, THREAD_STACK_SIZE, arg);
+	aThread = _beginthread(func,
+				NULL,
+				OS2_STACKSIZE(_pythread_stacksize),
+				arg);
 
 	if (aThread == -1) {
 		success = -1;
@@ -274,3 +280,37 @@ void PyThread_release_lock(PyThread_type_lock aLock)
 	DosExitCritSec();
 #endif
 }
+
+/* minimum/maximum thread stack sizes supported */
+#define THREAD_MIN_STACKSIZE	0x8000		/* 32kB */
+#define THREAD_MAX_STACKSIZE	0x2000000	/* 32MB */
+
+/* set the thread stack size.
+ * Return 1 if an exception is pending, 0 otherwise.
+ */
+static int
+_pythread_os2_set_stacksize(size_t size)
+{
+	/* set to default */
+	if (size == 0) {
+		_pythread_stacksize = 0;
+		return 0;
+	}
+
+	/* valid range? */
+	if (size >= THREAD_MIN_STACKSIZE && size < THREAD_MAX_STACKSIZE) {
+		_pythread_stacksize = size;
+		return 0;
+	}
+	else {
+		char warning[128];
+		snprintf(warning,
+			 128,
+			 "thread stack size of %#x bytes not supported on OS/2",
+			 size);
+		return PyErr_Warn(PyExc_RuntimeWarning, warning);
+	}
+}
+
+#undef THREAD_SET_STACKSIZE
+#define THREAD_SET_STACKSIZE(x)	_pythread_os2_set_stacksize(x)
diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h
index c29a61c..e2907e0 100644
--- a/Python/thread_pthread.h
+++ b/Python/thread_pthread.h
@@ -12,6 +12,24 @@
 #endif
 #include <signal.h>
 
+/* The POSIX spec requires that use of pthread_attr_setstacksize
+   be conditional on _POSIX_THREAD_ATTR_STACKSIZE being defined. */
+#ifdef _POSIX_THREAD_ATTR_STACKSIZE
+#ifndef THREAD_STACK_SIZE
+#define	THREAD_STACK_SIZE	0	/* use default stack size */
+#endif
+/* for safety, ensure a viable minimum stacksize */
+#define	THREAD_STACK_MIN	0x8000	/* 32kB */
+#if THREAD_STACK_MIN < PTHREAD_STACK_MIN
+#undef THREAD_STACK_MIN
+#define	THREAD_STACK_MIN	PTHREAD_STACK_MIN
+#endif
+#else  /* !_POSIX_THREAD_ATTR_STACKSIZE */
+#ifdef THREAD_STACK_SIZE
+#error "THREAD_STACK_SIZE defined but _POSIX_THREAD_ATTR_STACKSIZE undefined"
+#endif
+#endif
+
 /* The POSIX spec says that implementations supporting the sem_*
    family of functions must indicate this by defining
    _POSIX_SEMAPHORES. */   
@@ -138,6 +156,10 @@ PyThread_start_new_thread(void (*func)(void *), void *arg)
 #if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
 	pthread_attr_t attrs;
 #endif
+#if defined(THREAD_STACK_SIZE)
+	size_t	tss;
+#endif
+
 	dprintf(("PyThread_start_new_thread called\n"));
 	if (!initialized)
 		PyThread_init_thread();
@@ -145,8 +167,15 @@ PyThread_start_new_thread(void (*func)(void *), void *arg)
 #if defined(THREAD_STACK_SIZE) || defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
 	pthread_attr_init(&attrs);
 #endif
-#ifdef THREAD_STACK_SIZE
-	pthread_attr_setstacksize(&attrs, THREAD_STACK_SIZE);
+#if defined(THREAD_STACK_SIZE)
+	tss = (_pythread_stacksize != 0) ? _pythread_stacksize
+					 : THREAD_STACK_SIZE;
+	if (tss != 0) {
+		if (pthread_attr_setstacksize(&attrs, tss) != 0) {
+			pthread_attr_destroy(&attrs);
+			return -1;
+		}
+	}
 #endif
 #if defined(PTHREAD_SYSTEM_SCHED_SUPPORTED)
         pthread_attr_setscope(&attrs, PTHREAD_SCOPE_SYSTEM);
@@ -460,3 +489,33 @@ PyThread_release_lock(PyThread_type_lock lock)
 }
 
 #endif /* USE_SEMAPHORES */
+
+/* set the thread stack size.
+ * Return 1 if an exception is pending, 0 otherwise.
+ */
+static int
+_pythread_pthread_set_stacksize(size_t size)
+{
+	/* set to default */
+	if (size == 0) {
+		_pythread_stacksize = 0;
+		return 0;
+	}
+
+	/* valid range? */
+	if (size >= THREAD_STACK_MIN) {
+		_pythread_stacksize = size;
+		return 0;
+	}
+	else {
+		char warning[128];
+		snprintf(warning,
+			 128,
+			 "thread stack size of %#x bytes not supported",
+			 size);
+		return PyErr_Warn(PyExc_RuntimeWarning, warning);
+	}
+}
+
+#undef THREAD_SET_STACKSIZE
+#define THREAD_SET_STACKSIZE(x)	_pythread_pthread_set_stacksize(x)