Implemented Martin's suggestion to clear the free lists during the garbage collection of the highest generation.

15 files changed, 131 insertions, 25 deletions

diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst
index f20f14c..c104f89 100644
--- a/Doc/c-api/method.rst
+++ b/Doc/c-api/method.rst
@@ -63,3 +63,10 @@ There are some useful functions that are useful for working with method objects.
 .. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth)
 
    Macro version of :cfunc:`PyMethod_Self` which avoids error checking.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6
diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst
index b25adb3..1fcf5fc 100644
--- a/Doc/c-api/tuple.rst
+++ b/Doc/c-api/tuple.rst
@@ -115,3 +115,10 @@ Tuple Objects
 
    .. versionchanged:: 2.2
       Removed unused third parameter, *last_is_sticky*.
+
+
+.. cfunction:: int PyMethod_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6
diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst
index 7c570ab..5e5d1a2 100644
--- a/Doc/c-api/unicode.rst
+++ b/Doc/c-api/unicode.rst
@@ -89,6 +89,13 @@ access internal read-only data of Unicode objects:
    Return a pointer to the internal buffer of the object. *o* has to be a
    :ctype:`PyUnicodeObject` (not checked).
 
+
+.. cfunction:: int PyUnicode_ClearFreeList(void)
+
+   Clear the free list. Return the total number of freed items.
+
+   .. versionadded:: 2.6
+
 Unicode provides many different character properties. The most often needed ones
 are available through these macros which are mapped to C functions depending on
 the Python configuration.
diff --git a/Include/classobject.h b/Include/classobject.h
index 8f8db7d..118dd09 100644
--- a/Include/classobject.h
+++ b/Include/classobject.h
@@ -74,6 +74,7 @@ PyAPI_FUNC(PyObject *) _PyInstance_Lookup(PyObject *pinst, PyObject *name);
 
 PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *);
 
+PyAPI_FUNC(int) PyMethod_ClearFreeList(void);
 
 #ifdef __cplusplus
 }
diff --git a/Include/frameobject.h b/Include/frameobject.h
index 794f651..297b66d 100644
--- a/Include/frameobject.h
+++ b/Include/frameobject.h
@@ -75,6 +75,8 @@ PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int);
 PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
 PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
 
+PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/methodobject.h b/Include/methodobject.h
index 810a69c..6e160b6 100644
--- a/Include/methodobject.h
+++ b/Include/methodobject.h
@@ -85,6 +85,8 @@ typedef struct {
     PyObject    *m_module; /* The __module__ attribute, can be anything */
 } PyCFunctionObject;
 
+PyAPI_FUNC(int) PyCFunction_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/tupleobject.h b/Include/tupleobject.h
index 1b51b1b..58479ee 100644
--- a/Include/tupleobject.h
+++ b/Include/tupleobject.h
@@ -52,6 +52,8 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
 /* Macro, *only* to be used to fill in brand new tuples */
 #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
 
+PyAPI_FUNC(int) PyTuple_ClearFreeList(void);
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index 8809b34..d7ed0fa 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -209,6 +209,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
 # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
 # define _PyUnicode_Init _PyUnicodeUCS2_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
 # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
@@ -295,6 +296,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
 # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
 # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
 # define _PyUnicode_Init _PyUnicodeUCS4_Init
+# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
 # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
 # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
 # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
@@ -403,6 +405,8 @@ extern const unsigned char _Py_ascii_whitespace[];
 extern "C" {
 #endif
 
+PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
+
 /* --- Unicode Type ------------------------------------------------------- */
 
 typedef struct {
diff --git a/Misc/NEWS b/Misc/NEWS
index fe4b89f..134ac7e 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
 Core and builtins
 -----------------
 
+- Clear all free list during a gc.collect() of the highest generation in order
+  to allow pymalloc to free more arenas. Python may give back memory to the
+  OS earlier.
+
 - Issue #2045: Fix an infinite recursion triggered when printing a subclass of
   collections.defaultdict, if its default_factory is set to a bound method.
 
diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c
index 522cc89..4f8c85f 100644
--- a/Modules/gcmodule.c
+++ b/Modules/gcmodule.c
@@ -19,6 +19,7 @@
 */
 
 #include "Python.h"
+#include "frameobject.h"	/* for PyFrame_ClearFreeList */
 
 /* Get an object's GC head */
 #define AS_GC(o) ((PyGC_Head *)(o)-1)
@@ -722,6 +723,21 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old)
 	}
 }
 
+/* Clear all free lists
+ * All free lists are cleared during the collection of the highest generation.
+ * Allocated items in the free list may keep a pymalloc arena occupied.
+ * Clearing the free lists may give back memory to the OS earlier.
+ */
+static void
+clear_freelists(void)
+{
+	(void)PyMethod_ClearFreeList();
+	(void)PyFrame_ClearFreeList();
+	(void)PyCFunction_ClearFreeList();
+	(void)PyTuple_ClearFreeList();
+	(void)PyUnicode_ClearFreeList();
+}
+
 /* This is the main function.  Read this to understand how the
  * collection process works. */
 static Py_ssize_t
@@ -874,6 +890,12 @@ collect(int generation)
 	 */
 	(void)handle_finalizers(&finalizers, old);
 
+	/* Clear free list only during the collection of the higest
+	 * generation */
+	if (generation == NUM_GENERATIONS-1) {
+		clear_freelists();
+	}
+
 	if (PyErr_Occurred()) {
 		if (gc_str == NULL)
 			gc_str = PyString_FromString("garbage collection");
diff --git a/Objects/classobject.c b/Objects/classobject.c
index 759027b..6c77153 100644
--- a/Objects/classobject.c
+++ b/Objects/classobject.c
@@ -2626,9 +2626,11 @@ PyTypeObject PyMethod_Type = {
 
 /* Clear out the free list */
 
-void
-PyMethod_Fini(void)
+int
+PyMethod_ClearFreeList(void)
 {
+	int freelist_size = numfree;
+	
 	while (free_list) {
 		PyMethodObject *im = free_list;
 		free_list = (PyMethodObject *)(im->im_self);
@@ -2636,4 +2638,11 @@ PyMethod_Fini(void)
 		numfree--;
 	}
 	assert(numfree == 0);
+	return freelist_size;
+}
+
+void
+PyMethod_Fini(void)
+{
+	(void)PyMethod_ClearFreeList();
 }
diff --git a/Objects/frameobject.c b/Objects/frameobject.c
index df9c2e0..025431e 100644
--- a/Objects/frameobject.c
+++ b/Objects/frameobject.c
@@ -889,10 +889,11 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 }
 
 /* Clear out the free list */
-
-void
-PyFrame_Fini(void)
+int
+PyFrame_ClearFreeList(void)
 {
+	int freelist_size = numfree;
+	
 	while (free_list != NULL) {
 		PyFrameObject *f = free_list;
 		free_list = free_list->f_back;
@@ -900,6 +901,13 @@ PyFrame_Fini(void)
 		--numfree;
 	}
 	assert(numfree == 0);
+	return freelist_size;
+}
+
+void
+PyFrame_Fini(void)
+{
+	(void)PyFrame_ClearFreeList();
 	Py_XDECREF(builtin_object);
 	builtin_object = NULL;
 }
diff --git a/Objects/methodobject.c b/Objects/methodobject.c
index 0d9cf1c..d661c47 100644
--- a/Objects/methodobject.c
+++ b/Objects/methodobject.c
@@ -353,9 +353,11 @@ Py_FindMethod(PyMethodDef *methods, PyObject *self, const char *name)
 
 /* Clear out the free list */
 
-void
-PyCFunction_Fini(void)
+int
+PyCFunction_ClearFreeList(void)
 {
+	int freelist_size = numfree;
+	
 	while (free_list) {
 		PyCFunctionObject *v = free_list;
 		free_list = (PyCFunctionObject *)(v->m_self);
@@ -363,6 +365,13 @@ PyCFunction_Fini(void)
 		numfree--;
 	}
 	assert(numfree == 0);
+	return freelist_size;
+}
+
+void
+PyCFunction_Fini(void)
+{
+	(void)PyCFunction_ClearFreeList();
 }
 
 /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),
diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c
index dfd2a16..e9cb3ef 100644
--- a/Objects/tupleobject.c
+++ b/Objects/tupleobject.c
@@ -832,19 +832,18 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
 	return 0;
 }
 
-void
-PyTuple_Fini(void)
+int
+PyTuple_ClearFreeList(void)
 {
+	int freelist_size = 0;
 #if PyTuple_MAXSAVESIZE > 0
 	int i;
-
-	Py_XDECREF(free_list[0]);
-	free_list[0] = NULL;
-
 	for (i = 1; i < PyTuple_MAXSAVESIZE; i++) {
 		PyTupleObject *p, *q;
 		p = free_list[i];
+		freelist_size += numfree[i];
 		free_list[i] = NULL;
+		numfree[i] = 0;
 		while (p) {
 			q = p;
 			p = (PyTupleObject *)(p->ob_item[0]);
@@ -852,6 +851,20 @@ PyTuple_Fini(void)
 		}
 	}
 #endif
+	return freelist_size;
+}
+	
+void
+PyTuple_Fini(void)
+{
+#if PyTuple_MAXSAVESIZE > 0
+	/* empty tuples are used all over the place and applications may
+	 * rely on the fact that an empty tuple is a singleton. */
+	Py_XDECREF(free_list[0]);
+	free_list[0] = NULL;
+
+	(void)PyTuple_ClearFreeList();
+#endif
 }
 
 /*********************** Tuple Iterator **************************/
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 74b4796..3e15f53 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -8853,10 +8853,29 @@ void _PyUnicode_Init(void)
 
 /* Finalize the Unicode implementation */
 
+int
+PyUnicode_ClearFreeList(void)
+{
+    int freelist_size = numfree;
+    PyUnicodeObject *u;
+
+    for (u = free_list; u != NULL;) {
+	PyUnicodeObject *v = u;
+	u = *(PyUnicodeObject **)u;
+	if (v->str)
+	    PyMem_DEL(v->str);
+	Py_XDECREF(v->defenc);
+	PyObject_Del(v);
+	numfree--;
+    }
+    free_list = NULL;
+    assert(numfree == 0);
+    return freelist_size;
+}
+
 void
 _PyUnicode_Fini(void)
 {
-    PyUnicodeObject *u;
     int i;
 
     Py_XDECREF(unicode_empty);
@@ -8868,17 +8887,7 @@ _PyUnicode_Fini(void)
 	    unicode_latin1[i] = NULL;
 	}
     }
-
-    for (u = free_list; u != NULL;) {
-	PyUnicodeObject *v = u;
-	u = *(PyUnicodeObject **)u;
-	if (v->str)
-	    PyMem_DEL(v->str);
-	Py_XDECREF(v->defenc);
-	PyObject_Del(v);
-    }
-    free_list = NULL;
-    numfree = 0;
+    (void)PyUnicode_ClearFreeList();
 }
 
 #ifdef __cplusplus