From 3b718a79af900fdacaf0b825137f69eadc753765 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Thu, 14 Feb 2008 12:47:33 +0000 Subject: Implemented Martin's suggestion to clear the free lists during the garbage collection of the highest generation. --- Doc/c-api/method.rst | 7 +++++++ Doc/c-api/tuple.rst | 7 +++++++ Doc/c-api/unicode.rst | 7 +++++++ Include/classobject.h | 1 + Include/frameobject.h | 2 ++ Include/methodobject.h | 2 ++ Include/tupleobject.h | 2 ++ Include/unicodeobject.h | 4 ++++ Misc/NEWS | 4 ++++ Modules/gcmodule.c | 22 ++++++++++++++++++++++ Objects/classobject.c | 13 +++++++++++-- Objects/frameobject.c | 14 +++++++++++--- Objects/methodobject.c | 13 +++++++++++-- Objects/tupleobject.c | 25 +++++++++++++++++++------ Objects/unicodeobject.c | 33 +++++++++++++++++++++------------ 15 files changed, 131 insertions(+), 25 deletions(-) diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index f20f14c..c104f89 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -63,3 +63,10 @@ There are some useful functions that are useful for working with method objects. .. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth) Macro version of :cfunc:`PyMethod_Self` which avoids error checking. + + +.. cfunction:: int PyMethod_ClearFreeList(void) + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 2.6 diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index b25adb3..1fcf5fc 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -115,3 +115,10 @@ Tuple Objects .. versionchanged:: 2.2 Removed unused third parameter, *last_is_sticky*. + + +.. cfunction:: int PyMethod_ClearFreeList(void) + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 2.6 diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 7c570ab..5e5d1a2 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -89,6 +89,13 @@ access internal read-only data of Unicode objects: Return a pointer to the internal buffer of the object. *o* has to be a :ctype:`PyUnicodeObject` (not checked). + +.. cfunction:: int PyUnicode_ClearFreeList(void) + + Clear the free list. Return the total number of freed items. + + .. versionadded:: 2.6 + Unicode provides many different character properties. The most often needed ones are available through these macros which are mapped to C functions depending on the Python configuration. diff --git a/Include/classobject.h b/Include/classobject.h index 8f8db7d..118dd09 100644 --- a/Include/classobject.h +++ b/Include/classobject.h @@ -74,6 +74,7 @@ PyAPI_FUNC(PyObject *) _PyInstance_Lookup(PyObject *pinst, PyObject *name); PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *); +PyAPI_FUNC(int) PyMethod_ClearFreeList(void); #ifdef __cplusplus } diff --git a/Include/frameobject.h b/Include/frameobject.h index 794f651..297b66d 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -75,6 +75,8 @@ PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int); PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); +PyAPI_FUNC(int) PyFrame_ClearFreeList(void); + #ifdef __cplusplus } #endif diff --git a/Include/methodobject.h b/Include/methodobject.h index 810a69c..6e160b6 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -85,6 +85,8 @@ typedef struct { PyObject *m_module; /* The __module__ attribute, can be anything */ } PyCFunctionObject; +PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); + #ifdef __cplusplus } #endif diff --git a/Include/tupleobject.h b/Include/tupleobject.h index 1b51b1b..58479ee 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -52,6 +52,8 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); /* Macro, *only* to be used to fill in brand new tuples */ #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v) +PyAPI_FUNC(int) PyTuple_ClearFreeList(void); + #ifdef __cplusplus } #endif diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h index 8809b34..d7ed0fa 100644 --- a/Include/unicodeobject.h +++ b/Include/unicodeobject.h @@ -209,6 +209,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE; # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString # define _PyUnicode_Fini _PyUnicodeUCS2_Fini # define _PyUnicode_Init _PyUnicodeUCS2_Init +# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit @@ -295,6 +296,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE; # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString # define _PyUnicode_Fini _PyUnicodeUCS4_Fini # define _PyUnicode_Init _PyUnicodeUCS4_Init +# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit @@ -403,6 +405,8 @@ extern const unsigned char _Py_ascii_whitespace[]; extern "C" { #endif +PyAPI_FUNC(int) PyUnicode_ClearFreeList(void); + /* --- Unicode Type ------------------------------------------------------- */ typedef struct { diff --git a/Misc/NEWS b/Misc/NEWS index fe4b89f..134ac7e 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1? Core and builtins ----------------- +- Clear all free list during a gc.collect() of the highest generation in order + to allow pymalloc to free more arenas. Python may give back memory to the + OS earlier. + - Issue #2045: Fix an infinite recursion triggered when printing a subclass of collections.defaultdict, if its default_factory is set to a bound method. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 522cc89..4f8c85f 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -19,6 +19,7 @@ */ #include "Python.h" +#include "frameobject.h" /* for PyFrame_ClearFreeList */ /* Get an object's GC head */ #define AS_GC(o) ((PyGC_Head *)(o)-1) @@ -722,6 +723,21 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) } } +/* Clear all free lists + * All free lists are cleared during the collection of the highest generation. + * Allocated items in the free list may keep a pymalloc arena occupied. + * Clearing the free lists may give back memory to the OS earlier. + */ +static void +clear_freelists(void) +{ + (void)PyMethod_ClearFreeList(); + (void)PyFrame_ClearFreeList(); + (void)PyCFunction_ClearFreeList(); + (void)PyTuple_ClearFreeList(); + (void)PyUnicode_ClearFreeList(); +} + /* This is the main function. Read this to understand how the * collection process works. */ static Py_ssize_t @@ -874,6 +890,12 @@ collect(int generation) */ (void)handle_finalizers(&finalizers, old); + /* Clear free list only during the collection of the higest + * generation */ + if (generation == NUM_GENERATIONS-1) { + clear_freelists(); + } + if (PyErr_Occurred()) { if (gc_str == NULL) gc_str = PyString_FromString("garbage collection"); diff --git a/Objects/classobject.c b/Objects/classobject.c index 759027b..6c77153 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -2626,9 +2626,11 @@ PyTypeObject PyMethod_Type = { /* Clear out the free list */ -void -PyMethod_Fini(void) +int +PyMethod_ClearFreeList(void) { + int freelist_size = numfree; + while (free_list) { PyMethodObject *im = free_list; free_list = (PyMethodObject *)(im->im_self); @@ -2636,4 +2638,11 @@ PyMethod_Fini(void) numfree--; } assert(numfree == 0); + return freelist_size; +} + +void +PyMethod_Fini(void) +{ + (void)PyMethod_ClearFreeList(); } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index df9c2e0..025431e 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -889,10 +889,11 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear) } /* Clear out the free list */ - -void -PyFrame_Fini(void) +int +PyFrame_ClearFreeList(void) { + int freelist_size = numfree; + while (free_list != NULL) { PyFrameObject *f = free_list; free_list = free_list->f_back; @@ -900,6 +901,13 @@ PyFrame_Fini(void) --numfree; } assert(numfree == 0); + return freelist_size; +} + +void +PyFrame_Fini(void) +{ + (void)PyFrame_ClearFreeList(); Py_XDECREF(builtin_object); builtin_object = NULL; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 0d9cf1c..d661c47 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -353,9 +353,11 @@ Py_FindMethod(PyMethodDef *methods, PyObject *self, const char *name) /* Clear out the free list */ -void -PyCFunction_Fini(void) +int +PyCFunction_ClearFreeList(void) { + int freelist_size = numfree; + while (free_list) { PyCFunctionObject *v = free_list; free_list = (PyCFunctionObject *)(v->m_self); @@ -363,6 +365,13 @@ PyCFunction_Fini(void) numfree--; } assert(numfree == 0); + return freelist_size; +} + +void +PyCFunction_Fini(void) +{ + (void)PyCFunction_ClearFreeList(); } /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index dfd2a16..e9cb3ef 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -832,19 +832,18 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) return 0; } -void -PyTuple_Fini(void) +int +PyTuple_ClearFreeList(void) { + int freelist_size = 0; #if PyTuple_MAXSAVESIZE > 0 int i; - - Py_XDECREF(free_list[0]); - free_list[0] = NULL; - for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { PyTupleObject *p, *q; p = free_list[i]; + freelist_size += numfree[i]; free_list[i] = NULL; + numfree[i] = 0; while (p) { q = p; p = (PyTupleObject *)(p->ob_item[0]); @@ -852,6 +851,20 @@ PyTuple_Fini(void) } } #endif + return freelist_size; +} + +void +PyTuple_Fini(void) +{ +#if PyTuple_MAXSAVESIZE > 0 + /* empty tuples are used all over the place and applications may + * rely on the fact that an empty tuple is a singleton. */ + Py_XDECREF(free_list[0]); + free_list[0] = NULL; + + (void)PyTuple_ClearFreeList(); +#endif } /*********************** Tuple Iterator **************************/ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 74b4796..3e15f53 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -8853,10 +8853,29 @@ void _PyUnicode_Init(void) /* Finalize the Unicode implementation */ +int +PyUnicode_ClearFreeList(void) +{ + int freelist_size = numfree; + PyUnicodeObject *u; + + for (u = free_list; u != NULL;) { + PyUnicodeObject *v = u; + u = *(PyUnicodeObject **)u; + if (v->str) + PyMem_DEL(v->str); + Py_XDECREF(v->defenc); + PyObject_Del(v); + numfree--; + } + free_list = NULL; + assert(numfree == 0); + return freelist_size; +} + void _PyUnicode_Fini(void) { - PyUnicodeObject *u; int i; Py_XDECREF(unicode_empty); @@ -8868,17 +8887,7 @@ _PyUnicode_Fini(void) unicode_latin1[i] = NULL; } } - - for (u = free_list; u != NULL;) { - PyUnicodeObject *v = u; - u = *(PyUnicodeObject **)u; - if (v->str) - PyMem_DEL(v->str); - Py_XDECREF(v->defenc); - PyObject_Del(v); - } - free_list = NULL; - numfree = 0; + (void)PyUnicode_ClearFreeList(); } #ifdef __cplusplus -- cgit v0.12