From 422051a3675271e179995ad4a0f056ff94395d55 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 4 Feb 2008 18:00:12 +0000 Subject: Patch #1953 I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers. The patch also renames sys._cleartypecache to sys._clear_type_cache --- Doc/c-api/float.rst | 9 +++++++++ Doc/c-api/int.rst | 9 +++++++++ Doc/library/sys.rst | 24 ++++++++++++++++++++++-- Include/floatobject.h | 2 ++ Include/intobject.h | 3 +++ Lib/test/regrtest.py | 2 +- Lib/test/test_sys.py | 18 ++++++++++++++++++ Misc/NEWS | 6 +++++- Objects/floatobject.c | 30 ++++++++++++++++++++++------- Objects/intobject.c | 52 +++++++++++++++++++++++++++++++++------------------ Python/sysmodule.c | 47 +++++++++++++++++++++++++++++++++------------- 11 files changed, 160 insertions(+), 42 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index 505c19e..bb4f74a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -84,3 +84,12 @@ Floating Point Objects Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`. .. versionadded:: 2.6 + + +.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum) + + Compact the float free list. *bc* is the number of allocated blocks before + blocks are freed, *bf* is the number of freed blocks and *sum* is the number + of remaining objects in the blocks. + + .. versionadded:: 2.6 diff --git a/Doc/c-api/int.rst b/Doc/c-api/int.rst index 526083b..94bf380 100644 --- a/Doc/c-api/int.rst +++ b/Doc/c-api/int.rst @@ -120,3 +120,12 @@ Plain Integer Objects Return the system's idea of the largest integer it can handle (:const:`LONG_MAX`, as defined in the system header files). + + +.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum) + + Compact the integer free list. *bc* is the number of allocated blocks before + blocks are freed, *bf* is the number of freed blocks and *sum* is the number + of remaining objects in the blocks. + + .. versionadded:: 2.6 diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 7c88251..17e3890 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -58,9 +58,29 @@ always available. A string containing the copyright pertaining to the Python interpreter. -.. function:: _cleartypecache() +.. function:: _compact_freelists() - Clear the internal type lookup cache. + Compact the free lists of integers and floats by deallocating unused blocks. + It can reduce the memory usage of the Python process several tenth of + thousands of integers or floats have been allocated at once. + + The return value is a tuple of tuples each containing three elements, + amount of used objects, total block count before the blocks are deallocated + and amount of freed blocks. The first tuple refers to ints, the second to + floats. + + This function should be used for specialized purposes only. + + .. versionadded:: 2.6 + + +.. function:: _clear_type_cache() + + Clear the internal type cache. The type cache is used to speed up attribute + and method lookups. Use the function *only* to drop unnecessary references + during reference leak debugging. + + This function should be used for internal and specialized purposes only. .. versionadded:: 2.6 diff --git a/Include/floatobject.h b/Include/floatobject.h index 1f4510b..0e5f6a9 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -101,6 +101,8 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void); PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); +/* free list api */ +PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *); #ifdef __cplusplus } diff --git a/Include/intobject.h b/Include/intobject.h index 51d3e1b..7b50525 100644 --- a/Include/intobject.h +++ b/Include/intobject.h @@ -59,6 +59,9 @@ PyAPI_FUNC(long) PyInt_GetMax(void); PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(char *, char **, int); +/* free list api */ +PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *); + #ifdef __cplusplus } #endif diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 72d1039..b6aa96b 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -710,7 +710,7 @@ def dash_R_cleanup(fs, ps, pic, abcs): sys.path_importer_cache.update(pic) # clear type cache - sys._cleartypecache() + sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index aafbfa3..a01ea3e 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -363,6 +363,24 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(type(getattr(sys.flags, attr)), int, attr) self.assert_(repr(sys.flags)) + def test_clear_type_cache(self): + sys._clear_type_cache() + + def test_compact_freelists(self): + sys._compact_freelists() + r = sys._compact_freelists() + # freed blocks shouldn't change + self.assertEqual(r[0][2], 0) + self.assertEqual(r[1][2], 0) + # fill freelists + ints = list(range(12000)) + floats = [float(i) for i in ints] + del ints + del floats + # should free more than 200 blocks each + r = sys._compact_freelists() + self.assert_(r[0][2] > 200, r[0][2]) + self.assert_(r[1][2] > 200, r[1][2]) def test_main(): test.test_support.run_unittest(SysModuleTest) diff --git a/Misc/NEWS b/Misc/NEWS index 6291729..2466c6d 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1? Core and builtins ----------------- +- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions + ´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´ + to compact the internal free lists of pre-allocted ints and floats. + - Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls. Python expected the return type int but the fork familie returns pi_t. @@ -21,7 +25,7 @@ Core and builtins - Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak detection -- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the +- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the internal lookup cache for ref leak tests. - Patch #1473257: generator objects gain a gi_code attribute. This is the diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 5e9371d..19149af 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1609,17 +1609,15 @@ _PyFloat_Init(void) } void -PyFloat_Fini(void) +PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) { PyFloatObject *p; PyFloatBlock *list, *next; unsigned i; - int bc, bf; /* block count, number of freed blocks */ - int frem, fsum; /* remaining unfreed floats per block, total */ + size_t bc = 0, bf = 0; /* block count, number of freed blocks */ + size_t fsum = 0; /* total unfreed ints */ + int frem; /* remaining unfreed ints per block */ - bc = 0; - bf = 0; - fsum = 0; list = block_list; block_list = NULL; free_list = NULL; @@ -1654,6 +1652,22 @@ PyFloat_Fini(void) fsum += frem; list = next; } + *pbc = bc; + *pbf = bf; + *bsum = fsum; +} + +void +PyFloat_Fini(void) +{ + PyFloatObject *p; + PyFloatBlock *list; + unsigned i; + size_t bc, bf; /* block count, number of freed blocks */ + size_t fsum; /* total unfreed floats per block */ + + PyFloat_CompactFreeList(&bc, &bf, &fsum); + if (!Py_VerboseFlag) return; fprintf(stderr, "# cleanup floats"); @@ -1662,7 +1676,9 @@ PyFloat_Fini(void) } else { fprintf(stderr, - ": %d unfreed float%s in %d out of %d block%s\n", + ": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %" + PY_FORMAT_SIZE_T "d out of %" + PY_FORMAT_SIZE_T "d block%s\n", fsum, fsum == 1 ? "" : "s", bc - bf, bc, bc == 1 ? "" : "s"); } diff --git a/Objects/intobject.c b/Objects/intobject.c index edb8e4f..7c2a6fb 100644 --- a/Objects/intobject.c +++ b/Objects/intobject.c @@ -1202,28 +1202,15 @@ _PyInt_Init(void) } void -PyInt_Fini(void) +PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) { PyIntObject *p; PyIntBlock *list, *next; - int i; unsigned int ctr; - int bc, bf; /* block count, number of freed blocks */ - int irem, isum; /* remaining unfreed ints per block, total */ + size_t bc = 0, bf = 0; /* block count, number of freed blocks */ + size_t isum = 0; /* total unfreed ints */ + int irem; /* remaining unfreed ints per block */ -#if NSMALLNEGINTS + NSMALLPOSINTS > 0 - PyIntObject **q; - - i = NSMALLNEGINTS + NSMALLPOSINTS; - q = small_ints; - while (--i >= 0) { - Py_XDECREF(*q); - *q++ = NULL; - } -#endif - bc = 0; - bf = 0; - isum = 0; list = block_list; block_list = NULL; free_list = NULL; @@ -1268,6 +1255,33 @@ PyInt_Fini(void) isum += irem; list = next; } + + *pbc = bc; + *pbf = bf; + *bsum = isum; +} + +void +PyInt_Fini(void) +{ + PyIntObject *p; + PyIntBlock *list; + unsigned int ctr; + size_t bc, bf; /* block count, number of freed blocks */ + size_t isum; /* total unfreed ints per block */ + +#if NSMALLNEGINTS + NSMALLPOSINTS > 0 + int i; + PyIntObject **q; + + i = NSMALLNEGINTS + NSMALLPOSINTS; + q = small_ints; + while (--i >= 0) { + Py_XDECREF(*q); + *q++ = NULL; + } +#endif + PyInt_CompactFreeList(&bc, &bf, &isum); if (!Py_VerboseFlag) return; fprintf(stderr, "# cleanup ints"); @@ -1276,7 +1290,9 @@ PyInt_Fini(void) } else { fprintf(stderr, - ": %d unfreed int%s in %d out of %d block%s\n", + ": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %" + PY_FORMAT_SIZE_T "d out of %" + PY_FORMAT_SIZE_T "d block%s\n", isum, isum == 1 ? "" : "s", bc - bf, bc, bc == 1 ? "" : "s"); } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 41830b9..8e27844 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -754,17 +754,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\ 10. Number of stack pops performed by call_function()" ); -static PyObject * -sys_cleartypecache(PyObject* self, PyObject* args) -{ - PyType_ClearCache(); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(cleartypecache_doc, -"_cleartypecache() -> None\n\ -Clear the internal type lookup cache."); - #ifdef __cplusplus extern "C" { #endif @@ -783,12 +772,44 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *); } #endif +static PyObject * +sys_clear_type_cache(PyObject* self, PyObject* args) +{ + PyType_ClearCache(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(sys_clear_type_cache__doc__, +"_clear_type_cache() -> None\n\ +Clear the internal type lookup cache."); + + +static PyObject * +sys_compact_freelists(PyObject* self, PyObject* args) +{ + size_t isum, ibc, ibf; + size_t fsum, fbc, fbf; + + PyInt_CompactFreeList(&ibc, &ibf, &isum); + PyFloat_CompactFreeList(&fbc, &fbf, &fsum); + + return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf, + fsum, fbc, fbf); + +} + +PyDoc_STRVAR(sys_compact_freelists__doc__, +"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\ +Compact the free lists of ints and floats."); + static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ {"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS, callstats_doc}, - {"_cleartypecache", sys_cleartypecache, METH_NOARGS, - cleartypecache_doc}, + {"_clear_type_cache", sys_clear_type_cache, METH_NOARGS, + sys_clear_type_cache__doc__}, + {"_compact_freelists", sys_compact_freelists, METH_NOARGS, + sys_compact_freelists__doc__}, {"_current_frames", sys_current_frames, METH_NOARGS, current_frames_doc}, {"displayhook", sys_displayhook, METH_O, displayhook_doc}, -- cgit v0.12