From 2fe77060eb2b322da925b50ffe3c471258736cee Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Sun, 6 Jul 2008 03:35:58 +0000 Subject: - Issue #2862: Make int and float freelist management consistent with other freelists. Changes their CompactFreeList apis into ClearFreeList apis and calls them via gc.collect(). --- Doc/c-api/float.rst | 7 +++--- Doc/c-api/int.rst | 7 +++--- Doc/library/gc.rst | 6 ++++++ Doc/library/sys.rst | 16 -------------- Include/floatobject.h | 2 +- Include/intobject.h | 2 +- Misc/NEWS | 4 ++++ Modules/gcmodule.c | 2 ++ Objects/floatobject.c | 43 +++++++++++++++--------------------- Objects/intobject.c | 60 +++++++++++++++++++++------------------------------ Python/sysmodule.c | 20 ----------------- 11 files changed, 62 insertions(+), 107 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index bb4f74a..f705297 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -86,10 +86,9 @@ Floating Point Objects .. versionadded:: 2.6 -.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum) +.. cfunction:: int PyFloat_ClearFreeList(void) - Compact the float free list. *bc* is the number of allocated blocks before - blocks are freed, *bf* is the number of freed blocks and *sum* is the number - of remaining objects in the blocks. + Clear the float free list. Return the number of items that could not + be freed. .. versionadded:: 2.6 diff --git a/Doc/c-api/int.rst b/Doc/c-api/int.rst index 94bf380..ef82100 100644 --- a/Doc/c-api/int.rst +++ b/Doc/c-api/int.rst @@ -122,10 +122,9 @@ Plain Integer Objects (:const:`LONG_MAX`, as defined in the system header files). -.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum) +.. cfunction:: int PyInt_ClearFreeList(void) - Compact the integer free list. *bc* is the number of allocated blocks before - blocks are freed, *bf* is the number of freed blocks and *sum* is the number - of remaining objects in the blocks. + Clear the integer free list. Return the number of items that could not + be freed. .. versionadded:: 2.6 diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 70e4a6b..9ebbf06 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -47,6 +47,12 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 2.5 The optional *generation* argument was added. + .. versionchanged:: 2.6 + The free lists maintained for a number of builtin types are cleared + whenever a full collection or collection of the highest generation (2) + is run. Not all items in some free lists may be freed due to the + particular implementation, in particular :class:`int` and :class:`float`. + .. function:: set_debug(flags) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 2e39633..22397f2 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -58,22 +58,6 @@ always available. A string containing the copyright pertaining to the Python interpreter. -.. function:: _compact_freelists() - - Compact the free lists of integers and floats by deallocating unused blocks. - It can reduce the memory usage of the Python process several tenth of - thousands of integers or floats have been allocated at once. - - The return value is a tuple of tuples each containing three elements, - amount of used objects, total block count before the blocks are deallocated - and amount of freed blocks. The first tuple refers to ints, the second to - floats. - - This function should be used for specialized purposes only. - - .. versionadded:: 2.6 - - .. function:: _clear_type_cache() Clear the internal type cache. The type cache is used to speed up attribute diff --git a/Include/floatobject.h b/Include/floatobject.h index 04978be..60ede40 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -113,7 +113,7 @@ PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); /* free list api */ -PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *); +PyAPI_FUNC(int) PyFloat_ClearFreeList(void); /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ diff --git a/Include/intobject.h b/Include/intobject.h index 329ff45..78746a6 100644 --- a/Include/intobject.h +++ b/Include/intobject.h @@ -60,7 +60,7 @@ PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int); PyAPI_FUNC(long) PyOS_strtol(char *, char **, int); /* free list api */ -PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *); +PyAPI_FUNC(int) PyInt_ClearFreeList(void); /* Convert an integer to the given base. Returns a string. If base is 2, 8 or 16, add the proper prefix '0b', '0o' or '0x'. diff --git a/Misc/NEWS b/Misc/NEWS index ce43943..715746c 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -29,6 +29,10 @@ Core and Builtins would not cause a syntax error. This was regression from 2.4 caused by the switch to the new compiler. +- Issue #2862: Make int and float freelist management consistent with other + freelists. Changes their CompactFreeList apis into ClearFreeList apis and + calls them via gc.collect(). + Library ------- diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 4f8c85f..b8f9c31 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -736,6 +736,8 @@ clear_freelists(void) (void)PyCFunction_ClearFreeList(); (void)PyTuple_ClearFreeList(); (void)PyUnicode_ClearFreeList(); + (void)PyInt_ClearFreeList(); + (void)PyFloat_ClearFreeList(); } /* This is the main function. Read this to understand how the diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 83401f2..45cb905 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1608,30 +1608,28 @@ _PyFloat_Init(void) PyStructSequence_InitType(&FloatInfoType, &floatinfo_desc); } -void -PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) +int +PyFloat_ClearFreeList(void) { PyFloatObject *p; PyFloatBlock *list, *next; - unsigned i; - size_t bc = 0, bf = 0; /* block count, number of freed blocks */ - size_t fsum = 0; /* total unfreed ints */ - int frem; /* remaining unfreed ints per block */ + int i; + int u; /* remaining unfreed ints per block */ + int freelist_size = 0; list = block_list; block_list = NULL; free_list = NULL; while (list != NULL) { - bc++; - frem = 0; + u = 0; for (i = 0, p = &list->objects[0]; i < N_FLOATOBJECTS; i++, p++) { if (PyFloat_CheckExact(p) && Py_REFCNT(p) != 0) - frem++; + u++; } next = list->next; - if (frem) { + if (u) { list->next = block_list; block_list = list; for (i = 0, p = &list->objects[0]; @@ -1646,15 +1644,12 @@ PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) } } else { - PyMem_FREE(list); /* XXX PyObject_FREE ??? */ - bf++; + PyMem_FREE(list); } - fsum += frem; + freelist_size += u; list = next; } - *pbc = bc; - *pbf = bf; - *bsum = fsum; + return freelist_size; } void @@ -1662,25 +1657,21 @@ PyFloat_Fini(void) { PyFloatObject *p; PyFloatBlock *list; - unsigned i; - size_t bc, bf; /* block count, number of freed blocks */ - size_t fsum; /* total unfreed floats per block */ + int i; + int u; /* total unfreed floats per block */ - PyFloat_CompactFreeList(&bc, &bf, &fsum); + u = PyFloat_ClearFreeList(); if (!Py_VerboseFlag) return; fprintf(stderr, "# cleanup floats"); - if (!fsum) { + if (!u) { fprintf(stderr, "\n"); } else { fprintf(stderr, - ": %" PY_FORMAT_SIZE_T "d unfreed float%s in %" - PY_FORMAT_SIZE_T "d out of %" - PY_FORMAT_SIZE_T "d block%s\n", - fsum, fsum == 1 ? "" : "s", - bc - bf, bc, bc == 1 ? "" : "s"); + ": %d unfreed float%s\n", + u, u == 1 ? "" : "s"); } if (Py_VerboseFlag > 1) { list = block_list; diff --git a/Objects/intobject.c b/Objects/intobject.c index f98aee0..e73c921 100644 --- a/Objects/intobject.c +++ b/Objects/intobject.c @@ -1296,35 +1296,33 @@ _PyInt_Init(void) return 1; } -void -PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) +int +PyInt_ClearFreeList(void) { PyIntObject *p; PyIntBlock *list, *next; - unsigned int ctr; - size_t bc = 0, bf = 0; /* block count, number of freed blocks */ - size_t isum = 0; /* total unfreed ints */ - int irem; /* remaining unfreed ints per block */ + int i; + int u; /* remaining unfreed ints per block */ + int freelist_size = 0; list = block_list; block_list = NULL; free_list = NULL; while (list != NULL) { - bc++; - irem = 0; - for (ctr = 0, p = &list->objects[0]; - ctr < N_INTOBJECTS; - ctr++, p++) { + u = 0; + for (i = 0, p = &list->objects[0]; + i < N_INTOBJECTS; + i++, p++) { if (PyInt_CheckExact(p) && p->ob_refcnt != 0) - irem++; + u++; } next = list->next; - if (irem) { + if (u) { list->next = block_list; block_list = list; - for (ctr = 0, p = &list->objects[0]; - ctr < N_INTOBJECTS; - ctr++, p++) { + for (i = 0, p = &list->objects[0]; + i < N_INTOBJECTS; + i++, p++) { if (!PyInt_CheckExact(p) || p->ob_refcnt == 0) { Py_TYPE(p) = (struct _typeobject *) @@ -1345,15 +1343,12 @@ PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) } else { PyMem_FREE(list); - bf++; } - isum += irem; + freelist_size += u; list = next; } - *pbc = bc; - *pbf = bf; - *bsum = isum; + return freelist_size; } void @@ -1361,12 +1356,10 @@ PyInt_Fini(void) { PyIntObject *p; PyIntBlock *list; - unsigned int ctr; - size_t bc, bf; /* block count, number of freed blocks */ - size_t isum; /* total unfreed ints per block */ + int i; + int u; /* total unfreed ints per block */ #if NSMALLNEGINTS + NSMALLPOSINTS > 0 - int i; PyIntObject **q; i = NSMALLNEGINTS + NSMALLPOSINTS; @@ -1376,27 +1369,24 @@ PyInt_Fini(void) *q++ = NULL; } #endif - PyInt_CompactFreeList(&bc, &bf, &isum); + u = PyInt_ClearFreeList(); if (!Py_VerboseFlag) return; fprintf(stderr, "# cleanup ints"); - if (!isum) { + if (!u) { fprintf(stderr, "\n"); } else { fprintf(stderr, - ": %" PY_FORMAT_SIZE_T "d unfreed int%s in %" - PY_FORMAT_SIZE_T "d out of %" - PY_FORMAT_SIZE_T "d block%s\n", - isum, isum == 1 ? "" : "s", - bc - bf, bc, bc == 1 ? "" : "s"); + ": %d unfreed int%s\n", + u, u == 1 ? "" : "s"); } if (Py_VerboseFlag > 1) { list = block_list; while (list != NULL) { - for (ctr = 0, p = &list->objects[0]; - ctr < N_INTOBJECTS; - ctr++, p++) { + for (i = 0, p = &list->objects[0]; + i < N_INTOBJECTS; + i++, p++) { if (PyInt_CheckExact(p) && p->ob_refcnt != 0) /* XXX(twouters) cast refcount to long until %zd is universally diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 64ea89f..5cfb488 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -829,32 +829,12 @@ PyDoc_STRVAR(sys_clear_type_cache__doc__, Clear the internal type lookup cache."); -static PyObject * -sys_compact_freelists(PyObject* self, PyObject* args) -{ - size_t isum, ibc, ibf; - size_t fsum, fbc, fbf; - - PyInt_CompactFreeList(&ibc, &ibf, &isum); - PyFloat_CompactFreeList(&fbc, &fbf, &fsum); - - return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf, - fsum, fbc, fbf); - -} - -PyDoc_STRVAR(sys_compact_freelists__doc__, -"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\ -Compact the free lists of ints and floats."); - static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ {"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS, callstats_doc}, {"_clear_type_cache", sys_clear_type_cache, METH_NOARGS, sys_clear_type_cache__doc__}, - {"_compact_freelists", sys_compact_freelists, METH_NOARGS, - sys_compact_freelists__doc__}, {"_current_frames", sys_current_frames, METH_NOARGS, current_frames_doc}, {"displayhook", sys_displayhook, METH_O, displayhook_doc}, -- cgit v0.12