From 15ebc88d87d2ff8f520581a9f6a6816d78a7e504 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Mon, 4 Feb 2008 18:48:49 +0000 Subject: Merged revisions 60481,60485,60489-60492,60494-60496,60498-60499,60501-60503,60505-60506,60508-60509,60523-60524,60532,60543,60545,60547-60548,60552-60567 via svnmerge from svn+ssh://pythondev@svn.python.org/python/trunk ........ r60553 | neal.norwitz | 2008-02-03 17:53:09 +0100 (Sun, 03 Feb 2008) | 1 line Ignore leaky warnings from test_asynchat ........ r60555 | christian.heimes | 2008-02-03 20:51:13 +0100 (Sun, 03 Feb 2008) | 1 line Another int -> pid_t case ........ r60560 | amaury.forgeotdarc | 2008-02-03 23:51:43 +0100 (Sun, 03 Feb 2008) | 6 lines Ensure that PySet_Add() operates on a newly created frozenset, like PyTuple_SetItem does. Add PyFrozenSet_Check(), which was not needed before; The list of Py*Set_Check* macros seems to be complete now. Add missing NEWS entries about all this. ........ r60563 | amaury.forgeotdarc | 2008-02-04 00:14:32 +0100 (Mon, 04 Feb 2008) | 2 lines Nasty typo in setobject.h ........ r60564 | amaury.forgeotdarc | 2008-02-04 00:15:32 +0100 (Mon, 04 Feb 2008) | 3 lines Correct test_mailbox on win32: since the test sets a custom 'colon' attribute to the main mailbox, copy it to secondary mailbox instances. ........ r60565 | amaury.forgeotdarc | 2008-02-04 00:57:24 +0100 (Mon, 04 Feb 2008) | 2 lines Let test_socketserver pass on win32, which does not have AF_UNIX sockets. ........ r60566 | jeffrey.yasskin | 2008-02-04 02:04:35 +0100 (Mon, 04 Feb 2008) | 2 lines Make int() and long() fall back to __trunc__(). See issue 2002. ........ r60567 | christian.heimes | 2008-02-04 19:00:12 +0100 (Mon, 04 Feb 2008) | 3 lines Patch #1953 I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers. The patch also renames sys._cleartypecache to sys._clear_type_cache ........ --- Doc/c-api/float.rst | 6 ++++ Doc/c-api/set.rst | 7 +++++ Doc/library/sys.rst | 23 ++++++++++++++-- Include/abstract.h | 13 +++++++++ Include/floatobject.h | 3 ++ Include/setobject.h | 6 +++- Lib/rational.py | 2 -- Lib/test/regrtest.py | 2 +- Lib/test/test_builtin.py | 52 ++++++++++++++++++++++++++++++++++- Lib/test/test_mailbox.py | 1 + Lib/test/test_socketserver.py | 15 +++++----- Lib/test/test_sys.py | 14 ++++++++++ Misc/build.sh | 2 +- Modules/posixmodule.c | 3 +- Objects/abstract.c | 64 +++++++++++++++++++++++++++++++++++++++++++ Objects/floatobject.c | 30 +++++++++++++++----- Objects/setobject.c | 7 ++++- Python/sysmodule.c | 44 ++++++++++++++++++++--------- 18 files changed, 257 insertions(+), 37 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b0647bc..8fff0d0 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -72,3 +72,9 @@ Floating Point Objects .. cfunction:: double PyFloat_GetMin(void) Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`. + +.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum) + + Compact the float free list. *bc* is the number of allocated blocks before + blocks are freed, *bf* is the number of freed blocks and *sum* is the number + of remaining objects in the blocks. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 83d9532..14d8eab 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -56,6 +56,13 @@ the constructor functions work with any iterable Python object. .. versionadded:: 2.6 +.. cfunction:: int PyFrozenSet_Check(PyObject *p) + + Return true if *p* is a :class:`frozenset` object or an instance of a + subtype. + + .. versionadded:: 2.6 + .. cfunction:: int PyAnySet_Check(PyObject *p) Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 5fdff44..02a1d20 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -54,9 +54,28 @@ always available. A string containing the copyright pertaining to the Python interpreter. -.. function:: _cleartypecache() +.. function:: _compact_freelists() - Clear the internal type lookup cache. + Compact the free list of floats by deallocating unused blocks. + It can reduce the memory usage of the Python process several tenth of + thousands of integers or floats have been allocated at once. + + The return value is a tuple of tuples each containing three elements, + amount of used objects, total block count before the blocks are deallocated + and amount of freed blocks. + + This function should be used for specialized purposes only. + + .. versionadded:: 2.6 + + +.. function:: _clear_type_cache() + + Clear the internal type cache. The type cache is used to speed up attribute + and method lookups. Use the function *only* to drop unnecessary references + during reference leak debugging. + + This function should be used for internal and specialized purposes only. .. versionadded:: 2.6 diff --git a/Include/abstract.h b/Include/abstract.h index dc21692..a496817 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -793,6 +793,19 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/ PyAPI_FUNC(Py_ssize_t) PyNumber_AsSsize_t(PyObject *o, PyObject *exc); /* + Returns the Integral instance converted to an int. The + instance is expected to be int or long or have an __int__ + method. Steals integral's reference. error_format will be + used to create the TypeError if integral isn't actually an + Integral instance. error_format should be a format string + that can accept a char* naming integral's type. + */ + + PyAPI_FUNC(PyObject *) _PyNumber_ConvertIntegralToInt( + PyObject *integral, + const char* error_format); + + /* Returns the object converted to Py_ssize_t by going through PyNumber_Index first. If an overflow error occurs while converting the int-or-long to Py_ssize_t, then the second argument diff --git a/Include/floatobject.h b/Include/floatobject.h index f18933b..459b029 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -91,6 +91,9 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void); PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le); PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); +/* free list api */ +PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *); + #ifdef __cplusplus } #endif diff --git a/Include/setobject.h b/Include/setobject.h index c8e2702..60bd73b 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -75,7 +75,11 @@ PyAPI_DATA(PyTypeObject) PySetIter_Type; PyType_IsSubtype(Py_TYPE(ob), &PySet_Type) || \ PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) #define PySet_Check(ob) \ - (Py_TYPE(ob) == &PySet_Type || PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) + (Py_TYPE(ob) == &PySet_Type || \ + PyType_IsSubtype(Py_TYPE(ob), &PySet_Type)) +#define PyFrozenSet_Check(ob) \ + (Py_TYPE(ob) == &PyFrozenSet_Type || \ + PyType_IsSubtype(Py_TYPE(ob), &PyFrozenSet_Type)) PyAPI_FUNC(PyObject *) PySet_New(PyObject *); PyAPI_FUNC(PyObject *) PyFrozenSet_New(PyObject *); diff --git a/Lib/rational.py b/Lib/rational.py index e2fed23..6002964 100755 --- a/Lib/rational.py +++ b/Lib/rational.py @@ -406,8 +406,6 @@ class Rational(RationalAbc): else: return a.numerator // a.denominator - __int__ = __trunc__ - def __floor__(a): """Will be math.floor(a) in 3.0.""" return a.numerator // a.denominator diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 5934e6b..9b6f14a 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -753,7 +753,7 @@ def dash_R_cleanup(fs, ps, pic, abcs): sys.path_importer_cache.update(pic) # clear type cache - sys._cleartypecache() + sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index cc31e93..1b9edc0 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -861,6 +861,14 @@ class BuiltinTest(unittest.TestCase): def test_intconversion(self): # Test __int__() + class ClassicMissingMethods: + pass + self.assertRaises(TypeError, int, ClassicMissingMethods()) + + class MissingMethods(object): + pass + self.assertRaises(TypeError, int, MissingMethods()) + class Foo0: def __int__(self): return 42 @@ -892,6 +900,49 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(int(Foo4()), 42) self.assertRaises(TypeError, int, Foo5()) + class Classic: + pass + for base in (object, Classic): + class IntOverridesTrunc(base): + def __int__(self): + return 42 + def __trunc__(self): + return -12 + self.assertEqual(int(IntOverridesTrunc()), 42) + + class JustTrunc(base): + def __trunc__(self): + return 42 + self.assertEqual(int(JustTrunc()), 42) + + for trunc_result_base in (object, Classic): + class Integral(trunc_result_base): + def __int__(self): + return 42 + + class TruncReturnsNonInt(base): + def __trunc__(self): + return Integral() + self.assertEqual(int(TruncReturnsNonInt()), 42) + + class NonIntegral(trunc_result_base): + def __trunc__(self): + # Check that we avoid infinite recursion. + return NonIntegral() + + class TruncReturnsNonIntegral(base): + def __trunc__(self): + return NonIntegral() + try: + int(TruncReturnsNonIntegral()) + except TypeError as e: + self.assertEquals(str(e), + "__trunc__ returned non-Integral" + " (type NonIntegral)") + else: + self.fail("Failed to raise TypeError with %s" % + ((base, trunc_result_base),)) + def test_iter(self): self.assertRaises(TypeError, iter) self.assertRaises(TypeError, iter, 42, 42) @@ -1095,7 +1146,6 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) - def test_longconversion(self): # Test __long__() class Foo0: diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 918116a..925f878 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -517,6 +517,7 @@ class TestMaildir(TestMailbox): class FakeMessage(mailbox.MaildirMessage): pass box = mailbox.Maildir(self._path, factory=FakeMessage) + box.colon = self._box.colon msg2 = box.get_message(key) self.assert_(isinstance(msg2, FakeMessage)) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 6a65b6c..0fccdd9 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -51,13 +51,14 @@ class MyDatagramHandler(MyMixinHandler, SocketServer.DatagramRequestHandler): pass -class ForkingUnixStreamServer(SocketServer.ForkingMixIn, - SocketServer.UnixStreamServer): - pass - -class ForkingUnixDatagramServer(SocketServer.ForkingMixIn, - SocketServer.UnixDatagramServer): - pass +if HAVE_UNIX_SOCKETS: + class ForkingUnixStreamServer(SocketServer.ForkingMixIn, + SocketServer.UnixStreamServer): + pass + + class ForkingUnixDatagramServer(SocketServer.ForkingMixIn, + SocketServer.UnixDatagramServer): + pass class MyMixinServer: diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 8979889..11251b7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -330,6 +330,20 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(type(getattr(sys.flags, attr)), int, attr) self.assert_(repr(sys.flags)) + def test_clear_type_cache(self): + sys._clear_type_cache() + + def test_compact_freelists(self): + sys._compact_freelists() + r = sys._compact_freelists() + # freed blocks shouldn't change + self.assertEqual(r[0][2], 0) + # fill freelists + floats = [float(i) for i in range(12000)] + del floats + # should free more than 200 blocks + r = sys._compact_freelists() + self.assert_(r[0][2] > 200, r[0][2]) def test_main(): test.test_support.run_unittest(SysModuleTest) diff --git a/Misc/build.sh b/Misc/build.sh index 8eca6ba..91a8cbc 100755 --- a/Misc/build.sh +++ b/Misc/build.sh @@ -67,7 +67,7 @@ REFLOG="build/reflog.txt.out" # Note: test_XXX (none currently) really leak, but are disabled # so we don't send spam. Any test which really leaks should only # be listed here if there are also test cases under Lib/test/leakers. -LEAKY_TESTS="test_(cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)" +LEAKY_TESTS="test_(asynchat|cmd_line|popen2|socket|sys|threadsignals|urllib2_localnet)" # These tests always fail, so skip them so we don't get false positives. _ALWAYS_SKIP="" diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fc989fe..b21e93b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4626,7 +4626,8 @@ Return the process group associated with the terminal given by a fd."); static PyObject * posix_tcgetpgrp(PyObject *self, PyObject *args) { - int fd, pgid; + int fd; + pid_t pgid; if (!PyArg_ParseTuple(args, "i:tcgetpgrp", &fd)) return NULL; pgid = tcgetpgrp(fd); diff --git a/Objects/abstract.c b/Objects/abstract.c index 755cb0e..b50b43e 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1242,6 +1242,40 @@ PyNumber_AsSsize_t(PyObject *item, PyObject *err) } +PyObject * +_PyNumber_ConvertIntegralToInt(PyObject *integral, const char* error_format) +{ + static PyObject *int_name = NULL; + if (int_name == NULL) { + int_name = PyUnicode_InternFromString("__int__"); + if (int_name == NULL) + return NULL; + } + + if (integral && !PyLong_Check(integral)) { + /* Don't go through tp_as_number->nb_int to avoid + hitting the classic class fallback to __trunc__. */ + PyObject *int_func = PyObject_GetAttr(integral, int_name); + if (int_func == NULL) { + PyErr_Clear(); /* Raise a different error. */ + goto non_integral_error; + } + Py_DECREF(integral); + integral = PyEval_CallObject(int_func, NULL); + Py_DECREF(int_func); + if (integral && !PyLong_Check(integral)) { + goto non_integral_error; + } + } + return integral; + +non_integral_error: + PyErr_Format(PyExc_TypeError, error_format, Py_TYPE(integral)->tp_name); + Py_DECREF(integral); + return NULL; +} + + /* Add a check for embedded NULL-bytes in the argument. */ static PyObject * long_from_string(const char *s, Py_ssize_t len) @@ -1265,9 +1299,17 @@ PyObject * PyNumber_Long(PyObject *o) { PyNumberMethods *m; + static PyObject *trunc_name = NULL; + PyObject *trunc_func; const char *buffer; Py_ssize_t buffer_len; + if (trunc_name == NULL) { + trunc_name = PyUnicode_InternFromString("__trunc__"); + if (trunc_name == NULL) + return NULL; + } + if (o == NULL) return null_error(); if (PyLong_CheckExact(o)) { @@ -1287,6 +1329,7 @@ PyNumber_Long(PyObject *o) return res; } if (m && m->nb_long) { /* This should include subclasses of long */ + /* Classic classes always take this branch. */ PyObject *res = m->nb_long(o); if (res && !PyLong_Check(res)) { PyErr_Format(PyExc_TypeError, @@ -1299,6 +1342,27 @@ PyNumber_Long(PyObject *o) } if (PyLong_Check(o)) /* A long subclass without nb_long */ return _PyLong_Copy((PyLongObject *)o); + trunc_func = PyObject_GetAttr(o, trunc_name); + if (trunc_func) { + PyObject *truncated = PyEval_CallObject(trunc_func, NULL); + PyObject *int_instance; + Py_DECREF(trunc_func); + /* __trunc__ is specified to return an Integral type, + but long() needs to return a long. */ + int_instance = _PyNumber_ConvertIntegralToInt( + truncated, + "__trunc__ returned non-Integral (type %.200s)"); + return int_instance; + } + PyErr_Clear(); /* It's not an error if o.__trunc__ doesn't exist. */ + + if (PyString_Check(o)) + /* need to do extra error checking that PyLong_FromString() + * doesn't do. In particular long('9.5') must raise an + * exception, not truncate the float. + */ + return long_from_string(PyString_AS_STRING(o), + PyString_GET_SIZE(o)); if (PyUnicode_Check(o)) /* The above check is done in PyLong_FromUnicode(). */ return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o), diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 9b499ba..e074a37 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1540,17 +1540,15 @@ _PyFloat_Init(void) } void -PyFloat_Fini(void) +PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum) { PyFloatObject *p; PyFloatBlock *list, *next; unsigned i; - int bc, bf; /* block count, number of freed blocks */ - int frem, fsum; /* remaining unfreed floats per block, total */ + size_t bc = 0, bf = 0; /* block count, number of freed blocks */ + size_t fsum = 0; /* total unfreed ints */ + int frem; /* remaining unfreed ints per block */ - bc = 0; - bf = 0; - fsum = 0; list = block_list; block_list = NULL; free_list = NULL; @@ -1585,6 +1583,22 @@ PyFloat_Fini(void) fsum += frem; list = next; } + *pbc = bc; + *pbf = bf; + *bsum = fsum; +} + +void +PyFloat_Fini(void) +{ + PyFloatObject *p; + PyFloatBlock *list; + unsigned i; + size_t bc, bf; /* block count, number of freed blocks */ + size_t fsum; /* total unfreed floats per block */ + + PyFloat_CompactFreeList(&bc, &bf, &fsum); + if (!Py_VerboseFlag) return; fprintf(stderr, "# cleanup floats"); @@ -1593,7 +1607,9 @@ PyFloat_Fini(void) } else { fprintf(stderr, - ": %d unfreed float%s in %d out of %d block%s\n", + ": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %" + PY_FORMAT_SIZE_T "d out of %" + PY_FORMAT_SIZE_T "d block%s\n", fsum, fsum == 1 ? "" : "s", bc - bf, bc, bc == 1 ? "" : "s"); } diff --git a/Objects/setobject.c b/Objects/setobject.c index 4cf47bd..c827434 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2173,7 +2173,8 @@ PySet_Discard(PyObject *set, PyObject *key) int PySet_Add(PyObject *anyset, PyObject *key) { - if (!PyAnySet_Check(anyset)) { + if (!PySet_Check(anyset) && + (!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) { PyErr_BadInternalCall(); return -1; } @@ -2277,6 +2278,10 @@ test_c_api(PySetObject *so) f = PyFrozenSet_New(dup); assertRaises(PySet_Clear(f) == -1, PyExc_SystemError); assertRaises(_PySet_Update(f, dup) == -1, PyExc_SystemError); + assert(PySet_Add(f, elem) == 0); + Py_INCREF(f); + assertRaises(PySet_Add(f, elem) == -1, PyExc_SystemError); + Py_DECREF(f); Py_DECREF(f); /* Exercise direct iteration */ diff --git a/Python/sysmodule.c b/Python/sysmodule.c index d3ec827..aceb2a3 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -730,17 +730,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\ 10. Number of stack pops performed by call_function()" ); -static PyObject * -sys_cleartypecache(PyObject* self, PyObject* args) -{ - PyType_ClearCache(); - Py_RETURN_NONE; -} - -PyDoc_STRVAR(cleartypecache_doc, -"_cleartypecache() -> None\n\ -Clear the internal type lookup cache."); - #ifdef __cplusplus extern "C" { #endif @@ -759,12 +748,41 @@ extern PyObject *_Py_GetDXProfile(PyObject *, PyObject *); } #endif +static PyObject * +sys_clear_type_cache(PyObject* self, PyObject* args) +{ + PyType_ClearCache(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(sys_clear_type_cache__doc__, +"_clear_type_cache() -> None\n\ +Clear the internal type lookup cache."); + + +static PyObject * +sys_compact_freelists(PyObject* self, PyObject* args) +{ + size_t fsum, fbc, fbf; + + PyFloat_CompactFreeList(&fbc, &fbf, &fsum); + + return Py_BuildValue("((kkk))", fsum, fbc, fbf); + +} + +PyDoc_STRVAR(sys_compact_freelists__doc__, +"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks),)\n\ +Compact the free lists of floats."); + static PyMethodDef sys_methods[] = { /* Might as well keep this in alphabetic order */ {"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS, callstats_doc}, - {"_cleartypecache", sys_cleartypecache, METH_NOARGS, - cleartypecache_doc}, + {"_clear_type_cache", sys_clear_type_cache, METH_NOARGS, + sys_clear_type_cache__doc__}, + {"_compact_freelists", sys_compact_freelists, METH_NOARGS, + sys_compact_freelists__doc__}, {"_current_frames", sys_current_frames, METH_NOARGS, current_frames_doc}, {"displayhook", sys_displayhook, METH_O, displayhook_doc}, -- cgit v0.12