diff options
35 files changed, 483 insertions, 171 deletions
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index e37d57e..7324f31 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -670,6 +670,13 @@ always available. .. versionadded:: 3.4 +.. function:: getunicodeinternedsize() + + Return the number of unicode objects that have been interned. + + .. versionadded:: 3.12 + + .. function:: getandroidapilevel() Return the build time API version of Android as an integer. diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index f940665..b98b715 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1129,6 +1129,24 @@ New Features to replace the legacy-api :c:func:`!PyErr_Display`. (Contributed by Irit Katriel in :gh:`102755`). +* :pep:`683`: Introduced Immortal Objects to Python which allows objects + to bypass reference counts and introduced changes to the C-API: + + - ``_Py_IMMORTAL_REFCNT``: The reference count that defines an object + as immortal. + - ``_Py_IsImmortal`` Checks if an object has the immortal reference count. + - ``PyObject_HEAD_INIT`` This will now initialize reference count to + ``_Py_IMMORTAL_REFCNT`` when used with ``Py_BUILD_CORE``. + - ``SSTATE_INTERNED_IMMORTAL`` An identifier for interned unicode objects + that are immortal. + - ``SSTATE_INTERNED_IMMORTAL_STATIC`` An identifier for interned unicode + objects that are immortal and static + - ``sys.getunicodeinternedsize`` This returns the total number of unicode + objects that have been interned. This is now needed for refleak.py to + correctly track reference counts and allocated blocks + + (Contributed by Eddie Elizondo in :gh:`84436`.) + Porting to Python 3.12 ---------------------- @@ -1293,8 +1311,7 @@ Removed * :c:func:`!PyUnicode_GetSize` * :c:func:`!PyUnicode_GET_DATA_SIZE` -* Remove the ``PyUnicode_InternImmortal()`` function and the - ``SSTATE_INTERNED_IMMORTAL`` macro. +* Remove the ``PyUnicode_InternImmortal()`` function macro. (Contributed by Victor Stinner in :gh:`85858`.) * Remove ``Jython`` compatibility hacks from several stdlib modules and tests. diff --git a/Include/boolobject.h b/Include/boolobject.h index ca21fbf..976fa35 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -11,8 +11,7 @@ PyAPI_DATA(PyTypeObject) PyBool_Type; #define PyBool_Check(x) Py_IS_TYPE((x), &PyBool_Type) -/* Py_False and Py_True are the only two bools in existence. -Don't forget to apply Py_INCREF() when returning either!!! */ +/* Py_False and Py_True are the only two bools in existence. */ /* Don't use these directly */ PyAPI_DATA(PyLongObject) _Py_FalseStruct; @@ -31,8 +30,8 @@ PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) /* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_NewRef(Py_True) -#define Py_RETURN_FALSE return Py_NewRef(Py_False) +#define Py_RETURN_TRUE return Py_True +#define Py_RETURN_FALSE return Py_False /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 75a74ff..3394726 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -98,9 +98,16 @@ typedef struct { Py_ssize_t length; /* Number of code points in the string */ Py_hash_t hash; /* Hash value; -1 if not set */ struct { - /* If interned is set, the two references from the - dictionary to this object are *not* counted in ob_refcnt. */ - unsigned int interned:1; + /* If interned is non-zero, the two references from the + dictionary to this object are *not* counted in ob_refcnt. + The possible values here are: + 0: Not Interned + 1: Interned + 2: Interned and Immortal + 3: Interned, Immortal, and Static + This categorization allows the runtime to determine the right + cleanup mechanism at runtime shutdown. */ + unsigned int interned:2; /* Character size: - PyUnicode_1BYTE_KIND (1): @@ -135,7 +142,7 @@ typedef struct { unsigned int ascii:1; /* Padding to ensure that PyUnicode_DATA() is always aligned to 4 bytes (see issue #19537 on m68k). */ - unsigned int :26; + unsigned int :25; } state; } PyASCIIObject; @@ -183,6 +190,8 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( /* Interning state. */ #define SSTATE_NOT_INTERNED 0 #define SSTATE_INTERNED_MORTAL 1 +#define SSTATE_INTERNED_IMMORTAL 2 +#define SSTATE_INTERNED_IMMORTAL_STATIC 3 /* Use only if you know it's a string */ static inline unsigned int PyUnicode_CHECK_INTERNED(PyObject *op) { diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 14dfd9e..fdfa80b 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -8,15 +8,13 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_object.h" // _PyObject_IMMORTAL_REFCNT - #ifdef Py_DEBUG static inline void _PyStaticObject_CheckRefcnt(PyObject *obj) { - if (Py_REFCNT(obj) < _PyObject_IMMORTAL_REFCNT) { + if (Py_REFCNT(obj) < _Py_IMMORTAL_REFCNT) { _PyObject_ASSERT_FAILED_MSG(obj, "immortal object has less refcnt than expected " - "_PyObject_IMMORTAL_REFCNT"); + "_Py_IMMORTAL_REFCNT"); } } #endif diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index 137a046..fe86581 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -245,7 +245,7 @@ _PyLong_FlipSign(PyLongObject *op) { #define _PyLong_DIGIT_INIT(val) \ { \ - .ob_base = _PyObject_IMMORTAL_INIT(&PyLong_Type), \ + .ob_base = _PyObject_HEAD_INIT(&PyLong_Type) \ .long_value = { \ .lv_tag = TAG_FROM_SIGN_AND_SIZE( \ (val) == 0 ? 0 : ((val) < 0 ? -1 : 1), \ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b3d496e..2ca0478 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,21 +14,25 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime -/* This value provides *effective* immortality, meaning the object should never - be deallocated (until runtime finalization). See PEP 683 for more details about - immortality, as well as a proposed mechanism for proper immortality. */ -#define _PyObject_IMMORTAL_REFCNT 999999999 - -#define _PyObject_IMMORTAL_INIT(type) \ - { \ - .ob_refcnt = _PyObject_IMMORTAL_REFCNT, \ - .ob_type = (type), \ - } -#define _PyVarObject_IMMORTAL_INIT(type, size) \ - { \ - .ob_base = _PyObject_IMMORTAL_INIT(type), \ - .ob_size = size, \ - } +/* We need to maintain an internal copy of Py{Var}Object_HEAD_INIT to avoid + designated initializer conflicts in C++20. If we use the deinition in + object.h, we will be mixing designated and non-designated initializers in + pycore objects which is forbiddent in C++20. However, if we then use + designated initializers in object.h then Extensions without designated break. + Furthermore, we can't use designated initializers in Extensions since these + are not supported pre-C++20. Thus, keeping an internal copy here is the most + backwards compatible solution */ +#define _PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + .ob_refcnt = _Py_IMMORTAL_REFCNT, \ + .ob_type = (type) \ + }, +#define _PyVarObject_HEAD_INIT(type, size) \ + { \ + .ob_base = _PyObject_HEAD_INIT(type) \ + .ob_size = size \ + }, PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( const char *func, @@ -61,9 +65,20 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) } #define _Py_RefcntAdd(op, n) _Py_RefcntAdd(_PyObject_CAST(op), n) +static inline void _Py_SetImmortal(PyObject *op) +{ + if (op) { + op->ob_refcnt = _Py_IMMORTAL_REFCNT; + } +} +#define _Py_SetImmortal(op) _Py_SetImmortal(_PyObject_CAST(op)) + static inline void _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); @@ -82,6 +97,9 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) static inline void _Py_DECREF_NO_DEALLOC(PyObject *op) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_DEC_REFTOTAL(_PyInterpreterState_GET()); diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 5b09a45..d8425b3 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -76,13 +76,13 @@ extern PyTypeObject _PyExc_MemoryError; .latin1 = _Py_str_latin1_INIT, \ }, \ .tuple_empty = { \ - .ob_base = _PyVarObject_IMMORTAL_INIT(&PyTuple_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&PyTuple_Type, 0) \ }, \ .hamt_bitmap_node_empty = { \ - .ob_base = _PyVarObject_IMMORTAL_INIT(&_PyHamt_BitmapNode_Type, 0) \ + .ob_base = _PyVarObject_HEAD_INIT(&_PyHamt_BitmapNode_Type, 0) \ }, \ .context_token_missing = { \ - .ob_base = _PyObject_IMMORTAL_INIT(&_PyContextTokenMissing_Type), \ + .ob_base = _PyObject_HEAD_INIT(&_PyContextTokenMissing_Type) \ }, \ }, \ }, \ @@ -116,11 +116,11 @@ extern PyTypeObject _PyExc_MemoryError; .singletons = { \ ._not_used = 1, \ .hamt_empty = { \ - .ob_base = _PyObject_IMMORTAL_INIT(&_PyHamt_Type), \ + .ob_base = _PyObject_HEAD_INIT(&_PyHamt_Type) \ .h_root = (PyHamtNode*)&_Py_SINGLETON(hamt_bitmap_node_empty), \ }, \ .last_resort_memory_error = { \ - _PyObject_IMMORTAL_INIT(&_PyExc_MemoryError), \ + _PyObject_HEAD_INIT(&_PyExc_MemoryError) \ }, \ }, \ }, \ @@ -138,7 +138,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyBytes_SIMPLE_INIT(CH, LEN) \ { \ - _PyVarObject_IMMORTAL_INIT(&PyBytes_Type, (LEN)), \ + _PyVarObject_HEAD_INIT(&PyBytes_Type, (LEN)) \ .ob_shash = -1, \ .ob_sval = { (CH) }, \ } @@ -149,7 +149,7 @@ extern PyTypeObject _PyExc_MemoryError; #define _PyUnicode_ASCII_BASE_INIT(LITERAL, ASCII) \ { \ - .ob_base = _PyObject_IMMORTAL_INIT(&PyUnicode_Type), \ + .ob_base = _PyObject_HEAD_INIT(&PyUnicode_Type) \ .length = sizeof(LITERAL) - 1, \ .hash = -1, \ .state = { \ diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index ff97b9a..1bb0f36 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -12,6 +12,7 @@ extern "C" { #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); +Py_ssize_t _PyUnicode_InternedSize(void); /* runtime lifecycle */ diff --git a/Include/object.h b/Include/object.h index 2943a60..66c3df0 100644 --- a/Include/object.h +++ b/Include/object.h @@ -78,12 +78,76 @@ whose size is determined when the object is allocated. /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; -#define PyObject_HEAD_INIT(type) \ - { _PyObject_EXTRA_INIT \ - 1, (type) }, +/* +Immortalization: + +The following indicates the immortalization strategy depending on the amount +of available bits in the reference count field. All strategies are backwards +compatible but the specific reference count value or immortalization check +might change depending on the specializations for the underlying system. + +Proper deallocation of immortal instances requires distinguishing between +statically allocated immortal instances vs those promoted by the runtime to be +immortal. The latter should be the only instances that require +cleanup during runtime finalization. +*/ + +#if SIZEOF_VOID_P > 4 +/* +In 64+ bit systems, an object will be marked as immortal by setting all of the +lower 32 bits of the reference count field, which is equal to: 0xFFFFFFFF + +Using the lower 32 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases will use saturated arithmetic, taking advantage of +having all the lower 32 bits set, which will avoid the reference count to go +beyond the refcount limit. Immortality checks for reference count decreases will +be done by checking the bit sign flag in the lower 32 bits. +*/ +#define _Py_IMMORTAL_REFCNT UINT_MAX + +#else +/* +In 32 bit systems, an object will be marked as immortal by setting all of the +lower 30 bits of the reference count field, which is equal to: 0x3FFFFFFF -#define PyVarObject_HEAD_INIT(type, size) \ - { PyObject_HEAD_INIT(type) (size) }, +Using the lower 30 bits makes the value backwards compatible by allowing +C-Extensions without the updated checks in Py_INCREF and Py_DECREF to safely +increase and decrease the objects reference count. The object would lose its +immortality, but the execution would still be correct. + +Reference count increases and decreases will first go through an immortality +check by comparing the reference count field to the immortality reference count. +*/ +#define _Py_IMMORTAL_REFCNT (UINT_MAX >> 2) +#endif + +// Make all internal uses of PyObject_HEAD_INIT immortal while preserving the +// C-API expectation that the refcnt will be set to 1. +#ifdef Py_BUILD_CORE +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { _Py_IMMORTAL_REFCNT }, \ + (type) \ + }, +#else +#define PyObject_HEAD_INIT(type) \ + { \ + _PyObject_EXTRA_INIT \ + { 1 }, \ + (type) \ + }, +#endif /* Py_BUILD_CORE */ + +#define PyVarObject_HEAD_INIT(type, size) \ + { \ + PyObject_HEAD_INIT(type) \ + (size) \ + }, /* PyObject_VAR_HEAD defines the initial segment of all variable-size * container objects. These end with a declaration of an array with 1 @@ -101,7 +165,12 @@ whose size is determined when the object is allocated. */ struct _object { _PyObject_HEAD_EXTRA - Py_ssize_t ob_refcnt; + union { + Py_ssize_t ob_refcnt; +#if SIZEOF_VOID_P > 4 + PY_UINT32_T ob_refcnt_split[2]; +#endif + }; PyTypeObject *ob_type; }; @@ -152,6 +221,15 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif +static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) +{ +#if SIZEOF_VOID_P > 4 + return _Py_CAST(PY_INT32_T, op->ob_refcnt) < 0; +#else + return op->ob_refcnt == _Py_IMMORTAL_REFCNT; +#endif +} +#define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; @@ -162,6 +240,13 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { + // This immortal check is for code that is unaware of immortal objects. + // The runtime tracks these objects and we should avoid as much + // as possible having extensions inadvertently change the refcnt + // of an immortalized object. + if (_Py_IsImmortal(ob)) { + return; + } ob->ob_refcnt = refcnt; } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 @@ -524,19 +609,33 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -static inline void Py_INCREF(PyObject *op) +static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000 // Stable ABI for Python 3.10 built in debug mode. _Py_IncRef(op); #else - _Py_INCREF_STAT_INC(); // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. +#if SIZEOF_VOID_P > 4 + // Portable saturated add, branching on the carry flag and set low bits + PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T new_refcnt = cur_refcnt + 1; + if (new_refcnt == 0) { + return; + } + op->ob_refcnt_split[PY_BIG_ENDIAN] = new_refcnt; +#else + // Explicitly check immortality against the immortal value + if (_Py_IsImmortal(op)) { + return; + } + op->ob_refcnt++; +#endif + _Py_INCREF_STAT_INC(); #ifdef Py_REF_DEBUG _Py_INC_REFTOTAL(); -#endif // Py_REF_DEBUG - op->ob_refcnt++; +#endif #endif } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 @@ -553,6 +652,9 @@ static inline void Py_DECREF(PyObject *op) { #elif defined(Py_REF_DEBUG) static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { + if (_Py_IsImmortal(op)) { + return; + } _Py_DECREF_STAT_INC(); _Py_DEC_REFTOTAL(); if (--op->ob_refcnt != 0) { @@ -567,11 +669,14 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) #define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op)) #else -static inline void Py_DECREF(PyObject *op) +static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op) { - _Py_DECREF_STAT_INC(); // Non-limited C API and limited C API for Python 3.9 and older access // directly PyObject.ob_refcnt. + if (_Py_IsImmortal(op)) { + return; + } + _Py_DECREF_STAT_INC(); if (--op->ob_refcnt == 0) { _Py_Dealloc(op); } @@ -721,7 +826,7 @@ PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) /* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_NewRef(Py_None) +#define Py_RETURN_NONE return Py_None /* Py_NotImplemented is a singleton used to signal that an operation is @@ -731,7 +836,7 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented /* Rich comparison opcodes */ #define Py_LT 0 diff --git a/Include/pyport.h b/Include/pyport.h index eef0fe1..5e226f5 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -184,7 +184,6 @@ typedef Py_ssize_t Py_ssize_clean_t; # define Py_LOCAL_INLINE(type) static inline type #endif -// bpo-28126: Py_MEMCPY is kept for backwards compatibility, #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_MEMCPY memcpy #endif diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 868f9f8..834daa4 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -1,27 +1,31 @@ import sys import types -import unittest +# Note: This test file can't import `unittest` since the runtime can't +# currently guarantee that it will not leak memory. Doing so will mark +# the test as passing but with reference leaks. This can safely import +# the `unittest` library once there's a strict guarantee of no leaks +# during runtime shutdown. # bpo-46417: Test that structseq types used by the sys module are still # valid when Py_Finalize()/Py_Initialize() are called multiple times. -class TestStructSeq(unittest.TestCase): +class TestStructSeq: # test PyTypeObject members - def check_structseq(self, obj_type): + def _check_structseq(self, obj_type): # ob_refcnt - self.assertGreaterEqual(sys.getrefcount(obj_type), 1) + assert sys.getrefcount(obj_type) > 1 # tp_base - self.assertTrue(issubclass(obj_type, tuple)) + assert issubclass(obj_type, tuple) # tp_bases - self.assertEqual(obj_type.__bases__, (tuple,)) + assert obj_type.__bases__ == (tuple,) # tp_dict - self.assertIsInstance(obj_type.__dict__, types.MappingProxyType) + assert isinstance(obj_type.__dict__, types.MappingProxyType) # tp_mro - self.assertEqual(obj_type.__mro__, (obj_type, tuple, object)) + assert obj_type.__mro__ == (obj_type, tuple, object) # tp_name - self.assertIsInstance(type.__name__, str) + assert isinstance(type.__name__, str) # tp_subclasses - self.assertEqual(obj_type.__subclasses__(), []) + assert obj_type.__subclasses__() == [] def test_sys_attrs(self): for attr_name in ( @@ -32,23 +36,23 @@ class TestStructSeq(unittest.TestCase): 'thread_info', # ThreadInfoType 'version_info', # VersionInfoType ): - with self.subTest(attr=attr_name): - attr = getattr(sys, attr_name) - self.check_structseq(type(attr)) + attr = getattr(sys, attr_name) + self._check_structseq(type(attr)) def test_sys_funcs(self): func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType if hasattr(sys, 'getwindowsversion'): func_names.append('getwindowsversion') # WindowsVersionType for func_name in func_names: - with self.subTest(func=func_name): - func = getattr(sys, func_name) - obj = func() - self.check_structseq(type(obj)) + func = getattr(sys, func_name) + obj = func() + self._check_structseq(type(obj)) try: - unittest.main() + tests = TestStructSeq() + tests.test_sys_attrs() + tests.test_sys_funcs() except SystemExit as exc: if exc.args[0] != 0: raise diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index 4298fa8..2de8c6c 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -73,9 +73,10 @@ def dash_R(ns, test_name, test_func): fd_deltas = [0] * repcount getallocatedblocks = sys.getallocatedblocks gettotalrefcount = sys.gettotalrefcount + getunicodeinternedsize = sys.getunicodeinternedsize fd_count = os_helper.fd_count # initialize variables to make pyflakes quiet - rc_before = alloc_before = fd_before = 0 + rc_before = alloc_before = fd_before = interned_before = 0 if not ns.quiet: print("beginning", repcount, "repetitions", file=sys.stderr) @@ -91,9 +92,13 @@ def dash_R(ns, test_name, test_func): dash_R_cleanup(fs, ps, pic, zdc, abcs) support.gc_collect() - # Read memory statistics immediately after the garbage collection - alloc_after = getallocatedblocks() - rc_after = gettotalrefcount() + # Read memory statistics immediately after the garbage collection. + # Also, readjust the reference counts and alloc blocks by ignoring + # any strings that might have been interned during test_func. These + # strings will be deallocated at runtime shutdown + interned_after = getunicodeinternedsize() + alloc_after = getallocatedblocks() - interned_after + rc_after = gettotalrefcount() - interned_after * 2 fd_after = fd_count() if not ns.quiet: @@ -106,6 +111,7 @@ def dash_R(ns, test_name, test_func): alloc_before = alloc_after rc_before = rc_after fd_before = fd_after + interned_before = interned_after if not ns.quiet: print(file=sys.stderr) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e7a79bc..04dd8ff 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -28,7 +28,7 @@ from textwrap import dedent from types import AsyncGeneratorType, FunctionType, CellType from operator import neg from test import support -from test.support import (swap_attr, maybe_get_event_loop_policy) +from test.support import (cpython_only, swap_attr, maybe_get_event_loop_policy) from test.support.os_helper import (EnvironmentVarGuard, TESTFN, unlink) from test.support.script_helper import assert_python_ok from test.support.warnings_helper import check_warnings @@ -2370,6 +2370,28 @@ class ShutdownTest(unittest.TestCase): self.assertEqual(["before", "after"], out.decode().splitlines()) +@cpython_only +class ImmortalTests(unittest.TestCase): + def test_immortal(self): + none_refcount = sys.getrefcount(None) + true_refcount = sys.getrefcount(True) + false_refcount = sys.getrefcount(False) + smallint_refcount = sys.getrefcount(100) + + # Assert that all of these immortal instances have large ref counts. + self.assertGreater(none_refcount, 2 ** 15) + self.assertGreater(true_refcount, 2 ** 15) + self.assertGreater(false_refcount, 2 ** 15) + self.assertGreater(smallint_refcount, 2 ** 15) + + # Confirm that the refcount doesn't change even with a new ref to them. + l = [None, True, False, 100] + self.assertEqual(sys.getrefcount(None), none_refcount) + self.assertEqual(sys.getrefcount(True), true_refcount) + self.assertEqual(sys.getrefcount(False), false_refcount) + self.assertEqual(sys.getrefcount(100), smallint_refcount) + + class TestType(unittest.TestCase): def test_new_type(self): A = type('A', (), {}) diff --git a/Lib/test/test_ctypes/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py index 49571f9..de8989e 100644 --- a/Lib/test/test_ctypes/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -46,7 +46,8 @@ class PythonAPITestCase(unittest.TestCase): pythonapi.PyLong_AsLong.restype = c_long res = pythonapi.PyLong_AsLong(42) - self.assertEqual(grc(res), ref42 + 1) + # Small int refcnts don't change + self.assertEqual(grc(res), ref42) del res self.assertEqual(grc(42), ref42) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 1aebe1b..611cd27 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -385,7 +385,8 @@ class SysModuleTest(unittest.TestCase): self.assertRaises(TypeError, sys.getrefcount) c = sys.getrefcount(None) n = None - self.assertEqual(sys.getrefcount(None), c+1) + # Singleton refcnts don't change + self.assertEqual(sys.getrefcount(None), c) del n self.assertEqual(sys.getrefcount(None), c) if hasattr(sys, "gettotalrefcount"): diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 333b976..95944c7 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -600,9 +600,15 @@ class BasicTest(BaseTest): ld_library_path_env = "DYLD_LIBRARY_PATH" else: ld_library_path_env = "LD_LIBRARY_PATH" - subprocess.check_call(cmd, - env={"PYTHONPATH": pythonpath, - ld_library_path_env: ld_library_path}) + # Note that in address sanitizer mode, the current runtime + # implementation leaks memory due to not being able to correctly + # clean all unicode objects during runtime shutdown. Therefore, + # this uses subprocess.run instead of subprocess.check_call to + # maintain the core of the test while not failing due to the refleaks. + # This should be able to use check_call once all refleaks are fixed. + subprocess.run(cmd, + env={"PYTHONPATH": pythonpath, + ld_library_path_env: ld_library_path}) envpy = os.path.join(self.env_dir, self.bindir, self.exe) # Now check the venv created from the non-installed python has # correct zip path in pythonpath. diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst b/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst new file mode 100644 index 0000000..c4d8ce7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-04-02-22-14-57.gh-issue-84436.hvMgwF.rst @@ -0,0 +1,3 @@ +The implementation of PEP-683 which adds Immortal Objects by using a fixed +reference count that skips reference counting to make objects truly +immutable. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 4eaa549..1d00fc3 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -418,8 +418,20 @@ validate_list(PyGC_Head *head, enum flagstates flags) static void update_refs(PyGC_Head *containers) { + PyGC_Head *next; PyGC_Head *gc = GC_NEXT(containers); - for (; gc != containers; gc = GC_NEXT(gc)) { + + while (gc != containers) { + next = GC_NEXT(gc); + /* Move any object that might have become immortal to the + * permanent generation as the reference count is not accurately + * reflecting the actual number of live references to this object + */ + if (_Py_IsImmortal(FROM_GC(gc))) { + gc_list_move(gc, &get_gc_state()->permanent_generation.head); + gc = next; + continue; + } gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been @@ -440,6 +452,7 @@ update_refs(PyGC_Head *containers) * check instead of an assert? */ _PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0); + gc = next; } } diff --git a/Objects/boolobject.c b/Objects/boolobject.c index 9d8e956..597a76f 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -145,10 +145,14 @@ static PyNumberMethods bool_as_number = { 0, /* nb_index */ }; -static void _Py_NO_RETURN -bool_dealloc(PyObject* Py_UNUSED(ignore)) +static void +bool_dealloc(PyObject *boolean) { - _Py_FatalRefcountError("deallocating True or False"); + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref Booleans out of existence. Instead, + * since bools are immortal, re-set the reference count. + */ + _Py_SetImmortal(boolean); } /* The type object for bool. Note that this cannot be subclassed! */ diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index ef9e65e..33aa9c3 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -258,9 +258,12 @@ _Py_bytes_istitle(const char *cptr, Py_ssize_t len) const unsigned char *e; int cased, previous_is_cased; - /* Shortcut for single character strings */ - if (len == 1) - return PyBool_FromLong(Py_ISUPPER(*p)); + if (len == 1) { + if (Py_ISUPPER(*p)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; + } /* Special case for empty strings */ if (len == 0) diff --git a/Objects/longobject.c b/Objects/longobject.c index bb4eac0..d98bbbb 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -52,8 +52,7 @@ static PyObject * get_small_int(sdigit ival) { assert(IS_SMALL_INT(ival)); - PyObject *v = (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival]; - return Py_NewRef(v); + return (PyObject *)&_PyLong_SMALL_INTS[_PY_NSMALLNEGINTS + ival]; } static PyLongObject * @@ -3271,6 +3270,27 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } +static void +long_dealloc(PyObject *self) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref small Ints out of existence. Instead, + * since small Ints are immortal, re-set the reference count. + */ + PyLongObject *pylong = (PyLongObject*)self; + if (pylong && _PyLong_IsCompact(pylong)) { + stwodigits ival = medium_value(pylong); + if (IS_SMALL_INT(ival)) { + PyLongObject *small_pylong = (PyLongObject *)get_small_int((sdigit)ival); + if (pylong == small_pylong) { + _Py_SetImmortal(self); + return; + } + } + } + Py_TYPE(self)->tp_free(self); +} + static Py_hash_t long_hash(PyLongObject *v) { @@ -6233,7 +6253,7 @@ PyTypeObject PyLong_Type = { "int", /* tp_name */ offsetof(PyLongObject, long_value.ob_digit), /* tp_basicsize */ sizeof(digit), /* tp_itemsize */ - 0, /* tp_dealloc */ + long_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ diff --git a/Objects/object.c b/Objects/object.c index e26f737..e508881 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1754,10 +1754,14 @@ none_repr(PyObject *op) return PyUnicode_FromString("None"); } -static void _Py_NO_RETURN -none_dealloc(PyObject* Py_UNUSED(ignore)) +static void +none_dealloc(PyObject* none) { - _Py_FatalRefcountError("deallocating None"); + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref None out of existence. Instead, + * since None is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(none); } static PyObject * @@ -1823,7 +1827,7 @@ PyTypeObject _PyNone_Type = { "NoneType", 0, 0, - none_dealloc, /*tp_dealloc*/ /*never called*/ + none_dealloc, /*tp_dealloc*/ 0, /*tp_vectorcall_offset*/ 0, /*tp_getattr*/ 0, /*tp_setattr*/ @@ -1860,8 +1864,9 @@ PyTypeObject _PyNone_Type = { }; PyObject _Py_NoneStruct = { - _PyObject_EXTRA_INIT - 1, &_PyNone_Type + _PyObject_EXTRA_INIT + { _Py_IMMORTAL_REFCNT }, + &_PyNone_Type }; /* NotImplemented is an object that can be used to signal that an @@ -1894,13 +1899,14 @@ notimplemented_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) Py_RETURN_NOTIMPLEMENTED; } -static void _Py_NO_RETURN -notimplemented_dealloc(PyObject* ignore) +static void +notimplemented_dealloc(PyObject *notimplemented) { /* This should never get called, but we also don't want to SEGV if - * we accidentally decref NotImplemented out of existence. + * we accidentally decref NotImplemented out of existence. Instead, + * since Notimplemented is an immortal object, re-set the reference count. */ - Py_FatalError("deallocating NotImplemented"); + _Py_SetImmortal(notimplemented); } static int @@ -1962,7 +1968,8 @@ PyTypeObject _PyNotImplemented_Type = { PyObject _Py_NotImplementedStruct = { _PyObject_EXTRA_INIT - 1, &_PyNotImplemented_Type + { _Py_IMMORTAL_REFCNT }, + &_PyNotImplemented_Type }; extern PyTypeObject _Py_GenericAliasIterType; @@ -2143,7 +2150,8 @@ new_reference(PyObject *op) if (_PyRuntime.tracemalloc.config.tracing) { _PyTraceMalloc_NewReference(op); } - Py_SET_REFCNT(op, 1); + // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 + op->ob_refcnt = 1; #ifdef Py_TRACE_REFS _Py_AddToAllObjects(op, 1); #endif diff --git a/Objects/setobject.c b/Objects/setobject.c index fcdda2a..58f0ae7 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -2543,6 +2543,7 @@ static PyTypeObject _PySetDummy_Type = { }; static PyObject _dummy_struct = { - _PyObject_EXTRA_INIT - 2, &_PySetDummy_Type + _PyObject_EXTRA_INIT + { _Py_IMMORTAL_REFCNT }, + &_PySetDummy_Type }; diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 584ebce..e6776ac 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -29,6 +29,16 @@ ellipsis_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return Py_NewRef(Py_Ellipsis); } +static void +ellipsis_dealloc(PyObject *ellipsis) +{ + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref Ellipsis out of existence. Instead, + * since Ellipsis is an immortal object, re-set the reference count. + */ + _Py_SetImmortal(ellipsis); +} + static PyObject * ellipsis_repr(PyObject *op) { @@ -51,7 +61,7 @@ PyTypeObject PyEllipsis_Type = { "ellipsis", /* tp_name */ 0, /* tp_basicsize */ 0, /* tp_itemsize */ - 0, /*never called*/ /* tp_dealloc */ + ellipsis_dealloc, /* tp_dealloc */ 0, /* tp_vectorcall_offset */ 0, /* tp_getattr */ 0, /* tp_setattr */ @@ -89,7 +99,8 @@ PyTypeObject PyEllipsis_Type = { PyObject _Py_EllipsisObject = { _PyObject_EXTRA_INIT - 1, &PyEllipsis_Type + { _Py_IMMORTAL_REFCNT }, + &PyEllipsis_Type }; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 9ea458f..85bcd05 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -318,27 +318,11 @@ _PyType_InitCache(PyInterpreterState *interp) entry->version = 0; // Set to None so _PyType_Lookup() can use Py_SETREF(), // rather than using slower Py_XSETREF(). - // (See _PyType_FixCacheRefcounts() about the refcount.) entry->name = Py_None; entry->value = NULL; } } -// This is the temporary fix used by pycore_create_interpreter(), -// in pylifecycle.c. _PyType_InitCache() is called before the GIL -// has been created (for the main interpreter) and without the -// "current" thread state set. This causes crashes when the -// reftotal is updated, so we don't modify the refcount in -// _PyType_InitCache(), and instead do it later by calling -// _PyType_FixCacheRefcounts(). -// XXX This workaround should be removed once we have immortal -// objects (PEP 683). -void -_PyType_FixCacheRefcounts(void) -{ - _Py_RefcntAdd(Py_None, (1 << MCACHE_SIZE_EXP)); -} - static unsigned int _PyType_ClearCache(PyInterpreterState *interp) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 85e5ae7..fd056e3 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -228,14 +228,18 @@ static inline PyObject* unicode_new_empty(void) to strings in this dictionary are *not* counted in the string's ob_refcnt. When the interned string reaches a refcnt of 0 the string deallocation function will delete the reference from this dictionary. - Another way to look at this is that to say that the actual reference - count of a string is: s->ob_refcnt + (s->state ? 2 : 0) */ static inline PyObject *get_interned_dict(PyInterpreterState *interp) { return _Py_INTERP_CACHED_OBJECT(interp, interned_strings); } +Py_ssize_t +_PyUnicode_InternedSize() +{ + return PyObject_Length(get_interned_dict(_PyInterpreterState_GET())); +} + static int init_interned_dict(PyInterpreterState *interp) { @@ -1538,30 +1542,19 @@ find_maxchar_surrogates(const wchar_t *begin, const wchar_t *end, static void unicode_dealloc(PyObject *unicode) { - PyInterpreterState *interp = _PyInterpreterState_GET(); #ifdef Py_DEBUG if (!unicode_is_finalizing() && unicode_is_singleton(unicode)) { _Py_FatalRefcountError("deallocating an Unicode singleton"); } #endif + /* This should never get called, but we also don't want to SEGV if + * we accidentally decref an immortal string out of existence. Since + * the string is an immortal object, just re-set the reference count. + */ if (PyUnicode_CHECK_INTERNED(unicode)) { - /* Revive the dead object temporarily. PyDict_DelItem() removes two - references (key and value) which were ignored by - PyUnicode_InternInPlace(). Use refcnt=3 rather than refcnt=2 - to prevent calling unicode_dealloc() again. Adjust refcnt after - PyDict_DelItem(). */ - assert(Py_REFCNT(unicode) == 0); - Py_SET_REFCNT(unicode, 3); - PyObject *interned = get_interned_dict(interp); - assert(interned != NULL); - if (PyDict_DelItem(interned, unicode) != 0) { - _PyErr_WriteUnraisableMsg("deletion of interned string failed", - NULL); - } - assert(Py_REFCNT(unicode) == 1); - Py_SET_REFCNT(unicode, 0); + _Py_SetImmortal(unicode); + return; } - if (_PyUnicode_HAS_UTF8_MEMORY(unicode)) { PyObject_Free(_PyUnicode_UTF8(unicode)); } @@ -14637,11 +14630,21 @@ _PyUnicode_InternInPlace(PyInterpreterState *interp, PyObject **p) return; } - /* The two references in interned dict (key and value) are not counted by - refcnt. unicode_dealloc() and _PyUnicode_ClearInterned() take care of - this. */ - Py_SET_REFCNT(s, Py_REFCNT(s) - 2); - _PyUnicode_STATE(s).interned = 1; + if (_Py_IsImmortal(s)) { + _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL_STATIC; + return; + } +#ifdef Py_REF_DEBUG + /* The reference count value excluding the 2 references from the + interned dictionary should be excluded from the RefTotal. The + decrements to these objects will not be registered so they + need to be accounted for in here. */ + for (Py_ssize_t i = 0; i < Py_REFCNT(s) - 2; i++) { + _Py_DecRefTotal(_PyInterpreterState_GET()); + } +#endif + _Py_SetImmortal(s); + _PyUnicode_STATE(*p).interned = SSTATE_INTERNED_IMMORTAL; } void @@ -14681,10 +14684,20 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) } assert(PyDict_CheckExact(interned)); - /* Interned unicode strings are not forcibly deallocated; rather, we give - them their stolen references back, and then clear and DECREF the - interned dict. */ - + /* TODO: + * Currently, the runtime is not able to guarantee that it can exit without + * allocations that carry over to a future initialization of Python within + * the same process. i.e: + * ./python -X showrefcount -c 'import itertools' + * [237 refs, 237 blocks] + * + * Therefore, this should remain disabled for until there is a strict guarantee + * that no memory will be left after `Py_Finalize`. + */ +#ifdef Py_DEBUG + /* For all non-singleton interned strings, restore the two valid references + to that instance from within the intern string dictionary and let the + normal reference counting process clean up these instances. */ #ifdef INTERNED_STATS fprintf(stderr, "releasing %zd interned strings\n", PyDict_GET_SIZE(interned)); @@ -14694,15 +14707,27 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) Py_ssize_t pos = 0; PyObject *s, *ignored_value; while (PyDict_Next(interned, &pos, &s, &ignored_value)) { - assert(PyUnicode_CHECK_INTERNED(s)); - // Restore the two references (key and value) ignored - // by PyUnicode_InternInPlace(). - Py_SET_REFCNT(s, Py_REFCNT(s) + 2); + assert(PyUnicode_IS_READY(s)); + switch (PyUnicode_CHECK_INTERNED(s)) { + case SSTATE_INTERNED_IMMORTAL: + // Skip the Immortal Instance check and restore + // the two references (key and value) ignored + // by PyUnicode_InternInPlace(). + s->ob_refcnt = 2; #ifdef INTERNED_STATS - total_length += PyUnicode_GET_LENGTH(s); + total_length += PyUnicode_GET_LENGTH(s); #endif - - _PyUnicode_STATE(s).interned = 0; + break; + case SSTATE_INTERNED_IMMORTAL_STATIC: + break; + case SSTATE_INTERNED_MORTAL: + /* fall through */ + case SSTATE_NOT_INTERNED: + /* fall through */ + default: + Py_UNREACHABLE(); + } + _PyUnicode_STATE(s).interned = SSTATE_NOT_INTERNED; } #ifdef INTERNED_STATS fprintf(stderr, @@ -14710,6 +14735,12 @@ _PyUnicode_ClearInterned(PyInterpreterState *interp) total_length); #endif + struct _Py_unicode_state *state = &interp->unicode; + struct _Py_unicode_ids *ids = &state->ids; + for (Py_ssize_t i=0; i < ids->size; i++) { + Py_XINCREF(ids->array[i]); + } +#endif /* Py_DEBUG */ clear_interned_dict(interp); } diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 0071711..f78ba41 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1911,14 +1911,13 @@ static int test_unicode_id_init(void) str1 = _PyUnicode_FromId(&PyId_test_unicode_id_init); assert(str1 != NULL); - assert(Py_REFCNT(str1) == 1); + assert(_Py_IsImmortal(str1)); str2 = PyUnicode_FromString("test_unicode_id_init"); assert(str2 != NULL); assert(PyUnicode_Compare(str1, str2) == 0); - // str1 is a borrowed reference Py_DECREF(str2); Py_Finalize(); diff --git a/Python/ceval.c b/Python/ceval.c index d8495da..3588350 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -53,8 +53,11 @@ #undef Py_DECREF #define Py_DECREF(arg) \ do { \ - _Py_DECREF_STAT_INC(); \ PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ if (--op->ob_refcnt == 0) { \ destructor dealloc = Py_TYPE(op)->tp_dealloc; \ (*dealloc)(op); \ @@ -77,8 +80,11 @@ #undef _Py_DECREF_SPECIALIZED #define _Py_DECREF_SPECIALIZED(arg, dealloc) \ do { \ - _Py_DECREF_STAT_INC(); \ PyObject *op = _PyObject_CAST(arg); \ + if (_Py_IsImmortal(op)) { \ + break; \ + } \ + _Py_DECREF_STAT_INC(); \ if (--op->ob_refcnt == 0) { \ destructor d = (destructor)(dealloc); \ d(op); \ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index 46252dd..7a7c188 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -912,6 +912,34 @@ exit: return return_value; } +PyDoc_STRVAR(sys_getunicodeinternedsize__doc__, +"getunicodeinternedsize($module, /)\n" +"--\n" +"\n" +"Return the number of elements of the unicode interned dictionary"); + +#define SYS_GETUNICODEINTERNEDSIZE_METHODDEF \ + {"getunicodeinternedsize", (PyCFunction)sys_getunicodeinternedsize, METH_NOARGS, sys_getunicodeinternedsize__doc__}, + +static Py_ssize_t +sys_getunicodeinternedsize_impl(PyObject *module); + +static PyObject * +sys_getunicodeinternedsize(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + _return_value = sys_getunicodeinternedsize_impl(module); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + PyDoc_STRVAR(sys__getframe__doc__, "_getframe($module, depth=0, /)\n" "--\n" @@ -1387,4 +1415,4 @@ exit: #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=5c761f14326ced54 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d598acc26237fbe input=a9049054013a1b77]*/ diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 2a3b2b8..8334f59 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -16,13 +16,13 @@ static PyObject DISABLE = { - .ob_refcnt = _PyObject_IMMORTAL_REFCNT, + .ob_refcnt = _Py_IMMORTAL_REFCNT, .ob_type = &PyBaseObject_Type }; PyObject _PyInstrumentation_MISSING = { - .ob_refcnt = _PyObject_IMMORTAL_REFCNT, + .ob_refcnt = _Py_IMMORTAL_REFCNT, .ob_type = &PyBaseObject_Type }; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index cf345bd..e509e63 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -324,7 +324,7 @@ sys_trace_exception_handled( PyTypeObject _PyLegacyEventHandler_Type = { - _PyVarObject_IMMORTAL_INIT(&PyType_Type, 0), + PyVarObject_HEAD_INIT(&PyType_Type, 0) "sys.legacy_event_handler", sizeof(_PyLegacyEventHandler), .tp_dealloc = (destructor)PyObject_Free, diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index d6627bc..a510c9b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -808,11 +808,6 @@ pycore_interp_init(PyThreadState *tstate) PyStatus status; PyObject *sysmod = NULL; - // This is a temporary fix until we have immortal objects. - // (See _PyType_InitCache() in typeobject.c.) - extern void _PyType_FixCacheRefcounts(void); - _PyType_FixCacheRefcounts(); - // Create singletons before the first PyType_Ready() call, since // PyType_Ready() uses singletons like the Unicode empty string (tp_doc) // and the empty tuple singletons (tp_bases). diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 4d693a1..1e42e8d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1874,6 +1874,18 @@ sys_getallocatedblocks_impl(PyObject *module) return _Py_GetAllocatedBlocks(); } +/*[clinic input] +sys.getunicodeinternedsize -> Py_ssize_t + +Return the number of elements of the unicode interned dictionary +[clinic start generated code]*/ + +static Py_ssize_t +sys_getunicodeinternedsize_impl(PyObject *module) +/*[clinic end generated code: output=ad0e4c9738ed4129 input=726298eaa063347a]*/ +{ + return _PyUnicode_InternedSize(); +} /*[clinic input] sys._getframe @@ -2243,6 +2255,7 @@ static PyMethodDef sys_methods[] = { SYS_GETDEFAULTENCODING_METHODDEF SYS_GETDLOPENFLAGS_METHODDEF SYS_GETALLOCATEDBLOCKS_METHODDEF + SYS_GETUNICODEINTERNEDSIZE_METHODDEF SYS_GETFILESYSTEMENCODING_METHODDEF SYS_GETFILESYSTEMENCODEERRORS_METHODDEF #ifdef Py_TRACE_REFS diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index aba5fec..5cfef5c 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -142,7 +142,7 @@ class Printer: def object_head(self, typename: str) -> None: with self.block(".ob_base =", ","): - self.write(f".ob_refcnt = 999999999,") + self.write(f".ob_refcnt = _Py_IMMORTAL_REFCNT,") self.write(f".ob_type = &{typename},") def object_var_head(self, typename: str, size: int) -> None: |