gh-108634: Py_TRACE_REFS uses a hash table (#108663)

Python built with "configure --with-trace-refs" (tracing references)
is now ABI compatible with Python release build and debug build.
Moreover, it now also supports the Limited API.

Change Py_TRACE_REFS build:

* Remove _PyObject_EXTRA_INIT macro.
* The PyObject structure no longer has two extra members (_ob_prev
  and _ob_next).
* Use a hash table (_Py_hashtable_t) to trace references (all
  objects): PyInterpreterState.object_state.refchain.
* Py_TRACE_REFS build is now ABI compatible with release build and
  debug build.
* Limited C API extensions can now be built with Py_TRACE_REFS:
  xxlimited, xxlimited_35, _testclinic_limited.
* No longer rename PyModule_Create2() and PyModule_FromDefAndSpec2()
  functions to PyModule_Create2TraceRefs() and
  PyModule_FromDefAndSpec2TraceRefs().
* _Py_PrintReferenceAddresses() is now called before
  finalize_interp_delete() which deletes the refchain hash table.
* test_tracemalloc find_trace() now also filters by size to ignore
  the memory allocated by _PyRefchain_Trace().

Test changes for Py_TRACE_REFS:

* Add test.support.Py_TRACE_REFS constant.
* Add test_sys.test_getobjects() to test sys.getobjects() function.
* test_exceptions skips test_recursion_normalizing_with_no_memory()
  and test_memory_error_in_PyErr_PrintEx() if Python is built with
  Py_TRACE_REFS.
* test_repl skips test_no_memory().
* test_capi skisp test_set_nomemory().

4 files changed, 17 insertions, 15 deletions

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 30dd717..971067e 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -3110,7 +3110,7 @@ _PyBuiltin_Init(PyInterpreterState *interp)
      * result, programs leaking references to None and False (etc)
      * couldn't be diagnosed by examining sys.getobjects(0).
      */
-#define ADD_TO_ALL(OBJECT) _Py_AddToAllObjects((PyObject *)(OBJECT), 0)
+#define ADD_TO_ALL(OBJECT) _Py_AddToAllObjects((PyObject *)(OBJECT))
 #else
 #define ADD_TO_ALL(OBJECT) (void)0
 #endif
diff --git a/Python/hashtable.c b/Python/hashtable.c
index 4e22a1a..8f5e816 100644
--- a/Python/hashtable.c
+++ b/Python/hashtable.c
@@ -226,7 +226,6 @@ _Py_hashtable_set(_Py_hashtable_t *ht, const void *key, void *value)
     assert(entry == NULL);
 #endif
 
-
     entry = ht->alloc.malloc(sizeof(_Py_hashtable_entry_t));
     if (entry == NULL) {
         /* memory allocation failed */
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 7d362af..ee5d498 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1956,31 +1956,30 @@ Py_FinalizeEx(void)
     // XXX Ensure finalizer errors are handled properly.
 
     finalize_interp_clear(tstate);
-    finalize_interp_delete(tstate->interp);
-
-#ifdef Py_REF_DEBUG
-    if (show_ref_count) {
-        _PyDebug_PrintTotalRefs();
-    }
-    _Py_FinalizeRefTotal(runtime);
-#endif
-    _Py_FinalizeAllocatedBlocks(runtime);
 
 #ifdef Py_TRACE_REFS
     /* Display addresses (& refcnts) of all objects still alive.
      * An address can be used to find the repr of the object, printed
-     * above by _Py_PrintReferences.
-     */
-
+     * above by _Py_PrintReferences. */
     if (dump_refs) {
         _Py_PrintReferenceAddresses(tstate->interp, stderr);
     }
-
     if (dump_refs_fp != NULL) {
         _Py_PrintReferenceAddresses(tstate->interp, dump_refs_fp);
         fclose(dump_refs_fp);
     }
 #endif /* Py_TRACE_REFS */
+
+    finalize_interp_delete(tstate->interp);
+
+#ifdef Py_REF_DEBUG
+    if (show_ref_count) {
+        _PyDebug_PrintTotalRefs();
+    }
+    _Py_FinalizeRefTotal(runtime);
+#endif
+    _Py_FinalizeAllocatedBlocks(runtime);
+
 #ifdef WITH_PYMALLOC
     if (malloc_stats) {
         _PyObject_DebugMallocStats(stderr);
diff --git a/Python/pystate.c b/Python/pystate.c
index 01651d7..4a8808f 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -674,6 +674,7 @@ init_interpreter(PyInterpreterState *interp,
                 _obmalloc_pools_INIT(interp->obmalloc.pools);
         memcpy(&interp->obmalloc.pools.used, temp, sizeof(temp));
     }
+
     _PyObject_InitState(interp);
 
     _PyEval_InitState(interp, pending_lock);
@@ -1001,6 +1002,9 @@ PyInterpreterState_Delete(PyInterpreterState *interp)
     if (interp->id_mutex != NULL) {
         PyThread_free_lock(interp->id_mutex);
     }
+
+    _PyObject_FiniState(interp);
+
     free_interpreter(interp);
 }