From 5ac9e6eee5ed18172d70d28cf438df0be4e3b83d Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Tue, 10 Jul 2018 17:19:53 +0900 Subject: bpo-33597: Reduce PyGC_Head size (GH-7043) --- Include/objimpl.h | 106 ++-- .../2018-05-28-21-17-31.bpo-33597.r0ToM4.rst | 1 + Modules/gcmodule.c | 579 +++++++++++++-------- Objects/object.c | 13 +- 4 files changed, 436 insertions(+), 263 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-05-28-21-17-31.bpo-33597.r0ToM4.rst diff --git a/Include/objimpl.h b/Include/objimpl.h index a38906c..4eeb8df 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -251,76 +251,88 @@ PyAPI_FUNC(PyVarObject *) _PyObject_GC_Resize(PyVarObject *, Py_ssize_t); /* GC information is stored BEFORE the object structure. */ #ifndef Py_LIMITED_API -typedef union _gc_head { - struct { - union _gc_head *gc_next; - union _gc_head *gc_prev; - Py_ssize_t gc_refs; - } gc; - double dummy; /* force worst-case alignment */ +typedef struct { + // Pointer to next object in the list. + // 0 means the object is not tracked + uintptr_t _gc_next; + + // Pointer to previous object in the list. + // Lowest two bits are used for flags documented later. + uintptr_t _gc_prev; } PyGC_Head; extern PyGC_Head *_PyGC_generation0; #define _Py_AS_GC(o) ((PyGC_Head *)(o)-1) +/* Bit flags for _gc_prev */ /* Bit 0 is set when tp_finalize is called */ -#define _PyGC_REFS_MASK_FINALIZED (1 << 0) -/* The (N-1) most significant bits contain the gc state / refcount */ -#define _PyGC_REFS_SHIFT (1) -#define _PyGC_REFS_MASK (((size_t) -1) << _PyGC_REFS_SHIFT) - -#define _PyGCHead_REFS(g) ((g)->gc.gc_refs >> _PyGC_REFS_SHIFT) -#define _PyGCHead_SET_REFS(g, v) do { \ - (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK) \ - | (((size_t)(v)) << _PyGC_REFS_SHIFT); \ +#define _PyGC_PREV_MASK_FINALIZED (1) +/* Bit 1 is set when the object is in generation which is GCed currently. */ +#define _PyGC_PREV_MASK_COLLECTING (2) +/* The (N-2) most significant bits contain the real address. */ +#define _PyGC_PREV_SHIFT (2) +#define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) + +// Lowest bit of _gc_next is used for flags only in GC. +// But it is always 0 for normal code. +#define _PyGCHead_NEXT(g) ((PyGC_Head*)(g)->_gc_next) +#define _PyGCHead_SET_NEXT(g, p) ((g)->_gc_next = (uintptr_t)(p)) + +// Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. +#define _PyGCHead_PREV(g) ((PyGC_Head*)((g)->_gc_prev & _PyGC_PREV_MASK)) +#define _PyGCHead_SET_PREV(g, p) do { \ + assert(((uintptr_t)p & ~_PyGC_PREV_MASK) == 0); \ + (g)->_gc_prev = ((g)->_gc_prev & ~_PyGC_PREV_MASK) \ + | ((uintptr_t)(p)); \ } while (0) -#define _PyGCHead_DECREF(g) ((g)->gc.gc_refs -= 1 << _PyGC_REFS_SHIFT) -#define _PyGCHead_FINALIZED(g) (((g)->gc.gc_refs & _PyGC_REFS_MASK_FINALIZED) != 0) -#define _PyGCHead_SET_FINALIZED(g, v) do { \ - (g)->gc.gc_refs = ((g)->gc.gc_refs & ~_PyGC_REFS_MASK_FINALIZED) \ - | (v != 0); \ - } while (0) +#define _PyGCHead_FINALIZED(g) (((g)->_gc_prev & _PyGC_PREV_MASK_FINALIZED) != 0) +#define _PyGCHead_SET_FINALIZED(g) ((g)->_gc_prev |= _PyGC_PREV_MASK_FINALIZED) #define _PyGC_FINALIZED(o) _PyGCHead_FINALIZED(_Py_AS_GC(o)) -#define _PyGC_SET_FINALIZED(o, v) _PyGCHead_SET_FINALIZED(_Py_AS_GC(o), v) - -#define _PyGC_REFS(o) _PyGCHead_REFS(_Py_AS_GC(o)) - -#define _PyGC_REFS_UNTRACKED (-2) -#define _PyGC_REFS_REACHABLE (-3) -#define _PyGC_REFS_TENTATIVELY_UNREACHABLE (-4) - -/* Tell the GC to track this object. NB: While the object is tracked the - * collector it must be safe to call the ob_traverse method. */ +#define _PyGC_SET_FINALIZED(o) _PyGCHead_SET_FINALIZED(_Py_AS_GC(o)) + +/* Tell the GC to track this object. + * + * NB: While the object is tracked by the collector, it must be safe to call the + * ob_traverse method. + * + * Internal note: _PyGC_generation0->_gc_prev doesn't have any bit flags + * because it's not object header. So we don't use _PyGCHead_PREV() and + * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. + */ #define _PyObject_GC_TRACK(o) do { \ PyGC_Head *g = _Py_AS_GC(o); \ - if (_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED) \ + if (g->_gc_next != 0) { \ Py_FatalError("GC object already tracked"); \ - _PyGCHead_SET_REFS(g, _PyGC_REFS_REACHABLE); \ - g->gc.gc_next = _PyGC_generation0; \ - g->gc.gc_prev = _PyGC_generation0->gc.gc_prev; \ - g->gc.gc_prev->gc.gc_next = g; \ - _PyGC_generation0->gc.gc_prev = g; \ + } \ + assert((g->_gc_prev & _PyGC_PREV_MASK_COLLECTING) == 0); \ + PyGC_Head *last = (PyGC_Head*)(_PyGC_generation0->_gc_prev); \ + _PyGCHead_SET_NEXT(last, g); \ + _PyGCHead_SET_PREV(g, last); \ + _PyGCHead_SET_NEXT(g, _PyGC_generation0); \ + _PyGC_generation0->_gc_prev = (uintptr_t)g; \ } while (0); /* Tell the GC to stop tracking this object. - * gc_next doesn't need to be set to NULL, but doing so is a good - * way to provoke memory errors if calling code is confused. + * + * Internal note: This may be called while GC. So _PyGC_PREV_MASK_COLLECTING must + * be cleared. But _PyGC_PREV_MASK_FINALIZED bit is kept. */ #define _PyObject_GC_UNTRACK(o) do { \ PyGC_Head *g = _Py_AS_GC(o); \ - assert(_PyGCHead_REFS(g) != _PyGC_REFS_UNTRACKED); \ - _PyGCHead_SET_REFS(g, _PyGC_REFS_UNTRACKED); \ - g->gc.gc_prev->gc.gc_next = g->gc.gc_next; \ - g->gc.gc_next->gc.gc_prev = g->gc.gc_prev; \ - g->gc.gc_next = NULL; \ + PyGC_Head *prev = _PyGCHead_PREV(g); \ + PyGC_Head *next = _PyGCHead_NEXT(g); \ + assert(next != NULL); \ + _PyGCHead_SET_NEXT(prev, next); \ + _PyGCHead_SET_PREV(next, prev); \ + g->_gc_next = 0; \ + g->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; \ } while (0); /* True if the object is currently tracked by the GC. */ -#define _PyObject_GC_IS_TRACKED(o) \ - (_PyGC_REFS(o) != _PyGC_REFS_UNTRACKED) +#define _PyObject_GC_IS_TRACKED(o) (_Py_AS_GC(o)->_gc_next != 0) /* True if the object may be tracked by the GC in the future, or already is. This can be useful to implement some optimizations. */ diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-05-28-21-17-31.bpo-33597.r0ToM4.rst b/Misc/NEWS.d/next/Core and Builtins/2018-05-28-21-17-31.bpo-33597.r0ToM4.rst new file mode 100644 index 0000000..b6baab2 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-05-28-21-17-31.bpo-33597.r0ToM4.rst @@ -0,0 +1 @@ +Reduce ``PyGC_Head`` size from 3 words to 2 words. diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index 7d23fc2..e3e290c 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -36,6 +36,72 @@ module gc [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=b5c9690ecc842d79]*/ +#define GC_DEBUG (0) /* Enable more asserts */ + +#define GC_NEXT _PyGCHead_NEXT +#define GC_PREV _PyGCHead_PREV + +// update_refs() set this bit for all objects in current generation. +// subtract_refs() and move_unreachable() uses this to distinguish +// visited object is in GCing or not. +// +// move_unreachable() removes this flag from reachable objects. +// Only unreachable objects have this flag. +// +// No objects in interpreter have this flag after GC ends. +#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING + +// Lowest bit of _gc_next is used for UNREACHABLE flag. +// +// This flag represents the object is in unreachable list in move_unreachable() +// +// Although this flag is used only in move_unreachable(), move_unreachable() +// doesn't clear this flag to skip unnecessary iteration. +// move_legacy_finalizers() removes this flag instead. +// Between them, unreachable list is not normal list and we can not use +// most gc_list_* functions for it. +#define NEXT_MASK_UNREACHABLE (1) + +static inline int +gc_is_collecting(PyGC_Head *g) +{ + return (g->_gc_prev & PREV_MASK_COLLECTING) != 0; +} + +static inline void +gc_clear_collecting(PyGC_Head *g) +{ + g->_gc_prev &= ~PREV_MASK_COLLECTING; +} + +static inline Py_ssize_t +gc_get_refs(PyGC_Head *g) +{ + return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT); +} + +static inline void +gc_set_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK) + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_reset_refs(PyGC_Head *g, Py_ssize_t refs) +{ + g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED) + | PREV_MASK_COLLECTING + | ((uintptr_t)(refs) << _PyGC_PREV_SHIFT); +} + +static inline void +gc_decref(PyGC_Head *g) +{ + assert(gc_get_refs(g) > 0); + g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; +} + /* Get an object's GC head */ #define AS_GC(o) ((PyGC_Head *)(o)-1) @@ -63,104 +129,116 @@ _PyGC_Initialize(struct _gc_runtime_state *state) #define _GEN_HEAD(n) (&state->generations[n].head) struct gc_generation generations[NUM_GENERATIONS] = { - /* PyGC_Head, threshold, count */ - {{{_GEN_HEAD(0), _GEN_HEAD(0), 0}}, 700, 0}, - {{{_GEN_HEAD(1), _GEN_HEAD(1), 0}}, 10, 0}, - {{{_GEN_HEAD(2), _GEN_HEAD(2), 0}}, 10, 0}, + /* PyGC_Head, threshold, count */ + {{(uintptr_t)_GEN_HEAD(0), (uintptr_t)_GEN_HEAD(0)}, 700, 0}, + {{(uintptr_t)_GEN_HEAD(1), (uintptr_t)_GEN_HEAD(1)}, 10, 0}, + {{(uintptr_t)_GEN_HEAD(2), (uintptr_t)_GEN_HEAD(2)}, 10, 0}, }; for (int i = 0; i < NUM_GENERATIONS; i++) { state->generations[i] = generations[i]; }; state->generation0 = GEN_HEAD(0); struct gc_generation permanent_generation = { - {{&state->permanent_generation.head, &state->permanent_generation.head, 0}}, 0, 0 + {(uintptr_t)&state->permanent_generation.head, + (uintptr_t)&state->permanent_generation.head}, 0, 0 }; state->permanent_generation = permanent_generation; } -/*-------------------------------------------------------------------------- -gc_refs values. - -Between collections, every gc'ed object has one of two gc_refs values: +/* +_gc_prev values +--------------- -GC_UNTRACKED - The initial state; objects returned by PyObject_GC_Malloc are in this - state. The object doesn't live in any generation list, and its - tp_traverse slot must not be called. +Between collections, _gc_prev is used for doubly linked list. -GC_REACHABLE - The object lives in some generation list, and its tp_traverse is safe to - call. An object transitions to GC_REACHABLE when PyObject_GC_Track - is called. +Lowest two bits of _gc_prev are used for flags. +PREV_MASK_COLLECTING is used only while collecting and cleared before GC ends +or _PyObject_GC_UNTRACK() is called. -During a collection, gc_refs can temporarily take on other states: +During a collection, _gc_prev is temporary used for gc_refs, and the gc list +is singly linked until _gc_prev is restored. ->= 0 +gc_refs At the start of a collection, update_refs() copies the true refcount to gc_refs, for each object in the generation being collected. subtract_refs() then adjusts gc_refs so that it equals the number of times an object is referenced directly from outside the generation being collected. - gc_refs remains >= 0 throughout these steps. -GC_TENTATIVELY_UNREACHABLE +PREV_MASK_COLLECTING + Objects in generation being collected are marked PREV_MASK_COLLECTING in + update_refs(). + + +_gc_next values +--------------- + +_gc_next takes these values: + +0 + The object is not tracked + +!= 0 + Pointer to the next object in the GC list. + Additionally, lowest bit is used temporary for + NEXT_MASK_UNREACHABLE flag described below. + +NEXT_MASK_UNREACHABLE move_unreachable() then moves objects not reachable (whether directly or - indirectly) from outside the generation into an "unreachable" set. - Objects that are found to be reachable have gc_refs set to GC_REACHABLE - again. Objects that are found to be unreachable have gc_refs set to - GC_TENTATIVELY_UNREACHABLE. It's "tentatively" because the pass doing - this can't be sure until it ends, and GC_TENTATIVELY_UNREACHABLE may - transition back to GC_REACHABLE. - - Only objects with GC_TENTATIVELY_UNREACHABLE still set are candidates - for collection. If it's decided not to collect such an object (e.g., - it has a __del__ method), its gc_refs is restored to GC_REACHABLE again. ----------------------------------------------------------------------------- -*/ -#define GC_UNTRACKED _PyGC_REFS_UNTRACKED -#define GC_REACHABLE _PyGC_REFS_REACHABLE -#define GC_TENTATIVELY_UNREACHABLE _PyGC_REFS_TENTATIVELY_UNREACHABLE + indirectly) from outside the generation into an "unreachable" set and + set this flag. + + Objects that are found to be reachable have gc_refs set to 1. + When this flag is set for the reachable object, the object must be in + "unreachable" set. + The flag is unset and the object is moved back to "reachable" set. -#define IS_TRACKED(o) (_PyGC_REFS(o) != GC_UNTRACKED) -#define IS_REACHABLE(o) (_PyGC_REFS(o) == GC_REACHABLE) -#define IS_TENTATIVELY_UNREACHABLE(o) ( \ - _PyGC_REFS(o) == GC_TENTATIVELY_UNREACHABLE) + move_legacy_finalizers() will remove this flag from "unreachable" set. +*/ /*** list functions ***/ -static void +static inline void gc_list_init(PyGC_Head *list) { - list->gc.gc_prev = list; - list->gc.gc_next = list; + // List header must not have flags. + // We can assign pointer by simple cast. + list->_gc_prev = (uintptr_t)list; + list->_gc_next = (uintptr_t)list; } -static int +static inline int gc_list_is_empty(PyGC_Head *list) { - return (list->gc.gc_next == list); + return (list->_gc_next == (uintptr_t)list); } -#if 0 -/* This became unused after gc_list_move() was introduced. */ /* Append `node` to `list`. */ -static void +static inline void gc_list_append(PyGC_Head *node, PyGC_Head *list) { - node->gc.gc_next = list; - node->gc.gc_prev = list->gc.gc_prev; - node->gc.gc_prev->gc.gc_next = node; - list->gc.gc_prev = node; + PyGC_Head *last = (PyGC_Head *)list->_gc_prev; + + // last <-> node + _PyGCHead_SET_PREV(node, last); + _PyGCHead_SET_NEXT(last, node); + + // node <-> list + _PyGCHead_SET_NEXT(node, list); + list->_gc_prev = (uintptr_t)node; } -#endif /* Remove `node` from the gc list it's currently in. */ -static void +static inline void gc_list_remove(PyGC_Head *node) { - node->gc.gc_prev->gc.gc_next = node->gc.gc_next; - node->gc.gc_next->gc.gc_prev = node->gc.gc_prev; - node->gc.gc_next = NULL; /* object is not currently tracked */ + PyGC_Head *prev = GC_PREV(node); + PyGC_Head *next = GC_NEXT(node); + + _PyGCHead_SET_NEXT(prev, next); + _PyGCHead_SET_PREV(next, prev); + + node->_gc_next = 0; /* object is not currently tracked */ } /* Move `node` from the gc list it's currently in (which is not explicitly @@ -170,30 +248,38 @@ gc_list_remove(PyGC_Head *node) static void gc_list_move(PyGC_Head *node, PyGC_Head *list) { - PyGC_Head *new_prev; - PyGC_Head *current_prev = node->gc.gc_prev; - PyGC_Head *current_next = node->gc.gc_next; /* Unlink from current list. */ - current_prev->gc.gc_next = current_next; - current_next->gc.gc_prev = current_prev; + PyGC_Head *from_prev = GC_PREV(node); + PyGC_Head *from_next = GC_NEXT(node); + _PyGCHead_SET_NEXT(from_prev, from_next); + _PyGCHead_SET_PREV(from_next, from_prev); + /* Relink at end of new list. */ - new_prev = node->gc.gc_prev = list->gc.gc_prev; - new_prev->gc.gc_next = list->gc.gc_prev = node; - node->gc.gc_next = list; + // list must not have flags. So we can skip macros. + PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev; + _PyGCHead_SET_PREV(node, to_prev); + _PyGCHead_SET_NEXT(to_prev, node); + list->_gc_prev = (uintptr_t)node; + _PyGCHead_SET_NEXT(node, list); } /* append list `from` onto list `to`; `from` becomes an empty list */ static void gc_list_merge(PyGC_Head *from, PyGC_Head *to) { - PyGC_Head *tail; assert(from != to); if (!gc_list_is_empty(from)) { - tail = to->gc.gc_prev; - tail->gc.gc_next = from->gc.gc_next; - tail->gc.gc_next->gc.gc_prev = tail; - to->gc.gc_prev = from->gc.gc_prev; - to->gc.gc_prev->gc.gc_next = to; + PyGC_Head *to_tail = GC_PREV(to); + PyGC_Head *from_head = GC_NEXT(from); + PyGC_Head *from_tail = GC_PREV(from); + assert(from_head != from); + assert(from_tail != from); + + _PyGCHead_SET_NEXT(to_tail, from_head); + _PyGCHead_SET_PREV(from_head, to_tail); + + _PyGCHead_SET_NEXT(from_tail, to); + _PyGCHead_SET_PREV(to, from_tail); } gc_list_init(from); } @@ -203,7 +289,7 @@ gc_list_size(PyGC_Head *list) { PyGC_Head *gc; Py_ssize_t n = 0; - for (gc = list->gc.gc_next; gc != list; gc = gc->gc.gc_next) { + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { n++; } return n; @@ -216,7 +302,7 @@ static int append_objects(PyObject *py_list, PyGC_Head *gc_list) { PyGC_Head *gc; - for (gc = gc_list->gc.gc_next; gc != gc_list; gc = gc->gc.gc_next) { + for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); if (op != py_list) { if (PyList_Append(py_list, op)) { @@ -227,20 +313,39 @@ append_objects(PyObject *py_list, PyGC_Head *gc_list) return 0; } +#if GC_DEBUG +// validate_list checks list consistency. And it works as document +// describing when expected_mask is set / unset. +static void +validate_list(PyGC_Head *head, uintptr_t expected_mask) +{ + PyGC_Head *prev = head; + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + assert(GC_NEXT(gc) != NULL); + assert(GC_PREV(gc) == prev); + assert((gc->_gc_prev & PREV_MASK_COLLECTING) == expected_mask); + prev = gc; + gc = GC_NEXT(gc); + } + assert(prev == GC_PREV(head)); +} +#else +#define validate_list(x,y) do{}while(0) +#endif + /*** end of list stuff ***/ -/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 for all objects - * in containers, and is GC_REACHABLE for all tracked gc objects not in - * containers. +/* Set all gc_refs = ob_refcnt. After this, gc_refs is > 0 and + * PREV_MASK_COLLECTING bit is set for all objects in containers. */ static void update_refs(PyGC_Head *containers) { - PyGC_Head *gc = containers->gc.gc_next; - for (; gc != containers; gc = gc->gc.gc_next) { - assert(_PyGCHead_REFS(gc) == GC_REACHABLE); - _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); + PyGC_Head *gc = GC_NEXT(containers); + for (; gc != containers; gc = GC_NEXT(gc)) { + gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc))); /* Python's cyclic gc should never see an incoming refcount * of 0: if something decref'ed to 0, it should have been * deallocated immediately at that time. @@ -259,7 +364,7 @@ update_refs(PyGC_Head *containers) * so serious that maybe this should be a release-build * check instead of an assert? */ - assert(_PyGCHead_REFS(gc) != 0); + assert(gc_get_refs(gc) != 0); } } @@ -274,9 +379,9 @@ visit_decref(PyObject *op, void *data) * generation being collected, which can be recognized * because only they have positive gc_refs. */ - assert(_PyGCHead_REFS(gc) != 0); /* else refcount was too small */ - if (_PyGCHead_REFS(gc) > 0) - _PyGCHead_DECREF(gc); + if (gc_is_collecting(gc)) { + gc_decref(gc); + } } return 0; } @@ -290,8 +395,8 @@ static void subtract_refs(PyGC_Head *containers) { traverseproc traverse; - PyGC_Head *gc = containers->gc.gc_next; - for (; gc != containers; gc=gc->gc.gc_next) { + PyGC_Head *gc = GC_NEXT(containers); + for (; gc != containers; gc = GC_NEXT(gc)) { traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; (void) traverse(FROM_GC(gc), (visitproc)visit_decref, @@ -303,71 +408,84 @@ subtract_refs(PyGC_Head *containers) static int visit_reachable(PyObject *op, PyGC_Head *reachable) { - if (PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - const Py_ssize_t gc_refs = _PyGCHead_REFS(gc); + if (!PyObject_IS_GC(op)) { + return 0; + } - if (gc_refs == 0) { - /* This is in move_unreachable's 'young' list, but - * the traversal hasn't yet gotten to it. All - * we need to do is tell move_unreachable that it's - * reachable. - */ - _PyGCHead_SET_REFS(gc, 1); - } - else if (gc_refs == GC_TENTATIVELY_UNREACHABLE) { - /* This had gc_refs = 0 when move_unreachable got - * to it, but turns out it's reachable after all. - * Move it back to move_unreachable's 'young' list, - * and move_unreachable will eventually get to it - * again. - */ - gc_list_move(gc, reachable); - _PyGCHead_SET_REFS(gc, 1); - } - /* Else there's nothing to do. - * If gc_refs > 0, it must be in move_unreachable's 'young' - * list, and move_unreachable will eventually get to it. - * If gc_refs == GC_REACHABLE, it's either in some other - * generation so we don't care about it, or move_unreachable - * already dealt with it. - * If gc_refs == GC_UNTRACKED, it must be ignored. + PyGC_Head *gc = AS_GC(op); + const Py_ssize_t gc_refs = gc_get_refs(gc); + + // Ignore untracked objects and objects in other generation. + if (gc->_gc_next == 0 || !gc_is_collecting(gc)) { + return 0; + } + + if (gc->_gc_next & NEXT_MASK_UNREACHABLE) { + /* This had gc_refs = 0 when move_unreachable got + * to it, but turns out it's reachable after all. + * Move it back to move_unreachable's 'young' list, + * and move_unreachable will eventually get to it + * again. + */ + // Manually unlink gc from unreachable list because + PyGC_Head *prev = GC_PREV(gc); + PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); + assert(prev->_gc_next & NEXT_MASK_UNREACHABLE); + assert(next->_gc_next & NEXT_MASK_UNREACHABLE); + prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE + _PyGCHead_SET_PREV(next, prev); + + gc_list_append(gc, reachable); + gc_set_refs(gc, 1); + } + else if (gc_refs == 0) { + /* This is in move_unreachable's 'young' list, but + * the traversal hasn't yet gotten to it. All + * we need to do is tell move_unreachable that it's + * reachable. */ - else { - assert(gc_refs > 0 - || gc_refs == GC_REACHABLE - || gc_refs == GC_UNTRACKED); - } + gc_set_refs(gc, 1); + } + /* Else there's nothing to do. + * If gc_refs > 0, it must be in move_unreachable's 'young' + * list, and move_unreachable will eventually get to it. + */ + else { + assert(gc_refs > 0); } return 0; } /* Move the unreachable objects from young to unreachable. After this, - * all objects in young have gc_refs = GC_REACHABLE, and all objects in - * unreachable have gc_refs = GC_TENTATIVELY_UNREACHABLE. All tracked - * gc objects not in young or unreachable still have gc_refs = GC_REACHABLE. + * all objects in young don't have PREV_MASK_COLLECTING flag and + * unreachable have the flag. * All objects in young after this are directly or indirectly reachable * from outside the original young; and all objects in unreachable are * not. + * + * This function restores _gc_prev pointer. young and unreachable are + * doubly linked list after this function. + * But _gc_next in unreachable list has NEXT_MASK_UNREACHABLE flag. + * So we can not gc_list_* functions for unreachable until we remove the flag. */ static void move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) { - PyGC_Head *gc = young->gc.gc_next; + // previous elem in the young list, used for restore gc_prev. + PyGC_Head *prev = young; + PyGC_Head *gc = GC_NEXT(young); - /* Invariants: all objects "to the left" of us in young have gc_refs - * = GC_REACHABLE, and are indeed reachable (directly or indirectly) - * from outside the young list as it was at entry. All other objects - * from the original young "to the left" of us are in unreachable now, - * and have gc_refs = GC_TENTATIVELY_UNREACHABLE. All objects to the + /* Invariants: all objects "to the left" of us in young are reachable + * (directly or indirectly) from outside the young list as it was at entry. + * + * All other objects from the original young "to the left" of us are in + * unreachable now, and have NEXT_MASK_UNREACHABLE. All objects to the * left of us in 'young' now have been scanned, and no objects here * or to the right have been scanned yet. */ while (gc != young) { - PyGC_Head *next; - - if (_PyGCHead_REFS(gc)) { + if (gc_get_refs(gc)) { /* gc is definitely reachable from outside the * original 'young'. Mark it as such, and traverse * its pointers to find any other objects that may @@ -378,15 +496,17 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) */ PyObject *op = FROM_GC(gc); traverseproc traverse = Py_TYPE(op)->tp_traverse; - assert(_PyGCHead_REFS(gc) > 0); - _PyGCHead_SET_REFS(gc, GC_REACHABLE); + assert(gc_get_refs(gc) > 0); + // NOTE: visit_reachable may change gc->_gc_next when + // young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before! (void) traverse(op, - (visitproc)visit_reachable, - (void *)young); - next = gc->gc.gc_next; - if (PyTuple_CheckExact(op)) { - _PyTuple_MaybeUntrack(op); - } + (visitproc)visit_reachable, + (void *)young); + // relink gc_prev to prev element. + _PyGCHead_SET_PREV(gc, prev); + // gc is not COLLECTING state aftere here. + gc_clear_collecting(gc); + prev = gc; } else { /* This *may* be unreachable. To make progress, @@ -396,9 +516,37 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) * visit_reachable will eventually move gc back into * young if that's so, and we'll see it again. */ - next = gc->gc.gc_next; - gc_list_move(gc, unreachable); - _PyGCHead_SET_REFS(gc, GC_TENTATIVELY_UNREACHABLE); + // Move gc to unreachable. + // No need to gc->next->prev = prev because it is single linked. + prev->_gc_next = gc->_gc_next; + + // We can't use gc_list_append() here because we use + // NEXT_MASK_UNREACHABLE here. + PyGC_Head *last = GC_PREV(unreachable); + // NOTE: Since all objects in unreachable set has + // NEXT_MASK_UNREACHABLE flag, we set it unconditionally. + // But this may set the flat to unreachable too. + // move_legacy_finalizers() should care about it. + last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); + _PyGCHead_SET_PREV(gc, last); + gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); + unreachable->_gc_prev = (uintptr_t)gc; + } + gc = (PyGC_Head*)prev->_gc_next; + } + // young->_gc_prev must be last element remained in the list. + young->_gc_prev = (uintptr_t)prev; +} + +static void +untrack_tuples(PyGC_Head *head) +{ + PyGC_Head *next, *gc = GC_NEXT(head); + while (gc != head) { + PyObject *op = FROM_GC(gc); + next = GC_NEXT(gc); + if (PyTuple_CheckExact(op)) { + _PyTuple_MaybeUntrack(op); } gc = next; } @@ -408,12 +556,13 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) static void untrack_dicts(PyGC_Head *head) { - PyGC_Head *next, *gc = head->gc.gc_next; + PyGC_Head *next, *gc = GC_NEXT(head); while (gc != head) { PyObject *op = FROM_GC(gc); - next = gc->gc.gc_next; - if (PyDict_CheckExact(op)) + next = GC_NEXT(gc); + if (PyDict_CheckExact(op)) { _PyDict_MaybeUntrack(op); + } gc = next; } } @@ -426,27 +575,29 @@ has_legacy_finalizer(PyObject *op) } /* Move the objects in unreachable with tp_del slots into `finalizers`. - * Objects moved into `finalizers` have gc_refs set to GC_REACHABLE; the - * objects remaining in unreachable are left at GC_TENTATIVELY_UNREACHABLE. + * + * This function also removes NEXT_MASK_UNREACHABLE flag + * from _gc_next in unreachable. */ static void move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) { - PyGC_Head *gc; - PyGC_Head *next; + PyGC_Head *gc, *next; + unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; /* March over unreachable. Move objects with finalizers into * `finalizers`. */ - for (gc = unreachable->gc.gc_next; gc != unreachable; gc = next) { + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyObject *op = FROM_GC(gc); - assert(IS_TENTATIVELY_UNREACHABLE(op)); - next = gc->gc.gc_next; + assert(gc->_gc_next & NEXT_MASK_UNREACHABLE); + gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; if (has_legacy_finalizer(op)) { + gc_clear_collecting(gc); gc_list_move(gc, finalizers); - _PyGCHead_SET_REFS(gc, GC_REACHABLE); } } } @@ -456,10 +607,10 @@ static int visit_move(PyObject *op, PyGC_Head *tolist) { if (PyObject_IS_GC(op)) { - if (IS_TENTATIVELY_UNREACHABLE(op)) { - PyGC_Head *gc = AS_GC(op); + PyGC_Head *gc = AS_GC(op); + if (gc_is_collecting(gc)) { gc_list_move(gc, tolist); - _PyGCHead_SET_REFS(gc, GC_REACHABLE); + gc_clear_collecting(gc); } } return 0; @@ -472,8 +623,8 @@ static void move_legacy_finalizer_reachable(PyGC_Head *finalizers) { traverseproc traverse; - PyGC_Head *gc = finalizers->gc.gc_next; - for (; gc != finalizers; gc = gc->gc.gc_next) { + PyGC_Head *gc = GC_NEXT(finalizers); + for (; gc != finalizers; gc = GC_NEXT(gc)) { /* Note that the finalizers list may grow during this. */ traverse = Py_TYPE(FROM_GC(gc))->tp_traverse; (void) traverse(FROM_GC(gc), @@ -513,12 +664,11 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * make another pass over wrcb_to_call, invoking callbacks, after this * pass completes. */ - for (gc = unreachable->gc.gc_next; gc != unreachable; gc = next) { + for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { PyWeakReference **wrlist; op = FROM_GC(gc); - assert(IS_TENTATIVELY_UNREACHABLE(op)); - next = gc->gc.gc_next; + next = GC_NEXT(gc); if (! PyType_SUPPORTS_WEAKREFS(Py_TYPE(op))) continue; @@ -572,9 +722,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * to imagine how calling it later could create a problem for us. wr * is moved to wrcb_to_call in this case. */ - if (IS_TENTATIVELY_UNREACHABLE(wr)) + if (gc_is_collecting(AS_GC(wr))) { continue; - assert(IS_REACHABLE(wr)); + } /* Create a new reference so that wr can't go away * before we can process it again. @@ -597,9 +747,8 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) PyObject *temp; PyObject *callback; - gc = wrcb_to_call.gc.gc_next; + gc = (PyGC_Head*)wrcb_to_call._gc_next; op = FROM_GC(gc); - assert(IS_REACHABLE(op)); assert(PyWeakref_Check(op)); wr = (PyWeakReference *)op; callback = wr->wr_callback; @@ -624,12 +773,13 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) * ours). */ Py_DECREF(op); - if (wrcb_to_call.gc.gc_next == gc) { + if (wrcb_to_call._gc_next == (uintptr_t)gc) { /* object is still alive -- move it */ gc_list_move(gc, old); } - else + else { ++num_freed; + } } return num_freed; @@ -652,7 +802,7 @@ debug_cycle(const char *msg, PyObject *op) static void handle_legacy_finalizers(PyGC_Head *finalizers, PyGC_Head *old) { - PyGC_Head *gc = finalizers->gc.gc_next; + PyGC_Head *gc = GC_NEXT(finalizers); assert(!PyErr_Occurred()); if (_PyRuntime.gc.garbage == NULL) { @@ -660,7 +810,7 @@ handle_legacy_finalizers(PyGC_Head *finalizers, PyGC_Head *old) if (_PyRuntime.gc.garbage == NULL) Py_FatalError("gc couldn't create gc.garbage list"); } - for (; gc != finalizers; gc = gc->gc.gc_next) { + for (; gc != finalizers; gc = GC_NEXT(gc)) { PyObject *op = FROM_GC(gc); if ((_PyRuntime.gc.debug & DEBUG_SAVEALL) || has_legacy_finalizer(op)) { @@ -695,13 +845,13 @@ finalize_garbage(PyGC_Head *collectable) gc_list_init(&seen); while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = collectable->gc.gc_next; + PyGC_Head *gc = GC_NEXT(collectable); PyObject *op = FROM_GC(gc); gc_list_move(gc, &seen); if (!_PyGCHead_FINALIZED(gc) && PyType_HasFeature(Py_TYPE(op), Py_TPFLAGS_HAVE_FINALIZE) && (finalize = Py_TYPE(op)->tp_finalize) != NULL) { - _PyGCHead_SET_FINALIZED(gc, 1); + _PyGCHead_SET_FINALIZED(gc); Py_INCREF(op); finalize(op); assert(!PyErr_Occurred()); @@ -717,30 +867,26 @@ finalize_garbage(PyGC_Head *collectable) static int check_garbage(PyGC_Head *collectable) { + int ret = 0; PyGC_Head *gc; - for (gc = collectable->gc.gc_next; gc != collectable; - gc = gc->gc.gc_next) { - _PyGCHead_SET_REFS(gc, Py_REFCNT(FROM_GC(gc))); - assert(_PyGCHead_REFS(gc) != 0); + for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { + // Use gc_refs and break gc_prev again. + gc_set_refs(gc, Py_REFCNT(FROM_GC(gc))); + assert(gc_get_refs(gc) != 0); } subtract_refs(collectable); - for (gc = collectable->gc.gc_next; gc != collectable; - gc = gc->gc.gc_next) { - assert(_PyGCHead_REFS(gc) >= 0); - if (_PyGCHead_REFS(gc) != 0) - return -1; - } - return 0; -} - -static void -revive_garbage(PyGC_Head *collectable) -{ - PyGC_Head *gc; - for (gc = collectable->gc.gc_next; gc != collectable; - gc = gc->gc.gc_next) { - _PyGCHead_SET_REFS(gc, GC_REACHABLE); + PyGC_Head *prev = collectable; + for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) { + assert(gc_get_refs(gc) >= 0); + if (gc_get_refs(gc) != 0) { + ret = -1; + } + // Restore gc_prev here. + _PyGCHead_SET_PREV(gc, prev); + gc_clear_collecting(gc); + prev = gc; } + return ret; } /* Break reference cycles by clearing the containers involved. This is @@ -754,9 +900,11 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) assert(!PyErr_Occurred()); while (!gc_list_is_empty(collectable)) { - PyGC_Head *gc = collectable->gc.gc_next; + PyGC_Head *gc = GC_NEXT(collectable); PyObject *op = FROM_GC(gc); + assert(Py_REFCNT(FROM_GC(gc)) > 0); + if (_PyRuntime.gc.debug & DEBUG_SAVEALL) { assert(_PyRuntime.gc.garbage != NULL); if (PyList_Append(_PyRuntime.gc.garbage, op) < 0) { @@ -775,10 +923,9 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old) Py_DECREF(op); } } - if (collectable->gc.gc_next == gc) { + if (GC_NEXT(collectable) == gc) { /* object is still alive, move it, it may die later */ gc_list_move(gc, old); - _PyGCHead_SET_REFS(gc, GC_REACHABLE); } } } @@ -857,12 +1004,14 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, else old = young; + validate_list(young, 0); + validate_list(old, 0); /* Using ob_refcnt and gc_refs, calculate which objects in the * container set are reachable from outside the set (i.e., have a * refcount greater than 0 when all the references within the * set are taken into account). */ - update_refs(young); + update_refs(young); // gc_prev is used for gc_refs subtract_refs(young); /* Leave everything reachable from outside young in young, and move @@ -872,8 +1021,10 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, * so it's more efficient to move the unreachable things. */ gc_list_init(&unreachable); - move_unreachable(young, &unreachable); + move_unreachable(young, &unreachable); // gc_prev is pointer again + validate_list(young, 0); + untrack_tuples(young); /* Move reachable objects to next generation. */ if (young != old) { if (generation == NUM_GENERATIONS - 2) { @@ -893,6 +1044,8 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, * legacy finalizers (e.g. tp_del) can't safely be deleted. */ gc_list_init(&finalizers); + // NEXT_MASK_UNREACHABLE is cleared here. + // After move_legacy_finalizers(), unreachable is normal list. move_legacy_finalizers(&unreachable, &finalizers); /* finalizers contains the unreachable objects with a legacy finalizer; * unreachable objects reachable *from* those are also uncollectable, @@ -900,11 +1053,13 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, */ move_legacy_finalizer_reachable(&finalizers); + validate_list(&finalizers, 0); + validate_list(&unreachable, PREV_MASK_COLLECTING); + /* Collect statistics on collectable objects found and print * debugging information. */ - for (gc = unreachable.gc.gc_next; gc != &unreachable; - gc = gc->gc.gc_next) { + for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { m++; if (_PyRuntime.gc.debug & DEBUG_COLLECTABLE) { debug_cycle("collectable", FROM_GC(gc)); @@ -914,11 +1069,13 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, /* Clear weakrefs and invoke callbacks as necessary. */ m += handle_weakrefs(&unreachable, old); + validate_list(old, 0); + validate_list(&unreachable, PREV_MASK_COLLECTING); + /* Call tp_finalize on objects which have one. */ finalize_garbage(&unreachable); - if (check_garbage(&unreachable)) { - revive_garbage(&unreachable); + if (check_garbage(&unreachable)) { // clear PREV_MASK_COLLECTING here gc_list_merge(&unreachable, old); } else { @@ -931,9 +1088,7 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, /* Collect statistics on uncollectable objects found and print * debugging information. */ - for (gc = finalizers.gc.gc_next; - gc != &finalizers; - gc = gc->gc.gc_next) { + for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; if (_PyRuntime.gc.debug & DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); @@ -956,6 +1111,7 @@ collect(int generation, Py_ssize_t *n_collected, Py_ssize_t *n_uncollectable, * this if they insist on creating this type of structure. */ handle_legacy_finalizers(&finalizers, old); + validate_list(old, 0); /* Clear free list only during the collection of the highest * generation */ @@ -1263,7 +1419,7 @@ gc_referrers_for(PyObject *objs, PyGC_Head *list, PyObject *resultlist) PyGC_Head *gc; PyObject *obj; traverseproc traverse; - for (gc = list->gc.gc_next; gc != list; gc = gc->gc.gc_next) { + for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { obj = FROM_GC(gc); traverse = Py_TYPE(obj)->tp_traverse; if (obj == objs || obj == resultlist) @@ -1423,7 +1579,7 @@ gc_is_tracked(PyObject *module, PyObject *obj) { PyObject *result; - if (PyObject_IS_GC(obj) && IS_TRACKED(obj)) + if (PyObject_IS_GC(obj) && _PyObject_GC_IS_TRACKED(obj)) result = Py_True; else result = Py_False; @@ -1696,8 +1852,9 @@ PyObject_GC_UnTrack(void *op) /* Obscure: the Py_TRASHCAN mechanism requires that we be able to * call PyObject_GC_UnTrack twice on an object. */ - if (IS_TRACKED(op)) + if (_PyObject_GC_IS_TRACKED(op)) { _PyObject_GC_UNTRACK(op); + } } static PyObject * @@ -1715,8 +1872,9 @@ _PyObject_GC_Alloc(int use_calloc, size_t basicsize) g = (PyGC_Head *)PyObject_Malloc(size); if (g == NULL) return PyErr_NoMemory(); - g->gc.gc_refs = 0; - _PyGCHead_SET_REFS(g, GC_UNTRACKED); + assert(((uintptr_t)g & 3) == 0); // g must be aligned 4bytes boundary + g->_gc_next = 0; + g->_gc_prev = 0; _PyRuntime.gc.generations[0].count++; /* number of allocated GC objects */ if (_PyRuntime.gc.generations[0].count > _PyRuntime.gc.generations[0].threshold && _PyRuntime.gc.enabled && @@ -1774,7 +1932,7 @@ _PyObject_GC_Resize(PyVarObject *op, Py_ssize_t nitems) { const size_t basicsize = _PyObject_VAR_SIZE(Py_TYPE(op), nitems); PyGC_Head *g = AS_GC(op); - assert(!IS_TRACKED(op)); + assert(!_PyObject_GC_IS_TRACKED(op)); if (basicsize > PY_SSIZE_T_MAX - sizeof(PyGC_Head)) return (PyVarObject *)PyErr_NoMemory(); g = (PyGC_Head *)PyObject_REALLOC(g, sizeof(PyGC_Head) + basicsize); @@ -1789,8 +1947,9 @@ void PyObject_GC_Del(void *op) { PyGC_Head *g = AS_GC(op); - if (IS_TRACKED(op)) + if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + } if (_PyRuntime.gc.generations[0].count > 0) { _PyRuntime.gc.generations[0].count--; } diff --git a/Objects/object.c b/Objects/object.c index 3eb4810..2471f6b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -284,8 +284,9 @@ PyObject_CallFinalizer(PyObject *self) return; tp->tp_finalize(self); - if (PyType_IS_GC(tp)) - _PyGC_SET_FINALIZED(self, 1); + if (PyType_IS_GC(tp)) { + _PyGC_SET_FINALIZED(self); + } } int @@ -2095,7 +2096,7 @@ _PyTrash_deposit_object(PyObject *op) assert(PyObject_IS_GC(op)); assert(!_PyObject_GC_IS_TRACKED(op)); assert(op->ob_refcnt == 0); - _Py_AS_GC(op)->gc.gc_prev = (PyGC_Head *)_PyRuntime.gc.trash_delete_later; + _PyGCHead_SET_PREV(_Py_AS_GC(op), _PyRuntime.gc.trash_delete_later); _PyRuntime.gc.trash_delete_later = op; } @@ -2107,7 +2108,7 @@ _PyTrash_thread_deposit_object(PyObject *op) assert(PyObject_IS_GC(op)); assert(!_PyObject_GC_IS_TRACKED(op)); assert(op->ob_refcnt == 0); - _Py_AS_GC(op)->gc.gc_prev = (PyGC_Head *) tstate->trash_delete_later; + _PyGCHead_SET_PREV(_Py_AS_GC(op), tstate->trash_delete_later); tstate->trash_delete_later = op; } @@ -2122,7 +2123,7 @@ _PyTrash_destroy_chain(void) destructor dealloc = Py_TYPE(op)->tp_dealloc; _PyRuntime.gc.trash_delete_later = - (PyObject*) _Py_AS_GC(op)->gc.gc_prev; + (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but @@ -2160,7 +2161,7 @@ _PyTrash_thread_destroy_chain(void) destructor dealloc = Py_TYPE(op)->tp_dealloc; tstate->trash_delete_later = - (PyObject*) _Py_AS_GC(op)->gc.gc_prev; + (PyObject*) _PyGCHead_PREV(_Py_AS_GC(op)); /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but -- cgit v0.12