From 8aa64cc45bff516a6db1f3a3c037cbcce9417fea Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 28 Aug 2021 11:09:21 -0700 Subject: bpo-44962: Fix a race in WeakKeyDict, WeakValueDict and WeakSet when two threads attempt to commit the last pending removal (GH-27921) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: Traceback (most recent call last): File "/home/graingert/projects/asyncio-demo/demo.py", line 36, in sys.exit(main()) File "/home/graingert/projects/asyncio-demo/demo.py", line 30, in main test_all_tasks_threading() File "/home/graingert/projects/asyncio-demo/demo.py", line 24, in test_all_tasks_threading results.append(f.result()) File "/usr/lib/python3.10/concurrent/futures/_base.py", line 438, in result return self.__get_result() File "/usr/lib/python3.10/concurrent/futures/_base.py", line 390, in __get_result raise self._exception File "/usr/lib/python3.10/concurrent/futures/thread.py", line 52, in run result = self.fn(*self.args, **self.kwargs) File "/usr/lib/python3.10/asyncio/runners.py", line 47, in run _cancel_all_tasks(loop) File "/usr/lib/python3.10/asyncio/runners.py", line 56, in _cancel_all_tasks to_cancel = tasks.all_tasks(loop) File "/usr/lib/python3.10/asyncio/tasks.py", line 53, in all_tasks tasks = list(_all_tasks) File "/usr/lib/python3.10/_weakrefset.py", line 60, in __iter__ with _IterationGuard(self): File "/usr/lib/python3.10/_weakrefset.py", line 33, in __exit__ w._commit_removals() File "/usr/lib/python3.10/_weakrefset.py", line 57, in _commit_removals discard(l.pop()) IndexError: pop from empty list Also fixes: Exception ignored in: weakref callback .remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: Exception ignored in: weakref callback .remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: Exception ignored in: weakref callback .remove at 0x00007fe82245d2e0> Traceback (most recent call last): File "/usr/lib/pypy3/lib-python/3/weakref.py", line 390, in remove del self.data[k] KeyError: See: https://github.com/agronholm/anyio/issues/362GH-issuecomment-904424310 See also: https://bugs.python.org/issue29519 Co-authored-by: Łukasz Langa (cherry picked from commit 206b21ed9f64fedff67bfea7cf73e423e3e32393) Co-authored-by: Thomas Grainger --- Lib/_weakrefset.py | 10 +++++--- Lib/weakref.py | 29 +++++++++++++++------- .../2021-08-23-19-55-08.bpo-44962.J00ftt.rst | 1 + 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index b267780..2a27684 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -51,10 +51,14 @@ class WeakSet: self.update(data) def _commit_removals(self): - l = self._pending_removals + pop = self._pending_removals.pop discard = self.data.discard - while l: - discard(l.pop()) + while True: + try: + item = pop() + except IndexError: + return + discard(item) def __iter__(self): with _IterationGuard(self): diff --git a/Lib/weakref.py b/Lib/weakref.py index a968139..994ea8a 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -119,14 +119,17 @@ class WeakValueDictionary(_collections_abc.MutableMapping): self.data = {} self.update(other, **kw) - def _commit_removals(self): - l = self._pending_removals + def _commit_removals(self, _atomic_removal=_remove_dead_weakref): + pop = self._pending_removals.pop d = self.data # We shouldn't encounter any KeyError, because this method should # always be called *before* mutating the dict. - while l: - key = l.pop() - _remove_dead_weakref(d, key) + while True: + try: + key = pop() + except IndexError: + return + _atomic_removal(d, key) def __getitem__(self, key): if self._pending_removals: @@ -370,7 +373,10 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): if self._iterating: self._pending_removals.append(k) else: - del self.data[k] + try: + del self.data[k] + except KeyError: + pass self._remove = remove # A list of dead weakrefs (keys to be removed) self._pending_removals = [] @@ -384,11 +390,16 @@ class WeakKeyDictionary(_collections_abc.MutableMapping): # because a dead weakref never compares equal to a live weakref, # even if they happened to refer to equal objects. # However, it means keys may already have been removed. - l = self._pending_removals + pop = self._pending_removals.pop d = self.data - while l: + while True: + try: + key = pop() + except IndexError: + return + try: - del d[l.pop()] + del d[key] except KeyError: pass diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst new file mode 100644 index 0000000..6b4b9df --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-08-23-19-55-08.bpo-44962.J00ftt.rst @@ -0,0 +1 @@ +Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes asyncio.create_task and fixes a data loss in asyncio.run where shutdown_asyncgens is not run \ No newline at end of file -- cgit v0.12