From 57594aae5e6155d30b9192a4cd85333584339445 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sun, 6 Sep 2026 09:15:08 +0300 Subject: [PATCH 1/3] gh-154511: Consolidate IDLE's mouse wheel handling in util (GH-156974) Move wheel_event there from idlelib.tree and add x11_buttons(widget) and bind_wheel(widget, func), used by the editor, the tree and test_sidebar. wheel_event now reads the direction from the event, not the platform. Move its test from test_tree, and test the fix_ functions of util too. Co-authored-by: Claude Opus 5 --- Lib/idlelib/editor.py | 8 +- Lib/idlelib/idle_test/test_sidebar.py | 15 +- Lib/idlelib/idle_test/test_tree.py | 29 +--- Lib/idlelib/idle_test/test_util.py | 152 ++++++++++++++++++ Lib/idlelib/tree.py | 35 +--- Lib/idlelib/util.py | 49 ++++++ ...-09-05-16-00-00.gh-issue-154511.hV2wQn.rst | 5 + 7 files changed, 219 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/IDLE/2026-09-05-16-00-00.gh-issue-154511.hV2wQn.rst diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 89e1725e15f30b0..2b4b95e053ba10b 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -26,8 +26,7 @@ from idlelib import query from idlelib import replace from idlelib import search -from idlelib.tree import wheel_event -from idlelib.util import py_extensions +from idlelib.util import bind_wheel, py_extensions, wheel_event from idlelib import window from idlelib.help import _get_dochome @@ -115,10 +114,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): # Elsewhere, use right-click for popup menus. text.bind("<3>",self.right_menu_event) - text.bind('', wheel_event) - if text._windowingsystem == 'x11': - text.bind('', wheel_event) - text.bind('', wheel_event) + bind_wheel(text, wheel_event) text.bind('', self.handle_winconfig) text.bind("<>", self.cut) text.bind("<>", self.copy) diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index df84554c4f1a515..848f12a5d7c1850 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -14,7 +14,8 @@ from idlelib.percolator import Percolator import idlelib.pyshell from idlelib.pyshell import PyShell, PyShellFileList -from idlelib.util import fix_scaling, fix_word_breaks, fix_x11_paste +from idlelib.util import (fix_scaling, fix_word_breaks, fix_x11_paste, + x11_buttons) import idlelib.sidebar from idlelib.sidebar import get_end_linenumber, get_lineno @@ -689,15 +690,13 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Simulate a mouse wheel notch. Tk 8.7 replaced the X11 - # / wheel events with (whose delta is - # platform-dependent); older Tk on X11 still uses the button events. - x11_buttons = (sidebar.canvas._windowingsystem == 'x11' - and tk.TkVersion < 8.7) + # Simulate a mouse wheel notch with the events that Tk sends for + # one; the delta of a event is platform-dependent. + buttons = x11_buttons(sidebar.canvas) delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 # Scroll up. - if x11_buttons: + if buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=delta) @@ -705,7 +704,7 @@ def test_mousewheel(self): self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down. - if x11_buttons: + if buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=-delta) diff --git a/Lib/idlelib/idle_test/test_tree.py b/Lib/idlelib/idle_test/test_tree.py index b3e4c10cf9e38e7..9be9abee361f083 100644 --- a/Lib/idlelib/idle_test/test_tree.py +++ b/Lib/idlelib/idle_test/test_tree.py @@ -4,7 +4,7 @@ import unittest from test.support import requires requires('gui') -from tkinter import Tk, EventType, SCROLL +from tkinter import Tk class TreeTest(unittest.TestCase): @@ -29,32 +29,5 @@ def test_init(self): node.expand() -class TestScrollEvent(unittest.TestCase): - - def test_wheel_event(self): - # Fake widget class containing `yview` only. - class _Widget: - def __init__(widget, *expected): - widget.expected = expected - def yview(widget, *args): - self.assertTupleEqual(widget.expected, args) - # Fake event class - class _Event: - pass - # (type, delta, num, amount) - tests = ((EventType.MouseWheel, 120, -1, -5), - (EventType.MouseWheel, -120, -1, 5), - (EventType.ButtonPress, -1, 4, -5), - (EventType.ButtonPress, -1, 5, 5)) - - event = _Event() - for ty, delta, num, amount in tests: - event.type = ty - event.delta = delta - event.num = num - res = tree.wheel_event(event, _Widget(SCROLL, amount, "units")) - self.assertEqual(res, "break") - - if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_util.py b/Lib/idlelib/idle_test/test_util.py index 20721fe980c784e..d90a0784075d3e0 100644 --- a/Lib/idlelib/idle_test/test_util.py +++ b/Lib/idlelib/idle_test/test_util.py @@ -1,14 +1,166 @@ """Test util, coverage 100%""" +import sys import unittest +from unittest import mock +from test.support import requires +from test.support.isolation import runInSubprocess +import tkinter +from tkinter import EventType from idlelib import util +from idlelib.idle_test.mock_tk import Event class UtilTest(unittest.TestCase): + def test_extensions(self): for extension in {'.pyi', '.py', '.pyw'}: self.assertIn(extension, util.py_extensions) + @unittest.skipUnless(sys.platform == 'win32', 'Windows only') + @runInSubprocess() + def test_fix_win_hidpi(self): + # Awareness is process-wide and cannot be undone. + import ctypes + PROCESS_DPI_UNAWARE = 0 + util.fix_win_hidpi() + awareness = ctypes.c_int() + ctypes.OleDLL('shcore').GetProcessDpiAwareness( + None, ctypes.byref(awareness)) + self.assertNotEqual(awareness.value, PROCESS_DPI_UNAWARE) + + +class WheelTest(unittest.TestCase): + "Test the wheel functions with a widget on this display." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tkinter.Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def setUp(self): + self.text = tkinter.Text(self.root) + self.addCleanup(self.text.destroy) + + def test_x11_buttons(self): + # Only X11 before Tk 8.7 sends the wheel as button events. + text = self.text + if text._windowingsystem == 'x11' and tkinter.TkVersion < 8.7: + self.assertTrue(util.x11_buttons(text)) + else: + self.assertFalse(util.x11_buttons(text)) + + def test_bind_wheel(self): + # The events Tk sends here are the ones bound. + text = self.text + util.bind_wheel(text, util.wheel_event) + if util.x11_buttons(text): + self.assertEqual(sorted(text.bind()), + ['', '']) + else: + self.assertEqual(sorted(text.bind()), ['']) + + +class WheelEventTest(unittest.TestCase): + "Test the direction and the amount of the scroll." + + # An unmapped widget has no height and does not scroll by lines, + # so record the yview call instead of a real scroll. + def event(self, event_type, delta=0, num='??'): + # Tk leaves num '??' for a wheel event and delta 0 for a button. + return Event(type=event_type, delta=delta, num=num, + widget=mock.Mock()) + + def scroll(self, event, widget=None): + "Return the arguments of the yview call." + self.assertEqual(util.wheel_event(event, widget), 'break') + scrolled = event.widget if widget is None else widget + scrolled.yview.assert_called_once() + return scrolled.yview.call_args.args + + def test_mousewheel(self): + # Delta is positive for up on all systems. + for delta in 120, 1, 1200: + self.assertEqual(self.scroll(self.event(EventType.MouseWheel, + delta)), + ('scroll', -5, 'units')) + self.assertEqual(self.scroll(self.event(EventType.MouseWheel, + -delta)), + ('scroll', 5, 'units')) + + def test_buttons(self): + self.assertEqual(self.scroll(self.event(EventType.ButtonPress, num=4)), + ('scroll', -5, 'units')) + self.assertEqual(self.scroll(self.event(EventType.ButtonPress, num=5)), + ('scroll', 5, 'units')) + + def test_widget_argument(self): + # A tree label scrolls the canvas, not itself. + event = self.event(EventType.MouseWheel, 120) + canvas = mock.Mock() + self.assertEqual(self.scroll(event, canvas), ('scroll', -5, 'units')) + event.widget.yview.assert_not_called() + + +class FixTest(unittest.TestCase): + "Test the fix_ functions, which need a display." + + @classmethod + def setUpClass(cls): + requires('gui') + cls.root = tkinter.Tk() + cls.root.withdraw() + + @classmethod + def tearDownClass(cls): + cls.root.destroy() + del cls.root + + def test_fix_scaling(self): + from tkinter import font + root = self.root + self.addCleanup(root.tk_scaling, root.tk_scaling()) + # Both fonts go with the root; Font.delete_font is a flag. + pixels = font.Font(root=root, name='TestPixelFont', size=-16) + points = font.Font(root=root, name='TestPointFont', size=12) + + root.tk_scaling(1.0) + util.fix_scaling(root) # No scaling, no change. + self.assertEqual(int(pixels['size']), -16) + + root.tk_scaling(2.0) + util.fix_scaling(root) # A size in pixels becomes one in points. + self.assertEqual(int(pixels['size']), 12) # round(-0.75 * -16) + self.assertEqual(int(points['size']), 12) # Points are left alone. + + def test_fix_word_breaks(self): + root = self.root + util.fix_word_breaks(root) + self.assertEqual(root.tk.call('set', 'tcl_wordchars'), r'\w') + self.assertEqual(root.tk.call('set', 'tcl_nonwordchars'), r'\W') + + def test_fix_x11_paste(self): + root = self.root + classes = 'Text', 'Entry', 'Spinbox' + before = {cls: root.bind_class(cls, '<>') for cls in classes} + util.fix_x11_paste(root) + for cls in classes: + with self.subTest(cls=cls): + after = root.bind_class(cls, '<>') + if root._windowingsystem == 'x11': + # Deleting the selection makes paste replace it. + self.assertEqual( + after, + 'catch {%W delete sel.first sel.last}\n' + before[cls]) + else: + self.assertEqual(after, before[cls]) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/tree.py b/Lib/idlelib/tree.py index 182ce7189614daf..cd32f04b7c34abe 100644 --- a/Lib/idlelib/tree.py +++ b/Lib/idlelib/tree.py @@ -20,6 +20,7 @@ from tkinter.ttk import Frame, Scrollbar from idlelib.config import idleConf +from idlelib.util import bind_wheel, wheel_event from idlelib import zoomheight ICONDIR = "Icons" @@ -56,30 +57,6 @@ def listicons(icondir=ICONDIR): column = 0 root.images = images -def wheel_event(event, widget=None): - """Handle scrollwheel event. - - For wheel up, event.delta = 120*n on Windows, -1*n on darwin, - where n can be > 1 if one scrolls fast. Flicking the wheel - generates up to maybe 20 events with n up to 10 or more 1. - Macs use wheel down (delta = 1*n) to scroll up, so positive - delta means to scroll up on both systems. - - X-11 sends Control-Button-4,5 events instead. - - The widget parameter is needed so browser label bindings can pass - the underlying canvas. - - This function depends on widget.yview to not be overridden by - a subclass. - """ - up = {EventType.MouseWheel: event.delta > 0, - EventType.ButtonPress: event.num == 4} - lines = -5 if up[event.type] else 5 - widget = event.widget if widget is None else widget - widget.yview(SCROLL, lines, 'units') - return 'break' - class TreeNode: @@ -285,10 +262,7 @@ def drawtext(self): anchor="nw", window=self.label) self.label.bind("<1>", self.select_or_edit) self.label.bind("", self.flip) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) - if self.label._windowingsystem == 'x11': - self.label.bind("", lambda e: wheel_event(e, self.canvas)) - self.label.bind("", lambda e: wheel_event(e, self.canvas)) + bind_wheel(self.label, lambda e: wheel_event(e, self.canvas)) self.text_id = id if TreeNode.dy == 0: # The first row doesn't matter what the dy is, just measure its @@ -466,10 +440,7 @@ def __init__(self, master, **opts): self.canvas.bind("", self.page_down) self.canvas.bind("", self.unit_up) self.canvas.bind("", self.unit_down) - self.canvas.bind("", wheel_event) - if self.canvas._windowingsystem == 'x11': - self.canvas.bind("", wheel_event) - self.canvas.bind("", wheel_event) + bind_wheel(self.canvas, wheel_event) #if isinstance(master, Toplevel) or isinstance(master, Tk): self.canvas.bind("", self.zoom_height) self.canvas.focus_set() diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index e884e38bef90bf1..f408daf728d4a97 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -65,6 +65,55 @@ def fix_x11_paste(root): root.bind_class(cls, '<>')) +# Mouse wheel handling. + +def x11_buttons(widget): + """Return whether Tk reports wheel rotations to widget as button events. + + On X11, Tk 8.6 and older report a mouse wheel rotation as a + or event. Tk 8.7 and newer report it as a + event, as Tk always did on Windows and macOS. Which of + the two a widget gets depends on its windowing system, which is a + property of its display, so a widget is needed, not just the version. + """ + from tkinter import TkVersion + return TkVersion < 8.7 and widget._windowingsystem == 'x11' + + +def bind_wheel(widget, func): # Called in editor and tree. + "Bind func to the events that Tk sends widget for a wheel rotation." + if x11_buttons(widget): + widget.bind('', func) + widget.bind('', func) + else: + widget.bind('', func) + + +def wheel_event(event, widget=None): + """Handle a scrollwheel event by scrolling 5 lines. + + For a event, event.delta is 120*n on Windows and X11, + and -1*n on macOS, where n can be > 1 if one scrolls fast. Flicking + the wheel generates up to maybe 20 events with n up to 10 or more. + Macs use wheel down (delta = 1*n) to scroll up, so positive delta + means to scroll up on all systems. + + A or event (see x11_buttons) says up or down + by its number, and has no delta; a wheel event has no number. + + The widget parameter is needed so tree label bindings can pass the + underlying canvas. If tree is replaced by ttk.Treeview, it can go. + + This function depends on widget.yview to not be overridden by + a subclass. + """ + up = event.num == 4 if event.num in (4, 5) else event.delta > 0 + lines = -5 if up else 5 + widget = event.widget if widget is None else widget + widget.yview('scroll', lines, 'units') + return 'break' + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_util', verbosity=2) diff --git a/Misc/NEWS.d/next/IDLE/2026-09-05-16-00-00.gh-issue-154511.hV2wQn.rst b/Misc/NEWS.d/next/IDLE/2026-09-05-16-00-00.gh-issue-154511.hV2wQn.rst new file mode 100644 index 000000000000000..591c1582fa8d75a --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-09-05-16-00-00.gh-issue-154511.hV2wQn.rst @@ -0,0 +1,5 @@ +Consolidate IDLE's mouse wheel handling in ``idlelib.util``. +``wheel_event`` moves there from ``idlelib.tree`` and joins ``x11_buttons``, +which tells whether Tk reports wheel rotations to a widget as +````/```` events, and ``bind_wheel``, which binds +whichever events Tk sends. From b707ce61968161a444f18e45910d1a25f1181ab7 Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Sun, 6 Sep 2026 13:30:51 +0300 Subject: [PATCH 2/3] gh-156310: Make the iter() sequence fallback iterator safe in free-threaded build (#156311) Co-authored-by: Neil Schemenauer --- .../test_free_threading/test_iteration.py | 58 ++++++++++++++++ Lib/test/test_iter.py | 66 +++++++++++++++++++ ...08-24-12-00-00.gh-issue-156310.pSqIter.rst | 7 ++ Objects/iterobject.c | 49 +++++++++----- 4 files changed, 165 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2026-08-24-12-00-00.gh-issue-156310.pSqIter.rst diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index 44d3e9ccfdd14e0..540e8edcc676285 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -1,6 +1,8 @@ +import sys import threading import unittest from test import support +from test.support import threading_helper # The race conditions these tests were written for only happen every now and # then, even with the current numbers. To find rare race conditions, bumping @@ -112,6 +114,62 @@ def worker(): self.assert_iterator_results(results, list(seq)) +class ContendedSeqIterExhaustionTest(unittest.TestCase): + """Test draining a shared iter() fallback iterator (PySeqIter_Type). + + Sequences implementing __getitem__ but not __iter__ iterate through + PySeqIter_Type. Unlike the other tests in this file, this uses a + tiny sequence and many rounds so that many threads reach the racy + exhaustion path simultaneously (see gh-156310, where this + use-after-freed the sequence). + """ + + class Seq: + def __init__(self, n): + self.n = n + + def __getitem__(self, i): + if i >= self.n: + raise IndexError(i) + return i + + @support.refcount_test + def test_shared_iterator_exhaustion(self): + nthreads = 8 + nrounds = 20 if support.check_sanitizer(thread=True) else 100 + seq = self.Seq(4) + expected = set(range(seq.n)) + refcount_before = sys.getrefcount(seq) + + def drain(it, barrier, results): + items = [] + barrier.wait() + for item in it: + items.append(item) + results.extend(items) + + for _ in range(nrounds): + it = iter(seq) + barrier = threading.Barrier(nthreads) + results = [] + threads = [ + threading.Thread(target=drain, args=(it, barrier, results)) + for _ in range(nthreads) + ] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + self.assertIsNone(cm.exc_value) + del it + # Threads may see duplicate or missing items, but never + # invented ones. + self.assertEqual(set(results) - expected, set()) + + # A double-DECREF of the sequence does not always crash; it + # reliably shows up as a sagging reference count. + self.assertEqual(sys.getrefcount(seq), refcount_before) + + class ContendedRangeIterationTest(ContendedTupleIterationTest): def make_testdata(self, n): return range(n) diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index be9d0a709f2f4a7..fe8617309da29d0 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -2,6 +2,7 @@ import sys import unittest +from test import support from test.support import cpython_only from test.support.os_helper import TESTFN, unlink from test.support import check_free_after_iterating, ALWAYS_EQ, NEVER_EQ @@ -249,6 +250,71 @@ def test_mutating_seq_class_exhausted_iter(self): self.assertEqual(list(empit), [5, 6]) self.assertEqual(list(a), [0, 1, 2, 3, 4, 5, 6]) + @support.refcount_test + def test_seq_class_reentrant_exhaustion(self): + # gh-156310: a re-entrant next() from inside __getitem__ (or from + # __del__ of the IndexError instance) that exhausts the iterator + # used to make the outer next() DECREF the sequence a second time. + it = None + + class ReentrantGetItem: + def __init__(self): + self.calls = 0 + + def __getitem__(self, i): + self.calls += 1 + if self.calls == 1: + for _ in it: + pass + raise IndexError(i) + + seq = ReentrantGetItem() + refcount = sys.getrefcount(seq) + it = iter(seq) + self.assertEqual(list(it), []) + del it + support.gc_collect() + self.assertEqual(sys.getrefcount(seq), refcount) + + class ReentrantIndexError(IndexError): + def __del__(self): + try: + next(it) + except StopIteration: + pass + + class RaiseReentrant: + def __getitem__(self, i): + raise ReentrantIndexError(i) + + seq = RaiseReentrant() + refcount = sys.getrefcount(seq) + it = iter(seq) + self.assertEqual(list(it), []) + del it + support.gc_collect() + self.assertEqual(sys.getrefcount(seq), refcount) + + # An outer __getitem__ that succeeds after a re-entrant next() + # exhausted the iterator must not revive it. + class ReviveGetItem: + def __init__(self): + self.calls = 0 + + def __getitem__(self, i): + self.calls += 1 + if self.calls == 1: + for _ in it: + pass + if i >= 3: + raise IndexError(i) + return i + + it = iter(ReviveGetItem()) + self.assertEqual(next(it), 0) + self.assertEqual(list(it), []) + self.assertEqual(it.__length_hint__(), 0) + def test_reduce_mutating_builtins_iter(self): # This is a reproducer of issue #101765 # where iter `__reduce__` calls could lead to a segfault or SystemError diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-24-12-00-00.gh-issue-156310.pSqIter.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-24-12-00-00.gh-issue-156310.pSqIter.rst new file mode 100644 index 000000000000000..440929d62333a6f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-24-12-00-00.gh-issue-156310.pSqIter.rst @@ -0,0 +1,7 @@ +Fix memory safety issues in the :func:`iter` fallback for objects that +implement :meth:`~object.__getitem__` without :meth:`~object.__iter__` +(``PySeqIter_Type``). Sharing an iterator between threads in the +free-threaded build could use the underlying sequence after it was freed, +and re-entrant exhaustion in the default build could decrement the sequence's +reference count twice. Concurrent iteration may still see duplicate or +missing items, but it no longer corrupts the interpreter state. diff --git a/Objects/iterobject.c b/Objects/iterobject.c index 0394227cd482dbc..b5783c92c8eb689 100644 --- a/Objects/iterobject.c +++ b/Objects/iterobject.c @@ -7,14 +7,16 @@ #include "pycore_genobject.h" // _PyCoro_GetAwaitableIter() #include "pycore_iterobject.h" // _PyCallIter_NewEx() #include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyThreadState_GET() typedef struct { PyObject_HEAD - Py_ssize_t it_index; - PyObject *it_seq; /* Set to NULL when iterator is exhausted */ + Py_ssize_t it_index; /* -1 when iterator is exhausted */ + PyObject *it_seq; /* Set to NULL when iterator is exhausted + (in the default build) */ } seqiterobject; PyObject * @@ -61,26 +63,41 @@ iter_iternext(PyObject *iterator) assert(PySeqIter_Check(iterator)); it = (seqiterobject *)iterator; + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index < 0) + return NULL; seq = it->it_seq; +#ifndef Py_GIL_DISABLED if (seq == NULL) return NULL; - if (it->it_index == PY_SSIZE_T_MAX) { +#endif + if (index == PY_SSIZE_T_MAX) { PyErr_SetString(PyExc_OverflowError, "iter index too large"); return NULL; } - result = PySequence_GetItem(seq, it->it_index); + result = PySequence_GetItem(seq, index); if (result != NULL) { - it->it_index++; + /* PySequence_GetItem() can exhaust the iterator re-entrantly. + * Preserve the exhaustion sentinel if it is observed. Concurrent + * exhaustion can still race with the store, but remains memory-safe + * because the sequence stays alive. */ + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index) >= 0) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index + 1); + } return result; } if (PyErr_ExceptionMatches(PyExc_IndexError) || PyErr_ExceptionMatches(PyExc_StopIteration)) { + /* Mark the iterator exhausted before anything that can run + * arbitrary code. */ + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, -1); +#ifndef Py_GIL_DISABLED + Py_CLEAR(it->it_seq); +#endif PyErr_Clear(); - it->it_seq = NULL; - Py_DECREF(seq); } return NULL; } @@ -91,7 +108,8 @@ iter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) seqiterobject *it = (seqiterobject*)op; Py_ssize_t seqsize, len; - if (it->it_seq) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index >= 0 && it->it_seq != NULL) { if (_PyObject_HasLen(it->it_seq)) { seqsize = PySequence_Size(it->it_seq); if (seqsize == -1) @@ -100,7 +118,7 @@ iter_len(PyObject *op, PyObject *Py_UNUSED(ignored)) else { Py_RETURN_NOTIMPLEMENTED; } - len = seqsize - it->it_index; + len = seqsize - index; if (len >= 0) return PyLong_FromSsize_t(len); } @@ -119,8 +137,9 @@ iter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) * call must be before access of iterator pointers. * see issue #101765 */ - if (it->it_seq != NULL) - return Py_BuildValue("N(O)n", iter, it->it_seq, it->it_index); + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index); + if (index >= 0 && it->it_seq != NULL) + return Py_BuildValue("N(O)n", iter, it->it_seq, index); else return Py_BuildValue("N(())", iter); } @@ -134,10 +153,10 @@ iter_setstate(PyObject *op, PyObject *state) Py_ssize_t index = PyLong_AsSsize_t(state); if (index == -1 && PyErr_Occurred()) return NULL; - if (it->it_seq != NULL) { - if (index < 0) - index = 0; - it->it_index = index; + if (index < 0) + index = 0; + if (it->it_seq && FT_ATOMIC_LOAD_SSIZE_RELAXED(it->it_index) >= 0) { + FT_ATOMIC_STORE_SSIZE_RELAXED(it->it_index, index); } Py_RETURN_NONE; } From 83dbe6ae9f47f51948184f612d8443bd7ac79ff2 Mon Sep 17 00:00:00 2001 From: Timofei Ivankov <128279579+deadlovelll@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:33:49 +0300 Subject: [PATCH 3/3] gh-156512: Fix asyncio calling connection_lost() twice from resume_writing() (#156657) --- Lib/asyncio/selector_events.py | 8 +++- Lib/test/test_asyncio/test_selector_events.py | 43 +++++++++++++++++++ ...-08-30-13-25-53.gh-issue-156512.tVTOIV.rst | 2 + 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-30-13-25-53.gh-issue-156512.tVTOIV.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 16d5c1b6f0a3e19..a682063d9d3d6e3 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1127,7 +1127,9 @@ def _write_sendmsg(self): self._loop._remove_writer(self._sock_fd) if self._empty_waiter is not None: self._empty_waiter.set_result(None) - if self._closing: + # gh-156512: don't let _call_connection_lost be called twice + if self._closing and not self._conn_lost: + self._conn_lost += 1 self._call_connection_lost(None) elif self._eof: self._sock.shutdown(socket.SHUT_WR) @@ -1173,7 +1175,9 @@ def _write_send(self): self._loop._remove_writer(self._sock_fd) if self._empty_waiter is not None: self._empty_waiter.set_result(None) - if self._closing: + # gh-156512: don't let _call_connection_lost be called twice + if self._closing and not self._conn_lost: + self._conn_lost += 1 self._call_connection_lost(None) elif self._eof: self._sock.shutdown(socket.SHUT_WR) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index a323084d262ebfe..74322105a7c7577 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1195,6 +1195,49 @@ def _resume_writing(): self.assertEqual(transport.get_write_buffer_size(), 0) self.assertTrue(self.protocol.connection_lost.called) + def test_write_ready_resume_writing_closes(self): + # gh-156512: closing from resume_writing() must not lose the connection twice + self.sock.send.return_value = 2 + + def _resume_writing(): + transport.close() + + self.protocol.resume_writing.side_effect = _resume_writing + self.loop.call_exception_handler = mock.Mock() + + transport = self.socket_transport() + transport.set_write_buffer_limits(high=1, low=0) + transport.write(b'data') + + self.loop.writers[7]._run() + test_utils.run_briefly(self.loop) + + self.assertEqual(self.protocol.connection_lost.call_count, 1) + self.loop.call_exception_handler.assert_not_called() + + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') + def test_write_sendmsg_resume_writing_closes(self): + # gh-156512: same as above, for the sendmsg write path + self.sock.send.return_value = 2 + self.sock.sendmsg.return_value = 2 + + def _resume_writing(): + transport.close() + + self.protocol.resume_writing.side_effect = _resume_writing + self.loop.call_exception_handler = mock.Mock() + + transport = self.socket_transport(sendmsg=True) + transport.set_write_buffer_limits(high=1, low=0) + transport.write(b'data') + + self.loop.writers[7]._run() + test_utils.run_briefly(self.loop) + + self.assertEqual(self.protocol.connection_lost.call_count, 1) + self.loop.call_exception_handler.assert_not_called() + + class SelectorSocketTransportBufferedProtocolTests(test_utils.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2026-08-30-13-25-53.gh-issue-156512.tVTOIV.rst b/Misc/NEWS.d/next/Library/2026-08-30-13-25-53.gh-issue-156512.tVTOIV.rst new file mode 100644 index 000000000000000..16d6b7bdeae9859 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-30-13-25-53.gh-issue-156512.tVTOIV.rst @@ -0,0 +1,2 @@ +Fix :mod:`asyncio` losing a connection twice when ``resume_writing()`` +closes the transport.