GH-103082: Filter LINE events in VM, to simplify tool implementation.

Include/internal/pycore_frame.h

@@ -19,7 +19,6 @@ struct _frame {
  struct _PyInterpreterFrame *f_frame; /* points to the frame data */
  PyObject *f_trace; /* Trace function */
  int f_lineno; /* Current line number. Only valid if non-zero */
- int f_last_traced_line; /* The last line traced for this frame */
  char f_trace_lines; /* Emit per-line trace events? */
  char f_trace_opcodes; /* Emit per-opcode trace events? */
  char f_fast_as_locals; /* Have the fast locals of this frame been converted to a dict? */

Include/internal/pycore_instruments.h

@@ -69,13 +69,13 @@ _Py_call_instrumentation(PyThreadState *tstate, int event,
 
 extern int
 _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame,
- _Py_CODEUNIT *instr);
+ _Py_CODEUNIT *instr, _Py_CODEUNIT *prev);
 
 extern int
 _Py_call_instrumentation_instruction(
  PyThreadState *tstate, _PyInterpreterFrame* frame, _Py_CODEUNIT *instr);
 
-int
+_Py_CODEUNIT *
 _Py_call_instrumentation_jump(
  PyThreadState *tstate, int event,
  _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, _Py_CODEUNIT *target);
@@ -100,6 +100,7 @@ extern int
 _Py_Instrumentation_GetLine(PyCodeObject *code, int index);
 
 extern PyObject _PyInstrumentation_MISSING;
+extern PyObject _PyInstrumentation_DISABLE;
 
 #ifdef __cplusplus
 }

Lib/test/test_monitoring.py

@@ -524,7 +524,7 @@ def test_lines_loop(self):
  sys.monitoring.set_events(TEST_TOOL, 0)
  sys.monitoring.register_callback(TEST_TOOL, E.LINE, None)
  start = LineMonitoringTest.test_lines_loop.__code__.co_firstlineno
- self.assertEqual(events, [start+7, 21, 22, 22, 21, start+8])
+ self.assertEqual(events, [start+7, 21, 22, 21, 22, 21, start+8])
  finally:
  sys.monitoring.set_events(TEST_TOOL, 0)
  sys.monitoring.register_callback(TEST_TOOL, E.LINE, None)
@@ -1050,6 +1050,8 @@ def func3():
 def line_from_offset(code, offset):
  for start, end, line in code.co_lines():
  if start <= offset < end:
+ if line is None:
+ return f"[offset={offset}]"
  return line - code.co_firstlineno
  return -1
 
@@ -1072,9 +1074,20 @@ class BranchRecorder(JumpRecorder):
  event_type = E.BRANCH
  name = "branch"
 
+class ReturnRecorder:
+
+ event_type = E.PY_RETURN
+
+ def __init__(self, events):
+ self.events = events
+
+ def __call__(self, code, offset, val):
+ self.events.append(("return", val))
+
 
 JUMP_AND_BRANCH_RECORDERS = JumpRecorder, BranchRecorder
 JUMP_BRANCH_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder
+FLOW_AND_LINE_RECORDERS = JumpRecorder, BranchRecorder, LineRecorder, ExceptionRecorder, ReturnRecorder
 
 class TestBranchAndJumpEvents(CheckEvents):
  maxDiff = None
@@ -1098,7 +1111,6 @@ def func():
  ('jump', 'func', 4, 2),
  ('branch', 'func', 2, 2)])
 
-
  self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
  ('line', 'check_events', 10),
  ('line', 'func', 1),
@@ -1108,15 +1120,62 @@ def func():
  ('branch', 'func', 3, 6),
  ('line', 'func', 6),
  ('jump', 'func', 6, 2),
+ ('line', 'func', 2),
  ('branch', 'func', 2, 2),
  ('line', 'func', 3),
  ('branch', 'func', 3, 4),
  ('line', 'func', 4),
  ('jump', 'func', 4, 2),
+ ('line', 'func', 2),
  ('branch', 'func', 2, 2),
+ ('line', 'check_events', 11)])
+
+ def test_except_star(self):
+
+ class Foo:
+ def meth(self):
+ pass
+
+ def func():
+ try:
+ try:
+ raise KeyError
+ except* Exception as e:
+ f = Foo(); f.meth()
+ except KeyError:
+ pass
+
+
+ self.check_events(func, recorders = JUMP_BRANCH_AND_LINE_RECORDERS, expected = [
+ ('line', 'check_events', 10),
+ ('line', 'func', 1),
  ('line', 'func', 2),
+ ('line', 'func', 3),
+ ('line', 'func', 4),
+ ('branch', 'func', 4, 4),
+ ('line', 'func', 5),
+ ('line', 'meth', 1),
+ ('jump', 'func', 5, 5),
+ ('jump', 'func', 5, '[offset=114]'),
+ ('branch', 'func', '[offset=120]', '[offset=122]'),
  ('line', 'check_events', 11)])
 
+ self.check_events(func, recorders = FLOW_AND_LINE_RECORDERS, expected = [
+ ('line', 'check_events', 10),
+ ('line', 'func', 1),
+ ('line', 'func', 2),
+ ('line', 'func', 3),
+ ('raise', KeyError),
+ ('line', 'func', 4),
+ ('branch', 'func', 4, 4),
+ ('line', 'func', 5),
+ ('line', 'meth', 1),
+ ('return', None),
+ ('jump', 'func', 5, 5),
+ ('jump', 'func', 5, '[offset=114]'),
+ ('branch', 'func', '[offset=120]', '[offset=122]'),
+ ('return', None),
+ ('line', 'check_events', 11)])
 
 class TestSetGetEvents(MonitoringTestBase, unittest.TestCase):
 

Lib/test/test_pdb.py

@@ -1793,8 +1793,9 @@ def test_pdb_issue_gh_101517():
  ... 'continue'
  ... ]):
  ... test_function()
- > <doctest test.test_pdb.test_pdb_issue_gh_101517[0]>(5)test_function()
- -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace()
+ --Return--
+ > <doctest test.test_pdb.test_pdb_issue_gh_101517[0]>(None)test_function()->None
+ -> Warning: lineno is None
  (Pdb) continue
  """
 

Lib/test/test_sys.py

@@ -1446,7 +1446,7 @@ class C(object): pass
  def func():
  return sys._getframe()
  x = func()
- check(x, size('3Pii3c7P2ic??2P'))
+ check(x, size('3Pi3c7P2ic??2P'))
  # function
  def func(): pass
  check(func, size('14Pi'))

Lib/test/test_sys_settrace.py

@@ -2867,6 +2867,5 @@ def func(arg = 1):
  sys.settrace(None)
 
 
-
 if __name__ == "__main__":
  unittest.main()

Misc/NEWS.d/next/Core and Builtins/2023-05-10-20-52-29.gh-issue-103082.y3LG5Q.rst

@@ -0,0 +1,5 @@
+Change behavior of ``sys.monitoring.events.LINE`` events in
+``sys.monitoring``: Line events now occur when a new line is reached
+dynamically, instead of using a static approximation, as before. This makes
+the behavior very similar to that of "line" events in ``sys.settrace``. This
+should ease porting of tools from 3.11 to 3.12.

Objects/frameobject.c

@@ -831,7 +831,6 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
  start_stack = pop_value(start_stack);
  }
  /* Finally set the new lasti and return OK. */
- f->f_last_traced_line = new_lineno;
  f->f_lineno = 0;
  f->f_frame->prev_instr = _PyCode_CODE(f->f_frame->f_code) + best_addr;
  return 0;
@@ -854,7 +853,6 @@ frame_settrace(PyFrameObject *f, PyObject* v, void *closure)
  }
  if (v != f->f_trace) {
  Py_XSETREF(f->f_trace, Py_XNewRef(v));
- f->f_last_traced_line = -1;
  }
  return 0;
 }
@@ -1056,7 +1054,6 @@ _PyFrame_New_NoTrack(PyCodeObject *code)
  f->f_trace_opcodes = 0;
  f->f_fast_as_locals = 0;
  f->f_lineno = 0;
- f->f_last_traced_line = -1;
  return f;
 }
 

Python/bytecodes.c

@@ -3288,28 +3288,6 @@ dummy_func(
  assert(oparg >= 2);
  }
 
- inst(INSTRUMENTED_LINE, ( -- )) {
- _Py_CODEUNIT *here = next_instr-1;
- _PyFrame_SetStackPointer(frame, stack_pointer);
- int original_opcode = _Py_call_instrumentation_line(
- tstate, frame, here);
- stack_pointer = _PyFrame_GetStackPointer(frame);
- if (original_opcode < 0) {
- next_instr = here+1;
- goto error;
- }
- next_instr = frame->prev_instr;
- if (next_instr != here) {
- DISPATCH();
- }
- if (_PyOpcode_Caches[original_opcode]) {
- _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
- INCREMENT_ADAPTIVE_COUNTER(cache->counter);
- }
- opcode = original_opcode;
- DISPATCH_GOTO();
- }
-
  inst(INSTRUMENTED_INSTRUCTION, ( -- )) {
  int next_opcode = _Py_call_instrumentation_instruction(
  tstate, frame, next_instr-1);

Python/ceval.c

@@ -775,6 +775,41 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 
 #include "generated_cases.c.h"
 
+ /* INSTRUMENTED_LINE has to be here, rather than in bytecodes.c,
+ * because it needs to capture frame->prev_instr before it is updated,
+ * as happens in the standard instruction prologue.
+ */
+#if USE_COMPUTED_GOTOS
+ TARGET_INSTRUMENTED_LINE:
+#else
+ case INSTRUMENTED_LINE:
+#endif
+ {
+ _Py_CODEUNIT *prev = frame->prev_instr;
+ _Py_CODEUNIT *here = frame->prev_instr = next_instr;
+ _PyFrame_SetStackPointer(frame, stack_pointer);
+ int original_opcode = _Py_call_instrumentation_line(
+ tstate, frame, here, prev);
+ stack_pointer = _PyFrame_GetStackPointer(frame);
+ if (original_opcode < 0) {
+ next_instr = here+1;
+ goto error;
+ }
+ next_instr = frame->prev_instr;
+ if (next_instr != here) {
+ DISPATCH();
+ }
+ if (_PyOpcode_Caches[original_opcode]) {
+ _PyBinaryOpCache *cache = (_PyBinaryOpCache *)(next_instr+1);
+ /* Prevent the underlying instruction from specializing
+ * and overwriting the instrumentation. */
+ INCREMENT_ADAPTIVE_COUNTER(cache->counter);
+ }
+ opcode = original_opcode;
+ DISPATCH_GOTO();
+ }
+
+
 #if USE_COMPUTED_GOTOS
  _unknown_opcode:
 #else

Python/ceval_macros.h

@@ -334,11 +334,10 @@ do { \
 #define INSTRUMENTED_JUMP(src, dest, event) \
 do { \
  _PyFrame_SetStackPointer(frame, stack_pointer); \
-int err = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \
+next_instr = _Py_call_instrumentation_jump(tstate, event, frame, src, dest); \
  stack_pointer = _PyFrame_GetStackPointer(frame); \
- if (err) { \
+ if (next_instr == NULL) { \
  next_instr = (dest)+1; \
  goto error; \
  } \
- next_instr = frame->prev_instr; \
 } while (0);