From f91879b25cf77a29870e31ba8bbaace51e84f565 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Wed, 21 Oct 2026 18:21:51 -0700 Subject: [PATCH 1/3] Trace the bodies of async functions and async generators --- .../Compiler/Ast/FunctionDefinition.cs | 10 +- .../IronPython/Compiler/GeneratorRewriter.cs | 11 +- src/core/IronPython/Compiler/Parser.cs | 11 +- .../Runtime/Exceptions/TraceBack.cs | 6 + src/core/IronPython/Runtime/FunctionCode.cs | 43 +- src/dlr | 2 +- tests/suite/test_async.py | 47 +++ tests/suite/test_async_trace.py | 392 ++++++++++++++++++ tests/suite/test_generator.py | 49 +++ 9 files changed, 552 insertions(+), 19 deletions(-) create mode 100644 tests/suite/test_async_trace.py diff --git a/src/core/IronPython/Compiler/Ast/FunctionDefinition.cs b/src/core/IronPython/Compiler/Ast/FunctionDefinition.cs index ad6a45e65..13b31ae4b 100644 --- a/src/core/IronPython/Compiler/Ast/FunctionDefinition.cs +++ b/src/core/IronPython/Compiler/Ast/FunctionDefinition.cs @@ -681,11 +681,9 @@ private LightLambdaExpression CreateFunctionLambda() { List statements = new List(); - // Add beginning sequence point. For async functions this body is deferred into the nested Async/AsyncEnumerable lambda, - // and the outer factory (which runs first, and is the only one traced today) re-emits its own entry point at StartIndex; - // anchor this one at HeaderIndex (end of the signature) so the two occupy distinct source locations should the deferred - // body ever become traceable. For single-line signatures both still land on the same line (differing only in column). - statements.Add(MakeFunctionEntrySequencePoint(IsAsync ? HeaderIndex : StartIndex)); + // Add beginning sequence point: the location of the 'call' trace event. Async functions included, since this is their body too; + // their outer factory, which only creates the coroutine, is not traced (as in CPython). + statements.Add(MakeFunctionEntrySequencePoint(StartIndex)); // For generators/coroutines, we need to do a check before the first statement for Generator.Throw() / Generator.Close(). @@ -740,7 +738,6 @@ private LightLambdaExpression CreateFunctionLambda() { var throwSlot = AsyncThrowSlot; body = MSAst.Expression.Block( [cts, excBox, sendSlot, throwSlot], - MakeFunctionEntrySequencePoint(StartIndex), MSAst.Expression.Assign(cts, MSAst.Expression.New(typeof(CancellationTokenSource))), MSAst.Expression.Assign(excBox, MSAst.Expression.New(typeof(StrongBox))), MSAst.Expression.Assign(sendSlot, MSAst.Expression.New(typeof(StrongBox))), @@ -760,7 +757,6 @@ private LightLambdaExpression CreateFunctionLambda() { // rather than whatever context happened to be current at construction. body = MSAst.Expression.Block( [cts, excBox], - MakeFunctionEntrySequencePoint(StartIndex), MSAst.Expression.Assign(cts, MSAst.Expression.New(typeof(CancellationTokenSource))), MSAst.Expression.Assign(excBox, MSAst.Expression.New(typeof(StrongBox))), Ast.Call( diff --git a/src/core/IronPython/Compiler/GeneratorRewriter.cs b/src/core/IronPython/Compiler/GeneratorRewriter.cs index 69784249c..85c1ae827 100644 --- a/src/core/IronPython/Compiler/GeneratorRewriter.cs +++ b/src/core/IronPython/Compiler/GeneratorRewriter.cs @@ -373,6 +373,11 @@ protected override Expression VisitTry(TryExpression node) { if (catchYields != tryYields) { var block = new List(); + // A deferred handler runs if its variable is set, so the variables must be reset on each entry into the try: they are + // lifted, and would otherwise keep the exception of an earlier pass (e.g. of a loop) whose handler did not reach its end. + // A resumption into a deferred handler is routed past the reset, which keeps the exception for the rest of the handler. + var resetDeferredVars = new List(); + block.Add(MakeYieldRouter(node.Body.Type, tryYields, catchYields, tryStart)); block.Add(null); // empty slot to fill in later @@ -395,7 +400,8 @@ protected override Expression VisitTry(TryExpression node) { // the catch block had one. It needs to be hoisted because // the catch might contain yields. var deferredVar = c.Variable ?? Expression.Variable(c.Test, null); - LiftVariable(deferredVar); + LiftVariable(deferredVar); + resetDeferredVars.Add(DelayedAssign(Visit(deferredVar), AstUtils.Constant(null, deferredVar.Type))); // We need to ensure that filters can access the exception // variable @@ -431,7 +437,8 @@ protected override Expression VisitTry(TryExpression node) { ); } - block[1] = Expression.MakeTry(null, @try, null, null, new ReadOnlyCollection(handlers)); + resetDeferredVars.Add(Expression.MakeTry(null, @try, null, null, new ReadOnlyCollection(handlers))); + block[1] = Expression.Block(resetDeferredVars); @try = Expression.Block(block); Debug.Assert(@try.Type == node.Body.Type); handlers = Array.Empty(); // so we don't reuse these diff --git a/src/core/IronPython/Compiler/Parser.cs b/src/core/IronPython/Compiler/Parser.cs index 97251d5b6..f2eb69fa7 100644 --- a/src/core/IronPython/Compiler/Parser.cs +++ b/src/core/IronPython/Compiler/Parser.cs @@ -1126,7 +1126,7 @@ private Statement ParseDecorated() { res = cls; } else if (PeekToken() == Tokens.KeywordAsyncToken) { Eat(TokenKind.KeywordAsync); - FunctionDefinition fnc = ParseFuncDef(true); + FunctionDefinition fnc = ParseFuncDef(asyncStart: GetStart()); fnc.Decorators = decorators.ToArray(); res = fnc; } else { @@ -1139,10 +1139,11 @@ private Statement ParseDecorated() { // funcdef: [decorators] 'def' NAME parameters ':' suite // parameters: '(' [typedargslist] ')' - // this gets called with "def" as the look-ahead - private FunctionDefinition ParseFuncDef(bool isAsync = false) { + // this gets called with "def" as the look-ahead; asyncStart is the start of the preceding 'async' keyword, if any + private FunctionDefinition ParseFuncDef(int? asyncStart = null) { + bool isAsync = asyncStart.HasValue; Eat(TokenKind.KeywordDef); - var start = GetStart(); + var start = asyncStart ?? GetStart(); // like in CPython, an async function starts at its 'async' keyword string name = ReadName(); Eat(TokenKind.LeftParenthesis); @@ -1469,7 +1470,7 @@ private Statement ParseAsyncStmt(bool onlyAllowDef = false) { switch (PeekToken().Kind) { case TokenKind.KeywordDef: - return ParseFuncDef(true); + return ParseFuncDef(asyncStart: start); case TokenKind.KeywordWith: if (onlyAllowDef) goto default; return ParseAsyncWithStmt(start); diff --git a/src/core/IronPython/Runtime/Exceptions/TraceBack.cs b/src/core/IronPython/Runtime/Exceptions/TraceBack.cs index 1700743cc..e97a5ea0a 100644 --- a/src/core/IronPython/Runtime/Exceptions/TraceBack.cs +++ b/src/core/IronPython/Runtime/Exceptions/TraceBack.cs @@ -222,6 +222,12 @@ private void SetLineNumber(int newLineNum) { } FunctionCode funcCode = _debugProperties.Code; +#if FEATURE_NET_ASYNC + // The frame of a traced async body reports its location but cannot change it. + if ((funcCode.Flags & FunctionAttributes.Coroutine) != 0) { + throw PythonOps.ValueError("can't jump in an async function or async generator"); + } +#endif Dictionary> loopAndFinallyLocations = _debugProperties.LoopAndFinallyLocations; Dictionary handlerLocations = _debugProperties.HandlerLocations; diff --git a/src/core/IronPython/Runtime/FunctionCode.cs b/src/core/IronPython/Runtime/FunctionCode.cs index f46e8b1b8..e70e151af 100644 --- a/src/core/IronPython/Runtime/FunctionCode.cs +++ b/src/core/IronPython/Runtime/FunctionCode.cs @@ -754,11 +754,15 @@ private LambdaExpression GetGeneratorOrNormalLambdaTracing(PythonContext context // Under .NET-async, async bodies are lowered before they reach here and must NOT go through the generator rewriter: // * plain async (Coroutine, no Generator) via AsyncExpression, // * async generators (Generator+Coroutine) via AsyncEnumerableExpression. - // Only a plain generator (Generator without Coroutine) is rewritten here. - if ((Flags & FunctionAttributes.Generator) == 0 || (Flags & FunctionAttributes.Coroutine) != 0) { -#else - if ((Flags & (FunctionAttributes.Generator | FunctionAttributes.Coroutine)) == 0) { + // As in CPython, calling an async function only creates the coroutine (or async generator) object, so the outer factory lambda + // is not traced at all. The lowered async body is traced instead, by its own transform, whenever it runs: its Async/AsyncEnumerable + // node is swapped for a copy carrying this code's DebugContext and payload. See async-tracing-proposal.md. + if ((Flags & FunctionAttributes.Coroutine) != 0) { + var lambda = (LambdaExpression)Compiler.Ast.Node.RemoveFrame(_lambda.GetLambda()); + return (LambdaExpression)new AsyncDebugHookInjector(context.DebugContext, debugInfo).Visit(lambda); + } #endif + if ((Flags & (FunctionAttributes.Generator | FunctionAttributes.Coroutine)) == 0) { return context.DebugContext.TransformLambda((LambdaExpression)Compiler.Ast.Node.RemoveFrame(_lambda.GetLambda()), debugInfo); } @@ -1204,4 +1208,35 @@ public Dictionary LoopOrFinallyIds { } } } + +#if FEATURE_NET_ASYNC + /// + /// Swaps a lowered / for a debug-enabled copy. + /// + /// + /// Used to build the tracing variant of an async function: the debug-enabled node instruments its own body when reduced, so the + /// async body becomes traceable while the outer factory stays untraced. A function's own async node is the only one in its lambda + /// — nested defs compile to their own lambdas/codes — so a single replacement is made, and the node is not descended into. + /// + internal sealed class AsyncDebugHookInjector : ExpressionVisitor { + private readonly DebugContext _debugContext; + private readonly DebugLambdaInfo _debugLambdaInfo; + + internal AsyncDebugHookInjector(DebugContext debugContext, DebugLambdaInfo debugLambdaInfo) { + _debugContext = debugContext; + _debugLambdaInfo = debugLambdaInfo; + } + + protected override Expression VisitExtension(Expression node) { + switch (node) { + case AsyncExpression ae: + return ae.WithDebugInfo(_debugContext, _debugLambdaInfo); + case AsyncEnumerableExpression aee: + return aee.WithDebugInfo(_debugContext, _debugLambdaInfo); + default: + return base.VisitExtension(node); + } + } + } +#endif } diff --git a/src/dlr b/src/dlr index 4b380fe1d..922753f8c 160000 --- a/src/dlr +++ b/src/dlr @@ -1 +1 @@ -Subproject commit 4b380fe1dce52f95f64b4725b679fba5e4b0ca06 +Subproject commit 922753f8ccbf5186c15899116d5a407b6b1723fb diff --git a/tests/suite/test_async.py b/tests/suite/test_async.py index 22d4f5c43..f19a8c9d0 100644 --- a/tests/suite/test_async.py +++ b/tests/suite/test_async.py @@ -48,6 +48,12 @@ async def foo(): return 42 self.assertEqual(run_coro(foo()), 42) + def test_location_starts_at_async_keyword(self): + # As in CPython. It is also where a coroutine's 'call' trace event is reported. + import ast + node = ast.parse("x = 1\nasync def f():\n pass\n").body[1] + self.assertEqual((node.lineno, node.col_offset), (2, 0)) + def test_coroutine_type(self): async def foo(): return 1 @@ -171,6 +177,47 @@ async def test(): self.assertEqual(run_coro(test()), 'caught: boom') + def test_await_in_handler_left_by_continue(self): + """A handler that awaits and then leaves its iteration does not run again in a later iteration that does not raise.""" + async def noop(): + pass + + async def test(): + log = [] + for i in range(3): + try: + if i == 0: + raise ValueError() + log.append(('try', i)) + except ValueError: + await noop() + log.append(('except', i)) + continue + return log + + self.assertEqual(run_coro(test()), [('except', 0), ('try', 1), ('try', 2)]) + + def test_await_in_handler_left_by_break(self): + """As above, with the handler breaking out of an inner loop that the outer loop enters again.""" + async def noop(): + pass + + async def test(): + log = [] + for j in range(2): + for i in range(2): + try: + if j == 0: + raise ValueError() + log.append(('try', j, i)) + except ValueError: + await noop() + log.append(('except', j, i)) + break + return log + + self.assertEqual(run_coro(test()), [('except', 0, 0), ('try', 1, 0), ('try', 1, 1)]) + class AsyncWithTest(unittest.TestCase): """Tests for async with statement.""" diff --git a/tests/suite/test_async_trace.py b/tests/suite/test_async_trace.py new file mode 100644 index 000000000..c0f6d4d5c --- /dev/null +++ b/tests/suite/test_async_trace.py @@ -0,0 +1,392 @@ +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the Apache 2.0 License. +# See the LICENSE file in the project root for more information. + +""" +Tests for sys.settrace tracing of async functions and async generators. + +They also run on CPython (checked with 3.7 to 3.15), which serves as the reference: put src/core/IronPython/Lib on PYTHONPATH for iptest. +""" + +import sys +import unittest + +from iptest import run_test, is_cli, is_netcoreapp + +if is_cli: + import clr + clr.AddReference("IronPython") + from IronPython.Runtime.Operations import PythonOps + + +def drive(coro): + """Drive a coroutine to completion without an event loop; return its result.""" + try: + while True: + coro.send(None) + except StopIteration as e: + return e.value + + +def stack_depth(): + """The depth of the thread's Python frame stack (the one sys._getframe walks).""" + if is_cli: + return PythonOps.GetFunctionStack().Count + + depth, frame = 0, sys._getframe(1) + while frame is not None: + depth += 1 + frame = frame.f_back + return depth + + +class Recorder: + """ + Trace function recording the events of the given functions as (event, name, line), with lines relative to the + function's first line, and the argument of each 'return' event as (name, arg). + + It ignores the 'exception' event CPython reports for the StopIteration that ends an awaited coroutine (with no traceback). + IronPython does not report it, as no exception is involved. + Debuggers based on bdb, like pdb, skip that event anyway, and coverage tools do not use exception events. + """ + def __init__(self, *functions): + self.codes = {f.__code__ for f in functions} + self.events = [] + self.returns = [] + + def __call__(self, frame, event, arg): + code = frame.f_code + if code in self.codes and not (event == "exception" and arg[0] is StopIteration and arg[2] is None): + self.events.append((event, code.co_name, frame.f_lineno - code.co_firstlineno)) + if event == "return": + self.returns.append((code.co_name, arg)) + return self + + def run(self, fn, *args): + sys.settrace(self) + try: + return fn(*args) + finally: + sys.settrace(None) + + def drive(self, async_fn, *args): + """ + Create the coroutine async_fn(*args) and drive it, both under tracing. (Unless run with -X:Tracing, IronPython decides + whether a coroutine or generator body is traced when it is created.) + """ + return self.run(lambda: drive(async_fn(*args))) + + +@unittest.skipIf(is_cli and not is_netcoreapp, "tests tracing of the .NET async implementation (FEATURE_NET_ASYNC)") +class AsyncTraceTest(unittest.TestCase): + + def test_creating_coroutine_is_not_traced(self): + async def coro(): + return 1 + + rec = Recorder(coro) + c = rec.run(coro) + self.assertEqual(rec.events, []) + + self.assertEqual(rec.run(drive, c), 1) + self.assertEqual(rec.events, [("call", "coro", 0), ("line", "coro", 1), ("return", "coro", 1)]) + + def test_call_location_matches_sync_function(self): + # Like for a synchronous function (and in CPython), 'call' is at the first line of the header, also when it spans several lines. + def f(a, + b): + return a + + async def cf(a, + b): + return a + + rec = Recorder(f, cf) + rec.run(f, 1, 2) + rec.drive(cf, 1, 2) + self.assertEqual(rec.events, [ + ("call", "f", 0), ("line", "f", 2), ("return", "f", 2), + ("call", "cf", 0), ("line", "cf", 2), ("return", "cf", 2), + ]) + + def test_coroutine_events(self): + # Awaits of coroutines that complete synchronously do not suspend the awaiting coroutine: no 'return'/'call' pairs at them. + async def inner(x): + return x * 2 + + async def coro(n): + total = 0 + for i in range(n): + total += await inner(i) + return total + + rec = Recorder(inner, coro) + self.assertEqual(rec.drive(coro, 2), 2) + inner_events = [("call", "inner", 0), ("line", "inner", 1), ("return", "inner", 1)] + self.assertEqual(rec.events, [ + ("call", "coro", 0), ("line", "coro", 1), ("line", "coro", 2), ("line", "coro", 3), + *inner_events, + ("line", "coro", 2), ("line", "coro", 3), + *inner_events, + ("line", "coro", 2), ("line", "coro", 4), ("return", "coro", 4), + ]) + self.assertEqual(rec.returns, [("inner", 0), ("inner", 2), ("coro", 2)]) + + def test_async_generator_traced_per_resumption(self): + # As for a generator in CPython: 'call' when resumed (at the line it resumes at), 'return' when it yields (with the value). + async def agen(): + yield 1 + yield 2 + + async def consume(out): + async for v in agen(): + out.append(v) + + rec = Recorder(agen) + out = [] + rec.run(drive, consume(out)) + self.assertEqual(out, [1, 2]) + self.assertEqual(rec.events, [ + ("call", "agen", 0), ("line", "agen", 1), ("return", "agen", 1), + ("call", "agen", 1), ("line", "agen", 2), ("return", "agen", 2), + ("call", "agen", 2), ("return", "agen", 2), + ]) + if is_cli: + # CPython reports an internal wrapper (async_generator_wrapped_value) of the yielded value instead + self.assertEqual(rec.returns, [("agen", 1), ("agen", 2), ("agen", None)]) + + def test_abandoned_async_generator_leaves_no_frame(self): + async def agen(): + yield 1 + yield 2 + + async def consume(ag): + async for v in ag: + break + + def runner(): + before = stack_depth() + ag = agen() + drive(consume(ag)) + # The generator is kept alive until tracing ends, so that CPython does not finalize it (PEP 525) while traced. + return before, stack_depth(), sys._getframe().f_code.co_name, ag + + rec = Recorder(agen) + before, after, name, ag = rec.run(runner) + self.assertEqual(after, before) + self.assertEqual(name, "runner") + self.assertEqual(rec.events, [("call", "agen", 0), ("line", "agen", 1), ("return", "agen", 1)]) + + def test_interleaved_async_generators(self): + async def g1(): + yield 1 + yield 2 + + async def g2(): + yield 10 + yield 20 + yield 30 + + async def zip2(a, b, out): + while True: + try: + x = await a.__anext__() + y = await b.__anext__() + except StopAsyncIteration: + break + out.append((x, y)) + + def runner(out): + before = stack_depth() + a, b = g1(), g2() + drive(zip2(a, b, out)) + # The generators are kept alive until tracing ends, so that CPython does not finalize g2 (PEP 525) while traced. + return before, stack_depth(), (a, b) + + rec = Recorder(g1, g2) + out = [] + before, after, generators = rec.run(runner, out) + self.assertEqual(out, [(1, 10), (2, 20)]) + self.assertEqual(after, before) + self.assertEqual([e for e in rec.events if e[1] == "g1"], [ + ("call", "g1", 0), ("line", "g1", 1), ("return", "g1", 1), + ("call", "g1", 1), ("line", "g1", 2), ("return", "g1", 2), + ("call", "g1", 2), ("return", "g1", 2), + ]) + self.assertEqual([e for e in rec.events if e[1] == "g2"], [ + ("call", "g2", 0), ("line", "g2", 1), ("return", "g2", 1), + ("call", "g2", 1), ("line", "g2", 2), ("return", "g2", 2), + ]) + + @unittest.skipUnless(is_cli, "awaits a .NET Task") + def test_resumption_on_another_thread(self): + # The coroutine suspends on the driving thread and resumes on a thread-pool thread (which has no trace function, + # as sys.settrace is per thread): its frame must not stay behind on the driving thread. + from System.Threading.Tasks import Task + + async def coro(): + a = 1 + await Task.Delay(20) + return a + + def runner(): + before = stack_depth() + result = drive(coro()) + return result, before, stack_depth(), sys._getframe().f_code.co_name + + rec = Recorder(coro) + result, before, after, name = rec.run(runner) + self.assertEqual(result, 1) + self.assertEqual(after, before) + self.assertEqual(name, "runner") + self.assertEqual(rec.events, [("call", "coro", 0), ("line", "coro", 1), ("line", "coro", 2), ("return", "coro", 2)]) + + def test_caught_exception_is_reported(self): + async def coro(): + try: + raise ValueError() + except ValueError: + pass + return 1 + + rec = Recorder(coro) + self.assertEqual(rec.drive(coro), 1) + self.assertEqual(rec.events, [ + ("call", "coro", 0), ("line", "coro", 1), ("line", "coro", 2), ("exception", "coro", 2), + ("line", "coro", 3), ("line", "coro", 4), ("line", "coro", 5), ("return", "coro", 5), + ]) + + def test_escaping_exception_is_reported_once(self): + async def coro(): + raise KeyError() + + rec = Recorder(coro) + with self.assertRaises(KeyError): + rec.drive(coro) + self.assertEqual(rec.events, [("call", "coro", 0), ("line", "coro", 1), ("exception", "coro", 1), ("return", "coro", 1)]) + + def test_tracing_does_not_rerun_exception_handler(self): + # A handler that leaves its loop iteration early must not run again in a later iteration that does not raise; the handler has no + # await, so it is only tracing that puts a suspension point (its debug points) into it. + async def left_by_continue(): + log = [] + for i in range(3): + try: + if i == 0: + raise ValueError() + log.append(("try", i)) + except ValueError: + log.append(("except", i)) + continue + return log + + async def left_by_break(): + log = [] + for j in range(2): + for i in range(2): + try: + if j == 0: + raise ValueError() + log.append(("try", j, i)) + except ValueError: + log.append(("except", j, i)) + break + return log + + with self.subTest("continue"): + self.assertEqual(Recorder(left_by_continue).drive(left_by_continue), [("except", 0), ("try", 1), ("try", 2)]) + with self.subTest("break"): + self.assertEqual(Recorder(left_by_break).drive(left_by_break), [("except", 0, 0), ("try", 1, 0), ("try", 1, 1)]) + + def test_trace_function_exception_surfaces_in_body(self): + class Boom(Exception): + pass + + log = [] + + async def coro(): + try: + x = 1 + finally: + log.append("finally") + + def tracer(frame, event, arg): + if frame.f_code is coro.__code__ and event == "line" and frame.f_lineno == coro.__code__.co_firstlineno + 2: + raise Boom() + return tracer + + sys.settrace(tracer) + try: + with self.assertRaises(Boom): + drive(coro()) + finally: + sys.settrace(None) + self.assertEqual(log, ["finally"]) + + @unittest.skipIf(not is_cli and sys.version_info < (3, 11), "CPython before 3.11 raises a trace function's exception outside the resumed frame") + def test_trace_function_exception_on_resumption_surfaces_in_async_generator(self): + # The trace function raises on the 'call' event of the generator's second resumption, right after its first yield: as in CPython, + # the exception surfaces at that yield, inside the generator, and ends the iteration. + class Boom(Exception): + pass + + log = [] + + async def agen(): + try: + yield 1 + log.append("resumed") + yield 2 + finally: + log.append("finally") + + async def consume(out): + async for v in agen(): + out.append(v) + + calls = [] + + def tracer(frame, event, arg): + if frame.f_code is agen.__code__ and event == "call": + calls.append(event) + if len(calls) == 2: + raise Boom() + return tracer + + out = [] + sys.settrace(tracer) + try: + with self.assertRaises(Boom): + drive(consume(out)) + finally: + sys.settrace(None) + self.assertEqual(out, [1]) + self.assertEqual(log, ["finally"]) + + @unittest.skipUnless(is_cli, "an IronPython limitation: CPython can jump in async frames") + def test_setting_lineno_is_refused(self): + errors = [] + + async def coro(): + x = 1 + x = 2 + return x + + def tracer(frame, event, arg): + if frame.f_code is coro.__code__ and event == "line" and frame.f_lineno == coro.__code__.co_firstlineno + 2: + try: + frame.f_lineno += 1 + except Exception as e: + errors.append(type(e)) + return tracer + + sys.settrace(tracer) + try: + result = drive(coro()) + finally: + sys.settrace(None) + self.assertEqual(errors, [ValueError]) + self.assertEqual(result, 2) + + +run_test(__name__) diff --git a/tests/suite/test_generator.py b/tests/suite/test_generator.py index c384e3657..aec2c049c 100644 --- a/tests/suite/test_generator.py +++ b/tests/suite/test_generator.py @@ -534,4 +534,53 @@ def g(): yield from (lambda: [1, 2])() self.assertEqual(list(g()), [1, 2]) + def test_yield_in_handler_left_early(self): + """A handler that yields and then leaves its iteration does not run again in a later iteration that does not raise.""" + def left_by_continue(): + for i in range(3): + try: + if i == 0: + raise ValueError() + yield ('try', i) + except ValueError: + yield ('except', i) + continue + + def left_by_break(): + for j in range(2): + for i in range(2): + try: + if j == 0: + raise ValueError() + yield ('try', j, i) + except ValueError: + yield ('except', j, i) + break + + self.assertEqual(list(left_by_continue()), [('except', 0), ('try', 1), ('try', 2)]) + self.assertEqual(list(left_by_break()), [('except', 0, 0), ('try', 1, 0), ('try', 1, 1)]) + + def test_yield_in_handler_keeps_exception(self): + """The exception being handled survives the handler's yields, for the handler body and for a bare raise.""" + def uses_exception(): + try: + raise ValueError('v') + except ValueError as e: + yield 1 + yield e.args[0] + + def reraises(): + try: + raise KeyError('k') + except KeyError: + yield 1 + raise + + self.assertEqual(list(uses_exception()), [1, 'v']) + g = reraises() + self.assertEqual(next(g), 1) + with self.assertRaises(KeyError) as cm: + next(g) + self.assertEqual(cm.exception.args, ('k',)) + run_test(__name__) From 601c6814d44009b894869b62706dbab52dfdfed1 Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Thu, 22 Oct 2026 12:33:05 -0700 Subject: [PATCH 2/3] Ignore test_asyncgen on .NET Fmwk and Mono only --- tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini b/tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini index 302605c7a..90cf362e5 100644 --- a/tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini +++ b/tests/IronPython.Tests/Cases/IronPythonCasesManifest.ini @@ -5,7 +5,6 @@ Redirect=false Timeout=120000 # 2 minute timeout [IronPython.test_asyncgen] -Ignore=true RunCondition='$(FRAMEWORK)' <> '.NETFramework,Version=v4.6.2' Reason=Requires IAsyncEnumerable & IAsyncDisposable From 3564b5996f2b447da04f521c74bb8b456124041d Mon Sep 17 00:00:00 2001 From: Pavel Koneski Date: Fri, 23 Oct 2026 14:52:16 -0700 Subject: [PATCH 3/3] Clean up async tracing comments in FunctionCode --- src/core/IronPython/Runtime/FunctionCode.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/IronPython/Runtime/FunctionCode.cs b/src/core/IronPython/Runtime/FunctionCode.cs index e70e151af..ccb68ac70 100644 --- a/src/core/IronPython/Runtime/FunctionCode.cs +++ b/src/core/IronPython/Runtime/FunctionCode.cs @@ -755,8 +755,7 @@ private LambdaExpression GetGeneratorOrNormalLambdaTracing(PythonContext context // * plain async (Coroutine, no Generator) via AsyncExpression, // * async generators (Generator+Coroutine) via AsyncEnumerableExpression. // As in CPython, calling an async function only creates the coroutine (or async generator) object, so the outer factory lambda - // is not traced at all. The lowered async body is traced instead, by its own transform, whenever it runs: its Async/AsyncEnumerable - // node is swapped for a copy carrying this code's DebugContext and payload. See async-tracing-proposal.md. + // is not traced at all. The async body is traced instead. if ((Flags & FunctionAttributes.Coroutine) != 0) { var lambda = (LambdaExpression)Compiler.Ast.Node.RemoveFrame(_lambda.GetLambda()); return (LambdaExpression)new AsyncDebugHookInjector(context.DebugContext, debugInfo).Visit(lambda); @@ -794,7 +793,7 @@ private LambdaExpression GetGeneratorOrNormalLambdaTracing(PythonContext context private LightLambdaExpression GetGeneratorOrNormalLambda() { LightLambdaExpression finalCode; #if FEATURE_NET_ASYNC - // See GetGeneratorOrNormalLambda's sibling in MakeDebuggableFunctionCode: only a plain generator + // See the sibling GetGeneratorOrNormalLambdaTracing: only a plain generator // (Generator without Coroutine) is rewritten; plain async and async generators are lowered earlier. if ((Flags & FunctionAttributes.Generator) == 0 || (Flags & FunctionAttributes.Coroutine) != 0) { #else