From b241e15840292e21f6b21e2d218b34cf8f94deb7 Mon Sep 17 00:00:00 2001 From: rimkusaurimas Date: Wed, 6 May 2026 11:58:32 +0200 Subject: [PATCH 1/3] fix(intercept): add intercept_context() context manager to prevent ContextVar leak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `set_interceptor_context` is fire-and-forget — once called it never resets the underlying ContextVars. Inside an async agent loop, the tool function and the subsequent LLM call run in the same `asyncio.Task`, so the tag set inside the tool keeps applying to LLM calls fired after the tool returns. Those LLM calls end up recorded in `provably_intercepts` with the tool's `action_name`. Two downstream symptoms in user code that calls `build_handoff_payload`: 1. `load_latest_intercept_payload(pg_url, action_name, agent_id)` does `ORDER BY created_at DESC LIMIT 1` and returns the most recent row matching the (agent_id, action_name) key — which, because of the leak, is the LLM POST that fired AFTER the tool, not the tool's own GET. The claim's `request_payload` then carries the LLM provider URL and trips the trust gate's "endpoints missing from trusted snapshot" check. 2. `get_intercept_row_id(agent_id, action_name)` returns the same wrong row id, so the Provably query record indexes the LLM completion JSON. The verbatim claim comparison in `evaluate_handoff` then always returns CAUGHT even when the agent's claim and the actual data agree. Fix: add `intercept_context()` — a contextlib context manager that calls `ContextVar.set` on enter, captures the tokens, and `reset` on exit. Same pattern as the existing `provably_self_egress()` manager. `set_interceptor_context` is kept for backward compatibility; its docstring now carries a warning recommending `intercept_context` for tool bodies. Includes a regression test that pins the leaky behavior of `set_interceptor_context` and a parallel test that proves `intercept_context` does not leak. 88/88 tests pass (was 82, +6 new). --- src/provably/intercept/__init__.py | 2 + src/provably/intercept/interceptor.py | 46 +++++++- tests/unit/test_intercept_context.py | 147 ++++++++++++++++++++++++++ 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_intercept_context.py diff --git a/src/provably/intercept/__init__.py b/src/provably/intercept/__init__.py index ac1d063..d901603 100644 --- a/src/provably/intercept/__init__.py +++ b/src/provably/intercept/__init__.py @@ -8,6 +8,7 @@ disable, enable, init_interceptor, + intercept_context, is_enabled, set_intercept_body_hook, set_intercept_url_allowlist, @@ -22,6 +23,7 @@ "disable", "enable", "init_interceptor", + "intercept_context", "is_enabled", "set_intercept_body_hook", "set_intercept_url_allowlist", diff --git a/src/provably/intercept/interceptor.py b/src/provably/intercept/interceptor.py index 01afa0f..d2b0cbd 100644 --- a/src/provably/intercept/interceptor.py +++ b/src/provably/intercept/interceptor.py @@ -3,7 +3,8 @@ from __future__ import annotations import threading -from collections.abc import Callable +from collections.abc import Callable, Generator +from contextlib import contextmanager from contextvars import ContextVar from typing import Any @@ -77,6 +78,13 @@ def set_interceptor_context(*, agent_id: str, action_name: str, intercept_index: Uses :class:`contextvars.ContextVar` so concurrent tasks/threads each see their own values. Call this immediately before invoking the agent action whose HTTP traffic should be tagged. + .. warning:: + **Fire-and-forget**: this setter never resets. Inside an async agent loop the + ContextVar persists past the tool boundary into subsequent LLM calls running in the + same :class:`asyncio.Task`, which causes those LLM calls to be tagged with the + tool's ``action_name``. Prefer :func:`intercept_context` for any code path that + runs inside an agent framework's tool execution. + Args: agent_id: Logical agent identifier; recorded in ``provably_intercepts.agent_id``. action_name: Action name; recorded in ``provably_intercepts.action_name``. @@ -88,6 +96,42 @@ def set_interceptor_context(*, agent_id: str, action_name: str, intercept_index: _ctx_intercept_index.set(intercept_index) +@contextmanager +def intercept_context( + *, agent_id: str, action_name: str, intercept_index: int = 0 +) -> Generator[None, None, None]: + """Scoped tagging for HTTP traffic emitted inside the ``with`` block. + + Drop-in replacement for :func:`set_interceptor_context` that auto-resets the + underlying :class:`contextvars.ContextVar` values on block exit, so the tag does not + leak into surrounding LLM calls in the same :class:`asyncio.Task`. + + Use this for any HTTP emitted from inside an agent framework's tool function:: + + @function_tool + def get_temperature(): + with intercept_context(agent_id="demo", action_name="get_weather"): + return requests.get(...).json() + + Nesting is supported: the prior values are restored on exit, not cleared. + + Args: + agent_id: Logical agent identifier; recorded in ``provably_intercepts.agent_id``. + action_name: Action name; recorded in ``provably_intercepts.action_name``. + intercept_index: Per-action sequence number used by the simulation hook to address a + specific intercept (e.g. "mutate the second response of action X"). Default ``0``. + """ + t_agent = _ctx_agent_id.set(agent_id) + t_action = _ctx_action_name.set(action_name) + t_index = _ctx_intercept_index.set(intercept_index) + try: + yield + finally: + _ctx_intercept_index.reset(t_index) + _ctx_action_name.reset(t_action) + _ctx_agent_id.reset(t_agent) + + def take_last_intercept_row_id() -> int | None: """Pop the row id from the most recent ``provably_intercepts`` INSERT (single-shot, thread-safe).""" global _last_intercept_row_id diff --git a/tests/unit/test_intercept_context.py b/tests/unit/test_intercept_context.py new file mode 100644 index 0000000..9a2547d --- /dev/null +++ b/tests/unit/test_intercept_context.py @@ -0,0 +1,147 @@ +"""Tests for ``intercept_context`` and the ContextVar-leak it fixes. + +The leak: ``set_interceptor_context`` writes to a ContextVar with no reset. Inside an +agent loop the tool function and the subsequent LLM call run in the same asyncio.Task, +so the tag set inside the tool keeps applying to LLM calls fired after the tool +returns. ``intercept_context`` is the scoped fix. +""" + +from __future__ import annotations + +import asyncio + +import provably.intercept.interceptor as interceptor +from provably.intercept import intercept_context, set_interceptor_context + + +def _reset_ctx() -> None: + """Bring ContextVars back to defaults for test isolation (no SDK API for this — by design).""" + set_interceptor_context(agent_id="", action_name="", intercept_index=0) + + +def test_intercept_context_sets_values_inside_block() -> None: + _reset_ctx() + try: + with intercept_context(agent_id="ag-1", action_name="get_weather", intercept_index=2): + assert interceptor._ctx_agent_id.get() == "ag-1" + assert interceptor._ctx_action_name.get() == "get_weather" + assert interceptor._ctx_intercept_index.get() == 2 + finally: + _reset_ctx() + + +def test_intercept_context_resets_to_default_on_exit() -> None: + """When called with no prior values, exit restores the default empty/zero state.""" + _reset_ctx() + try: + with intercept_context(agent_id="ag-1", action_name="get_weather"): + pass + assert interceptor._ctx_agent_id.get() == "" + assert interceptor._ctx_action_name.get() == "" + assert interceptor._ctx_intercept_index.get() == 0 + finally: + _reset_ctx() + + +def test_intercept_context_restores_prior_values_on_exit() -> None: + """Nesting: exit restores whatever values were set BEFORE the context manager entered.""" + _reset_ctx() + try: + set_interceptor_context(agent_id="outer-ag", action_name="outer-action", intercept_index=7) + with intercept_context(agent_id="inner-ag", action_name="inner-action", intercept_index=99): + assert interceptor._ctx_action_name.get() == "inner-action" + assert interceptor._ctx_agent_id.get() == "outer-ag" + assert interceptor._ctx_action_name.get() == "outer-action" + assert interceptor._ctx_intercept_index.get() == 7 + finally: + _reset_ctx() + + +def test_intercept_context_resets_even_on_exception() -> None: + _reset_ctx() + try: + try: + with intercept_context(agent_id="x", action_name="y"): + raise RuntimeError("boom") + except RuntimeError: + pass + assert interceptor._ctx_agent_id.get() == "" + assert interceptor._ctx_action_name.get() == "" + finally: + _reset_ctx() + + +# --------------------------------------------------------------------------- +# Regression: agent-loop scenario (LLM → tool → LLM in one asyncio.Task). +# +# We make three "HTTP calls" — a fake LLM POST, a tool GET that tags itself as +# ``get_weather``, then a second LLM POST. We capture what tag each "call" sees by +# reading the ContextVar (which is exactly what _insert_row does). +# --------------------------------------------------------------------------- + + +def _record_what_insert_row_would_see() -> tuple[str, str]: + """Mirror of interceptor._insert_row's tag-reading logic.""" + return ( + interceptor._ctx_agent_id.get() or "unknown", + interceptor._ctx_action_name.get() or "unknown", + ) + + +async def _agent_loop_with_set_interceptor_context() -> list[tuple[str, str]]: + """Reproduces the bug: tool calls set_interceptor_context fire-and-forget.""" + rows: list[tuple[str, str]] = [] + rows.append(_record_what_insert_row_would_see()) # LLM turn 1 + # tool body: + set_interceptor_context(agent_id="demo", action_name="get_weather") + rows.append(_record_what_insert_row_would_see()) # tool GET — tagged correctly + # tool returns; agent continues with another LLM call in the SAME task: + rows.append(_record_what_insert_row_would_see()) # LLM turn 2 + return rows + + +async def _agent_loop_with_intercept_context() -> list[tuple[str, str]]: + """Same scenario, but the tool uses the scoped intercept_context manager.""" + rows: list[tuple[str, str]] = [] + rows.append(_record_what_insert_row_would_see()) # LLM turn 1 + # tool body: + with intercept_context(agent_id="demo", action_name="get_weather"): + rows.append(_record_what_insert_row_would_see()) # tool GET — tagged correctly + # tool returns; tag is reset: + rows.append(_record_what_insert_row_would_see()) # LLM turn 2 + return rows + + +def test_set_interceptor_context_leaks_into_subsequent_calls() -> None: + """Documents the bug — turn-2 LLM call inherits the tool's action_name. + + This is the failure mode that produces wrong ``claim_urls`` (LLM URL instead of the + data-API URL) in handoff payloads built by ``payload_builder``, and the always-CAUGHT + outcome in ``evaluate_handoff`` (because the wrong intercept row gets indexed).""" + _reset_ctx() + try: + rows = asyncio.run(_agent_loop_with_set_interceptor_context()) + # turn 1: untagged + assert rows[0] == ("unknown", "unknown") + # tool: tagged + assert rows[1] == ("demo", "get_weather") + # turn 2: STILL tagged → this is the leak that produces wrong claim_urls / always-CAUGHT + assert rows[2] == ("demo", "get_weather"), ( + "Expected leak to persist with set_interceptor_context — if this assertion " + "starts failing it means set_interceptor_context now scopes itself, in which " + "case the deprecation note in its docstring should be removed." + ) + finally: + _reset_ctx() + + +def test_intercept_context_does_not_leak_into_subsequent_calls() -> None: + """The fix: turn-2 LLM call goes back to ``"unknown"`` after the tool's ``with`` block.""" + _reset_ctx() + try: + rows = asyncio.run(_agent_loop_with_intercept_context()) + assert rows[0] == ("unknown", "unknown") # turn 1 + assert rows[1] == ("demo", "get_weather") # tool + assert rows[2] == ("unknown", "unknown") # turn 2 — fixed + finally: + _reset_ctx() From e00b561801cb56517eadba1ebb34c40cd79530e4 Mon Sep 17 00:00:00 2001 From: rimkusaurimas Date: Wed, 6 May 2026 12:09:10 +0200 Subject: [PATCH 2/3] fix(intercept): remove set_interceptor_context (no backward compat shim) The fire-and-forget setter was the leaky API; ship intercept_context as the single supported way to tag intercepts. Drop the deprecation shim and update the regression test to demonstrate the leak via raw ContextVar.set() instead of the removed public function. 88/88 tests still pass. --- src/provably/__init__.py | 4 +- src/provably/intercept/__init__.py | 2 - src/provably/intercept/interceptor.py | 32 +------ tests/unit/test_intercept_context.py | 120 +++++++++++++------------- 4 files changed, 65 insertions(+), 93 deletions(-) diff --git a/src/provably/__init__.py b/src/provably/__init__.py index 5466478..5383031 100644 --- a/src/provably/__init__.py +++ b/src/provably/__init__.py @@ -20,10 +20,10 @@ disable, enable, init_interceptor, + intercept_context, is_enabled, set_intercept_body_hook, set_intercept_url_allowlist, - set_interceptor_context, take_last_intercept_row_id, ) from provably.runtime import configure_indexing @@ -59,6 +59,7 @@ "field_descriptions", "init_interceptor", "initialize_runtime", + "intercept_context", "is_enabled", "is_trusted_endpoint", "list_trusted_endpoints", @@ -67,6 +68,5 @@ "post_handoff", "set_intercept_body_hook", "set_intercept_url_allowlist", - "set_interceptor_context", "take_last_intercept_row_id", ] diff --git a/src/provably/intercept/__init__.py b/src/provably/intercept/__init__.py index d901603..c74962b 100644 --- a/src/provably/intercept/__init__.py +++ b/src/provably/intercept/__init__.py @@ -12,7 +12,6 @@ is_enabled, set_intercept_body_hook, set_intercept_url_allowlist, - set_interceptor_context, take_last_intercept_row_id, ) from .interceptor import ( @@ -27,6 +26,5 @@ "is_enabled", "set_intercept_body_hook", "set_intercept_url_allowlist", - "set_interceptor_context", "take_last_intercept_row_id", ] diff --git a/src/provably/intercept/interceptor.py b/src/provably/intercept/interceptor.py index d2b0cbd..7f68f3d 100644 --- a/src/provably/intercept/interceptor.py +++ b/src/provably/intercept/interceptor.py @@ -72,39 +72,15 @@ def set_intercept_url_allowlist(urls: list[str] | None) -> None: _url_allowlist.discard("") -def set_interceptor_context(*, agent_id: str, action_name: str, intercept_index: int = 0) -> None: - """Bind per-call context that subsequent intercepts will tag onto their inserted rows. - - Uses :class:`contextvars.ContextVar` so concurrent tasks/threads each see their own values. - Call this immediately before invoking the agent action whose HTTP traffic should be tagged. - - .. warning:: - **Fire-and-forget**: this setter never resets. Inside an async agent loop the - ContextVar persists past the tool boundary into subsequent LLM calls running in the - same :class:`asyncio.Task`, which causes those LLM calls to be tagged with the - tool's ``action_name``. Prefer :func:`intercept_context` for any code path that - runs inside an agent framework's tool execution. - - Args: - agent_id: Logical agent identifier; recorded in ``provably_intercepts.agent_id``. - action_name: Action name; recorded in ``provably_intercepts.action_name``. - intercept_index: Per-action sequence number used by the simulation hook to address a - specific intercept (e.g. "mutate the second response of action X"). Default ``0``. - """ - _ctx_agent_id.set(agent_id) - _ctx_action_name.set(action_name) - _ctx_intercept_index.set(intercept_index) - - @contextmanager def intercept_context( *, agent_id: str, action_name: str, intercept_index: int = 0 ) -> Generator[None, None, None]: """Scoped tagging for HTTP traffic emitted inside the ``with`` block. - Drop-in replacement for :func:`set_interceptor_context` that auto-resets the - underlying :class:`contextvars.ContextVar` values on block exit, so the tag does not - leak into surrounding LLM calls in the same :class:`asyncio.Task`. + Sets the underlying :class:`contextvars.ContextVar` values on enter and resets them + on exit, so the tag does not leak into surrounding LLM calls running in the same + :class:`asyncio.Task`. Use this for any HTTP emitted from inside an agent framework's tool function:: @@ -113,7 +89,7 @@ def get_temperature(): with intercept_context(agent_id="demo", action_name="get_weather"): return requests.get(...).json() - Nesting is supported: the prior values are restored on exit, not cleared. + Nesting is supported: prior values are restored on exit, not cleared. Args: agent_id: Logical agent identifier; recorded in ``provably_intercepts.agent_id``. diff --git a/tests/unit/test_intercept_context.py b/tests/unit/test_intercept_context.py index 9a2547d..633459e 100644 --- a/tests/unit/test_intercept_context.py +++ b/tests/unit/test_intercept_context.py @@ -1,9 +1,9 @@ -"""Tests for ``intercept_context`` and the ContextVar-leak it fixes. +"""Tests for ``intercept_context`` and the ContextVar-leak it prevents. -The leak: ``set_interceptor_context`` writes to a ContextVar with no reset. Inside an -agent loop the tool function and the subsequent LLM call run in the same asyncio.Task, -so the tag set inside the tool keeps applying to LLM calls fired after the tool -returns. ``intercept_context`` is the scoped fix. +The leak: a naked ``ContextVar.set()`` inside an agent loop's tool function persists past +the tool boundary into subsequent LLM calls running in the same ``asyncio.Task``, because +async tasks share their ContextVar cell unless an explicit ``reset(token)`` is paired with +the original ``set``. ``intercept_context`` is the scoped fix. """ from __future__ import annotations @@ -11,28 +11,30 @@ import asyncio import provably.intercept.interceptor as interceptor -from provably.intercept import intercept_context, set_interceptor_context +from provably.intercept import intercept_context -def _reset_ctx() -> None: - """Bring ContextVars back to defaults for test isolation (no SDK API for this — by design).""" - set_interceptor_context(agent_id="", action_name="", intercept_index=0) +def _reset_ctx_for_test_isolation() -> None: + """Bring the ContextVars back to their declared defaults between tests.""" + interceptor._ctx_agent_id.set("") + interceptor._ctx_action_name.set("") + interceptor._ctx_intercept_index.set(0) def test_intercept_context_sets_values_inside_block() -> None: - _reset_ctx() + _reset_ctx_for_test_isolation() try: with intercept_context(agent_id="ag-1", action_name="get_weather", intercept_index=2): assert interceptor._ctx_agent_id.get() == "ag-1" assert interceptor._ctx_action_name.get() == "get_weather" assert interceptor._ctx_intercept_index.get() == 2 finally: - _reset_ctx() + _reset_ctx_for_test_isolation() def test_intercept_context_resets_to_default_on_exit() -> None: """When called with no prior values, exit restores the default empty/zero state.""" - _reset_ctx() + _reset_ctx_for_test_isolation() try: with intercept_context(agent_id="ag-1", action_name="get_weather"): pass @@ -40,25 +42,25 @@ def test_intercept_context_resets_to_default_on_exit() -> None: assert interceptor._ctx_action_name.get() == "" assert interceptor._ctx_intercept_index.get() == 0 finally: - _reset_ctx() + _reset_ctx_for_test_isolation() def test_intercept_context_restores_prior_values_on_exit() -> None: """Nesting: exit restores whatever values were set BEFORE the context manager entered.""" - _reset_ctx() + _reset_ctx_for_test_isolation() try: - set_interceptor_context(agent_id="outer-ag", action_name="outer-action", intercept_index=7) - with intercept_context(agent_id="inner-ag", action_name="inner-action", intercept_index=99): - assert interceptor._ctx_action_name.get() == "inner-action" - assert interceptor._ctx_agent_id.get() == "outer-ag" - assert interceptor._ctx_action_name.get() == "outer-action" - assert interceptor._ctx_intercept_index.get() == 7 + with intercept_context(agent_id="outer", action_name="outer-action", intercept_index=7): + with intercept_context(agent_id="inner", action_name="inner-action", intercept_index=99): + assert interceptor._ctx_action_name.get() == "inner-action" + assert interceptor._ctx_agent_id.get() == "outer" + assert interceptor._ctx_action_name.get() == "outer-action" + assert interceptor._ctx_intercept_index.get() == 7 finally: - _reset_ctx() + _reset_ctx_for_test_isolation() def test_intercept_context_resets_even_on_exception() -> None: - _reset_ctx() + _reset_ctx_for_test_isolation() try: try: with intercept_context(agent_id="x", action_name="y"): @@ -68,80 +70,76 @@ def test_intercept_context_resets_even_on_exception() -> None: assert interceptor._ctx_agent_id.get() == "" assert interceptor._ctx_action_name.get() == "" finally: - _reset_ctx() + _reset_ctx_for_test_isolation() # --------------------------------------------------------------------------- # Regression: agent-loop scenario (LLM → tool → LLM in one asyncio.Task). # -# We make three "HTTP calls" — a fake LLM POST, a tool GET that tags itself as -# ``get_weather``, then a second LLM POST. We capture what tag each "call" sees by -# reading the ContextVar (which is exactly what _insert_row does). +# We simulate three "HTTP calls" by reading the ContextVars (which is exactly what +# ``_insert_row`` does at intercept time). The "tool" body sets the tag using +# ``intercept_context``; the second LLM call must NOT inherit the tool's tag. # --------------------------------------------------------------------------- -def _record_what_insert_row_would_see() -> tuple[str, str]: - """Mirror of interceptor._insert_row's tag-reading logic.""" +def _read_what_insert_row_would_record() -> tuple[str, str]: return ( interceptor._ctx_agent_id.get() or "unknown", interceptor._ctx_action_name.get() or "unknown", ) -async def _agent_loop_with_set_interceptor_context() -> list[tuple[str, str]]: - """Reproduces the bug: tool calls set_interceptor_context fire-and-forget.""" +async def _agent_loop_with_naked_set() -> list[tuple[str, str]]: + """Demonstrates WHY the context manager exists: a fire-and-forget ``ContextVar.set`` + inside the tool persists into the subsequent LLM call in the same Task.""" rows: list[tuple[str, str]] = [] - rows.append(_record_what_insert_row_would_see()) # LLM turn 1 - # tool body: - set_interceptor_context(agent_id="demo", action_name="get_weather") - rows.append(_record_what_insert_row_would_see()) # tool GET — tagged correctly + rows.append(_read_what_insert_row_would_record()) # LLM turn 1 + # tool body uses naked .set() (the buggy pattern): + interceptor._ctx_agent_id.set("demo") + interceptor._ctx_action_name.set("get_weather") + rows.append(_read_what_insert_row_would_record()) # tool GET # tool returns; agent continues with another LLM call in the SAME task: - rows.append(_record_what_insert_row_would_see()) # LLM turn 2 + rows.append(_read_what_insert_row_would_record()) # LLM turn 2 return rows async def _agent_loop_with_intercept_context() -> list[tuple[str, str]]: - """Same scenario, but the tool uses the scoped intercept_context manager.""" + """The fix: scoping with ``intercept_context`` resets the tag on tool exit.""" rows: list[tuple[str, str]] = [] - rows.append(_record_what_insert_row_would_see()) # LLM turn 1 - # tool body: + rows.append(_read_what_insert_row_would_record()) # LLM turn 1 with intercept_context(agent_id="demo", action_name="get_weather"): - rows.append(_record_what_insert_row_would_see()) # tool GET — tagged correctly - # tool returns; tag is reset: - rows.append(_record_what_insert_row_would_see()) # LLM turn 2 + rows.append(_read_what_insert_row_would_record()) # tool GET + rows.append(_read_what_insert_row_would_record()) # LLM turn 2 return rows -def test_set_interceptor_context_leaks_into_subsequent_calls() -> None: - """Documents the bug — turn-2 LLM call inherits the tool's action_name. - - This is the failure mode that produces wrong ``claim_urls`` (LLM URL instead of the - data-API URL) in handoff payloads built by ``payload_builder``, and the always-CAUGHT - outcome in ``evaluate_handoff`` (because the wrong intercept row gets indexed).""" - _reset_ctx() +def test_naked_ctx_var_set_leaks_into_subsequent_calls() -> None: + """Documents WHY ``intercept_context`` must reset on exit. If an agent loop sets a + ContextVar directly inside a tool, the tag persists into the next LLM call in the + same asyncio.Task — producing wrong ``claim_urls`` and always-CAUGHT outcomes + downstream.""" + _reset_ctx_for_test_isolation() try: - rows = asyncio.run(_agent_loop_with_set_interceptor_context()) - # turn 1: untagged - assert rows[0] == ("unknown", "unknown") - # tool: tagged - assert rows[1] == ("demo", "get_weather") - # turn 2: STILL tagged → this is the leak that produces wrong claim_urls / always-CAUGHT - assert rows[2] == ("demo", "get_weather"), ( - "Expected leak to persist with set_interceptor_context — if this assertion " - "starts failing it means set_interceptor_context now scopes itself, in which " - "case the deprecation note in its docstring should be removed." + rows = asyncio.run(_agent_loop_with_naked_set()) + assert rows[0] == ("unknown", "unknown") # turn 1 + assert rows[1] == ("demo", "get_weather") # tool + assert rows[2] == ("demo", "get_weather"), ( # turn 2 — leaks + "Naked ContextVar.set should leak into subsequent calls in the same Task. " + "If this assertion ever starts failing it means asyncio's ContextVar " + "semantics changed, in which case revisit the rationale for " + "intercept_context." ) finally: - _reset_ctx() + _reset_ctx_for_test_isolation() def test_intercept_context_does_not_leak_into_subsequent_calls() -> None: """The fix: turn-2 LLM call goes back to ``"unknown"`` after the tool's ``with`` block.""" - _reset_ctx() + _reset_ctx_for_test_isolation() try: rows = asyncio.run(_agent_loop_with_intercept_context()) assert rows[0] == ("unknown", "unknown") # turn 1 assert rows[1] == ("demo", "get_weather") # tool assert rows[2] == ("unknown", "unknown") # turn 2 — fixed finally: - _reset_ctx() + _reset_ctx_for_test_isolation() From c037911be5bc8f394294d2a0521e18653571ce76 Mon Sep 17 00:00:00 2001 From: rimkusaurimas Date: Wed, 6 May 2026 12:36:10 +0200 Subject: [PATCH 3/3] docs(intercept): clarify intercept_context usage + add CAUGHT troubleshooting checklist Two doc-only additions surfaced by a debugging session where multiple unrelated misuses of the SDK all produced the same CAUGHT outcome: - intercept_context docstring: explicit "must be used with `with`" callout warning that a bare call is a no-op, plus a note that agent_id must match the intercept_agent_id passed to build_handoff_payload. - README `### eval`: short troubleshooting checklist for unexpected CAUGHT covering the four common causes (tool body never ran, bare context-manager call, agent_id mismatch, wrong row-id helper). No code changes. 88/88 tests still pass. --- README.md | 9 +++++++++ src/provably/intercept/interceptor.py | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/README.md b/README.md index 389c962..ff9033e 100644 --- a/README.md +++ b/README.md @@ -303,6 +303,15 @@ Outcome semantics: - **`CAUGHT`** — at least one claim disagreed with the indexed value or a proof failed. - **`ERROR`** — the evaluator could not run (missing config, Provably backend unreachable, transient server error). Not evidence of tampering — the system was unhealthy, not the agent. +#### Getting `CAUGHT` and you don't expect to be? + +`CAUGHT` means the indexed value the evaluator pulled from `provably_intercepts` doesn't match the claim. In practice when this surprises you, it's almost always one of: + +1. **The tool body never ran.** `@function_tool` (or any agent-framework decorator) only registers the function — you still need an agent loop (e.g. `Runner.run(...)`) to invoke it. Bare LLM calls don't execute tools. +2. **`intercept_context(...)` was called without `with`.** It's a context manager; a bare call is a no-op (see the function's docstring). +3. **`agent_id` mismatch.** The `agent_id` you pass to `intercept_context(...)` inside the tool must match the `intercept_agent_id` you pass to `build_handoff_payload(...)` (default `"fetch_and_claim"`). Mismatch → the lookup misses → empty `request_payload`. +4. **Wrong row-id helper.** Use `get_intercept_row_id(agent_id, action_name)` to pick the row tagged with your action. `take_last_intercept_row_id()` returns the **globally** last insert (typically the final LLM POST), which is rarely what you want. + Comparison modes (the `VerificationMode` type): | Mode | Comparison | diff --git a/src/provably/intercept/interceptor.py b/src/provably/intercept/interceptor.py index 7f68f3d..d90e810 100644 --- a/src/provably/intercept/interceptor.py +++ b/src/provably/intercept/interceptor.py @@ -82,6 +82,13 @@ def intercept_context( on exit, so the tag does not leak into surrounding LLM calls running in the same :class:`asyncio.Task`. + .. important:: + **Must be used as a ``with`` statement.** A bare call like + ``intercept_context(agent_id="demo", action_name="get_weather")`` is a no-op + (returns a context-manager object that is immediately discarded; the body never + runs and no ContextVar is set). Subsequent intercepts will be tagged + ``("unknown", "unknown")``. + Use this for any HTTP emitted from inside an agent framework's tool function:: @function_tool @@ -93,6 +100,10 @@ def get_temperature(): Args: agent_id: Logical agent identifier; recorded in ``provably_intercepts.agent_id``. + **Must match** the ``intercept_agent_id`` you later pass to + :func:`provably.build_handoff_payload` (default ``"fetch_and_claim"``); + otherwise the (agent_id, action_name) lookup misses and the claim ends up + with no recorded request payload. action_name: Action name; recorded in ``provably_intercepts.action_name``. intercept_index: Per-action sequence number used by the simulation hook to address a specific intercept (e.g. "mutate the second response of action X"). Default ``0``.