From 174bb04917f56b3f14f03b894fba9356ae4df097 Mon Sep 17 00:00:00 2001 From: kaihackney-scale Date: Thu, 19 Nov 2026 17:03:27 -0500 Subject: [PATCH 1/2] feat(prompt_agent): let a post-turn hook give the user sim a note turn.ask_user(note=...) shows the note to an LLM user sim with the agent's reply, as what the person found checking the work themselves. The agent never sees it. A peer user sim gets it as one more text part after the reply; a built-in user sim gets it in its read as user_note. A human peer gets none. The note is recorded under metadata user_side_notes (peer) or in the stored read (built-in). Co-Authored-By: Claude Opus 5.5 --- .../task_steps/agent/user_sim_versions.py | 3 ++ .../task_step/task_steps/post_turn_hooks.py | 19 +++++--- .../task_step/task_steps/prompt_agent.py | 27 +++++++---- .../test_prompt_agent_built_in_user_sim.py | 23 ++++++++++ .../test_prompt_agent_post_turn_hooks.py | 45 +++++++++++++++++++ 5 files changed, 102 insertions(+), 15 deletions(-) diff --git a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py index c1bf4da53..785905870 100644 --- a/src/agent_env/task_step/task_steps/agent/user_sim_versions.py +++ b/src/agent_env/task_step/task_steps/agent/user_sim_versions.py @@ -370,6 +370,9 @@ def dated(r: dict, items: list) -> list[dict]: # an earlier read's items, each out += [f" [{_stamp(at(s), now)}] {s['text']}{attached_note(s.get('files'))}" for s in own] out.append("\nNEW FROM YOUR AI ASSISTANT (this is what you're answering):") out += [_message(m, _stamp(at(m), now)) for m in messages] + if read.get("user_note"): # from a post-turn hook: what the user saw checking the work, never sent to the assistant + out.append("\nWHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):") + out += [" " + line for line in read["user_note"].strip().splitlines()] out.append("\n" + CLOSING.format(first=_first(person))) out.append("Answer in JSON: " + ANSWER_LINE) return "\n".join(out) diff --git a/src/agent_env/task_step/task_steps/post_turn_hooks.py b/src/agent_env/task_step/task_steps/post_turn_hooks.py index 0afb54b71..9a8549f11 100644 --- a/src/agent_env/task_step/task_steps/post_turn_hooks.py +++ b/src/agent_env/task_step/task_steps/post_turn_hooks.py @@ -163,17 +163,17 @@ async def call_tool(self, tool: str, args: Optional[dict], role: str) -> dict: class _Answer: """The user side's answer this turn: asked at most once, then the same for every later call.""" - def __init__(self, ask_user: Callable[[], Awaitable[Answer]], + def __init__(self, ask_user: Callable[..., Awaitable[Answer]], user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> None: self.value: Optional[Answer] = None self._ask_user = ask_user self._user_triggers = user_triggers self._asking = asyncio.Lock() - async def ask_user(self) -> Answer: + async def ask_user(self, note: Optional[str] = None) -> Answer: async with self._asking: if self.value is None: - self.value = await self._ask_user() + self.value = await (self._ask_user(note=note) if note is not None else self._ask_user()) return self.value async def user_triggers(self) -> Optional[Answer]: @@ -204,9 +204,14 @@ def metadata(self) -> Mapping[str, Any]: """A read-only copy of the run's metadata, its secret keys stripped.""" return MappingProxyType(_strip_redacted_keys(self._context.metadata)) - async def ask_user(self) -> Answer: - """Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here.""" - return await self._answer.ask_user() + async def ask_user(self, note: Optional[str] = None) -> Answer: + """Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here. + + ``note`` is shown to an LLM user sim with the agent's reply, as something only it sees (what the person would + find by checking the work themselves); the agent never sees it. A human peer gets no note.""" + if note is not None and (not isinstance(note, str) or not note.strip()): + raise ValueError("a note for the user side is non-empty text") + return await self._answer.ask_user(note) async def user_triggers(self) -> Optional[Answer]: """Agent triggers alone; None, leaving no trace, when none fires.""" @@ -263,7 +268,7 @@ def _parts(message: Any, hook: PostTurnHook) -> list[dict]: async def run_post_turn_hooks(hooks: PostTurnHooks, *, context: TaskStepContext, step_id: str, number: int, last: bool, agent_reply: str, agent_parts: list[dict], - ask_user: Callable[[], Awaitable[Answer]], + ask_user: Callable[..., Awaitable[Answer]], user_triggers: Callable[[], Awaitable[Optional[Answer]]]) -> Optional[Answer]: """Run the hooks in order; the first to return a message or an end decides, and one that asked returns that.""" answer = _Answer(ask_user, user_triggers) diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 64f862c81..2aa383fa1 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -1109,6 +1109,10 @@ async def _get_trigger_state(self, deployed) -> dict: raise RuntimeError(f"could not read /triggers/state for env '{deployed.env_id}': {last_exc}") +# How a post-turn hook's note reaches a peer user sim: one more text part after the agent's reply. +PEER_NOTE = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: {note}]") + + class _UserSide: """Who answers the agent between turns: agent triggers first, then the LLM user simulator or a human peer.""" @@ -1128,10 +1132,11 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: self._shareable = shareable self._end_after: Optional[str] = None - async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) -> dict: + async def ask(self, number: int, solver_message: str, agent_parts: list[dict], note: Optional[str] = None) -> dict: """The user side's answer: ``{"message": parts}``, or ``{"end": reason}`` with any closing message to record. A user deployed by deploy_user_sim_agent may add ``end_after``: the agent answers the message, then the - conversation ends for that reason, and every later ask gets that end without asking anyone.""" + conversation ends for that reason, and every later ask gets that end without asking anyone. ``note`` (from a + post-turn hook) is shown to an LLM user sim with the reply, as something only it sees; a human peer gets none.""" if self._end_after is not None: return {"end": self._end_after} if self._decide_url is not None: @@ -1146,10 +1151,11 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict]) - return answer # no trigger fired -> fall through to the LLM user-sim for this turn (hybrid) if self._built_in is not None: - answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in) + answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in, + note=note) self._end_after = answer.get("end_after") return answer - return await self._ask_peer(number, agent_parts) + return await self._ask_peer(number, agent_parts, note) async def _ask_built_in(self, text: str) -> dict: """One turn of the user sim deploy_user_sim_agent deployed, in this conversation's session: ``{"text", @@ -1209,8 +1215,12 @@ async def triggers(self, number: int, solver_message: str) -> Optional[dict]: self._record("agent_trigger_firings", {"turn": number, "fired": resp.get("fired", [])}) return answer - async def _ask_peer(self, number: int, agent_parts: list[dict]) -> dict: + async def _ask_peer(self, number: int, agent_parts: list[dict], note: Optional[str] = None) -> dict: step = self._step + if note is not None and self._is_user_sim: + # after the reply, so the sim reads it as its own check of the work; the agent never sees this message + agent_parts = [*agent_parts, {"kind": "text", "text": PEER_NOTE.format(note=note.strip())}] + self._record("user_side_notes", {"turn": number, "note": note}) if self._url is None: self._url = get_config().get_default_human_a2a_url() user_a2a_task_id = uuid.uuid4().hex @@ -1291,12 +1301,13 @@ async def start(cls, step: PromptAgentTaskStep, context: TaskStepContext, record "opening": opening, "done_ends_run": True} return cls(step, context, person, opening, clock_envs) - async def ask(self, number: int, reply: str, ask_user_sim) -> dict: + async def ask(self, number: int, reply: str, ask_user_sim, note: Optional[str] = None) -> dict: """The user's answer to the agent's ``reply``: ``{"message": parts}``, with ``end_after`` when the agent's answer - to it ends the conversation, or ``{"end": reason}``.""" + to it ends the conversation, or ``{"end": reason}``. ``note`` goes in the read as ``user_note``: what the user + found checking the work themselves.""" now = await _clock_now(self._step, self._clock_envs) read = {"time": now, "assistant_messages": [{"at": now, "text": reply}], "phone": [], - "you_sent": self._you_sent, "done_ends_run": True} + "you_sent": self._you_sent, "done_ends_run": True, **({"user_note": note.strip()} if note else {})} self._you_sent = [] calls: list[dict] = [] diff --git a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py index 5eb2490d3..40d9b0af6 100644 --- a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py +++ b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py @@ -411,3 +411,26 @@ async def test_agent_triggers_registered_for_the_built_in_user_are_refused_befor "user sim aren't supported yet"): await _step().execute(ctx) assert "user_sim_reads" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_hook_note_is_in_the_users_read_after_the_reply_and_never_reaches_the_agent(monkeypatch): + from tst.unit.task_step.test_prompt_agent_post_turn_hooks import _hook + hook = _hook("notes", """ + async def post_turn(turn, params): + return None if turn.last else await turn.ask_user(note="The sheet has no total.") + """) + peers = Peers(monkeypatch, {AGENT: [text("Sheet's in the folder."), text("Added it.")], + USER: [said("reply", "can it have a total?")]}) + ctx = await _context(monkeypatch) + + await _step(turns=2, post_turn_hooks=[{"file_artifact_id": hook, "params": {}}]).execute(ctx) + + stored = _full(_reads(ctx)[0]) + assert stored["read"]["user_note"] == "The sheet has no total." + rendered = stored["input"] + assert rendered.index("Sheet's in the folder.") < rendered.index( + "WHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):\n The sheet has no total.") + sent_to_agent = [r[2]["params"]["message"]["parts"] for r in peers.requests(ctx) if r[1] == f"{AGENT}/a2a" + and r[2].get("method") == "message/send"] + assert all("no total" not in json.dumps(parts) for parts in sent_to_agent) diff --git a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py index f54d8a200..a03f80bb4 100644 --- a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py +++ b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py @@ -264,6 +264,51 @@ async def post_turn(turn, params): ] +NOTES = """ + async def post_turn(turn, params): + return None if turn.last else await turn.ask_user(note=f"the sheet has no total (turn {turn.number})") +""" + + +@pytest.mark.asyncio +async def test_a_hook_note_reaches_the_user_sim_after_the_reply_and_never_the_agent(monkeypatch): + peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], USER: [sim("add a total?")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _step(_hook("notes", NOTES), turns=2).execute(ctx) + + note = ("[Only you see this; the assistant doesn't. What you found when you checked its work yourself: " + "the sheet has no total (turn 1)]") + assert peers.requests(ctx)[1:] == [ + *exchange(AGENT, text("plan my week"), "", "", "task-1"), + *exchange(USER, [*text("one"), *text(note)], "", "", "task-2"), + *exchange(AGENT, text("add a total?"), "", "", "task-3"), + ] + assert ctx.metadata["user_side_notes"]["talk"] == [{"turn": 1, "note": "the sheet has no total (turn 1)"}] + + +@pytest.mark.asyncio +async def test_a_human_peer_gets_no_hook_note(monkeypatch): + monkeypatch.setenv("AGENT_ENV_HUMAN_A2A_URL", "http://human.test") + peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], "http://human.test": [text("go on")]}) + ctx = _context() + + await _step(_hook("notes", NOTES), turns=2).execute(ctx) + + assert peers.requests(ctx)[2:4] == exchange("http://human.test", text("one"), "", "", "task-2") + assert "user_side_notes" not in ctx.metadata + + +@pytest.mark.asyncio +@pytest.mark.parametrize("note", ["''", "' '", "7"]) +async def test_a_hook_note_that_is_not_text_fails_the_step(monkeypatch, note): + EnvPeers(monkeypatch, {AGENT: [text("one")], USER: [sim("ok")]}) + hook = _hook("bad", f"async def post_turn(turn, params):\n return await turn.ask_user(note={note})\n") + + with pytest.raises(ValueError, match="a note for the user side is non-empty text"): + await _step(hook, turns=2).execute(_context(user_card=_USER_SIM_CARD)) + + @pytest.mark.asyncio @pytest.mark.parametrize("returned, fragment", [ ("'wake'", "returned 'wake'"), From 0bd68349fc6f40654e34bd35942e7238c2a1d3f1 Mon Sep 17 00:00:00 2001 From: kaihackney-scale Date: Fri, 20 Nov 2026 18:59:40 -0500 Subject: [PATCH 2/2] fix(post_turn_hooks): a note that would be dropped raises; the built-in sim's notes are recorded too - The user side is asked once a turn: a note passed after it was already asked, without that note, now raises instead of being dropped with no trace. One passed after an agent trigger answered the turn has no one to reach, which is documented, not an error. - A note to a user sim deploy_user_sim_agent deployed is recorded in metadata["user_side_notes"] too, as the peer path's is, so there is one place to look for what the user sim was told. - ask's docstring says a note reaches no one when an agent trigger answers the turn or once end_after is set. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/post_turn_hooks.py | 14 ++++++- .../task_step/task_steps/prompt_agent.py | 6 ++- .../test_prompt_agent_built_in_user_sim.py | 2 + .../test_prompt_agent_post_turn_hooks.py | 42 +++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/agent_env/task_step/task_steps/post_turn_hooks.py b/src/agent_env/task_step/task_steps/post_turn_hooks.py index 9a8549f11..4a333d375 100644 --- a/src/agent_env/task_step/task_steps/post_turn_hooks.py +++ b/src/agent_env/task_step/task_steps/post_turn_hooks.py @@ -169,11 +169,19 @@ def __init__(self, ask_user: Callable[..., Awaitable[Answer]], self._ask_user = ask_user self._user_triggers = user_triggers self._asking = asyncio.Lock() + self._asked_user = False # the user sim (or human) was asked this turn, not an agent trigger + self._note: Optional[str] = None async def ask_user(self, note: Optional[str] = None) -> Answer: async with self._asking: if self.value is None: self.value = await (self._ask_user(note=note) if note is not None else self._ask_user()) + self._asked_user, self._note = True, note + elif self._asked_user and note is not None and note != self._note: + # The user side answered already; a note now would never reach it, so say so rather than drop it. + raise RuntimeError("the user side was already asked this turn" + + (" with another note" if self._note is not None else " without a note") + + "; a note passed now would never reach it") return self.value async def user_triggers(self) -> Optional[Answer]: @@ -208,7 +216,11 @@ async def ask_user(self, note: Optional[str] = None) -> Answer: """Today's user side: agent triggers, then a human or the LLM user sim, which sees only replies sent here. ``note`` is shown to an LLM user sim with the agent's reply, as something only it sees (what the person would - find by checking the work themselves); the agent never sees it. A human peer gets no note.""" + find by checking the work themselves); the agent never sees it. A human peer gets no note. + + The user side is asked once a turn and every later call gets that answer: a note passed after it was already + asked, by this hook or another, without that note raises RuntimeError rather than being dropped. A note has + no one to reach when an agent trigger answers the turn instead (``user_triggers`` fired first).""" if note is not None and (not isinstance(note, str) or not note.strip()): raise ValueError("a note for the user side is non-empty text") return await self._answer.ask_user(note) diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 2aa383fa1..652256a6f 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -1136,7 +1136,9 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict], n """The user side's answer: ``{"message": parts}``, or ``{"end": reason}`` with any closing message to record. A user deployed by deploy_user_sim_agent may add ``end_after``: the agent answers the message, then the conversation ends for that reason, and every later ask gets that end without asking anyone. ``note`` (from a - post-turn hook) is shown to an LLM user sim with the reply, as something only it sees; a human peer gets none.""" + post-turn hook) is shown to an LLM user sim with the reply, as something only it sees, and recorded in + ``metadata["user_side_notes"]``; a human peer gets none. Nobody sees it either when an agent trigger answers + the turn, or once ``end_after`` has ended the conversation.""" if self._end_after is not None: return {"end": self._end_after} if self._decide_url is not None: @@ -1151,6 +1153,8 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict], n return answer # no trigger fired -> fall through to the LLM user-sim for this turn (hybrid) if self._built_in is not None: + if note is not None: # where the peer path records it too: one place for what the user sim was told + self._record("user_side_notes", {"turn": number, "note": note}) answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in, note=note) self._end_after = answer.get("end_after") diff --git a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py index 40d9b0af6..f14ef59d9 100644 --- a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py +++ b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py @@ -428,6 +428,8 @@ async def post_turn(turn, params): stored = _full(_reads(ctx)[0]) assert stored["read"]["user_note"] == "The sheet has no total." + # recorded where the peer path records it, so there's one place to look for what the user sim was told + assert [n["note"] for n in ctx.metadata["user_side_notes"]["talk"]] == ["The sheet has no total."] rendered = stored["input"] assert rendered.index("Sheet's in the folder.") < rendered.index( "WHAT YOU FOUND WHEN YOU CHECKED ITS WORK YOURSELF (the assistant doesn't see this):\n The sheet has no total.") diff --git a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py index a03f80bb4..32b1b2688 100644 --- a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py +++ b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py @@ -616,3 +616,45 @@ async def test_with_no_armed_env_clock_stored_messages_carry_none(monkeypatch): poll_interval_seconds=0).execute(ctx) assert all("env_clock" not in m for m in _stored(ctx, peers)["messages"]) + + + +@pytest.mark.asyncio +async def test_a_note_passed_after_the_user_side_was_asked_raises_instead_of_being_dropped(): + from agent_env.task_step.task_steps.post_turn_hooks import _Answer + + asked = [] + + async def ask_user(note=None): + asked.append(note) + return {"message": text("ok")} + + async def no_trigger(): + return None + + answer = _Answer(ask_user, no_trigger) + assert await answer.ask_user() == {"message": text("ok")} + with pytest.raises(RuntimeError, match="already asked this turn without a note"): + await answer.ask_user(note="the sheet has no total") + assert await answer.ask_user() == {"message": text("ok")} and asked == [None] # asked once + + noted = _Answer(ask_user, no_trigger) + await noted.ask_user(note="first") + assert await noted.ask_user(note="first") == {"message": text("ok")} # the same note is fine + with pytest.raises(RuntimeError, match="with another note"): + await noted.ask_user(note="second") + + +@pytest.mark.asyncio +async def test_a_note_after_an_agent_trigger_answered_the_turn_has_no_one_to_reach_and_isnt_an_error(): + from agent_env.task_step.task_steps.post_turn_hooks import _Answer + + async def ask_user(note=None): + raise AssertionError("the user side isn't asked once a trigger answered") + + async def fired(): + return {"message": text("nudge from a trigger")} + + answer = _Answer(ask_user, fired) + assert await answer.user_triggers() == {"message": text("nudge from a trigger")} + assert await answer.ask_user(note="the sheet has no total") == {"message": text("nudge from a trigger")}