diff --git a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py index 2098450d1..89cc8cc22 100644 --- a/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py +++ b/src/agent_env/task_step/task_steps/agent/rate_user_satisfaction.py @@ -341,6 +341,11 @@ def _always_on_session(record: dict) -> _Session: if record.get("end_reason") in ERROR_ENDS: session.skipped = f"the run ended on {record['end_reason']}" return session + if record.get("error_type") == "user_sim_invalid": + # The user-sim gate cut the run off as an error (on_fail "fail"): the user sim broke it, not the agent. A plain + # on_fail "end" sets no error, and the run is rated as one that ended. + session.skipped = "the user sim's answer still read as the assistant's after its rewrites" + return session timeline = record.get("timeline_tail") or [] if record.get("object_url"): try: @@ -369,6 +374,10 @@ async def _prompt_agent_session(context: TaskStepContext, run_id: str, response: if response.error_type == "user_sim_error": session.skipped = f"the user sim failed: {(response.error_message or '').removeprefix(SIM_FAILED_PREFIX)}" return session + if response.error_type == "user_sim_invalid": + # The user-sim gate cut the conversation off: the user sim, not the agent, broke it. + session.skipped = "the user sim's message still read as the assistant's after its rewrites" + return session if response.error_type or response.error_message: session.skipped = "the agent's turn failed" return session diff --git a/src/agent_env/task_step/task_steps/always_on.py b/src/agent_env/task_step/task_steps/always_on.py index 84f75763b..94329c869 100644 --- a/src/agent_env/task_step/task_steps/always_on.py +++ b/src/agent_env/task_step/task_steps/always_on.py @@ -48,7 +48,7 @@ closing wake, with the step's wrap-up line and the last-turn notice, and the run ends with end reason time_budget. - End reasons: end_time, user_done (one sitting), max_wakes, time_budget, finished_nothing_pending or stuck (nothing can wake the agent, with nothing pending or with something still pending), agent_turn_failed, agent_turn_timeout, - user_sim_error, step_error. With no end_time, a world that only repeats ends the run too (``IDLE_JUMPS_MAX``, in the + user_sim_error, user_sim_invalid, step_error. With no end_time, a world that only repeats ends the run too (``IDLE_JUMPS_MAX``, in the loop's docstring), so a recurring trigger that never wakes the agent can't move the clock forever. What was still pending is recorded (``pending_at_end``): replies held, messages the user never read, user lines and alerts never delivered, the agent's checks not yet due, contact replies in progress. - Graders read what the user saw. The prompt response is every message_user text in order, each with its pretend @@ -64,6 +64,12 @@ being when the reply reached the agent, so checks built on those records grade the run instead of skipping it. A reply that never reached it (her last read's, or one still held at the end) has ``delivered`` false and no ``user_reads_at``. +- ``user_sim_gate`` (user_sim_gate.py) checks each answer the user sim would send before the agent gets it. One that + reads as the assistant's is held back and the user sim, told why in its session, answers again, up to the gate's + ``max_rewrites``; each read's event records the checks (``user_sim_gate``: verdicts, rewrites, the first message). + An answer that still fails is never delivered and ends the run (user_sim_invalid, also recorded in + ``metadata["user_sim_invalid"][step_id]``): with ``on_fail`` "fail" it sets ``error_type`` too, so verifiers skip + the run; "end" ends it plainly; "send" delivers it anyway. Her last read's answer, never delivered, isn't checked. - The run's metadata keeps a summary in ``metadata["always_on"][step_id]`` (small enough for a running job to report back); the full timeline goes to the object store, at the summary's ``object_url``. Its ``user_sim`` names the user sim's session (``context_id``), which rate_user_satisfaction asks in once the run is over. @@ -123,6 +129,7 @@ fetch_native_trajectory, prescribe_formats, ) +from agent_env.task_step.task_steps.user_sim_gate import judge_call_config, judge_setup_failed, run_gate from agent_env.task_step.task_steps.agent.user_sim_common import iso as _iso, person_of, task_zone, utc as _utc from agent_env.task_step.task_steps.agent.user_sim_common import attached, file_name, for_agent, recorded_opening @@ -181,6 +188,7 @@ "agent_turn_failed": "The assistant's turn failed.", "agent_turn_timeout": "The assistant's turn timed out.", "user_sim_error": "The user simulator failed.", + "user_sim_invalid": "The user simulator's answer still read as the assistant's after its rewrites.", "step_error": "The run stopped on an error.", } # The run's metadata keeps a summary; the full timeline goes to the object store. When storing fails, the summary keeps @@ -213,6 +221,8 @@ # ================================================================================================================== class AlwaysOnTaskStep(PromptAgentTaskStep): type: ClassVar[str] = "always_on" + # Its own loop checks each user-sim answer before the agent gets it (_Run._gate, with user_sim_gate.run_gate). + USER_SIM_GATE_SUPPORTED: ClassVar[Optional[bool]] = True entity_refs = () field_scopes = {k: v for k, v in PromptAgentTaskStep.field_scopes.items() if k != "user_model"} @@ -277,8 +287,15 @@ def settle_cap_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SE def read_cap_seconds(self) -> float: """The longest one read's answer may take: READ_ASKS user sim turns, each with its grace and trajectory - fetch; an ask past it is refused, and an answer past it ends the read as a user sim error.""" - return READ_ASKS * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) + fetch, and with a user_sim_gate one more turn per rewrite and a judge call per check (every message, the + first and each rewrite); an ask past it is refused, and an answer past it ends the read as a user sim error.""" + asks, judging = READ_ASKS, 0.0 + gate = self.user_sim_gate + if gate is not None: + asks += gate.max_rewrites + if gate.judge_model: + judging = (gate.max_rewrites + 1) * gate.judge_timeout_seconds + return asks * (self.user_timeout_seconds + TURN_GRACE_SECONDS + TRAJECTORY_FETCH_SECONDS) + judging def wake_reserve_seconds(self, contact_seconds: float = CONTACT_TIMEOUT_DEFAULT_SECONDS) -> float: """The longest one turn of the loop, or the run's closing, can take, each part hard-capped: a clock jump's @@ -327,12 +344,16 @@ def to_dict(self) -> dict: base[k] = getattr(self, k) for k in self._OWN: base[k] = getattr(self, k) + if self.user_sim_gate is not None: + base["user_sim_gate"] = self.user_sim_gate.to_dict() return base @classmethod def from_dict(cls, data: dict) -> AlwaysOnTaskStep: fields = {k: data[k] for k in cls._PROMPT_FIELDS if k in data and data[k] is not None} own = {k: data[k] for k in cls._OWN if k in data and data[k] is not None} + if data.get("user_sim_gate") is not None: + fields["user_sim_gate"] = data["user_sim_gate"] return cls(**cls._base_from_dict(data), retry_config=data.get("retry_config"), **fields, **own) def task_problems(self, steps: list[TaskStep]) -> list[str]: @@ -377,6 +398,10 @@ def __init__(self, step: AlwaysOnTaskStep, context: TaskStepContext): self.held: list[dict] = [] # user replies held for their response time: {deliver_at, text} self.you_sent: list[dict] = [] # the user's own lines since their last read: {at, text} self.history: list[dict] = [] # every read and its answer, for the plug-in + # The conversation as the user-sim gate reads it, (role, text); and the gate's judge settings, set up once. + self.gate_history: list[tuple[str, str]] = [] + self.gate_call_config = None + self.gate_setup_error: Optional[str] = None self.replies = 0 self.gave_up = False self.final_sent = False @@ -477,6 +502,11 @@ async def go(self) -> None: raise StepContextError(f"always_on: the user sim agent '{us['agent_name']}' is not deployed") self.user_url = self.user_agent.a2a_url or self.user_agent.api_url self.plugin = await self._load_plugin(us.get("plugin")) + if self.step.user_sim_gate is not None and not callable(getattr(self.plugin, "parse", None)): + # A rewrite is the user sim's answer again, in the plugin's own format: without its parse, a rewrite + # would read as nothing and never reach the agent. + raise ValueError(f"always_on: user_sim_gate needs the user sim's plugin to define parse(raw), which reads " + f"one of its answers as answer() does; plugin {us.get('plugin')!r} has none") self.tz = ZoneInfo(self.zone) self.clock_envs = [d for d in ctx.deployed_envs if d.supports(EXT_CLOCK_URI, "state")] @@ -1015,6 +1045,8 @@ async def _capped(self, work, what: str, seconds: Optional[float] = None): async def _open(self) -> None: """The opening wake: the user's request, with what came due for the agent.""" self.you_sent.append(dict(self.opening)) + if self.opening.get("text"): + self.gate_history.append(("user", self.opening["text"])) opened_at = _utc(self.opening["at"]) await self._chat_user(self._opening_parts(lambda text: self._chat_part(opened_at, text, kind="opening"))) due = await take_due(self.notification_envs, self.recipient) if self.notification_envs else [] @@ -1506,6 +1538,13 @@ async def ask(text: str) -> dict: except TimeoutError: # the plug-in ran past the read's cap: a user sim error, as a failed ask is ans = {"outcome": "no_reply", "message": ""} calls.append({"text": None, "structured": None, "error": f"the read ran past its cap ({cap:.0f}s)"}) + self.gate_history += [("assistant", m["text"]) for m in seen] + gate_record = None + if (self.step.user_sim_gate is not None and not last and ans.get("outcome") in ("reply", "done") + and (ans.get("message") or "").strip() and not (calls and calls[-1].get("error"))): + ans, gate_record = await self._gate(ans, ask) + if ans.get("outcome") in ("reply", "done") and (ans.get("message") or "").strip(): + self.gate_history.append(("user", ans["message"])) self.history.append({"read": read, "answer": ans}) mode = self.read_mode latency = self._latency(mode) @@ -1520,7 +1559,11 @@ async def ask(text: str) -> dict: user_sim_trajectory_urls=[c.get("trajectory_url") for c in calls], user_sim_turns=ans.get("turns", len(calls)), shown=ans.get("shown"), past_turn_cap=ans.get("past_turn_cap"), last_read=last, - retried=ans.get("retried"), first_answer=ans.get("first_answer")) + retried=ans.get("retried"), first_answer=ans.get("first_answer"), + **({"user_sim_gate": gate_record} if gate_record else {}), + # as prompt_agent's read: what reached the agent is ``message``, what she first wrote this + **({"first_message": gate_record["first_message"]} + if gate_record and "first_message" in gate_record else {})) outcome = ans.get("outcome") said = (ans.get("message") or "").strip() hook = {"hook": HOOK_NAME, "turn": self.wakes, "read_at": _iso(now), @@ -1556,6 +1599,58 @@ async def ask(text: str) -> dict: if calls and calls[-1].get("error") and not last: self.end_reason = self.end_reason or "user_sim_error" + async def _gate(self, ans: dict, ask) -> tuple[dict, dict]: + """The user sim's answer after the user-sim gate (user_sim_gate.py): each failing message is held back and the + user sim, told why through ``ask`` in its session, answers again, up to the gate's limit. Returns the answer to + use and the gate's record for the read. A message that still fails is never delivered: the answer becomes a + no_reply, and the run ends ("user_sim_invalid"; an error, so verifiers skip it, when ``on_fail`` is "fail"), + unless ``on_fail`` is "send".""" + gate = self.step.user_sim_gate + if self.gate_call_config is None and self.gate_setup_error is None and gate.judge_model: + try: + self.gate_call_config = judge_call_config(gate, self.ctx.metadata.get("user_overrides", {})) + except Exception as e: # as in prompt_agent: the patterns' verdict stands; with patterns off, it raises + self.gate_setup_error = judge_setup_failed(gate, e) + logger.warning(f"always_on: user-sim gate {self.gate_setup_error}; using patterns only") + latest = {"ans": ans} + verdicts: list[dict] = [] + rewrites: list[dict] = [] + + async def rewrite(note: str) -> Optional[str]: + got = await ask(note) + # The plugin's own parse: a plugin with another answer format reads its rewrite as it reads its answers. + new = self.plugin.parse(got["structured"] if isinstance(got.get("structured"), dict) else got.get("text")) + rewrites.append({**{k: new.get(k) for k in ("outcome", "message", "used", "schema_errors")}, + "user_sim_error": got.get("error")}) + if got.get("error"): # read() ends the run on it (user_sim_error), as on any failed ask + latest["ans"] = {**ans, "outcome": "no_reply", "message": "", "used": None} + return None + latest["ans"] = {**ans, **new} + said = (new.get("message") or "").strip() + return said if new.get("outcome") in ("reply", "done") and said else None + + def on_verdict(attempt: int, text: str, verdict) -> None: + verdicts.append({"attempt": attempt, "chars": len(text), **verdict.to_dict()}) + if not verdict.passed: + logger.warning(f"always_on wake {self.wakes}: user-sim answer failed the gate (attempt {attempt + 1}/" + f"{gate.max_rewrites + 1}): {'; '.join(verdict.reasons())[:300]}") + + outcome = await run_gate(gate, ans["message"], self.gate_history, rewrite, call_config=self.gate_call_config, + judge_unavailable=self.gate_setup_error, on_verdict=on_verdict) + record = {"verdicts": verdicts, "rewrites": rewrites, "invalid": outcome.invalid} + if rewrites: + record["first_message"] = ans["message"] + if not outcome.invalid: + return latest["ans"], record + reasons = outcome.reasons() + self.ctx.metadata.setdefault("user_sim_invalid", {})[self.step.id] = {"wake": self.wakes, "reasons": reasons} + if gate.on_fail == "fail": + self.error = (f"user-sim answer at wake {self.wakes} still reads as the assistant's after " + f"{gate.max_rewrites} rewrite(s): {'; '.join(reasons)[:500]}") + self.error_type = "user_sim_invalid" + self.end_reason = self.end_reason or "user_sim_invalid" + return {**latest["ans"], "outcome": "no_reply", "message": "", "used": None}, record + async def _ask_user_sim(self, text: str, deadline: float) -> dict: """One user sim turn on ``text``, cut off at its time limit and grace or at the read's ``deadline`` (time.monotonic()), whichever comes first.""" diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 2383c3d69..a67064cf4 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -50,6 +50,7 @@ from agent_env.task_step.task_step import RetryConfig, StepRole, TaskStep, TaskStepDependency from agent_env.task_step.task_steps.notification_delivery import delivered_parts, labels_zone, next_message, take_due from agent_env.task_step.task_steps.post_turn_hooks import PostTurnHooks, run_post_turn_hooks, validate_hook_specs +from agent_env.task_step.task_steps.user_sim_gate import UserSimGate, judge_call_config, judge_setup_failed, run_gate from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.user_sim_common import iso, person_of, task_zone, utc from agent_env.task_step.task_steps.agent.user_sim_common import attached, for_agent, opening_text, recorded_opening @@ -177,6 +178,16 @@ class PromptAgentTaskStep(TaskStep): FieldScope.USER)} DEFAULT_PROMPT_TIMEOUT_SECONDS: ClassVar[int] = 600 DEFAULT_POLL_INTERVAL_SECONDS: ClassVar[int] = 2 + # Whether this step type applies user_sim_gate. None: it does when it keeps prompt_agent's conversation; a + # subclass that runs its own loop sets True once that loop calls user_sim_gate.run_gate. + USER_SIM_GATE_SUPPORTED: ClassVar[Optional[bool]] = None + _CONVERSATION_METHODS: ClassVar[tuple[str, ...]] = ("_execute", "_execute_conversation", "_converse") + + @classmethod + def applies_user_sim_gate(cls) -> bool: + if cls.USER_SIM_GATE_SUPPORTED is not None: + return cls.USER_SIM_GATE_SUPPORTED + return all(getattr(cls, m) is getattr(PromptAgentTaskStep, m) for m in cls._CONVERSATION_METHODS) def __init__( self, @@ -212,6 +223,7 @@ def __init__( snapshot_config: Optional[SnapshotConfig] = None, live_trajectory: bool = False, post_turn_hooks: Optional[list[dict]] = None, + user_sim_gate: Optional[UserSimGate | dict] = None, ): # `prompt` is the legacy text-only field; `parts` is the multimodal # form that's the authoritative A2A wire shape. Exactly one must be set: @@ -258,6 +270,15 @@ def __init__( self.snapshot_config = snapshot_config self.live_trajectory = live_trajectory self.post_turn_hooks = post_turn_hooks + # Checks each user-sim message before the agent gets it (user_sim_gate.py). A dict is accepted + # too, as JSON task files and the kwargs form of put() carry it. + self.user_sim_gate = (UserSimGate.from_dict(user_sim_gate) if isinstance(user_sim_gate, dict) + else user_sim_gate) + if self.user_sim_gate is not None and not type(self).applies_user_sim_gate(): + raise ValueError( + f"{type(self).__name__} '{id}': user_sim_gate is applied by prompt_agent's own conversation, which " + f"this step type replaces; it would be ignored. Call user_sim_gate.run_gate from its loop and set " + f"USER_SIM_GATE_SUPPORTED = True, or leave user_sim_gate out") @staticmethod def _validate_parts(parts: list[dict]) -> None: @@ -317,6 +338,8 @@ def to_dict(self) -> dict: base["live_trajectory"] = True if self.post_turn_hooks is not None: base["post_turn_hooks"] = self.post_turn_hooks + if self.user_sim_gate is not None: + base["user_sim_gate"] = self.user_sim_gate.to_dict() return base @classmethod @@ -354,6 +377,7 @@ def from_dict(cls, data: dict) -> PromptAgentTaskStep: ), live_trajectory=data.get("live_trajectory", False), post_turn_hooks=data.get("post_turn_hooks"), + user_sim_gate=data.get("user_sim_gate"), ) def _trajectory_prefix(self) -> str: @@ -661,8 +685,12 @@ async def _converse( self, context, url=user_url, user_sim=user_sim, is_user_sim=is_user_sim, conversation_id=conversation_id, decide_url=decide_url, shareable=sent_to_target, decides_quietly=card_request_accepts(decide_method.get("request") or {}, _QUIET_DECIDE), + opening_text="\n".join(p.get("text", "") for p in initial_parts if p.get("kind") == "text"), built_in=sim, usage=usage, ) + if self.user_sim_gate is not None and not is_user_sim: + logger.warning(f"prompt_agent '{self.id}': user_sim_gate is set but the user side isn't a user-sim " + "agent-env deployed; no message is checked") # Set by a user's done with a message: the agent answers it, then the conversation ends for this reason. end_after: Optional[str] = None @@ -846,6 +874,10 @@ def record_turn() -> None: # As always_on marks it: verifiers skip the run instead of grading the agent's latest draft as its answer. error_type = "user_sim_error" final_error_for_response = final_error_for_response or f"The user simulator failed: {sim.error}" + if not error_type and user_side.gate_failure is not None: + # The conversation was cut off by the gate: verifiers skip it rather than grade the agent's latest draft. + error_type = "user_sim_invalid" + final_error_for_response = final_error_for_response or f"user_sim_gate: {user_side.gate_failure}" context.prompt_responses.append(PromptResponse( prompt_id=self.prompt_id, response=final_terminal.response_text, @@ -868,6 +900,9 @@ def record_turn() -> None: usage=usage.entries(), )) + if user_side.gate_failure is not None: + raise RuntimeError(f"user_sim_gate: {user_side.gate_failure}") + if final_state == TaskState.failed: detail_parts = [] if final_terminal.error_type: @@ -1117,7 +1152,8 @@ class _UserSide: def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: Optional[str], user_sim: Optional[DeployedAgent], is_user_sim: bool, conversation_id: str, decide_url: Optional[str], decides_quietly: bool, shareable: set[str], - built_in: Optional[_BuiltInUserSim] = None, usage: Optional[StepUsage] = None) -> None: + opening_text: str = "", built_in: Optional[_BuiltInUserSim] = None, + usage: Optional[StepUsage] = None) -> None: self._step = step self._built_in = built_in self._usage = usage @@ -1129,6 +1165,13 @@ def __init__(self, step: PromptAgentTaskStep, context: TaskStepContext, *, url: self._decide_url = decide_url self._decides_quietly = decides_quietly self._shareable = shareable + # The conversation as the user-sim gate reads it: (role, text), role "user" or "assistant". + self._history: list[tuple[str, str]] = [("user", opening_text)] if opening_text else [] + self._gate = step.user_sim_gate if is_user_sim else None + self._gate_call_config = None + self._gate_judge_setup_error: Optional[str] = None + # Set once a user-sim message fails the gate after its rewrites, with on_fail "fail". + self.gate_failure: Optional[str] = None self._end_after: Optional[str] = None async def ask(self, number: int, solver_message: str, agent_parts: list[dict], note: Optional[str] = None) -> dict: @@ -1137,7 +1180,17 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict], n conversation ends for that reason, and every later ask gets that end without asking anyone. ``note`` (from a post-turn hook) is shown to an LLM user sim with the reply, as something only it sees, and recorded in ``metadata["user_side_notes"]``; a human peer gets none. Nobody sees it either when an agent trigger answers - the turn, or once ``end_after`` has ended the conversation.""" + the turn, or once ``end_after`` has ended the conversation. It goes with the first message only, never with a + user-sim gate's rewrite request, and the gate's judge is shown it.""" + self._history.append(("assistant", solver_message or "")) + answer = await self._answer(number, agent_parts, solver_message, note) + if "message" in answer and "end" not in answer: + self._history.append(("user", "\n".join(p.get("text", "") for p in answer["message"] + if p.get("kind") == "text"))) + return answer + + async def _answer(self, number: int, agent_parts: list[dict], solver_message: str, + note: Optional[str] = None) -> dict: if self._end_after is not None: return {"end": self._end_after} if self._decide_url is not None: @@ -1156,6 +1209,11 @@ async def ask(self, number: int, solver_message: str, agent_parts: list[dict], n self._record("user_side_notes", {"turn": number, "note": note}) answer = await self._built_in.ask(number, _agent_text(solver_message, agent_parts), self._ask_built_in, note=note) + if self._gate is not None and "message" in answer: + async def rewrite_built_in(why: str) -> dict: # the gate's request alone: the hook's note went once + return await self._built_in.rewrite(number, why, self._ask_built_in) + + answer = await self._gated(number, answer, rewrite_built_in, hook_note=note) self._end_after = answer.get("end_after") return answer return await self._ask_peer(number, agent_parts, note) @@ -1219,6 +1277,56 @@ async def triggers(self, number: int, solver_message: str) -> Optional[dict]: return answer async def _ask_peer(self, number: int, agent_parts: list[dict], note: Optional[str] = None) -> dict: + answer = await self._send_to_peer(number, agent_parts, note) + if self._gate is None or "end" in answer: + return answer + + async def rewrite_peer(why: str) -> dict: # the gate's request alone: the hook's note went with the first send + return await self._send_to_peer(number, [{"kind": "text", "text": why}]) + + return await self._gated(number, answer, rewrite_peer, hook_note=note) + + async def _gated(self, number: int, answer: dict, ask_rewrite, hook_note: Optional[str] = None) -> dict: + """Check the user-sim's message; while it fails, ask for a rewrite with ``ask_rewrite(note)``, which returns an + answer shaped as ``ask``'s, up to the gate's limit (``run_gate``). The judge is shown ``hook_note``, what a + post-turn hook told the user sim it found, so a message reporting that check reads as the user's.""" + gate = self._gate + if self._gate_call_config is None and self._gate_judge_setup_error is None and gate.judge_model: + # Judge setup (its key, its endpoint) failing leaves the patterns' verdict standing, with the error + # recorded on it (run_gate); with patterns off it fails the step (judge_setup_failed). + try: + self._gate_call_config = judge_call_config(gate, self._context.metadata.get("user_overrides", {})) + except Exception as e: + self._gate_judge_setup_error = judge_setup_failed(gate, e) + logger.warning(f"user-sim gate {self._gate_judge_setup_error}; using patterns only") + latest = {"answer": answer} + + async def rewrite(note: str) -> Optional[str]: + reply = await ask_rewrite(note) + latest["answer"] = reply + return None if "end" in reply else _text_of(reply["message"]) + + def on_verdict(attempt: int, text: str, verdict) -> None: + self._record("user_sim_gate", {"turn": number, "attempt": attempt, "chars": len(text), **verdict.to_dict()}) + if not verdict.passed: + logger.warning(f"Turn {number}: user-sim message failed the gate (attempt {attempt + 1}/" + f"{gate.max_rewrites + 1}): {'; '.join(verdict.reasons())[:300]}") + + outcome = await run_gate(gate, _text_of(answer["message"]), self._history, rewrite, + call_config=self._gate_call_config, judge_unavailable=self._gate_judge_setup_error, + on_verdict=on_verdict, user_note=hook_note) + if not outcome.invalid: + return latest["answer"] + self._context.metadata.setdefault("user_sim_invalid", {})[self._step.id] = { + "turn": number, "reasons": outcome.reasons()} + if self._built_in is not None: + self._built_in.withhold() + if gate.on_fail == "fail": + self.gate_failure = (f"user-sim message at turn {number} still reads as the assistant's after " + f"{gate.max_rewrites} rewrite(s): {'; '.join(outcome.reasons())[:500]}") + return {"end": "user_sim_invalid"} + + async def _send_to_peer(self, number: int, agent_parts: list[dict], note: Optional[str] = None) -> dict: step = self._step if note is not None and self._is_user_sim: # after the reply, so the sim reads it as its own check of the work; the agent never sees this message @@ -1282,6 +1390,10 @@ def _record_usage(self, message: dict) -> protocol.TerminalResponse: return term +def _text_of(parts: list[dict]) -> str: + return "\n".join(p.get("text", "") for p in parts if p.get("kind") == "text") + + class _BuiltInUserSim: """The user deployed by deploy_user_sim_agent, read as always_on reads it, in one sitting: each agent reply is one read, done ends the conversation, and there is no phone.""" @@ -1347,6 +1459,51 @@ async def ask(text: str) -> dict: return {"message": [{"kind": "text", "text": ans["message"]}], "end_after": "user_done"} return {"end": "user_done" if outcome == "done" else "user_no_reply"} + def withhold(self) -> None: + """The gate held the read's answer back for good (its rewrites failed too): as in always_on, the read's record + becomes a no_reply with no message, keeping ``first_message`` and ``gate_rewrites``.""" + reads = self._context.metadata.get("user_sim_reads", {}).get(self._step.id) or [] + if reads: + reads[-1].setdefault("first_message", reads[-1].get("message")) + reads[-1].update(outcome="no_reply", message="", used=None) + + async def rewrite(self, number: int, note: str, ask_user_sim) -> dict: + """The user's answer again, after the user-sim gate held its message back with ``note``, in the same session; + shaped as ``ask``'s. It replaces the held answer in the reply count, so ``max_replies`` holds whichever answer is + sent. As always_on's does, the read's record holds the answer delivered (``outcome``, ``message``, ``used``, + ``schema_errors`` are the latest rewrite's), keeps what the user sim first answered as ``first_message``, and + lists each rewrite under ``gate_rewrites``.""" + got = await ask_user_sim(note) + ans = user_sim_versions.parse(got["structured"] if isinstance(got.get("structured"), dict) else got.get("text")) + error = got.get("error") + rewrite = {k: ans.get(k) for k in ("outcome", "message", "used", "schema_errors")} + rewrite.update(user_sim_error=error, user_sim_trajectory_url=got.get("trajectory_url")) + reads = self._context.metadata.get("user_sim_reads", {}).get(self._step.id) or [] + if reads: + reads[-1].setdefault("first_message", reads[-1].get("message")) + reads[-1].setdefault("gate_rewrites", []).append(rewrite) + reads[-1].update({k: v for k, v in rewrite.items() if k in ("outcome", "message", "used", "schema_errors")}) + outcome, said = ans.get("outcome"), (ans.get("message") or "").strip() + if self.history: + held = self.history[-1]["answer"] + # ask counted the held answer if it was a reply; count this one in its place. + held_counted = held.get("outcome") == "reply" and bool((held.get("message") or "").strip()) + self.replies += int(outcome == "reply" and bool(said) and not error) - int(held_counted) + # What the user is shown later counts this answer as hers: the one that reached the agent. + self.history[-1]["answer"] = {**held, **ans} + logger.info(f"Turn {number} user sim rewrite: outcome={outcome} error={error}") + if error: + self.error = error + return {"end": "user_sim_error"} + if outcome == "reply" and said: + if self._max_replies and self.replies >= int(self._max_replies): + logger.info(f"Turn {number} user sim: reply {self.replies} of max_replies {self._max_replies}") + return {"message": [{"kind": "text", "text": ans["message"]}], "end_after": "user_gave_up"} + return {"message": [{"kind": "text", "text": ans["message"]}]} + if outcome == "done" and said: + return {"message": [{"kind": "text", "text": ans["message"]}], "end_after": "user_done"} + return {"end": "user_done" if outcome == "done" else "user_no_reply"} + async def _record(self, number: int, read: dict, ans: dict, calls: list[dict], error: Optional[str]) -> None: """Record the read as always_on does: a summary in ``metadata["user_sim_reads"][step_id]``, the whole read (what the user sim was shown, and its raw answers) in the object store at the summary's ``object_url``.""" diff --git a/src/agent_env/task_step/task_steps/user_sim_gate.py b/src/agent_env/task_step/task_steps/user_sim_gate.py new file mode 100644 index 000000000..7778a3b10 --- /dev/null +++ b/src/agent_env/task_step/task_steps/user_sim_gate.py @@ -0,0 +1,409 @@ +"""A check on every simulated-user message before prompt_agent sends it to the agent. + +An LLM playing the user can slip into the assistant's role: it answers its own question, reports work as +done, or offers to do the agent's job. Once one such message is in the conversation the rest of the run +is no longer a test of the agent. The gate checks each user-sim message, asks the user-sim to rewrite one +that fails, and, if the rewrites fail too, ends the conversation and marks the run. + +Two checks, either of which can fail a message: + +- ``patterns``: phrases and shapes that belong to the assistant's side (a report of work done, a + formatted deliverable, offers to do the work). Free, and deliberately narrow. +- ``judge_model``: one model call that reads the last turns and the candidate message and says whether + the message is written as the user. + +With both, a judge that reads the message as the user's clears a hit on a phrase alone (``PHRASE_REASONS``), which +counts at any length; a hit on a report's shape and length stays a failure. + +Opt in with ``user_sim_gate`` on a ``prompt_agent`` step whose user is a user-sim agent-env deployed: a peer +user-sim, or the built-in one deploy_user_sim_agent deploys, whose rewrite is parsed as its own answer. ``always_on`` +applies it to each of the built-in user-sim's answers. A step type that runs its own conversation loop applies the gate +with ``run_gate``, and sets ``USER_SIM_GATE_SUPPORTED``:: + + {"judge_model": "", "max_rewrites": 2, "on_fail": "fail"} +""" +from __future__ import annotations + +import json +import logging +import re +from dataclasses import asdict, dataclass, field +from typing import Awaitable, Callable, ClassVar, Optional + +logger = logging.getLogger(__name__) + +ON_FAIL = ("fail", "end", "send") + +# ── Patterns ────────────────────────────────────────────────────────────────────────────────────────── + +_REPORT_OPENING = re.compile( + r"^\s*(?:\*\*)?(?:done\b|all done|checked\b|short (?:answer|version)\b" + r"|here'?s (?:what (?:i|we) (?:found|did|checked|changed)|the (?:full |short |whole )?(?:picture|summary|breakdown" + r"|rundown|update|status)|where (?:things|we|it|you) stands?|how (?:it|things) (?:stands?|looks?))" + r"|summary\b|update:|findings\b|tl;?dr\b)", + re.I, +) +_WORK_DONE = re.compile( + r"\bI(?:'ve| have)? (?:checked|reviewed|pulled|updated|put together|drafted|flagged|verified|" + r"cross-checked|went through|compared|audited|analy[sz]ed|reconciled|fixed|created|added|" + r"rewrote|built|calculated|ran)\b", + re.I, +) +_NOW_LIVE = re.compile(r"\b(?:now live|is now (?:up|live))\b", re.I) +# A user says these about someone else's work too ("the invoice has been sent to me already"), so they count only on a +# message long enough to be a report. +_DELIVERED = re.compile( + r"\b(?:(?:emails?|messages?|drafts?|replies|invites?|notes?) (?:sent|drafted|posted|scheduled)\b|incorporated\b|" + r"(?:the|a|my) (?:cleaner|updated|revised|corrected|final) version\b|has been (?:updated|created|sent|posted))", + re.I, +) +# Phrases the assistant says and a user doesn't, at any length. A user does say "I don't have access to that folder", +# "Should I send you the spreadsheet?", "I'll pull the numbers tonight", "I can send it to you", "I apologize for the +# confusion" and "send me the results", so those aren't here. +_AS_ASSISTANT = re.compile( + r"\b(?:as an ai|i'?m an? (?:ai|assistant|language model)|my previous (?:answer|response|reply)|" + r"i apologi[sz]e for my (?:previous|earlier|last) (?:answer|response|reply|message)|" + r"let me know if you'?d like me to|would you like me to|want me to (?:draft|send|update|create|go ahead)|" + r"standing by|ready when you are|parked until|" + r"anything else (?:you(?:'d)? (?:want|like) (?:me )?to|to (?:cover|look at|dig into))|" + r"what do you want (?:me )?to do next|corrections (?:noted|taken)|" + r"how (?:can|may) i (?:help|assist)|i'?m (?:here|ready) to help|what would you like (?:me )?to (?:work on|do) today)\b", + re.I, +) +_TABLE = re.compile(r"^\s*\|?\s*:?-{3,}:?\s*\|", re.M) +_HEADER = re.compile(r"^\s*#{1,4} \S", re.M) +_BOLD_BULLET = re.compile(r"^\s*(?:[-*•]|\d+\.)\s+\*\*", re.M) + + +class PatternLimits: + """How long or how formatted a message must be before a shape-based pattern counts. Phrases that only the + assistant says (offers, "standing by", "now live") count at any length; shapes a user can also write (an + opening like "Done.", a table, "I checked") count only on a message long enough to be a report.""" + + REPORT_OPENING_MIN_CHARS: ClassVar[int] = 300 + DELIVERED_MIN_CHARS: ClassVar[int] = 300 + FORMATTED_MIN_CHARS: ClassVar[int] = 600 + FORMATTED_MIN_HEADERS: ClassVar[int] = 2 + FORMATTED_MIN_BOLD_BULLETS: ClassVar[int] = 3 + WORK_REPORT_MIN_CHARS: ClassVar[int] = 500 + WORK_REPORT_MIN_PHRASES: ClassVar[int] = 2 + + +TALKS_LIKE_THE_ASSISTANT = "it talks like the assistant (offers to do the work, waits on the user, or speaks as the AI)" +NOW_LIVE = "it says something is now live" +# The reasons a phrase gives at any length: a judge that reads the message as the user's clears them. The rest need a +# report's shape and length, and stay failures whatever the judge says. +PHRASE_REASONS = frozenset({TALKS_LIKE_THE_ASSISTANT, NOW_LIVE}) + + +def pattern_reasons(text: str) -> list[str]: + """Why ``text`` reads as the assistant's message, one reason per pattern that matched; empty if none.""" + lim = PatternLimits + reasons = [] + n = len(text) + if _AS_ASSISTANT.search(text): + reasons.append(TALKS_LIKE_THE_ASSISTANT) + if _REPORT_OPENING.search(text) and n > lim.REPORT_OPENING_MIN_CHARS: + reasons.append("it opens like a report of work done") + if _NOW_LIVE.search(text): + reasons.append(NOW_LIVE) + if _DELIVERED.search(text) and n > lim.DELIVERED_MIN_CHARS: + reasons.append("it reports something delivered or revised") + formatted = (_TABLE.search(text) or len(_HEADER.findall(text)) >= lim.FORMATTED_MIN_HEADERS + or len(_BOLD_BULLET.findall(text)) >= lim.FORMATTED_MIN_BOLD_BULLETS) + if formatted and n > lim.FORMATTED_MIN_CHARS: + reasons.append("it is a formatted deliverable (a table, headings or bold bullet points)") + work = _WORK_DONE.findall(text) + if work and (n > lim.WORK_REPORT_MIN_CHARS or len(work) >= lim.WORK_REPORT_MIN_PHRASES): + reasons.append("it reports checking or building things itself") + return reasons + + +# ── Judge ───────────────────────────────────────────────────────────────────────────────────────────── + +_JUDGE_PROMPT = """You check one message in a test conversation. A model is playing a USER who gave an AI ASSISTANT \ +a task. The assistant does the work; the user asks, answers questions the assistant put to them, decides, approves, \ +pushes back, or ends the conversation. + +Decide whether the CANDIDATE message is written as the user. It is NOT written as the user when it does the \ +assistant's job: it answers the question the user should be asking, reports work as done or findings it produced \ +("Done. Here's what I found", "I've updated the page"), presents analysis or a deliverable, apologizes for "my \ +previous answer", offers to do work ("I'll draft it", "Standing by", "Want me to send it?"), or speaks as the AI. + +It IS written as the user when it asks, instructs, approves, pushes back, answers a question the assistant asked, \ +or shares the user's own knowledge, even if it is long, formatted, quotes the assistant's numbers, or pastes an \ +email or document. Length and markdown alone are not evidence. + +Judge only whose voice the message is in. A user message that repeats an earlier one, ignores or contradicts what \ +the assistant said, states a decision or a fact the user holds, or seems off topic is still written as the user. + +CONVERSATION SO FAR (most recent last): +{history} +{user_note} +CANDIDATE (the user's next message): +<<< +{candidate} +>>> + +Answer in JSON: {{"in_role": true or false, "reason": "one sentence citing the candidate"}}""" + +# What a post-turn hook told the user it found (prompt_agent's turn.ask_user(note=...)): a user who says it is +# reporting their own check, not the assistant's work. +_USER_NOTE = """ +WHAT THE USER FOUND checking the work themselves just now (only the user saw this, not the assistant). Saying it in \ +the user's own words ("I checked the folder and the sheet isn't there") is written as the user: +<<< +{note} +>>> +""" + +_JUDGE_SCHEMA = { + "type": "object", + "properties": {"in_role": {"type": "boolean"}, "reason": {"type": "string"}}, + "required": ["in_role", "reason"], + "additionalProperties": False, +} + + +def render_history(history: list[tuple[str, str]], turns: int, max_chars: int) -> str: + """The last ``turns`` messages as ``USER:`` / ``ASSISTANT:`` blocks, each cut to ``max_chars``; none for 0.""" + out = [] + for role, text in (history[-turns:] if turns else []): # history[-0:] would be all of it + body = text if len(text) <= max_chars else text[:max_chars] + f"\n[… {len(text) - max_chars} more characters]" + out.append(f"{role.upper()}:\n{body}") + return "\n\n".join(out) or "(nothing yet)" + + +def parse_judge(content: str) -> Optional[dict]: + """The judge's JSON object, tolerating text around it; None if there's none with a boolean ``in_role``.""" + start = content.find("{") + if start < 0: + return None + try: + obj = json.JSONDecoder().raw_decode(content[start:])[0] + except ValueError: + return None + return obj if isinstance(obj, dict) and isinstance(obj.get("in_role"), bool) else None + + +# ── The gate ────────────────────────────────────────────────────────────────────────────────────────── + +@dataclass +class UserSimGate: + """How ``prompt_agent`` checks user-sim messages; see the module docstring.""" + + patterns: bool = True + judge_model: Optional[str] = None + max_rewrites: int = 2 + # What happens when a message still fails after its rewrites: "fail" ends the conversation and fails the + # step, "end" ends the conversation and lets the step finish, "send" sends it anyway (record only). + on_fail: str = "fail" + context_turns: int = 4 + # The judge alone fails only a message at least this long; a shorter one also needs a pattern. Swaps are long or + # match a pattern, and without the scenario the judge misreads short scripted answers that correct the agent. + judge_min_chars: int = 300 + context_chars: int = 3000 + judge_timeout_seconds: float = 60 + + # How much of a judge error and of the judge's reason a verdict keeps. + RECORDED_ERROR_CHARS: ClassVar[int] = 300 + RECORDED_REASON_CHARS: ClassVar[int] = 500 + + def __post_init__(self) -> None: + """Refuse a setting of the wrong type or range here, when the task loads, rather than mid-conversation.""" + def need(ok: bool, name: str, what: str) -> None: + if not ok: + raise ValueError(f"user_sim_gate.{name} must be {what}, not {getattr(self, name)!r}") + + def whole(v: object) -> bool: # an int, and not a bool (which is one in Python) + return isinstance(v, int) and not isinstance(v, bool) + + need(isinstance(self.patterns, bool), "patterns", "true or false") + need(self.judge_model is None or (isinstance(self.judge_model, str) and bool(self.judge_model.strip())), + "judge_model", "a model name or null") + need(whole(self.max_rewrites) and self.max_rewrites >= 0, "max_rewrites", "a whole number >= 0") + need(self.on_fail in ON_FAIL, "on_fail", f"one of {ON_FAIL}") + need(whole(self.context_turns) and self.context_turns >= 0, "context_turns", "a whole number >= 0") + need(whole(self.judge_min_chars) and self.judge_min_chars >= 0, "judge_min_chars", "a whole number >= 0") + need(whole(self.context_chars) and self.context_chars > 0, "context_chars", "a whole number > 0") + need(isinstance(self.judge_timeout_seconds, (int, float)) and not isinstance(self.judge_timeout_seconds, bool) + and self.judge_timeout_seconds > 0, "judge_timeout_seconds", "a number > 0") + if not self.patterns and not self.judge_model: + raise ValueError("user_sim_gate needs patterns, a judge_model, or both") + + @classmethod + def from_dict(cls, data: Optional[dict]) -> Optional[UserSimGate]: + """None (no gate) for None; the defaults for {}, as for ``UserSimGate()``.""" + if data is None: + return None + known = set(cls.__dataclass_fields__) + unknown = set(data) - known + if unknown: + raise ValueError(f"user_sim_gate: unknown keys {sorted(unknown)}") + return cls(**data) + + def to_dict(self) -> dict: + return asdict(self) + + async def check(self, candidate: str, history: list[tuple[str, str]], *, call_config=None, + judge_unavailable: Optional[str] = None, user_note: Optional[str] = None) -> GateVerdict: + """Check one message. ``history`` is the conversation so far as ``(role, text)``, role user/assistant. + A judge that reads the message as the user's clears a hit on a phrase alone (``PHRASE_REASONS``); a shape hit + stays a failure. A judge call that errors or can't be parsed doesn't fail the message: the patterns' verdict + stands. So does a judge that couldn't be set up (``judge_unavailable``, the reason), which isn't called.""" + reasons = pattern_reasons(candidate) if self.patterns else [] + verdict = GateVerdict(passed=not reasons, pattern_reasons=reasons) + if self.judge_model and judge_unavailable is not None: + verdict.judge_error = judge_unavailable[: self.RECORDED_ERROR_CHARS] + return verdict + if self.judge_model: + try: + judged = await (self._judge(candidate, history, call_config, user_note=user_note) if user_note + else self._judge(candidate, history, call_config)) + except Exception as e: # the judge is a second opinion; never let it take the conversation down + logger.warning(f"user-sim gate judge call failed ({type(e).__name__}: {e}); using patterns only") + verdict.judge_error = f"{type(e).__name__}: {e}"[: self.RECORDED_ERROR_CHARS] + return verdict + if judged is None: + verdict.judge_error = "unparseable judge output" + return verdict + verdict.judge_in_role = judged["in_role"] + verdict.judge_reason = str(judged.get("reason", ""))[: self.RECORDED_REASON_CHARS] + if not judged["in_role"] and (reasons or len(candidate) >= self.judge_min_chars): + verdict.passed = False + elif judged["in_role"] and reasons and set(reasons) <= PHRASE_REASONS: + verdict.passed = True # a phrase alone, which the judge reads as the user's + verdict.cleared_by_judge = True + return verdict + + async def _judge(self, candidate: str, history: list[tuple[str, str]], route, + user_note: Optional[str] = None) -> Optional[dict]: + """One judge call through core's one model-call path (config/model_call.py), on ``route`` + (``judge_call_config``'s), or on the configured endpoint when none was given.""" + from agent_env.config import model_call + + prompt = _JUDGE_PROMPT.format( + history=render_history(history, self.context_turns, self.context_chars), + candidate=candidate[: self.context_chars * 2], + user_note=_USER_NOTE.format(note=user_note.strip()) if user_note else "", + ) + if route is None: + route = judge_call_config(self, {}) + reply = await model_call.complete( + route, [{"role": "user", "content": prompt}], timeout=self.judge_timeout_seconds, + response_format={"type": "json_schema", + "json_schema": {"name": "user_sim_gate", "schema": _JUDGE_SCHEMA, "strict": True}}) + return parse_judge(reply.content) + + +@dataclass +class GateVerdict: + passed: bool + pattern_reasons: list[str] = field(default_factory=list) + judge_in_role: Optional[bool] = None + judge_reason: Optional[str] = None + judge_error: Optional[str] = None + cleared_by_judge: Optional[bool] = None # the judge read a phrase-only hit as the user's + + def reasons(self) -> list[str]: + out = list(self.pattern_reasons) + if self.judge_in_role is False and self.judge_reason: + out.append(self.judge_reason) + return out + + def to_dict(self) -> dict: + return {k: v for k, v in asdict(self).items() if v not in (None, [])} + + +def rewrite_request(verdict: GateVerdict) -> str: + """What the user-sim is told when its message was held back.""" + why = "; ".join(verdict.reasons()) or "it reads as the assistant's message" + return ( + "[Conversation system note, not from the assistant.] Your last message was not sent to the assistant, " + f"because it reads as the assistant's reply rather than yours: {why}. You are the user. The assistant does " + "the work and reports it; you ask, answer what you were asked, decide, push back, or end. Write your next " + "message to the assistant again, as yourself, in the same reply format as before." + ) + + +@dataclass +class GateOutcome: + """How one user-sim message fared: the text to send (the last one checked), every verdict in order, and + ``invalid`` when it still failed after the rewrites and ``on_fail`` isn't "send". ``stopped`` is set when the + rewrite callback ended the exchange instead of returning a new message.""" + + text: Optional[str] + verdicts: list[GateVerdict] = field(default_factory=list) + invalid: bool = False + stopped: bool = False + + def reasons(self) -> list[str]: + return self.verdicts[-1].reasons() if self.verdicts else [] + + +async def run_gate( + gate: UserSimGate, + text: str, + history: list[tuple[str, str]], + rewrite: Callable[[str], Awaitable[Optional[str]]], + *, + call_config=None, + judge_unavailable: Optional[str] = None, + on_verdict: Optional[Callable[[int, str, GateVerdict], None]] = None, + user_note: Optional[str] = None, +) -> GateOutcome: + """Check ``text``; while it fails, send the user-sim ``rewrite_request`` through ``rewrite`` and check its new + message, up to ``gate.max_rewrites`` times. ``rewrite`` returns the new message, or None when the user side + ended instead. ``judge_unavailable`` is why the judge couldn't be set up, if it couldn't (see ``check``). + ``on_verdict(attempt, text, verdict)`` sees every check, for recording. ``user_note`` is what a post-turn hook + told the user sim it found, shown to the judge. Any conversation loop + can call this; ``prompt_agent`` does, between a user-sim's reply and the agent's next turn.""" + outcome = GateOutcome(text=text) + for attempt in range(gate.max_rewrites + 1): + extra = {} if judge_unavailable is None else {"judge_unavailable": judge_unavailable} + if user_note: # what a post-turn hook told the user sim it found: the judge reads a report of it as the user's + extra["user_note"] = user_note + verdict = await gate.check(text, history, call_config=call_config, **extra) + outcome.verdicts.append(verdict) + if on_verdict is not None: + on_verdict(attempt, text, verdict) + if verdict.passed: + return outcome + if attempt == gate.max_rewrites: + break + new_text = await rewrite(rewrite_request(verdict)) + if new_text is None: + outcome.stopped = True + return outcome + text = outcome.text = new_text + outcome.invalid = gate.on_fail != "send" + return outcome + + +JUDGE_KEY_HOW = "judge_litellm_api_key, or litellm_api_key, in the run's overrides" + + +def judge_setup_failed(gate: UserSimGate, error: Exception) -> str: + """What a step records when the gate's judge can't be set up (``judge_call_config`` raised), so the patterns' + verdict stands. With patterns off nothing would check the user sim's messages for the rest of the run, and a + setup failure won't recover, so that raises instead.""" + reason = f"judge setup: {type(error).__name__}: {error}" + if not gate.patterns: + raise RuntimeError(f"user_sim_gate: {reason}; with patterns off, nothing would check the user sim's " + "messages, so the step fails rather than run unchecked") from error + return reason[: gate.RECORDED_ERROR_CHARS] + + +def judge_call_config(gate: UserSimGate, user_overrides: dict): + """The route of the gate's judge calls (a ``model_call.ModelRoute``), honoring a run's judge or LiteLLM + overrides as the rubrics verifier does; None without a judge. Raises ConfigError, as a setup failure, for an + endpoint the run gives no key for.""" + if not gate.judge_model: + return None + from agent_env.config import model_call + + return model_call.resolve_route( + gate.judge_model, + api_key=user_overrides.get("judge_litellm_api_key") or user_overrides.get("litellm_api_key"), + api_base=user_overrides.get("judge_litellm_base_url") or user_overrides.get("litellm_base_url"), + who="user_sim_gate", key_how=JUDGE_KEY_HOW) diff --git a/tst/unit/task_step/always_on_harness.py b/tst/unit/task_step/always_on_harness.py index 8e40c3bcb..e68b6a19b 100644 --- a/tst/unit/task_step/always_on_harness.py +++ b/tst/unit/task_step/always_on_harness.py @@ -20,6 +20,7 @@ from agent_env.env.gateway.triggers import TriggerEngine from agent_env.task_step.context import DeployedAgent, TaskStepContext from agent_env.task_step.task_steps import always_on as ao +from agent_env.task_step.task_steps.agent import user_sim_versions from agent_env.task_step.task_steps.agent.deploy_user_sim_agent import USER_SIMS_KEY GATEWAY = "http://gateway.test" @@ -295,4 +296,6 @@ async def answer(version, person, history, read, ask): rig.reads.append(read) return rig.answers.pop(0) if rig.answers else {"outcome": "no_reply", "message": ""} + parse = staticmethod(user_sim_versions.parse) # its answers (user_texts) are in our format + return Plugin diff --git a/tst/unit/task_step/test_always_on_budget.py b/tst/unit/task_step/test_always_on_budget.py index a5bb85aa3..269951dec 100644 --- a/tst/unit/task_step/test_always_on_budget.py +++ b/tst/unit/task_step/test_always_on_budget.py @@ -258,3 +258,14 @@ async def hangs_once(run): with pytest.raises(RuntimeError, match="the opening wake ran past its worst case"): await asyncio.wait_for(rig.run(rig.step(), ctx), 10) assert ctx.metadata[ao.ALWAYS_ON_KEY]["week"]["end_reason"] == "step_error" + + +@pytest.mark.parametrize("gate", [{"patterns": True, "max_rewrites": 2}, + {"judge_model": "j", "max_rewrites": 3, "judge_timeout_seconds": 45}], + ids=["patterns", "judge"]) +def test_a_gated_read_has_time_for_every_rewrite_and_judge_call_and_the_reserve_grows_with_it(gate): + plain, gated = _step(), _step(user_sim_gate=gate) + turn = plain.user_timeout_seconds + ao.TURN_GRACE_SECONDS + ao.TRAJECTORY_FETCH_SECONDS + judging = (gate["max_rewrites"] + 1) * gate["judge_timeout_seconds"] if "judge_model" in gate else 0 + assert gated.read_cap_seconds() == (ao.READ_ASKS + gate["max_rewrites"]) * turn + judging + assert gated.wake_reserve_seconds() - plain.wake_reserve_seconds() == gated.read_cap_seconds() - plain.read_cap_seconds() diff --git a/tst/unit/task_step/test_always_on_user_sim_gate.py b/tst/unit/task_step/test_always_on_user_sim_gate.py new file mode 100644 index 000000000..38e944b47 --- /dev/null +++ b/tst/unit/task_step/test_always_on_user_sim_gate.py @@ -0,0 +1,170 @@ +"""always_on with ``user_sim_gate``: an answer of the user sim's that reads as the assistant's is held back, the user +sim answers again in its session, and an answer that still fails never reaches the agent and ends the run.""" + +from __future__ import annotations + +import json + +import pytest + +from tst.unit.task_step.always_on_harness import Rig +from tst.unit.task_step.test_prompt_agent_user_side import _SWAP + +pytestmark = pytest.mark.usefixtures("local_stores") + +NOTE = "[Conversation system note, not from the assistant.] Your last message was not sent" + + +def _answer(outcome: str, message: str = "") -> str: + return json.dumps({"outcome": outcome, "message": message, "used": "none"}) + + +def _messages_once(n, texts, rig): + if n == 1: + rig.message("here's the plan for the weekend") + return "sent" + rig.quiet() + return "" + + +def _gated(rig: Rig, **gate): + return rig.step(max_wakes=4, user_sim_gate={"patterns": True, **gate}) + + +def _got(rig: Rig, text: str) -> bool: + return any(text in t for wake in rig.sent for t in wake) + + +@pytest.mark.asyncio +async def test_an_answer_that_reads_as_the_assistants_is_held_back_and_the_rewrite_reaches_the_agent(monkeypatch, + tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": _SWAP}], + user_texts=(_answer("reply", "do those numbers hold up?"),)) + await rig.arm() + ctx = await rig.run(_gated(rig)) + + assert [c["parts"][0]["text"][:len(NOTE)] for c in rig.user_calls] == [NOTE] # asked again, in her session + assert _got(rig, "do those numbers hold up?") and not _got(rig, "None of the three reported numbers") + (read, *_) = rig.events(ctx, "user_sim_read") + assert read["message"] == "do those numbers hold up?" + gate = read["user_sim_gate"] + assert [(v["attempt"], v["passed"]) for v in gate["verdicts"]] == [(0, False), (1, True)] + assert gate["first_message"] == _SWAP and gate["invalid"] is False + assert read["first_message"] == _SWAP # where prompt_agent's read keeps it too + assert [r["message"] for r in gate["rewrites"]] == ["do those numbers hold up?"] + assert "user_sim_invalid" not in ctx.metadata + assert rig.record(ctx)["end_reason"] != "user_sim_invalid" + + +@pytest.mark.asyncio +async def test_an_answer_that_still_fails_never_reaches_the_agent_and_ends_the_run_as_an_error(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": _SWAP}], + user_texts=(_answer("reply", _SWAP), _answer("reply", _SWAP))) + await rig.arm() + ctx = await rig.run(_gated(rig)) + + assert not _got(rig, "None of the three reported numbers") + assert len(rig.sent) == 1 # no wake after hers: the run ended at her read + record = rig.record(ctx) + assert record["end_reason"] == "user_sim_invalid" + assert record["error_type"] == "user_sim_invalid" # verifiers skip a run that broke, as on a user sim error + assert ctx.metadata["user_sim_invalid"]["week"]["wake"] == 1 + (read,) = rig.events(ctx, "user_sim_read") + assert read["outcome"] == "no_reply" and read["user_sim_gate"]["invalid"] is True + assert [v["passed"] for v in read["user_sim_gate"]["verdicts"]] == [False, False, False] + + +@pytest.mark.asyncio +async def test_on_fail_end_ends_the_run_without_marking_it_an_error(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": _SWAP}], + user_texts=(_answer("reply", _SWAP),)) + await rig.arm() + ctx = await rig.run(_gated(rig, max_rewrites=1, on_fail="end")) + + record = rig.record(ctx) + assert record["end_reason"] == "user_sim_invalid" and record["error_type"] is None + assert not _got(rig, "None of the three reported numbers") + + +@pytest.mark.asyncio +async def test_an_in_role_answer_passes_with_no_rewrite(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": "shorter, please"}]) + await rig.arm() + ctx = await rig.run(_gated(rig)) + (read, *_) = rig.events(ctx, "user_sim_read") + assert read["user_sim_gate"] == {"verdicts": [{"attempt": 0, "chars": 15, "passed": True}], "rewrites": [], + "invalid": False} + assert rig.user_calls == [] + + +@pytest.mark.asyncio +async def test_without_the_gate_nothing_is_checked(monkeypatch, tmp_path): + plain = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": _SWAP}]) + await plain.arm() + ctx = await plain.run(plain.step(max_wakes=4)) + assert "user_sim_gate" not in plain.events(ctx, "user_sim_read")[0] + assert _got(plain, "None of the three reported numbers") # unchanged without the gate + + + +@pytest.mark.asyncio +async def test_a_judge_only_gate_whose_judge_cant_be_set_up_fails_the_step(monkeypatch, tmp_path): + # A setup failure won't recover, and with patterns off nothing else would check her answers. + def no_endpoint(model, **kw): + raise KeyError("no endpoint configured for judge-model") + + monkeypatch.setattr("agent_env.config.model_call.resolve_route", no_endpoint) + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, answers=[{"outcome": "reply", "message": "looks good"}]) + await rig.arm() + + with pytest.raises(Exception, match="judge setup: KeyError.*with patterns off"): + await rig.run(rig.step(max_wakes=4, user_sim_gate={"patterns": False, "judge_model": "judge-model"})) + assert not _got(rig, "looks good") + + +class _OwnFormatPlugin: + """A plugin whose user sim answers "REPLY: " instead of our JSON, with its own parse.""" + + def __init__(self, answers): + self.answers = list(answers) + + def setup(self, version, person): + return {"system_prompt": "you are linda", "output_format": None} + + async def answer(self, version, person, history, read, ask): + return self.answers.pop(0) + + @staticmethod + def parse(raw): + text = raw if isinstance(raw, str) else "" + return {"outcome": "reply", "message": text.removeprefix("REPLY: "), "used": "none"} if text else {} + + +@pytest.mark.asyncio +async def test_a_plugins_rewrite_is_read_with_its_own_parse_and_reaches_the_agent(monkeypatch, tmp_path): + rig = Rig(monkeypatch, tmp_path, agent=_messages_once, user_texts=("REPLY: do those numbers hold up?",)) + rig.plugin = _OwnFormatPlugin([{"outcome": "reply", "message": _SWAP}]) + await rig.arm() + ctx = await rig.run(_gated(rig)) + + assert _got(rig, "do those numbers hold up?") # parsed its way, not read as an empty no_reply + (read, *_) = rig.events(ctx, "user_sim_read") + assert read["message"] == "do those numbers hold up?" + + +@pytest.mark.asyncio +async def test_a_gate_with_a_plugin_that_cant_read_its_rewrites_is_refused_when_the_run_starts(monkeypatch, tmp_path): + class NoParse: + @staticmethod + def setup(version, person): + return {"system_prompt": "you are linda", "output_format": None} + + @staticmethod + async def answer(version, person, history, read, ask): + return {"outcome": "reply", "message": "ok"} + + rig = Rig(monkeypatch, tmp_path, agent=_messages_once) + rig.plugin = NoParse + await rig.arm() + with pytest.raises(ValueError, match="user_sim_gate needs the user sim's plugin to define parse"): + await rig.run(_gated(rig)) diff --git a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py index b35933747..996b80fef 100644 --- a/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py +++ b/tst/unit/task_step/test_prompt_agent_built_in_user_sim.py @@ -17,7 +17,8 @@ from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep, StoredTrajectory from agent_env.task_step.task_steps.verifiers.agent_prompt_response_verifier import AgentPromptResponseVerifierTaskStep from tst.unit.task_step.test_deploy_user_sim_agent import HANNAH, _published, _step as _deploy_step -from tst.unit.task_step.test_prompt_agent_user_side import AGENT, HANG, HUMAN, USER, Peers, _stored, exchange, text +from tst.unit.task_step.test_prompt_agent_user_side import ( + _SWAP, AGENT, HANG, HUMAN, USER, Peers, _stored, exchange, text) pytestmark = pytest.mark.usefixtures("local_stores") @@ -414,6 +415,99 @@ async def test_agent_triggers_registered_for_the_built_in_user_are_refused_befor assert "user_sim_reads" not in ctx.metadata +# ---- the user-sim gate ---------------------------------------------------------------------------------------------- +def _gated(turns: int = 3, **gate) -> PromptAgentTaskStep: + return _step(turns=turns, user_sim_gate={"patterns": True, **gate}) + + +def _to(peers: Peers, ctx: TaskStepContext, url: str) -> list[str]: + sent = [r for r in peers.requests(ctx) if r[2] and r[2].get("method") == "message/send"] + return [r[2]["params"]["message"]["parts"][0]["text"] for r in sent if r[1].startswith(url)] + + +@pytest.mark.asyncio +async def test_the_gate_holds_back_a_built_in_users_answer_that_reads_as_the_assistants_and_she_answers_again( + monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two"), text("you're welcome")], + USER: [said("reply", _SWAP), said("reply", "do those numbers hold up?", "D1"), + said("done", "great, thanks")]}) + ctx = await _context(monkeypatch) + + await _gated().execute(ctx) + + to_user, to_agent = _to(peers, ctx, USER), _to(peers, ctx, AGENT) + assert to_user[1].startswith("[Conversation system note, not from the assistant.] Your last message was not sent") + assert to_agent == ["plan my parents' visit", "do those numbers hold up?", "great, thanks"] # no swap reached it + verdicts = ctx.metadata["user_sim_gate"]["talk"] + assert [(v["turn"], v["attempt"], v["passed"]) for v in verdicts] == [(1, 0, False), (1, 1, True), (2, 0, True)] + first = _reads(ctx)[0] + # As in always_on: the read holds what reached the agent, and keeps what she first answered for whoever grades her. + assert (first["message"], first["first_message"]) == ("do those numbers hold up?", _SWAP) + assert [(r["outcome"], r["message"], r["used"]) for r in first["gate_rewrites"]] == [ + ("reply", "do those numbers hold up?", "D1")] + assert "user_sim_invalid" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_built_in_users_answer_that_still_fails_ends_the_run_and_fails_the_step(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("reply", _SWAP)] * 3}) + ctx = await _context(monkeypatch) + + with pytest.raises(RuntimeError, match=r"user_sim_gate: user-sim message at turn 1 still reads as the assistant's"): + await _gated().execute(ctx) + + assert [v["passed"] for v in ctx.metadata["user_sim_gate"]["talk"]] == [False, False, False] + assert ctx.metadata["user_sim_invalid"]["talk"]["turn"] == 1 + assert len(_reads(ctx)[0]["gate_rewrites"]) == 2 + # Nothing reached the agent, so the read is a no_reply with no message, as in always_on. + read = _reads(ctx)[0] + assert (read["outcome"], read["message"], read["first_message"]) == ("no_reply", "", _SWAP) + # Verifiers skip a response with an error_type, so a cut-off conversation isn't graded as the agent's answer. + assert ctx.prompt_responses[-1].error_type == "user_sim_invalid" + assert ctx.prompt_responses[-1].error_message.startswith("user_sim_gate: user-sim message at turn 1") + assert _to(peers, ctx, AGENT) == ["plan my parents' visit"] # nothing she wrote reached the agent + + +@pytest.mark.asyncio +async def test_a_rewrite_that_says_nothing_ends_the_conversation_as_her_no_reply(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [said("reply", _SWAP), said("no_reply")]}) + ctx = await _context(monkeypatch) + + await _gated().execute(ctx) + + assert _to(peers, ctx, AGENT) == ["plan my parents' visit"] + assert "user_sim_invalid" not in ctx.metadata + assert [(v["attempt"], v["passed"]) for v in ctx.metadata["user_sim_gate"]["talk"]] == [(0, False)] + + +@pytest.mark.asyncio +async def test_a_done_rewritten_into_a_reply_counts_toward_max_replies(monkeypatch): + # Her first answer is a done, which isn't counted; its rewrite is a reply, which is, and it is her last. + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], + USER: [said("done", _SWAP), said("reply", "do those numbers hold up?")]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "max_replies": 1}) + + result = await _gated().execute(ctx) + + assert _to(peers, ctx, AGENT) == ["plan my parents' visit", "do those numbers hold up?"] + assert len(_to(peers, ctx, USER)) == 2 # her read and the rewrite: she isn't asked again past the limit + assert result.prompt_responses[-1].response == "draft two" + assert result.prompt_responses[-1].error_type is None + + +@pytest.mark.asyncio +async def test_a_reply_rewritten_into_a_done_no_longer_counts(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two"), text("draft three")], + USER: [said("reply", _SWAP), said("reply", "shorter, please"), + said("reply", "and cheaper")]}) + ctx = await _context(monkeypatch, persona={**HANNAH, "max_replies": 2}) + + await _gated().execute(ctx) + + # The held reply and its rewrite count once, so her second reply is the one that reaches the limit. + assert _to(peers, ctx, AGENT) == ["plan my parents' visit", "shorter, please", "and cheaper"] + + @pytest.mark.asyncio async def test_a_hook_note_is_in_the_users_read_after_the_reply_and_never_reaches_the_agent(monkeypatch): from tst.unit.task_step.test_prompt_agent_post_turn_hooks import _hook diff --git a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py index 32b1b2688..f78b97f61 100644 --- a/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py +++ b/tst/unit/task_step/test_prompt_agent_post_turn_hooks.py @@ -658,3 +658,30 @@ async def fired(): answer = _Answer(ask_user, fired) assert await answer.user_triggers() == {"message": text("nudge from a trigger")} assert await answer.ask_user(note="the sheet has no total") == {"message": text("nudge from a trigger")} + + +@pytest.mark.asyncio +async def test_with_a_user_sim_gate_the_note_goes_with_the_first_send_only_and_the_judge_is_shown_it(monkeypatch): + from agent_env.task_step.task_steps import user_sim_gate + from tst.unit.task_step.test_prompt_agent_user_side import _SWAP + + shown = [] + + async def judge(self, candidate, history, call_config, user_note=None): + shown.append(user_note) + return {"in_role": not candidate.startswith("Done."), "reason": "judged"} + + monkeypatch.setattr(user_sim_gate.UserSimGate, "_judge", judge) + monkeypatch.setattr("agent_env.config.model_call.resolve_route", lambda model, **kw: None) + peers = EnvPeers(monkeypatch, {AGENT: [text("one"), text("two")], USER: [sim(_SWAP), sim("add a total?")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _step(_hook("notes", NOTES), turns=2, + user_sim_gate={"patterns": True, "judge_model": "judge-model"}).execute(ctx) + + to_user = [r[2]["params"]["message"]["parts"] for r in peers.requests(ctx) + if r[1] == f"{USER}/a2a" and r[2] and r[2].get("method") == "message/send"] + assert [len(parts) for parts in to_user] == [2, 1] # the reply and the note; then the rewrite request alone + assert "the sheet has no total" in to_user[0][1]["text"] + assert "the sheet has no total" not in json.dumps(to_user[1]) + assert shown == ["the sheet has no total (turn 1)"] * 2 # the judge is shown it for the message and its rewrite diff --git a/tst/unit/task_step/test_prompt_agent_user_side.py b/tst/unit/task_step/test_prompt_agent_user_side.py index 0aba179a8..9b0b06417 100644 --- a/tst/unit/task_step/test_prompt_agent_user_side.py +++ b/tst/unit/task_step/test_prompt_agent_user_side.py @@ -479,3 +479,179 @@ async def test_with_no_user_agent_the_configured_human_peer_answers(monkeypatch) *exchange(HUMAN, text("one"), "", "", "task-2"), *exchange(AGENT, text("go on"), "", "", "task-3"), ] + + +# ---- the user-sim gate ---------------------------------------------------------------------------------------------- + +_SWAP = ("Done. Here's what I found when I checked the figures against the records:\n\n" + "| Metric | Reported | Actual |\n|---|---|---|\n| Installs | 1,400 | 901 |\n| Retention | 43% | 25% |\n\n" + "None of the three reported numbers hold up, and I've updated the summary page to say so. " * 2) + + +_TAKEOVER = "I'll handle the rest of it from here, so there's no need for you to keep going on this. " * 4 + + +def _gated(turns: int = 2, **gate) -> PromptAgentTaskStep: + return _step(turns=turns, user_agent_name="human_agent", user_sim_gate={"patterns": True, **gate}) + + +@pytest.mark.asyncio +async def test_the_gate_lets_a_user_message_through_and_records_its_verdict(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("make it shorter")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _gated().execute(ctx) + + assert peers.requests(ctx)[-2] == send(AGENT, text("make it shorter"), "", "") + assert ctx.metadata["user_sim_gate"] == {"talk": [{"turn": 1, "attempt": 0, "chars": 15, "passed": True}]} + assert "user_sim_invalid" not in ctx.metadata + + +@pytest.mark.asyncio +async def test_a_message_that_reads_as_the_assistants_is_held_back_and_rewritten(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], + USER: [sim(_SWAP), sim("do those numbers hold up?")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _gated().execute(ctx) + + sent = [r for r in peers.requests(ctx) if r[2] and r[2].get("method") == "message/send"] + to_user = [r[2]["params"]["message"]["parts"][0]["text"] for r in sent if r[1].startswith(USER)] + to_agent = [r[2]["params"]["message"]["parts"] for r in sent if r[1].startswith(AGENT)] + assert to_user[0] == "draft one" + assert to_user[1].startswith("[Conversation system note, not from the assistant.] Your last message was not sent") + assert to_agent == [text("plan my week"), text("do those numbers hold up?")] # the swap never reached it + verdicts = ctx.metadata["user_sim_gate"]["talk"] + assert [(v["attempt"], v["passed"]) for v in verdicts] == [(0, False), (1, True)] + assert "it opens like a report of work done" in verdicts[0]["pattern_reasons"] + + +@pytest.mark.asyncio +async def test_a_message_that_still_fails_after_its_rewrites_ends_the_run_and_fails_the_step(monkeypatch): + peers = Peers(monkeypatch, {AGENT: [text("draft one")], USER: [sim(_SWAP), sim(_SWAP), sim(_SWAP)]}) + ctx = _context(user_card=_USER_SIM_CARD) + + with pytest.raises(RuntimeError, match=r"user_sim_gate: user-sim message at turn 1 still reads as the assistant's"): + await _gated(turns=3).execute(ctx) + + assert [v["passed"] for v in ctx.metadata["user_sim_gate"]["talk"]] == [False, False, False] + assert ctx.metadata["user_sim_invalid"]["talk"]["turn"] == 1 + assert _stored(ctx, peers)["status"] == "closed" + assert [m["role"] for m in _stored(ctx, peers)["messages"]] == ["user", "agent"] # nothing more was sent + assert ctx.prompt_responses[-1].response == "draft one" # the run kept its record + assert ctx.prompt_responses[-1].error_type == "user_sim_invalid" # which verifiers skip + + +@pytest.mark.asyncio +async def test_on_fail_end_closes_the_conversation_without_failing_the_step(monkeypatch): + Peers(monkeypatch, {AGENT: [text("draft one")], USER: [sim(_SWAP), sim(_SWAP)]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _gated(turns=3, max_rewrites=1, on_fail="end").execute(ctx) + + assert ctx.metadata["user_sim_invalid"]["talk"]["turn"] == 1 + assert len(ctx.metadata["user_sim_gate"]["talk"]) == 2 + + +@pytest.mark.asyncio +async def test_the_judge_fails_a_message_the_patterns_pass_and_sees_the_conversation(monkeypatch): + from agent_env.task_step.task_steps import user_sim_gate + + seen = [] + + async def judge(self, candidate, history, call_config): + seen.append((candidate, list(history))) + return {"in_role": candidate != _TAKEOVER, "reason": "it offers to do the work"} + + monkeypatch.setattr(user_sim_gate.UserSimGate, "_judge", judge) + Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], + USER: [sim(_TAKEOVER), sim("shorter, please")]}) + ctx = _context(user_card=_USER_SIM_CARD) + monkeypatch.setattr("agent_env.config.model_call.resolve_route", lambda model, **kw: None) + + await _gated(judge_model="judge-model").execute(ctx) + + verdicts = ctx.metadata["user_sim_gate"]["talk"] + assert [(v["passed"], v.get("judge_in_role")) for v in verdicts] == [(False, False), (True, True)] + assert verdicts[0]["judge_reason"] == "it offers to do the work" + assert seen[0][1] == [("user", "plan my week"), ("assistant", "draft one")] + + +@pytest.mark.asyncio +async def test_a_judge_that_errors_leaves_the_patterns_verdict(monkeypatch): + from agent_env.task_step.task_steps import user_sim_gate + + async def judge(self, candidate, history, call_config): + raise TimeoutError("judge timed out") + + monkeypatch.setattr(user_sim_gate.UserSimGate, "_judge", judge) + Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("shorter, please")]}) + ctx = _context(user_card=_USER_SIM_CARD) + monkeypatch.setattr("agent_env.config.model_call.resolve_route", lambda model, **kw: None) + + await _gated(judge_model="judge-model").execute(ctx) + + (verdict,) = ctx.metadata["user_sim_gate"]["talk"] + assert verdict["passed"] is True + assert verdict["judge_error"] == "TimeoutError: judge timed out" + + +def test_the_gate_round_trips_through_the_step_dict_and_rejects_bad_settings(): + step = _gated(judge_model="judge-model", max_rewrites=1) + again = PromptAgentTaskStep.from_dict(step.to_dict()) + assert again.user_sim_gate == step.user_sim_gate + assert again.user_sim_gate.max_rewrites == 1 + assert "user_sim_gate" not in _step().to_dict() + defaults = _step(user_sim_gate={}).user_sim_gate # empty settings turn the gate on with its defaults + assert defaults is not None and defaults.patterns and defaults.judge_model is None and defaults.max_rewrites == 2 + with pytest.raises(ValueError, match="unknown keys"): + _step(user_sim_gate={"judge": "x"}) + with pytest.raises(ValueError, match="on_fail"): + _step(user_sim_gate={"on_fail": "retry"}) + with pytest.raises(ValueError, match="needs patterns"): + _step(user_sim_gate={"patterns": False}) + + +@pytest.mark.asyncio +async def test_a_judge_whose_setup_fails_leaves_the_patterns_verdict_and_is_not_called(monkeypatch): + from agent_env.task_step.task_steps import user_sim_gate + + called = [] + + async def judge(self, candidate, history, call_config): + called.append(candidate) + return {"in_role": True} + + def no_endpoint(model, **kw): + raise KeyError("no endpoint configured for judge-model") + + monkeypatch.setattr(user_sim_gate.UserSimGate, "_judge", judge) + monkeypatch.setattr("agent_env.config.model_call.resolve_route", no_endpoint) + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], + USER: [sim(_SWAP), sim("shorter, please")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + await _gated(judge_model="judge-model").execute(ctx) + + verdicts = ctx.metadata["user_sim_gate"]["talk"] + assert [v["passed"] for v in verdicts] == [False, True] # the patterns still held the swap back + assert all(v["judge_error"].startswith("judge setup: KeyError") for v in verdicts) + assert called == [] + assert peers.requests(ctx)[-2] == send(AGENT, text("shorter, please"), "", "") + + +@pytest.mark.asyncio +async def test_a_judge_only_gate_whose_setup_fails_fails_the_step_before_any_unchecked_message_reaches_the_agent( + monkeypatch): + # A setup failure won't recover, and with patterns off nothing else would check the user sim's messages. + def no_endpoint(model, **kw): + raise KeyError("no endpoint configured for judge-model") + + monkeypatch.setattr("agent_env.config.model_call.resolve_route", no_endpoint) + peers = Peers(monkeypatch, {AGENT: [text("draft one"), text("draft two")], USER: [sim("shorter, please")]}) + ctx = _context(user_card=_USER_SIM_CARD) + + with pytest.raises(RuntimeError, match="judge setup: KeyError.*with patterns off"): + await _gated(patterns=False, judge_model="judge-model").execute(ctx) + + assert not any(r == send(AGENT, text("shorter, please"), "", "") for r in peers.requests(ctx)) diff --git a/tst/unit/task_step/test_user_sim_gate.py b/tst/unit/task_step/test_user_sim_gate.py new file mode 100644 index 000000000..8481b3a7f --- /dev/null +++ b/tst/unit/task_step/test_user_sim_gate.py @@ -0,0 +1,278 @@ +"""The user-sim gate's own checks: the patterns, the judge's output parsing and the history it is shown.""" + +import pytest + +from agent_env.task_step.task_steps.user_sim_gate import ( + GateVerdict, parse_judge, pattern_reasons, render_history, rewrite_request) + +_FILLER = " The details are in the shared folder and the numbers come from last month's records." * 6 + + +def test_the_patterns_pass_what_users_write(): + for message in [ + "and the numbers she gave the board, do those hold up?", + "ok. who actually worked on it, and what does each of them cost per month?", + "Here's the email my manager sent, pasted so you have it:\n\n> Please hold the order until Friday." + _FILLER, + "No, go ahead and send it. Use the second draft.", + "I checked my inbox and there's nothing from them yet.", + "I don't have access to that folder.", + "Should I send you the spreadsheet?", + "I'll pull the numbers from my bank tonight.", + "I can send it to you after lunch.", + "I apologize for the confusion, I meant March.", + "The invoice has been sent to me already.", + "My manager said the page has been updated.", + "ok, send me the results when you have them", + ]: + assert pattern_reasons(message) == [], message + + +def test_the_patterns_catch_the_assistants_side(): + cases = { + "Done. Here's what I found when I went through the files." + _FILLER: "it opens like a report of work done", + "Here's where things stand after the changes." + _FILLER: "it opens like a report of work done", + "Understood. Standing by.": "it talks like the assistant", + "Want me to draft the reply to them now?": "it talks like the assistant", + "I'm ready to help! What would you like to work on today?": "it talks like the assistant", + "The page is now live, with the corrected totals." : "it says something is now live", + "I apologize for my previous answer, here is the corrected one.": "it talks like the assistant", + "Thanks for those corrections, incorporated. The cleaner version:" + _FILLER: "it reports something delivered", + "| Item | Was | Is |\n|---|---|---|\n| A | 1 | 2 |\n" + _FILLER * 2: "it is a formatted deliverable", + "I've reviewed the contract and I've compared it with the invoice." + _FILLER: "it reports checking or building", + } + for message, expected in cases.items(): + reasons = pattern_reasons(message) + assert any(r.startswith(expected) for r in reasons), (message[:60], reasons) + + +def test_the_judge_output_is_read_through_surrounding_text_and_rejected_without_a_boolean(): + assert parse_judge('{"in_role": false, "reason": "reports work"}') == {"in_role": False, "reason": "reports work"} + assert parse_judge('Sure.\n{"in_role": true, "reason": "asks"} trailing') == {"in_role": True, "reason": "asks"} + assert parse_judge('{"in_role": "no"}') is None + assert parse_judge("no json here") is None + + +def test_the_history_shows_the_last_turns_each_cut_to_size(): + history = [("user", "first"), ("assistant", "x" * 50), ("user", "second"), ("assistant", "third")] + shown = render_history(history, turns=3, max_chars=10) + assert shown.startswith("ASSISTANT:\nxxxxxxxxxx\n[… 40 more characters]") + assert "first" not in shown and shown.endswith("ASSISTANT:\nthird") + assert render_history([], turns=4, max_chars=10) == "(nothing yet)" + assert render_history(history, turns=0, max_chars=10) == "(nothing yet)" # context_turns 0: no history + + +def test_the_rewrite_request_names_why_and_says_it_is_not_from_the_assistant(): + note = rewrite_request(GateVerdict(passed=False, pattern_reasons=["it opens like a report of work done"], + judge_in_role=False, judge_reason="it answers its own question")) + assert note.startswith("[Conversation system note, not from the assistant.]") + assert "it opens like a report of work done; it answers its own question" in note + + +def test_the_judge_alone_fails_only_a_long_message(): + import asyncio + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + class Gate(UserSimGate): + async def _judge(self, candidate, history, call_config): + return {"in_role": False, "reason": "reads as analysis"} + + gate = Gate(judge_model="judge-model") + short = asyncio.run(gate.check("Contracted physicians are required in two states.", [])) + long = asyncio.run(gate.check("The two states that need contracted physicians are these." + _FILLER, [])) + assert (short.passed, short.judge_in_role) == (True, False) # recorded, not failed + assert long.passed is False + + +# ---- run_gate, for any conversation loop ------------------------------------------------------------------------------ + +def _scripted_gate(verdicts, **settings): + from agent_env.task_step.task_steps.user_sim_gate import GateVerdict, UserSimGate + + class Gate(UserSimGate): + async def check(self, candidate, history, *, call_config=None, judge_unavailable=None): + return GateVerdict(passed=verdicts.pop(0), pattern_reasons=["it opens like a report of work done"]) + + return Gate(**settings) + + +def test_run_gate_rewrites_until_a_message_passes(): + import asyncio + from agent_env.task_step.task_steps.user_sim_gate import run_gate + + asked, seen = [], [] + + async def rewrite(note): + asked.append(note) + return f"rewrite {len(asked)}" + + outcome = asyncio.run(run_gate(_scripted_gate([False, False, True]), "draft", [], rewrite, + on_verdict=lambda attempt, text, v: seen.append((attempt, text, v.passed)))) + assert (outcome.text, outcome.invalid, outcome.stopped) == ("rewrite 2", False, False) + assert seen == [(0, "draft", False), (1, "rewrite 1", False), (2, "rewrite 2", True)] + assert asked[0].startswith("[Conversation system note, not from the assistant.]") + + +def test_run_gate_marks_a_message_that_never_passes_and_honors_send(): + import asyncio + from agent_env.task_step.task_steps.user_sim_gate import run_gate + + async def rewrite(note): + return "still the assistant" + + failed = asyncio.run(run_gate(_scripted_gate([False, False], max_rewrites=1), "draft", [], rewrite)) + sent = asyncio.run(run_gate(_scripted_gate([False, False], max_rewrites=1, on_fail="send"), "draft", [], rewrite)) + assert (failed.invalid, len(failed.verdicts)) == (True, 2) + assert (sent.invalid, sent.text) == (False, "still the assistant") + + +def test_run_gate_stops_when_the_user_side_ends_instead_of_rewriting(): + import asyncio + from agent_env.task_step.task_steps.user_sim_gate import run_gate + + async def rewrite(note): + return None + + outcome = asyncio.run(run_gate(_scripted_gate([False]), "draft", [], rewrite)) + assert (outcome.stopped, outcome.invalid, len(outcome.verdicts)) == (True, False, 1) + + +# ---- step types that run their own conversation ------------------------------------------------------------------ + +def test_a_step_type_that_replaces_the_conversation_refuses_the_gate_unless_it_applies_it(): + import pytest + from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + + class OwnLoop(PromptAgentTaskStep): + async def _execute(self, context): + return context + + class OwnLoopWithGate(OwnLoop): + USER_SIM_GATE_SUPPORTED = True + + with pytest.raises(ValueError, match="OwnLoop 'talk': user_sim_gate is applied by prompt_agent's own conversation"): + OwnLoop(id="talk", version=None, prompt="hi", user_sim_gate={}) + assert OwnLoop(id="talk", version=None, prompt="hi").user_sim_gate is None + assert OwnLoopWithGate(id="talk", version=None, prompt="hi", user_sim_gate={}).user_sim_gate is not None + assert PromptAgentTaskStep.applies_user_sim_gate() and not OwnLoop.applies_user_sim_gate() + + +def test_an_always_on_step_keeps_its_gate_through_the_step_dict(): + from agent_env.task_step.task_steps.always_on import AlwaysOnTaskStep + + step = AlwaysOnTaskStep("week", None, user_sim="user-sim", prompt_id="ask", agent_name="solver", + user_sim_gate={"max_rewrites": 1, "on_fail": "end"}) + again = AlwaysOnTaskStep.from_dict(step.to_dict()) + + assert again.user_sim_gate == step.user_sim_gate and again.user_sim_gate.on_fail == "end" + plain = AlwaysOnTaskStep("week", None, user_sim="user-sim", prompt_id="ask", agent_name="solver") + assert "user_sim_gate" not in plain.to_dict() + assert AlwaysOnTaskStep.from_dict(plain.to_dict()).user_sim_gate is None + + +def test_a_judge_that_couldnt_be_set_up_isnt_called_and_the_patterns_verdict_stands(): + import asyncio + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + class NeverCalled(UserSimGate): + async def _judge(self, candidate, history, call_config): + raise AssertionError("the judge was called") + + for patterns in (True, False): + gate = NeverCalled(patterns=patterns, judge_model="judge-model") + verdict = asyncio.run(gate.check("shorter, please", [], judge_unavailable="judge setup: KeyError: x")) + assert (verdict.passed, verdict.judge_error) == (True, "judge setup: KeyError: x") + + +def test_the_judge_calls_through_the_one_model_call_path_on_the_runs_route(monkeypatch): + """The gate's judge goes through config/model_call.py (route and complete), so its routing (a Claude model drops + an endpoint's trailing /v1, an unplaceable name goes as openai/) and its cost record are core's.""" + import asyncio + + from agent_env.config import model_call + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate, judge_call_config + + resolved, sent = [], [] + + def resolve_route(model, *, api_key, api_base, who, key_how): + resolved.append((model, api_key, api_base, who)) + return "route" + + async def complete(route, messages, *, timeout, response_format=None): + sent.append((route, timeout, response_format["json_schema"]["name"])) + return model_call.ModelReply('{"in_role": true, "reason": "ok"}', "m", None, None, None) + + monkeypatch.setattr(model_call, "resolve_route", resolve_route) + monkeypatch.setattr(model_call, "complete", complete) + gate = UserSimGate(judge_model="claude-sonnet-5-5", judge_timeout_seconds=45) + route = judge_call_config(gate, {"judge_litellm_api_key": "secret:k", "litellm_base_url": "https://llm.example/v1"}) + + judged = asyncio.run(gate._judge("thanks, looks good", [], route)) + + assert resolved == [("claude-sonnet-5-5", "secret:k", "https://llm.example/v1", "user_sim_gate")] + assert sent == [("route", 45, "user_sim_gate")] and judged["in_role"] is True + + + +def _judged(in_role): + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + class Gate(UserSimGate): + async def _judge(self, candidate, history, call_config): + return {"in_role": in_role, "reason": "judged"} + + return Gate(judge_model="judge-model") + + +def test_a_judge_reading_it_as_the_user_clears_a_phrase_but_not_a_reports_shape(): + import asyncio + + phrase = "Understood. Standing by." + shape = "Done. Here's what I found when I went through the files." + _FILLER + cleared = asyncio.run(_judged(True).check(phrase, [])) + assert (cleared.passed, cleared.cleared_by_judge) == (True, True) and cleared.pattern_reasons + assert asyncio.run(_judged(True).check(shape, [])).passed is False # shape + length: a failure whatever the judge + assert asyncio.run(_judged(False).check(phrase, [])).passed is False + + + +@pytest.mark.parametrize("settings, field", [ + ({"max_rewrites": 1.5}, "max_rewrites"), ({"max_rewrites": True}, "max_rewrites"), + ({"max_rewrites": -1}, "max_rewrites"), ({"judge_min_chars": None}, "judge_min_chars"), + ({"patterns": "false"}, "patterns"), ({"context_turns": "4"}, "context_turns"), + ({"context_chars": 0}, "context_chars"), ({"judge_timeout_seconds": 0}, "judge_timeout_seconds"), + ({"judge_model": ""}, "judge_model"), ({"on_fail": "skip"}, "on_fail"), +], ids=lambda v: str(v)) +def test_a_setting_of_the_wrong_type_or_range_is_refused_when_the_task_loads(settings, field): + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + with pytest.raises(ValueError, match=f"user_sim_gate.{field} must be"): + UserSimGate.from_dict(settings) + + +def test_good_settings_load_and_round_trip(): + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + gate = UserSimGate.from_dict({"patterns": False, "judge_model": "j", "max_rewrites": 0, "judge_timeout_seconds": 30}) + assert UserSimGate.from_dict(gate.to_dict()) == gate + + + +def test_the_judge_is_shown_what_the_user_found_when_a_hook_told_it(monkeypatch): + import asyncio + + from agent_env.config import model_call + from agent_env.task_step.task_steps.user_sim_gate import UserSimGate + + prompts = [] + + async def complete(route, messages, *, timeout, response_format=None): + prompts.append(messages[0]["content"]) + return model_call.ModelReply('{"in_role": true, "reason": "ok"}', "m", None, None, None) + + monkeypatch.setattr(model_call, "complete", complete) + gate = UserSimGate(judge_model="judge-model") + asyncio.run(gate._judge("i checked and the sheet isn't there", [], "route", user_note="no sheet in the folder")) + asyncio.run(gate._judge("i checked and the sheet isn't there", [], "route")) + + assert "WHAT THE USER FOUND" in prompts[0] and "no sheet in the folder" in prompts[0] + assert "WHAT THE USER FOUND" not in prompts[1] diff --git a/tst/unit/task_step/test_user_sim_satisfaction.py b/tst/unit/task_step/test_user_sim_satisfaction.py index 8c9a39e40..1153accd9 100644 --- a/tst/unit/task_step/test_user_sim_satisfaction.py +++ b/tst/unit/task_step/test_user_sim_satisfaction.py @@ -519,6 +519,21 @@ async def test_a_prompt_agent_chat_the_user_sim_broke_is_not_rated(monkeypatch): assert rating["skipped"] == "the user sim failed: state failed" and rating["score"] is None +@pytest.mark.asyncio +async def test_a_prompt_agent_chat_the_user_sim_gate_cut_off_is_not_rated_and_the_agent_isnt_blamed(monkeypatch): + from agent_env.task_step.context import PromptResponse + + ctx = await _context(monkeypatch) + ctx.metadata.setdefault("a2a_conversations", {})["talk"] = None + ctx.prompt_responses.append(PromptResponse(prompt_id="p", response="draft", error_type="user_sim_invalid", + error_message="user_sim_gate: user-sim message at turn 1 ...", + step_id="talk")) + await _hannah_rater().execute(ctx) + rating = ctx.metadata[SATISFACTION_KEY]["talk"] + assert rating["skipped"] == "the user sim's message still read as the assistant's after its rewrites" + assert rating["score"] is None + + @pytest.mark.asyncio async def test_a_prompt_agent_chat_whose_agent_turn_failed_is_not_rated(monkeypatch): Peers(monkeypatch, {AGENT: [("failed", text("it broke"))], USER: []}) @@ -527,3 +542,15 @@ async def test_a_prompt_agent_chat_whose_agent_turn_failed_is_not_rated(monkeypa await _step().execute(ctx) await _hannah_rater().execute(ctx) assert ctx.metadata[SATISFACTION_KEY]["talk"]["skipped"] == "the agent's turn failed" + + + +@pytest.mark.parametrize("record, skipped", [ + ({"end_reason": "user_sim_invalid", "error_type": "user_sim_invalid"}, + "the user sim's answer still read as the assistant's after its rewrites"), + ({"end_reason": "user_sim_invalid"}, None), # on_fail "end": no error, so the run is rated as one that ended +], ids=["on_fail fail", "on_fail end"]) +def test_an_always_on_run_the_gate_failed_as_an_error_is_not_rated(record, skipped): + from agent_env.task_step.task_steps.agent import rate_user_satisfaction as rus + + assert rus._always_on_session({"user_sim": {"name": "linda"}, **record}).skipped == skipped