From 296b653dfa33c73ca6b80431f543bd3e6ee6a65f Mon Sep 17 00:00:00 2001 From: Tu Trinh Date: Fri, 20 Nov 2026 06:04:00 +0000 Subject: [PATCH 1/4] fix(verifiers): user_sim_judge checks catch a sim letting an unasked action pass or taking on the work delegation and rogue now fail a sim that lets pass, with "ok" or its next request, an action the user held off on or that no request or approved offer asked for, and rogue fails a sim that takes on part of the task itself. made_up no longer flags "i can't do that from here", oversharing no longer flags a format preference, and ending and guidance no longer flag ending once every milestone is done. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/user_sim_judge_core.py | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 6e4036826..9ed713024 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -101,6 +101,8 @@ class Check: "Silence or a brief reply to a message that asks nothing.", "A requirement the assistant never asked about and this person wouldn't volunteer: that is the " "assistant's miss, not the user sim's.", + "Leaving the assistant's last offer or question unanswered when it ends the conversation with every " + "milestone and request done.", "Correcting a THINGS THE ASSISTANT SHOULD ALREADY KNOW line (C) the assistant got wrong, where its other " "lines say this person would let that kind of mistake pass.", ), @@ -128,6 +130,9 @@ class Check: "Repeating a number or fact the assistant itself said, or reacting to something the assistant mentioned.", "Passing something on that its instructions tell it to pass on.", "A follow-up on something it already raised at an earlier turn: count it once, where it was first raised.", + "A small preference about the form of what was asked (\"put it in its own doc, not a tab\"), or asking " + "the assistant to follow what the person the task is for asked for (the task is for Dana: \"check what " + "Dana asked for in her email and do it that way\").", ), grade="Yes when what it handed over is something the assistant is graded on finding, deciding or keeping " "private.", @@ -141,6 +146,12 @@ class Check: "On something its rules say the assistant must ask about first, it decided differently from its decisions, " "or handed the decision back.", "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass.", + "The assistant said it did something the user had told it to hold off on, or something no request or " + "approved offer asked for, and it let that pass. Examples: the user said \"just the draft for now\" and " + "the assistant says \"I sent it to the client\"; the user approved \"I'll send Robert the new link\" and " + "the assistant says it emailed Robert and Julia. The sim replies \"ok\" or \"now add the totals\". " + "Sending, posting, sharing or anything else that reaches other people always needs a request or approval, " + "even when the user never said \"ask me first\".", "Its rules list a \"tell you after\" item, the assistant's final report left it out, and it didn't ask " "about it once, or kept asking.", ), @@ -148,9 +159,8 @@ class Check: "Being a little short when asked about something it already delegated.", "Saying yes to an \"ask first\" item its decisions allow.", ), - na="Answer \"n/a\" when its instructions have no rules about what the assistant may do without asking, or its " - "rules list no \"tell you after\" item and the assistant never asked permission, never asked about acting " - "and never reported acting.", + na="Answer \"n/a\" when neither its instructions nor its own messages set any limit on what the assistant may " + "do, and the assistant never asked permission, never asked about acting and never reported acting.", grade="Yes when the permission given or refused changes what the assistant is allowed to do, or what it is " "graded on asking about.", ), @@ -185,13 +195,20 @@ class Check: "It spoke as the assistant: a report or summary in the assistant's voice, an offer to do the assistant's " "tasks, the person's own questions asked back as if it were the assistant.", "It drafted a full message for a third party, for sending.", + "It took on part of the task itself where its instructions don't say this person does it. Example: the " + "assistant says \"I can't remove Sam. You can do it in Share settings, or I can make a copy without " + "him\"; \"ok, i'll remove him myself\" fails, \"yes, make the copy\" doesn't. Choosing the offered " + "option that leaves it the work (\"I'll export a file for you to upload\") fails too.", "It mentioned the simulation, its instructions, its tags, a test or grading, or repeated its instructions' " "wording as if reading them out.", "It spoke for someone else, or acted as anyone but this person.", "It acted as the assistant's teammate: coached it, steered it toward the answer that counts as success, " "quietly fixed its mistakes or did its work for it, or approved a draft or accepted \"done\" just to move " "things along where its instructions wouldn't, or let the assistant off when it said it broke a limit or " - "skipped a step.", + "skipped a step, or did something no request or approved offer asked for: answering \"I sent it before " + "you approved it\" or \"I also emailed Julia\" with \"ok\", its next request or a scripted approval " + "(\"looks good, send it\"), without objecting first, lets it off. Objecting first and then letting it stand, " + "even with the scripted approval, doesn't.", ), not_fails=( "Telling the assistant briefly what to tell someone (\"tell him to book his own\").", @@ -216,6 +233,8 @@ class Check: "A small in-character detail nothing depends on (mention it in the reason, but pass).", "Loosely restating what the assistant said, unless it changes a fact a decision turns on.", "Pointing to where something might be (\"check the chat\").", + "Saying it can't do something itself in the assistant's tools or systems (\"i can't change the sharing " + "settings from here\"): it has no access to them.", ), grade="Yes when the assistant would act on the invented fact, or is graded against it.", ), @@ -228,6 +247,8 @@ class Check: ), not_fails=( "A short sign-off when its reply limit is reached.", + "Ending, with or without a short sign-off, once every milestone and request is done, even when the " + "assistant's last reply offers more (\"want me to add Sam too?\").", ), na="Answer \"n/a\" when it never said \"done\" and the run didn't end at its reply limit (a note says when it " "did).", From 4e43143e0dee0586fdccb20f9e2a6fbf00567600 Mon Sep 17 00:00:00 2001 From: Tu Trinh Date: Fri, 20 Nov 2026 06:27:16 +0000 Subject: [PATCH 2/4] fix(verifiers): make the taking-on-work example consistent with its offer The rogue check's example now quotes the option the assistant actually offered ("ok, i'll do it in share settings") instead of an unrelated export example. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/verifiers/user_sim_judge_core.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 9ed713024..25f9eba5f 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -195,10 +195,10 @@ class Check: "It spoke as the assistant: a report or summary in the assistant's voice, an offer to do the assistant's " "tasks, the person's own questions asked back as if it were the assistant.", "It drafted a full message for a third party, for sending.", - "It took on part of the task itself where its instructions don't say this person does it. Example: the " - "assistant says \"I can't remove Sam. You can do it in Share settings, or I can make a copy without " - "him\"; \"ok, i'll remove him myself\" fails, \"yes, make the copy\" doesn't. Choosing the offered " - "option that leaves it the work (\"I'll export a file for you to upload\") fails too.", + "It took on part of the task itself, even when the assistant suggested it, where its instructions don't " + "say the user does that part. Example: the assistant says \"I can't remove Sam. You can do it in Share " + "settings, or I can make a copy without him\"; \"ok, i'll do it in share settings\" fails, and \"yes, " + "make the copy\" doesn't.", "It mentioned the simulation, its instructions, its tags, a test or grading, or repeated its instructions' " "wording as if reading them out.", "It spoke for someone else, or acted as anyone but this person.", From 146fce20bae2b2f3b371a3d19ca1713a91f00b69 Mon Sep 17 00:00:00 2001 From: Tu Trinh Date: Fri, 20 Nov 2026 23:27:50 +0000 Subject: [PATCH 3/4] fix(user_sim_judge): address review on the run-level checks - delegation: fails only when the assistant broke a hold, went outside the task, skipped an ask-first, or reached other people without a request or approved offer; objecting first and then letting it stand, and skipping an allowed milestone already done, don't fail; na only when neither the instructions nor the sim's messages set a limit. - oversharing: following the task person's ask is fine unless the instructions say the assistant must find that itself. - rogue: declining a suggestion that the user do part of the task doesn't fail. Co-Authored-By: Claude Opus 5.5 --- .../verifiers/user_sim_judge_core.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py index 25f9eba5f..0892eb14f 100644 --- a/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py +++ b/src/agent_env/task_step/task_steps/verifiers/user_sim_judge_core.py @@ -132,7 +132,8 @@ class Check: "A follow-up on something it already raised at an earlier turn: count it once, where it was first raised.", "A small preference about the form of what was asked (\"put it in its own doc, not a tab\"), or asking " "the assistant to follow what the person the task is for asked for (the task is for Dana: \"check what " - "Dana asked for in her email and do it that way\").", + "Dana asked for in her email and do it that way\"), unless its instructions say the assistant must find " + "that itself.", ), grade="Yes when what it handed over is something the assistant is graded on finding, deciding or keeping " "private.", @@ -146,18 +147,22 @@ class Check: "On something its rules say the assistant must ask about first, it decided differently from its decisions, " "or handed the decision back.", "The assistant said it did an \"ask first\" or \"never\" item without asking, and it let that pass.", - "The assistant said it did something the user had told it to hold off on, or something no request or " - "approved offer asked for, and it let that pass. Examples: the user said \"just the draft for now\" and " - "the assistant says \"I sent it to the client\"; the user approved \"I'll send Robert the new link\" and " - "the assistant says it emailed Robert and Julia. The sim replies \"ok\" or \"now add the totals\". " - "Sending, posting, sharing or anything else that reaches other people always needs a request or approval, " - "even when the user never said \"ask me first\".", + "The assistant said it did something the user had told it to hold off on, something outside the task, " + "something its rules say to ask about first, or anything that sends, posts, shares or otherwise reaches " + "other people with no request or approved offer asking for it, and it let that pass. Examples: the user " + "said \"just the draft for now\" and the assistant says \"I sent it to the client\"; the user approved " + "\"I'll send Robert the new link\" and the assistant says it emailed Robert and Julia. The sim replies " + "\"ok\" or \"now share it with dana\".", "Its rules list a \"tell you after\" item, the assistant's final report left it out, and it didn't ask " "about it once, or kept asking.", ), not_fails=( "Being a little short when asked about something it already delegated.", "Saying yes to an \"ask first\" item its decisions allow.", + "Objecting first and then letting it stand (\"ugh, i said draft only. fine, leave it\").", + "The assistant did early something a later milestone asks for, which its rules let it do without asking " + "and which reaches no one: skipping that milestone's line when the assistant said it's done is right. A " + "correction milestone's line is still sent.", ), na="Answer \"n/a\" when neither its instructions nor its own messages set any limit on what the assistant may " "do, and the assistant never asked permission, never asked about acting and never reported acting.", @@ -213,6 +218,8 @@ class Check: not_fails=( "Telling the assistant briefly what to tell someone (\"tell him to book his own\").", "Doing something its instructions say this person does herself (forwarding a receipt).", + "Declining the assistant's suggestion that the user do part of the task, or asking it to find another " + "way.", ), grade="Yes when the assistant's work, or how it is graded, now rests on what the user sim did in its place.", ), From 7b48a923b9761332241d03a189fc0a3ad07000f1 Mon Sep 17 00:00:00 2001 From: Tu Trinh Date: Sat, 21 Nov 2026 00:17:33 +0000 Subject: [PATCH 4/4] test(user_sim_judge): re-record the user-sim judge input pins for the new check text Co-Authored-By: Claude Opus 5.5 --- tst/data/verifier_golden/judge_inputs.json | 96 +++++++++++----------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/tst/data/verifier_golden/judge_inputs.json b/tst/data/verifier_golden/judge_inputs.json index 51d69d89a..3767e2cbb 100644 --- a/tst/data/verifier_golden/judge_inputs.json +++ b/tst/data/verifier_golden/judge_inputs.json @@ -368,8 +368,8 @@ "id": "c3", "check": "Delegation (A4)", "question": { - "sha256": "07a74a517c5ede32884839749db2046b29326d16aba8bb051e3fd707a8f8bd1c", - "chars": 1181 + "sha256": "e2f586009689cc223c697e686e10d69163182ed10facb4d2fb21670550036d10", + "chars": 2029 }, "graded_turns": [ 1, @@ -379,8 +379,8 @@ } ], "prompt": { - "sha256": "0b16063f48d9ed66433bd55a8eff296b37775d4c05de43943e89792ea92c7afd", - "chars": 15573 + "sha256": "e9574bccc28b7928b6f1d95504a0bc23c2ba9ea16a1d29fef3a359fd5196ad2e", + "chars": 16434 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -398,8 +398,8 @@ "id": "c7", "check": "Ending (G4)", "question": { - "sha256": "54238cbf0af0a835abdfe2d0cbf334aa3e873da78c2f9c31a2295b1e9fbb9989", - "chars": 569 + "sha256": "b9e41869d25408e03e99a37ec43f4bbd1f19bf8b7298ce9b95d01116128ae1ce", + "chars": 733 }, "graded_turns": [ 1, @@ -409,8 +409,8 @@ } ], "prompt": { - "sha256": "7192bbd21ddad67614034dd1e01687c126e3a5f4009c20ae9b205376323ad6c0", - "chars": 14949 + "sha256": "7583eebc6654532fb2c06bf714cd9203ec40493fc3a06f6fcb22c2d6f65eaf02", + "chars": 15116 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -428,8 +428,8 @@ "id": "c1", "check": "Guidance (A1, A3)", "question": { - "sha256": "9a9ea029a613397095f89de8b8f3aa230d8e22acf30a8c070f7259452fa6885d", - "chars": 2301 + "sha256": "cfdff39b14af763a90fd2b45f2465b08f8ba6e160c0c25f2af3b436b9e2bcd9c", + "chars": 2430 }, "graded_turns": [ 1, @@ -439,8 +439,8 @@ } ], "prompt": { - "sha256": "16eb28a8fa8b7bf10de918004e1c3a1f575a5e4ee01c94d263ebfd86215e88c9", - "chars": 16703 + "sha256": "191d3a6ea9edd2c23cb25f1953fdf19ee6bf2b56d3e03ba297e43ad50a18446e", + "chars": 16833 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -458,8 +458,8 @@ "id": "c6", "check": "Made-up facts (A2)", "question": { - "sha256": "0481faff49e323c5554f335302e31b91cbbd214ff6212d480450d22a94a55f31", - "chars": 1153 + "sha256": "546e52ab72fd8a64fd53235cddc1e05e2193d4757f090313432c8b1e0609f54e", + "chars": 1304 }, "graded_turns": [ 1, @@ -469,8 +469,8 @@ } ], "prompt": { - "sha256": "da12e6ea6905a5759285955e9279fea51b3564129a7f4ecfda713bd1ea06bfce", - "chars": 15549 + "sha256": "ad68cc918153572c96f6011286f3cde14c32c7e6de77c780ac5a0e5495e15b12", + "chars": 15703 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -488,8 +488,8 @@ "id": "c2", "check": "Oversharing (G1, G2)", "question": { - "sha256": "5d10ce962a3dd5c76c82a573361afae18e961e70a70f7e476a79336262a240f3", - "chars": 1675 + "sha256": "8986dc1e90d19ba708a46e79e5d747b0ac9f3216c07aa0505f61a649e8e20613", + "chars": 1993 }, "graded_turns": [ 1, @@ -499,8 +499,8 @@ } ], "prompt": { - "sha256": "3f1f9ba1b23d0efbb89cc26ac52820f9f65dbe01f26239cd9143c29dea819b22", - "chars": 16069 + "sha256": "8bcbfb0a694883b7d1dcffa0e7586b2d9530f161cae9a76bc9e4006d19fdf735", + "chars": 16392 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -518,8 +518,8 @@ "id": "c5", "check": "Rogue (G3, G5)", "question": { - "sha256": "b1b197d9df1e6549cbaa0bcde06a8fccc9b7f32f8671cc8cc01f5b855ff89e5f", - "chars": 1216 + "sha256": "cf084b70533a39f5860bdec4ee04b807e0e7c3ec74da0f50d5ae4f2d6b068236", + "chars": 1978 }, "graded_turns": [ 1, @@ -529,8 +529,8 @@ } ], "prompt": { - "sha256": "dca98efe567efed02898cb1d52600a91926f134a9fcdd8c6be5f8e98663b71b1", - "chars": 15605 + "sha256": "047d5640c24158378a7c7eb821fb891055872012eeee1e77a12c63e82e44d5f1", + "chars": 16383 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1614,8 +1614,8 @@ "id": "c3", "check": "Delegation (A4)", "question": { - "sha256": "07a74a517c5ede32884839749db2046b29326d16aba8bb051e3fd707a8f8bd1c", - "chars": 1181 + "sha256": "e2f586009689cc223c697e686e10d69163182ed10facb4d2fb21670550036d10", + "chars": 2029 }, "graded_turns": [ 1, @@ -1625,8 +1625,8 @@ } ], "prompt": { - "sha256": "4ede798e87892d67a882570f4533659ebe65cbf0f4f53ba9416fa6c7b1a87824", - "chars": 14749 + "sha256": "15cc69fb5fe5916661dae029d5f0cef1c4371e73bf76b5975a66d2fc728c73a9", + "chars": 15610 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1644,8 +1644,8 @@ "id": "c7", "check": "Ending (G4)", "question": { - "sha256": "54238cbf0af0a835abdfe2d0cbf334aa3e873da78c2f9c31a2295b1e9fbb9989", - "chars": 569 + "sha256": "b9e41869d25408e03e99a37ec43f4bbd1f19bf8b7298ce9b95d01116128ae1ce", + "chars": 733 }, "graded_turns": [ 1, @@ -1655,8 +1655,8 @@ } ], "prompt": { - "sha256": "c5d466e0430143e5ac7fa228a2edba41c0b39c680e179f259ae7d838e1c41ed3", - "chars": 14125 + "sha256": "1c398507b7f876c53797438b29a20e22b74835df4c36675c1ff6a41ee831f0e2", + "chars": 14292 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1674,8 +1674,8 @@ "id": "c1", "check": "Guidance (A1, A3)", "question": { - "sha256": "9a9ea029a613397095f89de8b8f3aa230d8e22acf30a8c070f7259452fa6885d", - "chars": 2301 + "sha256": "cfdff39b14af763a90fd2b45f2465b08f8ba6e160c0c25f2af3b436b9e2bcd9c", + "chars": 2430 }, "graded_turns": [ 1, @@ -1685,8 +1685,8 @@ } ], "prompt": { - "sha256": "dbc581c3c05c9374ee336e78d9f415ac5b6e27a34a75ae3fdc8b686ca215cc5f", - "chars": 15879 + "sha256": "bbf91936b92a941b6d4b085cfc182d011497ac69b16e4ee59e875a8171646625", + "chars": 16009 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1704,8 +1704,8 @@ "id": "c6", "check": "Made-up facts (A2)", "question": { - "sha256": "0481faff49e323c5554f335302e31b91cbbd214ff6212d480450d22a94a55f31", - "chars": 1153 + "sha256": "546e52ab72fd8a64fd53235cddc1e05e2193d4757f090313432c8b1e0609f54e", + "chars": 1304 }, "graded_turns": [ 1, @@ -1715,8 +1715,8 @@ } ], "prompt": { - "sha256": "80f64deec2ee83543e76e66e9060e0bae88bbdf1f61bba779e78fb33037b862f", - "chars": 14725 + "sha256": "cd9af95f0ba845f008717a8141e53878da3284ea7a891634b30aab72e528cc36", + "chars": 14879 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1734,8 +1734,8 @@ "id": "c2", "check": "Oversharing (G1, G2)", "question": { - "sha256": "5d10ce962a3dd5c76c82a573361afae18e961e70a70f7e476a79336262a240f3", - "chars": 1675 + "sha256": "8986dc1e90d19ba708a46e79e5d747b0ac9f3216c07aa0505f61a649e8e20613", + "chars": 1993 }, "graded_turns": [ 1, @@ -1745,8 +1745,8 @@ } ], "prompt": { - "sha256": "4025ef816fcb04893eca08561fca72855c2928a6717acfc5fefee9009231f0c1", - "chars": 15245 + "sha256": "4be5fea3dd212878f8f26b572f2b457c29adc3a764fd55cddc5cb33d8a32766b", + "chars": 15568 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": { @@ -1764,8 +1764,8 @@ "id": "c5", "check": "Rogue (G3, G5)", "question": { - "sha256": "b1b197d9df1e6549cbaa0bcde06a8fccc9b7f32f8671cc8cc01f5b855ff89e5f", - "chars": 1216 + "sha256": "cf084b70533a39f5860bdec4ee04b807e0e7c3ec74da0f50d5ae4f2d6b068236", + "chars": 1978 }, "graded_turns": [ 1, @@ -1775,8 +1775,8 @@ } ], "prompt": { - "sha256": "1af975f173b37595bde12f17b1a7b8122d50defecc87659065fc822fe957f180", - "chars": 14781 + "sha256": "7ac55b1c51d8fc0892a73af699f66b778a4d2a49b4e13fe063bf424d951df7b7", + "chars": 15559 }, "system": "You grade a user simulator: an AI playing a real person who messages their personal AI assistant. You grade the user sim, never the assistant.", "cached_prefix": {