From 6dbacb9b41f5b55ddeab208bc6268051e6881902 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 02:15:11 -0500 Subject: [PATCH 1/6] fix(training): refuse an OpenEnv episode whose first-turn tools differ from its config group's probed task --- AGENTS.md | 8 ++- src/agent_env/training/_chat.py | 20 +++++-- src/agent_env/training/_openenv.py | 38 +++++++++--- src/agent_env/training/nemo_gym/prepare.py | 7 ++- src/agent_env/training/nemo_gym/server.py | 8 ++- src/agent_env/training/skyrl/env.py | 15 ++++- src/agent_env/training/skyrl/prepare.py | 18 ++++-- src/agent_env/training/trl.py | 7 ++- src/agent_env/training/verl/loop.py | 5 +- src/agent_env/training/verl/prepare.py | 18 ++++-- tst/nemo_gym/server_test.py | 6 +- tst/skyrl/generator_test.py | 16 ++++- tst/unit/training/_support.py | 60 ++++++++++++++++++- tst/unit/training/chat_test.py | 68 +++++++++++++++++++++- tst/unit/training/conftest.py | 2 +- tst/unit/training/nemo_gym_test.py | 12 +++- tst/unit/training/skyrl_test.py | 15 ++++- tst/unit/training/trl_test.py | 16 +++++ tst/unit/training/verl_test.py | 15 ++++- tst/verl/loop_test.py | 11 ++++ 20 files changed, 317 insertions(+), 48 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index e03fa8703..055c47acd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,13 +153,15 @@ share. Shared: - `_openenv.py`: the session on a server `openenv/` makes that each helper plays an episode through (`Session`: a background start, tool calls, the turns left open ended until scored), with the Task API reads, the per-toolset probe, - `tasks_and_tools` (what the row writers read: the tasks, the split names checked, each toolset's tools) and an - in-process `Server`. + `tasks_and_tools` (what the row writers read: the tasks, the split names checked, each toolset's tools, and for a + toolset whose tasks apply server config the probed task's first-turn tools, which rows carry as `first_turn_tools` + and an episode's first turn must then offer exactly) and an in-process `Server`. - `_tools.py`: turns an env's tools into typed, documented Python methods from their JSON Schema, as transformers' `get_json_schema` reads them. - `_chat.py`, the rows and episode the Miles and slime helpers share: - `write_rows` and its `command` write a server's tasks as `.jsonl` rows of chat `messages` and `metadata` - (the episode's split, index, task id and tools, and the server it plays on with its deadlines and turn cap). + (the episode's split, index, task id and tools, `first_turn_tools` when its task applies server config, and the + server it plays on with its deadlines and turn cap). - `play` plays a row's episode, the model chatting at a trainer's OpenAI chat endpoint that records its turns, with the row's tools. It answers a call whose arguments aren't a JSON object as the model's error, and returns the reward, or flags an episode the env or the session failed, whose model missed the row's `model_timeout` or that ended diff --git a/src/agent_env/training/_chat.py b/src/agent_env/training/_chat.py index 0d71d554c..4ccd76dcb 100644 --- a/src/agent_env/training/_chat.py +++ b/src/agent_env/training/_chat.py @@ -14,7 +14,17 @@ import click import httpx -from agent_env.training._openenv import Session, Shared, function_tool, key, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + Session, + Shared, + exactly, + function_tool, + key, + messages, + tasks_and_tools, + token_in, +) @click.command() @@ -50,8 +60,8 @@ def write_rows(url: str, out: str | Path, *, tool_timeout: float = 60.0, reset_t rollout may take, and the environment variable holding the token a server given one asks for, read in each of the trainer's rollout processes.""" url = url.rstrip("/") - _, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + _, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", + timeout=reset_timeout, token=token_in(token_var)) tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} server = {"url": url, "tool_timeout": tool_timeout, "reset_timeout": reset_timeout, "model_timeout": model_timeout, "max_turns": max_turns, **({"token_var": token_var} if token_var else {})} @@ -61,6 +71,7 @@ def write_rows(url: str, out: str | Path, *, tool_timeout: float = 60.0, reset_t (out / f"{split}.jsonl").write_text("".join(json.dumps({ "messages": messages(task["system_prompt"], task["prompt"]), "metadata": {"split": split, "index": task["index"], "task_id": task["id"], "tools": tools[key(task)], + **({FIRST_TURN_TOOLS: firsts[key(task)]} if key(task) in firsts else {}), "agentenv": server}, }) + "\n" for task in described)) return out @@ -89,7 +100,8 @@ async def play(base_url: str, prompt: list[dict[str, Any]], request_kwargs: dict server, tools = row["agentenv"], row["tools"] episode = _Episode(Shared(server["url"], frozenset(tool["function"]["name"] for tool in tools), server["tool_timeout"], server["reset_timeout"], patience=server["reset_timeout"], - waits_for_server=True, token=token_in(server.get("token_var")))) + waits_for_server=True, token=token_in(server.get("token_var")), + exactly=exactly(row.get(FIRST_TURN_TOOLS)))) counts = {"agentenv_tool_calls": 0, "agentenv_invalid_calls": 0} try: await episode._start(row["split"], int(row["index"]), row["task_id"]) diff --git a/src/agent_env/training/_openenv.py b/src/agent_env/training/_openenv.py index 8b96ae404..3d9aaf8af 100644 --- a/src/agent_env/training/_openenv.py +++ b/src/agent_env/training/_openenv.py @@ -28,6 +28,7 @@ FAILED = "The environment failed to answer, so this episode won't be scored." ENDED = "The episode has ended." END_TURN = END_TURN_TOOL.name +FIRST_TURN_TOOLS = "first_turn_tools" CONNECT_SECONDS = 60.0 CLOSE_SECONDS = 300.0 #: How much longer than a call's deadline the client waits for the server to end the call, before dropping the session. @@ -64,8 +65,9 @@ class NamedToolset: class Shared: """What a session needs of its server: where it is, the tools the model is shown, which an episode's first turn must offer no more than, the deadlines, how long to wait for room on the server, whether to wait for a server that - refuses connections, as one restarting does, as long as a reset may take, and the token it asks for. The key makes - it equal to its pickled copies only.""" + refuses connections, as one restarting does, as long as a reset may take, the token it asks for, and for a toolset + whose tasks apply server config, the tools its probed task offered at its first turn, which an episode's first turn + must offer exactly. The key makes it equal to its pickled copies only.""" url: str offered: frozenset[str] @@ -74,6 +76,7 @@ class Shared: patience: float = REOPEN_PATIENCE waits_for_server: bool = False token: str | None = field(default=None, repr=False) + exactly: frozenset[str] | None = None key: str = field(default_factory=lambda: uuid.uuid4().hex) @@ -156,6 +159,10 @@ async def _begin(self, rollout: Rollout, split: str, index: int, expected: str | if not (offered := frozenset(tool["name"] for tool in listed.observation["tools"])) <= self._shared.offered: raise Misgrouped(f"task {played!r} offers the tools {sorted(offered)}, not the " f"{sorted(self._shared.offered)} {self.offered_by}") + if (exactly := self._shared.exactly) is not None and offered != exactly: + raise Misgrouped(f"task {played!r} offers the tools {sorted(offered)} at its first turn, not the " + f"{sorted(exactly)} its toolset's first task does: their server configs reveal different " + "tools, so serve them in separate bundles") async def _step(self, name: str, arguments: dict[str, Any]) -> str: async with self._lock: @@ -391,16 +398,33 @@ def read_tasks(url: str, *, local: str, token: str | None = None) -> tuple[str, def tasks_and_tools(url: str, *, local: str, timeout: float, token: str | None = None - ) -> tuple[str, dict[str, list[dict[str, Any]]], dict[str, tuple[dict[str, Any], ...]]]: + ) -> tuple[str, dict[str, list[dict[str, Any]]], dict[str, tuple[dict[str, Any], ...]], + dict[str, list[str]]]: """What a writer of a split's rows reads: the server's env name, each split's tasks, refusing a split whose name - can't name a data file, and every tool each toolset's turns offer, by the toolset's key, found by playing its - first task through (``probe``), so reading deploys those.""" + can't name a data file, every tool each toolset's turns offer, by the toolset's key, found by playing its first + task through (``probe``), so reading deploys those, and for each toolset whose tasks apply server config, the + tools that task offered at its first turn, which a row carries as ``FIRST_TURN_TOOLS``.""" name, tasks = read_tasks(url, local=local, token=token) if unfit := [split for split in tasks if not _FILE.fullmatch(split)]: raise ValueError(f"the server's splits {unfit} can't name a data file: a split's name must be letters, " "digits, '.', '_' and '-', not starting with '.'") - return name, tasks, {each: asyncio.run(probe(url, toolset, timeout, token=token))[1] - for each, toolset in toolsets(name, tasks).items()} + probed = {each: asyncio.run(probe(url, toolset, timeout, token=token)) + for each, toolset in toolsets(name, tasks).items()} + return (name, tasks, {each: tools for each, (_, tools) in probed.items()}, + {each: sorted(first) for each, (first, _) in probed.items() if applies_config(each)}) + + +def applies_config(each: str) -> bool: + """Whether the tasks of the toolset keyed ``each`` apply server config, whose replies can reveal tools.""" + toolset: Toolset = json.loads(each) + return toolset.get("config", False) + + +def exactly(names: list[str] | str | None) -> frozenset[str] | None: + """A row's ``FIRST_TURN_TOOLS``, a list or its JSON, as ``Shared.exactly``; ``None`` for a row without it.""" + if isinstance(names, str): + names = json.loads(names) + return None if names is None else frozenset(names) def toolsets(name: str, tasks: dict[str, list[dict[str, Any]]]) -> dict[str, NamedToolset]: diff --git a/src/agent_env/training/nemo_gym/prepare.py b/src/agent_env/training/nemo_gym/prepare.py index 2b8f404f4..482486d66 100644 --- a/src/agent_env/training/nemo_gym/prepare.py +++ b/src/agent_env/training/nemo_gym/prepare.py @@ -12,7 +12,7 @@ import yaml from agent_env.openenv import build_app -from agent_env.training._openenv import Server, key, messages, tasks_and_tools +from agent_env.training._openenv import FIRST_TURN_TOOLS, Server, key, messages, tasks_and_tools #: The routes a resources server serves itself, which a tool's route can't take. GYM_ROUTES = frozenset({"seed_session", "verify", "close_session", "aggregate_metrics", "reverify_mode"}) @@ -67,8 +67,8 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non served = None if url else Server(build_app(source, reward=summed if reward_components else reward, sandbox=sandbox, max_concurrent_envs=1)) try: - env_name, tasks, probed = tasks_and_tools(url or served.url, local="name the bundle's folder instead of its " - "server", timeout=reset_timeout) + env_name, tasks, probed, firsts = tasks_and_tools(url or served.url, local="name the bundle's folder instead " + "of its server", timeout=reset_timeout) tools = {each: [_function(tool) for tool in found] for each, found in probed.items()} finally: if served is not None: @@ -96,6 +96,7 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non "responses_create_params": {"input": messages(task["system_prompt"], task["prompt"]), "tools": tools[key(task)]}, "agent_ref": agent, "split": split, "index": task["index"], "task_id": task["id"], + **({FIRST_TURN_TOOLS: firsts[key(task)]} if key(task) in firsts else {}), }) + "\n" for task in described)) return server diff --git a/src/agent_env/training/nemo_gym/server.py b/src/agent_env/training/nemo_gym/server.py index e245675fc..e688e9159 100644 --- a/src/agent_env/training/nemo_gym/server.py +++ b/src/agent_env/training/nemo_gym/server.py @@ -41,7 +41,7 @@ "Python 3.13.14 or later as NeMo Gym is") from exc from agent_env.openenv import build_app -from agent_env.training._openenv import STOP_SECONDS, Server, Session, Shared +from agent_env.training._openenv import STOP_SECONDS, Server, Session, Shared, exactly from agent_env.training.nemo_gym.prepare import REWARD_AND_COMPONENTS, summed logger = logging.getLogger(__name__) @@ -89,7 +89,8 @@ def _checked(self) -> "AgentEnvResourcesServerConfig": class AgentEnvSeedSessionRequest(BaseSeedSessionRequest): - """A row: the tools it shows the model, and the task it plays.""" + """A row: the tools it shows the model, the task it plays, and for a task that applies server config, the tools its + first turn must offer exactly.""" model_config = ConfigDict(extra="allow") @@ -97,6 +98,7 @@ class AgentEnvSeedSessionRequest(BaseSeedSessionRequest): split: str index: int task_id: str | None = None + first_turn_tools: list[str] | None = None class AgentEnvVerifyRequest(BaseVerifyRequest): @@ -139,7 +141,7 @@ async def seed_session(self, request: Request, body: AgentEnvSeedSessionRequest) session_id = request.session[SESSION_ID_KEY] offered = frozenset(tool["name"] for tool in body.responses_create_params.get("tools") or ()) session = _Session(Shared(self._url, offered, self.config.tool_timeout, self.config.reset_timeout, - patience=self.config.reset_timeout)) + patience=self.config.reset_timeout, exactly=exactly(body.first_turn_tools))) if (previous := self._sessions.pop(session_id, None)) is not None: await previous.close() self._sessions[session_id] = session diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index 43b934439..0c9538bea 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -16,7 +16,17 @@ raise ImportError("agent_env.training.skyrl.env runs in SkyRL's trainer: install 'agentenv-framework[skyrl]' into " "SkyRL's environment") from exc -from agent_env.training._openenv import CLOSE_SECONDS, GRACE_SECONDS, Rollout, Session, Shared, on_session_loop, token_in +from agent_env.training._openenv import ( + CLOSE_SECONDS, + FIRST_TURN_TOOLS, + GRACE_SECONDS, + Rollout, + Session, + Shared, + exactly, + on_session_loop, + token_in, +) from agent_env.training.skyrl.calls import Call, Unparsed, parse logger = logging.getLogger(__name__) @@ -37,7 +47,8 @@ def __init__(self, env_config: Any = None, extras: dict[str, Any] | None = None) + 2 * GRACE_SECONDS + CLOSE_SECONDS self._session = _Session(Shared(server["url"], frozenset(self._tools), server["tool_timeout"], server["reset_timeout"], patience=server["reset_timeout"], - waits_for_server=True, token=token_in(server.get("token_var")))) + waits_for_server=True, token=token_in(server.get("token_var")), + exactly=exactly(self._episode.get(FIRST_TURN_TOOLS)))) self._scores: dict[str, float] = {} self._run = self._unparsed = 0 self._failed = False diff --git a/src/agent_env/training/skyrl/prepare.py b/src/agent_env/training/skyrl/prepare.py index a1fe2ca43..e7d36357f 100644 --- a/src/agent_env/training/skyrl/prepare.py +++ b/src/agent_env/training/skyrl/prepare.py @@ -13,7 +13,15 @@ import pyarrow.parquet as pq from transformers import AutoTokenizer -from agent_env.training._openenv import function_tool, key, longest_prompt, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + function_tool, + key, + longest_prompt, + messages, + tasks_and_tools, + token_in, +) from agent_env.training.skyrl.calls import FORMATS #: The ``env_class`` every row plays under, and the id the env is registered at. @@ -24,7 +32,7 @@ ("env_class", pa.string()), ("data_source", pa.string()), ("episode", pa.struct([("split", pa.string()), ("index", pa.int64()), ("task_id", pa.string()), - ("tools", pa.string())])), + ("tools", pa.string()), (FIRST_TURN_TOOLS, pa.string())])), ("server", pa.struct([("url", pa.string()), ("tool_timeout", pa.float64()), ("reset_timeout", pa.float64()), ("attempts", pa.int64()), ("tool_format", pa.string()), ("token_var", pa.string())])), ]) @@ -52,8 +60,8 @@ def write_dataset(url: str, out: str | Path, *, tokenizer: str, tool_format: str if tool_format not in FORMATS: raise ValueError(f"tool_format must be one of {', '.join(FORMATS)}, not {tool_format!r}") url = url.rstrip("/") - env_name, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + env_name, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app " + "first", timeout=reset_timeout, token=token_in(token_var)) template = AutoTokenizer.from_pretrained(tokenizer) kwargs = chat_template_kwargs or {} tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} @@ -64,7 +72,7 @@ def write_dataset(url: str, out: str | Path, *, tokenizer: str, tool_format: str "env_class": AGENT, "data_source": f"{AGENT}/{env_name}", "episode": {"split": split, "index": task["index"], "task_id": task["id"], - "tools": json.dumps(tools[key(task)])}, + "tools": json.dumps(tools[key(task)]), FIRST_TURN_TOOLS: json.dumps(firsts[key(task)]) if key(task) in firsts else None}, "server": server, } for task in described] for split, described in tasks.items()} longest = _longest(rows, template, max_prompt_length, kwargs) diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index fc1de3f8b..efe10e853 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -39,6 +39,7 @@ Server, Session, Shared, + applies_config, key, messages, on_session_loop, @@ -153,12 +154,12 @@ def close(self) -> None: _stop(self._server) async def _environments(self) -> list[type[_Environment]]: - return [await self._environment(toolset) for toolset in self._toolsets.values()] + return [await self._environment(toolset, applies_config(each)) for each, toolset in self._toolsets.items()] - async def _environment(self, toolset: NamedToolset) -> type[_Environment]: + async def _environment(self, toolset: NamedToolset, config: bool) -> type[_Environment]: first, tools = await probe(self.url, toolset, self._reset_timeout) shared = Shared(url=self.url, offered=first, tool_timeout=self._tool_timeout, reset_timeout=self._reset_timeout, - waits_for_server=True) + waits_for_server=True, exactly=first if config else None) self._shared.append(shared) return _environment_class(toolset.name, tools, shared) diff --git a/src/agent_env/training/verl/loop.py b/src/agent_env/training/verl/loop.py index d7d69109c..b2088000c 100644 --- a/src/agent_env/training/verl/loop.py +++ b/src/agent_env/training/verl/loop.py @@ -23,7 +23,7 @@ raise ImportError("agent_env.training.verl.loop runs in verl's agent loop workers: install " "'agentenv-framework[verl]' into verl's environment") from exc -from agent_env.training._openenv import Rollout, Session, Shared, attempt_numbers, token_in +from agent_env.training._openenv import FIRST_TURN_TOOLS, Rollout, Session, Shared, attempt_numbers, exactly, token_in logger = logging.getLogger(__name__) #: The tools whose answers verl was seen to cut, each warned of once a process. @@ -82,7 +82,8 @@ async def run(self, sampling_params: dict[str, Any], **kwargs: Any) -> AgentLoop row = self._row = kwargs["extra_info"] schemas = json.loads(row["tools"]) shared = Shared(self._url, frozenset(schema["function"]["name"] for schema in schemas), self._tool_timeout, - self._reset_timeout, patience=self._reset_timeout, waits_for_server=True, token=self._token) + self._reset_timeout, patience=self._reset_timeout, waits_for_server=True, token=self._token, + exactly=exactly(row.get(FIRST_TURN_TOOLS))) for attempt in attempt_numbers(self._attempts, self._reset_timeout): session = _Session(shared) try: diff --git a/src/agent_env/training/verl/prepare.py b/src/agent_env/training/verl/prepare.py index ec9737f45..a99c3ac1c 100644 --- a/src/agent_env/training/verl/prepare.py +++ b/src/agent_env/training/verl/prepare.py @@ -14,7 +14,15 @@ import yaml from transformers import AutoTokenizer -from agent_env.training._openenv import function_tool, key, longest_prompt, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + function_tool, + key, + longest_prompt, + messages, + tasks_and_tools, + token_in, +) #: The ``agent_name`` every row plays under, and the name ``agent_loop.yaml`` registers the loop at. AGENT = "agentenv" @@ -25,7 +33,7 @@ ("prompt", pa.list_(pa.struct([("role", pa.string()), ("content", pa.string())]))), ("agent_name", pa.string()), ("extra_info", pa.struct([("split", pa.string()), ("index", pa.int64()), ("task_id", pa.string()), - ("tools", pa.string())])), + ("tools", pa.string()), (FIRST_TURN_TOOLS, pa.string())])), ]) @@ -58,15 +66,15 @@ def write_dataset(url: str, out: str | Path, *, tool_timeout: float = 60.0, rese if tokenizer is None and (max_prompt_length is not None or chat_template_kwargs): raise ValueError("max_prompt_length and chat_template_kwargs count tokens, so they take the model's tokenizer") url = url.rstrip("/") - env_name, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + env_name, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app " + "first", timeout=reset_timeout, token=token_in(token_var)) tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} rows = {split: [{ "data_source": f"agentenv/{env_name}", "prompt": messages(task["system_prompt"], task["prompt"]), "agent_name": AGENT, "extra_info": {"split": split, "index": task["index"], "task_id": task["id"], - "tools": json.dumps(tools[key(task)])}, + "tools": json.dumps(tools[key(task)]), FIRST_TURN_TOOLS: json.dumps(firsts[key(task)]) if key(task) in firsts else None}, } for task in described] for split, described in tasks.items()} longest = None if tokenizer is None else _longest(rows, AutoTokenizer.from_pretrained(tokenizer), max_prompt_length, chat_template_kwargs or {}) diff --git a/tst/nemo_gym/server_test.py b/tst/nemo_gym/server_test.py index ecc588a42..862144ca0 100644 --- a/tst/nemo_gym/server_test.py +++ b/tst/nemo_gym/server_test.py @@ -70,6 +70,8 @@ def test_end_turn_answers_with_the_next_turns_prompt_and_a_turn_left_open_ends_e ({"a": PLAYED}, {"task_id": "b"}, "the episode plays task 'a', not the row's 'b'"), ({"a": PLAYED}, {"tools": [FUNCTION_END_TURN]}, "task 'a' offers the tools ['add_item', 'end_turn'], not the ['end_turn'] the row shows the model"), + ({"a": PLAYED}, {"first_turn_tools": ["add_item", "end_turn", "take_item"]}, + "at its first turn, not the ['add_item', 'end_turn', 'take_item'] its toolset's first task does"), ]) def test_a_rollout_the_environment_fails_or_that_isnt_its_rows_is_masked(serve, tasks, row, why): server = serve(tasks) @@ -264,9 +266,9 @@ def _app(**config): ).setup_webserver() -def _row(task_id="a", tools=(ADD_ITEM, FUNCTION_END_TURN)): +def _row(task_id="a", tools=(ADD_ITEM, FUNCTION_END_TURN), **extra): return {"responses_create_params": {"input": [{"role": "user", "content": "Add a crate."}], "tools": list(tools)}, - "split": "all", "index": 0, "task_id": task_id} + "split": "all", "index": 0, "task_id": task_id, **extra} def _verify(client, row, reply): diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index ec34f855e..1d4cd3181 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -5,11 +5,12 @@ import json from types import SimpleNamespace +import pyarrow as pa import pyarrow.parquet as pq import pytest from transformers import AutoTokenizer -from agent_env.training.skyrl.prepare import token_vars, write_dataset +from agent_env.training.skyrl.prepare import ROW, token_vars, write_dataset from tst.unit.task.episode import run_test from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN from tst.unit.training._support import PLAYED @@ -236,6 +237,19 @@ async def hanging(self, context): assert (output["env_metrics"][0]["failed"], output["env_metrics"][0]["attempts"]) == (1, 1) +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_raises( + serve, tmp_path, tool_format, tokenizer): + server = serve({"a": PLAYED}) + rows = _rows(server.url, tmp_path, tool_format) + written = pq.read_table(rows).to_pylist() + written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) + pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) + + with pytest.raises(ValueError, match="their server configs reveal different tools"): + _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) + server.stop() + + def test_an_episode_that_ends_without_failing_or_a_reward_raises(serve, tmp_path, tool_format, tokenizer): server = serve({"a": PLAYED}, reward=lambda scores: None) rows = _rows(server.url, tmp_path, tool_format) diff --git a/tst/unit/training/_support.py b/tst/unit/training/_support.py index 12ae529c8..cdc57ae6d 100644 --- a/tst/unit/training/_support.py +++ b/tst/unit/training/_support.py @@ -4,12 +4,14 @@ import pytest +from agent_env.config.runtime import Config from agent_env.openenv.app import _app from agent_env.openenv.reward import _score_at from agent_env.openenv.environment import Served, ServedTask from agent_env.task import EpisodeSpec, Task +from agent_env.task_step.task_step import TaskStep from agent_env.training._openenv import Server -from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN +from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN, _Env #: A task the model plays by adding a crate and saying so, which its verifier scores 1. PLAYED = [ITEMS, ALICE, TURN, REPLIES] @@ -41,3 +43,59 @@ def served(tasks, splits=None, max_concurrent_envs=8, name="dock", reward=_score yield served for server in servers: server.stop() + + +class _Config(TaskStep): + """Stands in for ``apply_server_config`` on ``env_id``, whose reply reveals the tools ``reveal``.""" + + type = "apply_server_config" + entity_refs = () + revealed: list[str] = [] + applied = 0 + + def __init__(self, id, version, env_id="items", reveal=(), depends_on=None, fail_task_on_error=True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id, self.reveal = env_id, list(reveal) + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data), env_id=data.get("env_id", "items"), reveal=data.get("reveal", ())) + + async def execute(self, context): + _Config.revealed[:] = self.reveal + _Config.applied += 1 + return context + + +class _Offered: + def __iter__(self): + return iter(["add_item", *_Config.revealed]) + + +@pytest.fixture +def reveals(monkeypatch, tools): + """A config step whose reply reveals ``reveal``, given ``args``; each episode starts with nothing revealed, and + ``reveals.applied()`` counts the config steps run.""" + registry = Config().task_step_registry() + monkeypatch.setattr(Config, "task_step_registry", lambda self: {**registry, _Config.type: _Config}) + deploy = _Env.execute + + async def deploying(self, context): + _Config.revealed.clear() + return await deploy(self, context) + + monkeypatch.setattr(_Env, "execute", deploying) + monkeypatch.setattr(_Config, "revealed", []) + monkeypatch.setattr(_Config, "applied", 0) + tools().offered = _Offered() + + def config(*reveal, **args): + return {"id": "cfg", "type": _Config.type, "env_id": "items", "reveal": list(reveal), + "directives": [{"service": "items", "uri": "urn:example:set-errors/v1", "args": args}]} + + config.applied = lambda: _Config.applied + return config + + +def revealing(config, *reveal, **args): + return [ITEMS, ALICE, config(*reveal, **args), TURN, REPLIES] diff --git a/tst/unit/training/chat_test.py b/tst/unit/training/chat_test.py index ea9475be5..378fcc0ec 100644 --- a/tst/unit/training/chat_test.py +++ b/tst/unit/training/chat_test.py @@ -5,6 +5,7 @@ import asyncio import json +import re from types import SimpleNamespace import pytest @@ -19,8 +20,9 @@ from agent_env.training.miles.__main__ import main as miles_main from agent_env.training._chat import keep_scored_groups, play, reward, write_rows from agent_env.training.slime.__main__ import main as slime_main +from tst.unit.openenv.environment_test import AGAIN from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -107,6 +109,70 @@ def test_the_agent_plays_a_rows_task_through_the_session_and_returns_its_reward( assert second["messages"][-1]["content"] +def _rows(server, tmp_path): + folder = write_rows(server.url, tmp_path / "rows") + return {row["metadata"]["task_id"]: row for row in map(json.loads, (folder / "all.jsonl").read_text().splitlines())} + + +def _played(model, row): + return asyncio.run(play(model.base_url, row["messages"], SAMPLING, row["metadata"])) + + +@pytest.mark.parametrize("reveal", [("take_item",), ()], ids=["reveal-the-same", "reveal-nothing"]) +def test_tasks_whose_configs_reveal_the_same_tools_share_one_probe_whatever_their_args_and_play( + serve, session, tmp_path, reveals, reveal): + server = serve({"a": revealing(reveals, *reveal, rate=0.1), "b": revealing(reveals, *reveal, rate=0.5), + "plain": PLAYED}) + rows = _rows(server, tmp_path) + probes = reveals.applied() + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + played = _played(model, rows["b"]) + server.stop() + + assert probes == 1 + assert rows["a"]["metadata"]["first_turn_tools"] == rows["b"]["metadata"]["first_turn_tools"] == sorted( + ["add_item", "end_turn", *reveal]) + assert "first_turn_tools" not in rows["plain"]["metadata"] + assert played["reward"] == 1.0 + + +def test_a_config_between_turns_is_compared_on_the_first_turns_tools_not_on_every_turns(serve, session, tmp_path, + reveals): + server = serve({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES]}) + row = _rows(server, tmp_path)["a"] + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + played = _played(model, row) + server.stop() + + assert [tool["function"]["name"] for tool in row["metadata"]["tools"]] == ["add_item", "end_turn", "take_item"] + assert row["metadata"]["first_turn_tools"] == ["add_item", "end_turn"] + assert "agentenv_failure" not in played + + +@pytest.mark.parametrize("probed, later, refusal", [ + (["take_item"], [], r"\['add_item', 'end_turn'\] at its first turn, not the \['add_item', 'end_turn', " + r"'take_item'\] its toolset's first task does: their server configs reveal different tools"), + ([], ["take_item"], r"offers the tools \['add_item', 'end_turn', 'take_item'\], not the \['add_item', 'end_turn'\]"), +], ids=["fewer", "more"]) +def test_a_task_whose_config_reveals_other_tools_than_its_probed_tasks_is_refused_and_an_older_row_plays( + serve, session, tmp_path, reveals, probed, later, refusal): + server = serve({"a": revealing(reveals, *probed), "b": revealing(reveals, *later)}) + rows = _rows(server, tmp_path) + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + refused = _played(model, rows["b"]) + older = {**rows["b"], "metadata": {k: v for k, v in rows["b"]["metadata"].items() if k != "first_turn_tools"}} + played = _played(model, older) + server.stop() + + assert refused["reward"] == 0.0 + assert re.search(refusal, refused["agentenv_failure"]) + assert ("agentenv_failure" in played) == (later != []) + + def test_a_call_whose_arguments_arent_an_object_is_answered_and_the_episode_goes_on(serve, session, tmp_path): server = serve({"a": PLAYED}) row = _row(server, tmp_path) diff --git a/tst/unit/training/conftest.py b/tst/unit/training/conftest.py index 8183c3c0d..9779445f3 100644 --- a/tst/unit/training/conftest.py +++ b/tst/unit/training/conftest.py @@ -4,4 +4,4 @@ from tst.unit.openenv.environment_test import tools # noqa: F401 from tst.unit.task.episode.conftest import _disable_network # noqa: F401 from tst.unit.task.episode.run_test import teardowns # noqa: F401 -from tst.unit.training._support import serve # noqa: F401 +from tst.unit.training._support import reveals, serve # noqa: F401 diff --git a/tst/unit/training/nemo_gym_test.py b/tst/unit/training/nemo_gym_test.py index ed31c0ad1..af0be8374 100644 --- a/tst/unit/training/nemo_gym_test.py +++ b/tst/unit/training/nemo_gym_test.py @@ -23,7 +23,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing STYLE = {**REPLIES, "id": "style", "verifier_id": "style"} ADD_ITEM = {"type": "function", "name": "add_item", "description": "add_item.", "parameters": {"type": "object"}, @@ -84,6 +84,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + folder = write_server(server.url, tmp_path, name="dock") + server.stop() + + assert [row.get("first_turn_tools") for row in _rows(folder / "data" / "all.jsonl")] == [ + ["add_item", "end_turn", "take_item"], None] + + def test_a_split_whose_name_isnt_a_file_name_is_refused_before_anything_is_written(serve, tmp_path): server = serve({"a": PLAYED}, {"../escape": ["a"]}) diff --git a/tst/unit/training/skyrl_test.py b/tst/unit/training/skyrl_test.py index 2e7a30fe2..ce8e5a7c5 100644 --- a/tst/unit/training/skyrl_test.py +++ b/tst/unit/training/skyrl_test.py @@ -24,7 +24,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -87,7 +87,8 @@ def test_a_servers_tasks_become_a_row_per_task_with_its_tools_in_its_system_prom {"role": "user", "content": "Add a barrel."}], "env_class": "agentenv", "data_source": "agentenv/Dock Yard", - "episode": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN]}, + "episode": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN], + "first_turn_tools": None}, "server": {"url": server.url, "tool_timeout": 30.0, "reset_timeout": 90.0, "attempts": 2, "tool_format": "qwen3_coder", "token_var": None}, } @@ -131,6 +132,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, tokenizer, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + out = write_dataset(server.url, tmp_path, tokenizer=tokenizer()).folder + server.stop() + + assert [row["episode"]["first_turn_tools"] for row in _rows(out / "all.parquet")] == [ + json.dumps(["add_item", "end_turn", "take_item"]), None] + + def test_a_split_the_server_keeps_empty_has_a_rows_columns(serve, tmp_path, tokenizer): server = serve({"a": PLAYED}, {"empty": []}) diff --git a/tst/unit/training/trl_test.py b/tst/unit/training/trl_test.py index dd7868342..05dcb14cb 100644 --- a/tst/unit/training/trl_test.py +++ b/tst/unit/training/trl_test.py @@ -23,6 +23,7 @@ from tst.unit.bundle._support import layout from tst.unit.openenv.environment_test import AGAIN from tst.unit.task.episode.run_test import ALICE, ITEMS, PANTRY, PANTRY_MCP, REPLIES, TURN, _tool_sessions +from tst.unit.training._support import revealing from tst.unit.training.tools_test import called_by_qwen3, shown_to_qwen3 PLAYED = [ITEMS, ALICE, TURN, REPLIES] @@ -278,6 +279,21 @@ def test_a_rollout_whose_tools_arent_its_classs_stops_training(served, monkeypat environment.get_reward() +@pytest.mark.parametrize("later, stops", [(["take_item"], False), ([], True)], ids=["same-reveal", "fewer"]) +def test_a_rollout_whose_config_reveals_fewer_tools_than_its_classs_stops_training(served, reveals, later, stops): + env = served({"a": revealing(reveals, "take_item", rate=0.1), "b": revealing(reveals, *later, rate=0.5)}) + environment = env.factory() + + environment.reset(**env.dataset[1]) + environment.add_item(item="crate") + environment.end_turn(response="Added a crate.") + if stops: + with pytest.raises(RuntimeError, match=r"their server configs reveal different tools"): + environment.get_reward() + else: + assert environment.get_reward() == 1.0 + + class _Slow(TaskStep): """Takes a second, as a verifier that grades the episode after its last turn does.""" diff --git a/tst/unit/training/verl_test.py b/tst/unit/training/verl_test.py index d472ce5c3..fcd38894d 100644 --- a/tst/unit/training/verl_test.py +++ b/tst/unit/training/verl_test.py @@ -23,7 +23,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -50,7 +50,8 @@ def test_a_servers_tasks_become_a_row_per_task_of_each_split_and_the_agent_loop_ "data_source": "agentenv/Dock Yard", "prompt": [{"role": "system", "content": "You keep the items."}, {"role": "user", "content": "Add a barrel."}], "agent_name": "agentenv", - "extra_info": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN]}, + "extra_info": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN], + "first_turn_tools": None}, } assert [(row["extra_info"]["index"], row["extra_info"]["task_id"]) for row in _rows(out / "all.parquet")] == [ (0, "a"), (1, "b")] @@ -69,6 +70,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + out = write_dataset(server.url, tmp_path).folder + server.stop() + + assert [row["extra_info"]["first_turn_tools"] for row in _rows(out / "all.parquet")] == [ + json.dumps(["add_item", "end_turn", "take_item"]), None] + + def test_a_split_the_server_keeps_empty_has_a_rows_columns(serve, tmp_path): server = serve({"a": PLAYED}, {"empty": []}) diff --git a/tst/verl/loop_test.py b/tst/verl/loop_test.py index 267377a6f..ebed08c86 100644 --- a/tst/verl/loop_test.py +++ b/tst/verl/loop_test.py @@ -119,6 +119,17 @@ def test_a_rollout_the_env_fails_every_time_raises_so_verl_leaves_it_out(serve, broken.stop() +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_is_refused( + serve, tmp_path, tokenizer): + server = serve({"a": PLAYED}) + row = _row(write_dataset(server.url, tmp_path)) + row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) + + with pytest.raises(verl_loop.RolloutFailed, match="their server configs reveal different tools"): + asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) + server.stop() + + def test_a_rollout_isnt_played_again_once_its_time_is_up(serve, tmp_path, tokenizer, monkeypatch): server = serve({"a": PLAYED}) row = _row(write_dataset(server.url, tmp_path)) From ad43e8950393fabccb938956fa4a2a7c716f156c Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 02:30:19 -0500 Subject: [PATCH 2/6] address review round 2: test a matching stored first_turn_tools and the multi-turn more case, refusal texts name both directions, AGENTS.md row lines --- AGENTS.md | 7 ++++--- src/agent_env/training/_openenv.py | 6 ++---- src/agent_env/training/skyrl/env.py | 2 +- src/agent_env/training/trl.py | 8 ++++---- tst/skyrl/generator_test.py | 7 ++++++- tst/unit/training/chat_test.py | 20 ++++++++++++++++++++ tst/verl/loop_test.py | 6 +++++- 7 files changed, 42 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 055c47acd..c6837f38e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,14 +182,14 @@ Per trainer: - `python -m agent_env.training.nemo_gym SOURCE ROOT` writes a bundle's NeMo Gym resources server into a Gym root, `NEMO_GYM_EXTRA_ROOTS`'s or a Gym checkout's: `app.py`; a `pyproject.toml`, since uv applies the nearest one's `[tool.uv]` and a checkout's `exclude-dependencies` would drop `cryptography`; a config with `simple_agent` and the - `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools. + `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools (and `first_turn_tools`). - `server.AgentEnvResourcesServer` is that server, run in the venv Gym builds: a row's seed starts its episode, a catch-all route calls its tools, and `verify` ends the turns left open with the model's last reply and returns the reward, masking a rollout the environment failed. With `reward_components` it also returns each score by step id, their sum the reward, for NeMo RL's GDPO. - `verl/` (`verl` extra, installed into verl's own environment): - `python -m agent_env.training.verl URL OUT` writes a server's tasks as verl's `.parquet` rows (the task's - prompt, `agent_name: agentenv`, the row's tools in `extra_info`) and `agent_loop.yaml`. It reads a server that asks + prompt, `agent_name: agentenv`, the row's tools and `first_turn_tools` in `extra_info`) and `agent_loop.yaml`. It reads a server that asks for a token with the one `--token-var` names, and with `--tokenizer` counts each prompt with its tools as verl renders it (`--chat-template-kwargs`), refusing rows over `--max-prompt-length` before writing. - `loop.AgentEnvAgentLoop` is verl's `ToolAgentLoop` with the row's tools: each call a call on the row's episode, the @@ -203,7 +203,8 @@ Per trainer: `max_tool_response_length`. - `skyrl/` (`skyrl` extra, installed into SkyRL's own environment): - `python -m agent_env.training.skyrl URL OUT --tokenizer MODEL` writes a server's tasks as SkyRL's `.parquet` - rows (`env_class: agentenv`, the episode's split, index, task id and tools, the server's settings). Each row has its + rows (`env_class: agentenv`, the episode's split, index, task id, tools and `first_turn_tools`, the server's + settings). Each row has its tools rendered into its system prompt by the model's chat template, checked to render as `tools=` would, since SkyRL's generator passes the template no tools; rows over `--max-prompt-length`, counted as SkyRL's filter and generator count them, are refused. diff --git a/src/agent_env/training/_openenv.py b/src/agent_env/training/_openenv.py index 3d9aaf8af..2ac2f1726 100644 --- a/src/agent_env/training/_openenv.py +++ b/src/agent_env/training/_openenv.py @@ -93,8 +93,8 @@ def __init__(self, url: str, limit: str): class Misgrouped(Failed): - """An episode offering a tool the model isn't shown: a toolset that misses what tells tasks apart, which no rollout - fixes.""" + """An episode offering a tool the model isn't shown, or with server config offering other first-turn tools than its + toolset's probed task: a toolset that misses what tells tasks apart, which no rollout fixes.""" @dataclass @@ -415,13 +415,11 @@ def tasks_and_tools(url: str, *, local: str, timeout: float, token: str | None = def applies_config(each: str) -> bool: - """Whether the tasks of the toolset keyed ``each`` apply server config, whose replies can reveal tools.""" toolset: Toolset = json.loads(each) return toolset.get("config", False) def exactly(names: list[str] | str | None) -> frozenset[str] | None: - """A row's ``FIRST_TURN_TOOLS``, a list or its JSON, as ``Shared.exactly``; ``None`` for a row without it.""" if isinstance(names, str): names = json.loads(names) return None if names is None else frozenset(names) diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index 0c9538bea..a8f23fec2 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -102,7 +102,7 @@ def _stepped(self, action: str) -> BaseTextEnvStepOutput: if rollout.misgrouped: self.close() raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools of its toolset's first " - "task, so a task must offer no tool that one doesn't") + "task, which its episode must offer") raise _Failed(rollout.failure) if not rollout.done: return BaseTextEnvStepOutput(observations=answers, reward=0.0, done=False, metadata={}) diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index efe10e853..8e46db7f9 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -196,8 +196,8 @@ def reset(self, *, split: str, index: int, task_id: str | None = None, **_: Any) on_session_loop(self._start(split, index, task_id), CONNECT_SECONDS + CLOSE_SECONDS) def get_reward(self) -> float | None: - """The episode's reward, or ``None`` when it ended in an error. Raises when the episode's first turn offered a - tool its toolset's first task doesn't, which no rollout fixes.""" + """The episode's reward, or ``None`` when it ended in an error. Raises when the episode is misgrouped + (``Misgrouped``), which no rollout fixes.""" rollout, self._rollout = self._rollout, None if rollout is None: return None @@ -206,8 +206,8 @@ def get_reward(self) -> float | None: except Exception as e: rollout.failure = rollout.failure or why(e) if rollout.misgrouped: - raise RuntimeError(f"{rollout.failure}. A task must offer no tool its toolset's first task doesn't, since TRL shows " - "the model a class's: their toolset misses what tells them apart") + raise RuntimeError(f"{rollout.failure}. TRL shows the model its class's tools, so a task's episode must " + "offer them: their toolset misses what tells them apart") if rollout.failure is not None: logger.warning("rollout left unscored: %s", rollout.failure) return None diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index 1d4cd3181..6514866a8 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -237,7 +237,7 @@ async def hanging(self, context): assert (output["env_metrics"][0]["failed"], output["env_metrics"][0]["attempts"]) == (1, 1) -def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_raises( +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_raises_and_a_match_plays( serve, tmp_path, tool_format, tokenizer): server = serve({"a": PLAYED}) rows = _rows(server.url, tmp_path, tool_format) @@ -247,8 +247,13 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ with pytest.raises(ValueError, match="their server configs reveal different tools"): _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) + written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) + pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) + output = _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) server.stop() + assert sum(output["rewards"][0]) == 1.0 + def test_an_episode_that_ends_without_failing_or_a_reward_raises(serve, tmp_path, tool_format, tokenizer): server = serve({"a": PLAYED}, reward=lambda scores: None) diff --git a/tst/unit/training/chat_test.py b/tst/unit/training/chat_test.py index 378fcc0ec..6c1fe1a59 100644 --- a/tst/unit/training/chat_test.py +++ b/tst/unit/training/chat_test.py @@ -145,11 +145,31 @@ def test_a_config_between_turns_is_compared_on_the_first_turns_tools_not_on_ever model = session(_call('{"item": "crate"}'), _says("Added a crate.")) played = _played(model, row) + stored = {**row, "metadata": {**row["metadata"], "first_turn_tools": json.dumps(row["metadata"]["first_turn_tools"])}} + played_from_its_json = _played(model, stored) server.stop() assert [tool["function"]["name"] for tool in row["metadata"]["tools"]] == ["add_item", "end_turn", "take_item"] assert row["metadata"]["first_turn_tools"] == ["add_item", "end_turn"] assert "agentenv_failure" not in played + assert "agentenv_failure" not in played_from_its_json + + +def test_a_task_revealing_at_its_first_turn_what_its_probed_task_reveals_later_is_refused(serve, session, tmp_path, + reveals): + server = serve({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES], + "b": [ITEMS, ALICE, reveals("take_item"), {**TURN, "depends_on": ["cfg"]}, AGAIN, REPLIES]}) + rows = _rows(server, tmp_path) + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + refused = _played(model, rows["b"]) + server.stop() + + assert [tool["function"]["name"] for tool in rows["b"]["metadata"]["tools"]] == [ + "add_item", "end_turn", "take_item"] + assert re.search(r"\['add_item', 'end_turn', 'take_item'\] at its first turn, not the \['add_item', 'end_turn'\]", + refused["agentenv_failure"]) @pytest.mark.parametrize("probed, later, refusal", [ diff --git a/tst/verl/loop_test.py b/tst/verl/loop_test.py index ebed08c86..386e29739 100644 --- a/tst/verl/loop_test.py +++ b/tst/verl/loop_test.py @@ -119,7 +119,7 @@ def test_a_rollout_the_env_fails_every_time_raises_so_verl_leaves_it_out(serve, broken.stop() -def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_is_refused( +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_is_refused_and_a_match_plays( serve, tmp_path, tokenizer): server = serve({"a": PLAYED}) row = _row(write_dataset(server.url, tmp_path)) @@ -127,8 +127,12 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ with pytest.raises(verl_loop.RolloutFailed, match="their server configs reveal different tools"): asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) + row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) + output = asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) server.stop() + assert output.reward_score == 1.0 + def test_a_rollout_isnt_played_again_once_its_time_is_up(serve, tmp_path, tokenizer, monkeypatch): server = serve({"a": PLAYED}) From 11ce9542eb67a93806f38f7d60c13f2e6b389583 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 02:48:44 -0500 Subject: [PATCH 3/6] wip r4 --- AGENTS.md | 8 ++++---- src/agent_env/training/skyrl/env.py | 3 ++- src/agent_env/training/trl.py | 5 +++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c6837f38e..6e55f15a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -182,14 +182,14 @@ Per trainer: - `python -m agent_env.training.nemo_gym SOURCE ROOT` writes a bundle's NeMo Gym resources server into a Gym root, `NEMO_GYM_EXTRA_ROOTS`'s or a Gym checkout's: `app.py`; a `pyproject.toml`, since uv applies the nearest one's `[tool.uv]` and a checkout's `exclude-dependencies` would drop `cryptography`; a config with `simple_agent` and the - `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools (and `first_turn_tools`). + `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools (and `first_turn_tools` when it applies server config). - `server.AgentEnvResourcesServer` is that server, run in the venv Gym builds: a row's seed starts its episode, a catch-all route calls its tools, and `verify` ends the turns left open with the model's last reply and returns the reward, masking a rollout the environment failed. With `reward_components` it also returns each score by step id, their sum the reward, for NeMo RL's GDPO. - `verl/` (`verl` extra, installed into verl's own environment): - `python -m agent_env.training.verl URL OUT` writes a server's tasks as verl's `.parquet` rows (the task's - prompt, `agent_name: agentenv`, the row's tools and `first_turn_tools` in `extra_info`) and `agent_loop.yaml`. It reads a server that asks + prompt, `agent_name: agentenv`, the row's tools and `first_turn_tools`, null without server config, in `extra_info`) and `agent_loop.yaml`. It reads a server that asks for a token with the one `--token-var` names, and with `--tokenizer` counts each prompt with its tools as verl renders it (`--chat-template-kwargs`), refusing rows over `--max-prompt-length` before writing. - `loop.AgentEnvAgentLoop` is verl's `ToolAgentLoop` with the row's tools: each call a call on the row's episode, the @@ -203,8 +203,8 @@ Per trainer: `max_tool_response_length`. - `skyrl/` (`skyrl` extra, installed into SkyRL's own environment): - `python -m agent_env.training.skyrl URL OUT --tokenizer MODEL` writes a server's tasks as SkyRL's `.parquet` - rows (`env_class: agentenv`, the episode's split, index, task id, tools and `first_turn_tools`, the server's - settings). Each row has its + rows (`env_class: agentenv`, the episode's split, index, task id, tools and `first_turn_tools` (null without + server config), the server's settings). Each row has its tools rendered into its system prompt by the model's chat template, checked to render as `tools=` would, since SkyRL's generator passes the template no tools; rows over `--max-prompt-length`, counted as SkyRL's filter and generator count them, are refused. diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index a8f23fec2..a5afc19cb 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -102,7 +102,8 @@ def _stepped(self, action: str) -> BaseTextEnvStepOutput: if rollout.misgrouped: self.close() raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools of its toolset's first " - "task, which its episode must offer") + "task: an episode must offer no tool beyond them, and with server config exactly " + "them at its first turn") raise _Failed(rollout.failure) if not rollout.done: return BaseTextEnvStepOutput(observations=answers, reward=0.0, done=False, metadata={}) diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index 8e46db7f9..51959211c 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -206,8 +206,9 @@ def get_reward(self) -> float | None: except Exception as e: rollout.failure = rollout.failure or why(e) if rollout.misgrouped: - raise RuntimeError(f"{rollout.failure}. TRL shows the model its class's tools, so a task's episode must " - "offer them: their toolset misses what tells them apart") + raise RuntimeError(f"{rollout.failure}. TRL shows the model its class's tools, so an episode must offer no " + "tool beyond them, and with server config exactly them at its first turn: their toolset " + "misses what tells them apart") if rollout.failure is not None: logger.warning("rollout left unscored: %s", rollout.failure) return None From 850601c4a8236b0a7994bc7f9708787ab8be7073 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 03:01:43 -0500 Subject: [PATCH 4/6] wip r5 --- src/agent_env/training/skyrl/env.py | 5 ++--- src/agent_env/training/trl.py | 5 ++--- tst/skyrl/generator_test.py | 4 +++- tst/unit/training/trl_test.py | 17 +++++++++++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index a5afc19cb..5e6e3062f 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -101,9 +101,8 @@ def _stepped(self, action: str) -> BaseTextEnvStepOutput: if rollout.failure is not None: if rollout.misgrouped: self.close() - raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools of its toolset's first " - "task: an episode must offer no tool beyond them, and with server config exactly " - "them at its first turn") + raise ValueError(f"{self._task()}: {rollout.failure}. Its row was written from its toolset's first " + "task, so its toolset misses what tells its tasks apart") raise _Failed(rollout.failure) if not rollout.done: return BaseTextEnvStepOutput(observations=answers, reward=0.0, done=False, metadata={}) diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index 51959211c..14693dbf2 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -206,9 +206,8 @@ def get_reward(self) -> float | None: except Exception as e: rollout.failure = rollout.failure or why(e) if rollout.misgrouped: - raise RuntimeError(f"{rollout.failure}. TRL shows the model its class's tools, so an episode must offer no " - "tool beyond them, and with server config exactly them at its first turn: their toolset " - "misses what tells them apart") + raise RuntimeError(f"{rollout.failure}. TRL shows every task of a toolset the tools found by playing its " + "first task, so this toolset misses what tells its tasks apart") if rollout.failure is not None: logger.warning("rollout left unscored: %s", rollout.failure) return None diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index 6514866a8..8980d3a63 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -245,7 +245,9 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) - with pytest.raises(ValueError, match="their server configs reveal different tools"): + with pytest.raises(ValueError, match="their server configs reveal different tools, so serve them in separate " + "bundles. Its row was written from its toolset's first task, so its toolset " + "misses what tells its tasks apart"): _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) diff --git a/tst/unit/training/trl_test.py b/tst/unit/training/trl_test.py index 05dcb14cb..aed767654 100644 --- a/tst/unit/training/trl_test.py +++ b/tst/unit/training/trl_test.py @@ -294,6 +294,23 @@ def test_a_rollout_whose_config_reveals_fewer_tools_than_its_classs_stops_traini assert environment.get_reward() == 1.0 +def test_a_rollout_revealing_at_its_first_turn_what_its_classs_first_task_reveals_later_stops_training( + served, reveals): + env = served({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES], + "b": [ITEMS, ALICE, reveals("take_item"), {**TURN, "depends_on": ["cfg"]}, AGAIN, REPLIES]}) + environment = env.factory() + + environment.reset(**env.dataset[1]) + with pytest.raises(RuntimeError) as stopped: + environment.get_reward() + + assert str(stopped.value) == ( + "task 'b' offers the tools ['add_item', 'end_turn', 'take_item'], not the ['add_item', 'end_turn'] its " + "toolset's first task does. TRL shows every task of a toolset the tools found by playing its first task, so " + "this toolset misses what tells its tasks apart") + + class _Slow(TaskStep): """Takes a second, as a verifier that grades the episode after its last turn does.""" From 113e3941e9a442a829f59bdb43b4ea55b5f91635 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 03:34:44 -0500 Subject: [PATCH 5/6] wip r6 --- src/agent_env/training/skyrl/env.py | 4 ++-- tst/skyrl/generator_test.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index 5e6e3062f..7cbab76f2 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -101,8 +101,8 @@ def _stepped(self, action: str) -> BaseTextEnvStepOutput: if rollout.failure is not None: if rollout.misgrouped: self.close() - raise ValueError(f"{self._task()}: {rollout.failure}. Its row was written from its toolset's first " - "task, so its toolset misses what tells its tasks apart") + raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools found by playing its " + "toolset's first task, so its toolset misses what tells its tasks apart") raise _Failed(rollout.failure) if not rollout.done: return BaseTextEnvStepOutput(observations=answers, reward=0.0, done=False, metadata={}) diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index 8980d3a63..e2867bb13 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -246,8 +246,8 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) with pytest.raises(ValueError, match="their server configs reveal different tools, so serve them in separate " - "bundles. Its row was written from its toolset's first task, so its toolset " - "misses what tells its tasks apart"): + "bundles. Its row lists the tools found by playing its toolset's first task, " + "so its toolset misses what tells its tasks apart"): _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) From d778fb383b6acdf3e05925be88a746a42cc7b403 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 03:46:58 -0500 Subject: [PATCH 6/6] wip r7 --- src/agent_env/training/_openenv.py | 4 ++-- tst/skyrl/generator_test.py | 2 +- tst/unit/training/chat_test.py | 2 +- tst/unit/training/trl_test.py | 3 ++- tst/verl/loop_test.py | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/agent_env/training/_openenv.py b/src/agent_env/training/_openenv.py index 2ac2f1726..6f5eb81e7 100644 --- a/src/agent_env/training/_openenv.py +++ b/src/agent_env/training/_openenv.py @@ -161,8 +161,8 @@ async def _begin(self, rollout: Rollout, split: str, index: int, expected: str | f"{sorted(self._shared.offered)} {self.offered_by}") if (exactly := self._shared.exactly) is not None and offered != exactly: raise Misgrouped(f"task {played!r} offers the tools {sorted(offered)} at its first turn, not the " - f"{sorted(exactly)} its toolset's first task does: their server configs reveal different " - "tools, so serve them in separate bundles") + f"{sorted(exactly)} its toolset's first task does there, so serve them in separate " + "bundles") async def _step(self, name: str, arguments: dict[str, Any]) -> str: async with self._lock: diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index e2867bb13..c4cf0d0c5 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -245,7 +245,7 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) - with pytest.raises(ValueError, match="their server configs reveal different tools, so serve them in separate " + with pytest.raises(ValueError, match="its toolset's first task does there, so serve them in separate " "bundles. Its row lists the tools found by playing its toolset's first task, " "so its toolset misses what tells its tasks apart"): _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) diff --git a/tst/unit/training/chat_test.py b/tst/unit/training/chat_test.py index 6c1fe1a59..71cc9b8d6 100644 --- a/tst/unit/training/chat_test.py +++ b/tst/unit/training/chat_test.py @@ -174,7 +174,7 @@ def test_a_task_revealing_at_its_first_turn_what_its_probed_task_reveals_later_i @pytest.mark.parametrize("probed, later, refusal", [ (["take_item"], [], r"\['add_item', 'end_turn'\] at its first turn, not the \['add_item', 'end_turn', " - r"'take_item'\] its toolset's first task does: their server configs reveal different tools"), + r"'take_item'\] its toolset's first task does there, so serve them in separate bundles$"), ([], ["take_item"], r"offers the tools \['add_item', 'end_turn', 'take_item'\], not the \['add_item', 'end_turn'\]"), ], ids=["fewer", "more"]) def test_a_task_whose_config_reveals_other_tools_than_its_probed_tasks_is_refused_and_an_older_row_plays( diff --git a/tst/unit/training/trl_test.py b/tst/unit/training/trl_test.py index aed767654..61c313506 100644 --- a/tst/unit/training/trl_test.py +++ b/tst/unit/training/trl_test.py @@ -288,7 +288,8 @@ def test_a_rollout_whose_config_reveals_fewer_tools_than_its_classs_stops_traini environment.add_item(item="crate") environment.end_turn(response="Added a crate.") if stops: - with pytest.raises(RuntimeError, match=r"their server configs reveal different tools"): + with pytest.raises(RuntimeError, match=r"its toolset's first task does there, so serve them in separate bundles\. " + r"TRL shows every task"): environment.get_reward() else: assert environment.get_reward() == 1.0 diff --git a/tst/verl/loop_test.py b/tst/verl/loop_test.py index 386e29739..096db8266 100644 --- a/tst/verl/loop_test.py +++ b/tst/verl/loop_test.py @@ -125,7 +125,7 @@ def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_ row = _row(write_dataset(server.url, tmp_path)) row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) - with pytest.raises(verl_loop.RolloutFailed, match="their server configs reveal different tools"): + with pytest.raises(verl_loop.RolloutFailed, match="its toolset's first task does there, so serve them in separate bundles"): asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) output = asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1))