diff --git a/AGENTS.md b/AGENTS.md index e03fa8703..6e55f15a9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -153,13 +153,15 @@ share. Shared: - `_openenv.py`: the session on a server `openenv/` makes that each helper plays an episode through (`Session`: a background start, tool calls, the turns left open ended until scored), with the Task API reads, the per-toolset probe, - `tasks_and_tools` (what the row writers read: the tasks, the split names checked, each toolset's tools) and an - in-process `Server`. + `tasks_and_tools` (what the row writers read: the tasks, the split names checked, each toolset's tools, and for a + toolset whose tasks apply server config the probed task's first-turn tools, which rows carry as `first_turn_tools` + and an episode's first turn must then offer exactly) and an in-process `Server`. - `_tools.py`: turns an env's tools into typed, documented Python methods from their JSON Schema, as transformers' `get_json_schema` reads them. - `_chat.py`, the rows and episode the Miles and slime helpers share: - `write_rows` and its `command` write a server's tasks as `.jsonl` rows of chat `messages` and `metadata` - (the episode's split, index, task id and tools, and the server it plays on with its deadlines and turn cap). + (the episode's split, index, task id and tools, `first_turn_tools` when its task applies server config, and the + server it plays on with its deadlines and turn cap). - `play` plays a row's episode, the model chatting at a trainer's OpenAI chat endpoint that records its turns, with the row's tools. It answers a call whose arguments aren't a JSON object as the model's error, and returns the reward, or flags an episode the env or the session failed, whose model missed the row's `model_timeout` or that ended @@ -180,14 +182,14 @@ Per trainer: - `python -m agent_env.training.nemo_gym SOURCE ROOT` writes a bundle's NeMo Gym resources server into a Gym root, `NEMO_GYM_EXTRA_ROOTS`'s or a Gym checkout's: `app.py`; a `pyproject.toml`, since uv applies the nearest one's `[tool.uv]` and a checkout's `exclude-dependencies` would drop `cryptography`; a config with `simple_agent` and the - `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools. + `legacy_agent` environment server; and `data/.jsonl` rows with each task's tools (and `first_turn_tools` when it applies server config). - `server.AgentEnvResourcesServer` is that server, run in the venv Gym builds: a row's seed starts its episode, a catch-all route calls its tools, and `verify` ends the turns left open with the model's last reply and returns the reward, masking a rollout the environment failed. With `reward_components` it also returns each score by step id, their sum the reward, for NeMo RL's GDPO. - `verl/` (`verl` extra, installed into verl's own environment): - `python -m agent_env.training.verl URL OUT` writes a server's tasks as verl's `.parquet` rows (the task's - prompt, `agent_name: agentenv`, the row's tools in `extra_info`) and `agent_loop.yaml`. It reads a server that asks + prompt, `agent_name: agentenv`, the row's tools and `first_turn_tools`, null without server config, in `extra_info`) and `agent_loop.yaml`. It reads a server that asks for a token with the one `--token-var` names, and with `--tokenizer` counts each prompt with its tools as verl renders it (`--chat-template-kwargs`), refusing rows over `--max-prompt-length` before writing. - `loop.AgentEnvAgentLoop` is verl's `ToolAgentLoop` with the row's tools: each call a call on the row's episode, the @@ -201,7 +203,8 @@ Per trainer: `max_tool_response_length`. - `skyrl/` (`skyrl` extra, installed into SkyRL's own environment): - `python -m agent_env.training.skyrl URL OUT --tokenizer MODEL` writes a server's tasks as SkyRL's `.parquet` - rows (`env_class: agentenv`, the episode's split, index, task id and tools, the server's settings). Each row has its + rows (`env_class: agentenv`, the episode's split, index, task id, tools and `first_turn_tools` (null without + server config), the server's settings). Each row has its tools rendered into its system prompt by the model's chat template, checked to render as `tools=` would, since SkyRL's generator passes the template no tools; rows over `--max-prompt-length`, counted as SkyRL's filter and generator count them, are refused. diff --git a/src/agent_env/training/_chat.py b/src/agent_env/training/_chat.py index 0d71d554c..4ccd76dcb 100644 --- a/src/agent_env/training/_chat.py +++ b/src/agent_env/training/_chat.py @@ -14,7 +14,17 @@ import click import httpx -from agent_env.training._openenv import Session, Shared, function_tool, key, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + Session, + Shared, + exactly, + function_tool, + key, + messages, + tasks_and_tools, + token_in, +) @click.command() @@ -50,8 +60,8 @@ def write_rows(url: str, out: str | Path, *, tool_timeout: float = 60.0, reset_t rollout may take, and the environment variable holding the token a server given one asks for, read in each of the trainer's rollout processes.""" url = url.rstrip("/") - _, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + _, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", + timeout=reset_timeout, token=token_in(token_var)) tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} server = {"url": url, "tool_timeout": tool_timeout, "reset_timeout": reset_timeout, "model_timeout": model_timeout, "max_turns": max_turns, **({"token_var": token_var} if token_var else {})} @@ -61,6 +71,7 @@ def write_rows(url: str, out: str | Path, *, tool_timeout: float = 60.0, reset_t (out / f"{split}.jsonl").write_text("".join(json.dumps({ "messages": messages(task["system_prompt"], task["prompt"]), "metadata": {"split": split, "index": task["index"], "task_id": task["id"], "tools": tools[key(task)], + **({FIRST_TURN_TOOLS: firsts[key(task)]} if key(task) in firsts else {}), "agentenv": server}, }) + "\n" for task in described)) return out @@ -89,7 +100,8 @@ async def play(base_url: str, prompt: list[dict[str, Any]], request_kwargs: dict server, tools = row["agentenv"], row["tools"] episode = _Episode(Shared(server["url"], frozenset(tool["function"]["name"] for tool in tools), server["tool_timeout"], server["reset_timeout"], patience=server["reset_timeout"], - waits_for_server=True, token=token_in(server.get("token_var")))) + waits_for_server=True, token=token_in(server.get("token_var")), + exactly=exactly(row.get(FIRST_TURN_TOOLS)))) counts = {"agentenv_tool_calls": 0, "agentenv_invalid_calls": 0} try: await episode._start(row["split"], int(row["index"]), row["task_id"]) diff --git a/src/agent_env/training/_openenv.py b/src/agent_env/training/_openenv.py index 8b96ae404..6f5eb81e7 100644 --- a/src/agent_env/training/_openenv.py +++ b/src/agent_env/training/_openenv.py @@ -28,6 +28,7 @@ FAILED = "The environment failed to answer, so this episode won't be scored." ENDED = "The episode has ended." END_TURN = END_TURN_TOOL.name +FIRST_TURN_TOOLS = "first_turn_tools" CONNECT_SECONDS = 60.0 CLOSE_SECONDS = 300.0 #: How much longer than a call's deadline the client waits for the server to end the call, before dropping the session. @@ -64,8 +65,9 @@ class NamedToolset: class Shared: """What a session needs of its server: where it is, the tools the model is shown, which an episode's first turn must offer no more than, the deadlines, how long to wait for room on the server, whether to wait for a server that - refuses connections, as one restarting does, as long as a reset may take, and the token it asks for. The key makes - it equal to its pickled copies only.""" + refuses connections, as one restarting does, as long as a reset may take, the token it asks for, and for a toolset + whose tasks apply server config, the tools its probed task offered at its first turn, which an episode's first turn + must offer exactly. The key makes it equal to its pickled copies only.""" url: str offered: frozenset[str] @@ -74,6 +76,7 @@ class Shared: patience: float = REOPEN_PATIENCE waits_for_server: bool = False token: str | None = field(default=None, repr=False) + exactly: frozenset[str] | None = None key: str = field(default_factory=lambda: uuid.uuid4().hex) @@ -90,8 +93,8 @@ def __init__(self, url: str, limit: str): class Misgrouped(Failed): - """An episode offering a tool the model isn't shown: a toolset that misses what tells tasks apart, which no rollout - fixes.""" + """An episode offering a tool the model isn't shown, or with server config offering other first-turn tools than its + toolset's probed task: a toolset that misses what tells tasks apart, which no rollout fixes.""" @dataclass @@ -156,6 +159,10 @@ async def _begin(self, rollout: Rollout, split: str, index: int, expected: str | if not (offered := frozenset(tool["name"] for tool in listed.observation["tools"])) <= self._shared.offered: raise Misgrouped(f"task {played!r} offers the tools {sorted(offered)}, not the " f"{sorted(self._shared.offered)} {self.offered_by}") + if (exactly := self._shared.exactly) is not None and offered != exactly: + raise Misgrouped(f"task {played!r} offers the tools {sorted(offered)} at its first turn, not the " + f"{sorted(exactly)} its toolset's first task does there, so serve them in separate " + "bundles") async def _step(self, name: str, arguments: dict[str, Any]) -> str: async with self._lock: @@ -391,16 +398,31 @@ def read_tasks(url: str, *, local: str, token: str | None = None) -> tuple[str, def tasks_and_tools(url: str, *, local: str, timeout: float, token: str | None = None - ) -> tuple[str, dict[str, list[dict[str, Any]]], dict[str, tuple[dict[str, Any], ...]]]: + ) -> tuple[str, dict[str, list[dict[str, Any]]], dict[str, tuple[dict[str, Any], ...]], + dict[str, list[str]]]: """What a writer of a split's rows reads: the server's env name, each split's tasks, refusing a split whose name - can't name a data file, and every tool each toolset's turns offer, by the toolset's key, found by playing its - first task through (``probe``), so reading deploys those.""" + can't name a data file, every tool each toolset's turns offer, by the toolset's key, found by playing its first + task through (``probe``), so reading deploys those, and for each toolset whose tasks apply server config, the + tools that task offered at its first turn, which a row carries as ``FIRST_TURN_TOOLS``.""" name, tasks = read_tasks(url, local=local, token=token) if unfit := [split for split in tasks if not _FILE.fullmatch(split)]: raise ValueError(f"the server's splits {unfit} can't name a data file: a split's name must be letters, " "digits, '.', '_' and '-', not starting with '.'") - return name, tasks, {each: asyncio.run(probe(url, toolset, timeout, token=token))[1] - for each, toolset in toolsets(name, tasks).items()} + probed = {each: asyncio.run(probe(url, toolset, timeout, token=token)) + for each, toolset in toolsets(name, tasks).items()} + return (name, tasks, {each: tools for each, (_, tools) in probed.items()}, + {each: sorted(first) for each, (first, _) in probed.items() if applies_config(each)}) + + +def applies_config(each: str) -> bool: + toolset: Toolset = json.loads(each) + return toolset.get("config", False) + + +def exactly(names: list[str] | str | None) -> frozenset[str] | None: + if isinstance(names, str): + names = json.loads(names) + return None if names is None else frozenset(names) def toolsets(name: str, tasks: dict[str, list[dict[str, Any]]]) -> dict[str, NamedToolset]: diff --git a/src/agent_env/training/nemo_gym/prepare.py b/src/agent_env/training/nemo_gym/prepare.py index 2b8f404f4..482486d66 100644 --- a/src/agent_env/training/nemo_gym/prepare.py +++ b/src/agent_env/training/nemo_gym/prepare.py @@ -12,7 +12,7 @@ import yaml from agent_env.openenv import build_app -from agent_env.training._openenv import Server, key, messages, tasks_and_tools +from agent_env.training._openenv import FIRST_TURN_TOOLS, Server, key, messages, tasks_and_tools #: The routes a resources server serves itself, which a tool's route can't take. GYM_ROUTES = frozenset({"seed_session", "verify", "close_session", "aggregate_metrics", "reverify_mode"}) @@ -67,8 +67,8 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non served = None if url else Server(build_app(source, reward=summed if reward_components else reward, sandbox=sandbox, max_concurrent_envs=1)) try: - env_name, tasks, probed = tasks_and_tools(url or served.url, local="name the bundle's folder instead of its " - "server", timeout=reset_timeout) + env_name, tasks, probed, firsts = tasks_and_tools(url or served.url, local="name the bundle's folder instead " + "of its server", timeout=reset_timeout) tools = {each: [_function(tool) for tool in found] for each, found in probed.items()} finally: if served is not None: @@ -96,6 +96,7 @@ def write_server(source: str | Path, root: str | Path, *, name: str | None = Non "responses_create_params": {"input": messages(task["system_prompt"], task["prompt"]), "tools": tools[key(task)]}, "agent_ref": agent, "split": split, "index": task["index"], "task_id": task["id"], + **({FIRST_TURN_TOOLS: firsts[key(task)]} if key(task) in firsts else {}), }) + "\n" for task in described)) return server diff --git a/src/agent_env/training/nemo_gym/server.py b/src/agent_env/training/nemo_gym/server.py index e245675fc..e688e9159 100644 --- a/src/agent_env/training/nemo_gym/server.py +++ b/src/agent_env/training/nemo_gym/server.py @@ -41,7 +41,7 @@ "Python 3.13.14 or later as NeMo Gym is") from exc from agent_env.openenv import build_app -from agent_env.training._openenv import STOP_SECONDS, Server, Session, Shared +from agent_env.training._openenv import STOP_SECONDS, Server, Session, Shared, exactly from agent_env.training.nemo_gym.prepare import REWARD_AND_COMPONENTS, summed logger = logging.getLogger(__name__) @@ -89,7 +89,8 @@ def _checked(self) -> "AgentEnvResourcesServerConfig": class AgentEnvSeedSessionRequest(BaseSeedSessionRequest): - """A row: the tools it shows the model, and the task it plays.""" + """A row: the tools it shows the model, the task it plays, and for a task that applies server config, the tools its + first turn must offer exactly.""" model_config = ConfigDict(extra="allow") @@ -97,6 +98,7 @@ class AgentEnvSeedSessionRequest(BaseSeedSessionRequest): split: str index: int task_id: str | None = None + first_turn_tools: list[str] | None = None class AgentEnvVerifyRequest(BaseVerifyRequest): @@ -139,7 +141,7 @@ async def seed_session(self, request: Request, body: AgentEnvSeedSessionRequest) session_id = request.session[SESSION_ID_KEY] offered = frozenset(tool["name"] for tool in body.responses_create_params.get("tools") or ()) session = _Session(Shared(self._url, offered, self.config.tool_timeout, self.config.reset_timeout, - patience=self.config.reset_timeout)) + patience=self.config.reset_timeout, exactly=exactly(body.first_turn_tools))) if (previous := self._sessions.pop(session_id, None)) is not None: await previous.close() self._sessions[session_id] = session diff --git a/src/agent_env/training/skyrl/env.py b/src/agent_env/training/skyrl/env.py index 43b934439..7cbab76f2 100644 --- a/src/agent_env/training/skyrl/env.py +++ b/src/agent_env/training/skyrl/env.py @@ -16,7 +16,17 @@ raise ImportError("agent_env.training.skyrl.env runs in SkyRL's trainer: install 'agentenv-framework[skyrl]' into " "SkyRL's environment") from exc -from agent_env.training._openenv import CLOSE_SECONDS, GRACE_SECONDS, Rollout, Session, Shared, on_session_loop, token_in +from agent_env.training._openenv import ( + CLOSE_SECONDS, + FIRST_TURN_TOOLS, + GRACE_SECONDS, + Rollout, + Session, + Shared, + exactly, + on_session_loop, + token_in, +) from agent_env.training.skyrl.calls import Call, Unparsed, parse logger = logging.getLogger(__name__) @@ -37,7 +47,8 @@ def __init__(self, env_config: Any = None, extras: dict[str, Any] | None = None) + 2 * GRACE_SECONDS + CLOSE_SECONDS self._session = _Session(Shared(server["url"], frozenset(self._tools), server["tool_timeout"], server["reset_timeout"], patience=server["reset_timeout"], - waits_for_server=True, token=token_in(server.get("token_var")))) + waits_for_server=True, token=token_in(server.get("token_var")), + exactly=exactly(self._episode.get(FIRST_TURN_TOOLS)))) self._scores: dict[str, float] = {} self._run = self._unparsed = 0 self._failed = False @@ -90,8 +101,8 @@ def _stepped(self, action: str) -> BaseTextEnvStepOutput: if rollout.failure is not None: if rollout.misgrouped: self.close() - raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools of its toolset's first " - "task, so a task must offer no tool that one doesn't") + raise ValueError(f"{self._task()}: {rollout.failure}. Its row lists the tools found by playing its " + "toolset's first task, so its toolset misses what tells its tasks apart") raise _Failed(rollout.failure) if not rollout.done: return BaseTextEnvStepOutput(observations=answers, reward=0.0, done=False, metadata={}) diff --git a/src/agent_env/training/skyrl/prepare.py b/src/agent_env/training/skyrl/prepare.py index a1fe2ca43..e7d36357f 100644 --- a/src/agent_env/training/skyrl/prepare.py +++ b/src/agent_env/training/skyrl/prepare.py @@ -13,7 +13,15 @@ import pyarrow.parquet as pq from transformers import AutoTokenizer -from agent_env.training._openenv import function_tool, key, longest_prompt, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + function_tool, + key, + longest_prompt, + messages, + tasks_and_tools, + token_in, +) from agent_env.training.skyrl.calls import FORMATS #: The ``env_class`` every row plays under, and the id the env is registered at. @@ -24,7 +32,7 @@ ("env_class", pa.string()), ("data_source", pa.string()), ("episode", pa.struct([("split", pa.string()), ("index", pa.int64()), ("task_id", pa.string()), - ("tools", pa.string())])), + ("tools", pa.string()), (FIRST_TURN_TOOLS, pa.string())])), ("server", pa.struct([("url", pa.string()), ("tool_timeout", pa.float64()), ("reset_timeout", pa.float64()), ("attempts", pa.int64()), ("tool_format", pa.string()), ("token_var", pa.string())])), ]) @@ -52,8 +60,8 @@ def write_dataset(url: str, out: str | Path, *, tokenizer: str, tool_format: str if tool_format not in FORMATS: raise ValueError(f"tool_format must be one of {', '.join(FORMATS)}, not {tool_format!r}") url = url.rstrip("/") - env_name, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + env_name, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app " + "first", timeout=reset_timeout, token=token_in(token_var)) template = AutoTokenizer.from_pretrained(tokenizer) kwargs = chat_template_kwargs or {} tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} @@ -64,7 +72,7 @@ def write_dataset(url: str, out: str | Path, *, tokenizer: str, tool_format: str "env_class": AGENT, "data_source": f"{AGENT}/{env_name}", "episode": {"split": split, "index": task["index"], "task_id": task["id"], - "tools": json.dumps(tools[key(task)])}, + "tools": json.dumps(tools[key(task)]), FIRST_TURN_TOOLS: json.dumps(firsts[key(task)]) if key(task) in firsts else None}, "server": server, } for task in described] for split, described in tasks.items()} longest = _longest(rows, template, max_prompt_length, kwargs) diff --git a/src/agent_env/training/trl.py b/src/agent_env/training/trl.py index fc1de3f8b..14693dbf2 100644 --- a/src/agent_env/training/trl.py +++ b/src/agent_env/training/trl.py @@ -39,6 +39,7 @@ Server, Session, Shared, + applies_config, key, messages, on_session_loop, @@ -153,12 +154,12 @@ def close(self) -> None: _stop(self._server) async def _environments(self) -> list[type[_Environment]]: - return [await self._environment(toolset) for toolset in self._toolsets.values()] + return [await self._environment(toolset, applies_config(each)) for each, toolset in self._toolsets.items()] - async def _environment(self, toolset: NamedToolset) -> type[_Environment]: + async def _environment(self, toolset: NamedToolset, config: bool) -> type[_Environment]: first, tools = await probe(self.url, toolset, self._reset_timeout) shared = Shared(url=self.url, offered=first, tool_timeout=self._tool_timeout, reset_timeout=self._reset_timeout, - waits_for_server=True) + waits_for_server=True, exactly=first if config else None) self._shared.append(shared) return _environment_class(toolset.name, tools, shared) @@ -195,8 +196,8 @@ def reset(self, *, split: str, index: int, task_id: str | None = None, **_: Any) on_session_loop(self._start(split, index, task_id), CONNECT_SECONDS + CLOSE_SECONDS) def get_reward(self) -> float | None: - """The episode's reward, or ``None`` when it ended in an error. Raises when the episode's first turn offered a - tool its toolset's first task doesn't, which no rollout fixes.""" + """The episode's reward, or ``None`` when it ended in an error. Raises when the episode is misgrouped + (``Misgrouped``), which no rollout fixes.""" rollout, self._rollout = self._rollout, None if rollout is None: return None @@ -205,8 +206,8 @@ def get_reward(self) -> float | None: except Exception as e: rollout.failure = rollout.failure or why(e) if rollout.misgrouped: - raise RuntimeError(f"{rollout.failure}. A task must offer no tool its toolset's first task doesn't, since TRL shows " - "the model a class's: their toolset misses what tells them apart") + raise RuntimeError(f"{rollout.failure}. TRL shows every task of a toolset the tools found by playing its " + "first task, so this toolset misses what tells its tasks apart") if rollout.failure is not None: logger.warning("rollout left unscored: %s", rollout.failure) return None diff --git a/src/agent_env/training/verl/loop.py b/src/agent_env/training/verl/loop.py index d7d69109c..b2088000c 100644 --- a/src/agent_env/training/verl/loop.py +++ b/src/agent_env/training/verl/loop.py @@ -23,7 +23,7 @@ raise ImportError("agent_env.training.verl.loop runs in verl's agent loop workers: install " "'agentenv-framework[verl]' into verl's environment") from exc -from agent_env.training._openenv import Rollout, Session, Shared, attempt_numbers, token_in +from agent_env.training._openenv import FIRST_TURN_TOOLS, Rollout, Session, Shared, attempt_numbers, exactly, token_in logger = logging.getLogger(__name__) #: The tools whose answers verl was seen to cut, each warned of once a process. @@ -82,7 +82,8 @@ async def run(self, sampling_params: dict[str, Any], **kwargs: Any) -> AgentLoop row = self._row = kwargs["extra_info"] schemas = json.loads(row["tools"]) shared = Shared(self._url, frozenset(schema["function"]["name"] for schema in schemas), self._tool_timeout, - self._reset_timeout, patience=self._reset_timeout, waits_for_server=True, token=self._token) + self._reset_timeout, patience=self._reset_timeout, waits_for_server=True, token=self._token, + exactly=exactly(row.get(FIRST_TURN_TOOLS))) for attempt in attempt_numbers(self._attempts, self._reset_timeout): session = _Session(shared) try: diff --git a/src/agent_env/training/verl/prepare.py b/src/agent_env/training/verl/prepare.py index ec9737f45..a99c3ac1c 100644 --- a/src/agent_env/training/verl/prepare.py +++ b/src/agent_env/training/verl/prepare.py @@ -14,7 +14,15 @@ import yaml from transformers import AutoTokenizer -from agent_env.training._openenv import function_tool, key, longest_prompt, messages, tasks_and_tools, token_in +from agent_env.training._openenv import ( + FIRST_TURN_TOOLS, + function_tool, + key, + longest_prompt, + messages, + tasks_and_tools, + token_in, +) #: The ``agent_name`` every row plays under, and the name ``agent_loop.yaml`` registers the loop at. AGENT = "agentenv" @@ -25,7 +33,7 @@ ("prompt", pa.list_(pa.struct([("role", pa.string()), ("content", pa.string())]))), ("agent_name", pa.string()), ("extra_info", pa.struct([("split", pa.string()), ("index", pa.int64()), ("task_id", pa.string()), - ("tools", pa.string())])), + ("tools", pa.string()), (FIRST_TURN_TOOLS, pa.string())])), ]) @@ -58,15 +66,15 @@ def write_dataset(url: str, out: str | Path, *, tool_timeout: float = 60.0, rese if tokenizer is None and (max_prompt_length is not None or chat_template_kwargs): raise ValueError("max_prompt_length and chat_template_kwargs count tokens, so they take the model's tokenizer") url = url.rstrip("/") - env_name, tasks, probed = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app first", - timeout=reset_timeout, token=token_in(token_var)) + env_name, tasks, probed, firsts = tasks_and_tools(url, local="serve the bundle with agent_env.openenv.build_app " + "first", timeout=reset_timeout, token=token_in(token_var)) tools = {each: [function_tool(tool) for tool in found] for each, found in probed.items()} rows = {split: [{ "data_source": f"agentenv/{env_name}", "prompt": messages(task["system_prompt"], task["prompt"]), "agent_name": AGENT, "extra_info": {"split": split, "index": task["index"], "task_id": task["id"], - "tools": json.dumps(tools[key(task)])}, + "tools": json.dumps(tools[key(task)]), FIRST_TURN_TOOLS: json.dumps(firsts[key(task)]) if key(task) in firsts else None}, } for task in described] for split, described in tasks.items()} longest = None if tokenizer is None else _longest(rows, AutoTokenizer.from_pretrained(tokenizer), max_prompt_length, chat_template_kwargs or {}) diff --git a/tst/nemo_gym/server_test.py b/tst/nemo_gym/server_test.py index ecc588a42..862144ca0 100644 --- a/tst/nemo_gym/server_test.py +++ b/tst/nemo_gym/server_test.py @@ -70,6 +70,8 @@ def test_end_turn_answers_with_the_next_turns_prompt_and_a_turn_left_open_ends_e ({"a": PLAYED}, {"task_id": "b"}, "the episode plays task 'a', not the row's 'b'"), ({"a": PLAYED}, {"tools": [FUNCTION_END_TURN]}, "task 'a' offers the tools ['add_item', 'end_turn'], not the ['end_turn'] the row shows the model"), + ({"a": PLAYED}, {"first_turn_tools": ["add_item", "end_turn", "take_item"]}, + "at its first turn, not the ['add_item', 'end_turn', 'take_item'] its toolset's first task does"), ]) def test_a_rollout_the_environment_fails_or_that_isnt_its_rows_is_masked(serve, tasks, row, why): server = serve(tasks) @@ -264,9 +266,9 @@ def _app(**config): ).setup_webserver() -def _row(task_id="a", tools=(ADD_ITEM, FUNCTION_END_TURN)): +def _row(task_id="a", tools=(ADD_ITEM, FUNCTION_END_TURN), **extra): return {"responses_create_params": {"input": [{"role": "user", "content": "Add a crate."}], "tools": list(tools)}, - "split": "all", "index": 0, "task_id": task_id} + "split": "all", "index": 0, "task_id": task_id, **extra} def _verify(client, row, reply): diff --git a/tst/skyrl/generator_test.py b/tst/skyrl/generator_test.py index ec34f855e..c4cf0d0c5 100644 --- a/tst/skyrl/generator_test.py +++ b/tst/skyrl/generator_test.py @@ -5,11 +5,12 @@ import json from types import SimpleNamespace +import pyarrow as pa import pyarrow.parquet as pq import pytest from transformers import AutoTokenizer -from agent_env.training.skyrl.prepare import token_vars, write_dataset +from agent_env.training.skyrl.prepare import ROW, token_vars, write_dataset from tst.unit.task.episode import run_test from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN from tst.unit.training._support import PLAYED @@ -236,6 +237,26 @@ async def hanging(self, context): assert (output["env_metrics"][0]["failed"], output["env_metrics"][0]["attempts"]) == (1, 1) +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_raises_and_a_match_plays( + serve, tmp_path, tool_format, tokenizer): + server = serve({"a": PLAYED}) + rows = _rows(server.url, tmp_path, tool_format) + written = pq.read_table(rows).to_pylist() + written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) + pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) + + with pytest.raises(ValueError, match="its toolset's first task does there, so serve them in separate " + "bundles. Its row lists the tools found by playing its toolset's first task, " + "so its toolset misses what tells its tasks apart"): + _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) + written[0]["episode"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) + pq.write_table(pa.Table.from_pylist(written, schema=ROW), rows) + output = _generate(rows, _Engine(tokenizer, adds_a_crate(tool_format)), tokenizer) + server.stop() + + assert sum(output["rewards"][0]) == 1.0 + + def test_an_episode_that_ends_without_failing_or_a_reward_raises(serve, tmp_path, tool_format, tokenizer): server = serve({"a": PLAYED}, reward=lambda scores: None) rows = _rows(server.url, tmp_path, tool_format) diff --git a/tst/unit/training/_support.py b/tst/unit/training/_support.py index 12ae529c8..cdc57ae6d 100644 --- a/tst/unit/training/_support.py +++ b/tst/unit/training/_support.py @@ -4,12 +4,14 @@ import pytest +from agent_env.config.runtime import Config from agent_env.openenv.app import _app from agent_env.openenv.reward import _score_at from agent_env.openenv.environment import Served, ServedTask from agent_env.task import EpisodeSpec, Task +from agent_env.task_step.task_step import TaskStep from agent_env.training._openenv import Server -from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN +from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN, _Env #: A task the model plays by adding a crate and saying so, which its verifier scores 1. PLAYED = [ITEMS, ALICE, TURN, REPLIES] @@ -41,3 +43,59 @@ def served(tasks, splits=None, max_concurrent_envs=8, name="dock", reward=_score yield served for server in servers: server.stop() + + +class _Config(TaskStep): + """Stands in for ``apply_server_config`` on ``env_id``, whose reply reveals the tools ``reveal``.""" + + type = "apply_server_config" + entity_refs = () + revealed: list[str] = [] + applied = 0 + + def __init__(self, id, version, env_id="items", reveal=(), depends_on=None, fail_task_on_error=True): + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id, self.reveal = env_id, list(reveal) + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data), env_id=data.get("env_id", "items"), reveal=data.get("reveal", ())) + + async def execute(self, context): + _Config.revealed[:] = self.reveal + _Config.applied += 1 + return context + + +class _Offered: + def __iter__(self): + return iter(["add_item", *_Config.revealed]) + + +@pytest.fixture +def reveals(monkeypatch, tools): + """A config step whose reply reveals ``reveal``, given ``args``; each episode starts with nothing revealed, and + ``reveals.applied()`` counts the config steps run.""" + registry = Config().task_step_registry() + monkeypatch.setattr(Config, "task_step_registry", lambda self: {**registry, _Config.type: _Config}) + deploy = _Env.execute + + async def deploying(self, context): + _Config.revealed.clear() + return await deploy(self, context) + + monkeypatch.setattr(_Env, "execute", deploying) + monkeypatch.setattr(_Config, "revealed", []) + monkeypatch.setattr(_Config, "applied", 0) + tools().offered = _Offered() + + def config(*reveal, **args): + return {"id": "cfg", "type": _Config.type, "env_id": "items", "reveal": list(reveal), + "directives": [{"service": "items", "uri": "urn:example:set-errors/v1", "args": args}]} + + config.applied = lambda: _Config.applied + return config + + +def revealing(config, *reveal, **args): + return [ITEMS, ALICE, config(*reveal, **args), TURN, REPLIES] diff --git a/tst/unit/training/chat_test.py b/tst/unit/training/chat_test.py index ea9475be5..71cc9b8d6 100644 --- a/tst/unit/training/chat_test.py +++ b/tst/unit/training/chat_test.py @@ -5,6 +5,7 @@ import asyncio import json +import re from types import SimpleNamespace import pytest @@ -19,8 +20,9 @@ from agent_env.training.miles.__main__ import main as miles_main from agent_env.training._chat import keep_scored_groups, play, reward, write_rows from agent_env.training.slime.__main__ import main as slime_main +from tst.unit.openenv.environment_test import AGAIN from tst.unit.task.episode.run_test import ALICE, ITEMS, REPLIES, TURN -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -107,6 +109,90 @@ def test_the_agent_plays_a_rows_task_through_the_session_and_returns_its_reward( assert second["messages"][-1]["content"] +def _rows(server, tmp_path): + folder = write_rows(server.url, tmp_path / "rows") + return {row["metadata"]["task_id"]: row for row in map(json.loads, (folder / "all.jsonl").read_text().splitlines())} + + +def _played(model, row): + return asyncio.run(play(model.base_url, row["messages"], SAMPLING, row["metadata"])) + + +@pytest.mark.parametrize("reveal", [("take_item",), ()], ids=["reveal-the-same", "reveal-nothing"]) +def test_tasks_whose_configs_reveal_the_same_tools_share_one_probe_whatever_their_args_and_play( + serve, session, tmp_path, reveals, reveal): + server = serve({"a": revealing(reveals, *reveal, rate=0.1), "b": revealing(reveals, *reveal, rate=0.5), + "plain": PLAYED}) + rows = _rows(server, tmp_path) + probes = reveals.applied() + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + played = _played(model, rows["b"]) + server.stop() + + assert probes == 1 + assert rows["a"]["metadata"]["first_turn_tools"] == rows["b"]["metadata"]["first_turn_tools"] == sorted( + ["add_item", "end_turn", *reveal]) + assert "first_turn_tools" not in rows["plain"]["metadata"] + assert played["reward"] == 1.0 + + +def test_a_config_between_turns_is_compared_on_the_first_turns_tools_not_on_every_turns(serve, session, tmp_path, + reveals): + server = serve({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES]}) + row = _rows(server, tmp_path)["a"] + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + played = _played(model, row) + stored = {**row, "metadata": {**row["metadata"], "first_turn_tools": json.dumps(row["metadata"]["first_turn_tools"])}} + played_from_its_json = _played(model, stored) + server.stop() + + assert [tool["function"]["name"] for tool in row["metadata"]["tools"]] == ["add_item", "end_turn", "take_item"] + assert row["metadata"]["first_turn_tools"] == ["add_item", "end_turn"] + assert "agentenv_failure" not in played + assert "agentenv_failure" not in played_from_its_json + + +def test_a_task_revealing_at_its_first_turn_what_its_probed_task_reveals_later_is_refused(serve, session, tmp_path, + reveals): + server = serve({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES], + "b": [ITEMS, ALICE, reveals("take_item"), {**TURN, "depends_on": ["cfg"]}, AGAIN, REPLIES]}) + rows = _rows(server, tmp_path) + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + refused = _played(model, rows["b"]) + server.stop() + + assert [tool["function"]["name"] for tool in rows["b"]["metadata"]["tools"]] == [ + "add_item", "end_turn", "take_item"] + assert re.search(r"\['add_item', 'end_turn', 'take_item'\] at its first turn, not the \['add_item', 'end_turn'\]", + refused["agentenv_failure"]) + + +@pytest.mark.parametrize("probed, later, refusal", [ + (["take_item"], [], r"\['add_item', 'end_turn'\] at its first turn, not the \['add_item', 'end_turn', " + r"'take_item'\] its toolset's first task does there, so serve them in separate bundles$"), + ([], ["take_item"], r"offers the tools \['add_item', 'end_turn', 'take_item'\], not the \['add_item', 'end_turn'\]"), +], ids=["fewer", "more"]) +def test_a_task_whose_config_reveals_other_tools_than_its_probed_tasks_is_refused_and_an_older_row_plays( + serve, session, tmp_path, reveals, probed, later, refusal): + server = serve({"a": revealing(reveals, *probed), "b": revealing(reveals, *later)}) + rows = _rows(server, tmp_path) + model = session(_call('{"item": "crate"}'), _says("Added a crate.")) + + refused = _played(model, rows["b"]) + older = {**rows["b"], "metadata": {k: v for k, v in rows["b"]["metadata"].items() if k != "first_turn_tools"}} + played = _played(model, older) + server.stop() + + assert refused["reward"] == 0.0 + assert re.search(refusal, refused["agentenv_failure"]) + assert ("agentenv_failure" in played) == (later != []) + + def test_a_call_whose_arguments_arent_an_object_is_answered_and_the_episode_goes_on(serve, session, tmp_path): server = serve({"a": PLAYED}) row = _row(server, tmp_path) diff --git a/tst/unit/training/conftest.py b/tst/unit/training/conftest.py index 8183c3c0d..9779445f3 100644 --- a/tst/unit/training/conftest.py +++ b/tst/unit/training/conftest.py @@ -4,4 +4,4 @@ from tst.unit.openenv.environment_test import tools # noqa: F401 from tst.unit.task.episode.conftest import _disable_network # noqa: F401 from tst.unit.task.episode.run_test import teardowns # noqa: F401 -from tst.unit.training._support import serve # noqa: F401 +from tst.unit.training._support import reveals, serve # noqa: F401 diff --git a/tst/unit/training/nemo_gym_test.py b/tst/unit/training/nemo_gym_test.py index ed31c0ad1..af0be8374 100644 --- a/tst/unit/training/nemo_gym_test.py +++ b/tst/unit/training/nemo_gym_test.py @@ -23,7 +23,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing STYLE = {**REPLIES, "id": "style", "verifier_id": "style"} ADD_ITEM = {"type": "function", "name": "add_item", "description": "add_item.", "parameters": {"type": "object"}, @@ -84,6 +84,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + folder = write_server(server.url, tmp_path, name="dock") + server.stop() + + assert [row.get("first_turn_tools") for row in _rows(folder / "data" / "all.jsonl")] == [ + ["add_item", "end_turn", "take_item"], None] + + def test_a_split_whose_name_isnt_a_file_name_is_refused_before_anything_is_written(serve, tmp_path): server = serve({"a": PLAYED}, {"../escape": ["a"]}) diff --git a/tst/unit/training/skyrl_test.py b/tst/unit/training/skyrl_test.py index 2e7a30fe2..ce8e5a7c5 100644 --- a/tst/unit/training/skyrl_test.py +++ b/tst/unit/training/skyrl_test.py @@ -24,7 +24,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -87,7 +87,8 @@ def test_a_servers_tasks_become_a_row_per_task_with_its_tools_in_its_system_prom {"role": "user", "content": "Add a barrel."}], "env_class": "agentenv", "data_source": "agentenv/Dock Yard", - "episode": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN]}, + "episode": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN], + "first_turn_tools": None}, "server": {"url": server.url, "tool_timeout": 30.0, "reset_timeout": 90.0, "attempts": 2, "tool_format": "qwen3_coder", "token_var": None}, } @@ -131,6 +132,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, tokenizer, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + out = write_dataset(server.url, tmp_path, tokenizer=tokenizer()).folder + server.stop() + + assert [row["episode"]["first_turn_tools"] for row in _rows(out / "all.parquet")] == [ + json.dumps(["add_item", "end_turn", "take_item"]), None] + + def test_a_split_the_server_keeps_empty_has_a_rows_columns(serve, tmp_path, tokenizer): server = serve({"a": PLAYED}, {"empty": []}) diff --git a/tst/unit/training/trl_test.py b/tst/unit/training/trl_test.py index dd7868342..61c313506 100644 --- a/tst/unit/training/trl_test.py +++ b/tst/unit/training/trl_test.py @@ -23,6 +23,7 @@ from tst.unit.bundle._support import layout from tst.unit.openenv.environment_test import AGAIN from tst.unit.task.episode.run_test import ALICE, ITEMS, PANTRY, PANTRY_MCP, REPLIES, TURN, _tool_sessions +from tst.unit.training._support import revealing from tst.unit.training.tools_test import called_by_qwen3, shown_to_qwen3 PLAYED = [ITEMS, ALICE, TURN, REPLIES] @@ -278,6 +279,39 @@ def test_a_rollout_whose_tools_arent_its_classs_stops_training(served, monkeypat environment.get_reward() +@pytest.mark.parametrize("later, stops", [(["take_item"], False), ([], True)], ids=["same-reveal", "fewer"]) +def test_a_rollout_whose_config_reveals_fewer_tools_than_its_classs_stops_training(served, reveals, later, stops): + env = served({"a": revealing(reveals, "take_item", rate=0.1), "b": revealing(reveals, *later, rate=0.5)}) + environment = env.factory() + + environment.reset(**env.dataset[1]) + environment.add_item(item="crate") + environment.end_turn(response="Added a crate.") + if stops: + with pytest.raises(RuntimeError, match=r"its toolset's first task does there, so serve them in separate bundles\. " + r"TRL shows every task"): + environment.get_reward() + else: + assert environment.get_reward() == 1.0 + + +def test_a_rollout_revealing_at_its_first_turn_what_its_classs_first_task_reveals_later_stops_training( + served, reveals): + env = served({"a": [ITEMS, ALICE, TURN, {**reveals("take_item"), "depends_on": ["turn"]}, + {**AGAIN, "depends_on": ["cfg"]}, REPLIES], + "b": [ITEMS, ALICE, reveals("take_item"), {**TURN, "depends_on": ["cfg"]}, AGAIN, REPLIES]}) + environment = env.factory() + + environment.reset(**env.dataset[1]) + with pytest.raises(RuntimeError) as stopped: + environment.get_reward() + + assert str(stopped.value) == ( + "task 'b' offers the tools ['add_item', 'end_turn', 'take_item'], not the ['add_item', 'end_turn'] its " + "toolset's first task does. TRL shows every task of a toolset the tools found by playing its first task, so " + "this toolset misses what tells its tasks apart") + + class _Slow(TaskStep): """Takes a second, as a verifier that grades the episode after its last turn does.""" diff --git a/tst/unit/training/verl_test.py b/tst/unit/training/verl_test.py index d472ce5c3..fcd38894d 100644 --- a/tst/unit/training/verl_test.py +++ b/tst/unit/training/verl_test.py @@ -23,7 +23,7 @@ TURN, _tool_sessions, ) -from tst.unit.training._support import PLAYED +from tst.unit.training._support import PLAYED, revealing ADD_ITEM = {"type": "function", "function": {"name": "add_item", "description": "add_item.", "parameters": {"type": "object"}}} @@ -50,7 +50,8 @@ def test_a_servers_tasks_become_a_row_per_task_of_each_split_and_the_agent_loop_ "data_source": "agentenv/Dock Yard", "prompt": [{"role": "system", "content": "You keep the items."}, {"role": "user", "content": "Add a barrel."}], "agent_name": "agentenv", - "extra_info": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN]}, + "extra_info": {"split": "smoke", "index": 0, "task_id": "b", "tools": [ADD_ITEM, FUNCTION_END_TURN], + "first_turn_tools": None}, } assert [(row["extra_info"]["index"], row["extra_info"]["task_id"]) for row in _rows(out / "all.parquet")] == [ (0, "a"), (1, "b")] @@ -69,6 +70,16 @@ def test_each_toolset_is_probed_once_and_a_rows_tools_are_its_tasks(serve, tmp_p assert len(teardowns.finished) == 2 +def test_a_row_of_a_task_with_server_config_keeps_its_toolsets_first_turn_tools(serve, tmp_path, reveals): + server = serve({"a": revealing(reveals, "take_item"), "plain": PLAYED}) + + out = write_dataset(server.url, tmp_path).folder + server.stop() + + assert [row["extra_info"]["first_turn_tools"] for row in _rows(out / "all.parquet")] == [ + json.dumps(["add_item", "end_turn", "take_item"]), None] + + def test_a_split_the_server_keeps_empty_has_a_rows_columns(serve, tmp_path): server = serve({"a": PLAYED}, {"empty": []}) diff --git a/tst/verl/loop_test.py b/tst/verl/loop_test.py index 267377a6f..096db8266 100644 --- a/tst/verl/loop_test.py +++ b/tst/verl/loop_test.py @@ -119,6 +119,21 @@ def test_a_rollout_the_env_fails_every_time_raises_so_verl_leaves_it_out(serve, broken.stop() +def test_a_rollout_whose_first_turn_offers_other_tools_than_its_rows_first_turn_tools_is_refused_and_a_match_plays( + serve, tmp_path, tokenizer): + server = serve({"a": PLAYED}) + row = _row(write_dataset(server.url, tmp_path)) + row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn", "take_item"]) + + with pytest.raises(verl_loop.RolloutFailed, match="its toolset's first task does there, so serve them in separate bundles"): + asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) + row["extra_info"]["first_turn_tools"] = json.dumps(["add_item", "end_turn"]) + output = asyncio.run(_play(server.url, row, _Model(tokenizer), tokenizer, attempts=1)) + server.stop() + + assert output.reward_score == 1.0 + + def test_a_rollout_isnt_played_again_once_its_time_is_up(serve, tmp_path, tokenizer, monkeypatch): server = serve({"a": PLAYED}) row = _row(write_dataset(server.url, tmp_path))