From 0deba588628d94c4619716db5b1d76afc0873fd4 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 05:36:03 -0500 Subject: [PATCH 1/2] fix(task-steps): a run-wide agent pick of another agent no longer takes the step's pinned version Co-Authored-By: Claude Fable 5.2 --- src/agent_env/cli/task/run.py | 2 +- .../task_step/task_steps/deploy_agent.py | 16 ++- .../task_step/task_steps/install_agent.py | 9 +- tst/unit/bundle/agent_override_test.py | 2 +- .../task_step/run_wide_agent_pick_test.py | 121 +++++++++++++++--- 5 files changed, 124 insertions(+), 26 deletions(-) diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index 64aff5183..8e7b84aa6 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -373,7 +373,7 @@ def _stored(task, context: TaskStepContext) -> str: @click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used") @click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)") @click.option("--a2a-agent-id", default=None, type=str, - help="Override A2A agent id for deploy_agent steps; a user sim's agent keeps its own") + help="Override A2A agent id for deploy_agent steps, at its latest version when it is not the step's own agent; a user sim's agent keeps its own") @click.option("--agent-sandbox", default=None, help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')") @click.option("--env-sandbox", default=None, diff --git a/src/agent_env/task_step/task_steps/deploy_agent.py b/src/agent_env/task_step/task_steps/deploy_agent.py index a4d8065cc..9d6ad1a3d 100644 --- a/src/agent_env/task_step/task_steps/deploy_agent.py +++ b/src/agent_env/task_step/task_steps/deploy_agent.py @@ -29,7 +29,7 @@ snapshot_load_call, ) from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store -from agent_env.config import get_config +from agent_env.config import ConfigError, get_config from agent_env.config.model import ModelRole, endpoint_host from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER @@ -136,6 +136,18 @@ def bound_deployment( return matches[0] if matches else None +def picks_another_agent(user_overrides: Mapping, stored_id: Optional[str]) -> bool: + """Whether a run-wide ``a2a_agent_id`` names an agent other than the step's own, which the step's pinned version + says nothing about.""" + picked = user_overrides.get("a2a_agent_id") + if not picked or stored_id: + return bool(picked) and picked != stored_id + try: + return picked != get_config().get_default_a2a_agent_id() + except ConfigError: # a pick needs no default agent, so an unreadable one must not fail the run + return True + + def _skill_fields(skill: dict) -> dict: """A ``skills`` entry as ``A2AAgent.register_skill`` arguments. ``content`` is an older name for ``skill_md``, and ``skill_s3_url`` and ``s3_uri`` for ``object_url``; ``skill_md`` is sent verbatim.""" @@ -330,7 +342,7 @@ def _resolve_agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[in return agent_id, step_overrides["a2a_agent_version"] if "a2a_agent_version" in user_overrides: return agent_id, user_overrides["a2a_agent_version"] - return agent_id, self.a2a_agent_version + return agent_id, None if picks_another_agent(user_overrides, self.a2a_agent_id) else self.a2a_agent_version def _run_system_prompt(self, context: TaskStepContext) -> Optional[str]: """The system prompt this run sets on the agent: ``system_prompt`` with the run's ```` placeholders diff --git a/src/agent_env/task_step/task_steps/install_agent.py b/src/agent_env/task_step/task_steps/install_agent.py index 13caa6499..0a21bc538 100644 --- a/src/agent_env/task_step/task_steps/install_agent.py +++ b/src/agent_env/task_step/task_steps/install_agent.py @@ -44,6 +44,7 @@ from agent_env.task_step.context import DeployedAgent, StepContextError, TaskStepContext from agent_env.entity.refs import EntityRef from agent_env.task_step.task_step import TaskStep, TaskStepDependency +from agent_env.task_step.task_steps.deploy_agent import picks_another_agent from agentenv_protocol.types import CONTROL_TOKEN_ENV logger = logging.getLogger(__name__) @@ -228,15 +229,17 @@ def preflight(self) -> list[str]: def _agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[int]]: """The agent id and version this run installs: this step's ``step_params`` override names both, else the - run-wide ``a2a_agent_id`` override or the step's agent, at the step's version.""" + step's agent at the step's version, or another agent the run-wide ``a2a_agent_id`` names at the run-wide + ``a2a_agent_version`` (latest when unset).""" step_overrides = self.step_param_overrides(context) if "a2a_agent_id" in step_overrides: return step_overrides["a2a_agent_id"], step_overrides.get("a2a_agent_version") # Honor `--a2a-agent-id` override (same pattern as DeployAgentTaskStep) — lets the # caller swap which agent gets installed without rewriting the task definition. user_overrides = context.metadata.get("user_overrides", {}) - a2a_agent_id = user_overrides.get("a2a_agent_id") or self.a2a_agent_id or get_config().get_default_a2a_agent_id() - return a2a_agent_id, self.a2a_agent_version + if picks_another_agent(user_overrides, self.a2a_agent_id): + return user_overrides["a2a_agent_id"], user_overrides.get("a2a_agent_version") + return self.a2a_agent_id or get_config().get_default_a2a_agent_id(), self.a2a_agent_version async def execute(self, context: TaskStepContext) -> TaskStepContext: from agent_env.providers.sandbox_providers.sandbox_provider import ( diff --git a/tst/unit/bundle/agent_override_test.py b/tst/unit/bundle/agent_override_test.py index 5159db696..ebfd85486 100644 --- a/tst/unit/bundle/agent_override_test.py +++ b/tst/unit/bundle/agent_override_test.py @@ -235,7 +235,7 @@ def test_an_install_takes_its_steps_override_over_the_run_wide_one_and_the_store assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide", **solver}})) == ( "solver", None) - assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide"}})) == ("run-wide", 3) + assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide"}})) == ("run-wide", None) assert step._agent_ref(TaskStepContext()) == ("stored", 3) diff --git a/tst/unit/task_step/run_wide_agent_pick_test.py b/tst/unit/task_step/run_wide_agent_pick_test.py index eecdf6e3b..94567c7a0 100644 --- a/tst/unit/task_step/run_wide_agent_pick_test.py +++ b/tst/unit/task_step/run_wide_agent_pick_test.py @@ -1,11 +1,20 @@ """A run-wide agent pick (``user_overrides.a2a_agent_id``: ``task run --a2a-agent-id``, a hub run's agent pick) reaches -every ``deploy_agent`` and ``install_agent`` step, a ``prompt_agent`` task's user agent included, exactly as on main. -A ``deploy_user_sim_agent`` step keeps its own agent through its own ``_resolve_agent_ref``, not anything these steps -read.""" +every ``deploy_agent`` and ``install_agent`` step, a ``prompt_agent`` task's user agent included. A step's pinned +version belongs to the agent the step names, so a pick of another agent never takes it. Each case runs the step and +reads what it asks the agent store for. A ``deploy_user_sim_agent`` step keeps its own agent through its own +``_resolve_agent_ref``, not anything these steps read.""" from __future__ import annotations -from agent_env.task_step.context import TaskStepContext +import asyncio +from types import SimpleNamespace + +import pytest + +from agent_env.a2a_agent import A2AAgent +from agent_env.config import ConfigError, get_config, reset_config +from agent_env.providers.sandbox_providers import sandbox_provider +from agent_env.task_step.context import DeployedSandbox, TaskStepContext from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep @@ -13,26 +22,100 @@ "a2a_agent_id": "theirs", "a2a_agent_version": 4} HUMAN = {"id": "deploy-human", "type": "deploy_agent", "env_ids": [], "agent_name": "human_agent", "a2a_agent_id": "sim", "a2a_agent_version": 2} +INSTALL = {"id": "install-human", "type": "install_agent", "agent_name": "human_agent", "a2a_agent_id": "sim", + "a2a_agent_version": 2, "sandbox_name": "world"} +UNNAMED = {"id": "deploy-default", "type": "deploy_agent", "env_ids": [], "agent_name": "solver", + "a2a_agent_version": 6} +UNNAMED_INSTALL = {"id": "install-default", "type": "install_agent", "agent_name": "solver", "a2a_agent_version": 6, + "sandbox_name": "world"} + + +class _Asked(Exception): + pass + + +@pytest.fixture +def asked(monkeypatch, local_stores) -> list: + """Each (id, version) a step asks the agent store for; the step stops there.""" + asked: list = [] + + def get(agent_id, version=None): + asked.append((agent_id, version)) + raise _Asked + + monkeypatch.setattr(A2AAgent, "get", staticmethod(get)) + monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: SimpleNamespace( + get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=SimpleNamespace()))) + return asked + + +async def _run(step_dict: dict, asked: list, **overrides) -> tuple: + step_class = InstallAgentTaskStep if step_dict["type"] == "install_agent" else DeployAgentTaskStep + context = TaskStepContext(instance_id="run-1", metadata={"user_overrides": overrides}, deployed_sandboxes=[ + DeployedSandbox(sandbox_name="world", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="local", + tunnel_urls={"8000": "http://127.0.0.1:8000"})]) + with pytest.raises(_Asked): + await step_class.from_dict(step_dict).execute(context) + return asked[-1] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"]) +async def test_a_step_with_no_pick_keeps_its_agent_and_pin(asked, step): + assert await _run(step, asked) == (step["a2a_agent_id"], step["a2a_agent_version"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"]) +async def test_a_pick_of_another_agent_does_not_take_the_steps_pinned_version(asked, step): + assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"]) +async def test_a_pick_of_another_agent_with_a_version_deploys_that_version(asked, step): + assert await _run(step, asked, a2a_agent_id="mine", a2a_agent_version=9) == ("mine", 9) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"]) +async def test_a_pick_of_the_steps_own_agent_keeps_its_pin(asked, step): + assert await _run(step, asked, a2a_agent_id=step["a2a_agent_id"]) == (step["a2a_agent_id"], step["a2a_agent_version"]) + +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [UNNAMED, UNNAMED_INSTALL], ids=lambda step: step["id"]) +async def test_a_step_naming_no_agent_keeps_its_pin_only_for_the_default_agent(asked, local_stores, step): + default = local_stores.get_default_a2a_agent_id() + assert await _run(step, asked) == (default, 6) + assert await _run(step, asked, a2a_agent_id=default) == (default, 6) + assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None) -def _picked(**overrides) -> TaskStepContext: - return TaskStepContext(instance_id="run-1", metadata={"user_overrides": {"a2a_agent_id": "mine", **overrides}}) +@pytest.mark.asyncio +@pytest.mark.parametrize("step", [UNNAMED, UNNAMED_INSTALL], ids=lambda step: step["id"]) +async def test_a_pick_runs_where_the_default_agent_cannot_be_read(asked, tmp_path, monkeypatch, step): + config = tmp_path / "config.toml" + config.write_text('[agents]\ndefault_a2a_agent_id = "env:AGENT_ENV_TEST_UNSET_DEFAULT_AGENT"\n') + monkeypatch.delenv("AGENT_ENV_TEST_UNSET_DEFAULT_AGENT", raising=False) + monkeypatch.setenv("AGENT_ENV_CONFIG", str(config)) + reset_config() + with pytest.raises(ConfigError): + get_config().get_default_a2a_agent_id() -def test_the_pick_reaches_a_prompt_agent_tasks_user_agent_too_at_each_steps_stored_version(): - ctx = _picked() - assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 4) - assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 2) + assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None) + assert await _run(step, asked, a2a_agent_id="mine", a2a_agent_version=9) == ("mine", 9) -def test_a_picked_version_applies_to_every_deploy(): - ctx = _picked(a2a_agent_version=9) - assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 9) - assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 9) +@pytest.mark.asyncio +async def test_a_picked_version_alone_applies_to_every_deploy_and_no_install(asked): + assert await _run(SOLVER, asked, a2a_agent_version=9) == ("theirs", 9) + assert await _run(HUMAN, asked, a2a_agent_version=9) == ("sim", 9) + assert await _run(INSTALL, asked, a2a_agent_version=9) == ("sim", 2) -def test_install_agent_takes_the_pick_at_its_stored_version(): - installed = InstallAgentTaskStep.from_dict({"id": "install-human", "type": "install_agent", - "agent_name": "human_agent", "a2a_agent_id": "sim", - "a2a_agent_version": 2, "sandbox_name": "world"}) - assert installed._agent_ref(_picked()) == ("mine", 2) +@pytest.mark.asyncio +async def test_a_steps_own_version_override_wins_over_the_pick(asked): + step_params = {"deploy-solver": {"a2a_agent_version": 5}} + assert await _run(SOLVER, asked, a2a_agent_id="mine", step_params=step_params) == ("mine", 5) + assert await _run(SOLVER, asked, a2a_agent_id="mine", a2a_agent_version=9, step_params=step_params) == ("mine", 5) From 61dcf8d3a7c597f5f79c061e5230ca370000fc36 Mon Sep 17 00:00:00 2001 From: Mateusz Cikowski Date: Fri, 20 Nov 2026 05:49:50 -0500 Subject: [PATCH 2/2] address review round 2: the pick's help text and the CLI test docstring state which version a picked agent runs at --- src/agent_env/cli/task/run.py | 2 +- tst/unit/cli/task_run_agent_pick_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/agent_env/cli/task/run.py b/src/agent_env/cli/task/run.py index 8e7b84aa6..9bf04f558 100644 --- a/src/agent_env/cli/task/run.py +++ b/src/agent_env/cli/task/run.py @@ -373,7 +373,7 @@ def _stored(task, context: TaskStepContext) -> str: @click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used") @click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)") @click.option("--a2a-agent-id", default=None, type=str, - help="Override A2A agent id for deploy_agent steps, at its latest version when it is not the step's own agent; a user sim's agent keeps its own") + help="Override A2A agent id for deploy_agent and install_agent steps; an agent other than the step's own runs at the run's a2a_agent_version, else its latest, never the step's pinned version; a user sim's agent keeps its own") @click.option("--agent-sandbox", default=None, help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')") @click.option("--env-sandbox", default=None, diff --git a/tst/unit/cli/task_run_agent_pick_test.py b/tst/unit/cli/task_run_agent_pick_test.py index c2b557162..35e70d076 100644 --- a/tst/unit/cli/task_run_agent_pick_test.py +++ b/tst/unit/cli/task_run_agent_pick_test.py @@ -1,4 +1,4 @@ -"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, at the deploy steps' stored versions, for +"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, with no version of its own, for each step to apply (tst/unit/task_step/run_wide_agent_pick_test.py); a step that plays the user keeps its own agent by its class, as it does for a hub run's pick."""