Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/agent_env/cli/task/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ def _stored(task, context: TaskStepContext) -> str:
@click.option("--judge-litellm-api-key", default=None, type=str, help="Override LITELLM_API_KEY used by rubrics_verifier judge agents (falls back to --litellm-api-key, then to config secret); a secret:KEY or env:NAME reference is resolved where it is used")
@click.option("--apply-trajectory-filter/--no-trajectory-filter", "apply_trajectory_filter", default=None, help="Force trajectory filtering on/off for rubrics_verifier (overrides the step's trajectory_filter setting)")
@click.option("--a2a-agent-id", default=None, type=str,
help="Override A2A agent id for deploy_agent steps; a user sim's agent keeps its own")
help="Override A2A agent id for deploy_agent and install_agent steps; an agent other than the step's own runs at the run's a2a_agent_version, else its latest, never the step's pinned version; a user sim's agent keeps its own")

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This help names "the run's a2a_agent_version", but task run has no flag that sets it: line 422 only writes user_overrides["a2a_agent_id"]. So after this PR, task run --a2a-agent-id mine on a task pinned to theirs v4 always gets mine at latest, resolved separately in each step, and the only way to pin it is a hand-written --context-json. Could you add --a2a-agent-version next to --a2a-agent-id (written into user_overrides), or say "latest; pin via --context-json user_overrides.a2a_agent_version" in the help?

@click.option("--agent-sandbox", default=None,
help="Override agent sandbox backend(s) for deploy_agent steps; comma-separated for fallback chain (e.g. 'modal,local')")
@click.option("--env-sandbox", default=None,
Expand Down
16 changes: 14 additions & 2 deletions src/agent_env/task_step/task_steps/deploy_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
snapshot_load_call,
)
from agent_env.a2a_agent.staging import StagedObjectStore, transfer_store
from agent_env.config import get_config
from agent_env.config import ConfigError, get_config
from agent_env.config.model import ModelRole, endpoint_host
from agent_env.env.env import DeployedEnv, DeployedSandboxEnv, Env
from agent_env.env.gateway import AGENT_ENV_ROLE_HEADER
Expand Down Expand Up @@ -136,6 +136,18 @@ def bound_deployment(
return matches[0] if matches else None


def picks_another_agent(user_overrides: Mapping, stored_id: Optional[str]) -> bool:
"""Whether a run-wide ``a2a_agent_id`` names an agent other than the step's own, which the step's pinned version
says nothing about."""
picked = user_overrides.get("a2a_agent_id")
if not picked or stored_id:
return bool(picked) and picked != stored_id
try:
return picked != get_config().get_default_a2a_agent_id()
except ConfigError: # a pick needs no default agent, so an unreadable one must not fail the run
return True


def _skill_fields(skill: dict) -> dict:
"""A ``skills`` entry as ``A2AAgent.register_skill`` arguments. ``content`` is an older name for
``skill_md``, and ``skill_s3_url`` and ``s3_uri`` for ``object_url``; ``skill_md`` is sent verbatim."""
Expand Down Expand Up @@ -330,7 +342,7 @@ def _resolve_agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[in
return agent_id, step_overrides["a2a_agent_version"]
if "a2a_agent_version" in user_overrides:
return agent_id, user_overrides["a2a_agent_version"]
return agent_id, self.a2a_agent_version
return agent_id, None if picks_another_agent(user_overrides, self.a2a_agent_id) else self.a2a_agent_version

def _run_system_prompt(self, context: TaskStepContext) -> Optional[str]:
"""The system prompt this run sets on the agent: ``system_prompt`` with the run's ``<seed>`` placeholders
Expand Down
9 changes: 6 additions & 3 deletions src/agent_env/task_step/task_steps/install_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
from agent_env.task_step.context import DeployedAgent, StepContextError, TaskStepContext
from agent_env.entity.refs import EntityRef
from agent_env.task_step.task_step import TaskStep, TaskStepDependency
from agent_env.task_step.task_steps.deploy_agent import picks_another_agent
from agentenv_protocol.types import CONTROL_TOKEN_ENV

logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -228,15 +229,17 @@ def preflight(self) -> list[str]:

def _agent_ref(self, context: TaskStepContext) -> tuple[str, Optional[int]]:
"""The agent id and version this run installs: this step's ``step_params`` override names both, else the
run-wide ``a2a_agent_id`` override or the step's agent, at the step's version."""
step's agent at the step's version, or another agent the run-wide ``a2a_agent_id`` names at the run-wide
``a2a_agent_version`` (latest when unset)."""
step_overrides = self.step_param_overrides(context)
if "a2a_agent_id" in step_overrides:
return step_overrides["a2a_agent_id"], step_overrides.get("a2a_agent_version")
# Honor `--a2a-agent-id` override (same pattern as DeployAgentTaskStep) — lets the
# caller swap which agent gets installed without rewriting the task definition.
user_overrides = context.metadata.get("user_overrides", {})
a2a_agent_id = user_overrides.get("a2a_agent_id") or self.a2a_agent_id or get_config().get_default_a2a_agent_id()
return a2a_agent_id, self.a2a_agent_version
if picks_another_agent(user_overrides, self.a2a_agent_id):
return user_overrides["a2a_agent_id"], user_overrides.get("a2a_agent_version")
return self.a2a_agent_id or get_config().get_default_a2a_agent_id(), self.a2a_agent_version

async def execute(self, context: TaskStepContext) -> TaskStepContext:
from agent_env.providers.sandbox_providers.sandbox_provider import (
Expand Down
2 changes: 1 addition & 1 deletion tst/unit/bundle/agent_override_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ def test_an_install_takes_its_steps_override_over_the_run_wide_one_and_the_store

assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide", **solver}})) == (
"solver", None)
assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide"}})) == ("run-wide", 3)
assert step._agent_ref(TaskStepContext(metadata={"user_overrides": {"a2a_agent_id": "run-wide"}})) == ("run-wide", None)
assert step._agent_ref(TaskStepContext()) == ("stored", 3)


Expand Down
2 changes: 1 addition & 1 deletion tst/unit/cli/task_run_agent_pick_test.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, at the deploy steps' stored versions, for
"""``task run --a2a-agent-id`` is a run-wide pick: the CLI leaves it run-wide, with no version of its own, for
each step to apply (tst/unit/task_step/run_wide_agent_pick_test.py); a step that plays the user keeps its own agent by
its class, as it does for a hub run's pick."""

Expand Down
121 changes: 102 additions & 19 deletions tst/unit/task_step/run_wide_agent_pick_test.py
Original file line number Diff line number Diff line change
@@ -1,38 +1,121 @@
"""A run-wide agent pick (``user_overrides.a2a_agent_id``: ``task run --a2a-agent-id``, a hub run's agent pick) reaches
every ``deploy_agent`` and ``install_agent`` step, a ``prompt_agent`` task's user agent included, exactly as on main.
A ``deploy_user_sim_agent`` step keeps its own agent through its own ``_resolve_agent_ref``, not anything these steps
read."""
every ``deploy_agent`` and ``install_agent`` step, a ``prompt_agent`` task's user agent included. A step's pinned
version belongs to the agent the step names, so a pick of another agent never takes it. Each case runs the step and
reads what it asks the agent store for. A ``deploy_user_sim_agent`` step keeps its own agent through its own
``_resolve_agent_ref``, not anything these steps read."""

from __future__ import annotations

from agent_env.task_step.context import TaskStepContext
import asyncio
from types import SimpleNamespace

import pytest

from agent_env.a2a_agent import A2AAgent
from agent_env.config import ConfigError, get_config, reset_config
from agent_env.providers.sandbox_providers import sandbox_provider
from agent_env.task_step.context import DeployedSandbox, TaskStepContext
from agent_env.task_step.task_steps.deploy_agent import DeployAgentTaskStep
from agent_env.task_step.task_steps.install_agent import InstallAgentTaskStep

SOLVER = {"id": "deploy-solver", "type": "deploy_agent", "env_ids": [], "agent_name": "solver",
"a2a_agent_id": "theirs", "a2a_agent_version": 4}
HUMAN = {"id": "deploy-human", "type": "deploy_agent", "env_ids": [], "agent_name": "human_agent",
"a2a_agent_id": "sim", "a2a_agent_version": 2}
INSTALL = {"id": "install-human", "type": "install_agent", "agent_name": "human_agent", "a2a_agent_id": "sim",
"a2a_agent_version": 2, "sandbox_name": "world"}
UNNAMED = {"id": "deploy-default", "type": "deploy_agent", "env_ids": [], "agent_name": "solver",
"a2a_agent_version": 6}
UNNAMED_INSTALL = {"id": "install-default", "type": "install_agent", "agent_name": "solver", "a2a_agent_version": 6,
"sandbox_name": "world"}


class _Asked(Exception):
pass


@pytest.fixture
def asked(monkeypatch, local_stores) -> list:
"""Each (id, version) a step asks the agent store for; the step stops there."""
asked: list = []

def get(agent_id, version=None):
asked.append((agent_id, version))
raise _Asked

monkeypatch.setattr(A2AAgent, "get", staticmethod(get))
monkeypatch.setattr(sandbox_provider, "build_sandbox_provider", lambda name: SimpleNamespace(
get_sandbox=lambda sandbox_id: asyncio.sleep(0, result=SimpleNamespace())))
return asked


async def _run(step_dict: dict, asked: list, **overrides) -> tuple:
step_class = InstallAgentTaskStep if step_dict["type"] == "install_agent" else DeployAgentTaskStep
context = TaskStepContext(instance_id="run-1", metadata={"user_overrides": overrides}, deployed_sandboxes=[
DeployedSandbox(sandbox_name="world", sandbox_id="sb-1", sandbox_mode="vm", sandbox_type="local",
tunnel_urls={"8000": "http://127.0.0.1:8000"})])
with pytest.raises(_Asked):
await step_class.from_dict(step_dict).execute(context)
return asked[-1]


@pytest.mark.asyncio
@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"])
async def test_a_step_with_no_pick_keeps_its_agent_and_pin(asked, step):
assert await _run(step, asked) == (step["a2a_agent_id"], step["a2a_agent_version"])


@pytest.mark.asyncio
@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"])
async def test_a_pick_of_another_agent_does_not_take_the_steps_pinned_version(asked, step):
assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None)


@pytest.mark.asyncio
@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"])
async def test_a_pick_of_another_agent_with_a_version_deploys_that_version(asked, step):
assert await _run(step, asked, a2a_agent_id="mine", a2a_agent_version=9) == ("mine", 9)


@pytest.mark.asyncio
@pytest.mark.parametrize("step", [SOLVER, HUMAN, INSTALL], ids=lambda step: step["id"])
async def test_a_pick_of_the_steps_own_agent_keeps_its_pin(asked, step):
assert await _run(step, asked, a2a_agent_id=step["a2a_agent_id"]) == (step["a2a_agent_id"], step["a2a_agent_version"])


@pytest.mark.asyncio
@pytest.mark.parametrize("step", [UNNAMED, UNNAMED_INSTALL], ids=lambda step: step["id"])
async def test_a_step_naming_no_agent_keeps_its_pin_only_for_the_default_agent(asked, local_stores, step):
default = local_stores.get_default_a2a_agent_id()
assert await _run(step, asked) == (default, 6)
assert await _run(step, asked, a2a_agent_id=default) == (default, 6)
assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None)

def _picked(**overrides) -> TaskStepContext:
return TaskStepContext(instance_id="run-1", metadata={"user_overrides": {"a2a_agent_id": "mine", **overrides}})

@pytest.mark.asyncio
@pytest.mark.parametrize("step", [UNNAMED, UNNAMED_INSTALL], ids=lambda step: step["id"])
async def test_a_pick_runs_where_the_default_agent_cannot_be_read(asked, tmp_path, monkeypatch, step):
config = tmp_path / "config.toml"
config.write_text('[agents]\ndefault_a2a_agent_id = "env:AGENT_ENV_TEST_UNSET_DEFAULT_AGENT"\n')
monkeypatch.delenv("AGENT_ENV_TEST_UNSET_DEFAULT_AGENT", raising=False)
monkeypatch.setenv("AGENT_ENV_CONFIG", str(config))
reset_config()
with pytest.raises(ConfigError):
get_config().get_default_a2a_agent_id()

def test_the_pick_reaches_a_prompt_agent_tasks_user_agent_too_at_each_steps_stored_version():
ctx = _picked()
assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 4)
assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 2)
assert await _run(step, asked, a2a_agent_id="mine") == ("mine", None)
assert await _run(step, asked, a2a_agent_id="mine", a2a_agent_version=9) == ("mine", 9)


def test_a_picked_version_applies_to_every_deploy():
ctx = _picked(a2a_agent_version=9)
assert DeployAgentTaskStep.from_dict(SOLVER)._resolve_agent_ref(ctx) == ("mine", 9)
assert DeployAgentTaskStep.from_dict(HUMAN)._resolve_agent_ref(ctx) == ("mine", 9)
@pytest.mark.asyncio
async def test_a_picked_version_alone_applies_to_every_deploy_and_no_install(asked):
assert await _run(SOLVER, asked, a2a_agent_version=9) == ("theirs", 9)
assert await _run(HUMAN, asked, a2a_agent_version=9) == ("sim", 9)
assert await _run(INSTALL, asked, a2a_agent_version=9) == ("sim", 2)


def test_install_agent_takes_the_pick_at_its_stored_version():
installed = InstallAgentTaskStep.from_dict({"id": "install-human", "type": "install_agent",
"agent_name": "human_agent", "a2a_agent_id": "sim",
"a2a_agent_version": 2, "sandbox_name": "world"})
assert installed._agent_ref(_picked()) == ("mine", 2)
@pytest.mark.asyncio
async def test_a_steps_own_version_override_wins_over_the_pick(asked):
step_params = {"deploy-solver": {"a2a_agent_version": 5}}
assert await _run(SOLVER, asked, a2a_agent_id="mine", step_params=step_params) == ("mine", 5)
assert await _run(SOLVER, asked, a2a_agent_id="mine", a2a_agent_version=9, step_params=step_params) == ("mine", 5)
Loading