From 80fc68139a1075fe62853da69f5da3cb543e1ad6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 14:49:53 -0800 Subject: [PATCH 1/6] feat(task-steps): prompt steps store each turn's atif/1 copy when the run prescribes it A run that prescribes atif/1 reads each solver turn's copy inline once the turn ends, off the turn's path, and stores it as .json in the run's folder for the agent, with its aux log, turn context and media; each turn's record says how it checked (valid, invalid, newer, unavailable), and a restart's next document is linked from the session's earlier one. Refs FD-6078. Co-Authored-By: Claude Opus 5.5 --- .agentenv/config.example.toml | 3 +- AGENTS.md | 7 +- src/agent_env/task_step/context.py | 21 +- .../task_step/snapshot_utils/atif_copies.py | 297 +++++++++++++ .../task_step/task_steps/prompt_agent.py | 10 + ...test_prompt_agent_live_trajectory_local.py | 22 +- .../test_prompt_agent_atif_copies.py | 393 ++++++++++++++++++ .../test_prompt_agent_live_trajectory.py | 5 +- .../task_step/verifiers/test_judge_inputs.py | 24 +- 9 files changed, 769 insertions(+), 13 deletions(-) create mode 100644 src/agent_env/task_step/snapshot_utils/atif_copies.py create mode 100644 tst/unit/task_step/test_prompt_agent_atif_copies.py diff --git a/.agentenv/config.example.toml b/.agentenv/config.example.toml index 7ca405d01..82966d756 100644 --- a/.agentenv/config.example.toml +++ b/.agentenv/config.example.toml @@ -62,7 +62,8 @@ agent_default = "local" # agents too # [trajectories] # formats = ["atif/1"] # the formats beside native a run prescribes to each agent whose trajectory it reads; -# # none by default, so agents build native only. Per run: user_overrides.trajectory_formats +# # none by default, so agents build native only. Per run: user_overrides.trajectory_formats. +# # With atif/1, prompt steps store each turn's copy under {prefix}atif/instance_id=// # [envs] # default_gateway_env_id = "default" # the gateway every gateway deploy runs diff --git a/AGENTS.md b/AGENTS.md index 48174a839..211676350 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,7 +122,12 @@ or one of them only where it is used. agent whose trajectory it reads (prompt_agent and always_on call the trajectory extension's `prescribe` on the agent they prompt before its conversation starts); a run's `user_overrides.trajectory_formats` beats it. None by default, so an agent builds native only, and an `atif/1` read of a conversation not prescribed it is refused - (`format_not_prescribed`). + (`format_not_prescribed`). A prompt step of a run that prescribes `atif/1` reads each turn's copy inline once the + turn ends, off the turn's path, and stores it as `.json` under `{prefix}atif/instance_id=//` + (`metadata["atif_copies"]`, `task_step/snapshot_utils/atif_copies.py`), overwritten each turn, with the turn's aux + log and turn context under `turns/` and the media it names pulled into `media/`; after a restart the session's + earlier document names the next in `continued_trajectory_ref`. Each turn's record says how its copy checked + (`AtifRecord`: `valid`, `invalid`, `newer`, `unavailable`), and a copy never fails the run; judges read native. - Secrets never live in the file: use `secret:KEY` or `env:NAME` references, resolved through `[stores.secret]`. - agent-env has no stage concept: "dev" versus "prod" is which file `AGENT_ENV_CONFIG` names. A diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 5bb8abdfb..604970159 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -235,13 +235,30 @@ def sandbox_overrides(sandbox: str) -> dict[str, str]: ) +class AtifRecord(TypedDict): + """A turn's atif/1 copy: the stored document and the step it reached at that turn (each turn's copy is a prefix + of the next), how it checked (``valid``, ``invalid`` with the rules it broke, ``newer`` than this worker reads as a + producer's, or ``unavailable`` with why in ``reason``), the versions that made and checked it, and the turn's aux + log and turn context.""" + + url: Optional[str] + through_step: Optional[int] + status: str + rules: list[str] + reason: Optional[str] + validator_version: Optional[str] + adapter_version: Optional[str] + aux_url: Optional[str] + turn_context_url: Optional[str] + + class TurnTrajectory(TypedDict): """What a turn's trajectory reads left, beside its native copy's url in the per-turn list: the native copy's format, as the agent's answer named it (None from an agent that names none), and its atif/1 copy, None when the - agent offers none.""" + run doesn't prescribe atif/1.""" native_format: Optional[str] - atif: Optional[dict[str, Any]] + atif: Optional[AtifRecord] @dataclass diff --git a/src/agent_env/task_step/snapshot_utils/atif_copies.py b/src/agent_env/task_step/snapshot_utils/atif_copies.py new file mode 100644 index 000000000..4e94ad174 --- /dev/null +++ b/src/agent_env/task_step/snapshot_utils/atif_copies.py @@ -0,0 +1,297 @@ +"""A run's atif/1 copies: when the run prescribes atif/1, each turn's copy of the conversation is read inline from the +agent once the turn ends and stored as ``.json`` in the run's folder for that agent, overwritten each +turn, with the turn's aux log and turn context beside it and the media it names pulled into its ``media/``. It's +checked as a producer's document and the turn records how it checked. The reads run one after another, off the turn's +path, so the next message isn't held for them; a copy that can't be read or stored never fails the run.""" +from __future__ import annotations + +import asyncio +import importlib.metadata +import json +import logging +import mimetypes +import re +import uuid +from collections.abc import Mapping +from typing import Any + +import httpx +from agentenv_protocol.a2a_agent import TrajectoryMediaResponse +from agentenv_protocol.a2a_agent.extensions import MAX_MEDIA_OBJECTS +from agentenv_protocol.atif import ( + FORMAT as ATIF, + MEDIA_PATH, + PROFILE_VERSION, + SCHEMA_VERSION, + Finding, + errors_in, + media_paths, + profile_of, + validate, +) +from agentenv_protocol.atif.validator import ERROR + +from agent_env.a2a_agent import A2AAgent +from agent_env.a2a_agent.object_transfer import ( + DEFAULT_TRAJECTORY_MAX_BYTES, + TRANSFER_TIMEOUT_SECONDS, + TrajectoryFormats, + TransferCall, + invoke_transfer, + write_object, +) +from agent_env.a2a_agent.staging import transfer_store +from agent_env.config import get_config +from agent_env.store.object_store import ObjectStore +from agent_env.store.object_store.object_store import issues_grants_to +from agent_env.task_step.snapshot_utils.agent_state_capture import trajectory_prefix_key +from agent_env.task_step.context import AtifRecord, DeployedAgent, TaskStepContext + +logger = logging.getLogger(__name__) + +#: Where a run records its atif/1 folders: ``{"run": , "agents": {: {"folder", "sessions"}}}``, each +#: session (A2A context) naming the document its last copy was stored as. +ATIF_COPIES_KEY = "atif_copies" +VALID, INVALID, NEWER, UNAVAILABLE = "valid", "invalid", "newer", "unavailable" +#: A trajectory id the worker stores a document under: the agent mints it, so it must be a plain file name. +_DOCUMENT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") +_VERSION = re.compile(r"(\d+)\.(\d+)") +_VALIDATOR_VERSION = importlib.metadata.version("agentenv-framework-protocol") + + +def atif_folder(context: TaskStepContext, agent_name: str) -> str: + """The run's folder for ``agent_name``'s atif/1 copies, ``{prefix}atif/instance_id={id}/{agent_name}/``, recorded + in ``metadata["atif_copies"]``; a run with no instance id gets one ``adhoc-`` for all its agents.""" + copies = context.metadata.setdefault(ATIF_COPIES_KEY, {}) + agent = copies.setdefault("agents", {}).setdefault(agent_name, {"sessions": {}}) + if "folder" not in agent: + run = copies.setdefault("run", context.instance_id or f"adhoc-{uuid.uuid4().hex[:12]}") + config = get_config() + folder = config.get_object_store().object_url( + f"{config.get_artifact_key_prefix()}atif/instance_id={run}/{agent_name}/") + agent["folder"] = folder if folder.endswith("/") else f"{folder}/" + return agent["folder"] + + +class _Unavailable(Exception): + def __init__(self, reason: str) -> None: + super().__init__(reason) + self.reason = reason + + +class AtifCopies: + """A step's atif/1 copies of one agent's conversation: ``turn`` starts each ended turn's, and ``records`` waits + for them all. A run that doesn't prescribe atif/1 records None for each turn; one that prescribes it to an agent + that doesn't offer it records ``unavailable``, ``format_not_offered``.""" + + def __init__(self, context: TaskStepContext, agent: DeployedAgent, a2a_url: str, + extension: Mapping[str, Any] | None, *, prescribed: bool) -> None: + self._context, self._agent, self._a2a_url, self._extension = context, agent, a2a_url, extension + self._prescribed = prescribed + self._offered = ATIF in TrajectoryFormats.of(extension).formats + self._turns: list[asyncio.Task[AtifRecord] | AtifRecord | None] = [] + self._last: asyncio.Task[AtifRecord] | None = None + #: The media paths the folder holds, listed once and kept as this step pulls more. + self._held: set[str] | None = None + + def turn(self, task_id: str, name: str) -> None: + """Copy the ended task ``task_id``'s atif/1 document, its aux log and turn context stored under ``name``.""" + if not self._prescribed: + self._turns.append(None) + elif not self._offered: + self._turns.append(_unavailable("format_not_offered")) + else: + self._last = asyncio.create_task(self._copy(task_id, name, self._last)) + self._turns.append(self._last) + + async def records(self) -> list[AtifRecord | None]: + """Each turn's record, once every copy has been stored; cancelled while waiting, it cancels the copies left.""" + try: + return [await turn if isinstance(turn, asyncio.Task) else turn for turn in self._turns] + except BaseException: + self.cancel() + raise + + def cancel(self) -> None: + for turn in self._turns: + if isinstance(turn, asyncio.Task): + turn.cancel() + + async def _copy(self, task_id: str, name: str, previous: asyncio.Task[AtifRecord] | None) -> AtifRecord: + if previous is not None: # each copy overwrites the one before, so they're stored in turn order + await asyncio.wait([previous]) + try: + return await self._stored(task_id, name) + except Exception as exc: + reason = _reason(exc) + logger.warning(f"agent '{self._agent.agent_name}': the atif/1 copy of task {task_id} is unavailable " + f"({reason}; continuing): {exc}") + return _unavailable(reason) + + async def _stored(self, task_id: str, name: str) -> AtifRecord: + _, path = A2AAgent.operation(self._extension, "get") + body = await invoke_transfer( + self._a2a_url + path, TransferCall("legacy", {"task_id": task_id, "format": ATIF}), verb="POST", + operation="atif/1 trajectory get", timeout=TRANSFER_TIMEOUT_SECONDS, headers=self._agent.control_headers) + document = body.get("trajectory") if isinstance(body, Mapping) else None + if not isinstance(document, Mapping): + raise _Unavailable("no_document") + trajectory_id = document.get("trajectory_id") + if not isinstance(trajectory_id, str) or not _DOCUMENT_ID.fullmatch(trajectory_id): + raise _Unavailable("invalid_trajectory_id") + store = get_config().get_object_store() + folder = trajectory_prefix_key(store, atif_folder(self._context, self._agent.agent_name)) + url = await asyncio.to_thread(store.put, f"{folder}{trajectory_id}.json", _json(document), + content_type="application/json", allow_overwrite=True) + aux_url = turn_context_url = None + if body.get("aux") is not None: + aux_url = await asyncio.to_thread(store.put, f"{folder}turns/{name}/aux.json", _json(body["aux"]), + content_type="application/json", allow_overwrite=True) + if body.get("turn_context") is not None: + turn_context_url = await asyncio.to_thread( + store.put, f"{folder}turns/{name}/turn_context.json", _json(body["turn_context"]), + content_type="application/json", allow_overwrite=True) + session_id = document.get("session_id") + if isinstance(session_id, str): + await self._link(store, folder, session_id, trajectory_id) + held = await self._pull_media(document, session_id, store, folder) + root = profile_of(document) + newer = _newer(document.get("schema_version"), SCHEMA_VERSION) or _newer(root.get("version"), PROFILE_VERSION) + errors = errors_in(await asyncio.to_thread(_findings, document, held, newer)) + status = NEWER if newer else INVALID if errors else VALID + if status == INVALID: + logger.warning(f"agent '{self._agent.agent_name}': the atif/1 copy of task {task_id} is invalid " + f"(continuing): {errors[0]}") + steps = document.get("steps") + last = steps[-1] if isinstance(steps, list) and steps and isinstance(steps[-1], Mapping) else {} + adapter = root.get("adapter") + return AtifRecord( + url=url, through_step=last.get("step_id") if isinstance(last.get("step_id"), int) else None, + status=status, rules=sorted({finding.rule for finding in errors}), + reason=str(errors[0])[:300] if errors else None, validator_version=_VALIDATOR_VERSION, + adapter_version=adapter.get("version") if isinstance(adapter, Mapping) else None, + aux_url=aux_url, turn_context_url=turn_context_url, + ) + + async def _link(self, store: ObjectStore, folder: str, session_id: str, trajectory_id: str) -> None: + """Point the session's earlier document, when a restart began this one, to this one: the agent that wrote + the earlier one may be gone, so the worker sets its ``continued_trajectory_ref``.""" + sessions = self._context.metadata[ATIF_COPIES_KEY]["agents"][self._agent.agent_name]["sessions"] + earlier = sessions.get(session_id) + sessions[session_id] = trajectory_id + if earlier is None or earlier == trajectory_id: + return + key = f"{folder}{earlier}.json" + document = json.loads(await asyncio.to_thread(store.read, key)) + document["continued_trajectory_ref"] = f"{trajectory_id}.json" + await asyncio.to_thread(store.put, key, _json(document), content_type="application/json", allow_overwrite=True) + + async def _pull_media(self, document: Mapping[str, Any], context_id: Any, store: ObjectStore, + folder: str) -> set[str]: + """The media paths the folder holds once the agent has pushed what ``document`` names and the folder lacks. + A file the agent answers missing or failed for may be stored already, by a push whose answer was lost, so the + store is asked before it's counted missing.""" + if self._held is None: + listed = await asyncio.to_thread(store.list, f"{folder}media/") + self._held = {key[len(folder):] for key in listed} + wanted = [path for path in media_paths(document) if MEDIA_PATH.match(path) and path not in self._held] + method, endpoint = A2AAgent.operation(self._extension, "media") + card, sandbox_type = self._agent.a2a_card, self._agent.sandbox_type + if not wanted or method is None or endpoint is None or ( + transfer_store(store, self._a2a_url, card, sandbox_type=sandbox_type) is store + and not issues_grants_to(store, sandbox_type)): + return self._held + infos = _media_infos(document) + cap = store.max_single_upload_bytes + wanted = [path for path in wanted if cap is None or (infos.get(path, {}).get("bytes") or 0) <= cap] + unconfirmed: list[str] = [] + for start in range(0, len(wanted), MAX_MEDIA_OBJECTS): + batch = wanted[start:start + MAX_MEDIA_OBJECTS] + grants = transfer_store(store, self._a2a_url, card, sandbox_type=sandbox_type) # one per call + objects = { + path: write_object(grants, grants.object_url(f"{folder}{path}"), media_type=_media_type(path, infos), + max_bytes=max(infos.get(path, {}).get("bytes") or DEFAULT_TRAJECTORY_MAX_BYTES, 1)) + .model_dump(mode="json") + for path in batch + } + try: + pushed = await invoke_transfer( + self._a2a_url + endpoint, TransferCall("objects", {"context_id": context_id, "objects": objects}), + verb="POST", operation="trajectory media", timeout=TRANSFER_TIMEOUT_SECONDS, + response_model=TrajectoryMediaResponse, store=grants, headers=self._agent.control_headers) + except Exception as exc: + logger.warning(f"agent '{self._agent.agent_name}': the media push of {len(batch)} files failed " + f"(continuing): {exc}") + unconfirmed += batch + continue + self._held.update(pushed.objects) + unconfirmed += [*pushed.missing, *pushed.failed] + for path in unconfirmed: + if await asyncio.to_thread(store.exists, f"{folder}{path}"): + self._held.add(path) + return self._held + + +def _findings(document: Mapping[str, Any], held: set[str], newer: bool) -> list[Finding]: + """The validator's findings, a reader's for a document newer than this worker's, with each media file the folder + doesn't hold as ``media-bytes``: the agent checked each file it pushed against its name, so held files aren't read + back.""" + found = validate(document, reader=newer) + found += [Finding("media-bytes", ERROR, "", f"{path} isn't stored beside the document") + for path in media_paths(document) if MEDIA_PATH.match(path) and path not in held] + return found + + +def _media_infos(document: Mapping[str, Any]) -> dict[str, dict[str, Any]]: + """Each media path's profile entry, with the media type of the part that names it when the entry has none.""" + infos: dict[str, dict[str, Any]] = {} + for trajectory in _trajectories(document): + for step in trajectory.get("steps") or []: + for holder in [step, *((step.get("observation") or {}).get("results") or [])]: + for path, info in (profile_of(holder).get("media") or {}).items(): + infos.setdefault(path, dict(info) if isinstance(info, Mapping) else {}) + content = holder.get("message") if holder is step else holder.get("content") + for part in content if isinstance(content, list) else []: + source = part.get("source") if isinstance(part, Mapping) else None + if isinstance(source, Mapping) and isinstance(source.get("path"), str): + info = infos.setdefault(source["path"], {}) + info.setdefault("media_type", source.get("media_type")) + return infos + + +def _trajectories(document: Mapping[str, Any]) -> list[Mapping[str, Any]]: + return [document, *(sub for nested in document.get("subagent_trajectories") or [] for sub in _trajectories(nested))] + + +def _media_type(path: str, infos: Mapping[str, Mapping[str, Any]]) -> str: + return infos.get(path, {}).get("media_type") or mimetypes.guess_type(path)[0] or "application/octet-stream" + + +def _newer(version: Any, ours: str) -> bool: + got, have = _VERSION.search(str(version or "")), _VERSION.search(ours) + return got is not None and (int(got[1]), int(got[2])) > (int(have[1]), int(have[2])) + + +def _json(value: Any) -> bytes: + return json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() + + +def _reason(exc: Exception) -> str: + if isinstance(exc, _Unavailable): + return exc.reason + if isinstance(exc, httpx.HTTPStatusError): + try: + code = (exc.response.json().get("error") or {}).get("code") + except (ValueError, AttributeError): + code = None + return code if isinstance(code, str) else f"http_{exc.response.status_code}" + if isinstance(exc, httpx.TimeoutException): + return "timeout" + if isinstance(exc, httpx.TransportError): + return "unreachable" + return type(exc).__name__ + + +def _unavailable(reason: str) -> AtifRecord: + return AtifRecord(url=None, through_step=None, status=UNAVAILABLE, rules=[], reason=reason, validator_version=None, + adapter_version=None, aux_url=None, turn_context_url=None) diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index b2de5b964..5d52eb7c9 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -15,6 +15,7 @@ import httpx from a2a.types import TaskState from agentenv_protocol.a2a_agent import card_request_accepts +from agentenv_protocol.atif import FORMAT as ATIF_FORMAT from agent_env.a2a_agent import A2AAgent, conversation_store from agent_env.a2a_agent import protocol @@ -62,6 +63,7 @@ trajectory_object_url, trajectory_prefix_key, ) +from agent_env.task_step.snapshot_utils.atif_copies import AtifCopies from agent_env.task_step.snapshot_utils.live_trajectory import LiveTurns from agent_env.task_step.snapshot_utils.snapshot_series import SnapshotConfig, SnapshotSeries @@ -645,6 +647,8 @@ async def _converse( source_agent_per_turn_prompt_parts: list[list[dict] | None] = [] traj_ext_cached = A2AAgent.find_extension(card, A2AAgent.EXT_TRAJECTORY) formats = TrajectoryFormats.of(traj_ext_cached) + atif_copies = AtifCopies(context, agent, target_url, traj_ext_cached, + prescribed=ATIF_FORMAT in prescribed_formats(context)) live = self._live_trajectory_target(context, card, target_url, trajectory_output_prefix) live_turns = ( LiveTurns(*live, poll_interval_seconds=self.poll_interval_seconds, headers=agent.control_headers, @@ -732,6 +736,7 @@ def record_turn() -> None: logger.warning(f"Turn {turn+1} trajectory fetch failed (continuing): {e}") per_turn_trajectory_urls.append(stored.url) per_turn_trajectories.append(TurnTrajectory(native_format=stored.format, atif=None)) + atif_copies.turn(sent_task_id, target_a2a_task_id) final_task_id = sent_task_id final_state = target_state @@ -786,10 +791,15 @@ def record_turn() -> None: logger.info(f"Turn {turn+1}: nothing to send the agent; ending multi-turn (nothing_to_send)") conversation_store.mark_closed(conversation_id) break + except BaseException: + atif_copies.cancel() + raise finally: # On failure too, so a reader of the last turn knows no other follows. if live_turns is not None: await live_turns.end() + for turn_record, atif in zip(per_turn_trajectories, await atif_copies.records()): + turn_record["atif"] = atif await self._persist_env_trigger_state(context) diff --git a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py index 3216fd1e6..6ca74353c 100644 --- a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py +++ b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py @@ -174,14 +174,18 @@ async def test_each_turn_is_stored_live_under_its_own_prefix_and_joins_to_its_fi @pytest.mark.asyncio async def test_a_deployed_agent_serves_its_native_format_and_atif_and_refuses_another(local_registry): + """And the run, which prescribes atif/1, stores the turn's copy in its folder with the image's file beside it.""" suffix = uuid.uuid4().hex[:8] native = "agentenv-echo-agent/v1" + image = _png(6, 4) agent = put_test_agent(f"trajectory-formats-agent-{suffix}") task = Task.put(id=f"trajectory-formats-{suffix}", steps=[ _deploy(agent, "solver"), PromptAgentTaskStep( - id="ask", version=None, prompt_id="ask", agent_name="solver", prompt="hello", - context_id=f"formats-{suffix}", + id="ask", version=None, prompt_id="ask", agent_name="solver", context_id=f"formats-{suffix}", + parts=[{"kind": "text", "text": "hello"}, + {"kind": "file", "file": {"bytes": base64.b64encode(image).decode(), "mimeType": "image/png", + "name": "shot.png"}}], ), ]) @@ -204,11 +208,23 @@ async def test_a_deployed_agent_serves_its_native_format_and_atif_and_refuses_an document = atif["trajectory"] assert (atif["format"], atif["schema_version"], atif["profile_version"]) == (ATIF, SCHEMA_VERSION, PROFILE_VERSION) assert errors_in(validate(document)) == [] - assert [step["message"] for step in document["steps"]] == ["hello", "Echo: hello"] + [shot] = media_paths(document) + assert document["steps"][0]["message"] == [{"type": "text", "text": "hello"}, + {"type": "image", "source": {"media_type": "image/png", "path": shot}}] + assert document["steps"][1]["message"].startswith("Echo: hello") assert (document["session_id"], atif["turn_context"]["task_id"]) == (f"formats-{suffix}", page["task_id"]) assert cursor["events"] == document["steps"] and cursor["header"]["trajectory_id"] == document["trajectory_id"] with pytest.raises(httpx.HTTPStatusError, match="HTTP 400: format_not_offered"): raise_for_extension_status(refused, operation="trajectory read") + [turn] = context.prompt_responses[-1].target_agent_per_turn_trajectories + record, store = turn["atif"], get_config().get_object_store() + folder = context.metadata["atif_copies"]["agents"]["solver"]["folder"] + assert (turn["native_format"], record["status"], record["rules"]) == (native, "valid", []) + assert record["url"] == f"{folder}{document['trajectory_id']}.json" + assert json.loads(store.get(record["url"])) == document + assert json.loads(store.get(record["turn_context_url"])) == atif["turn_context"] + assert store.get(f"{folder}{shot}") == image + assert errors_in(validate(document, media={shot: store.get(f"{folder}{shot}")})) == [] @pytest.mark.asyncio diff --git a/tst/unit/task_step/test_prompt_agent_atif_copies.py b/tst/unit/task_step/test_prompt_agent_atif_copies.py new file mode 100644 index 000000000..21b5bfae2 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_atif_copies.py @@ -0,0 +1,393 @@ +"""A prompt step stores each turn's atif/1 copy when the run prescribes atif/1: read inline from the agent once the turn +ends, stored as ``.json`` in the run's folder for the agent with its aux log, turn context and media, +checked, and recorded per turn beside the native copy, off the turn's path. Against a live SDK agent served in-process, +whose uploads land in a granting local store.""" +from __future__ import annotations + +import asyncio +import base64 +import json +import struct +import zlib +from pathlib import Path +from typing import Any, Callable +from urllib.parse import unquote + +import httpx +import pytest +from agentenv_protocol.a2a_agent import ( + TRAJECTORY_V1, + AgentEnvAgent, + AgentIdentity, + TaskRequest, + TaskResult, + a2a_agent, + enable, +) +from agentenv_protocol.atif import FORMAT as ATIF, Adapter, atif_adapter, errors_in, media_paths, native, validate + +from agent_env.config import configure +from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.snapshot_utils import atif_copies +from agent_env.task_step.task_steps import prompt_agent as pa +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.util.granting_object_store import GRANT_ORIGIN, GrantingObjectStore + +NATIVE = "shots/1" +_PREFIX = "prompt_agent_trajectories/prompt_id=solve/" + + +def _png(width: int, height: int) -> bytes: + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", pixels) + chunk(b"IEND", b"") + + +@atif_adapter(native_format=NATIVE, version="2.1.0", input_includes_cache=False) +class _Shots(Adapter): + """Each native shot a step: its caption and its screenshot.""" + + @native(type="shot") + def shot(self, record: dict, turn) -> None: + image = turn.media(base64.b64decode(record["png"]), "image/png") + turn.message(step=f"s{record['n']}", content=[f"shot {record['n']}", image]) + turn.step(key=f"s{record['n']}", llm_call_count=1) + + +def _app(*, atif: bool = True): + """An agent that, each turn, appends one shot, a new screenshot each time, and answers.""" + shots = iter(range(1, 1000)) + + @a2a_agent( + identity=AgentIdentity(name="solver", description="test", version="1"), + extensions=(enable(TRAJECTORY_V1, live=False, native=NATIVE, **({"atif": _Shots} if atif else {})),), + ) + class Agent(AgentEnvAgent): + async def run(self, request: TaskRequest) -> TaskResult: + request.trajectory.set_format(NATIVE) + n = next(shots) + request.trajectory.append({"type": "shot", "n": n, "png": base64.b64encode(_png(n, 1)).decode()}) + return TaskResult.text(f"shot {n}") + + return Agent().create_app() + + +class _Wire(httpx.AsyncBaseTransport): + """The agent at ``http://agent.test``, served in-process, and the store's grant URLs, which write into the store. + A test can hold the agent's atif/1 reads until it lets them go, rewrite their answers, or rewrite any answer.""" + + def __init__(self, store: GrantingObjectStore, *, atif: bool) -> None: + self.store = store + self._atif = atif + self.restart() + self.atif_reads: list[dict] = [] + self.media_requests: list[dict] = [] + self.prescriptions: list[dict] = [] + self.hold: asyncio.Event | None = None + #: Set once an atif/1 read has arrived, whether or not it's held. + self.read_arrived = asyncio.Event() + self.answered_atif_reads = 0 + self.rewrite_atif: Callable[[dict], dict] | None = None + self.answer: Callable[[httpx.Request, str, dict, httpx.Response], httpx.Response | None] | None = None + #: Paths the agent never receives a request for: each is answered 503. + self.unreachable: set[str] = set() + + def restart(self) -> None: + self.app = _app(atif=self._atif) + self._agent = httpx.ASGITransport(app=self.app) + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + body = await request.aread() + if request.url.host == httpx.URL(GRANT_ORIGIN).host: + self.store.put(unquote(request.url.path.lstrip("/")), body, content_type=request.headers["content-type"], + allow_overwrite=True) + return httpx.Response(200, request=request) + sent = json.loads(body) if body and request.method == "POST" else {} + path = request.url.path + if path in self.unreachable: + return httpx.Response(503, request=request) + atif_read = path == "/ext/trajectory" and sent.get("format") == ATIF and "after" not in sent + if atif_read: + self.atif_reads.append(sent) + self.read_arrived.set() + if self.hold is not None: + await self.hold.wait() + elif path == "/ext/trajectory/media": + self.media_requests.append(sent) + elif path == "/ext/trajectory/prescribe": + self.prescriptions.append(sent) + response = await self._agent.handle_async_request(request) + self.answered_atif_reads += atif_read + if atif_read and self.rewrite_atif is not None and response.status_code == 200: + await response.aread() + response = httpx.Response(200, json=self.rewrite_atif(json.loads(response.content)), request=request) + if self.answer is not None: + await response.aread() + response = self.answer(request, path, sent, response) or response + return response + + +class _Run: + def __init__(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, atif: bool = True) -> None: + monkeypatch.setenv("XDG_STATE_HOME", str(tmp_path / "state")) + self.root = tmp_path / "store" + self.store = GrantingObjectStore(str(self.root), reaches=True) + configure(object_store=self.store) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa.conversation_store, fn, lambda *a, **kw: None) + self.wire = _Wire(self.store, atif=atif) + self.asked: list[int] = [] + real_client = httpx.AsyncClient + monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: real_client( + *a, **{**kw, "transport": self.wire})) + + async def ask(user_side, number: int, reply: str, parts: list[dict]) -> dict: + self.asked.append(number) + return {"message": [{"kind": "text", "text": f"again {number}"}]} + + monkeypatch.setattr(pa._UserSide, "ask", ask) + + @property + def card(self) -> dict: + return self.wire.app.state.agentenv_a2a.card.model_dump(mode="json", exclude_none=True) + + def context(self, formats: list[str] | None = (ATIF,), **kwargs) -> TaskStepContext: + context = TaskStepContext(instance_id="ti-1", **kwargs) + if formats is not None: + context.metadata["user_overrides"] = {"trajectory_formats": list(formats)} + return context + + async def execute(self, context: TaskStepContext, *, card: dict | None = None, **step) -> PromptResponse: + context.deployed_agents[:] = [DeployedAgent(agent_name="solver", api_url="http://agent.test", + a2a_url="http://agent.test", a2a_card=card or self.card)] + step.setdefault("max_conversation_turns", 2) + step.setdefault("user_a2a_url", "http://user.test") # whose answers ``ask`` stands in for + await PromptAgentTaskStep(id="solve", version=None, prompt="go", agent_name="solver", poll_interval_seconds=0, + trajectory_output_prefix=self.store.object_url(_PREFIX), **step).execute(context) + return context.prompt_responses[-1] + + def read(self, url: str) -> Any: + return json.loads(self.store.get(url)) + + def folder(self, context: TaskStepContext) -> Path: + return self.root / self.store.get_object_key(context.metadata["atif_copies"]["agents"]["solver"]["folder"]) + + +def _atifs(response: PromptResponse) -> list[dict | None]: + return [turn["atif"] for turn in response.target_agent_per_turn_trajectories] + + +@pytest.mark.asyncio +async def test_a_run_that_prescribes_atif_stores_each_turns_copy_with_its_media_aux_log_and_turn_context( + monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + context = run.context() + + response = await run.execute(context) + + first, second = _atifs(response) + folder = context.metadata["atif_copies"]["agents"]["solver"]["folder"] + assert context.metadata["atif_copies"]["run"] == "ti-1" + assert folder == f"{run.store.object_url('atif/instance_id=ti-1/solver')}/" + document = run.read(second["url"]) + assert first["url"] == second["url"] == f"{folder}{document['trajectory_id']}.json" + assert [first["status"], second["status"]] == ["valid", "valid"] + assert first["rules"] == second["rules"] == [] and first["reason"] is None + assert first["adapter_version"] == "2.1.0" and first["validator_version"] == atif_copies._VALIDATOR_VERSION + assert first["through_step"] < second["through_step"] == document["steps"][-1]["step_id"] + assert document["session_id"] == response.a2a_context_id + assert len(media_paths(document)) == 2 + assert errors_in(validate(document, media=run.folder(context))) == [] + for record in (first, second): + assert record["aux_url"].startswith(f"{folder}turns/") and run.read(record["aux_url"]) == [] + assert run.read(record["turn_context_url"])["state"] == "completed" + assert [read["format"] for read in run.wire.atif_reads] == [ATIF, ATIF] + native_urls = response.target_agent_per_turn_trajectory_object_urls + assert [run.read(url)[0]["type"] for url in native_urls] == ["shot", "shot"] + assert [turn["native_format"] for turn in response.target_agent_per_turn_trajectories] == [NATIVE, NATIVE] + + +@pytest.mark.asyncio +async def test_a_run_that_prescribes_nothing_reads_and_stores_native_only(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + context = run.context(formats=None) + + response = await run.execute(context) + + assert _atifs(response) == [None, None] + assert run.wire.atif_reads == [] and run.wire.media_requests == [] + assert "atif_copies" not in context.metadata and not run.store.list("atif/") + + +@pytest.mark.asyncio +async def test_atif_prescribed_to_an_agent_that_doesnt_offer_it_is_recorded_unavailable(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path, atif=False) + + response = await run.execute(run.context()) + + assert [(atif["status"], atif["reason"]) for atif in _atifs(response)] == [ + ("unavailable", "format_not_offered")] * 2 + assert run.wire.atif_reads == [] + + +@pytest.mark.asyncio +async def test_a_read_the_agent_refuses_as_not_prescribed_is_recorded_unavailable_and_the_run_succeeds( + monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + run.wire.unreachable.add("/ext/trajectory/prescribe") + + response = await run.execute(run.context()) + + assert [(atif["status"], atif["reason"]) for atif in _atifs(response)] == [ + ("unavailable", "format_not_prescribed")] * 2 + assert response.error_type is None + + +@pytest.mark.asyncio +async def test_a_copy_that_fails_validation_is_stored_and_recorded_invalid_with_its_rules(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + + def stray_key(answer: dict) -> dict: + answer["trajectory"]["extra"]["elsewhere"] = {"x": 1} + return answer + + run.wire.rewrite_atif = stray_key + + response = await run.execute(run.context()) + + [first, second] = _atifs(response) + assert first["status"] == second["status"] == "invalid" + assert first["rules"] == ["outside-key"] and "outside-key" in first["reason"] + assert run.read(second["url"])["extra"]["elsewhere"] == {"x": 1} + assert response.error_type is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field, version", [("schema_version", "ATIF-v1.9"), ("profile", "1.1")]) +async def test_a_copy_newer_than_the_worker_is_stored_and_recorded_newer(monkeypatch, tmp_path, field, version): + run = _Run(monkeypatch, tmp_path) + + def newer(answer: dict) -> dict: + if field == "schema_version": + answer["trajectory"]["schema_version"] = version + else: + answer["trajectory"]["extra"]["agentenv"]["version"] = version + return answer + + run.wire.rewrite_atif = newer + + response = await run.execute(run.context()) + + assert [atif["status"] for atif in _atifs(response)] == ["newer", "newer"] + assert run.read(_atifs(response)[-1]["url"])["schema_version"] == ( + version if field == "schema_version" else "ATIF-v1.8") + + +@pytest.mark.asyncio +async def test_media_the_agent_cant_be_asked_for_makes_the_copy_invalid_under_media_bytes(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + card = run.card + trajectory = next(e for e in card["capabilities"]["extensions"] if e["uri"] == TRAJECTORY_V1.uri) + del trajectory["params"]["methods"]["media"] + + response = await run.execute(run.context(), card=card) + + assert [(atif["status"], atif["rules"]) for atif in _atifs(response)] == [("invalid", ["media-bytes"])] * 2 + assert run.wire.media_requests == [] + + +@pytest.mark.asyncio +async def test_a_file_whose_push_answer_was_lost_is_found_in_the_store_and_not_asked_for_again(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + lost: list[str] = [] + + def lose_first_push(request: httpx.Request, path: str, sent: dict, + response: httpx.Response) -> httpx.Response | None: + if path == "/ext/trajectory/media" and not lost: + lost.extend(sent["objects"]) + raise httpx.ReadTimeout("the answer was lost", request=request) + return None + + run.wire.answer = lose_first_push + context = run.context() + + response = await run.execute(context) + + assert [atif["status"] for atif in _atifs(response)] == ["valid", "valid"] + [first_shot] = lost + assert [list(request["objects"]) for request in run.wire.media_requests] == [ + [first_shot], [path for path in media_paths(run.read(_atifs(response)[-1]["url"])) if path != first_shot]] + + run.wire.media_requests.clear() + await run.execute(context, max_conversation_turns=1, context_id=None) + assert all(first_shot not in request["objects"] for request in run.wire.media_requests) + + +@pytest.mark.asyncio +async def test_a_restart_between_steps_links_the_sessions_earlier_document_to_the_next(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + context = run.context() + before = await run.execute(context, context_id="talk", max_conversation_turns=1) + run.wire.restart() + + after = await run.execute(context, context_id="talk", max_conversation_turns=1) + + earlier, later = run.read(_atifs(before)[0]["url"]), run.read(_atifs(after)[0]["url"]) + assert earlier["session_id"] == later["session_id"] == "talk" + assert earlier["trajectory_id"] != later["trajectory_id"] + assert earlier["continued_trajectory_ref"] == f"{later['trajectory_id']}.json" + assert "continued_trajectory_ref" not in later + assert errors_in(validate(earlier, media=run.folder(context))) == [] + assert context.metadata["atif_copies"]["agents"]["solver"]["sessions"] == {"talk": later["trajectory_id"]} + + +@pytest.mark.asyncio +async def test_the_user_sides_next_message_goes_out_before_the_turns_atif_copy_is_read(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + run.wire.hold = asyncio.Event() + real_ask = pa._UserSide.ask + + async def ask(user_side, number: int, reply: str, parts: list[dict]) -> dict: + await asyncio.wait_for(run.wire.read_arrived.wait(), 10) + assert run.wire.answered_atif_reads == 0 # the turn's read is still held, and the user side is asked anyway + run.wire.hold.set() + return await real_ask(user_side, number, reply, parts) + + monkeypatch.setattr(pa._UserSide, "ask", ask) + + response = await asyncio.wait_for(run.execute(run.context()), 30) + + assert run.asked == [1] and [atif["status"] for atif in _atifs(response)] == ["valid", "valid"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("trajectory_id", ["../elsewhere", "a/b", "", None]) +async def test_a_trajectory_id_that_isnt_a_plain_file_name_is_refused(monkeypatch, tmp_path, trajectory_id): + run = _Run(monkeypatch, tmp_path) + + def renamed(answer: dict) -> dict: + answer["trajectory"]["trajectory_id"] = trajectory_id + return answer + + run.wire.rewrite_atif = renamed + + response = await run.execute(run.context(), max_conversation_turns=1) + + assert [(atif["status"], atif["reason"]) for atif in _atifs(response)] == [ + ("unavailable", "invalid_trajectory_id")] + assert not run.store.list("atif/") + + +def test_a_run_without_an_instance_id_keeps_one_folder_run_for_all_its_agents(tmp_path): + configure(object_store=GrantingObjectStore(str(tmp_path))) + context = TaskStepContext() + + solver, judge = atif_copies.atif_folder(context, "solver"), atif_copies.atif_folder(context, "judge") + + run = context.metadata["atif_copies"]["run"] + assert run.startswith("adhoc-") and len(run) == len("adhoc-") + 12 + assert solver.endswith(f"atif/instance_id={run}/solver/") and judge.endswith(f"atif/instance_id={run}/judge/") + assert atif_copies.atif_folder(context, "solver") == solver diff --git a/tst/unit/task_step/test_prompt_agent_live_trajectory.py b/tst/unit/task_step/test_prompt_agent_live_trajectory.py index 5fde34290..a495c0fe4 100644 --- a/tst/unit/task_step/test_prompt_agent_live_trajectory.py +++ b/tst/unit/task_step/test_prompt_agent_live_trajectory.py @@ -127,9 +127,10 @@ async def test_a_prompt_step_prescribes_the_runs_formats_before_its_first_messag assert run.prescriptions == [{"formats": formats or []}] assert run.sent.index("/ext/trajectory/prescribe") < run.sent.index("message/send") assert context.metadata["trajectory_formats"]["solve"]["built"] == ["test-events/1", *(formats or [])] - [final] = run.final_reads + assert [read["format"] for read in run.final_reads] == ["test-events/1", *(formats or [])] async with httpx.AsyncClient() as client: - read = await client.post("http://agent.test/ext/trajectory", json={"task_id": final["task_id"], "format": ATIF}) + read = await client.post("http://agent.test/ext/trajectory", + json={"task_id": run.final_reads[0]["task_id"], "format": ATIF}) assert read.status_code == (200 if formats else 409) diff --git a/tst/unit/task_step/verifiers/test_judge_inputs.py b/tst/unit/task_step/verifiers/test_judge_inputs.py index a1c11247d..fa0cc4f84 100644 --- a/tst/unit/task_step/verifiers/test_judge_inputs.py +++ b/tst/unit/task_step/verifiers/test_judge_inputs.py @@ -2,7 +2,7 @@ prefix), what an agent judge is given (its prompt and every file staged in its sandbox) and what a scripted verify is called with, for runs whose native trajectories are stored as a run stores them. Each is what its golden file holds, recorded before prompt steps kept per-turn trajectory records, and stays so byte for byte when the run's prompt -responses carry those records, read back from the stored form. Re-record after a deliberate change with +responses carry those records and its atif/1 copies are stored, read back from the stored form. Re-record after a deliberate change with ``AGENTENV_TEST_RECORD_GOLDEN=1`` and review the file's diff. Several cases build their runs with other tests' setups (the always_on rig, user_sim_judge's fakes, the rubrics golden test's and run_context's helpers), so a change to one of those is re-recorded here too.""" @@ -14,6 +14,7 @@ import json import re from collections.abc import Awaitable, Callable +from importlib.resources import files from pathlib import Path from types import SimpleNamespace from typing import Any @@ -24,7 +25,9 @@ from agent_env.config import get_config from agent_env.env.env import DeployedEnv from agent_env.providers.sandbox_providers import sandbox_provider -from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.context import AtifRecord, DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.snapshot_utils.agent_state_capture import trajectory_prefix_key +from agent_env.task_step.snapshot_utils.atif_copies import atif_folder from agent_env.task_step.task_steps.verifiers import always_on_agent_verifier from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.verifiers.judge_utils.trajectory_filter import TrajectoryFilter @@ -63,12 +66,25 @@ def _tool_span(n: int) -> dict: r"/[0-9a-f]{12}") +#: An atif/1 copy a run that prescribes it stores beside each native one. +_ATIF = {**json.loads((files("agentenv_protocol.atif") / "fixtures" / "two_turns.json").read_text()), + "trajectory_id": "copy-1"} + + def _recorded_form(context: TaskStepContext) -> TaskStepContext: - """``context`` as a run that kept per-turn trajectory records leaves it, read back from its stored form.""" + """``context`` as a run that prescribes atif/1 and keeps per-turn trajectory records leaves it, its atif/1 copies + stored in the run's folder, read back from its stored form.""" + store = get_config().get_object_store() for response in context.prompt_responses: turns = response.target_agent_per_turn_trajectory_object_urls or [response.agent_trajectory_object_url] + folder = trajectory_prefix_key(store, atif_folder(context, response.agent_name or "solver")) + url = store.put(f"{folder}{_ATIF['trajectory_id']}.json", json.dumps(_ATIF).encode(), + content_type="application/json", allow_overwrite=True) response.target_agent_per_turn_trajectories = [ - {"native_format": "claude-code-stream-json", "atif": None} for _ in turns] + {"native_format": "claude-code-stream-json", + "atif": AtifRecord(url=url, through_step=n + 1, status="valid", rules=[], reason=None, + validator_version="1.0", adapter_version="1.0", aux_url=None, turn_context_url=None)} + for n, _ in enumerate(turns)] return TaskStepContext.from_dict(context.to_dict()) From fc3114d2681f61af658135202d58de268fcf5a71 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 15:27:03 -0800 Subject: [PATCH 2/6] fix(task-steps): a failed atif/1 link is retried, a malformed copy is invalid, and media are checked against the profile The session names a new document only once the earlier one links to it, so a later turn retries a link that failed. A copy whose shape can't be walked is stored and recorded invalid by the validator's findings, rather than unavailable. A held media file whose profile sha256 or size disagrees with its name or the stored file is media-bytes. Co-Authored-By: Claude Opus 5.5 --- .../task_step/snapshot_utils/atif_copies.py | 112 ++++++++++++------ .../test_prompt_agent_atif_copies.py | 61 ++++++++++ 2 files changed, 134 insertions(+), 39 deletions(-) diff --git a/src/agent_env/task_step/snapshot_utils/atif_copies.py b/src/agent_env/task_step/snapshot_utils/atif_copies.py index 4e94ad174..105e16568 100644 --- a/src/agent_env/task_step/snapshot_utils/atif_copies.py +++ b/src/agent_env/task_step/snapshot_utils/atif_copies.py @@ -25,7 +25,7 @@ SCHEMA_VERSION, Finding, errors_in, - media_paths, + media_file, profile_of, validate, ) @@ -91,8 +91,9 @@ def __init__(self, context: TaskStepContext, agent: DeployedAgent, a2a_url: str, self._offered = ATIF in TrajectoryFormats.of(extension).formats self._turns: list[asyncio.Task[AtifRecord] | AtifRecord | None] = [] self._last: asyncio.Task[AtifRecord] | None = None - #: The media paths the folder holds, listed once and kept as this step pulls more. - self._held: set[str] | None = None + #: The media files the folder holds and their sizes (None until read), listed once and kept as this step pulls + #: more. + self._held: dict[str, int | None] | None = None def turn(self, task_id: str, name: str) -> None: """Copy the ended task ``task_id``'s atif/1 document, its aux log and turn context stored under ``name``.""" @@ -151,13 +152,20 @@ async def _stored(self, task_id: str, name: str) -> AtifRecord: turn_context_url = await asyncio.to_thread( store.put, f"{folder}turns/{name}/turn_context.json", _json(body["turn_context"]), content_type="application/json", allow_overwrite=True) - session_id = document.get("session_id") - if isinstance(session_id, str): - await self._link(store, folder, session_id, trajectory_id) - held = await self._pull_media(document, session_id, store, folder) root = profile_of(document) newer = _newer(document.get("schema_version"), SCHEMA_VERSION) or _newer(root.get("version"), PROFILE_VERSION) - errors = errors_in(await asyncio.to_thread(_findings, document, held, newer)) + found = await asyncio.to_thread(validate, document, reader=newer) + try: + infos = _media_infos(document) + except (AttributeError, TypeError): # a shape no ATIF has, which the validator reports; nothing to walk + infos = None + if infos is not None: + session_id = document.get("session_id") + if isinstance(session_id, str): + await self._link(store, folder, session_id, trajectory_id) + held = await self._pull_media(infos, session_id, store, folder) + found += _media_findings(infos, held) + errors = errors_in(found) status = NEWER if newer else INVALID if errors else VALID if status == INVALID: logger.warning(f"agent '{self._agent.agent_name}': the atif/1 copy of task {task_id} is invalid " @@ -175,33 +183,48 @@ async def _stored(self, task_id: str, name: str) -> AtifRecord: async def _link(self, store: ObjectStore, folder: str, session_id: str, trajectory_id: str) -> None: """Point the session's earlier document, when a restart began this one, to this one: the agent that wrote - the earlier one may be gone, so the worker sets its ``continued_trajectory_ref``.""" + the earlier one may be gone, so the worker sets its ``continued_trajectory_ref``. The session names this one + only once that's done, so a link that fails is made by a later turn.""" sessions = self._context.metadata[ATIF_COPIES_KEY]["agents"][self._agent.agent_name]["sessions"] earlier = sessions.get(session_id) + if earlier is not None and earlier != trajectory_id: + key = f"{folder}{earlier}.json" + try: + document = json.loads(await asyncio.to_thread(store.read, key)) + document["continued_trajectory_ref"] = f"{trajectory_id}.json" + await asyncio.to_thread(store.put, key, _json(document), content_type="application/json", + allow_overwrite=True) + except Exception as exc: + logger.warning(f"agent '{self._agent.agent_name}': linking {earlier} to {trajectory_id} failed, so a " + f"later turn tries again (continuing): {exc}") + return sessions[session_id] = trajectory_id - if earlier is None or earlier == trajectory_id: - return - key = f"{folder}{earlier}.json" - document = json.loads(await asyncio.to_thread(store.read, key)) - document["continued_trajectory_ref"] = f"{trajectory_id}.json" - await asyncio.to_thread(store.put, key, _json(document), content_type="application/json", allow_overwrite=True) - - async def _pull_media(self, document: Mapping[str, Any], context_id: Any, store: ObjectStore, - folder: str) -> set[str]: - """The media paths the folder holds once the agent has pushed what ``document`` names and the folder lacks. - A file the agent answers missing or failed for may be stored already, by a push whose answer was lost, so the - store is asked before it's counted missing.""" + + async def _pull_media(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, store: ObjectStore, + folder: str) -> dict[str, int | None]: + """The media files the folder holds, with their sizes, once the agent has pushed what the document names + (``infos``, by path) and the folder lacks. A file the agent answers missing or failed for may be stored already, by a push whose answer + was lost, so the store is asked before it's counted missing; a held file's size is read from the store once.""" if self._held is None: listed = await asyncio.to_thread(store.list, f"{folder}media/") - self._held = {key[len(folder):] for key in listed} - wanted = [path for path in media_paths(document) if MEDIA_PATH.match(path) and path not in self._held] + self._held = {key[len(folder):]: None for key in listed} + held = await self._pushed(infos, context_id, store, folder, self._held) + for path in infos: + if path in held and held[path] is None: + metadata = await asyncio.to_thread(store.get_object_metadata, f"{folder}{path}") + held[path] = None if metadata is None else metadata.size + return held + + async def _pushed(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, store: ObjectStore, folder: str, + held: dict[str, int | None]) -> dict[str, int | None]: + """``held``, with what the agent pushed of the files the document names that it lacks.""" + wanted = [path for path in infos if MEDIA_PATH.match(path) and path not in held] method, endpoint = A2AAgent.operation(self._extension, "media") card, sandbox_type = self._agent.a2a_card, self._agent.sandbox_type if not wanted or method is None or endpoint is None or ( transfer_store(store, self._a2a_url, card, sandbox_type=sandbox_type) is store and not issues_grants_to(store, sandbox_type)): - return self._held - infos = _media_infos(document) + return held cap = store.max_single_upload_bytes wanted = [path for path in wanted if cap is None or (infos.get(path, {}).get("bytes") or 0) <= cap] unconfirmed: list[str] = [] @@ -224,26 +247,37 @@ async def _pull_media(self, document: Mapping[str, Any], context_id: Any, store: f"(continuing): {exc}") unconfirmed += batch continue - self._held.update(pushed.objects) + held.update({path: uploaded.size_bytes for path, uploaded in pushed.objects.items()}) unconfirmed += [*pushed.missing, *pushed.failed] for path in unconfirmed: - if await asyncio.to_thread(store.exists, f"{folder}{path}"): - self._held.add(path) - return self._held - - -def _findings(document: Mapping[str, Any], held: set[str], newer: bool) -> list[Finding]: - """The validator's findings, a reader's for a document newer than this worker's, with each media file the folder - doesn't hold as ``media-bytes``: the agent checked each file it pushed against its name, so held files aren't read - back.""" - found = validate(document, reader=newer) - found += [Finding("media-bytes", ERROR, "", f"{path} isn't stored beside the document") - for path in media_paths(document) if MEDIA_PATH.match(path) and path not in held] + metadata = await asyncio.to_thread(store.get_object_metadata, f"{folder}{path}") + if metadata is not None: + held[path] = metadata.size + return held + + +def _media_findings(infos: Mapping[str, Mapping[str, Any]], held: Mapping[str, int | None]) -> list[Finding]: + """``media-bytes`` for each media file the folder doesn't hold as the profile describes it: missing, a profile + sha256 other than its name's, or a profile size other than the stored file's. The agent checked each file it pushed + against its name, before and after the upload, so held files aren't read back.""" + found: list[Finding] = [] + for path, info in infos.items(): + if not MEDIA_PATH.match(path): + continue + if path not in held: + found.append(Finding("media-bytes", ERROR, "", f"{path} isn't stored beside the document")) + continue + if info.get("sha256") is not None and info["sha256"] != media_file(path): + found.append(Finding("media-bytes", ERROR, "", f"{path}'s profile gives sha256 {info['sha256']}")) + if info.get("bytes") is not None and held[path] is not None and info["bytes"] != held[path]: + found.append(Finding("media-bytes", ERROR, "", + f"{path}'s profile gives {info['bytes']} bytes, and the stored file has {held[path]}")) return found def _media_infos(document: Mapping[str, Any]) -> dict[str, dict[str, Any]]: - """Each media path's profile entry, with the media type of the part that names it when the entry has none.""" + """Each media file the document names (``media_paths``), by path, with its profile entry and the media type of + the part that names it when the entry has none.""" infos: dict[str, dict[str, Any]] = {} for trajectory in _trajectories(document): for step in trajectory.get("steps") or []: diff --git a/tst/unit/task_step/test_prompt_agent_atif_copies.py b/tst/unit/task_step/test_prompt_agent_atif_copies.py index 21b5bfae2..ca01acf76 100644 --- a/tst/unit/task_step/test_prompt_agent_atif_copies.py +++ b/tst/unit/task_step/test_prompt_agent_atif_copies.py @@ -391,3 +391,64 @@ def test_a_run_without_an_instance_id_keeps_one_folder_run_for_all_its_agents(tm assert run.startswith("adhoc-") and len(run) == len("adhoc-") + 12 assert solver.endswith(f"atif/instance_id={run}/solver/") and judge.endswith(f"atif/instance_id={run}/judge/") assert atif_copies.atif_folder(context, "solver") == solver + + +@pytest.mark.asyncio +async def test_a_link_that_fails_is_made_by_a_later_turn_and_its_turn_keeps_its_copy(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + context = run.context() + before = await run.execute(context, context_id="talk", max_conversation_turns=1) + earlier_url = _atifs(before)[0]["url"] + run.wire.restart() + real_read, failed = run.store.read, [] + + def read_failing_once(key: str) -> bytes: + if key.endswith(earlier_url.rsplit("/", 1)[1]) and not failed: + failed.append(key) + raise OSError("the store blinked") + return real_read(key) + + monkeypatch.setattr(run.store, "read", read_failing_once) + + after = await run.execute(context, context_id="talk", max_conversation_turns=2) + + assert failed and [atif["status"] for atif in _atifs(after)] == ["valid", "valid"] + later = run.read(_atifs(after)[-1]["url"]) + assert run.read(earlier_url)["continued_trajectory_ref"] == f"{later['trajectory_id']}.json" + + +@pytest.mark.asyncio +async def test_a_copy_that_doesnt_read_as_atif_is_stored_and_recorded_invalid(monkeypatch, tmp_path): + run = _Run(monkeypatch, tmp_path) + + def broken(answer: dict) -> dict: + answer["trajectory"]["steps"] = [None] + return answer + + run.wire.rewrite_atif = broken + + response = await run.execute(run.context(), max_conversation_turns=1) + + [atif] = _atifs(response) + assert (atif["status"], atif["rules"], atif["through_step"]) == ("invalid", ["atif"], None) + assert run.read(atif["url"])["steps"] == [None] and run.wire.media_requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["bytes", "sha256"]) +async def test_a_profile_that_misdescribes_a_stored_file_makes_the_copy_invalid(monkeypatch, tmp_path, field): + run = _Run(monkeypatch, tmp_path) + + def misdescribed(answer: dict) -> dict: + for step in answer["trajectory"]["steps"]: + for info in (step.get("extra") or {}).get("agentenv", {}).get("media", {}).values(): + info[field] = info[field] + 1 if field == "bytes" else "0" * 64 + return answer + + run.wire.rewrite_atif = misdescribed + + response = await run.execute(run.context(), max_conversation_turns=1) + + [atif] = _atifs(response) + assert (atif["status"], atif["rules"]) == ("invalid", ["media-bytes"]) + assert ("bytes" if field == "bytes" else "sha256") in atif["reason"] From e76cb71ef205d4bd148618ddd2d377b4e21c5863 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 15:30:15 -0800 Subject: [PATCH 3/6] fix(a2a-agent): what an agent staged before its call failed reaches the store A staged call whose answer is lost or refused still moves the objects the agent wrote into the store, as an upload through a grant does, so the atif/1 media pull finds a file the agent has already let go of. Names the atif/1 record's reason limit. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/object_transfer.py | 38 +++++++++++++------ .../task_step/snapshot_utils/atif_copies.py | 4 +- tst/unit/a2a_agent/staging_test.py | 35 +++++++++++++++++ 3 files changed, 64 insertions(+), 13 deletions(-) diff --git a/src/agent_env/a2a_agent/object_transfer.py b/src/agent_env/a2a_agent/object_transfer.py index 5ae20a70d..bc1fb1035 100644 --- a/src/agent_env/a2a_agent/object_transfer.py +++ b/src/agent_env/a2a_agent/object_transfer.py @@ -190,19 +190,24 @@ async def invoke_transfer( try: if staged is not None: await staged.push() - async with httpx.AsyncClient() as client: - send = client.post if verb == "POST" else client.put - resp = await send(url, json=call.payload, headers=headers, timeout=timeout) try: - raise_for_extension_status(resp, operation=operation, include_body=call.mode == "legacy") - except httpx.HTTPStatusError as exc: - hint = _STAGING_UNREACHABLE if staged is not None else ( - unreachable_hint(call.payload) if call.mode == "objects" else None - ) - if hint is None or not str(exc).endswith(f": {_TRANSFER_UNAVAILABLE}"): - raise - raise httpx.HTTPStatusError(f"{exc}. {hint}", request=exc.request, response=exc.response) from exc - body = resp.json() + async with httpx.AsyncClient() as client: + send = client.post if verb == "POST" else client.put + resp = await send(url, json=call.payload, headers=headers, timeout=timeout) + try: + raise_for_extension_status(resp, operation=operation, include_body=call.mode == "legacy") + except httpx.HTTPStatusError as exc: + hint = _STAGING_UNREACHABLE if staged is not None else ( + unreachable_hint(call.payload) if call.mode == "objects" else None + ) + if hint is None or not str(exc).endswith(f": {_TRANSFER_UNAVAILABLE}"): + raise + raise httpx.HTTPStatusError(f"{exc}. {hint}", request=exc.request, response=exc.response) from exc + body = resp.json() + except Exception: + if staged is not None: + await _collect_staged(staged, operation) + raise if staged is not None: await staged.pull() finally: @@ -213,6 +218,15 @@ async def invoke_transfer( return body +async def _collect_staged(staged: StagedObjectStore, operation: str) -> None: + """Move what the agent wrote before its call failed into the store, as an upload through a grant reaches it + whether or not the answer does; best-effort, so the call's own error is the one raised.""" + try: + await staged.pull() + except Exception as exc: + logger.warning(f"{operation}: collecting what the agent staged before the call failed: {exc}") + + def parse_response(model: type[_Response], body: Any, *, operation: str) -> _Response: """Validate an agent's ``operation`` response, ignoring fields this release does not know.""" try: diff --git a/src/agent_env/task_step/snapshot_utils/atif_copies.py b/src/agent_env/task_step/snapshot_utils/atif_copies.py index 105e16568..6fe5ca0b0 100644 --- a/src/agent_env/task_step/snapshot_utils/atif_copies.py +++ b/src/agent_env/task_step/snapshot_utils/atif_copies.py @@ -56,6 +56,8 @@ #: A trajectory id the worker stores a document under: the agent mints it, so it must be a plain file name. _DOCUMENT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") _VERSION = re.compile(r"(\d+)\.(\d+)") +#: How much of an invalid copy's first finding its record keeps as the reason; ``rules`` names every rule broken. +_REASON_CHARS = 300 _VALIDATOR_VERSION = importlib.metadata.version("agentenv-framework-protocol") @@ -176,7 +178,7 @@ async def _stored(self, task_id: str, name: str) -> AtifRecord: return AtifRecord( url=url, through_step=last.get("step_id") if isinstance(last.get("step_id"), int) else None, status=status, rules=sorted({finding.rule for finding in errors}), - reason=str(errors[0])[:300] if errors else None, validator_version=_VALIDATOR_VERSION, + reason=str(errors[0])[:_REASON_CHARS] if errors else None, validator_version=_VALIDATOR_VERSION, adapter_version=adapter.get("version") if isinstance(adapter, Mapping) else None, aux_url=aux_url, turn_context_url=turn_context_url, ) diff --git a/tst/unit/a2a_agent/staging_test.py b/tst/unit/a2a_agent/staging_test.py index 8be0b64a4..2959064e5 100644 --- a/tst/unit/a2a_agent/staging_test.py +++ b/tst/unit/a2a_agent/staging_test.py @@ -266,6 +266,41 @@ async def test_a_trajectory_upload_lands_in_the_store(agent, store): assert store.get(target) == b'[{"type": "result"}]' +def _losing_answers_to(app, path: str): + """``app``, whose answers to requests for ``path`` are lost once it has served them.""" + + async def losing(scope, receive, send): + if scope["type"] == "http" and scope["path"] == path: + async def dropped(message) -> None: + pass + + await app(scope, receive, dropped) + await PlainTextResponse("the answer was lost", status_code=504)(scope, receive, send) + return + await app(scope, receive, send) + + return losing + + +@pytest.mark.asyncio +async def test_what_the_agent_staged_reaches_the_store_when_its_answer_is_lost(agent, store, monkeypatch): + """As an upload through a grant does: the object the agent wrote is in the store whether or not its answer + arrives, so a caller that finds the agent has let go of it can still find it there.""" + card = _card(agent) + _, path = _method(card, TRAJECTORY_V1.uri, "get") + real_async_client = httpx._client.AsyncClient # the fixture already serves the agent through the patched one + losing = _losing_answers_to(agent, path) + monkeypatch.setattr(httpx, "AsyncClient", + lambda **kwargs: real_async_client(transport=httpx.ASGITransport(app=losing), **kwargs)) + granting = transfer_store(store, URL, card, sandbox_type="modal") + target = store.object_url("trajectories/t.json") + + with pytest.raises(httpx.HTTPStatusError, match="HTTP 504"): + await fetch_trajectory(URL + path, {"task_id": "t"}, upload=TrajectoryUpload.to(granting, target), store=granting) + + assert store.get(target) == b'[{"type": "result"}]' + + async def _enable_changelog(agent, store) -> tuple[str, staging.StagedNamespace]: card = _card(agent) granting = transfer_store(store, URL, card, sandbox_type="modal") From 49e915ab0079f9ee6c07c8997801d2cf626568b6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 16:19:53 -0800 Subject: [PATCH 4/6] fix(a2a-agent): a staged call that never reached the agent collects nothing Only a call that may have reached the agent, its answer lost or refused, has staged writes to collect; one that couldn't connect fails at once rather than after collection's retries. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/a2a_agent/object_transfer.py | 8 +++-- tst/unit/a2a_agent/staging_test.py | 38 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/agent_env/a2a_agent/object_transfer.py b/src/agent_env/a2a_agent/object_transfer.py index bc1fb1035..d2f5b69f1 100644 --- a/src/agent_env/a2a_agent/object_transfer.py +++ b/src/agent_env/a2a_agent/object_transfer.py @@ -204,8 +204,8 @@ async def invoke_transfer( raise raise httpx.HTTPStatusError(f"{exc}. {hint}", request=exc.request, response=exc.response) from exc body = resp.json() - except Exception: - if staged is not None: + except Exception as exc: + if staged is not None and not isinstance(exc, _UNSENT): await _collect_staged(staged, operation) raise if staged is not None: @@ -218,6 +218,10 @@ async def invoke_transfer( return body +#: Failures of a call that never reached the agent, which therefore staged nothing to collect. +_UNSENT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) + + async def _collect_staged(staged: StagedObjectStore, operation: str) -> None: """Move what the agent wrote before its call failed into the store, as an upload through a grant reaches it whether or not the answer does; best-effort, so the call's own error is the one raised.""" diff --git a/tst/unit/a2a_agent/staging_test.py b/tst/unit/a2a_agent/staging_test.py index 2959064e5..424402e91 100644 --- a/tst/unit/a2a_agent/staging_test.py +++ b/tst/unit/a2a_agent/staging_test.py @@ -301,6 +301,44 @@ async def test_what_the_agent_staged_reaches_the_store_when_its_answer_is_lost(a assert store.get(target) == b'[{"type": "result"}]' +class _Unreachable(httpx.AsyncBaseTransport): + """The agent, except that requests for ``path`` never reach it.""" + + def __init__(self, app, path: str) -> None: + self._agent, self._path = httpx.ASGITransport(app=app), path + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + if request.url.path == self._path: + raise httpx.ConnectError("unreachable", request=request) + return await self._agent.handle_async_request(request) + + +@pytest.mark.asyncio +async def test_a_staged_call_that_never_reached_the_agent_collects_nothing(agent, store, monkeypatch): + """So an unreachable agent's call fails at once, not after collection's retries.""" + card = _card(agent) + _, path = _method(card, TRAJECTORY_V1.uri, "get") + real_async_client = httpx._client.AsyncClient # the fixture already serves the agent through the patched one + monkeypatch.setattr(httpx, "AsyncClient", + lambda **kwargs: real_async_client(transport=_Unreachable(agent, path), **kwargs)) + collected: list[int] = [] + real_pull = StagedObjectStore.pull + + async def pull(self) -> None: + collected.append(1) + await real_pull(self) + + monkeypatch.setattr(StagedObjectStore, "pull", pull) + granting = transfer_store(store, URL, card, sandbox_type="modal") + + with pytest.raises(httpx.ConnectError): + await fetch_trajectory(URL + path, {"task_id": "t"}, + upload=TrajectoryUpload.to(granting, store.object_url("trajectories/t.json")), + store=granting) + + assert collected == [] + + async def _enable_changelog(agent, store) -> tuple[str, staging.StagedNamespace]: card = _card(agent) granting = transfer_store(store, URL, card, sandbox_type="modal") From 87074612d001fe1a93802221e0e8d60ce446ef4a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 16:34:40 -0800 Subject: [PATCH 5/6] refactor(task-steps): one media check in the protocol, self-filling atif/1 records, a shared prompt-step rig The validator takes files stored beside a document as StoredMedia and the reader exports media_refs, so the worker no longer re-walks a document's media or re-implements media-bytes. AtifCopies fills each turn's record itself; the atif_copies metadata and a record's status are typed; an agent's error code is read once (extension_error_code), so one the SDK wouldn't send isn't kept as a reason; invoke_transfer's send is its own function; prompt-step tests share prompt_agent_rig and tst/util/images. Co-Authored-By: Claude Opus 5.5 --- packages/agentenv-protocol/README.md | 6 +- .../src/agentenv_protocol/atif/PROFILE.md | 5 +- .../src/agentenv_protocol/atif/__init__.py | 21 +- .../src/agentenv_protocol/atif/reader.py | 71 +++++-- .../src/agentenv_protocol/atif/validator.py | 63 +++--- .../tests/test_atif_reader.py | 15 +- .../tests/test_atif_validator.py | 19 +- src/agent_env/a2a_agent/object_transfer.py | 32 +-- src/agent_env/a2a_agent/protocol.py | 23 ++- src/agent_env/task_step/context.py | 23 ++- .../task_step/snapshot_utils/atif_copies.py | 195 ++++++++---------- .../task_step/task_steps/prompt_agent.py | 8 +- ...test_prompt_agent_live_trajectory_local.py | 16 +- tst/unit/task_step/prompt_agent_rig.py | 38 ++++ .../test_prompt_agent_atif_copies.py | 42 ++-- .../test_prompt_agent_live_trajectory.py | 21 +- tst/util/images.py | 17 ++ 17 files changed, 365 insertions(+), 250 deletions(-) create mode 100644 tst/unit/task_step/prompt_agent_rig.py create mode 100644 tst/util/images.py diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 50c09edba..1b51b8e24 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -1307,8 +1307,10 @@ the profile's spec, and `fixtures/` holds its canonical examples. The package ne terminal state on its last step. Its `page()` is the live view, built when it's read: closed steps only, so a reader's earlier page is always a prefix of a later one. A step is rendered once a page serves it and kept, so a read costs one parse of the document, not a render of every step. -- **Readers** (`load`, `tasks`, `media_paths`, `strip_profile`, `is_prefix`) take any valid ATIF document, with or - without the profile, such as a Harbor task's trajectory.json. +- **Readers** (`load`, `tasks`, `media_paths`, `media_refs`, `strip_profile`, `is_prefix`) take any valid ATIF + document, with or without the profile, such as a Harbor task's trajectory.json. `media_refs` names each place a media + file is named, with the profile's details for it; `validate(document, media=StoredMedia(sizes))` checks those + details against files stored beside the document without reading their bytes. ## A2A agent framework diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 811658112..809d5385e 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -169,7 +169,10 @@ cross-check that reads them; per-part origins, when a reader needs them; and an producer's output to profile 1.0. `reader=True` checks someone else's document for reading instead: what a producer must not write only warns, and the document fails only where agentenv couldn't read it. `media` is the folder the document's media paths are relative to, the one holding `media/`, or the files by path; without it only the paths' form -is checked. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. A sub-agent's task is its spawning call's, found through the references, so an embedded sub-agent that no served step references isn't held to `live-unanswered`. On +is checked. `StoredMedia(sizes)` stands for files stored beside the document whose bytes were checked against their +names as they were stored, as agent-env's worker checks each file an agent pushes: `media-bytes` then holds the +profile's sha256 to each file's name and its size to the stored one, without reading the bytes. `media_refs` lists each +place a document names a media file, with the profile's details for it, as these checks walk them. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. A sub-agent's task is its spawning call's, found through the references, so an embedded sub-agent that no served step references isn't held to `live-unanswered`. On the command line: `python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]`, which prints each finding and exits 1 when one is an error. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index 85bfb47d5..00ff54d9f 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -46,20 +46,33 @@ read_profile, ) from .kit import KIT_RULES, KitResult, check_adapter -from .reader import MEDIA_PATH, Task, check, check_media, is_prefix, load, media_file, media_paths, strip_profile, tasks +from .reader import ( + MEDIA_PATH, + MediaRef, + Task, + check, + check_media, + is_prefix, + load, + media_file, + media_paths, + media_refs, + strip_profile, + tasks, +) from .runner import AdapterRun, Fed, RecordedTask, replay from .served import AuxEntry, ContentItem, MediaItem, ServedData, ServedError, ServedMedia, ServedTurn -from .validator import RULES, Finding, Rule, errors_in, validate, validate_live +from .validator import RULES, Finding, Rule, StoredMedia, errors_in, validate, validate_live __all__ = [ "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", "AtifError", "AtifTurn", "AudioSource", "aux", "AuxEntry", "check", "check_adapter", "check_media", "ContentItem", "ContentPart", "Document", "dump_profile", "errors_in", "Fed", "FinalMetrics", "FinalMetricsProfile", "Finding", "finish", "FORMAT", "ImageSource", "is_prefix", "KIT_RULES", "KitResult", "load", "Media", "MEDIA_PATH", "media_file", "media_paths", "MediaInfo", - "MediaItem", "Metrics", "MetricsProfile", "Native", "native", "Observation", "ObservationResult", + "MediaItem", "MediaRef", "media_refs", "Metrics", "MetricsProfile", "Native", "native", "Observation", "ObservationResult", "ObservationResultProfile", "PROFILE_KEY", "profile_of", "PROFILE_VERSION", "PROFILES", "RAW_ARGUMENTS", "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", "RULES", "SCHEMA_VERSION", "Selector", - "ServedData", "ServedError", "ServedMedia", "ServedTurn", "Step", "StepData", "StepError", "StepProfile", + "ServedData", "ServedError", "ServedMedia", "ServedTurn", "Step", "StepData", "StepError", "StepProfile", "StoredMedia", "strip_profile", "SubagentTrajectoryRef", "SubagentTrajectoryRefProfile", "Task", "tasks", "timestamp", "ToolCall", "ToolCallProfile", "Trajectory", "TrajectoryProfile", "TurnContext", "validate", "validate_live", "VOCABULARIES", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py index 6d493f7f5..522c588f6 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/reader.py @@ -85,14 +85,11 @@ def tasks(document: Mapping[str, Any]) -> list[Task]: return out -def _trajectories(document: Mapping[str, Any]) -> Iterator[Mapping[str, Any]]: - yield document - for sub in document.get("subagent_trajectories") or []: - yield from _trajectories(sub) - - -def _parts(value: Any) -> list[Mapping[str, Any]]: - return [p for p in value if isinstance(p, Mapping)] if isinstance(value, list) else [] +def _trajectories(document: Mapping[str, Any], prefix: str = "") -> Iterator[tuple[str, Mapping[str, Any]]]: + """The document and every sub-agent embedded in it, at any depth, with the path prefix of each.""" + yield prefix, document + for index, sub in enumerate(document.get("subagent_trajectories") or []): + yield from _trajectories(sub, f"{prefix}subagent_trajectories[{index}].") #: A media file's name, ``.``: what its bytes are, never a secret. @@ -118,21 +115,55 @@ def check_media(path: str, data: bytes) -> None: raise AtifError(f"{path}'s bytes have sha256 {digest}") +@dataclass(frozen=True) +class MediaRef: + """One place a document names a media file: its ``path``, ``where`` it's named (such as + ``steps[2].message[1].source.path``), the profile's details for it ({} without) and the media type of the part + that names it, else the details' own.""" + + path: str + where: str + info: Mapping[str, Any] + media_type: str | None + + +def media_refs(document: Mapping[str, Any]) -> list[MediaRef]: + """Every place the document and the sub-agents embedded in it name a media file, in order: in each step's message + and each result's content, the image and audio parts' paths, then the files the profile records beside them for + media ATIF has no part type for.""" + refs: list[MediaRef] = [] + for prefix, trajectory in _trajectories(document): + for index, step in enumerate(trajectory.get("steps") or []): + where = f"{prefix}steps[{index}]" + holders = [(where, step, step.get("message"), "message")] + [ + (f"{where}.observation.results[{n}]", result, result.get("content"), "content") + for n, result in enumerate((step.get("observation") or {}).get("results") or [])] + for owner, holder, content, field_name in holders: + details = profile_of(holder).get("media") or {} + details = details if isinstance(details, Mapping) else {} + named: set[str] = set() + for n, part in enumerate(content if isinstance(content, list) else []): + source = part.get("source") if isinstance(part, Mapping) else None + if isinstance(source, Mapping) and isinstance(source.get("path"), str): + path, info = source["path"], _details(details, source["path"]) + named.add(path) + refs.append(MediaRef(path, f"{owner}.{field_name}[{n}].source.path", info, + source.get("media_type") or info.get("media_type"))) + refs += [MediaRef(path, f"{owner}.extra.{PROFILE_KEY}.media[{path}]", _details(details, path), + _details(details, path).get("media_type")) + for path in details if path not in named] + return refs + + +def _details(details: Mapping[str, Any], path: str) -> Mapping[str, Any]: + info = details.get(path) + return info if isinstance(info, Mapping) else {} + + def media_paths(document: Mapping[str, Any]) -> list[str]: """Every media file the document names, once each, in the order they appear: image and audio parts' paths, and the files the profile records for media ATIF has no part type for.""" - seen: dict[str, None] = {} - for trajectory in _trajectories(document): - for step in trajectory.get("steps") or []: - holders = [step] + list((step.get("observation") or {}).get("results") or []) - for holder in holders: - for part in _parts(holder.get("message") if holder is step else holder.get("content")): - path = (part.get("source") or {}).get("path") - if isinstance(path, str): - seen.setdefault(path) - for path in profile_of(holder).get("media") or {}: - seen.setdefault(path) - return list(seen) + return list(dict.fromkeys(ref.path for ref in media_refs(document))) def strip_profile(document: Mapping[str, Any]) -> dict[str, Any]: diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 7675a0200..55bcd7e0c 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -26,7 +26,7 @@ StepError, profile_of, ) -from .reader import MEDIA_PATH, is_prefix, tasks +from .reader import MEDIA_PATH, _trajectories, is_prefix, media_refs, tasks ERROR = "error" WARNING = "warning" @@ -131,17 +131,26 @@ def no_constant(name: str) -> Any: return None, [Finding("json", ERROR, "", str(exc))] +@dataclass(frozen=True) +class StoredMedia: + """Media files stored beside a document by one who checked each file's bytes against its name, as the worker does + each file an agent pushes: their sizes by path, None where the store can't tell. Given these, ``validate`` holds + the profile's sha256 to each name and its size to the stored file's without reading the bytes.""" + + sizes: Mapping[str, int | None] + + def validate(document: Any, *, reader: bool = False, - media: Path | Mapping[str, bytes] | None = None) -> list[Finding]: + media: Path | Mapping[str, bytes] | StoredMedia | None = None) -> list[Finding]: """Every finding for ``document``. ``reader`` checks someone else's document for reading rather than a producer's - output. ``media`` is the folder the document's media paths are relative to, or the files by path; without it, only - the paths' form is checked.""" + output. ``media`` is the folder the document's media paths are relative to, the files by path, or the files stored + beside it (``StoredMedia``); without it, only the paths' form is checked.""" found = _Findings(reader) _check(document, found, media) return found.items -def _check(document: Any, found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: +def _check(document: Any, found: _Findings, media: Path | Mapping[str, bytes] | StoredMedia | None) -> None: if not isinstance(document, Mapping): found.add("json", "", "a document is a JSON object") return @@ -210,13 +219,6 @@ def validate_live(pages: Sequence[Mapping[str, Any] | None], final: Mapping[str, # What the document holds -def _trajectories(document: Mapping[str, Any], prefix: str = "") -> Iterator[tuple[str, Mapping[str, Any]]]: - """The document and every sub-agent embedded in it, at any depth, with the path prefix of each.""" - yield prefix, document - for index, sub in enumerate(document.get("subagent_trajectories") or []): - yield from _trajectories(sub, f"{prefix}subagent_trajectories[{index}].") - - def _count(value: Any) -> int | None: """A token count as ATIF's models read it, a numeric string included, or None when there is none.""" try: @@ -397,8 +399,10 @@ def _usage(document: Mapping[str, Any], found: _Findings) -> None: f"cache reads and writes ({cache}) exceed prompt_tokens ({prompt}), which include them") -def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | None) -> None: - for where, path, info in _media_paths(document): +def _media(document: Mapping[str, Any], found: _Findings, + media: Path | Mapping[str, bytes] | StoredMedia | None) -> None: + for ref in media_refs(document): + where, path, info = ref.where, ref.path, ref.info if "://" in path: found.add("media-path", where, f"{path!r} is a URL; the profile's media are files beside the document") continue @@ -407,6 +411,9 @@ def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[ found.add("media-path", where, f"{path!r} isn't media/.") if media is None: continue + if isinstance(media, StoredMedia): + _stored(found, where, path, named, info, media) + continue data = _media_bytes(media, path) if data is None: found.add("media-bytes", where, f"{path} isn't in the media given") @@ -420,24 +427,16 @@ def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[ found.add("media-bytes", where, f"the profile gives {info['bytes']} bytes, but the file has {len(data)}") -def _media_paths(document: Mapping[str, Any]) -> Iterator[tuple[str, str, Mapping[str, Any]]]: - """Each media path the document names, where, and the profile's details for it.""" - for where, step in _steps(document): - holders = [(f"{where}.message", step, step.get("message"))] + [ - (f"{where}.observation.results[{n}].content", result, result.get("content")) - for n, result in enumerate(_results(step))] - for at, holder, parts in holders: - details = profile_of(holder).get("media") or {} - details = details if isinstance(details, Mapping) else {} - paths = [(f"{at}[{n}].source.path", part["source"]["path"]) - for n, part in enumerate(parts if isinstance(parts, list) else []) - if isinstance(part, Mapping) and isinstance((part.get("source") or {}).get("path"), str)] - listed = {path for _, path in paths} - owner = at.rsplit(".", 1)[0] - paths += [(f"{owner}.extra.{PROFILE_KEY}.media[{path}]", path) for path in details if path not in listed] - for path_where, path in paths: - info = details.get(path) - yield path_where, path, info if isinstance(info, Mapping) else {} +def _stored(found: _Findings, where: str, path: str, named: re.Match[str] | None, info: Mapping[str, Any], + media: StoredMedia) -> None: + if path not in media.sizes: + found.add("media-bytes", where, f"{path} isn't in the media given") + return + if named and info.get("sha256") is not None and info["sha256"] != named["sha256"]: + found.add("media-bytes", where, f"the profile gives sha256 {info['sha256']}, but the name has {named['sha256']}") + size = media.sizes[path] + if info.get("bytes") is not None and size is not None and info["bytes"] != size: + found.add("media-bytes", where, f"the profile gives {info['bytes']} bytes, but the file has {size}") def _media_bytes(media: Path | Mapping[str, bytes], path: str) -> bytes | None: diff --git a/packages/agentenv-protocol/tests/test_atif_reader.py b/packages/agentenv-protocol/tests/test_atif_reader.py index ef16f4105..f5039d563 100644 --- a/packages/agentenv-protocol/tests/test_atif_reader.py +++ b/packages/agentenv-protocol/tests/test_atif_reader.py @@ -4,7 +4,7 @@ import pytest -from agentenv_protocol.atif import AtifError, check, is_prefix, load, media_paths, strip_profile, tasks +from agentenv_protocol.atif import AtifError, check, is_prefix, load, media_paths, media_refs, strip_profile, tasks from atif_cases import FIXTURES @@ -52,6 +52,19 @@ def test_media_paths_lists_every_file_once_including_the_ones_atif_has_no_part_f assert all((FIXTURES / path).exists() for path in media_paths(fixture(name))) +def test_media_refs_name_each_place_a_file_is_named_with_its_details_and_media_type(): + refs = media_refs(fixture("computer_use")) + assert list(dict.fromkeys(ref.path for ref in refs)) == media_paths(fixture("computer_use")) + part = next(ref for ref in refs if ref.where.endswith(".source.path")) + assert part.media_type and part.media_type.startswith("image/") and part.info.get("sha256") == part.path[6:70] + odd = media_refs(fixture("odd_parts")) + noted = [ref for ref in odd if ".extra.agentenv.media[" in ref.where] + assert noted and all(ref.media_type == ref.info.get("media_type") for ref in noted) + nested = {**fixture("coding"), "subagent_trajectories": [fixture("computer_use")]} + assert all(ref.where.startswith("subagent_trajectories[0].steps[") for ref in media_refs(nested)) + assert len(media_refs(nested)) == len(refs) + + def test_a_page_is_a_prefix_only_when_its_steps_lead_every_trajectory(): final = fixture("subagent") page = {**final, "steps": final["steps"][:2], diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index 5f61970bb..77f4232c7 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -9,7 +9,7 @@ import pytest -from agentenv_protocol.atif import KIT_RULES, RULES, errors_in, validate, validate_live +from agentenv_protocol.atif import KIT_RULES, RULES, StoredMedia, errors_in, media_refs, validate, validate_live from agentenv_protocol.atif.__main__ import main from atif_cases import CASES, FIXTURES, build @@ -142,6 +142,23 @@ def test_media_bytes_are_checked_against_their_name_and_profile(): assert "media-bytes" in _rules(errors_in(validate(document, media={path: b"not the screenshot"}))) +def test_stored_media_hold_the_profile_to_each_files_name_and_stored_size_without_its_bytes(): + document = _fixture("computer_use") + sizes = {ref.path: ref.info["bytes"] for ref in media_refs(document)} + assert validate(document, media=StoredMedia(sizes)) == [] + assert validate(document, media=StoredMedia(dict.fromkeys(sizes))) == [] # a store that can't tell sizes + [first, *_] = sizes + missing = errors_in(validate(document, media=StoredMedia({p: n for p, n in sizes.items() if p != first}))) + assert [(f.rule, f.message) for f in missing] == [("media-bytes", f"{first} isn't in the media given")] + resized = errors_in(validate(document, media=StoredMedia({**sizes, first: sizes[first] + 1}))) + assert [f.rule for f in resized] == ["media-bytes"] and "bytes" in resized[0].message + [holder] = [result for step in document["steps"] for result in (step.get("observation") or {}).get("results", []) + if first in (result.get("extra") or {}).get("agentenv", {}).get("media", {})] + holder["extra"]["agentenv"]["media"][first]["sha256"] = "0" * 64 + renamed = errors_in(validate(document, media=StoredMedia(sizes))) + assert [f.rule for f in renamed] == ["media-bytes"] and "but the name has" in renamed[0].message + + def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): document = _fixture("subagent") root = sum(s["metrics"]["prompt_tokens"] for s in document["steps"] if s.get("metrics")) diff --git a/src/agent_env/a2a_agent/object_transfer.py b/src/agent_env/a2a_agent/object_transfer.py index d2f5b69f1..2c0d0ae85 100644 --- a/src/agent_env/a2a_agent/object_transfer.py +++ b/src/agent_env/a2a_agent/object_transfer.py @@ -191,19 +191,8 @@ async def invoke_transfer( if staged is not None: await staged.push() try: - async with httpx.AsyncClient() as client: - send = client.post if verb == "POST" else client.put - resp = await send(url, json=call.payload, headers=headers, timeout=timeout) - try: - raise_for_extension_status(resp, operation=operation, include_body=call.mode == "legacy") - except httpx.HTTPStatusError as exc: - hint = _STAGING_UNREACHABLE if staged is not None else ( - unreachable_hint(call.payload) if call.mode == "objects" else None - ) - if hint is None or not str(exc).endswith(f": {_TRANSFER_UNAVAILABLE}"): - raise - raise httpx.HTTPStatusError(f"{exc}. {hint}", request=exc.request, response=exc.response) from exc - body = resp.json() + body = await _send(url, call, verb=verb, operation=operation, timeout=timeout, headers=headers, + staged=staged is not None) except Exception as exc: if staged is not None and not isinstance(exc, _UNSENT): await _collect_staged(staged, operation) @@ -218,6 +207,23 @@ async def invoke_transfer( return body +async def _send(url: str, call: TransferCall, *, verb: Literal["POST", "PUT"], operation: str, timeout: float, + headers: Mapping[str, str] | None, staged: bool) -> Any: + """The agent's answer to ``call``, raising for an error status with a hint when the agent couldn't reach the + objects the call named: its own staging when ``staged``, else the grants' store.""" + async with httpx.AsyncClient() as client: + send = client.post if verb == "POST" else client.put + resp = await send(url, json=call.payload, headers=headers, timeout=timeout) + try: + raise_for_extension_status(resp, operation=operation, include_body=call.mode == "legacy") + except httpx.HTTPStatusError as exc: + hint = _STAGING_UNREACHABLE if staged else unreachable_hint(call.payload) if call.mode == "objects" else None + if hint is None or not str(exc).endswith(f": {_TRANSFER_UNAVAILABLE}"): + raise + raise httpx.HTTPStatusError(f"{exc}. {hint}", request=exc.request, response=exc.response) from exc + return resp.json() + + #: Failures of a call that never reached the agent, which therefore staged nothing to collect. _UNSENT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout) diff --git a/src/agent_env/a2a_agent/protocol.py b/src/agent_env/a2a_agent/protocol.py index 74e9305f7..6a5dff9f5 100644 --- a/src/agent_env/a2a_agent/protocol.py +++ b/src/agent_env/a2a_agent/protocol.py @@ -45,6 +45,18 @@ class AgentUnreachableError(TimeoutError): _REFUSED = " (the agent refused its control token: the record it was sent from has none, or another deployment's)" +def extension_error_code(response: httpx.Response) -> str | None: + """The SDK error code an extension's error answer names, such as ``format_not_prescribed``, when it names one in + the SDK's form: the agent writes the body, so anything else is no code.""" + try: + body = response.json() + except ValueError: + return None + error = body.get("error") if isinstance(body, Mapping) else None + code = error.get("code") if isinstance(error, Mapping) else None + return code if isinstance(code, str) and _ERROR_CODE.fullmatch(code) else None + + def raise_for_extension_status( response: httpx.Response, *, operation: str, include_body: bool = False ) -> None: @@ -53,16 +65,7 @@ def raise_for_extension_status( try: response.raise_for_status() except httpx.HTTPStatusError as exc: - if include_body: - detail = response.text[:500] - else: - try: - body = response.json() - except ValueError: - body = None - error = body.get("error") if isinstance(body, Mapping) else None - code = error.get("code") if isinstance(error, Mapping) else None - detail = code if isinstance(code, str) and _ERROR_CODE.fullmatch(code) else "" + detail = response.text[:500] if include_body else extension_error_code(response) or "" status = f"{operation} failed with HTTP {response.status_code}" refused = _REFUSED if response.status_code == 401 else "" if refused: diff --git a/src/agent_env/task_step/context.py b/src/agent_env/task_step/context.py index 604970159..df1891de2 100644 --- a/src/agent_env/task_step/context.py +++ b/src/agent_env/task_step/context.py @@ -8,7 +8,7 @@ from collections.abc import Callable from contextvars import ContextVar from dataclasses import dataclass, field -from typing import Any, Optional, TypedDict +from typing import Any, Literal, Optional, TypedDict from agentenv_protocol import client as protocol_v1 @@ -235,6 +235,9 @@ def sandbox_overrides(sandbox: str) -> dict[str, str]: ) +AtifStatus = Literal["valid", "invalid", "newer", "unavailable"] + + class AtifRecord(TypedDict): """A turn's atif/1 copy: the stored document and the step it reached at that turn (each turn's copy is a prefix of the next), how it checked (``valid``, ``invalid`` with the rules it broke, ``newer`` than this worker reads as a @@ -243,7 +246,7 @@ class AtifRecord(TypedDict): url: Optional[str] through_step: Optional[int] - status: str + status: AtifStatus rules: list[str] reason: Optional[str] validator_version: Optional[str] @@ -252,6 +255,22 @@ class AtifRecord(TypedDict): turn_context_url: Optional[str] +class AtifAgentCopies(TypedDict): + """One agent's atif/1 copies in a run: the folder they're stored in, and the trajectory id of each session's (A2A + context's) latest document, which a restart's next document is linked from.""" + + folder: str + sessions: dict[str, str] + + +class AtifCopiesMetadata(TypedDict): + """``metadata["atif_copies"]``: the id a run's atif/1 folders are named by (its instance id, else one made for + it), and each agent's copies.""" + + run: str + agents: dict[str, AtifAgentCopies] + + class TurnTrajectory(TypedDict): """What a turn's trajectory reads left, beside its native copy's url in the per-turn list: the native copy's format, as the agent's answer named it (None from an agent that names none), and its atif/1 copy, None when the diff --git a/src/agent_env/task_step/snapshot_utils/atif_copies.py b/src/agent_env/task_step/snapshot_utils/atif_copies.py index 6fe5ca0b0..9c9353680 100644 --- a/src/agent_env/task_step/snapshot_utils/atif_copies.py +++ b/src/agent_env/task_step/snapshot_utils/atif_copies.py @@ -23,13 +23,13 @@ MEDIA_PATH, PROFILE_VERSION, SCHEMA_VERSION, - Finding, + MediaRef, + StoredMedia, errors_in, - media_file, + media_refs, profile_of, validate, ) -from agentenv_protocol.atif.validator import ERROR from agent_env.a2a_agent import A2AAgent from agent_env.a2a_agent.object_transfer import ( @@ -40,19 +40,30 @@ invoke_transfer, write_object, ) +from agent_env.a2a_agent.protocol import extension_error_code from agent_env.a2a_agent.staging import transfer_store from agent_env.config import get_config from agent_env.store.object_store import ObjectStore from agent_env.store.object_store.object_store import issues_grants_to +from agent_env.task_step.context import ( + AtifAgentCopies, + AtifCopiesMetadata, + AtifRecord, + AtifStatus, + DeployedAgent, + TaskStepContext, + TurnTrajectory, +) from agent_env.task_step.snapshot_utils.agent_state_capture import trajectory_prefix_key -from agent_env.task_step.context import AtifRecord, DeployedAgent, TaskStepContext logger = logging.getLogger(__name__) -#: Where a run records its atif/1 folders: ``{"run": , "agents": {: {"folder", "sessions"}}}``, each -#: session (A2A context) naming the document its last copy was stored as. +#: Where a run records its atif/1 folders (``AtifCopiesMetadata``). ATIF_COPIES_KEY = "atif_copies" -VALID, INVALID, NEWER, UNAVAILABLE = "valid", "invalid", "newer", "unavailable" +VALID: AtifStatus = "valid" +INVALID: AtifStatus = "invalid" +NEWER: AtifStatus = "newer" +UNAVAILABLE: AtifStatus = "unavailable" #: A trajectory id the worker stores a document under: the agent mints it, so it must be a plain file name. _DOCUMENT_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]{0,127}") _VERSION = re.compile(r"(\d+)\.(\d+)") @@ -64,15 +75,19 @@ def atif_folder(context: TaskStepContext, agent_name: str) -> str: """The run's folder for ``agent_name``'s atif/1 copies, ``{prefix}atif/instance_id={id}/{agent_name}/``, recorded in ``metadata["atif_copies"]``; a run with no instance id gets one ``adhoc-`` for all its agents.""" - copies = context.metadata.setdefault(ATIF_COPIES_KEY, {}) - agent = copies.setdefault("agents", {}).setdefault(agent_name, {"sessions": {}}) - if "folder" not in agent: - run = copies.setdefault("run", context.instance_id or f"adhoc-{uuid.uuid4().hex[:12]}") + return _copies_of(context, agent_name)["folder"] + + +def _copies_of(context: TaskStepContext, agent_name: str) -> AtifAgentCopies: + copies: AtifCopiesMetadata = context.metadata.setdefault(ATIF_COPIES_KEY, AtifCopiesMetadata( + run=context.instance_id or f"adhoc-{uuid.uuid4().hex[:12]}", agents={})) + if agent_name not in copies["agents"]: config = get_config() folder = config.get_object_store().object_url( - f"{config.get_artifact_key_prefix()}atif/instance_id={run}/{agent_name}/") - agent["folder"] = folder if folder.endswith("/") else f"{folder}/" - return agent["folder"] + f"{config.get_artifact_key_prefix()}atif/instance_id={copies['run']}/{agent_name}/") + copies["agents"][agent_name] = AtifAgentCopies(folder=folder if folder.endswith("/") else f"{folder}/", + sessions={}) + return copies["agents"][agent_name] class _Unavailable(Exception): @@ -82,43 +97,44 @@ def __init__(self, reason: str) -> None: class AtifCopies: - """A step's atif/1 copies of one agent's conversation: ``turn`` starts each ended turn's, and ``records`` waits - for them all. A run that doesn't prescribe atif/1 records None for each turn; one that prescribes it to an agent - that doesn't offer it records ``unavailable``, ``format_not_offered``.""" + """A step's atif/1 copies of one agent's conversation: ``turn`` starts each ended turn's and ``join`` waits for + them, each filling its turn's record. A run that doesn't prescribe atif/1 records None; one that prescribes it to + an agent that doesn't offer it, ``unavailable`` with ``format_not_offered``.""" def __init__(self, context: TaskStepContext, agent: DeployedAgent, a2a_url: str, extension: Mapping[str, Any] | None, *, prescribed: bool) -> None: self._context, self._agent, self._a2a_url, self._extension = context, agent, a2a_url, extension self._prescribed = prescribed self._offered = ATIF in TrajectoryFormats.of(extension).formats - self._turns: list[asyncio.Task[AtifRecord] | AtifRecord | None] = [] - self._last: asyncio.Task[AtifRecord] | None = None + self._pending: list[tuple[TurnTrajectory, asyncio.Task[AtifRecord]]] = [] #: The media files the folder holds and their sizes (None until read), listed once and kept as this step pulls #: more. self._held: dict[str, int | None] | None = None - def turn(self, task_id: str, name: str) -> None: - """Copy the ended task ``task_id``'s atif/1 document, its aux log and turn context stored under ``name``.""" + def turn(self, task_id: str, name: str, record: TurnTrajectory) -> None: + """Copy the ended task ``task_id``'s atif/1 document into ``record``, its aux log and turn context stored under + ``name``.""" if not self._prescribed: - self._turns.append(None) + record["atif"] = None elif not self._offered: - self._turns.append(_unavailable("format_not_offered")) + record["atif"] = _unavailable("format_not_offered") else: - self._last = asyncio.create_task(self._copy(task_id, name, self._last)) - self._turns.append(self._last) + previous = self._pending[-1][1] if self._pending else None + self._pending.append((record, asyncio.create_task(self._copy(task_id, name, previous)))) - async def records(self) -> list[AtifRecord | None]: - """Each turn's record, once every copy has been stored; cancelled while waiting, it cancels the copies left.""" + async def join(self) -> None: + """Fill each turn's record once its copy has been stored; cancelled while waiting, it cancels the copies + left.""" try: - return [await turn if isinstance(turn, asyncio.Task) else turn for turn in self._turns] + for record, copy in self._pending: + record["atif"] = await copy except BaseException: self.cancel() raise def cancel(self) -> None: - for turn in self._turns: - if isinstance(turn, asyncio.Task): - turn.cancel() + for _, copy in self._pending: + copy.cancel() async def _copy(self, task_id: str, name: str, previous: asyncio.Task[AtifRecord] | None) -> AtifRecord: if previous is not None: # each copy overwrites the one before, so they're stored in turn order @@ -144,30 +160,23 @@ async def _stored(self, task_id: str, name: str) -> AtifRecord: raise _Unavailable("invalid_trajectory_id") store = get_config().get_object_store() folder = trajectory_prefix_key(store, atif_folder(self._context, self._agent.agent_name)) - url = await asyncio.to_thread(store.put, f"{folder}{trajectory_id}.json", _json(document), - content_type="application/json", allow_overwrite=True) - aux_url = turn_context_url = None - if body.get("aux") is not None: - aux_url = await asyncio.to_thread(store.put, f"{folder}turns/{name}/aux.json", _json(body["aux"]), - content_type="application/json", allow_overwrite=True) - if body.get("turn_context") is not None: - turn_context_url = await asyncio.to_thread( - store.put, f"{folder}turns/{name}/turn_context.json", _json(body["turn_context"]), - content_type="application/json", allow_overwrite=True) + url = await _put(store, f"{folder}{trajectory_id}.json", document) + aux_url = None if body.get("aux") is None else await _put(store, f"{folder}turns/{name}/aux.json", body["aux"]) + turn_context_url = None if body.get("turn_context") is None else await _put( + store, f"{folder}turns/{name}/turn_context.json", body["turn_context"]) root = profile_of(document) newer = _newer(document.get("schema_version"), SCHEMA_VERSION) or _newer(root.get("version"), PROFILE_VERSION) - found = await asyncio.to_thread(validate, document, reader=newer) try: - infos = _media_infos(document) + refs = _first_refs(document) except (AttributeError, TypeError): # a shape no ATIF has, which the validator reports; nothing to walk - infos = None - if infos is not None: + refs = None + media = None + if refs is not None: session_id = document.get("session_id") if isinstance(session_id, str): await self._link(store, folder, session_id, trajectory_id) - held = await self._pull_media(infos, session_id, store, folder) - found += _media_findings(infos, held) - errors = errors_in(found) + media = StoredMedia(await self._pull_media(refs, session_id, store, folder)) + errors = errors_in(await asyncio.to_thread(validate, document, reader=newer, media=media)) status = NEWER if newer else INVALID if errors else VALID if status == INVALID: logger.warning(f"agent '{self._agent.agent_name}': the atif/1 copy of task {task_id} is invalid " @@ -187,40 +196,40 @@ async def _link(self, store: ObjectStore, folder: str, session_id: str, trajecto """Point the session's earlier document, when a restart began this one, to this one: the agent that wrote the earlier one may be gone, so the worker sets its ``continued_trajectory_ref``. The session names this one only once that's done, so a link that fails is made by a later turn.""" - sessions = self._context.metadata[ATIF_COPIES_KEY]["agents"][self._agent.agent_name]["sessions"] + sessions = _copies_of(self._context, self._agent.agent_name)["sessions"] earlier = sessions.get(session_id) if earlier is not None and earlier != trajectory_id: key = f"{folder}{earlier}.json" try: document = json.loads(await asyncio.to_thread(store.read, key)) document["continued_trajectory_ref"] = f"{trajectory_id}.json" - await asyncio.to_thread(store.put, key, _json(document), content_type="application/json", - allow_overwrite=True) + await _put(store, key, document) except Exception as exc: logger.warning(f"agent '{self._agent.agent_name}': linking {earlier} to {trajectory_id} failed, so a " f"later turn tries again (continuing): {exc}") return sessions[session_id] = trajectory_id - async def _pull_media(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, store: ObjectStore, + async def _pull_media(self, refs: Mapping[str, MediaRef], context_id: Any, store: ObjectStore, folder: str) -> dict[str, int | None]: """The media files the folder holds, with their sizes, once the agent has pushed what the document names - (``infos``, by path) and the folder lacks. A file the agent answers missing or failed for may be stored already, by a push whose answer - was lost, so the store is asked before it's counted missing; a held file's size is read from the store once.""" + (``refs``) and the folder lacks. A file the agent answers missing or failed for may be stored already, by a + push whose answer was lost, so the store is asked before it's counted missing; a held file's size is read from + the store once.""" if self._held is None: listed = await asyncio.to_thread(store.list, f"{folder}media/") self._held = {key[len(folder):]: None for key in listed} - held = await self._pushed(infos, context_id, store, folder, self._held) - for path in infos: + held = await self._pushed(refs, context_id, store, folder, self._held) + for path in refs: if path in held and held[path] is None: metadata = await asyncio.to_thread(store.get_object_metadata, f"{folder}{path}") held[path] = None if metadata is None else metadata.size return held - async def _pushed(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, store: ObjectStore, folder: str, + async def _pushed(self, refs: Mapping[str, MediaRef], context_id: Any, store: ObjectStore, folder: str, held: dict[str, int | None]) -> dict[str, int | None]: """``held``, with what the agent pushed of the files the document names that it lacks.""" - wanted = [path for path in infos if MEDIA_PATH.match(path) and path not in held] + wanted = [path for path in refs if MEDIA_PATH.match(path) and path not in held] method, endpoint = A2AAgent.operation(self._extension, "media") card, sandbox_type = self._agent.a2a_card, self._agent.sandbox_type if not wanted or method is None or endpoint is None or ( @@ -228,14 +237,14 @@ async def _pushed(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, and not issues_grants_to(store, sandbox_type)): return held cap = store.max_single_upload_bytes - wanted = [path for path in wanted if cap is None or (infos.get(path, {}).get("bytes") or 0) <= cap] + wanted = [path for path in wanted if cap is None or (refs[path].info.get("bytes") or 0) <= cap] unconfirmed: list[str] = [] for start in range(0, len(wanted), MAX_MEDIA_OBJECTS): batch = wanted[start:start + MAX_MEDIA_OBJECTS] grants = transfer_store(store, self._a2a_url, card, sandbox_type=sandbox_type) # one per call objects = { - path: write_object(grants, grants.object_url(f"{folder}{path}"), media_type=_media_type(path, infos), - max_bytes=max(infos.get(path, {}).get("bytes") or DEFAULT_TRAJECTORY_MAX_BYTES, 1)) + path: write_object(grants, grants.object_url(f"{folder}{path}"), media_type=_media_type(refs[path]), + max_bytes=max(refs[path].info.get("bytes") or DEFAULT_TRAJECTORY_MAX_BYTES, 1)) .model_dump(mode="json") for path in batch } @@ -258,49 +267,16 @@ async def _pushed(self, infos: Mapping[str, Mapping[str, Any]], context_id: Any, return held -def _media_findings(infos: Mapping[str, Mapping[str, Any]], held: Mapping[str, int | None]) -> list[Finding]: - """``media-bytes`` for each media file the folder doesn't hold as the profile describes it: missing, a profile - sha256 other than its name's, or a profile size other than the stored file's. The agent checked each file it pushed - against its name, before and after the upload, so held files aren't read back.""" - found: list[Finding] = [] - for path, info in infos.items(): - if not MEDIA_PATH.match(path): - continue - if path not in held: - found.append(Finding("media-bytes", ERROR, "", f"{path} isn't stored beside the document")) - continue - if info.get("sha256") is not None and info["sha256"] != media_file(path): - found.append(Finding("media-bytes", ERROR, "", f"{path}'s profile gives sha256 {info['sha256']}")) - if info.get("bytes") is not None and held[path] is not None and info["bytes"] != held[path]: - found.append(Finding("media-bytes", ERROR, "", - f"{path}'s profile gives {info['bytes']} bytes, and the stored file has {held[path]}")) - return found - - -def _media_infos(document: Mapping[str, Any]) -> dict[str, dict[str, Any]]: - """Each media file the document names (``media_paths``), by path, with its profile entry and the media type of - the part that names it when the entry has none.""" - infos: dict[str, dict[str, Any]] = {} - for trajectory in _trajectories(document): - for step in trajectory.get("steps") or []: - for holder in [step, *((step.get("observation") or {}).get("results") or [])]: - for path, info in (profile_of(holder).get("media") or {}).items(): - infos.setdefault(path, dict(info) if isinstance(info, Mapping) else {}) - content = holder.get("message") if holder is step else holder.get("content") - for part in content if isinstance(content, list) else []: - source = part.get("source") if isinstance(part, Mapping) else None - if isinstance(source, Mapping) and isinstance(source.get("path"), str): - info = infos.setdefault(source["path"], {}) - info.setdefault("media_type", source.get("media_type")) - return infos - - -def _trajectories(document: Mapping[str, Any]) -> list[Mapping[str, Any]]: - return [document, *(sub for nested in document.get("subagent_trajectories") or [] for sub in _trajectories(nested))] - - -def _media_type(path: str, infos: Mapping[str, Mapping[str, Any]]) -> str: - return infos.get(path, {}).get("media_type") or mimetypes.guess_type(path)[0] or "application/octet-stream" +def _first_refs(document: Mapping[str, Any]) -> dict[str, MediaRef]: + """Each media file the document names, by path, as the first place names it.""" + refs: dict[str, MediaRef] = {} + for ref in media_refs(document): + refs.setdefault(ref.path, ref) + return refs + + +def _media_type(ref: MediaRef) -> str: + return ref.media_type or mimetypes.guess_type(ref.path)[0] or "application/octet-stream" def _newer(version: Any, ours: str) -> bool: @@ -308,19 +284,16 @@ def _newer(version: Any, ours: str) -> bool: return got is not None and (int(got[1]), int(got[2])) > (int(have[1]), int(have[2])) -def _json(value: Any) -> bytes: - return json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() +async def _put(store: ObjectStore, key: str, value: Any) -> str: + body = json.dumps(value, separators=(",", ":"), ensure_ascii=False, allow_nan=False).encode() + return await asyncio.to_thread(store.put, key, body, content_type="application/json", allow_overwrite=True) def _reason(exc: Exception) -> str: if isinstance(exc, _Unavailable): return exc.reason if isinstance(exc, httpx.HTTPStatusError): - try: - code = (exc.response.json().get("error") or {}).get("code") - except (ValueError, AttributeError): - code = None - return code if isinstance(code, str) else f"http_{exc.response.status_code}" + return extension_error_code(exc.response) or f"http_{exc.response.status_code}" if isinstance(exc, httpx.TimeoutException): return "timeout" if isinstance(exc, httpx.TransportError): diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index 5d52eb7c9..c676a185e 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -735,8 +735,9 @@ def record_turn() -> None: except Exception as e: logger.warning(f"Turn {turn+1} trajectory fetch failed (continuing): {e}") per_turn_trajectory_urls.append(stored.url) - per_turn_trajectories.append(TurnTrajectory(native_format=stored.format, atif=None)) - atif_copies.turn(sent_task_id, target_a2a_task_id) + turn_record = TurnTrajectory(native_format=stored.format, atif=None) + per_turn_trajectories.append(turn_record) + atif_copies.turn(sent_task_id, target_a2a_task_id, turn_record) final_task_id = sent_task_id final_state = target_state @@ -798,8 +799,7 @@ def record_turn() -> None: # On failure too, so a reader of the last turn knows no other follows. if live_turns is not None: await live_turns.end() - for turn_record, atif in zip(per_turn_trajectories, await atif_copies.records()): - turn_record["atif"] = atif + await atif_copies.join() await self._persist_env_trigger_state(context) diff --git a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py index 6ca74353c..f107dc2ad 100644 --- a/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py +++ b/tst/integration/task_step/test_prompt_agent_live_trajectory_local.py @@ -13,13 +13,11 @@ import hashlib import json import re -import struct import shutil import socket import subprocess import time import uuid -import zlib import httpx import pytest @@ -41,6 +39,7 @@ from agent_env.task_step.task_steps.snapshot_agent_state import SnapshotAgentStateTaskStep from tst.integration.utils.registry import REGISTRY_IMAGE from tst.util.a2a_test_agent import put_test_agent +from tst.util.images import png pytestmark = pytest.mark.integration @@ -177,7 +176,7 @@ async def test_a_deployed_agent_serves_its_native_format_and_atif_and_refuses_an """And the run, which prescribes atif/1, stores the turn's copy in its folder with the image's file beside it.""" suffix = uuid.uuid4().hex[:8] native = "agentenv-echo-agent/v1" - image = _png(6, 4) + image = png(6, 4) agent = put_test_agent(f"trajectory-formats-agent-{suffix}") task = Task.put(id=f"trajectory-formats-{suffix}", steps=[ _deploy(agent, "solver"), @@ -288,15 +287,6 @@ async def test_a_model_key_a_command_prints_is_redacted_in_atif_and_kept_in_the_ assert errors_in(validate(atif["trajectory"])) == [] -def _png(width: int, height: int) -> bytes: - def chunk(kind: bytes, body: bytes) -> bytes: - return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) - - pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) - return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) - + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) - - @pytest.mark.asyncio async def test_a_deployed_agent_pushes_its_copys_media_under_exact_object_grants(local_registry): suffix = uuid.uuid4().hex[:8] @@ -305,7 +295,7 @@ async def test_a_deployed_agent_pushes_its_copys_media_under_exact_object_grants context = await Task.put(id=f"media-{suffix}", steps=[_deploy(agent, "solver")]).run() assert not context.metadata.get("failed_steps"), context.metadata.get("failed_steps") [deployed] = context.deployed_agents - image = _png(6, 4) + image = png(6, 4) parts = [{"kind": "text", "text": "look"}, {"kind": "file", "file": {"bytes": base64.b64encode(image).decode(), "mimeType": "image/png", "name": "shot.png"}}] diff --git a/tst/unit/task_step/prompt_agent_rig.py b/tst/unit/task_step/prompt_agent_rig.py new file mode 100644 index 000000000..1d3602583 --- /dev/null +++ b/tst/unit/task_step/prompt_agent_rig.py @@ -0,0 +1,38 @@ +"""What tests of a prompt step against an agent served in process share: the conversation store kept quiet, every +HTTP client sent to the agent, and the agent the step prompts, deployed from its own card.""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest + +from agent_env.task_step.context import DeployedAgent +from agent_env.task_step.task_steps import prompt_agent as pa + +#: The conversation store calls a prompt step makes, which these tests keep from reaching a store. +CONVERSATION_STORE_CALLS = ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", + "get_conversation") +AGENT_URL = "http://agent.test" + + +def quiet_conversations(monkeypatch: pytest.MonkeyPatch) -> None: + for name in CONVERSATION_STORE_CALLS: + monkeypatch.setattr(pa.conversation_store, name, lambda *args, **kwargs: None) + + +def route_clients(monkeypatch: pytest.MonkeyPatch, transport: httpx.AsyncBaseTransport, **options: Any) -> None: + """Send every ``httpx.AsyncClient`` in this process through ``transport``, with ``options`` such as event hooks.""" + real_client = httpx._client.AsyncClient # the class itself, so routing twice replaces rather than nests + monkeypatch.setattr(httpx, "AsyncClient", + lambda *args, **kwargs: real_client(*args, **{**kwargs, "transport": transport, **options})) + + +def card_of(app: Any) -> dict: + """An SDK agent app's card, as its well-known route serves it.""" + return app.state.agentenv_a2a.card.model_dump(mode="json", exclude_none=True) + + +def deployed(card: dict, *, name: str = "solver", url: str = AGENT_URL) -> DeployedAgent: + return DeployedAgent(agent_name=name, api_url=url, a2a_url=url, a2a_card=card) diff --git a/tst/unit/task_step/test_prompt_agent_atif_copies.py b/tst/unit/task_step/test_prompt_agent_atif_copies.py index ca01acf76..05f9451cc 100644 --- a/tst/unit/task_step/test_prompt_agent_atif_copies.py +++ b/tst/unit/task_step/test_prompt_agent_atif_copies.py @@ -7,8 +7,6 @@ import asyncio import base64 import json -import struct -import zlib from pathlib import Path from typing import Any, Callable from urllib.parse import unquote @@ -27,25 +25,18 @@ from agentenv_protocol.atif import FORMAT as ATIF, Adapter, atif_adapter, errors_in, media_paths, native, validate from agent_env.config import configure -from agent_env.task_step.context import DeployedAgent, PromptResponse, TaskStepContext +from agent_env.task_step.context import PromptResponse, TaskStepContext from agent_env.task_step.snapshot_utils import atif_copies from agent_env.task_step.task_steps import prompt_agent as pa from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.task_step.prompt_agent_rig import card_of, deployed, quiet_conversations, route_clients from tst.util.granting_object_store import GRANT_ORIGIN, GrantingObjectStore +from tst.util.images import png NATIVE = "shots/1" _PREFIX = "prompt_agent_trajectories/prompt_id=solve/" -def _png(width: int, height: int) -> bytes: - def chunk(kind: bytes, body: bytes) -> bytes: - return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) - - pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) - header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) - return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", pixels) + chunk(b"IEND", b"") - - @atif_adapter(native_format=NATIVE, version="2.1.0", input_includes_cache=False) class _Shots(Adapter): """Each native shot a step: its caption and its screenshot.""" @@ -69,7 +60,7 @@ class Agent(AgentEnvAgent): async def run(self, request: TaskRequest) -> TaskResult: request.trajectory.set_format(NATIVE) n = next(shots) - request.trajectory.append({"type": "shot", "n": n, "png": base64.b64encode(_png(n, 1)).decode()}) + request.trajectory.append({"type": "shot", "n": n, "png": base64.b64encode(png(n, 1)).decode()}) return TaskResult.text(f"shot {n}") return Agent().create_app() @@ -136,13 +127,10 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *, atif: boo self.root = tmp_path / "store" self.store = GrantingObjectStore(str(self.root), reaches=True) configure(object_store=self.store) - for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): - monkeypatch.setattr(pa.conversation_store, fn, lambda *a, **kw: None) + quiet_conversations(monkeypatch) self.wire = _Wire(self.store, atif=atif) self.asked: list[int] = [] - real_client = httpx.AsyncClient - monkeypatch.setattr(httpx, "AsyncClient", lambda *a, **kw: real_client( - *a, **{**kw, "transport": self.wire})) + route_clients(monkeypatch, self.wire) async def ask(user_side, number: int, reply: str, parts: list[dict]) -> dict: self.asked.append(number) @@ -152,7 +140,7 @@ async def ask(user_side, number: int, reply: str, parts: list[dict]) -> dict: @property def card(self) -> dict: - return self.wire.app.state.agentenv_a2a.card.model_dump(mode="json", exclude_none=True) + return card_of(self.wire.app) def context(self, formats: list[str] | None = (ATIF,), **kwargs) -> TaskStepContext: context = TaskStepContext(instance_id="ti-1", **kwargs) @@ -161,8 +149,7 @@ def context(self, formats: list[str] | None = (ATIF,), **kwargs) -> TaskStepCont return context async def execute(self, context: TaskStepContext, *, card: dict | None = None, **step) -> PromptResponse: - context.deployed_agents[:] = [DeployedAgent(agent_name="solver", api_url="http://agent.test", - a2a_url="http://agent.test", a2a_card=card or self.card)] + context.deployed_agents[:] = [deployed(card or self.card)] step.setdefault("max_conversation_turns", 2) step.setdefault("user_a2a_url", "http://user.test") # whose answers ``ask`` stands in for await PromptAgentTaskStep(id="solve", version=None, prompt="go", agent_name="solver", poll_interval_seconds=0, @@ -452,3 +439,16 @@ def misdescribed(answer: dict) -> dict: [atif] = _atifs(response) assert (atif["status"], atif["rules"]) == ("invalid", ["media-bytes"]) assert ("bytes" if field == "bytes" else "sha256") in atif["reason"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("code", ["x" * 5000, "Not A Code", 7]) +async def test_an_error_code_the_sdk_wouldnt_send_isnt_kept_as_the_reason(monkeypatch, tmp_path, code): + run = _Run(monkeypatch, tmp_path) + run.wire.answer = lambda request, path, sent, response: ( + httpx.Response(409, json={"error": {"code": code}}, request=request) + if path == "/ext/trajectory" and sent.get("format") == ATIF else None) + + response = await run.execute(run.context(), max_conversation_turns=1) + + assert [(atif["status"], atif["reason"]) for atif in _atifs(response)] == [("unavailable", "http_409")] diff --git a/tst/unit/task_step/test_prompt_agent_live_trajectory.py b/tst/unit/task_step/test_prompt_agent_live_trajectory.py index a495c0fe4..97080ebc3 100644 --- a/tst/unit/task_step/test_prompt_agent_live_trajectory.py +++ b/tst/unit/task_step/test_prompt_agent_live_trajectory.py @@ -20,10 +20,11 @@ from agent_env.config import configure from agent_env.store.object_store import LocalFilesystemObjectStore -from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.context import TaskStepContext from agent_env.task_step.snapshot_utils import live_trajectory from agent_env.task_step.task_steps import prompt_agent as pa from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep +from tst.unit.task_step.prompt_agent_rig import card_of, deployed, quiet_conversations, route_clients _PREFIX = "prompt_agent_trajectories/prompt_id=solve/" _EVENTS = [{"type": "system", "subtype": "init"}, {"type": "assistant", "step": 0}] @@ -73,18 +74,14 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch, tmp_path, *, live: bool, wai native: str | None = None, atif: bool = False) -> None: self.store = LocalFilesystemObjectStore(str(tmp_path)) configure(object_store=self.store) - for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): - monkeypatch.setattr(pa.conversation_store, fn, lambda *a, **kw: None) + quiet_conversations(monkeypatch) app = _agent(self.store, live=live, wait_for_chunk=wait_for_chunk, native=native, atif=atif).create_app() - self.card = app.state.agentenv_a2a.card.model_dump(mode="json", exclude_none=True) + self.card = card_of(app) self.cursor_reads: list[dict] = [] self.final_reads: list[dict] = [] #: Every request the step sent, in order: its path and, for a JSON-RPC call, its method. self.sent: list[str] = [] self.prescriptions: list[dict] = [] - transport = httpx.ASGITransport(app=app) - real_client = httpx.AsyncClient - async def record(request: httpx.Request) -> None: body = json.loads(await request.aread()) if request.method == "POST" else {} self.sent.append(body.get("method", request.url.path) if isinstance(body, dict) else request.url.path) @@ -93,17 +90,11 @@ async def record(request: httpx.Request) -> None: elif request.url.path == "/ext/trajectory/prescribe": self.prescriptions.append(body) - monkeypatch.setattr( - httpx, - "AsyncClient", - lambda *a, **kw: real_client(*a, transport=transport, event_hooks={"request": [record]}, **kw), - ) + route_clients(monkeypatch, httpx.ASGITransport(app=app), event_hooks={"request": [record]}) async def execute(self, step: PromptAgentTaskStep, context: TaskStepContext | None = None) -> TaskStepContext: context = context or TaskStepContext(instance_id="ti-1") - context.deployed_agents.append( - DeployedAgent(agent_name="solver", api_url="http://agent.test", a2a_url="http://agent.test", a2a_card=self.card) - ) + context.deployed_agents.append(deployed(self.card)) return await step.execute(context) def step(self, **kwargs) -> PromptAgentTaskStep: diff --git a/tst/util/images.py b/tst/util/images.py new file mode 100644 index 000000000..6932f2307 --- /dev/null +++ b/tst/util/images.py @@ -0,0 +1,17 @@ +"""Small real images for tests that move media: a PNG whose header a reader can size, built without Pillow.""" + +from __future__ import annotations + +import struct +import zlib + + +def png(width: int, height: int) -> bytes: + """A black ``width`` by ``height`` RGB PNG.""" + + def chunk(kind: bytes, body: bytes) -> bytes: + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body)) + + pixels = zlib.compress(b"".join(b"\x00" + b"\x00\x00\x00" * width for _ in range(height))) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", pixels) + chunk(b"IEND", b"")) From 5b70210f0698cfeda5b7ecddf0736c62437ed025 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 20 Nov 2026 16:58:25 -0800 Subject: [PATCH 6/6] fix(atif): a newer copy's media are still checked validate stops at a document whose schema ATIF's models reject, as a newer one's; validate_media gives the media-bytes findings alone for any document media_refs can walk, and the worker checks a copy's media with it. Co-Authored-By: Claude Opus 5.5 --- .../src/agentenv_protocol/atif/PROFILE.md | 3 ++- .../src/agentenv_protocol/atif/__init__.py | 4 +-- .../src/agentenv_protocol/atif/validator.py | 25 +++++++++++++++---- .../tests/test_atif_validator.py | 20 ++++++++++++++- .../task_step/snapshot_utils/atif_copies.py | 6 ++++- .../test_prompt_agent_atif_copies.py | 20 +++++++++++++++ 6 files changed, 68 insertions(+), 10 deletions(-) diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md index 809d5385e..442ee29d1 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/PROFILE.md @@ -171,7 +171,8 @@ must not write only warns, and the document fails only where agentenv couldn't r document's media paths are relative to, the one holding `media/`, or the files by path; without it only the paths' form is checked. `StoredMedia(sizes)` stands for files stored beside the document whose bytes were checked against their names as they were stored, as agent-env's worker checks each file an agent pushes: `media-bytes` then holds the -profile's sha256 to each file's name and its size to the stored one, without reading the bytes. `media_refs` lists each +profile's sha256 to each file's name and its size to the stored one, without reading the bytes. `validate_media(document, media)` gives the `media-bytes` findings alone, for a document +newer than these models, which `validate` stops at. `media_refs` lists each place a document names a media file, with the profile's details for it, as these checks walk them. `validate_live(pages, final)` checks what a live reader was served, page by page, with the final read. A sub-agent's task is its spawning call's, found through the references, so an embedded sub-agent that no served step references isn't held to `live-unanswered`. On the command line: `python -m agentenv_protocol.atif validate FILE [--media DIR] [--reader]`, which prints each finding and exits 1 when one is an error. diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py index 00ff54d9f..47f843feb 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/__init__.py @@ -62,7 +62,7 @@ ) from .runner import AdapterRun, Fed, RecordedTask, replay from .served import AuxEntry, ContentItem, MediaItem, ServedData, ServedError, ServedMedia, ServedTurn -from .validator import RULES, Finding, Rule, StoredMedia, errors_in, validate, validate_live +from .validator import RULES, Finding, Rule, StoredMedia, errors_in, validate, validate_live, validate_media __all__ = [ "Adapter", "AdapterRef", "AdapterRun", "AdapterSpec", "Agent", "AgentProfile", "Arrival", "atif_adapter", @@ -74,5 +74,5 @@ "read_profile", "ReasoningBlock", "RecordedTask", "replay", "Rule", "RULES", "SCHEMA_VERSION", "Selector", "ServedData", "ServedError", "ServedMedia", "ServedTurn", "Step", "StepData", "StepError", "StepProfile", "StoredMedia", "strip_profile", "SubagentTrajectoryRef", "SubagentTrajectoryRefProfile", "Task", "tasks", "timestamp", "ToolCall", - "ToolCallProfile", "Trajectory", "TrajectoryProfile", "TurnContext", "validate", "validate_live", "VOCABULARIES", + "ToolCallProfile", "Trajectory", "TrajectoryProfile", "TurnContext", "validate", "validate_live", "validate_media", "VOCABULARIES", ] diff --git a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py index 55bcd7e0c..8e8c536e9 100644 --- a/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py +++ b/packages/agentenv-protocol/src/agentenv_protocol/atif/validator.py @@ -399,18 +399,33 @@ def _usage(document: Mapping[str, Any], found: _Findings) -> None: f"cache reads and writes ({cache}) exceed prompt_tokens ({prompt}), which include them") +def validate_media(document: Mapping[str, Any], media: Path | Mapping[str, bytes] | StoredMedia, *, + reader: bool = False) -> list[Finding]: + """The ``media-bytes`` findings alone, for a document ``media_refs`` can walk, whether or not ATIF's models accept + it: a document newer than these models, which ``validate`` stops at, still names files a reader needs.""" + found = _Findings(reader) + _media_files(document, found, media) + return found.items + + def _media(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | StoredMedia | None) -> None: + for ref in media_refs(document): + if "://" in ref.path: + found.add("media-path", ref.where, + f"{ref.path!r} is a URL; the profile's media are files beside the document") + elif not MEDIA_PATH.match(ref.path): + found.add("media-path", ref.where, f"{ref.path!r} isn't media/.") + if media is not None: + _media_files(document, found, media) + + +def _media_files(document: Mapping[str, Any], found: _Findings, media: Path | Mapping[str, bytes] | StoredMedia) -> None: for ref in media_refs(document): where, path, info = ref.where, ref.path, ref.info if "://" in path: - found.add("media-path", where, f"{path!r} is a URL; the profile's media are files beside the document") continue named = MEDIA_PATH.match(path) - if not named: - found.add("media-path", where, f"{path!r} isn't media/.") - if media is None: - continue if isinstance(media, StoredMedia): _stored(found, where, path, named, info, media) continue diff --git a/packages/agentenv-protocol/tests/test_atif_validator.py b/packages/agentenv-protocol/tests/test_atif_validator.py index 77f4232c7..6de40b289 100644 --- a/packages/agentenv-protocol/tests/test_atif_validator.py +++ b/packages/agentenv-protocol/tests/test_atif_validator.py @@ -9,7 +9,16 @@ import pytest -from agentenv_protocol.atif import KIT_RULES, RULES, StoredMedia, errors_in, media_refs, validate, validate_live +from agentenv_protocol.atif import ( + KIT_RULES, + RULES, + StoredMedia, + errors_in, + media_refs, + validate, + validate_live, + validate_media, +) from agentenv_protocol.atif.__main__ import main from atif_cases import CASES, FIXTURES, build @@ -159,6 +168,15 @@ def test_stored_media_hold_the_profile_to_each_files_name_and_stored_size_withou assert [f.rule for f in renamed] == ["media-bytes"] and "but the name has" in renamed[0].message +def test_media_are_checked_on_their_own_for_a_document_newer_than_the_models(): + document = {**_fixture("computer_use"), "schema_version": "ATIF-v1.9"} + sizes = {ref.path: ref.info["bytes"] for ref in media_refs(document)} + assert _rules(errors_in(validate(document, reader=True, media=StoredMedia({})))) == {"atif"} + assert validate_media(document, StoredMedia(sizes), reader=True) == [] + assert _rules(errors_in(validate_media(document, StoredMedia({}), reader=True))) == {"media-bytes"} + assert validate_media(document, FIXTURES) == [] + + def test_totals_sum_the_sub_agents_steps_and_skip_a_count_a_step_lacks(): document = _fixture("subagent") root = sum(s["metrics"]["prompt_tokens"] for s in document["steps"] if s.get("metrics")) diff --git a/src/agent_env/task_step/snapshot_utils/atif_copies.py b/src/agent_env/task_step/snapshot_utils/atif_copies.py index 9c9353680..5bc016940 100644 --- a/src/agent_env/task_step/snapshot_utils/atif_copies.py +++ b/src/agent_env/task_step/snapshot_utils/atif_copies.py @@ -29,6 +29,7 @@ media_refs, profile_of, validate, + validate_media, ) from agent_env.a2a_agent import A2AAgent @@ -176,7 +177,10 @@ async def _stored(self, task_id: str, name: str) -> AtifRecord: if isinstance(session_id, str): await self._link(store, folder, session_id, trajectory_id) media = StoredMedia(await self._pull_media(refs, session_id, store, folder)) - errors = errors_in(await asyncio.to_thread(validate, document, reader=newer, media=media)) + found = await asyncio.to_thread(validate, document, reader=newer) + if media is not None: # on its own, since validate stops at a document newer than its models + found += await asyncio.to_thread(validate_media, document, media, reader=newer) + errors = errors_in(found) status = NEWER if newer else INVALID if errors else VALID if status == INVALID: logger.warning(f"agent '{self._agent.agent_name}': the atif/1 copy of task {task_id} is invalid " diff --git a/tst/unit/task_step/test_prompt_agent_atif_copies.py b/tst/unit/task_step/test_prompt_agent_atif_copies.py index 05f9451cc..2c1851842 100644 --- a/tst/unit/task_step/test_prompt_agent_atif_copies.py +++ b/tst/unit/task_step/test_prompt_agent_atif_copies.py @@ -273,6 +273,26 @@ def newer(answer: dict) -> dict: version if field == "schema_version" else "ATIF-v1.8") +@pytest.mark.asyncio +async def test_a_newer_copys_missing_media_are_in_its_rules(monkeypatch, tmp_path): + """ATIF's models stop at a newer schema, and the copy's media are still checked.""" + run = _Run(monkeypatch, tmp_path) + card = run.card + trajectory = next(e for e in card["capabilities"]["extensions"] if e["uri"] == TRAJECTORY_V1.uri) + del trajectory["params"]["methods"]["media"] + + def newer(answer: dict) -> dict: + answer["trajectory"]["schema_version"] = "ATIF-v1.9" + return answer + + run.wire.rewrite_atif = newer + + response = await run.execute(run.context(), card=card, max_conversation_turns=1) + + [atif] = _atifs(response) + assert (atif["status"], atif["rules"]) == ("newer", ["atif", "media-bytes"]) + + @pytest.mark.asyncio async def test_media_the_agent_cant_be_asked_for_makes_the_copy_invalid_under_media_bytes(monkeypatch, tmp_path): run = _Run(monkeypatch, tmp_path)